diff --git a/DATA.md b/DATA.md index c234f1a..f0b2bca 100644 --- a/DATA.md +++ b/DATA.md @@ -46,7 +46,8 @@ These collectors only run on Linux systems. | `system/dmesg_t.out` | `dmesg -T` | Kernel ring buffer with timestamps | | `system/free.out` | `free -h` | Memory usage summary | | `system/fstab.out` | `/etc/fstab` | Filesystem table | -| `system/hostname.out` | `hostname -f` | Fully qualified hostname | +| `system/hostname.out` | `hostname` | Kernel hostname (gethostname) | +| `system/hostname_fqdn.out` | `hostname -f` | Fully qualified hostname (resolver canonical name) | | `system/hypervisor.out` | `systemd-detect-virt` | Hypervisor detection | | `system/ifconfig.out` | `ifconfig -a` | Network interfaces (legacy) | | `system/interfaces.out` | `ip -o address` | Network interfaces (one-line) | @@ -181,7 +182,8 @@ These collectors only run on macOS systems. |------|--------|-------------| | `system/diskutil_info_all.out` | `diskutil info` (all disks) | Detailed disk information | | `system/diskutil_list.out` | `diskutil list` | Disk layout | -| `system/hostname.out` | `hostname` | Hostname | +| `system/hostname.out` | `hostname` | Kernel hostname (gethostname) | +| `system/hostname_fqdn.out` | `hostname -f` | Fully qualified hostname (resolver canonical name) | | `system/hypervisor.out` | `sysctl kern.hv_vmm_present` | Hypervisor detection | | `system/ifconfig.out` | `ifconfig -a` | Network interfaces | | `system/iostat.out` | `iostat -c 5 -w 1` | I/O statistics (5 samples) | @@ -222,7 +224,7 @@ Instance-level PostgreSQL collectors. Files stored in `postgresql/`. | File | Source | Description | |------|--------|-------------| | `postgresql/archiver.tsv` | `pg_stat_archiver` | WAL archiver statistics | -| `postgresql/available_extensions.tsv` | `pg_available_extensions` | Available extensions | +| `postgresql/available_extensions.tsv` | `pg_available_extension_versions` | Available extension versions with installed flag, trust and requires metadata | | `postgresql/bgwriter.tsv` | `pg_stat_bgwriter` | Background writer statistics | | `postgresql/blocking_locks.tsv` | Complex query | Blocking/blocked lock pairs | | `postgresql/checkpointer.tsv` | `pg_stat_checkpointer` | Checkpointer statistics | @@ -260,7 +262,9 @@ Instance-level PostgreSQL collectors. Files stored in `postgresql/`. | `postgresql/stat_progress_copy.tsv` | `pg_stat_progress_copy` | COPY progress (PG14+) | | `postgresql/stat_progress_create_index.tsv` | `pg_stat_progress_create_index` | CREATE INDEX progress (PG12+) | | `postgresql/stat_progress_vacuum.tsv` | `pg_stat_progress_vacuum` | VACUUM progress (PG9.6+) | +| `postgresql/stat_replication_slots.tsv` | `pg_stat_replication_slots` | Replication slot stats incl. spill counters (PG14+) | | `postgresql/stat_slru.tsv` | `pg_stat_slru` | SLRU cache statistics | +| `postgresql/stat_ssl.tsv` | `pg_stat_ssl` | Per-backend SSL/TLS state | | `postgresql/stat_statements_calls.tsv` | `pg_stat_statements` | Top 100 queries by call count | | `postgresql/stat_statements_max_time.tsv` | `pg_stat_statements` | Top 100 queries by max execution time | | `postgresql/stat_statements_total_time.tsv` | `pg_stat_statements` | Top 100 queries by total execution time | @@ -281,21 +285,24 @@ Collected for each accessible database. Files stored in `databases/{dbname}/`. | File | Source | Description | |------|--------|-------------| +| `bloat.tsv` | `pg_stats` heuristic | Table bloat estimate (heuristic) | | `extensions.tsv` | `pg_extension` | Installed extensions | | `funcs.tsv` | `pg_proc WHERE prokind='f'` | Functions | -| `indexes.tsv` | `pg_indexes` | Indexes | +| `indexes.tsv` | `pg_index` + `pg_stat_all_indexes` | Top 1000 indexes by size with semantic key, validity, scan counters and size | | `languages.tsv` | `pg_language` | Procedural languages | | `operators.tsv` | `pg_operator` | Operators | | `partitioned_tables.tsv` | `pg_partitioned_table` | Partitioned tables (PG10+) | | `partitions.tsv` | `pg_inherits` | Partition relationships | +| `pgstattuple.tsv` | `pgstattuple_approx()` | Authoritative bloat (when `pgstattuple` extension is installed) | | `procs.tsv` | `pg_proc WHERE prokind='p'` | Procedures (PG11+) | | `publication_tables.tsv` | `pg_publication_tables` | Tables in publications | | `publications.tsv` | `pg_publication` | Logical replication publications | | `schemas.tsv` | `pg_namespace` | Schemas | +| `sequences.tsv` | `pg_sequences` | Sequences with last/min/max values (PG10+) | | `stat_database.tsv` | `pg_stat_database` | Per-database statistics | | `statistics.tsv` | `pg_statistic_ext` | Extended statistics (PG10+) | | `subscription_tables.tsv` | `pg_subscription_rel` | Subscription relation states | -| `tables.tsv` | `pg_tables` | Tables | +| `tables.tsv` | `pg_class` + `pg_stat_all_tables` | Top 1000 tables by size with persistence, options, heap and table sizes, toast mapping, dead-tup counters and vacuum/analyze timestamps | | `triggers.tsv` | `pg_trigger` | Triggers | | `types.tsv` | `pg_type` | Data types | diff --git a/README.md b/README.md index aab373c..0a609ad 100644 --- a/README.md +++ b/README.md @@ -187,19 +187,20 @@ For a complete reference of all collected data, see [docs/data.md](docs/data.md) - **Configuration & files**: `pg_db_role_setting`, `pg_file_settings`, `pg_hba.conf`, `pg_hba_file_rules`, `pg_ident.conf`, `pg_settings`, `pg_tablespace`, `postgresql.auto.conf`, `postgresql.conf`, `recovery.conf`, `recovery.done` - **Activity & monitoring**: `pg_locks`, `pg_postmaster_start_time()`, `pg_prepared_xacts`, `pg_shmem_allocations`, `pg_stat_activity` -- **Statistics views**: `pg_stat_archiver`, `pg_stat_bgwriter`, `pg_stat_checkpointer` (PG17+), `pg_stat_database_conflicts`, `pg_stat_io` (PG16+), `pg_stat_slru`, `pg_stat_statements` (if installed), `pg_stat_wal` (PG14+), `pg_stat_wal_receiver` -- **Replication & WAL**: `pg_current_wal_lsn()`, `pg_replication_origin_status`, `pg_replication_slots`, `pg_stat_replication`, `pg_subscription` +- **Statistics views**: `pg_stat_archiver`, `pg_stat_bgwriter`, `pg_stat_checkpointer` (PG17+), `pg_stat_database_conflicts`, `pg_stat_io` (PG16+), `pg_stat_slru`, `pg_stat_ssl`, `pg_stat_statements` (if installed), `pg_stat_wal` (PG14+), `pg_stat_wal_receiver` +- **Replication & WAL**: `pg_current_wal_lsn()`, `pg_replication_origin_status`, `pg_replication_slots`, `pg_stat_replication`, `pg_stat_replication_slots` (PG14+, spill counters), `pg_subscription` - **Progress tracking**: `pg_stat_progress_analyze`, `pg_stat_progress_basebackup`, `pg_stat_progress_cluster`, `pg_stat_progress_copy`, `pg_stat_progress_create_index`, `pg_stat_progress_vacuum` -- **Catalog**: `pg_available_extensions`, `pg_database`, `pg_database_size()`, `pg_roles`, `pg_tablespace_size()`, `version()` +- **Catalog**: `pg_available_extension_versions`, `pg_database` (incl. `datfrozenxid`/`datminmxid` wraparound headroom and `datconnlimit`), `pg_database_size()`, `pg_roles`, `pg_tablespace_size()`, `version()` **Per-Database** -- **Schema objects**: `pg_indexes`, `pg_namespace`, `pg_operator`, `pg_tables`, `pg_type` +- **Schema objects**: `pg_class` + `pg_stat_all_tables` (tables incl. dead-tup counters, vacuum/analyze timestamps, `reloptions`, `reltoastrelid`, `relpersistence`), `pg_index` + `pg_stat_all_indexes` (indexes incl. validity, scan counters, size), `pg_namespace`, `pg_operator`, `pg_sequences`, `pg_type` - **Functions & procedures**: `pg_proc` - **Triggers & partitioning**: `pg_inherits`, `pg_partitioned_table`, `pg_trigger` - **Logical replication**: `pg_publication`, `pg_publication_tables`, `pg_subscription_rel` -- **Extensions**: `pg_extension`, `pg_language`, `pg_statistic_ext` +- **Extensions**: `pg_extension`, `pg_language`, `pg_statistic_ext`, `pgstattuple` (if installed — authoritative bloat estimate via `pgstattuple_approx()`) - **Statistics**: `pg_stat_database` (conflicts, deadlocks, temp files, stats reset) +- **Bloat estimate**: heuristic from `pg_stats` (always emitted) **[pg_statviz](https://github.com/vyruss/pg_statviz) Extension** (if present) diff --git a/docs/data.md b/docs/data.md index c234f1a..f0b2bca 100644 --- a/docs/data.md +++ b/docs/data.md @@ -46,7 +46,8 @@ These collectors only run on Linux systems. | `system/dmesg_t.out` | `dmesg -T` | Kernel ring buffer with timestamps | | `system/free.out` | `free -h` | Memory usage summary | | `system/fstab.out` | `/etc/fstab` | Filesystem table | -| `system/hostname.out` | `hostname -f` | Fully qualified hostname | +| `system/hostname.out` | `hostname` | Kernel hostname (gethostname) | +| `system/hostname_fqdn.out` | `hostname -f` | Fully qualified hostname (resolver canonical name) | | `system/hypervisor.out` | `systemd-detect-virt` | Hypervisor detection | | `system/ifconfig.out` | `ifconfig -a` | Network interfaces (legacy) | | `system/interfaces.out` | `ip -o address` | Network interfaces (one-line) | @@ -181,7 +182,8 @@ These collectors only run on macOS systems. |------|--------|-------------| | `system/diskutil_info_all.out` | `diskutil info` (all disks) | Detailed disk information | | `system/diskutil_list.out` | `diskutil list` | Disk layout | -| `system/hostname.out` | `hostname` | Hostname | +| `system/hostname.out` | `hostname` | Kernel hostname (gethostname) | +| `system/hostname_fqdn.out` | `hostname -f` | Fully qualified hostname (resolver canonical name) | | `system/hypervisor.out` | `sysctl kern.hv_vmm_present` | Hypervisor detection | | `system/ifconfig.out` | `ifconfig -a` | Network interfaces | | `system/iostat.out` | `iostat -c 5 -w 1` | I/O statistics (5 samples) | @@ -222,7 +224,7 @@ Instance-level PostgreSQL collectors. Files stored in `postgresql/`. | File | Source | Description | |------|--------|-------------| | `postgresql/archiver.tsv` | `pg_stat_archiver` | WAL archiver statistics | -| `postgresql/available_extensions.tsv` | `pg_available_extensions` | Available extensions | +| `postgresql/available_extensions.tsv` | `pg_available_extension_versions` | Available extension versions with installed flag, trust and requires metadata | | `postgresql/bgwriter.tsv` | `pg_stat_bgwriter` | Background writer statistics | | `postgresql/blocking_locks.tsv` | Complex query | Blocking/blocked lock pairs | | `postgresql/checkpointer.tsv` | `pg_stat_checkpointer` | Checkpointer statistics | @@ -260,7 +262,9 @@ Instance-level PostgreSQL collectors. Files stored in `postgresql/`. | `postgresql/stat_progress_copy.tsv` | `pg_stat_progress_copy` | COPY progress (PG14+) | | `postgresql/stat_progress_create_index.tsv` | `pg_stat_progress_create_index` | CREATE INDEX progress (PG12+) | | `postgresql/stat_progress_vacuum.tsv` | `pg_stat_progress_vacuum` | VACUUM progress (PG9.6+) | +| `postgresql/stat_replication_slots.tsv` | `pg_stat_replication_slots` | Replication slot stats incl. spill counters (PG14+) | | `postgresql/stat_slru.tsv` | `pg_stat_slru` | SLRU cache statistics | +| `postgresql/stat_ssl.tsv` | `pg_stat_ssl` | Per-backend SSL/TLS state | | `postgresql/stat_statements_calls.tsv` | `pg_stat_statements` | Top 100 queries by call count | | `postgresql/stat_statements_max_time.tsv` | `pg_stat_statements` | Top 100 queries by max execution time | | `postgresql/stat_statements_total_time.tsv` | `pg_stat_statements` | Top 100 queries by total execution time | @@ -281,21 +285,24 @@ Collected for each accessible database. Files stored in `databases/{dbname}/`. | File | Source | Description | |------|--------|-------------| +| `bloat.tsv` | `pg_stats` heuristic | Table bloat estimate (heuristic) | | `extensions.tsv` | `pg_extension` | Installed extensions | | `funcs.tsv` | `pg_proc WHERE prokind='f'` | Functions | -| `indexes.tsv` | `pg_indexes` | Indexes | +| `indexes.tsv` | `pg_index` + `pg_stat_all_indexes` | Top 1000 indexes by size with semantic key, validity, scan counters and size | | `languages.tsv` | `pg_language` | Procedural languages | | `operators.tsv` | `pg_operator` | Operators | | `partitioned_tables.tsv` | `pg_partitioned_table` | Partitioned tables (PG10+) | | `partitions.tsv` | `pg_inherits` | Partition relationships | +| `pgstattuple.tsv` | `pgstattuple_approx()` | Authoritative bloat (when `pgstattuple` extension is installed) | | `procs.tsv` | `pg_proc WHERE prokind='p'` | Procedures (PG11+) | | `publication_tables.tsv` | `pg_publication_tables` | Tables in publications | | `publications.tsv` | `pg_publication` | Logical replication publications | | `schemas.tsv` | `pg_namespace` | Schemas | +| `sequences.tsv` | `pg_sequences` | Sequences with last/min/max values (PG10+) | | `stat_database.tsv` | `pg_stat_database` | Per-database statistics | | `statistics.tsv` | `pg_statistic_ext` | Extended statistics (PG10+) | | `subscription_tables.tsv` | `pg_subscription_rel` | Subscription relation states | -| `tables.tsv` | `pg_tables` | Tables | +| `tables.tsv` | `pg_class` + `pg_stat_all_tables` | Top 1000 tables by size with persistence, options, heap and table sizes, toast mapping, dead-tup counters and vacuum/analyze timestamps | | `triggers.tsv` | `pg_trigger` | Triggers | | `types.tsv` | `pg_type` | Data types | diff --git a/docs/index.md b/docs/index.md index d5af0d5..e7d44cb 100644 --- a/docs/index.md +++ b/docs/index.md @@ -164,19 +164,20 @@ For a complete reference of all collected data, see [data.md](data.md). - **Configuration & files**: `pg_db_role_setting`, `pg_file_settings`, `pg_hba.conf`, `pg_hba_file_rules`, `pg_ident.conf`, `pg_settings`, `pg_tablespace`, `postgresql.auto.conf`, `postgresql.conf`, `recovery.conf`, `recovery.done` - **Activity & monitoring**: `pg_locks`, `pg_postmaster_start_time()`, `pg_prepared_xacts`, `pg_shmem_allocations`, `pg_stat_activity` -- **Statistics views**: `pg_stat_archiver`, `pg_stat_bgwriter`, `pg_stat_checkpointer` (PG17+), `pg_stat_database_conflicts`, `pg_stat_io` (PG16+), `pg_stat_slru`, `pg_stat_statements` (if installed), `pg_stat_wal` (PG14+), `pg_stat_wal_receiver` -- **Replication & WAL**: `pg_current_wal_lsn()`, `pg_replication_origin_status`, `pg_replication_slots`, `pg_stat_replication`, `pg_subscription` +- **Statistics views**: `pg_stat_archiver`, `pg_stat_bgwriter`, `pg_stat_checkpointer` (PG17+), `pg_stat_database_conflicts`, `pg_stat_io` (PG16+), `pg_stat_slru`, `pg_stat_ssl`, `pg_stat_statements` (if installed), `pg_stat_wal` (PG14+), `pg_stat_wal_receiver` +- **Replication & WAL**: `pg_current_wal_lsn()`, `pg_replication_origin_status`, `pg_replication_slots`, `pg_stat_replication`, `pg_stat_replication_slots` (PG14+, spill counters), `pg_subscription` - **Progress tracking**: `pg_stat_progress_analyze`, `pg_stat_progress_basebackup`, `pg_stat_progress_cluster`, `pg_stat_progress_copy`, `pg_stat_progress_create_index`, `pg_stat_progress_vacuum` -- **Catalog**: `pg_available_extensions`, `pg_database`, `pg_database_size()`, `pg_roles`, `pg_tablespace_size()`, `version()` +- **Catalog**: `pg_available_extension_versions`, `pg_database` (incl. `datfrozenxid`/`datminmxid` wraparound headroom and `datconnlimit`), `pg_database_size()`, `pg_roles`, `pg_tablespace_size()`, `version()` **Per-Database** -- **Schema objects**: `pg_indexes`, `pg_namespace`, `pg_operator`, `pg_tables`, `pg_type` +- **Schema objects**: `pg_class` + `pg_stat_all_tables` (tables incl. dead-tup counters, vacuum/analyze timestamps, `reloptions`, `reltoastrelid`, `relpersistence`), `pg_index` + `pg_stat_all_indexes` (indexes incl. validity, scan counters, size), `pg_namespace`, `pg_operator`, `pg_sequences`, `pg_type` - **Functions & procedures**: `pg_proc` - **Triggers & partitioning**: `pg_inherits`, `pg_partitioned_table`, `pg_trigger` - **Logical replication**: `pg_publication`, `pg_publication_tables`, `pg_subscription_rel` -- **Extensions**: `pg_extension`, `pg_language`, `pg_statistic_ext` +- **Extensions**: `pg_extension`, `pg_language`, `pg_statistic_ext`, `pgstattuple` (if installed — authoritative bloat estimate via `pgstattuple_approx()`) - **Statistics**: `pg_stat_database` (conflicts, deadlocks, temp files, stats reset) +- **Bloat estimate**: heuristic from `pg_stats` (always emitted) **[pg_statviz](https://github.com/vyruss/pg_statviz) Extension** (if present) diff --git a/postgres_tasks.go b/postgres_tasks.go index 7a376da..2ed902b 100644 --- a/postgres_tasks.go +++ b/postgres_tasks.go @@ -44,7 +44,10 @@ var postgresQueryTasks = []SimpleQueryTask{ { Name: "available_extensions", ArchivePath: "postgresql/available_extensions.tsv", - Query: "SELECT * FROM pg_available_extensions ORDER BY name", + Query: `SELECT name, version, installed, superuser, trusted, + relocatable, schema, requires, comment +FROM pg_available_extension_versions +ORDER BY name, version`, }, { Name: "bgwriter", @@ -105,7 +108,12 @@ WHERE NOT blocked_locks.granted`, { Name: "databases", ArchivePath: "postgresql/databases.tsv", - Query: "SELECT oid, datname, datdba, encoding, datcollate, datctype FROM pg_database ORDER BY datname", + Query: `SELECT oid, datname, datdba, encoding, datcollate, datctype, + datistemplate, datallowconn, datconnlimit, + datfrozenxid, age(datfrozenxid) AS frozenxid_age, + datminmxid, mxid_age(datminmxid) AS minmxid_age +FROM pg_database +ORDER BY datname`, }, { Name: "databases_blk", @@ -227,11 +235,26 @@ WHERE state != 'idle'`, ArchivePath: "postgresql/stat_progress_vacuum.tsv", Query: "SELECT * FROM pg_stat_progress_vacuum", }, + { + Name: "stat_replication_slots", + ArchivePath: "postgresql/stat_replication_slots.tsv", + Query: "SELECT * FROM pg_stat_replication_slots ORDER BY slot_name", + }, { Name: "stat_slru", ArchivePath: "postgresql/stat_slru.tsv", Query: "SELECT * FROM pg_stat_slru ORDER BY name", }, + { + Name: "stat_ssl", + ArchivePath: "postgresql/stat_ssl.tsv", + Query: `SELECT s.pid, s.ssl, s.version, s.cipher, s.bits, + s.client_dn, s.client_serial, s.issuer_dn, + a.usename, a.application_name, a.client_addr +FROM pg_stat_ssl s +LEFT JOIN pg_stat_activity a ON a.pid = s.pid +ORDER BY s.pid`, + }, { Name: "stat_statements_calls", ArchivePath: "postgresql/stat_statements_calls.tsv", @@ -298,6 +321,71 @@ WHERE state != 'idle'`, // Per-database query tasks (sorted alphabetically by name) // These are per-database tasks - ArchivePath will be formatted with dbname var perDatabaseQueryTasks = []SimpleQueryTask{ + { + Name: "bloat", + ArchivePath: "databases/%s/bloat.tsv", + Query: ` +SELECT current_database() AS current_database, + schemaname, + tablename, + ROUND((CASE WHEN otta = 0 THEN 0.0 + ELSE sml.relpages::FLOAT / otta END)::NUMERIC, 1) + AS table_bloat_ratio, + CASE WHEN relpages < otta THEN 0 + ELSE bs * (sml.relpages - otta)::BIGINT END AS wastedbytes, + iname, + ituples, + ipages, + iotta +FROM ( + SELECT schemaname, tablename, cc.reltuples, cc.relpages, bs, + CEIL((cc.reltuples * + ((datahdr + ma - + (CASE WHEN datahdr % ma = 0 THEN ma + ELSE datahdr % ma END)) + nullhdr2 + 4)) + / (bs - 20::FLOAT)) AS otta, + COALESCE(c2.relname, '?') AS iname, + COALESCE(c2.reltuples, 0) AS ituples, + COALESCE(c2.relpages, 0) AS ipages, + COALESCE(CEIL((c2.reltuples * (datahdr - 12)) + / (bs - 20::FLOAT)), 0) AS iotta + FROM ( + SELECT ma, bs, schemaname, tablename, + (datawidth + + (hdr + ma - + (CASE WHEN hdr % ma = 0 THEN ma + ELSE hdr % ma END)))::NUMERIC AS datahdr, + (maxfracsum * + (nullhdr + ma - + (CASE WHEN nullhdr % ma = 0 THEN ma + ELSE nullhdr % ma END))) AS nullhdr2 + FROM ( + SELECT schemaname, tablename, hdr, ma, bs, + SUM((1 - null_frac) * avg_width) AS datawidth, + MAX(null_frac) AS maxfracsum, + hdr + (SELECT 1 + COUNT(*) / 8 + FROM pg_stats s2 + WHERE null_frac <> 0 + AND s2.schemaname = s.schemaname + AND s2.tablename = s.tablename) AS nullhdr + FROM pg_stats s, + (SELECT (SELECT current_setting('block_size')::NUMERIC) + AS bs, + CASE WHEN SUBSTRING(v, 12, 3) + IN ('8.0', '8.1', '8.2') THEN 27 + ELSE 23 END AS hdr, + CASE WHEN v ~ 'mingw32' THEN 8 + ELSE 4 END AS ma + FROM (SELECT version() AS v) AS foo) AS constants + GROUP BY 1, 2, 3, 4, 5) AS foo) AS rs + JOIN pg_class cc ON cc.relname = rs.tablename + JOIN pg_namespace nn ON cc.relnamespace = nn.oid + AND nn.nspname = rs.schemaname + LEFT JOIN pg_index i ON indrelid = cc.oid + LEFT JOIN pg_class c2 ON c2.oid = i.indexrelid) AS sml +ORDER BY wastedbytes DESC, schemaname, tablename + `, + }, { Name: "extensions", ArchivePath: "databases/%s/extensions.tsv", @@ -312,9 +400,32 @@ var perDatabaseQueryTasks = []SimpleQueryTask{ Name: "indexes", ArchivePath: "databases/%s/indexes.tsv", Query: ` - SELECT schemaname, tablename, indexname, indexdef - FROM pg_indexes - ORDER BY schemaname, tablename, indexname + SELECT n.nspname AS schemaname, + t.relname AS tablename, + c.relname AS indexname, + pg_get_indexdef(i.indexrelid) AS indexdef, + i.indrelid::regclass AS indrelid, + i.indexrelid::regclass AS indexrelid, + i.indisunique, + i.indisprimary, + i.indisvalid, + i.indclass::text AS indclass, + i.indkey::text AS indkey, + pg_get_expr(i.indexprs, i.indrelid) AS indexprs, + pg_get_expr(i.indpred, i.indrelid) AS indpred, + pg_relation_size(i.indexrelid) AS index_size, + s.idx_scan, + s.idx_tup_read, + s.idx_tup_fetch + FROM pg_index i + JOIN pg_class c ON c.oid = i.indexrelid + JOIN pg_class t ON t.oid = i.indrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_stat_all_indexes s ON s.indexrelid = i.indexrelid + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + ORDER BY pg_relation_size(i.indexrelid) DESC NULLS LAST, schemaname, tablename, indexname + LIMIT 1000 `, }, { @@ -343,6 +454,28 @@ var perDatabaseQueryTasks = []SimpleQueryTask{ ORDER BY inhparent, inhseqno `, }, + { + Name: "pgstattuple", + ArchivePath: "databases/%s/pgstattuple.tsv", + Query: `SELECT n.nspname AS schemaname, + c.relname AS tablename, + p.table_len, + p.approx_tuple_count AS tuple_count, + p.approx_tuple_len AS tuple_len, + p.approx_tuple_percent AS tuple_percent, + p.dead_tuple_count, + p.dead_tuple_len, + p.dead_tuple_percent, + p.approx_free_space AS free_space, + p.approx_free_percent AS free_percent +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +CROSS JOIN LATERAL pgstattuple_approx(c.oid) p +WHERE c.relkind IN ('r', 'm') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' +ORDER BY p.dead_tuple_percent DESC NULLS LAST`, + }, { Name: "procs", ArchivePath: "databases/%s/procs.tsv", @@ -363,6 +496,16 @@ var perDatabaseQueryTasks = []SimpleQueryTask{ ArchivePath: "databases/%s/schemas.tsv", Query: "SELECT * FROM pg_namespace ORDER BY nspname", }, + { + Name: "sequences", + ArchivePath: "databases/%s/sequences.tsv", + Query: `SELECT schemaname, sequencename, + data_type::regtype::text AS data_type, + last_value, max_value, min_value, increment_by, + cycle, cache_size +FROM pg_sequences +ORDER BY schemaname, sequencename`, + }, { Name: "stat_database", ArchivePath: "databases/%s/stat_database.tsv", @@ -389,9 +532,38 @@ WHERE datname = current_database()`, Name: "tables", ArchivePath: "databases/%s/tables.tsv", Query: ` - SELECT schemaname, tablename, tableowner, tablespace, hasindexes, hasrules, hastriggers - FROM pg_tables - ORDER BY schemaname, tablename + SELECT n.nspname AS schemaname, + c.relname AS tablename, + pg_get_userbyid(c.relowner) AS tableowner, + t.spcname AS tablespace, + c.relhasindex AS hasindexes, + c.relhasrules AS hasrules, + c.relhastriggers AS hastriggers, + c.relpersistence, + c.reltuples, + c.reloptions, + pg_relation_size(c.oid) AS heap_size, + pg_table_size(c.oid) AS table_size, + c.reltoastrelid::regclass AS toast_table, + CASE WHEN c.reltoastrelid <> 0 + THEN pg_relation_size(c.reltoastrelid) END AS toast_size, + s.n_live_tup, + s.n_dead_tup, + s.n_mod_since_analyze, + s.n_ins_since_vacuum, + s.last_vacuum, + s.last_autovacuum, + s.last_analyze, + s.last_autoanalyze + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace + LEFT JOIN pg_stat_all_tables s ON s.relid = c.oid + WHERE c.relkind IN ('r', 'p') + AND n.nspname NOT IN ('pg_catalog', 'information_schema') + AND n.nspname NOT LIKE 'pg_toast%' + ORDER BY pg_table_size(c.oid) DESC NULLS LAST, schemaname, tablename + LIMIT 1000 `, }, { diff --git a/postgres_test.go b/postgres_test.go index 98182c8..0d7b291 100644 --- a/postgres_test.go +++ b/postgres_test.go @@ -69,7 +69,9 @@ func TestPostgreSQLCollectors(t *testing.T) { {"stat_progress_copy", "postgresql/stat_progress_copy.tsv"}, {"stat_progress_create_index", "postgresql/stat_progress_create_index.tsv"}, {"stat_progress_vacuum", "postgresql/stat_progress_vacuum.tsv"}, + {"stat_replication_slots", "postgresql/stat_replication_slots.tsv"}, {"stat_slru", "postgresql/stat_slru.tsv"}, + {"stat_ssl", "postgresql/stat_ssl.tsv"}, {"stat_statements_calls", "postgresql/stat_statements_calls.tsv"}, {"stat_statements_max_time", "postgresql/stat_statements_max_time.tsv"}, {"stat_statements_total_time", "postgresql/stat_statements_total_time.tsv"}, diff --git a/radar_test.go b/radar_test.go index 6d2fd6b..f55303d 100644 --- a/radar_test.go +++ b/radar_test.go @@ -520,6 +520,74 @@ func TestPerDatabaseTasksStructure(t *testing.T) { } } +// TestQueryTaskColumnsCoverage asserts that the queries we expanded continue +// to reference the column tokens we rely on. Catches accidental column +// removal during future edits to these query strings. +func TestQueryTaskColumnsCoverage(t *testing.T) { + checks := []struct { + taskList string + taskName string + mustContain []string + }{ + {"postgres", "databases", []string{ + "datfrozenxid", "datminmxid", "datconnlimit", + "datistemplate", "datallowconn", + }}, + {"perDB", "tables", []string{ + "n_live_tup", "n_dead_tup", "last_autovacuum", "last_analyze", + "reltuples", "reloptions", "reltoastrelid", "relpersistence", + "pg_relation_size", "pg_table_size", "LIMIT 1000", + }}, + {"perDB", "indexes", []string{ + "indrelid", "indclass", "indkey", "indisvalid", + "idx_scan", "pg_relation_size", "LIMIT 1000", + }}, + {"perDB", "sequences", []string{ + "pg_sequences", "last_value", "max_value", "min_value", "increment_by", + }}, + {"postgres", "stat_ssl", []string{ + "pg_stat_ssl", "ssl", "cipher", + }}, + {"postgres", "stat_replication_slots", []string{ + "pg_stat_replication_slots", + }}, + {"perDB", "bloat", []string{ + "table_bloat_ratio", "wastedbytes", + }}, + {"perDB", "pgstattuple", []string{ + "pgstattuple_approx", + }}, + } + + taskByName := func(list []SimpleQueryTask, name string) *SimpleQueryTask { + for i := range list { + if list[i].Name == name { + return &list[i] + } + } + return nil + } + + for _, c := range checks { + var task *SimpleQueryTask + switch c.taskList { + case "postgres": + task = taskByName(postgresQueryTasks, c.taskName) + case "perDB": + task = taskByName(perDatabaseQueryTasks, c.taskName) + } + if task == nil { + t.Errorf("task %q not found in %s tasks", c.taskName, c.taskList) + continue + } + for _, want := range c.mustContain { + if !strings.Contains(task.Query, want) { + t.Errorf("task %q query missing expected token %q", c.taskName, want) + } + } + } +} + // TestPgStatvizTasksStructure verifies all pg_statviz tasks have required fields func TestPgStatvizTasksStructure(t *testing.T) { for i, task := range pgStatvizQueryTasks { diff --git a/system_tasks_darwin.go b/system_tasks_darwin.go index 2879e2f..7787359 100644 --- a/system_tasks_darwin.go +++ b/system_tasks_darwin.go @@ -49,6 +49,12 @@ var systemCommandTasks = []SimpleCommandTask{ Command: "hostname", Args: []string{}, }, + { + Name: "hostname_fqdn", + ArchivePath: "system/hostname_fqdn.out", + Command: "hostname", + Args: []string{"-f"}, + }, { Name: "hypervisor-check", ArchivePath: "system/hypervisor.out", diff --git a/system_tasks_linux.go b/system_tasks_linux.go index 6d4ef07..7d1b4f0 100644 --- a/system_tasks_linux.go +++ b/system_tasks_linux.go @@ -65,6 +65,11 @@ var systemCommandTasks = []SimpleCommandTask{ Name: "hostname", ArchivePath: "system/hostname.out", Command: "hostname", + }, + { + Name: "hostname_fqdn", + ArchivePath: "system/hostname_fqdn.out", + Command: "hostname", Args: []string{"-f"}, }, { @@ -335,7 +340,7 @@ var systemCommandTasks = []SimpleCommandTask{ Name: "read_ahead", ArchivePath: "system/read_ahead.out", Command: "sh", - Args: []string{"-c", "blockdev --getra /dev/sd* /dev/nvme* 2>/dev/null"}, + Args: []string{"-c", "for d in /dev/sd* /dev/nvme* /dev/vd* /dev/xvd*; do [ -b \"$d\" ] && ra=$(blockdev --getra \"$d\" 2>/dev/null) && [ -n \"$ra\" ] && echo \"$d: $ra\"; done; exit 0"}, }, { Name: "transparent_hugepage",