Merge branch 'md_temp' into 'master'

buffers, pages, diagnostics: temperature columns, CLUSTER candidates, overview buffer stats

See merge request sysadmins/pgdu!5

Author: brainexe

go.mod

@@ -10,7 +10,6 @@ require (
 	github.com/jackc/pgx/v5 v5.10.0
 	github.com/muesli/termenv v0.16.0
 	github.com/spf13/pflag v1.0.10
-	golang.org/x/sync v0.22.0
 )
 
 require (
@@ -32,6 +31,7 @@ require (
 	github.com/muesli/cancelreader v0.2.2 // indirect
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/xo/terminfo v1.0.0 // indirect
+	golang.org/x/sync v0.22.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
 	golang.org/x/text v0.41.0 // indirect
 )

internal/pg/buffers.go

@@ -175,6 +175,38 @@ func (c *Client) TableBufferUsageCounts(ctx context.Context, db string, oid uint
 	return counts, blockSize, err
 }
 
+// PageBuffers maps the blocks of one main-fork window of a relation to their
+// shared-buffers state (usagecount, dirty). Blocks not currently cached are
+// absent from the map. Powers the page inspector's per-page temperature
+// column; callers treat failures as "no temperature data" rather than fatal.
+func (c *Client) PageBuffers(ctx context.Context, db string, oid uint32, start, count int32) (map[int64]PageBuffer, error) {
+	if err := c.EnsureBufferCache(ctx, db); err != nil {
+		return nil, err
+	}
+	pool, err := c.PoolFor(ctx, db)
+	if err != nil {
+		return nil, err
+	}
+	rows, err := pool.Query(ctx, sqlPageBuffers, oid, start, count)
+	if err != nil {
+		return nil, fmt.Errorf("page buffers for oid %d in %q: %w", oid, db, err)
+	}
+	defer rows.Close()
+	bufs := make(map[int64]PageBuffer)
+	for rows.Next() {
+		var blkno int64
+		var b PageBuffer
+		if err := rows.Scan(&blkno, &b.UsageCount, &b.Dirty); err != nil {
+			return nil, fmt.Errorf("page buffers for oid %d in %q: %w", oid, db, err)
+		}
+		bufs[blkno] = b
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("page buffers for oid %d in %q: %w", oid, db, err)
+	}
+	return bufs, nil
+}
+
 // BufferCacheSummary returns the cluster-wide shared_buffers occupancy split
 // between the current database, anything else, and free pages.
 func (c *Client) BufferCacheSummary(ctx context.Context, db string) (BufferCacheSummary, error) {

internal/pg/diagnostic_defs.go

@@ -98,6 +98,42 @@ var Diagnostics = []Diagnostic{
 			range scans, not single-row lookups — check the workload first, create
 			the BRIN, verify the plans still prune, then drop the btree.`,
 	},
+	{
+		Key:         "index_cluster_candidates",
+		PerDB:       true,
+		Title:       "CLUSTER candidates (fragmented)",
+		Category:    "index",
+		Description: "hot btree indexes on low-correlation columns (|corr| ≤ 0.5) whose scans return many scattered rows — candidates for CLUSTER / pg_repack",
+		SQL:         sqlDiagIndexClusterCandidates,
+		Bar:         "scatter_pct",
+		Sort:        "disk_pain",
+		Kinds: map[string]DiagColumnKind{
+			"scatter_pct":   DiagPercentBad,
+			"heap_miss_pct": DiagPercentBad,
+			"disk_pain":     DiagCount,
+			"scans":         DiagCount,
+			"tuples_read":   DiagCount,
+		},
+		Help: `Btree indexes whose scans return several rows each (tup_per_scan ≥ 5)
+			while the heap is ordered unlike the index (scatter_pct = (1−|corr|)×100;
+			100 means heap and index order are unrelated), so every scan fetches
+			rows scattered across many heap pages. The classic shape is a per-key
+			detail table — a player inventory, per-user events — always queried by
+			that key; rows_per_key confirms it. disk_pain, the default order, is
+			tuples_read × scatter × heap-miss ratio — an estimate of fetches that
+			were both scattered and served from disk; a fragmented table the
+			buffer cache fully absorbs (heap_miss_pct ≈ 0) scores near zero no
+			matter how hot its counters are. Ignore rows
+			with rows_per_key ≈ 1 unless the scans are range scans (cleanup by
+			timestamp, id ranges) — clustering only helps when scanned rows share
+			pages. Fix by rewriting the heap in index order: CLUSTER <table>
+			USING <index> (takes an ACCESS EXCLUSIVE lock) or pg_repack for an
+			online rewrite. Churny tables re-fragment over time, so re-run
+			periodically; clustered = t means the table was already CLUSTERed on
+			that index and has degraded since. scans/tuples_read/heap_miss_pct are
+			cumulative since the last stats reset, so freshly reset stats
+			understate hotness.`,
+	},
 	// "All indexes" was merged into "Index sizes" below (which now also carries
 	// scan counts and the unique flag, across all schemas), so it is no longer a
 	// separate entry. Kept commented for reference rather than deleted.

internal/pg/diagnostics.go

@@ -28,6 +28,7 @@ const (
 	DiagDuration                            // elapsed time in ms (Num): right-aligned, coloured by absolute magnitude band (ms→green, s→yellow, min→red)
 	DiagBackendState                        // pg_stat_activity state: coloured per value (active→green, idle-in-xact→yellow, aborted→red, idle→muted)
 	DiagPercentBad                          // 0–100 % where higher is worse: cell text graded green→red on an absolute scale (e.g. dead-tuple %, seq-scan %)
+	DiagCount                               // large cumulative counter: rendered humanized (1.2k/3.4M/5.1G) from Num; sorts and sums on the raw value
 )
 
 // DiagColumn describes one column of a diagnostic result set.

internal/pg/queries_buffers.go

@@ -78,6 +78,20 @@ LEFT   JOIN b ON b.usagecount = g.usagecount
 ORDER  BY g.usagecount
 `
 
+// sqlPageBuffers lists the shared buffers holding one window of a relation's
+// main fork: block number → usagecount/isdirty, feeding the page inspector's
+// per-page temperature column. pg_relation_filenode resolves rewritten and
+// mapped relations; reldatabase 0 covers shared catalogs. One full buffer-pool
+// scan per window — the same cost class as the aggregate queries above.
+const sqlPageBuffers = `
+SELECT relblocknumber::bigint, usagecount::int, isdirty
+FROM   pg_buffercache
+WHERE  relfilenode = pg_relation_filenode($1::oid::regclass)
+  AND  reldatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database()))
+  AND  relforknumber = 0
+  AND  relblocknumber >= $2::int AND relblocknumber < $2::int + $3::int
+`
+
 // sqlBufferStats reports per-table shared-buffer footprint and cumulative I/O
 // counters for one schema. Buffer footprint sums the heap, toast and every
 // index for the table, so the "biggest cache hog" answer matches the user's

internal/pg/queries_diag.go

@@ -238,6 +238,80 @@ WHERE am.amname = 'btree'                       -- only B-tree indexes
 ORDER BY abs(s.correlation) DESC, pg_relation_size(t.oid) DESC
 `
 
+// sqlDiagIndexClusterCandidates is the inverse of the BRIN query: btree indexes
+// whose leading column has *low* physical correlation (|correlation| ≤ 0.5) yet
+// whose scans return many rows each — the fragmented-detail-table pattern (e.g.
+// a player inventory always fetched by user id) where every index scan touches
+// scattered heap pages and a CLUSTER/pg_repack rewrite collapses them onto few.
+// Only the leading column matters (k.ord = 1): its correlation decides the heap
+// locality of a prefix scan. Unique/primary indexes stay in deliberately — the
+// tuples-per-scan floor already drops point lookups (~1 tuple/scan), and a PK
+// like (user_id, item_id) scanned by prefix is exactly the target. Expression
+// indexes drop out naturally (indkey attnum 0 has no pg_attribute/pg_stats row).
+// scatter_pct ((1−|corr|)×100, higher = worse) is the bar. heap_miss_pct
+// (per-table, from pg_statio_user_tables) separates fragmented tables that
+// actually hit disk from ones the buffer cache absorbs anyway, and the default
+// sort is disk_pain — tuples_read × scatter × heap-miss ratio — so the rows
+// whose scattered fetches actually cost disk reads rank first, not merely the
+// hottest counters on fully-cached tables.
+const sqlDiagIndexClusterCandidates = `
+SELECT
+    t.relname                                           AS table_name,
+    i.relname                                           AS index_name,
+    a.attname                                           AS column_name,
+    round(((1 - abs(s.correlation)) * 100)::numeric, 1) AS scatter_pct,
+    -- Fraction of the table's heap block reads that missed shared_buffers: a
+    -- fragmented table that lives entirely in cache costs little; one that
+    -- hits disk on scattered pages is the real CLUSTER payoff.
+    round(100 * sio.heap_blks_read::numeric
+          / nullif(sio.heap_blks_hit + sio.heap_blks_read, 0), 1) AS heap_miss_pct,
+    -- Composite ranking: tuple fetches × scattered fraction × disk-miss
+    -- fraction ≈ fetches that were both on a random page AND read from disk.
+    -- Table size needs no extra factor — a table the cache absorbs already
+    -- scores ~0 through the miss ratio. NULL (no heap I/O counted) sorts last.
+    round(st.idx_tup_read * (1 - abs(s.correlation))
+          * sio.heap_blks_read::numeric
+          / nullif(sio.heap_blks_hit + sio.heap_blks_read, 0)) AS disk_pain,
+    st.idx_scan                                         AS scans,
+    st.idx_tup_read                                     AS tuples_read,
+    round(st.idx_tup_read::numeric / st.idx_scan, 1)    AS tup_per_scan,
+    CASE
+        WHEN s.n_distinct > 0
+            THEN round((t.reltuples / s.n_distinct)::numeric, 1)
+        WHEN s.n_distinct < 0
+            THEN round((-1 / s.n_distinct)::numeric, 1)
+    END                                                 AS rows_per_key,
+    idx.indisclustered                                  AS clustered,
+    pg_size_pretty(pg_relation_size(t.oid))             AS table_size,
+    pg_size_pretty(pg_relation_size(i.oid))             AS index_size
+FROM pg_index idx
+JOIN pg_class i        ON i.oid = idx.indexrelid           -- the index
+JOIN pg_class t        ON t.oid = idx.indrelid             -- the table
+JOIN pg_namespace n    ON n.oid = t.relnamespace
+JOIN pg_am am          ON am.oid = i.relam                 -- access method
+-- pg_stat_user_indexes also scopes the result to user schemas, so no explicit
+-- pg_catalog/information_schema exclusion is needed here.
+JOIN pg_stat_user_indexes st ON st.indexrelid = idx.indexrelid
+JOIN pg_statio_user_tables sio ON sio.relid = t.oid
+JOIN LATERAL unnest(idx.indkey) WITH ORDINALITY AS k(attnum, ord) ON k.ord = 1
+JOIN pg_attribute a    ON a.attrelid = t.oid AND a.attnum = k.attnum
+LEFT JOIN pg_stats s   ON s.schemaname = n.nspname
+                       AND s.tablename  = t.relname
+                       AND s.attname    = a.attname
+WHERE am.amname = 'btree'                       -- only B-tree (CLUSTER's home turf)
+  AND idx.indisvalid
+  AND a.attnum > 0                              -- skip expression-index entries
+  AND t.reltuples > 100000                      -- only tables big enough to matter
+  AND s.correlation IS NOT NULL
+  AND abs(s.correlation) <= 0.5                 -- heap order does not follow the index
+  AND st.idx_scan > 0
+  -- Multi-row scans only, written multiplication-side to avoid division by
+  -- zero before the idx_scan > 0 predicate is applied: point lookups
+  -- (~1 tuple/scan) don't suffer from fragmentation and would be noise.
+  AND st.idx_tup_read >= 5 * st.idx_scan
+ORDER BY disk_pain DESC NULLS LAST
+`
+
 // sqlDiagIndexShowAll was the old per-index listing (public schema only). Its
 // useful columns (scan count, tuples read, unique flag) were folded into
 // sqlDiagIndexShowSize, so it is no longer registered. Kept commented for

internal/pg/queries_tableoverview.go

@@ -54,3 +54,44 @@ WHERE  n.nspname = $1
   AND  c.relkind IN ('r','m','p')
 ORDER  BY total_bytes DESC
 `
+
+// sqlTableStatsBuffers sums the current shared-buffer footprint (total and
+// dirty bytes) per table for one schema from pg_buffercache, covering the heap,
+// TOAST and every index — the same filenode aggregation as sqlBufferStats but
+// without the sizing/statio columns the overview already has. It is a separate
+// best-effort query rather than a join in sqlTableStats because pg_buffercache
+// is an extension needing pg_monitor: its absence must not take down the whole
+// overview, only blank the two buffer columns.
+const sqlTableStatsBuffers = `
+WITH bc AS (
+  SELECT relfilenode,
+         COUNT(*)                         AS bufs,
+         COUNT(*) FILTER (WHERE isdirty)  AS dirty_bufs
+  FROM   pg_buffercache
+  WHERE  reldatabase IN (0, (SELECT oid FROM pg_database WHERE datname = current_database()))
+  GROUP  BY relfilenode
+),
+filenodes AS (
+  SELECT c.oid AS tab_oid, pg_relation_filenode(c.oid) AS fn
+  FROM   pg_class c
+  JOIN   pg_namespace n ON n.oid = c.relnamespace
+  WHERE  n.nspname = $1 AND c.relkind IN ('r','m','p')
+  UNION ALL
+  SELECT c.oid, pg_relation_filenode(c.reltoastrelid)
+  FROM   pg_class c
+  JOIN   pg_namespace n ON n.oid = c.relnamespace
+  WHERE  n.nspname = $1 AND c.relkind IN ('r','m','p') AND c.reltoastrelid <> 0
+  UNION ALL
+  SELECT c.oid, pg_relation_filenode(i.indexrelid)
+  FROM   pg_class c
+  JOIN   pg_namespace n ON n.oid = c.relnamespace
+  JOIN   pg_index i ON i.indrelid = c.oid
+  WHERE  n.nspname = $1 AND c.relkind IN ('r','m','p')
+)
+SELECT f.tab_oid,
+       (COALESCE(SUM(bc.bufs), 0)       * current_setting('block_size')::int)::bigint AS buffered_bytes,
+       (COALESCE(SUM(bc.dirty_bufs), 0) * current_setting('block_size')::int)::bigint AS dirty_bytes
+FROM   filenodes f
+LEFT   JOIN bc ON bc.relfilenode = f.fn
+GROUP  BY f.tab_oid
+`

internal/pg/tableoverview.go

@@ -3,6 +3,7 @@ package pg
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/jackc/pgx/v5"
 )
@@ -11,12 +12,17 @@ import (
 // table in db.schema, with size, write/scan activity, cache-hit counters,
 // maintenance counters and storage options gathered in a single query. Result
 // is ordered by total size; the TUI re-sorts by the active column.
-func (c *Client) ListTableStats(ctx context.Context, db, schema string) ([]TableStat, error) {
+//
+// Two best-effort extras never fail the load: the current shared-buffer
+// footprint per table (pg_buffercache — extension, needs pg_monitor; absent →
+// BufsKnown stays false) and the database's stats_reset timestamp dating the
+// cumulative counters (zero when unavailable or never reset).
+func (c *Client) ListTableStats(ctx context.Context, db, schema string) ([]TableStat, time.Time, error) {
 	pool, err := c.PoolFor(ctx, db)
 	if err != nil {
-		return nil, err
+		return nil, time.Time{}, err
 	}
-	return collect(ctx, pool, fmt.Sprintf("list table stats in %q.%q", db, schema), sqlTableStats, []any{schema},
+	rows, err := collect(ctx, pool, fmt.Sprintf("list table stats in %q.%q", db, schema), sqlTableStats, []any{schema},
 		func(row pgx.CollectableRow) (TableStat, error) {
 			t := TableStat{DB: db, Schema: schema}
 			err := row.Scan(
@@ -34,4 +40,39 @@ func (c *Client) ListTableStats(ctx context.Context, db, schema string) ([]Table
 			)
 			return t, err
 		})
+	if err != nil {
+		return nil, time.Time{}, err
+	}
+
+	type bufRow struct {
+		oid             uint32
+		buffered, dirty int64
+	}
+	bufs := collectBestEffort(ctx, pool, sqlTableStatsBuffers, []any{schema},
+		func(r pgx.Rows) (bufRow, bool) {
+			var b bufRow
+			return b, r.Scan(&b.oid, &b.buffered, &b.dirty) == nil
+		})
+	if bufs != nil {
+		byOID := make(map[uint32]bufRow, len(bufs))
+		for _, b := range bufs {
+			byOID[b.oid] = b
+		}
+		// Every table in the schema gets a row from the LEFT JOIN, so a merged
+		// zero is a real "nothing cached", not missing data.
+		for i := range rows {
+			if b, ok := byOID[rows[i].OID]; ok {
+				rows[i].BufferedBytes = b.buffered
+				rows[i].DirtyBytes = b.dirty
+				rows[i].BufsKnown = true
+			}
+		}
+	}
+
+	var reset time.Time
+	var resetPtr *time.Time
+	if pool.QueryRow(ctx, sqlMaintTableStatsReset).Scan(&resetPtr) == nil && resetPtr != nil {
+		reset = *resetPtr
+	}
+	return rows, reset, nil
 }

internal/pg/triage.go

@@ -5,10 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"sort"
+	"sync"
 	"time"
 
-	"golang.org/x/sync/errgroup"
-
 	"pgdu/internal/humanize"
 )
 
@@ -157,8 +156,8 @@ func (c *Client) Triage(ctx context.Context) []TriageResult {
 		dbStats *DiagResult
 		dbErr   error
 	)
-	var pre errgroup.Group
-	pre.Go(func() error {
+	var pre sync.WaitGroup
+	pre.Go(func() {
 		cctx, cancel := context.WithTimeout(ctx, triageCheckTimeout)
 		defer cancel()
 		info, infoErr = c.Maintenance(cctx, "")
@@ -170,15 +169,13 @@ func (c *Client) Triage(ctx context.Context) []TriageResult {
 		if infoErr == nil && (info == nil || info.Version == "") {
 			infoErr = errors.New("server unreachable")
 		}
-		return nil
 	})
-	pre.Go(func() error {
+	pre.Go(func() {
 		cctx, cancel := context.WithTimeout(ctx, triageCheckTimeout)
 		defer cancel()
 		dbStats, dbErr = c.runTriageDiag(cctx, "", "database_stats")
-		return nil
 	})
-	_ = pre.Wait()
+	pre.Wait()
 
 	// mgrade/dgrade adapt a pure grader over a shared input into a check.run,
 	// short-circuiting to "could not evaluate" when that input failed to load.
@@ -231,10 +228,12 @@ func (c *Client) Triage(ctx context.Context) []TriageResult {
 	}
 
 	results := make([]TriageResult, len(checks))
-	g := new(errgroup.Group)
-	g.SetLimit(triageFanout)
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, triageFanout)
 	for i, chk := range checks {
-		g.Go(func() error {
+		wg.Go(func() {
+			sem <- struct{}{}
+			defer func() { <-sem }()
 			cctx, cancel := context.WithTimeout(ctx, triageCheckTimeout)
 			defer cancel()
 			sev, detail, err := chk.run(cctx)
@@ -248,10 +247,9 @@ func (c *Client) Triage(ctx context.Context) []TriageResult {
 				DiagKey:  chk.diagKey,
 				Target:   chk.target,
 			}
-			return nil
 		})
 	}
-	_ = g.Wait()
+	wg.Wait()
 
 	sort.SliceStable(results, func(a, b int) bool {
 		return results[a].Severity > results[b].Severity