From e3b24beaa7174efa66603253503366cbb48c40f8 Mon Sep 17 00:00:00 2001 From: chaadow Date: Thu, 16 Jul 2026 23:26:30 +0100 Subject: [PATCH 1/5] feat: add update_stats for HOT and new-page update breakdown ## Summary Add `update_stats` to report per-table HOT, same-page non-HOT, and new-page update metrics from `pg_stat_user_tables`, including fillfactor and percentages. ## Why Update-heavy workloads benefit from HOT updates, but diagnosing low HOT rates needs separating space pressure (`n_tup_newpage_upd`) from indexed-column updates. PostgreSQL 16 exposes that split; older versions still get a HOT vs non-HOT fallback. --- README.md | 15 +++++ lib/ruby-pg-extras.rb | 13 +++- lib/ruby_pg_extras/queries/update_stats.sql | 64 +++++++++++++++++++ .../queries/update_stats_legacy.sql | 39 +++++++++++ spec/smoke_spec.rb | 57 +++++++++++++++++ 5 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 lib/ruby_pg_extras/queries/update_stats.sql create mode 100644 lib/ruby_pg_extras/queries/update_stats_legacy.sql diff --git a/README.md b/README.md index 6da8bf8..4f410e5 100644 --- a/README.md +++ b/README.md @@ -725,6 +725,21 @@ RubyPgExtras.vacuum_io_stats This command surfaces cumulative I/O statistics for autovacuum-related VACUUM activity, based on the `pg_stat_io` view introduced in PostgreSQL 16 ([pg_stat_io documentation](https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-IO-VIEW)). It shows how many blocks autovacuum workers have read and written, how many buffer evictions and ring-buffer reuses occurred, and when the statistics were last reset; this is useful for determining whether autovacuum is responsible for I/O spikes, as described in the pganalyze article on `pg_stat_io` ([Tracking cumulative I/O activity by autovacuum and manual VACUUMs](https://pganalyze.com/blog/pg-stat-io#tracking-cumulative-io-activity-by-autovacuum-and-manual-vacuums)). On PostgreSQL versions below 16 this method returns a single informational row indicating that the feature is unavailable. +### `update_stats` + +```ruby + +RubyPgExtras.update_stats + + table | fillfactor | total_updates | hot_updates | hot_pct | same_page_non_hot_updates | same_page_non_hot_pct | new_page_updates | new_page_pct | same_page_pct | hot_given_same_page_pct +--------+------------+---------------+-------------+---------+---------------------------+-----------------------+------------------+--------------+---------------+------------------------- + users | 100 | 1250000 | 980000 | 78.40 | 45000 | 3.60 | 225000 | 18.00 | 82.00 | 95.61 + orders | 80 | 450000 | 410000 | 91.11 | 12000 | 2.67 | 28000 | 6.22 | 93.78 | 97.16 + (truncated results for brevity) +``` + +This command breaks down table updates into HOT, same-page non-HOT, and new-page updates using `pg_stat_user_tables` columns including `n_tup_newpage_upd` ([pg_stat_all_tables documentation](https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-ALL-TABLES-VIEW)). HOT updates require that changed columns are not indexed and that the new row version fits on the same page ([HOT updates in PostgreSQL for better performance](https://www.cybertec-postgresql.com/en/hot-updates-in-postgresql-for-better-performance/), [Heap-Only Tuples](https://www.postgresql.org/docs/current/storage-hot.html)). High `same_page_non_hot_pct` usually points to updates of indexed columns, while high `new_page_pct` often means pages are too full and lowering `fillfactor` (then rewriting the table with `VACUUM FULL` or `CLUSTER`) may help. These counters are cumulative and can be reset with PostgreSQL statistics-reset functions. On PostgreSQL versions below 16, where `n_tup_newpage_upd` is unavailable, the method returns a reduced breakdown of total, HOT, and non-HOT updates. + ### `kill_all` ```ruby diff --git a/lib/ruby-pg-extras.rb b/lib/ruby-pg-extras.rb index 17fa80a..87468f9 100644 --- a/lib/ruby-pg-extras.rb +++ b/lib/ruby-pg-extras.rb @@ -28,7 +28,7 @@ module RubyPgExtras records_rank seq_scans table_index_scans table_indexes_size table_size total_index_size total_table_size unused_indexes duplicate_indexes vacuum_stats vacuum_progress vacuum_io_stats - analyze_progress + analyze_progress update_stats kill_all kill_pid pg_stat_statements_reset buffercache_stats buffercache_usage ssl_used connections @@ -58,6 +58,8 @@ module RubyPgExtras vacuum_io_stats: {}, vacuum_io_stats_legacy: {}, analyze_progress: {}, + update_stats: { schema: DEFAULT_SCHEMA }, + update_stats_legacy: { schema: DEFAULT_SCHEMA }, buffercache_stats: { limit: 10 }, buffercache_usage: { limit: 20 }, unused_indexes: { max_scans: 50, schema: DEFAULT_SCHEMA }, @@ -119,6 +121,15 @@ def self.run_query_base(query_name:, conn:, exec_method:, in_format:, args: {}) end end + # The detailed update breakdown relies on n_tup_newpage_upd, available from PostgreSQL 16. + # Older versions fall back to the HOT/non-HOT breakdown in update_stats_legacy. + if query_name == :update_stats + server_version_num = conn.send(exec_method, "SHOW server_version_num").to_a[0].values[0].to_i + if server_version_num < 160000 + query_name = :update_stats_legacy + end + end + REQUIRED_ARGS.fetch(query_name) { [] }.each do |arg_name| if args[arg_name].nil? raise ArgumentError, "'#{arg_name}' is required" diff --git a/lib/ruby_pg_extras/queries/update_stats.sql b/lib/ruby_pg_extras/queries/update_stats.sql new file mode 100644 index 0000000..abde2cb --- /dev/null +++ b/lib/ruby_pg_extras/queries/update_stats.sql @@ -0,0 +1,64 @@ +/* HOT, same-page non-HOT, and new-page update statistics (PostgreSQL 16+) */ + +WITH table_stats AS ( + SELECT + s.relid, + s.schemaname, + s.relname, + s.n_tup_upd, + s.n_tup_hot_upd, + s.n_tup_newpage_upd, + COALESCE( + ( + SELECT option_value::integer + FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'fillfactor' + ), + 100 + ) AS fillfactor + FROM pg_stat_user_tables s + INNER JOIN pg_class c ON c.oid = s.relid + WHERE s.schemaname = '%{schema}' +) +SELECT + relname AS table, + fillfactor, + n_tup_upd AS total_updates, + n_tup_hot_upd AS hot_updates, + ROUND( + 100.0 * n_tup_hot_upd + / NULLIF(n_tup_upd, 0), + 2 + ) AS hot_pct, + n_tup_upd + - n_tup_hot_upd + - n_tup_newpage_upd + AS same_page_non_hot_updates, + ROUND( + 100.0 * ( + n_tup_upd + - n_tup_hot_upd + - n_tup_newpage_upd + ) + / NULLIF(n_tup_upd, 0), + 2 + ) AS same_page_non_hot_pct, + n_tup_newpage_upd AS new_page_updates, + ROUND( + 100.0 * n_tup_newpage_upd + / NULLIF(n_tup_upd, 0), + 2 + ) AS new_page_pct, + ROUND( + 100.0 * (n_tup_upd - n_tup_newpage_upd) + / NULLIF(n_tup_upd, 0), + 2 + ) AS same_page_pct, + ROUND( + 100.0 * n_tup_hot_upd + / NULLIF(n_tup_upd - n_tup_newpage_upd, 0), + 2 + ) AS hot_given_same_page_pct +FROM table_stats +WHERE n_tup_upd > 0 +ORDER BY n_tup_upd DESC; diff --git a/lib/ruby_pg_extras/queries/update_stats_legacy.sql b/lib/ruby_pg_extras/queries/update_stats_legacy.sql new file mode 100644 index 0000000..8f13777 --- /dev/null +++ b/lib/ruby_pg_extras/queries/update_stats_legacy.sql @@ -0,0 +1,39 @@ +/* HOT and non-HOT update statistics (PostgreSQL 15 and older) */ + +WITH table_stats AS ( + SELECT + s.relid, + s.relname, + s.n_tup_upd, + s.n_tup_hot_upd, + COALESCE( + ( + SELECT option_value::integer + FROM pg_options_to_table(c.reloptions) + WHERE option_name = 'fillfactor' + ), + 100 + ) AS fillfactor + FROM pg_stat_user_tables s + INNER JOIN pg_class c ON c.oid = s.relid + WHERE s.schemaname = '%{schema}' +) +SELECT + relname AS table, + fillfactor, + n_tup_upd AS total_updates, + n_tup_hot_upd AS hot_updates, + ROUND( + 100.0 * n_tup_hot_upd + / NULLIF(n_tup_upd, 0), + 2 + ) AS hot_pct, + n_tup_upd - n_tup_hot_upd AS non_hot_updates, + ROUND( + 100.0 * (n_tup_upd - n_tup_hot_upd) + / NULLIF(n_tup_upd, 0), + 2 + ) AS non_hot_pct +FROM table_stats +WHERE n_tup_upd > 0 +ORDER BY n_tup_upd DESC; diff --git a/spec/smoke_spec.rb b/spec/smoke_spec.rb index 2f8853f..03671c8 100644 --- a/spec/smoke_spec.rb +++ b/spec/smoke_spec.rb @@ -58,6 +58,63 @@ end end + describe "update_stats" do + it "returns a consistent HOT update breakdown" do + connection = RubyPgExtras.connection + server_version_num = connection.exec("SHOW server_version_num").to_a[0].values[0].to_i + + # Keep this fixture local so every run starts with fresh statistics counters + # and a controlled fillfactor for producing HOT and non-HOT updates. + connection.exec("DROP TABLE IF EXISTS update_stats_test") + connection.exec(<<~SQL) + CREATE TABLE update_stats_test ( + id INTEGER PRIMARY KEY, + value TEXT + ) WITH (fillfactor = 80) + SQL + connection.exec("INSERT INTO update_stats_test VALUES (1, 'before')") + # Updating the unindexed value column produces a HOT update. + connection.exec("UPDATE update_stats_test SET value = 'after' WHERE id = 1") + # Updating the primary key requires index maintenance, producing a non-HOT update. + connection.exec("UPDATE update_stats_test SET id = 2 WHERE id = 1") + + row = nil + # PostgreSQL publishes cumulative statistics asynchronously, particularly + # on older supported versions, so poll until both updates are visible. + 20.times do + row = RubyPgExtras.update_stats( + args: { schema: "public" }, + in_format: :hash, + ).find { |result| result["table"] == "update_stats_test" } + break if row && row["total_updates"].to_i == 2 + + sleep 0.1 + end + + expect(row).not_to be_nil + expect(row["fillfactor"].to_i).to eq(80) + expect(row["total_updates"].to_i).to eq(2) + expect(row["hot_updates"].to_i).to eq(1) + + if server_version_num >= 160000 + expect(row["same_page_non_hot_updates"].to_i).to eq(1) + expect(row["new_page_updates"].to_i).to eq(0) + expect(row["total_updates"].to_i).to eq( + row["hot_updates"].to_i + + row["same_page_non_hot_updates"].to_i + + row["new_page_updates"].to_i, + ) + else + expect(row["non_hot_updates"].to_i).to eq(1) + expect(row["total_updates"].to_i).to eq( + row["hot_updates"].to_i + row["non_hot_updates"].to_i, + ) + end + ensure + connection&.exec("DROP TABLE IF EXISTS update_stats_test") + end + end + describe "#database_url=" do it "setting custom database URL works" do RubyPgExtras.database_url = ENV.fetch("DATABASE_URL") From d73941740d3794a48ba3b4786af5326e66fe76f0 Mon Sep 17 00:00:00 2001 From: chaadow Date: Sat, 18 Jul 2026 00:03:41 +0100 Subject: [PATCH 2/5] feat: add new_page_updates check for diagnosing update performance ## Summary Introduce a new `new_page_updates` check in the `diagnose` module to identify tables in PostgreSQL 16 and newer that experience a high ratio of new-page updates. This check helps diagnose potential page-space pressure and the effectiveness of the table's `fillfactor`. ## Details - The check reports tables with at least 10,000 cumulative updates and 20% or more of updates resulting in new-page placements. - Users can override default thresholds using environment variables. - The implementation includes detailed reporting on new-page ratios, fillfactor, and HOT update percentages. ## Why This feature enhances the ability to diagnose performance issues related to update-heavy workloads, providing insights into how table configurations may impact update efficiency. --- README.md | 15 ++++ lib/ruby_pg_extras/diagnose_data.rb | 59 +++++++++++++++ spec/diagnose_data_spec.rb | 112 ++++++++++++++++++++++++++++ 3 files changed, 186 insertions(+) diff --git a/README.md b/README.md index 4f410e5..fadca21 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,21 @@ RubyPgExtras.diagnose Keep reading to learn about methods that `diagnose` uses under the hood. +### `new_page_updates` + +This is a `diagnose` check, not a standalone query method. On PostgreSQL 16 and newer, it uses the [`update_stats`](#update_stats) breakdown to flag tables where a significant share of updates placed the new row version on another heap page instead of staying on the original page. Those tables are worth reviewing for page-space pressure, row growth, and whether a lower table `fillfactor` would help. + +By default, a table is reported when it has at least 10,000 cumulative updates and 20% or more of its updates are new-page updates. The report includes each table's new-page ratio, current `fillfactor`, and the percentage of same-page updates that were HOT. A low HOT-among-same-page value suggests indexed-column changes are also preventing HOT, so lowering `fillfactor` alone may not be enough. + +You can override the default thresholds with environment variables: + +```ruby +ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"] = "5000" +ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] = "15" +``` + +The underlying counters are cumulative, so compare their values over time rather than treating a single snapshot as definitive. + ## Available methods ### `missing_fk_indexes` diff --git a/lib/ruby_pg_extras/diagnose_data.rb b/lib/ruby_pg_extras/diagnose_data.rb index ae0e792..26913d0 100644 --- a/lib/ruby_pg_extras/diagnose_data.rb +++ b/lib/ruby_pg_extras/diagnose_data.rb @@ -10,6 +10,8 @@ class DiagnoseData PG_EXTRAS_NULL_MIN_NULL_FRAC_PERCENT = 50 # 50% PG_EXTRAS_BLOAT_MIN_VALUE = 10 PG_EXTRAS_OUTLIERS_MIN_EXEC_RATIO = 33 # 33% + PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT = 20 # 20% + PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE = 10_000 def self.call new.call @@ -26,6 +28,7 @@ def call :unused_indexes, :null_indexes, :bloat, + :new_page_updates, :duplicate_indexes, ].yield_self do |checks| extensions_data = query_module.extensions(in_format: :hash) @@ -292,6 +295,62 @@ def duplicate_indexes end end + def new_page_updates + max_percent = ENV.fetch( + "PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT", + PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT, + ).to_f + min_sample = ENV.fetch( + "PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE", + PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE, + ).to_i + + tables = query_module.update_stats(in_format: :hash) + + if tables.any? && !tables.first.key?("new_page_pct") + return { + ok: true, + message: "New-page update analysis requires PostgreSQL 16 or newer.", + } + end + + tables = tables.select do |table| + table.fetch("total_updates").to_i >= min_sample && + table.fetch("new_page_pct").to_f >= max_percent + end + + if tables.empty? + { + ok: true, + message: "No tables with a high new-page update ratio detected.", + } + else + table_details = tables.map do |table| + <<~DETAIL.strip + '#{table.fetch("table")}': + new-page updates: #{table.fetch("new_page_pct")}% (#{table.fetch("new_page_updates")} of #{table.fetch("total_updates")}) + HOT among same-page updates: #{table.fetch("hot_given_same_page_pct")}% + fillfactor: #{table.fetch("fillfactor")} + DETAIL + end.join("\n\n") + + { + ok: false, + message: <<~MESSAGE.strip, + High new-page update ratios detected: + + #{table_details} + + A high new-page ratio means successor tuple versions often do not fit on their original heap page and therefore cannot be HOT. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate. + + The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are also preventing HOT, so changing fillfactor alone may not resolve the issue. + + These counters are cumulative; compare their deltas before and after a change. + MESSAGE + } + end + end + def outliers queries = query_module.outliers(in_format: :hash).select do |q| q.fetch("prop_exec_time").gsub("%", "").to_f >= PG_EXTRAS_OUTLIERS_MIN_EXEC_RATIO diff --git a/spec/diagnose_data_spec.rb b/spec/diagnose_data_spec.rb index 6fedda8..a2fd017 100644 --- a/spec/diagnose_data_spec.rb +++ b/spec/diagnose_data_spec.rb @@ -76,4 +76,116 @@ end end end + + describe "#new_page_updates" do + let(:diagnose_data) { described_class.new } + + it "reports tables exceeding the update sample and new-page thresholds" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "fillfactor" => "100", + "total_updates" => "10000", + "new_page_updates" => "2500", + "new_page_pct" => "25.00", + "hot_given_same_page_pct" => "90.00", + }, + { + "table" => "users", + "fillfactor" => "80", + "total_updates" => "9999", + "new_page_updates" => "3000", + "new_page_pct" => "30.00", + "hot_given_same_page_pct" => "95.00", + }, + ], + ) + + result = diagnose_data.send(:new_page_updates) + + expect(result).to eq( + ok: false, + message: <<~MESSAGE.strip, + High new-page update ratios detected: + + 'orders': + new-page updates: 25.00% (2500 of 10000) + HOT among same-page updates: 90.00% + fillfactor: 100 + + A high new-page ratio means successor tuple versions often do not fit on their original heap page and therefore cannot be HOT. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate. + + The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are also preventing HOT, so changing fillfactor alone may not resolve the issue. + + These counters are cumulative; compare their deltas before and after a change. + MESSAGE + ) + end + + it "does not report tables below either threshold" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "total_updates" => "10000", + "new_page_pct" => "19.99", + }, + { + "table" => "users", + "total_updates" => "9999", + "new_page_pct" => "25.00", + }, + ], + ) + + expect(diagnose_data.send(:new_page_updates)).to eq( + ok: true, + message: "No tables with a high new-page update ratio detected.", + ) + end + + it "allows overriding the thresholds with environment variables" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "fillfactor" => "100", + "total_updates" => "500", + "new_page_updates" => "75", + "new_page_pct" => "15.00", + "hot_given_same_page_pct" => "90.00", + }, + ], + ) + original_max_percent = ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] + original_min_sample = ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"] + ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] = "15" + ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"] = "500" + + expect(diagnose_data.send(:new_page_updates).fetch(:ok)).to eq(false) + ensure + ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] = original_max_percent + ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE"] = original_min_sample + end + + it "skips the check when update_stats returns the legacy breakdown" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "total_updates" => "10000", + "hot_updates" => "5000", + "hot_pct" => "50.00", + }, + ], + ) + + expect(diagnose_data.send(:new_page_updates)).to eq( + ok: true, + message: "New-page update analysis requires PostgreSQL 16 or newer.", + ) + end + + end end From 3c8293be263032e6c3c58144f81c2b4646935f1f Mon Sep 17 00:00:00 2001 From: chaadow Date: Sat, 18 Jul 2026 01:04:59 +0100 Subject: [PATCH 3/5] feat: add avg_row_bytes calculation to update_stats queries ## Summary Enhance the `update_stats` and `update_stats_legacy` SQL queries to include the calculation of average row size in bytes (`avg_row_bytes`). This metric is derived from the relation size divided by the number of tuples, providing better insights into table storage efficiency. ## Details - Added `reltuples` to the selected columns to facilitate the average row size calculation. - Implemented the `avg_row_bytes` calculation in both queries. - Updated smoke tests to verify the correctness of the new metric after running `ANALYZE`. ## Why This addition improves the diagnostic capabilities of the tool by allowing users to assess the average size of rows in their tables, which can be crucial for performance tuning and storage optimization. --- lib/ruby_pg_extras/queries/update_stats.sql | 5 +++++ lib/ruby_pg_extras/queries/update_stats_legacy.sql | 5 +++++ spec/smoke_spec.rb | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/lib/ruby_pg_extras/queries/update_stats.sql b/lib/ruby_pg_extras/queries/update_stats.sql index abde2cb..506902a 100644 --- a/lib/ruby_pg_extras/queries/update_stats.sql +++ b/lib/ruby_pg_extras/queries/update_stats.sql @@ -8,6 +8,7 @@ WITH table_stats AS ( s.n_tup_upd, s.n_tup_hot_upd, s.n_tup_newpage_upd, + c.reltuples, COALESCE( ( SELECT option_value::integer @@ -23,6 +24,10 @@ WITH table_stats AS ( SELECT relname AS table, fillfactor, + ROUND( + pg_relation_size(relid)::numeric + / NULLIF(reltuples, 0) + )::bigint AS avg_row_bytes, n_tup_upd AS total_updates, n_tup_hot_upd AS hot_updates, ROUND( diff --git a/lib/ruby_pg_extras/queries/update_stats_legacy.sql b/lib/ruby_pg_extras/queries/update_stats_legacy.sql index 8f13777..b7cf467 100644 --- a/lib/ruby_pg_extras/queries/update_stats_legacy.sql +++ b/lib/ruby_pg_extras/queries/update_stats_legacy.sql @@ -6,6 +6,7 @@ WITH table_stats AS ( s.relname, s.n_tup_upd, s.n_tup_hot_upd, + c.reltuples, COALESCE( ( SELECT option_value::integer @@ -21,6 +22,10 @@ WITH table_stats AS ( SELECT relname AS table, fillfactor, + ROUND( + pg_relation_size(relid)::numeric + / NULLIF(reltuples, 0) + )::bigint AS avg_row_bytes, n_tup_upd AS total_updates, n_tup_hot_upd AS hot_updates, ROUND( diff --git a/spec/smoke_spec.rb b/spec/smoke_spec.rb index 03671c8..6314f47 100644 --- a/spec/smoke_spec.rb +++ b/spec/smoke_spec.rb @@ -77,6 +77,9 @@ connection.exec("UPDATE update_stats_test SET value = 'after' WHERE id = 1") # Updating the primary key requires index maintenance, producing a non-HOT update. connection.exec("UPDATE update_stats_test SET id = 2 WHERE id = 1") + # avg_row_bytes divides heap size by pg_class.reltuples, which is only + # populated after ANALYZE (or VACUUM). + connection.exec("ANALYZE update_stats_test") row = nil # PostgreSQL publishes cumulative statistics asynchronously, particularly @@ -93,6 +96,7 @@ expect(row).not_to be_nil expect(row["fillfactor"].to_i).to eq(80) + expect(row["avg_row_bytes"].to_i).to be > 0 expect(row["total_updates"].to_i).to eq(2) expect(row["hot_updates"].to_i).to eq(1) From 079ad6787e897522396aae165521e910a0b375a5 Mon Sep 17 00:00:00 2001 From: chaadow Date: Sat, 18 Jul 2026 01:21:12 +0100 Subject: [PATCH 4/5] More polishes --- README.md | 10 +++++----- lib/ruby_pg_extras/diagnose_data.rb | 4 ++-- lib/ruby_pg_extras/queries/update_stats.sql | 10 ++++++---- lib/ruby_pg_extras/queries/update_stats_legacy.sql | 10 ++++++---- spec/diagnose_data_spec.rb | 9 +++++++-- spec/smoke_spec.rb | 6 +++--- 6 files changed, 29 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index fadca21..df60cdf 100644 --- a/README.md +++ b/README.md @@ -746,14 +746,14 @@ This command surfaces cumulative I/O statistics for autovacuum-related VACUUM ac RubyPgExtras.update_stats - table | fillfactor | total_updates | hot_updates | hot_pct | same_page_non_hot_updates | same_page_non_hot_pct | new_page_updates | new_page_pct | same_page_pct | hot_given_same_page_pct ---------+------------+---------------+-------------+---------+---------------------------+-----------------------+------------------+--------------+---------------+------------------------- - users | 100 | 1250000 | 980000 | 78.40 | 45000 | 3.60 | 225000 | 18.00 | 82.00 | 95.61 - orders | 80 | 450000 | 410000 | 91.11 | 12000 | 2.67 | 28000 | 6.22 | 93.78 | 97.16 + table | fillfactor | estimated_heap_bytes_per_live_row | total_updates | hot_updates | hot_pct | same_page_non_hot_updates | same_page_non_hot_pct | new_page_updates | new_page_pct | same_page_pct | hot_given_same_page_pct +--------+------------+-----------------------------------+---------------+-------------+---------+---------------------------+-----------------------+------------------+--------------+---------------+------------------------- + users | 100 | 256 | 1250000 | 980000 | 78.40 | 45000 | 3.60 | 225000 | 18.00 | 82.00 | 95.61 + orders | 80 | 128 | 450000 | 410000 | 91.11 | 12000 | 2.67 | 28000 | 6.22 | 93.78 | 97.16 (truncated results for brevity) ``` -This command breaks down table updates into HOT, same-page non-HOT, and new-page updates using `pg_stat_user_tables` columns including `n_tup_newpage_upd` ([pg_stat_all_tables documentation](https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-ALL-TABLES-VIEW)). HOT updates require that changed columns are not indexed and that the new row version fits on the same page ([HOT updates in PostgreSQL for better performance](https://www.cybertec-postgresql.com/en/hot-updates-in-postgresql-for-better-performance/), [Heap-Only Tuples](https://www.postgresql.org/docs/current/storage-hot.html)). High `same_page_non_hot_pct` usually points to updates of indexed columns, while high `new_page_pct` often means pages are too full and lowering `fillfactor` (then rewriting the table with `VACUUM FULL` or `CLUSTER`) may help. These counters are cumulative and can be reset with PostgreSQL statistics-reset functions. On PostgreSQL versions below 16, where `n_tup_newpage_upd` is unavailable, the method returns a reduced breakdown of total, HOT, and non-HOT updates. +This command breaks down table updates into HOT, same-page non-HOT, and new-page updates using `pg_stat_user_tables` columns including `n_tup_newpage_upd` ([pg_stat_all_tables documentation](https://www.postgresql.org/docs/current/monitoring-stats.html#MONITORING-PG-STAT-ALL-TABLES-VIEW)). HOT updates require that changed columns are not indexed and that the new row version fits on the same page ([HOT updates in PostgreSQL for better performance](https://www.cybertec-postgresql.com/en/hot-updates-in-postgresql-for-better-performance/), [Heap-Only Tuples](https://www.postgresql.org/docs/current/storage-hot.html)). High `same_page_non_hot_pct` usually points to updates of indexed columns, while high `new_page_pct` often means pages are too full and lowering `fillfactor` (then rewriting the table with `VACUUM FULL` or `CLUSTER`) may help. `estimated_heap_bytes_per_live_row` divides main-fork heap size by `pg_class.reltuples` when that estimate is positive; it reflects physical storage per estimated live row (including page overhead, fillfactor free space, and bloat) rather than logical tuple width, and is NULL until the table has been analyzed or vacuumed. Larger values often call for a lower `fillfactor`. These counters are cumulative and can be reset with PostgreSQL statistics-reset functions. On PostgreSQL versions below 16, where `n_tup_newpage_upd` is unavailable, the method returns a reduced breakdown of total, HOT, and non-HOT updates. ### `kill_all` diff --git a/lib/ruby_pg_extras/diagnose_data.rb b/lib/ruby_pg_extras/diagnose_data.rb index 26913d0..fc45319 100644 --- a/lib/ruby_pg_extras/diagnose_data.rb +++ b/lib/ruby_pg_extras/diagnose_data.rb @@ -341,9 +341,9 @@ def new_page_updates #{table_details} - A high new-page ratio means successor tuple versions often do not fit on their original heap page and therefore cannot be HOT. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate. + A high new-page ratio means many successor tuple versions were placed on another heap page and therefore could not be HOT. This commonly indicates insufficient reusable space on the original page. `n_tup_newpage_upd` records that placement directly; it does not identify the underlying reason or whether the update would otherwise have been HOT-eligible. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate. - The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are also preventing HOT, so changing fillfactor alone may not resolve the issue. + The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are preventing HOT on updates that did stay on the same page, so changing fillfactor alone may not resolve the issue. These counters are cumulative; compare their deltas before and after a change. MESSAGE diff --git a/lib/ruby_pg_extras/queries/update_stats.sql b/lib/ruby_pg_extras/queries/update_stats.sql index 506902a..afd8b93 100644 --- a/lib/ruby_pg_extras/queries/update_stats.sql +++ b/lib/ruby_pg_extras/queries/update_stats.sql @@ -24,10 +24,12 @@ WITH table_stats AS ( SELECT relname AS table, fillfactor, - ROUND( - pg_relation_size(relid)::numeric - / NULLIF(reltuples, 0) - )::bigint AS avg_row_bytes, + CASE + WHEN reltuples > 0 THEN + ROUND( + pg_relation_size(relid)::numeric / reltuples + )::bigint + END AS estimated_heap_bytes_per_live_row, n_tup_upd AS total_updates, n_tup_hot_upd AS hot_updates, ROUND( diff --git a/lib/ruby_pg_extras/queries/update_stats_legacy.sql b/lib/ruby_pg_extras/queries/update_stats_legacy.sql index b7cf467..bb1d676 100644 --- a/lib/ruby_pg_extras/queries/update_stats_legacy.sql +++ b/lib/ruby_pg_extras/queries/update_stats_legacy.sql @@ -22,10 +22,12 @@ WITH table_stats AS ( SELECT relname AS table, fillfactor, - ROUND( - pg_relation_size(relid)::numeric - / NULLIF(reltuples, 0) - )::bigint AS avg_row_bytes, + CASE + WHEN reltuples > 0 THEN + ROUND( + pg_relation_size(relid)::numeric / reltuples + )::bigint + END AS estimated_heap_bytes_per_live_row, n_tup_upd AS total_updates, n_tup_hot_upd AS hot_updates, ROUND( diff --git a/spec/diagnose_data_spec.rb b/spec/diagnose_data_spec.rb index a2fd017..2fba963 100644 --- a/spec/diagnose_data_spec.rb +++ b/spec/diagnose_data_spec.rb @@ -59,6 +59,11 @@ { table: "posts", column_name: "topic_id" }, ] } + + expect(RubyPgExtras) + .to receive(:update_stats) + .with(in_format: :hash) + .and_return([]) end it "works" do @@ -114,9 +119,9 @@ HOT among same-page updates: 90.00% fillfactor: 100 - A high new-page ratio means successor tuple versions often do not fit on their original heap page and therefore cannot be HOT. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate. + A high new-page ratio means many successor tuple versions were placed on another heap page and therefore could not be HOT. This commonly indicates insufficient reusable space on the original page. `n_tup_newpage_upd` records that placement directly; it does not identify the underlying reason or whether the update would otherwise have been HOT-eligible. Investigate page-space pressure, row growth, long-lived transactions, large update batches, and whether a lower table fillfactor is appropriate. - The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are also preventing HOT, so changing fillfactor alone may not resolve the issue. + The HOT-among-same-page percentage provides additional context: a low value suggests indexed-column changes are preventing HOT on updates that did stay on the same page, so changing fillfactor alone may not resolve the issue. These counters are cumulative; compare their deltas before and after a change. MESSAGE diff --git a/spec/smoke_spec.rb b/spec/smoke_spec.rb index 6314f47..8797a2d 100644 --- a/spec/smoke_spec.rb +++ b/spec/smoke_spec.rb @@ -77,8 +77,8 @@ connection.exec("UPDATE update_stats_test SET value = 'after' WHERE id = 1") # Updating the primary key requires index maintenance, producing a non-HOT update. connection.exec("UPDATE update_stats_test SET id = 2 WHERE id = 1") - # avg_row_bytes divides heap size by pg_class.reltuples, which is only - # populated after ANALYZE (or VACUUM). + # estimated_heap_bytes_per_live_row divides main-fork heap size by + # pg_class.reltuples, which is only populated after ANALYZE (or VACUUM). connection.exec("ANALYZE update_stats_test") row = nil @@ -96,7 +96,7 @@ expect(row).not_to be_nil expect(row["fillfactor"].to_i).to eq(80) - expect(row["avg_row_bytes"].to_i).to be > 0 + expect(row["estimated_heap_bytes_per_live_row"].to_i).to be > 0 expect(row["total_updates"].to_i).to eq(2) expect(row["hot_updates"].to_i).to eq(1) From 7888bd18bb15c642558ce57f4a278451a944de1a Mon Sep 17 00:00:00 2001 From: chaadow Date: Sat, 18 Jul 2026 18:33:35 +0100 Subject: [PATCH 5/5] feat: add low_hot_same_page check for diagnosing update performance ## Summary Introduce a new `low_hot_same_page` check in the `diagnose` module to identify tables in PostgreSQL 16 and newer that have a low ratio of HOT updates among same-page updates. This check helps diagnose potential issues with indexed column updates. ## Details - The check reports tables with at least 10,000 cumulative updates and fewer than 10% of updates being HOT. - Users can override default thresholds using environment variables. - The implementation includes detailed reporting on HOT ratios, same-page and new-page update percentages, and current fillfactor. ## Why This feature enhances the ability to diagnose performance issues related to update-heavy workloads, providing insights into how table configurations may impact update efficiency. --- README.md | 15 ++++ lib/ruby_pg_extras/diagnose_data.rb | 61 +++++++++++++ spec/diagnose_data_spec.rb | 135 ++++++++++++++++++++++++++++ 3 files changed, 211 insertions(+) diff --git a/README.md b/README.md index df60cdf..13e3b45 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,21 @@ ENV["PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT"] = "15" The underlying counters are cumulative, so compare their values over time rather than treating a single snapshot as definitive. +### `low_hot_same_page` + +This is a `diagnose` check, not a standalone query method. On PostgreSQL 16 and newer, it uses the [`update_stats`](#update_stats) breakdown to flag tables where updates that stayed on the original heap page were almost never HOT. That usually means those updates modified indexed columns, so lowering `fillfactor` alone will not help. + +By default, a table is reported when it has at least 10,000 cumulative updates and fewer than 10% of its same-page updates were HOT. The report includes each table's HOT-among-same-page ratio, same-page and new-page ratios, and current `fillfactor`. Review which columns your application updates and which indexes cover them; removing or adjusting indexes on frequently updated columns (or avoiding updating those columns) can restore HOT updates. + +You can override the default thresholds with environment variables: + +```ruby +ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"] = "5000" +ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"] = "5" +``` + +The underlying counters are cumulative, so compare their values over time rather than treating a single snapshot as definitive. + ## Available methods ### `missing_fk_indexes` diff --git a/lib/ruby_pg_extras/diagnose_data.rb b/lib/ruby_pg_extras/diagnose_data.rb index fc45319..3bb5e93 100644 --- a/lib/ruby_pg_extras/diagnose_data.rb +++ b/lib/ruby_pg_extras/diagnose_data.rb @@ -12,6 +12,8 @@ class DiagnoseData PG_EXTRAS_OUTLIERS_MIN_EXEC_RATIO = 33 # 33% PG_EXTRAS_NEW_PAGE_UPDATES_MAX_PERCENT = 20 # 20% PG_EXTRAS_NEW_PAGE_UPDATES_MIN_SAMPLE = 10_000 + PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT = 10 # 10% + PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE = 10_000 def self.call new.call @@ -29,6 +31,7 @@ def call :null_indexes, :bloat, :new_page_updates, + :low_hot_same_page, :duplicate_indexes, ].yield_self do |checks| extensions_data = query_module.extensions(in_format: :hash) @@ -351,6 +354,64 @@ def new_page_updates end end + def low_hot_same_page + min_percent = ENV.fetch( + "PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT", + PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT, + ).to_f + min_sample = ENV.fetch( + "PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE", + PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE, + ).to_i + + tables = query_module.update_stats(in_format: :hash) + + if tables.any? && !tables.first.key?("hot_given_same_page_pct") + return { + ok: true, + message: "HOT-among-same-page update analysis requires PostgreSQL 16 or newer.", + } + end + + tables = tables.select do |table| + hot_given_same_page_pct = table["hot_given_same_page_pct"] + next false if hot_given_same_page_pct.nil? + + table.fetch("total_updates").to_i >= min_sample && + hot_given_same_page_pct.to_f < min_percent + end + + if tables.empty? + { + ok: true, + message: "No tables with a low HOT-among-same-page update ratio detected.", + } + else + table_details = tables.map do |table| + <<~DETAIL.strip + '#{table.fetch("table")}': + HOT among same-page updates: #{table.fetch("hot_given_same_page_pct")}% + same-page updates: #{table.fetch("same_page_pct")}% + new-page updates: #{table.fetch("new_page_pct")}% + fillfactor: #{table.fetch("fillfactor")} + DETAIL + end.join("\n\n") + + { + ok: false, + message: <<~MESSAGE.strip, + Low HOT-among-same-page update ratios detected: + + #{table_details} + + A low HOT-among-same-page ratio means updates that stayed on the original heap page still could not be HOT. That usually means those updates modified indexed columns. Review which columns your application updates and which indexes cover them; removing or adjusting indexes on frequently updated columns (or avoiding updating those columns) can restore HOT updates and reduce index and vacuum overhead. + + These counters are cumulative; compare their deltas before and after a change. + MESSAGE + } + end + end + def outliers queries = query_module.outliers(in_format: :hash).select do |q| q.fetch("prop_exec_time").gsub("%", "").to_f >= PG_EXTRAS_OUTLIERS_MIN_EXEC_RATIO diff --git a/spec/diagnose_data_spec.rb b/spec/diagnose_data_spec.rb index 2fba963..2e19b22 100644 --- a/spec/diagnose_data_spec.rb +++ b/spec/diagnose_data_spec.rb @@ -63,6 +63,7 @@ expect(RubyPgExtras) .to receive(:update_stats) .with(in_format: :hash) + .twice .and_return([]) end @@ -191,6 +192,140 @@ message: "New-page update analysis requires PostgreSQL 16 or newer.", ) end + end + + describe "#low_hot_same_page" do + let(:diagnose_data) { described_class.new } + + it "reports tables below the HOT-among-same-page threshold" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "sessions", + "fillfactor" => "100", + "total_updates" => "10000", + "same_page_pct" => "95.61", + "new_page_pct" => "4.39", + "hot_given_same_page_pct" => "0.0", + }, + { + "table" => "orders", + "fillfactor" => "80", + "total_updates" => "10000", + "same_page_pct" => "90.00", + "new_page_pct" => "10.00", + "hot_given_same_page_pct" => "10.00", + }, + { + "table" => "users", + "fillfactor" => "100", + "total_updates" => "9999", + "same_page_pct" => "99.00", + "new_page_pct" => "1.00", + "hot_given_same_page_pct" => "0.0", + }, + ], + ) + + result = diagnose_data.send(:low_hot_same_page) + expect(result).to eq( + ok: false, + message: <<~MESSAGE.strip, + Low HOT-among-same-page update ratios detected: + + 'sessions': + HOT among same-page updates: 0.0% + same-page updates: 95.61% + new-page updates: 4.39% + fillfactor: 100 + + A low HOT-among-same-page ratio means updates that stayed on the original heap page still could not be HOT. That usually means those updates modified indexed columns. Review which columns your application updates and which indexes cover them; removing or adjusting indexes on frequently updated columns (or avoiding updating those columns) can restore HOT updates and reduce index and vacuum overhead. + + These counters are cumulative; compare their deltas before and after a change. + MESSAGE + ) + end + + it "does not report tables at or above the threshold" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "total_updates" => "10000", + "hot_given_same_page_pct" => "10.00", + }, + { + "table" => "users", + "total_updates" => "9999", + "hot_given_same_page_pct" => "0.0", + }, + ], + ) + + expect(diagnose_data.send(:low_hot_same_page)).to eq( + ok: true, + message: "No tables with a low HOT-among-same-page update ratio detected.", + ) + end + + it "skips tables with a NULL HOT-among-same-page ratio" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "total_updates" => "10000", + "hot_given_same_page_pct" => nil, + }, + ], + ) + + expect(diagnose_data.send(:low_hot_same_page)).to eq( + ok: true, + message: "No tables with a low HOT-among-same-page update ratio detected.", + ) + end + + it "allows overriding the thresholds with environment variables" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "sessions", + "fillfactor" => "100", + "total_updates" => "500", + "same_page_pct" => "90.00", + "new_page_pct" => "10.00", + "hot_given_same_page_pct" => "4.00", + }, + ], + ) + original_min_percent = ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"] + original_min_sample = ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"] + ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"] = "5" + ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"] = "500" + + expect(diagnose_data.send(:low_hot_same_page).fetch(:ok)).to eq(false) + ensure + ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_PERCENT"] = original_min_percent + ENV["PG_EXTRAS_LOW_HOT_SAME_PAGE_MIN_SAMPLE"] = original_min_sample + end + + it "skips the check when update_stats returns the legacy breakdown" do + allow(RubyPgExtras).to receive(:update_stats).with(in_format: :hash).and_return( + [ + { + "table" => "orders", + "total_updates" => "10000", + "hot_updates" => "5000", + "hot_pct" => "50.00", + }, + ], + ) + + expect(diagnose_data.send(:low_hot_same_page)).to eq( + ok: true, + message: "HOT-among-same-page update analysis requires PostgreSQL 16 or newer.", + ) + end end end