From 8c0c8ced8a64860f54587d429cb55d1efd1f7d95 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:04:19 -0300 Subject: [PATCH 01/21] docs: update shared_buffers guidance with pg_buffercache Clarify that 25% RAM is a starting point, not a strict rule. Recommend using pg_buffercache extension for workload-specific optimization. Add recent references for tuning methodology. Signed-off-by: Sebastian Webber --- rules.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rules.yml b/rules.yml index 8a5ca46..c05c872 100644 --- a/rules.yml +++ b/rules.yml @@ -1,10 +1,15 @@ categories: memory_related: shared_buffers: - abstract: This parameter allocate memory slots used by all process when the database starts. Mainly works as the disk cache and its similar to oracle's SGA buffer. + abstract: | + Allocates shared memory for caching data pages. Acts as PostgreSQL's main disk cache, similar to Oracle's SGA buffer. + + Start with **25% of RAM** as a baseline. For optimal tuning, use the **pg_buffercache extension** to analyze cache hit ratios for your specific workload. recomendations: Tuning Your PostgreSQL Server: https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server#shared_buffers - Tuning shared_buffers and wal_buffers: http://rhaas.blogspot.com.br/2012/03/tuning-sharedbuffers-and-walbuffers.html + Determining optimal shared_buffers using pg_buffercache: https://aws.amazon.com/blogs/database/determining-the-optimal-value-for-shared_buffers-using-the-pg_buffercache-extension-in-postgresql/ + Tuning shared_buffers for OLTP and data warehouse workloads: https://pganalyze.com/blog/5mins-postgres-tuning-shared-buffers-pgbench-TPROC-C-TPROC-H + PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components effective_cache_size: From 6df1d9cde41eb83f1f5ef8571dc828548af0c02b Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:09:04 -0300 Subject: [PATCH 02/21] docs: improve shared_buffers and effective_cache_size docs Add references explaining PostgreSQL double buffering mechanism and how shared_buffers works with OS page cache. Clarify that effective_cache_size is for planner estimates only, with formula for starting point calculation. Signed-off-by: Sebastian Webber --- rules.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index c05c872..bb809c6 100644 --- a/rules.yml +++ b/rules.yml @@ -9,11 +9,18 @@ categories: Tuning Your PostgreSQL Server: https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server#shared_buffers Determining optimal shared_buffers using pg_buffercache: https://aws.amazon.com/blogs/database/determining-the-optimal-value-for-shared_buffers-using-the-pg_buffercache-extension-in-postgresql/ Tuning shared_buffers for OLTP and data warehouse workloads: https://pganalyze.com/blog/5mins-postgres-tuning-shared-buffers-pgbench-TPROC-C-TPROC-H + "Introduction to Buffers in PostgreSQL": https://boringsql.com/posts/introduction-to-buffers/ + "PostgreSQL double buffering: understand the cache size": https://dev.to/franckpachot/postgresql-double-buffering-understand-the-cache-size-in-a-managed-service-oci-2oci PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components effective_cache_size: - abstract: This parameter does not allocate any resource, just tells to the query planner how much of the operating system cache are avaliable to use. Remember that shared_buffers needs to smaller than 8GB, then the query planner will prefer read the disk because it will be on memory. + abstract: | + Estimates total memory available for disk caching (PostgreSQL + OS cache). Used by query planner for cost estimates. + + Does **NOT allocate memory** - only informs the planner about available cache. + + A good starting point: `effective_cache_size = RAM - shared_buffers` recomendations: "effective_cache_size: A practical example": https://www.cybertec-postgresql.com/en/effective_cache_size-a-practical-example/ "effective_cache_size: What it means in PostgreSQL": https://www.cybertec-postgresql.com/en/effective_cache_size-what-it-means-in-postgresql/ From a01ea499cd38c865255c5a67d9d742a9cea22d23 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:19:37 -0300 Subject: [PATCH 03/21] docs: enhance work_mem documentation with OOM warnings Add explicit warning about potential OOM kills in Kubernetes and cloud environments. Include worst-case calculation example showing how work_mem can multiply to 102GB. Add references for temp file monitoring and per-session tuning. Update recommendations with 2025 best practices and remove outdated 2011 reference. Signed-off-by: Sebastian Webber --- rules.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/rules.yml b/rules.yml index bb809c6..2b0f21e 100644 --- a/rules.yml +++ b/rules.yml @@ -26,12 +26,23 @@ categories: "effective_cache_size: What it means in PostgreSQL": https://www.cybertec-postgresql.com/en/effective_cache_size-what-it-means-in-postgresql/ Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance work_mem: - abstract: This parameter defines how much a work_mem buffer can allocate. Each query can open many work_mem buffers when execute (normally one by subquery) if it uses any sort (or aggregate) operation. When work_mem its too small a temp file is created. + abstract: | + Memory per operation for sorts, hash joins, and aggregates. Each query can use **multiple work_mem buffers** simultaneously. + + **⚠️ Warning**: With high concurrency and large datasets, you can easily trigger **OOM kills** in Kubernetes pods or cloud instances. + + Maximum potential memory = `work_mem × operations × parallel_workers × connections` + + Example worst-case: 128MB × 3 operations × 2 workers × 100 connections = **102GB** + + Monitor temp file usage with `log_temp_files`. Consider **per-session** tuning (`SET work_mem`) for heavy queries instead of global settings. details: - Specifies the amount of memory to be used by internal sort operations and hash tables before writing to temporary disk files. The value defaults to four megabytes (4MB). Note that for a complex query, several sort or hash operations might be running in parallel; each operation will be allowed to use as much memory as this value specifies before it starts to write data into temporary files. Also, several running sessions could be doing such operations concurrently. Therefore, the total memory used could be many times the value of work_mem; it is necessary to keep this fact in mind when choosing the value. Sort operations are used for ORDER BY, DISTINCT, and merge joins. Hash tables are used in hash joins, hash-based aggregation, and hash-based processing of IN subqueries. recomendations: Configuring work_mem in Postgres: https://www.pgmustard.com/blog/work-mem - "Understaning postgresql.conf: WORK_MEM": https://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ + The surprising logic of the Postgres work_mem setting: https://pganalyze.com/blog/5mins-postgres-work-mem-tuning + Temporary files in PostgreSQL - identify and fix: https://klouddb.io/temporary-files-in-postgresql-steps-to-identify-and-fix-temp-file-issues/ + PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components maintenance_work_mem: From 6bec0ef09cacd87d57578c2d1b7867dde94f595c Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:35:43 -0300 Subject: [PATCH 04/21] docs: update maintenance_work_mem with version-specific info Add details about PostgreSQL version differences (1GB limit until v16, radix trees in v17+). Include autovacuum_max_workers multiply warning and reference to autovacuum_work_mem. Add AWS RDS autovacuum guide and 2025 best practices. Signed-off-by: Sebastian Webber --- rules.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index 2b0f21e..6adf682 100644 --- a/rules.yml +++ b/rules.yml @@ -46,11 +46,20 @@ categories: Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components maintenance_work_mem: - abstract: This parameter defines how much a maintenance operation (ALTER TABLE, VACUUM, REINDEX, AutoVACUUM worker, etc) buffer can use. + abstract: | + Memory for maintenance operations: **VACUUM**, **CREATE INDEX**, **ALTER TABLE**, and autovacuum workers. + + Can be set higher than work_mem since fewer concurrent maintenance operations run. + + **Important**: Total usage = `maintenance_work_mem × autovacuum_max_workers`. Consider using `autovacuum_work_mem` separately. + + **PostgreSQL ≤16**: 1GB limit (~179M dead tuples per pass). **PostgreSQL 17+**: No limit (uses radix trees). recomendations: Adjusting maintenance_work_mem: https://www.cybertec-postgresql.com/en/adjusting-maintenance_work_mem/ How Much maintenance_work_mem Do I Need?: http://rhaas.blogspot.com/2019/01/how-much-maintenanceworkmem-do-i-need.html Don't give Postgres too much memory (even on busy systems): https://vondra.me/posts/dont-give-postgres-too-much-memory-even-on-busy-systems/ + Understanding autovacuum in Amazon RDS for PostgreSQL: https://aws.amazon.com/blogs/database/understanding-autovacuum-in-amazon-rds-for-postgresql-environments/ + PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components checkpoint_related: From 1f494b779262eac14edec0e194e8a95bc44ce0a2 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:42:12 -0300 Subject: [PATCH 05/21] docs: update min_wal_size and max_wal_size guidance Clarify WAL recycling behavior and checkpoint triggering. Add recommendation to set max_wal_size for 1 hour of WAL with monitoring via pg_stat_bgwriter. Replace outdated 2016 reference with current best practices from official docs, EDB, Cybertec, and Crunchy Data. Signed-off-by: Sebastian Webber --- rules.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/rules.yml b/rules.yml index 6adf682..951e886 100644 --- a/rules.yml +++ b/rules.yml @@ -65,14 +65,28 @@ categories: checkpoint_related: min_wal_size: abstract: | - This parameter defines the minimum size of the pg_wal directory. pg_wal directory contains the WAL files. - Older versions refer to it as the pg_xlog directory. + Minimum size of pg_wal directory (pg_xlog in versions <10). WAL files are **recycled** rather than removed when below this threshold. + + Useful to handle **WAL spikes** during batch jobs or high write periods. recomendations: - "Configuration changes in 9.5: transaction log size": http://www.databasesoup.com/2016/01/configuration-changes-in-95-transaction.html + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html + "Checkpoint distance and amount of WAL": https://www.cybertec-postgresql.com/en/checkpoint-distance-and-amount-of-wal/ + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "Tuning Your Postgres Database for High Write Loads": https://www.crunchydata.com/blog/tuning-your-postgres-database-for-high-write-loads max_wal_size: abstract: | - This parameter defines the maximun size of the pg_wal directory. pg_wal directory contains the WAL files. - Older versions refer to it as the pg_xlog directory. + Triggers checkpoint when pg_wal exceeds this size. Larger values reduce checkpoint frequency but increase crash recovery time. + + **Recommendation**: Set to hold **1 hour of WAL**. Write-heavy systems may need significantly more. + + Monitor `pg_stat_bgwriter` to ensure most checkpoints are **timed** (not requested). + recomendations: + "Basics of Tuning Checkpoints": https://www.enterprisedb.com/blog/basics-tuning-checkpoints + "Tuning max_wal_size in PostgreSQL": https://www.enterprisedb.com/blog/tuning-maxwalsize-postgresql + "Checkpoint distance and amount of WAL": https://www.cybertec-postgresql.com/en/checkpoint-distance-and-amount-of-wal/ + "Tuning Your Postgres Database for High Write Loads": https://www.crunchydata.com/blog/tuning-your-postgres-database-for-high-write-loads + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices checkpoint_completion_target: abstract: This parameter defines a percentual of checkpoint_timeout as a target to write the CHECKPOINT data on the disk. recomendations: From 33ab67abbd003c353a6c35a19c9638785d04df34 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:50:50 -0300 Subject: [PATCH 06/21] docs: improve checkpoint_completion_target with example Add practical calculation example showing how 0.9 spreads writes over 4min 30s of a 5min interval. Include multiple monitoring resources for pg_stat_bgwriter. Replace outdated 2010 reference with current best practices and monitoring guides. Signed-off-by: Sebastian Webber --- rules.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/rules.yml b/rules.yml index 951e886..c928654 100644 --- a/rules.yml +++ b/rules.yml @@ -88,9 +88,20 @@ categories: "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices checkpoint_completion_target: - abstract: This parameter defines a percentual of checkpoint_timeout as a target to write the CHECKPOINT data on the disk. + abstract: | + Spreads checkpoint writes over this fraction of `checkpoint_timeout` to reduce I/O spikes. + + **Example**: `checkpoint_timeout = 5min` and `checkpoint_completion_target = 0.9` + → Checkpoint spreads writes over **270 seconds (4min 30s)**, leaving 30s buffer for sync overhead. + + Values higher than 0.9 risk checkpoint delays. Monitor via `pg_stat_bgwriter`. recomendations: - "Understaning postgresql.conf: CHECKPOINT_COMPLETION_TARGET": https://www.depesz.com/2010/11/03/checkpoint_completion_target/ + "Basics of Tuning Checkpoints": https://www.enterprisedb.com/blog/basics-tuning-checkpoints + "Measuring PostgreSQL Checkpoint Statistics": https://www.enterprisedb.com/blog/measuring-postgresql-checkpoint-statistics + "Checkpoints, Background Writer and monitoring": https://stormatics.tech/blogs/checkpoints-and-background-writer-and-how-to-monitor-it + "Deep dive into postgres stats: pg_stat_bgwriter": https://dataegret.com/2017/03/deep-dive-into-postgres-stats-pg_stat_bgwriter/ + "Understanding PostgreSQL Checkpoints": https://prateekcodes.com/understanding-postgres-checkpoints/ + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html wal_buffers: abstract: This parameter defines a buffer to store WAL changes before write it in the WAL file. recomendations: From 3a743a5ba582c895faf5a27101733c125c0e3941 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 19:58:17 -0300 Subject: [PATCH 07/21] feat: optimize wal_buffers for write-heavy workloads Set wal_buffers to 64MB for DW profile and 32MB for OLTP when shared_buffers > 8GB. Research shows these values can double performance for write-intensive systems vs auto-tune default. Add comprehensive table-driven tests covering all profiles and RAM sizes. Update documentation with profile-specific tuning recommendations and current best practices. Signed-off-by: Sebastian Webber --- pkg/category/checkpoint.go | 26 ++++++++++- pkg/rules/compute_test.go | 89 ++++++++++++++++++++++++++++++++++++++ rules.yml | 16 ++++++- 3 files changed, 128 insertions(+), 3 deletions(-) diff --git a/pkg/category/checkpoint.go b/pkg/category/checkpoint.go index e8cacfb..4516de2 100644 --- a/pkg/category/checkpoint.go +++ b/pkg/category/checkpoint.go @@ -3,6 +3,7 @@ package category import ( "github.com/pgconfig/api/pkg/input" "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" ) // CheckpointCfg is the checkpoint related category @@ -19,11 +20,34 @@ type CheckpointCfg struct { // For wal_buffers setting automatic by default. check this commit and the comments in the // function check_wal_buffers on https://github.com/postgres/postgres/commit/2594cf0e8c04406ffff19b1651c5a406d376657c#diff-0cf91b3df8a1bbd72140d10a0b4541b5R4915 func NewCheckpointCfg(in input.Input) *CheckpointCfg { + // Calculate shared_buffers to determine wal_buffers tuning + // Same calculation as in memory.go: totalRAM * MaxMemoryProfilePercent * SharedBufferPerc (25%) + maxMemoryProfilePercent := map[profile.Profile]float32{ + profile.Web: 1, + profile.OLTP: 1, + profile.DW: 1, + profile.Mixed: 0.5, + profile.Desktop: 0.2, + } + totalRAM := float32(in.TotalRAM) * maxMemoryProfilePercent[in.Profile] + sharedBuffers := bytes.Byte(totalRAM * 0.25) // SharedBufferPerc = 0.25 + + // DW (Data Warehouse) workloads benefit from larger wal_buffers + // for write-heavy operations + // OLTP with large shared_buffers (>8GB) indicates high concurrent writes + walBuffers := bytes.Byte(-1) // -1 means automatic tuning + + if in.Profile == profile.DW { + walBuffers = 64 * bytes.MB + } else if in.Profile == profile.OLTP && sharedBuffers > 8*bytes.GB { + walBuffers = 32 * bytes.MB + } + return &CheckpointCfg{ MinWALSize: bytes.Byte(2 * bytes.GB), MaxWALSize: bytes.Byte(3 * bytes.GB), CheckpointCompletionTarget: 0.9, - WALBuffers: -1, // -1 means automatic tuning + WALBuffers: walBuffers, CheckpointSegments: 16, } } diff --git a/pkg/rules/compute_test.go b/pkg/rules/compute_test.go index 61a2553..78e24ef 100644 --- a/pkg/rules/compute_test.go +++ b/pkg/rules/compute_test.go @@ -2,8 +2,97 @@ package rules import ( "testing" + + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" ) func TestCompute(t *testing.T) { t.SkipNow() } + +func TestComputeWalBuffers(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + totalRAM bytes.Byte + expectedWalBuffers bytes.Byte + description string + }{ + { + name: "DW profile always gets 64MB", + profile: profile.DW, + totalRAM: 100 * bytes.GB, + expectedWalBuffers: 64 * bytes.MB, + description: "Data Warehouse is write-heavy", + }, + { + name: "DW profile with small RAM still gets 64MB", + profile: profile.DW, + totalRAM: 8 * bytes.GB, + expectedWalBuffers: 64 * bytes.MB, + description: "DW always uses 64MB regardless of RAM", + }, + { + name: "OLTP with large shared_buffers gets 32MB", + profile: profile.OLTP, + totalRAM: 40 * bytes.GB, // shared_buffers = 40GB * 0.25 = 10GB > 8GB + expectedWalBuffers: 32 * bytes.MB, + description: "Large OLTP systems benefit from larger wal_buffers", + }, + { + name: "OLTP with small shared_buffers uses auto-tune", + profile: profile.OLTP, + totalRAM: 16 * bytes.GB, // shared_buffers = 16GB * 0.25 = 4GB < 8GB + expectedWalBuffers: -1, + description: "Small OLTP systems use auto-tuning", + }, + { + name: "Web profile uses auto-tune", + profile: profile.Web, + totalRAM: 100 * bytes.GB, + expectedWalBuffers: -1, + description: "Web workload uses default auto-tuning", + }, + { + name: "Mixed profile uses auto-tune", + profile: profile.Mixed, + totalRAM: 100 * bytes.GB, + expectedWalBuffers: -1, + description: "Mixed workload uses default auto-tuning", + }, + { + name: "Desktop profile uses auto-tune", + profile: profile.Desktop, + totalRAM: 16 * bytes.GB, + expectedWalBuffers: -1, + description: "Desktop workload uses default auto-tuning", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + TotalRAM: tt.totalRAM, + MaxConnections: 100, + DiskType: "ssd", + TotalCPU: 8, + PostgresVersion: 16.0, + } + + out, err := Compute(in) + if err != nil { + t.Fatalf("Compute failed: %v", err) + } + + if out.Checkpoint.WALBuffers != tt.expectedWalBuffers { + t.Errorf("%s: expected wal_buffers = %v, got %v", + tt.description, tt.expectedWalBuffers, out.Checkpoint.WALBuffers) + } + }) + } +} diff --git a/rules.yml b/rules.yml index c928654..00a175a 100644 --- a/rules.yml +++ b/rules.yml @@ -103,9 +103,21 @@ categories: "Understanding PostgreSQL Checkpoints": https://prateekcodes.com/understanding-postgres-checkpoints/ "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html wal_buffers: - abstract: This parameter defines a buffer to store WAL changes before write it in the WAL file. + abstract: | + Buffer for WAL data before flushing to disk. Default **-1** auto-tunes to 3% of `shared_buffers` (min 64kB, max 16MB). + + For **write-heavy workloads**, manual tuning can significantly improve performance: + - **DW profile**: 64MB (always, regardless of RAM) + - **OLTP profile**: 32MB (when shared_buffers > 8GB) + - **Others**: Auto-tune (default) + + WAL flushes at every commit, so extremely large values provide diminishing returns. recomendations: - Chapter 9 - Write Ahead Logging — WAL: http://www.interdb.jp/pg/pgsql09.html + "Tuning shared_buffers and wal_buffers": https://www.enterprisedb.com/blog/tuning-sharedbuffers-and-walbuffers + "Understanding shared_buffers, work_mem, and wal_buffers": https://www.postgresql.fastware.com/pzone/2024-06-understanding-shared-buffers-work-mem-and-wal-buffers-in-postgresql + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/runtime-config-wal.html + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "Tuning PostgreSQL for Write Heavy Workloads": https://www.cloudraft.io/blog/tuning-postgresql-for-write-heavy-workloads network_related: listen_addresses: abstract: This parameter defines a network address to bind to. From 00441091f1b41e676db8cde50ac4dcacecd161c9 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 20:01:50 -0300 Subject: [PATCH 08/21] docs: enhance network parameters with security and pooling Add security warnings for listen_addresses emphasizing localhost default and dangers of exposing to internet. Recommend specific IPs with pg_hba.conf or SSH tunnels. Expand max_connections guidance with connection pooling best practices. Include memory formula and PgBouncer references for high-concurrency scenarios. Signed-off-by: Sebastian Webber --- rules.yml | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/rules.yml b/rules.yml index 00a175a..7b8c413 100644 --- a/rules.yml +++ b/rules.yml @@ -120,11 +120,31 @@ categories: "Tuning PostgreSQL for Write Heavy Workloads": https://www.cloudraft.io/blog/tuning-postgresql-for-write-heavy-workloads network_related: listen_addresses: - abstract: This parameter defines a network address to bind to. + abstract: | + Network interfaces PostgreSQL listens on for connections. + + **Security**: Default is `localhost` (local-only). Never use `*` or `0.0.0.0` exposed to internet. + + Use specific IPs with `pg_hba.conf` rules, or SSH tunnels/VPN for remote access. + recomendations: + "PostgreSQL Connections and Authentication": https://www.postgresql.org/docs/current/runtime-config-connection.html + "PostgreSQL Security: 12 rules for database hardening": https://www.cybertec-postgresql.com/en/postgresql-security-things-to-avoid-in-real-life/ + "Postgres security best practices": https://www.bytebase.com/reference/postgres/how-to/postgres-security-best-practices/ max_connections: - abstract: This parameter defines a max connections allowed. + abstract: | + Maximum concurrent database connections. Each connection consumes memory (~10MB + work_mem per operation). + + **Best practice**: Use **connection pooling** (PgBouncer, pgpool) instead of high max_connections. + + With pooling: 20-50 connections. Without pooling: 100-200 (but review memory impact). + + Formula: `(RAM - shared_buffers) / (work_mem × avg_operations_per_query)` for rough estimate. recomendations: - Tuning max_connections in PostgreSQL: https://www.cybertec-postgresql.com/en/tuning-max_connections-in-postgresql/ + "Tuning max_connections in PostgreSQL": https://www.cybertec-postgresql.com/en/tuning-max_connections-in-postgresql/ + "Why you should use Connection Pooling": https://www.enterprisedb.com/postgres-tutorials/why-you-should-use-connection-pooling-when-setting-maxconnections-postgres + "PgBouncer for PostgreSQL: How Connection Pooling Solves Enterprise Slowdowns": https://www.percona.com/blog/pgbouncer-for-postgresql-how-connection-pooling-solves-enterprise-slowdowns/ + "Complete Guide to Fixing PostgreSQL Performance with PgBouncer": https://opstree.com/blog/2025/10/07/postgresql-performance-with-pgbouncer/ + "How To Optimize PostgreSQL For High Traffic": https://www.nilebits.com/blog/2025/06/postgresql-high-connections/ storage_type: random_page_cost: abstract: Sets the planner's estimate of the cost of a non-sequentially-fetched disk page. From 189251d6bef46646f58a03e5253ff8c428d337e3 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 20:03:53 -0300 Subject: [PATCH 09/21] feat: optimize min/max_wal_size per workload profile Tune WAL sizes based on workload characteristics: - DW (Data Warehouse): 4GB/16GB for write-heavy batch jobs - OLTP: 2GB/8GB for frequent transactions - Web: 1GB/4GB for moderate writes - Mixed: 2GB/6GB for balanced workload - Desktop: 512MB/2GB for low activity Add comprehensive tests validating all profile configurations. Aligns with best practice of holding ~1 hour of WAL for most systems while accommodating different write patterns. Signed-off-by: Sebastian Webber --- pkg/category/checkpoint.go | 32 ++++++++++++++- pkg/rules/compute_test.go | 82 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 112 insertions(+), 2 deletions(-) diff --git a/pkg/category/checkpoint.go b/pkg/category/checkpoint.go index 4516de2..40f85a5 100644 --- a/pkg/category/checkpoint.go +++ b/pkg/category/checkpoint.go @@ -43,9 +43,37 @@ func NewCheckpointCfg(in input.Input) *CheckpointCfg { walBuffers = 32 * bytes.MB } + // WAL size tuning per profile + // Recommended to hold ~1 hour of WAL for most systems + minWALSize := bytes.Byte(2 * bytes.GB) + maxWALSize := bytes.Byte(3 * bytes.GB) + + switch in.Profile { + case profile.DW: + // Data Warehouse: write-heavy with batch jobs + minWALSize = 4 * bytes.GB + maxWALSize = 16 * bytes.GB + case profile.OLTP: + // OLTP: frequent transactions + minWALSize = 2 * bytes.GB + maxWALSize = 8 * bytes.GB + case profile.Web: + // Web: moderate writes + minWALSize = 1 * bytes.GB + maxWALSize = 4 * bytes.GB + case profile.Mixed: + // Mixed: balanced workload + minWALSize = 2 * bytes.GB + maxWALSize = 6 * bytes.GB + case profile.Desktop: + // Desktop: low activity + minWALSize = 512 * bytes.MB + maxWALSize = 2 * bytes.GB + } + return &CheckpointCfg{ - MinWALSize: bytes.Byte(2 * bytes.GB), - MaxWALSize: bytes.Byte(3 * bytes.GB), + MinWALSize: minWALSize, + MaxWALSize: maxWALSize, CheckpointCompletionTarget: 0.9, WALBuffers: walBuffers, CheckpointSegments: 16, diff --git a/pkg/rules/compute_test.go b/pkg/rules/compute_test.go index 78e24ef..3b448a3 100644 --- a/pkg/rules/compute_test.go +++ b/pkg/rules/compute_test.go @@ -12,6 +12,88 @@ func TestCompute(t *testing.T) { t.SkipNow() } +func TestComputeWalSizes(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + totalRAM bytes.Byte + expectedMinWAL bytes.Byte + expectedMaxWAL bytes.Byte + description string + }{ + { + name: "DW profile gets large WAL sizes", + profile: profile.DW, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 4 * bytes.GB, + expectedMaxWAL: 16 * bytes.GB, + description: "Data Warehouse is write-heavy with batch jobs", + }, + { + name: "OLTP profile gets medium-large WAL sizes", + profile: profile.OLTP, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 2 * bytes.GB, + expectedMaxWAL: 8 * bytes.GB, + description: "OLTP has frequent transactions", + }, + { + name: "Web profile gets moderate WAL sizes", + profile: profile.Web, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 1 * bytes.GB, + expectedMaxWAL: 4 * bytes.GB, + description: "Web has moderate writes", + }, + { + name: "Mixed profile gets balanced WAL sizes", + profile: profile.Mixed, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 2 * bytes.GB, + expectedMaxWAL: 6 * bytes.GB, + description: "Mixed workload is balanced", + }, + { + name: "Desktop profile gets small WAL sizes", + profile: profile.Desktop, + totalRAM: 16 * bytes.GB, + expectedMinWAL: 512 * bytes.MB, + expectedMaxWAL: 2 * bytes.GB, + description: "Desktop has low activity", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + TotalRAM: tt.totalRAM, + MaxConnections: 100, + DiskType: "ssd", + TotalCPU: 8, + PostgresVersion: 16.0, + } + + out, err := Compute(in) + if err != nil { + t.Fatalf("Compute failed: %v", err) + } + + if out.Checkpoint.MinWALSize != tt.expectedMinWAL { + t.Errorf("%s: expected min_wal_size = %v, got %v", + tt.description, tt.expectedMinWAL, out.Checkpoint.MinWALSize) + } + + if out.Checkpoint.MaxWALSize != tt.expectedMaxWAL { + t.Errorf("%s: expected max_wal_size = %v, got %v", + tt.description, tt.expectedMaxWAL, out.Checkpoint.MaxWALSize) + } + }) + } +} + func TestComputeWalBuffers(t *testing.T) { tests := []struct { name string From 11b1034d1c12688f155d5e00966d86cbac1674e3 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 20:17:31 -0300 Subject: [PATCH 10/21] feat: optimize random_page_cost for DW analytical workloads Set random_page_cost to 1.8 for DW profile on SSD/SAN storage, as analytical queries often touch >10% of rows where sequential scans are more efficient than index scans. Other profiles maintain 1.1 for SSD to favor index scans. Add comprehensive table-driven tests covering all profile and storage combinations. Update documentation with 2025 debate on plan stability vs SSD optimization, and guidance on when sequential scans become more efficient. Signed-off-by: Sebastian Webber --- pkg/rules/storage.go | 7 ++++ pkg/rules/storage_test.go | 88 +++++++++++++++++++++++++++++++++++++++ rules.yml | 15 ++++++- 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/pkg/rules/storage.go b/pkg/rules/storage.go index cd114b2..46db1f8 100644 --- a/pkg/rules/storage.go +++ b/pkg/rules/storage.go @@ -3,6 +3,7 @@ package rules import ( "github.com/pgconfig/api/pkg/category" "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/profile" ) func computeStorage(in *input.Input, cfg *category.ExportCfg) (*category.ExportCfg, error) { @@ -21,6 +22,12 @@ func computeStorage(in *input.Input, cfg *category.ExportCfg) (*category.ExportC if in.DiskType != "HDD" { cfg.Storage.RandomPageCost = 1.1 + + // DW workloads: analytical queries often touch >10% of rows, + // where sequential scans are more efficient than index scans + if in.Profile == profile.DW { + cfg.Storage.RandomPageCost = 1.8 + } } return cfg, nil diff --git a/pkg/rules/storage_test.go b/pkg/rules/storage_test.go index f46f2a9..85a35f8 100644 --- a/pkg/rules/storage_test.go +++ b/pkg/rules/storage_test.go @@ -4,6 +4,9 @@ import ( "testing" "github.com/pgconfig/api/pkg/category" + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" ) func Test_computeStorage(t *testing.T) { @@ -38,3 +41,88 @@ func Test_computeStorage(t *testing.T) { t.Error("maintenance_io_concurrency should match effective_io_concurrency for HDD") } } + +func Test_computeStorageRandomPageCost(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + diskType string + expectedRandomPageCost float32 + description string + }{ + { + name: "DW profile with SSD gets higher random_page_cost", + profile: profile.DW, + diskType: "SSD", + expectedRandomPageCost: 1.8, + description: "DW analytical queries favor sequential scans", + }, + { + name: "DW profile with SAN gets higher random_page_cost", + profile: profile.DW, + diskType: "SAN", + expectedRandomPageCost: 1.8, + description: "DW analytical queries favor sequential scans on SAN too", + }, + { + name: "DW profile with HDD keeps default", + profile: profile.DW, + diskType: "HDD", + expectedRandomPageCost: 4.0, + description: "HDD uses PostgreSQL default", + }, + { + name: "OLTP profile with SSD gets low random_page_cost", + profile: profile.OLTP, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "OLTP favors index scans", + }, + { + name: "Web profile with SSD gets low random_page_cost", + profile: profile.Web, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "Web workload favors index scans", + }, + { + name: "Mixed profile with SSD gets low random_page_cost", + profile: profile.Mixed, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "Mixed workload uses general SSD value", + }, + { + name: "Desktop profile with SSD gets low random_page_cost", + profile: profile.Desktop, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "Desktop uses general SSD value", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + DiskType: tt.diskType, + TotalRAM: 100 * bytes.GB, + MaxConnections: 100, + TotalCPU: 8, + PostgresVersion: 16.0, + } + + out, err := computeStorage(&in, category.NewExportCfg(in)) + if err != nil { + t.Fatalf("computeStorage failed: %v", err) + } + + if out.Storage.RandomPageCost != tt.expectedRandomPageCost { + t.Errorf("%s: expected random_page_cost = %v, got %v", + tt.description, tt.expectedRandomPageCost, out.Storage.RandomPageCost) + } + }) + } +} diff --git a/rules.yml b/rules.yml index 7b8c413..6190407 100644 --- a/rules.yml +++ b/rules.yml @@ -147,9 +147,20 @@ categories: "How To Optimize PostgreSQL For High Traffic": https://www.nilebits.com/blog/2025/06/postgresql-high-connections/ storage_type: random_page_cost: - abstract: Sets the planner's estimate of the cost of a non-sequentially-fetched disk page. + abstract: | + Query planner's cost estimate for random disk access relative to sequential reads (`seq_page_cost = 1.0`). + + Lower values favor index scans, higher values favor sequential scans. Sequential scans become more efficient when queries return ~5-10% or more of table rows, common in analytical/DW workloads. + + **Debate (2025)**: Some experts advocate keeping higher values (4.0) for **plan stability** across cache states, while others recommend lower values (1.1-2.0) for SSD to favor index scans. + + Test with `EXPLAIN ANALYZE` to verify query plan choices for your workload. recomendations: - How a single PostgreSQL config change improved slow query performance by 50x: https://amplitude.engineering/how-a-single-postgresql-config-change-improved-slow-query-performance-by-50x-85593b8991b0 + "How a single PostgreSQL config change improved slow query performance by 50x": https://amplitude.engineering/how-a-single-postgresql-config-change-improved-slow-query-performance-by-50x-85593b8991b0 + "Better PostgreSQL performance on SSDs": https://www.cybertec-postgresql.com/en/better-postgresql-performance-on-ssds/ + "PostgreSQL with modern storage: what about a lower random_page_cost?": https://dev.to/aws-heroes/postgresql-with-modern-storage-what-about-a-lower-randompagecost-5b7f + "Postgres Scan Types in EXPLAIN Plans": https://www.crunchydata.com/blog/postgres-scan-types-in-explain-plans + "Tuning Your PostgreSQL Server": https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server effective_io_concurrency: abstract: Sets the number of concurrent disk I/O operations that PostgreSQL expects can be executed simultaneously. recomendations: From 281e49e987c5c2f9055f6f785fff99fc34497a99 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 20:33:07 -0300 Subject: [PATCH 11/21] docs: improve effective_io_concurrency documentation Clarify that parameter only affects bitmap heap scans and add context about when bitmap scans are used. Include note about PostgreSQL 18 default change from 1 to 16. Signed-off-by: Sebastian Webber --- rules.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index 6190407..9323b56 100644 --- a/rules.yml +++ b/rules.yml @@ -162,9 +162,16 @@ categories: "Postgres Scan Types in EXPLAIN Plans": https://www.crunchydata.com/blog/postgres-scan-types-in-explain-plans "Tuning Your PostgreSQL Server": https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server effective_io_concurrency: - abstract: Sets the number of concurrent disk I/O operations that PostgreSQL expects can be executed simultaneously. + abstract: | + Controls concurrent disk I/O operations for **bitmap heap scans** only. + + Bitmap scans are used when queries need to fetch moderate result sets (too many rows for index scans, too few for sequential scans) or when combining multiple indexes. They're more common in analytical workloads. + + PostgreSQL 18 changes the default from 1 to 16. Values above 200 show diminishing returns in benchmarks. recomendations: "PostgreSQL: effective_io_concurrency benchmarked": https://portavita.github.io/2019-07-19-PostgreSQL_effective_io_concurrency_benchmarked/ + "Bitmap Heap Scan - pganalyze": https://pganalyze.com/docs/explain/scan-nodes/bitmap-heap-scan + "PostgreSQL indexing: Index scan vs. Bitmap scan vs. Sequential scan (basics)": https://www.cybertec-postgresql.com/en/postgresql-indexing-index-scan-vs-bitmap-scan-vs-sequential-scan-basics/ io_method: abstract: Controls the asynchronous I/O implementation used by PostgreSQL. Options are "worker" (dedicated I/O worker processes), "io_uring" (Linux io_uring interface), and "sync" (traditional synchronous I/O). recomendations: From 4c202ef08dd12070303cd8cfd22eec8a7a69f5ec Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 20:43:20 -0300 Subject: [PATCH 12/21] docs: improve io_method documentation with usage guidance Clarify differences between worker, io_uring, and sync methods. Add recommendations on when to use each option and note that async I/O only affects reads in PostgreSQL 18. Signed-off-by: Sebastian Webber --- rules.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index 9323b56..fce2aa9 100644 --- a/rules.yml +++ b/rules.yml @@ -173,7 +173,16 @@ categories: "Bitmap Heap Scan - pganalyze": https://pganalyze.com/docs/explain/scan-nodes/bitmap-heap-scan "PostgreSQL indexing: Index scan vs. Bitmap scan vs. Sequential scan (basics)": https://www.cybertec-postgresql.com/en/postgresql-indexing-index-scan-vs-bitmap-scan-vs-sequential-scan-basics/ io_method: - abstract: Controls the asynchronous I/O implementation used by PostgreSQL. Options are "worker" (dedicated I/O worker processes), "io_uring" (Linux io_uring interface), and "sync" (traditional synchronous I/O). + abstract: | + Selects the async I/O implementation for read operations (PostgreSQL 18+). + + **worker** (default): Uses dedicated background processes. Best for most workloads, especially high-bandwidth sequential scans. Recommended as default. + + **io_uring** (Linux only): Kernel-level async I/O. Only switch after extensive testing proves benefit for your specific low-latency random-read patterns. Can hit file descriptor limits with high max_connections. + + **sync**: Traditional synchronous I/O. Slower than async methods - avoid unless debugging or testing. + + Note: Only affects reads. Writes, checkpoints, and WAL still use sync I/O. recomendations: "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ "Waiting for Postgres 18: Accelerating Disk Reads with Asynchronous I/O - pganalyze": https://pganalyze.com/blog/postgres-18-async-io From 227b6d91038d1db2406db7bb7fe98eb5ba68e2bd Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 20:50:30 -0300 Subject: [PATCH 13/21] docs: improve io_workers documentation with sizing guidance Add workload-specific recommendations for worker count based on CPU cores (10-40%). Explain when higher values are beneficial and how to monitor for saturation. Signed-off-by: Sebastian Webber --- rules.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index fce2aa9..99c4a93 100644 --- a/rules.yml +++ b/rules.yml @@ -190,7 +190,17 @@ categories: "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io "PostgreSQL 18: The AIO Revolution - dev.to": https://dev.to/mattleads/postgresql-18-the-aio-revolution-uuidv7-and-the-path-to-unprecedented-performance-415m io_workers: - abstract: Number of background I/O worker processes used when io_method is set to "worker". Determines how many concurrent I/O operations can be performed asynchronously. + abstract: | + Background worker processes for async I/O when `io_method = worker`. + + Default of 3 is too low for modern multi-core systems. Recommendation: **10-40% of CPU cores** depending on workload. + + Higher values benefit workloads with: + - Sequential scans (DW/analytical queries) + - High I/O latency (HDD vs SSD) + - High concurrent read operations + + Monitor `pg_stat_activity` for I/O wait events. If workers are saturated, increase this value. recomendations: "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ "Waiting for Postgres 18: Accelerating Disk Reads with Asynchronous I/O - pganalyze": https://pganalyze.com/blog/postgres-18-async-io From 314d81648864fbb7f78c0912cdc45ece24a944cf Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:02:03 -0300 Subject: [PATCH 14/21] docs: improve maintenance_io_concurrency documentation Clarify that parameter applies to VACUUM, CREATE INDEX, and ANALYZE. Add note about PostgreSQL 18 default change and relationship to effective_io_concurrency. Include workload-specific tuning references. Signed-off-by: Sebastian Webber --- rules.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index 99c4a93..0c33430 100644 --- a/rules.yml +++ b/rules.yml @@ -207,7 +207,17 @@ categories: "PostgreSQL 18: Better I/O performance with AIO - Cybertec": https://www.cybertec-postgresql.com/en/postgresql-18-better-i-o-performance-with-aio/ "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io maintenance_io_concurrency: - abstract: Sets the number of concurrent I/O operations that PostgreSQL expects can be executed simultaneously during maintenance operations such as VACUUM and CREATE INDEX. + abstract: | + Concurrent I/O operations for maintenance: **VACUUM**, **CREATE INDEX**, **ANALYZE**. + + Similar to `effective_io_concurrency` but for maintenance operations. PostgreSQL 18 changed default from 10 to 16. + + Typically set to match `effective_io_concurrency` based on storage capabilities. + recomendations: + "Quick Benchmark: ANALYZE vs. maintenance_io_concurrency": https://www.credativ.de/en/blog/postgresql-en/quick-benchmark-analyze-vs-maintenance_io_concurrency/ + "PostgreSQL 18 Asynchronous I/O: A Complete Guide": https://betterstack.com/community/guides/databases/postgresql-asynchronous-io/ + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "PostgreSQL Database Tuning for OLAP vs. OLTP Workloads": https://reintech.io/blog/postgresql-database-tuning-olap-vs-oltp io_combine_limit: abstract: Maximum size of adjacent blocks combined into one I/O request. recomendations: From 29b6c90bcca6bf2d3c698155d7079377b8321e3d Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:08:39 -0300 Subject: [PATCH 15/21] docs: improve io_combine_limit and io_max_combine_limit docs Clarify relationship between the two parameters and explain how larger values benefit sequential scans. Add note about PostgreSQL 18 requirement and data warehouse use cases. Signed-off-by: Sebastian Webber --- rules.yml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/rules.yml b/rules.yml index 0c33430..8174cc8 100644 --- a/rules.yml +++ b/rules.yml @@ -219,12 +219,18 @@ categories: "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices "PostgreSQL Database Tuning for OLAP vs. OLTP Workloads": https://reintech.io/blog/postgresql-database-tuning-olap-vs-oltp io_combine_limit: - abstract: Maximum size of adjacent blocks combined into one I/O request. + abstract: | + Maximum adjacent blocks combined into a single I/O request (PostgreSQL 18+). + + Larger values benefit sequential scans and bitmap heap scans by reducing I/O operations. Limited by `io_max_combine_limit` server setting. recomendations: - The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist (With Examples): https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples - PostgreSQL 18 Asynchronous Disk I/O - Deep Dive Into Implementation: https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ + "The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist": https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples + "PostgreSQL 18 Asynchronous Disk I/O - Deep Dive": https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ io_max_combine_limit: - abstract: Server-wide limit that clamps io_combine_limit, controlling the largest I/O size in operations that combine I/O. + abstract: | + Server-wide limit that clamps `io_combine_limit` (PostgreSQL 18+). + + Set at server startup. Controls the maximum I/O size for operations that combine I/O. Increase for data warehouses with large sequential scans. recomendations: "Allow io_combine_limit up to 1MB": https://www.postgresql.org/message-id/CA+hUKGKd=U1zSFYNjwBBrXV3NsqR2U8dCUrCBVeB0DQ8Vb8Dwg@mail.gmail.com "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ From c51e32f64de8a6b24e8d379d6475383f9298121d Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:11:26 -0300 Subject: [PATCH 16/21] docs: improve io_max_concurrency documentation Add read-ahead formula and explain relationship with other I/O parameters. Note memory pressure concerns with high concurrency and benefits for high-latency storage. Signed-off-by: Sebastian Webber --- rules.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/rules.yml b/rules.yml index 8174cc8..96d1684 100644 --- a/rules.yml +++ b/rules.yml @@ -240,15 +240,18 @@ categories: The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist (With Examples): https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples PostgreSQL 18 Asynchronous Disk I/O - Deep Dive Into Implementation: https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ io_max_concurrency: - abstract: Maximum number of concurrent I/O requests that can be in flight at once. + abstract: | + Hard limit on concurrent I/O operations per backend process (PostgreSQL 18+). + + Controls read-ahead with async I/O. Formula: `max read-ahead = effective_io_concurrency × io_combine_limit` + + Higher values benefit high-latency storage (cloud/EBS) with high IOPS. Watch memory usage - high concurrency increases memory pressure. recomendations: - "PostgreSQL 18 Beta 1 io_max_concurrency": https://www.postgresql.org/message-id/9673.1747250758@sss.pgh.pa.us + "PostgreSQL 18 Asynchronous I/O: A Complete Guide": https://betterstack.com/community/guides/databases/postgresql-asynchronous-io/ "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ "Waiting for Postgres 18: Accelerating Disk Reads with Asynchronous I/O - pganalyze": https://pganalyze.com/blog/postgres-18-async-io "PostgreSQL 18: Better I/O performance with AIO - Cybertec": https://www.cybertec-postgresql.com/en/postgresql-18-better-i-o-performance-with-aio/ "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io - The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist (With Examples): https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples - PostgreSQL 18 Asynchronous Disk I/O - Deep Dive Into Implementation: https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ file_copy_method: abstract: Selects the method used for copying files during CREATE DATABASE and ALTER DATABASE SET TABLESPACE operations. worker_related: From 510dfe26861932aff8a75bff3eb4d31485342cf2 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:17:27 -0300 Subject: [PATCH 17/21] docs: improve file_copy_method documentation Add recommendation to use clone method when filesystem supports it. Explain performance benefits (200-600ms for 100s of GB) and zero initial disk space consumption with CoW filesystems. Signed-off-by: Sebastian Webber --- rules.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/rules.yml b/rules.yml index 96d1684..ccb6014 100644 --- a/rules.yml +++ b/rules.yml @@ -253,7 +253,14 @@ categories: "PostgreSQL 18: Better I/O performance with AIO - Cybertec": https://www.cybertec-postgresql.com/en/postgresql-18-better-i-o-performance-with-aio/ "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io file_copy_method: - abstract: Selects the method used for copying files during CREATE DATABASE and ALTER DATABASE SET TABLESPACE operations. + abstract: | + Method for copying files during **CREATE DATABASE** and **ALTER DATABASE SET TABLESPACE** (PostgreSQL 18+). + + Recommendation: Use **clone** if your filesystem supports it - dramatically faster (200-600ms for 100s of GB) and initially consumes zero extra disk space. + recomendations: + "Instant database clones with PostgreSQL 18": https://boringsql.com/posts/instant-database-clones/ + "Instant Per-Branch Databases with PostgreSQL 18's clone": https://medium.com/axial-engineering/instant-per-branch-databases-with-postgresql-18s-clone-file-copy-and-copy-on-write-filesystems-1b1930bddbaa + "Git for Data: Instant PostgreSQL Database Cloning": https://vonng.com/en/pg/pg-clone/ worker_related: max_worker_processes: abstract: Sets the maximum number of background processes that the system can support. From 241f0a460908ceadc7e8046c4ccecc00d3baf081 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:20:58 -0300 Subject: [PATCH 18/21] docs: improve max_worker_processes documentation Explain worker pool concept and various consumers (parallel query, replication, extensions). Add recommendation to set based on CPU core count or at least 25% of vCPUs. Signed-off-by: Sebastian Webber --- rules.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/rules.yml b/rules.yml index ccb6014..9de4268 100644 --- a/rules.yml +++ b/rules.yml @@ -263,9 +263,19 @@ categories: "Git for Data: Instant PostgreSQL Database Cloning": https://vonng.com/en/pg/pg-clone/ worker_related: max_worker_processes: - abstract: Sets the maximum number of background processes that the system can support. + abstract: | + Maximum background worker processes (autovacuum, parallel query, replication, extensions). + + Pool from which all background workers are drawn. Must accommodate: + - Parallel query workers (`max_parallel_workers`) + - Logical replication workers + - Extensions (pg_stat_statements, etc.) + + Recommendation: Set to **CPU core count** or at least **25% of vCPUs**. Requires restart. recomendations: - "Comprehensive guide on how to tune database parameters and configuration in PostgreSQL": https://www.enterprisedb.com/postgres-tutorials/comprehensive-guide-how-tune-database-parameters-and-configuration-postgresql + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "PostgreSQL Performance Tuning: Key Parameters": https://www.tigerdata.com/learn/postgresql-performance-tuning-key-parameters + "Parallel Queries in Postgres - Crunchy Data": https://www.crunchydata.com/blog/parallel-queries-in-postgres max_parallel_workers_per_gather: abstract: Sets the maximum number of parallel processes per executor node. recomendations: From 582b6efdb2964eb43f6a20224ef9bcc74c9daec8 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:27:46 -0300 Subject: [PATCH 19/21] docs: improve max_parallel_workers_per_gather documentation Explain resource multiplication effect (N workers = N+1x resources). Add references for parallel query tuning and analytics workload optimization. Signed-off-by: Sebastian Webber --- rules.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rules.yml b/rules.yml index 9de4268..62cdeb7 100644 --- a/rules.yml +++ b/rules.yml @@ -277,9 +277,15 @@ categories: "PostgreSQL Performance Tuning: Key Parameters": https://www.tigerdata.com/learn/postgresql-performance-tuning-key-parameters "Parallel Queries in Postgres - Crunchy Data": https://www.crunchydata.com/blog/parallel-queries-in-postgres max_parallel_workers_per_gather: - abstract: Sets the maximum number of parallel processes per executor node. + abstract: | + Maximum parallel workers per query executor node. + + Each worker consumes resources individually (work_mem, CPU, I/O). A query with 4 workers uses 5x resources (1 leader + 4 workers). recomendations: - "Comprehensive guide on how to tune database parameters and configuration in PostgreSQL": https://www.enterprisedb.com/postgres-tutorials/comprehensive-guide-how-tune-database-parameters-and-configuration-postgresql + "Increasing max parallel workers per gather in Postgres": https://www.pgmustard.com/blog/max-parallel-workers-per-gather + "Postgres Tuning & Performance for Analytics Data": https://www.crunchydata.com/blog/postgres-tuning-and-performance-for-analytics-data + "Parallel Queries in Postgres": https://www.crunchydata.com/blog/parallel-queries-in-postgres + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices max_parallel_workers: abstract: Sets the maximum number of parallel workers that can be active at one time recomendations: From d49b2382bba9acfbc48e4bcec96febd69763faeb Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:28:14 -0300 Subject: [PATCH 20/21] docs: improve max_parallel_workers documentation Clarify system-wide limit and relationship with max_worker_processes. Add recommendation to set equal to CPU core count. Signed-off-by: Sebastian Webber --- rules.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rules.yml b/rules.yml index 62cdeb7..e792062 100644 --- a/rules.yml +++ b/rules.yml @@ -287,7 +287,13 @@ categories: "Parallel Queries in Postgres": https://www.crunchydata.com/blog/parallel-queries-in-postgres "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices max_parallel_workers: - abstract: Sets the maximum number of parallel workers that can be active at one time + abstract: | + Maximum parallel workers active system-wide across all queries. + + Limits total parallel workers from the `max_worker_processes` pool. Cannot exceed `max_worker_processes`. + + Recommendation: Set equal to **CPU core count** or `max_worker_processes`. recomendations: - "Comprehensive guide on how to tune database parameters and configuration in PostgreSQL": https://www.enterprisedb.com/postgres-tutorials/comprehensive-guide-how-tune-database-parameters-and-configuration-postgresql + "Parallel Queries in Postgres": https://www.crunchydata.com/blog/parallel-queries-in-postgres + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices From 0850b686a00215fe289c2162b48847a223f06e48 Mon Sep 17 00:00:00 2001 From: Sebastian Webber Date: Wed, 4 Feb 2026 21:30:19 -0300 Subject: [PATCH 21/21] feat: make worker parameters dynamic based on CPU and profile Scale worker parameters with CPU cores instead of using fixed values: - max_worker_processes: minimum 8 or CPU count - max_parallel_workers: minimum 8 or CPU count - max_parallel_workers_per_gather: 2 for transactional workloads (Desktop/Web/Mixed/OLTP), CPU/2 for DW (analytical workloads) Add comprehensive table-driven tests covering various CPU counts and workload profiles. Signed-off-by: Sebastian Webber --- pkg/category/worker.go | 44 ++++++++++-- pkg/category/worker_test.go | 135 ++++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 4 deletions(-) create mode 100644 pkg/category/worker_test.go diff --git a/pkg/category/worker.go b/pkg/category/worker.go index a220ee6..44c66aa 100644 --- a/pkg/category/worker.go +++ b/pkg/category/worker.go @@ -1,6 +1,9 @@ package category -import "github.com/pgconfig/api/pkg/input" +import ( + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/profile" +) // WorkerCfg is the main workers category type WorkerCfg struct { @@ -11,9 +14,42 @@ type WorkerCfg struct { // NewWorkerCfg creates a new Worker Configuration func NewWorkerCfg(in input.Input) *WorkerCfg { + // max_worker_processes: at least 8 (default), or CPU count + maxWorkerProcesses := max(8, in.TotalCPU) + + // max_parallel_workers: at least 8, or CPU count (limited by max_worker_processes) + maxParallelWorkers := max(8, in.TotalCPU) + + // max_parallel_workers_per_gather: varies by profile + // OLTP/transactional workloads keep default (2) + // DW/analytical workloads benefit from higher parallelism + maxParallelWorkerPerGather := 2 + if in.Profile == profile.DW { + // DW: use half of CPU cores, limited by max_parallel_workers + maxParallelWorkerPerGather = min(in.TotalCPU/2, maxParallelWorkers) + // Ensure at least 2 + if maxParallelWorkerPerGather < 2 { + maxParallelWorkerPerGather = 2 + } + } + return &WorkerCfg{ - MaxWorkerProcesses: 8, /* pg >= 9.4 */ - MaxParallelWorkerPerGather: 2, /* pg >= 9.6 */ - MaxParallelWorkers: 2, /* pg >= 10 */ + MaxWorkerProcesses: maxWorkerProcesses, + MaxParallelWorkerPerGather: maxParallelWorkerPerGather, + MaxParallelWorkers: maxParallelWorkers, + } +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func min(a, b int) int { + if a < b { + return a } + return b } diff --git a/pkg/category/worker_test.go b/pkg/category/worker_test.go new file mode 100644 index 0000000..f8c740d --- /dev/null +++ b/pkg/category/worker_test.go @@ -0,0 +1,135 @@ +package category + +import ( + "testing" + + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" +) + +func TestNewWorkerCfg(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + totalCPU int + expectedMaxWorkerProcesses int + expectedMaxParallelWorkers int + expectedMaxParallelWorkerPerGather int + description string + }{ + { + name: "Desktop with 4 cores", + profile: profile.Desktop, + totalCPU: 4, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 2, + description: "Small system uses minimum of 8 for worker processes", + }, + { + name: "Web with 8 cores", + profile: profile.Web, + totalCPU: 8, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 2, + description: "Web keeps default parallel workers per gather", + }, + { + name: "OLTP with 16 cores", + profile: profile.OLTP, + totalCPU: 16, + expectedMaxWorkerProcesses: 16, + expectedMaxParallelWorkers: 16, + expectedMaxParallelWorkerPerGather: 2, + description: "OLTP scales workers with CPU but keeps per-gather at 2", + }, + { + name: "Mixed with 16 cores", + profile: profile.Mixed, + totalCPU: 16, + expectedMaxWorkerProcesses: 16, + expectedMaxParallelWorkers: 16, + expectedMaxParallelWorkerPerGather: 2, + description: "Mixed workload uses default parallel workers per gather", + }, + { + name: "DW with 8 cores", + profile: profile.DW, + totalCPU: 8, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 4, + description: "DW uses CPU/2 for parallel workers per gather", + }, + { + name: "DW with 16 cores", + profile: profile.DW, + totalCPU: 16, + expectedMaxWorkerProcesses: 16, + expectedMaxParallelWorkers: 16, + expectedMaxParallelWorkerPerGather: 8, + description: "DW scales parallel workers per gather with CPU", + }, + { + name: "DW with 32 cores", + profile: profile.DW, + totalCPU: 32, + expectedMaxWorkerProcesses: 32, + expectedMaxParallelWorkers: 32, + expectedMaxParallelWorkerPerGather: 16, + description: "DW with many cores gets high parallelism", + }, + { + name: "DW with 2 cores ensures minimum", + profile: profile.DW, + totalCPU: 2, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 2, + description: "DW with few cores still gets minimum values", + }, + { + name: "Large system with 64 cores", + profile: profile.OLTP, + totalCPU: 64, + expectedMaxWorkerProcesses: 64, + expectedMaxParallelWorkers: 64, + expectedMaxParallelWorkerPerGather: 2, + description: "Large OLTP system scales workers but not per-gather", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + TotalCPU: tt.totalCPU, + TotalRAM: 16 * bytes.GB, + MaxConnections: 100, + DiskType: "SSD", + PostgresVersion: 16.0, + } + + cfg := NewWorkerCfg(in) + + if cfg.MaxWorkerProcesses != tt.expectedMaxWorkerProcesses { + t.Errorf("%s: expected max_worker_processes = %d, got %d", + tt.description, tt.expectedMaxWorkerProcesses, cfg.MaxWorkerProcesses) + } + + if cfg.MaxParallelWorkers != tt.expectedMaxParallelWorkers { + t.Errorf("%s: expected max_parallel_workers = %d, got %d", + tt.description, tt.expectedMaxParallelWorkers, cfg.MaxParallelWorkers) + } + + if cfg.MaxParallelWorkerPerGather != tt.expectedMaxParallelWorkerPerGather { + t.Errorf("%s: expected max_parallel_workers_per_gather = %d, got %d", + tt.description, tt.expectedMaxParallelWorkerPerGather, cfg.MaxParallelWorkerPerGather) + } + }) + } +}