diff --git a/pkg/category/checkpoint.go b/pkg/category/checkpoint.go index e8cacfb..40f85a5 100644 --- a/pkg/category/checkpoint.go +++ b/pkg/category/checkpoint.go @@ -3,6 +3,7 @@ package category import ( "github.com/pgconfig/api/pkg/input" "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" ) // CheckpointCfg is the checkpoint related category @@ -19,11 +20,62 @@ type CheckpointCfg struct { // For wal_buffers setting automatic by default. check this commit and the comments in the // function check_wal_buffers on https://github.com/postgres/postgres/commit/2594cf0e8c04406ffff19b1651c5a406d376657c#diff-0cf91b3df8a1bbd72140d10a0b4541b5R4915 func NewCheckpointCfg(in input.Input) *CheckpointCfg { + // Calculate shared_buffers to determine wal_buffers tuning + // Same calculation as in memory.go: totalRAM * MaxMemoryProfilePercent * SharedBufferPerc (25%) + maxMemoryProfilePercent := map[profile.Profile]float32{ + profile.Web: 1, + profile.OLTP: 1, + profile.DW: 1, + profile.Mixed: 0.5, + profile.Desktop: 0.2, + } + totalRAM := float32(in.TotalRAM) * maxMemoryProfilePercent[in.Profile] + sharedBuffers := bytes.Byte(totalRAM * 0.25) // SharedBufferPerc = 0.25 + + // DW (Data Warehouse) workloads benefit from larger wal_buffers + // for write-heavy operations + // OLTP with large shared_buffers (>8GB) indicates high concurrent writes + walBuffers := bytes.Byte(-1) // -1 means automatic tuning + + if in.Profile == profile.DW { + walBuffers = 64 * bytes.MB + } else if in.Profile == profile.OLTP && sharedBuffers > 8*bytes.GB { + walBuffers = 32 * bytes.MB + } + + // WAL size tuning per profile + // Recommended to hold ~1 hour of WAL for most systems + minWALSize := bytes.Byte(2 * bytes.GB) + maxWALSize := bytes.Byte(3 * bytes.GB) + + switch in.Profile { + case profile.DW: + // Data Warehouse: write-heavy with batch jobs + minWALSize = 4 * bytes.GB + maxWALSize = 16 * bytes.GB + case profile.OLTP: + // OLTP: frequent transactions + minWALSize = 2 * bytes.GB + maxWALSize = 8 * bytes.GB + case profile.Web: + // Web: moderate writes + minWALSize = 1 * bytes.GB + maxWALSize = 4 * bytes.GB + case profile.Mixed: + // Mixed: balanced workload + minWALSize = 2 * bytes.GB + maxWALSize = 6 * bytes.GB + case profile.Desktop: + // Desktop: low activity + minWALSize = 512 * bytes.MB + maxWALSize = 2 * bytes.GB + } + return &CheckpointCfg{ - MinWALSize: bytes.Byte(2 * bytes.GB), - MaxWALSize: bytes.Byte(3 * bytes.GB), + MinWALSize: minWALSize, + MaxWALSize: maxWALSize, CheckpointCompletionTarget: 0.9, - WALBuffers: -1, // -1 means automatic tuning + WALBuffers: walBuffers, CheckpointSegments: 16, } } diff --git a/pkg/category/worker.go b/pkg/category/worker.go index a220ee6..44c66aa 100644 --- a/pkg/category/worker.go +++ b/pkg/category/worker.go @@ -1,6 +1,9 @@ package category -import "github.com/pgconfig/api/pkg/input" +import ( + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/profile" +) // WorkerCfg is the main workers category type WorkerCfg struct { @@ -11,9 +14,42 @@ type WorkerCfg struct { // NewWorkerCfg creates a new Worker Configuration func NewWorkerCfg(in input.Input) *WorkerCfg { + // max_worker_processes: at least 8 (default), or CPU count + maxWorkerProcesses := max(8, in.TotalCPU) + + // max_parallel_workers: at least 8, or CPU count (limited by max_worker_processes) + maxParallelWorkers := max(8, in.TotalCPU) + + // max_parallel_workers_per_gather: varies by profile + // OLTP/transactional workloads keep default (2) + // DW/analytical workloads benefit from higher parallelism + maxParallelWorkerPerGather := 2 + if in.Profile == profile.DW { + // DW: use half of CPU cores, limited by max_parallel_workers + maxParallelWorkerPerGather = min(in.TotalCPU/2, maxParallelWorkers) + // Ensure at least 2 + if maxParallelWorkerPerGather < 2 { + maxParallelWorkerPerGather = 2 + } + } + return &WorkerCfg{ - MaxWorkerProcesses: 8, /* pg >= 9.4 */ - MaxParallelWorkerPerGather: 2, /* pg >= 9.6 */ - MaxParallelWorkers: 2, /* pg >= 10 */ + MaxWorkerProcesses: maxWorkerProcesses, + MaxParallelWorkerPerGather: maxParallelWorkerPerGather, + MaxParallelWorkers: maxParallelWorkers, + } +} + +func max(a, b int) int { + if a > b { + return a + } + return b +} + +func min(a, b int) int { + if a < b { + return a } + return b } diff --git a/pkg/category/worker_test.go b/pkg/category/worker_test.go new file mode 100644 index 0000000..f8c740d --- /dev/null +++ b/pkg/category/worker_test.go @@ -0,0 +1,135 @@ +package category + +import ( + "testing" + + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" +) + +func TestNewWorkerCfg(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + totalCPU int + expectedMaxWorkerProcesses int + expectedMaxParallelWorkers int + expectedMaxParallelWorkerPerGather int + description string + }{ + { + name: "Desktop with 4 cores", + profile: profile.Desktop, + totalCPU: 4, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 2, + description: "Small system uses minimum of 8 for worker processes", + }, + { + name: "Web with 8 cores", + profile: profile.Web, + totalCPU: 8, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 2, + description: "Web keeps default parallel workers per gather", + }, + { + name: "OLTP with 16 cores", + profile: profile.OLTP, + totalCPU: 16, + expectedMaxWorkerProcesses: 16, + expectedMaxParallelWorkers: 16, + expectedMaxParallelWorkerPerGather: 2, + description: "OLTP scales workers with CPU but keeps per-gather at 2", + }, + { + name: "Mixed with 16 cores", + profile: profile.Mixed, + totalCPU: 16, + expectedMaxWorkerProcesses: 16, + expectedMaxParallelWorkers: 16, + expectedMaxParallelWorkerPerGather: 2, + description: "Mixed workload uses default parallel workers per gather", + }, + { + name: "DW with 8 cores", + profile: profile.DW, + totalCPU: 8, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 4, + description: "DW uses CPU/2 for parallel workers per gather", + }, + { + name: "DW with 16 cores", + profile: profile.DW, + totalCPU: 16, + expectedMaxWorkerProcesses: 16, + expectedMaxParallelWorkers: 16, + expectedMaxParallelWorkerPerGather: 8, + description: "DW scales parallel workers per gather with CPU", + }, + { + name: "DW with 32 cores", + profile: profile.DW, + totalCPU: 32, + expectedMaxWorkerProcesses: 32, + expectedMaxParallelWorkers: 32, + expectedMaxParallelWorkerPerGather: 16, + description: "DW with many cores gets high parallelism", + }, + { + name: "DW with 2 cores ensures minimum", + profile: profile.DW, + totalCPU: 2, + expectedMaxWorkerProcesses: 8, + expectedMaxParallelWorkers: 8, + expectedMaxParallelWorkerPerGather: 2, + description: "DW with few cores still gets minimum values", + }, + { + name: "Large system with 64 cores", + profile: profile.OLTP, + totalCPU: 64, + expectedMaxWorkerProcesses: 64, + expectedMaxParallelWorkers: 64, + expectedMaxParallelWorkerPerGather: 2, + description: "Large OLTP system scales workers but not per-gather", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + TotalCPU: tt.totalCPU, + TotalRAM: 16 * bytes.GB, + MaxConnections: 100, + DiskType: "SSD", + PostgresVersion: 16.0, + } + + cfg := NewWorkerCfg(in) + + if cfg.MaxWorkerProcesses != tt.expectedMaxWorkerProcesses { + t.Errorf("%s: expected max_worker_processes = %d, got %d", + tt.description, tt.expectedMaxWorkerProcesses, cfg.MaxWorkerProcesses) + } + + if cfg.MaxParallelWorkers != tt.expectedMaxParallelWorkers { + t.Errorf("%s: expected max_parallel_workers = %d, got %d", + tt.description, tt.expectedMaxParallelWorkers, cfg.MaxParallelWorkers) + } + + if cfg.MaxParallelWorkerPerGather != tt.expectedMaxParallelWorkerPerGather { + t.Errorf("%s: expected max_parallel_workers_per_gather = %d, got %d", + tt.description, tt.expectedMaxParallelWorkerPerGather, cfg.MaxParallelWorkerPerGather) + } + }) + } +} diff --git a/pkg/rules/compute_test.go b/pkg/rules/compute_test.go index 61a2553..3b448a3 100644 --- a/pkg/rules/compute_test.go +++ b/pkg/rules/compute_test.go @@ -2,8 +2,179 @@ package rules import ( "testing" + + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" ) func TestCompute(t *testing.T) { t.SkipNow() } + +func TestComputeWalSizes(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + totalRAM bytes.Byte + expectedMinWAL bytes.Byte + expectedMaxWAL bytes.Byte + description string + }{ + { + name: "DW profile gets large WAL sizes", + profile: profile.DW, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 4 * bytes.GB, + expectedMaxWAL: 16 * bytes.GB, + description: "Data Warehouse is write-heavy with batch jobs", + }, + { + name: "OLTP profile gets medium-large WAL sizes", + profile: profile.OLTP, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 2 * bytes.GB, + expectedMaxWAL: 8 * bytes.GB, + description: "OLTP has frequent transactions", + }, + { + name: "Web profile gets moderate WAL sizes", + profile: profile.Web, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 1 * bytes.GB, + expectedMaxWAL: 4 * bytes.GB, + description: "Web has moderate writes", + }, + { + name: "Mixed profile gets balanced WAL sizes", + profile: profile.Mixed, + totalRAM: 100 * bytes.GB, + expectedMinWAL: 2 * bytes.GB, + expectedMaxWAL: 6 * bytes.GB, + description: "Mixed workload is balanced", + }, + { + name: "Desktop profile gets small WAL sizes", + profile: profile.Desktop, + totalRAM: 16 * bytes.GB, + expectedMinWAL: 512 * bytes.MB, + expectedMaxWAL: 2 * bytes.GB, + description: "Desktop has low activity", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + TotalRAM: tt.totalRAM, + MaxConnections: 100, + DiskType: "ssd", + TotalCPU: 8, + PostgresVersion: 16.0, + } + + out, err := Compute(in) + if err != nil { + t.Fatalf("Compute failed: %v", err) + } + + if out.Checkpoint.MinWALSize != tt.expectedMinWAL { + t.Errorf("%s: expected min_wal_size = %v, got %v", + tt.description, tt.expectedMinWAL, out.Checkpoint.MinWALSize) + } + + if out.Checkpoint.MaxWALSize != tt.expectedMaxWAL { + t.Errorf("%s: expected max_wal_size = %v, got %v", + tt.description, tt.expectedMaxWAL, out.Checkpoint.MaxWALSize) + } + }) + } +} + +func TestComputeWalBuffers(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + totalRAM bytes.Byte + expectedWalBuffers bytes.Byte + description string + }{ + { + name: "DW profile always gets 64MB", + profile: profile.DW, + totalRAM: 100 * bytes.GB, + expectedWalBuffers: 64 * bytes.MB, + description: "Data Warehouse is write-heavy", + }, + { + name: "DW profile with small RAM still gets 64MB", + profile: profile.DW, + totalRAM: 8 * bytes.GB, + expectedWalBuffers: 64 * bytes.MB, + description: "DW always uses 64MB regardless of RAM", + }, + { + name: "OLTP with large shared_buffers gets 32MB", + profile: profile.OLTP, + totalRAM: 40 * bytes.GB, // shared_buffers = 40GB * 0.25 = 10GB > 8GB + expectedWalBuffers: 32 * bytes.MB, + description: "Large OLTP systems benefit from larger wal_buffers", + }, + { + name: "OLTP with small shared_buffers uses auto-tune", + profile: profile.OLTP, + totalRAM: 16 * bytes.GB, // shared_buffers = 16GB * 0.25 = 4GB < 8GB + expectedWalBuffers: -1, + description: "Small OLTP systems use auto-tuning", + }, + { + name: "Web profile uses auto-tune", + profile: profile.Web, + totalRAM: 100 * bytes.GB, + expectedWalBuffers: -1, + description: "Web workload uses default auto-tuning", + }, + { + name: "Mixed profile uses auto-tune", + profile: profile.Mixed, + totalRAM: 100 * bytes.GB, + expectedWalBuffers: -1, + description: "Mixed workload uses default auto-tuning", + }, + { + name: "Desktop profile uses auto-tune", + profile: profile.Desktop, + totalRAM: 16 * bytes.GB, + expectedWalBuffers: -1, + description: "Desktop workload uses default auto-tuning", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + TotalRAM: tt.totalRAM, + MaxConnections: 100, + DiskType: "ssd", + TotalCPU: 8, + PostgresVersion: 16.0, + } + + out, err := Compute(in) + if err != nil { + t.Fatalf("Compute failed: %v", err) + } + + if out.Checkpoint.WALBuffers != tt.expectedWalBuffers { + t.Errorf("%s: expected wal_buffers = %v, got %v", + tt.description, tt.expectedWalBuffers, out.Checkpoint.WALBuffers) + } + }) + } +} diff --git a/pkg/rules/storage.go b/pkg/rules/storage.go index cd114b2..46db1f8 100644 --- a/pkg/rules/storage.go +++ b/pkg/rules/storage.go @@ -3,6 +3,7 @@ package rules import ( "github.com/pgconfig/api/pkg/category" "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/profile" ) func computeStorage(in *input.Input, cfg *category.ExportCfg) (*category.ExportCfg, error) { @@ -21,6 +22,12 @@ func computeStorage(in *input.Input, cfg *category.ExportCfg) (*category.ExportC if in.DiskType != "HDD" { cfg.Storage.RandomPageCost = 1.1 + + // DW workloads: analytical queries often touch >10% of rows, + // where sequential scans are more efficient than index scans + if in.Profile == profile.DW { + cfg.Storage.RandomPageCost = 1.8 + } } return cfg, nil diff --git a/pkg/rules/storage_test.go b/pkg/rules/storage_test.go index f46f2a9..85a35f8 100644 --- a/pkg/rules/storage_test.go +++ b/pkg/rules/storage_test.go @@ -4,6 +4,9 @@ import ( "testing" "github.com/pgconfig/api/pkg/category" + "github.com/pgconfig/api/pkg/input" + "github.com/pgconfig/api/pkg/input/bytes" + "github.com/pgconfig/api/pkg/input/profile" ) func Test_computeStorage(t *testing.T) { @@ -38,3 +41,88 @@ func Test_computeStorage(t *testing.T) { t.Error("maintenance_io_concurrency should match effective_io_concurrency for HDD") } } + +func Test_computeStorageRandomPageCost(t *testing.T) { + tests := []struct { + name string + profile profile.Profile + diskType string + expectedRandomPageCost float32 + description string + }{ + { + name: "DW profile with SSD gets higher random_page_cost", + profile: profile.DW, + diskType: "SSD", + expectedRandomPageCost: 1.8, + description: "DW analytical queries favor sequential scans", + }, + { + name: "DW profile with SAN gets higher random_page_cost", + profile: profile.DW, + diskType: "SAN", + expectedRandomPageCost: 1.8, + description: "DW analytical queries favor sequential scans on SAN too", + }, + { + name: "DW profile with HDD keeps default", + profile: profile.DW, + diskType: "HDD", + expectedRandomPageCost: 4.0, + description: "HDD uses PostgreSQL default", + }, + { + name: "OLTP profile with SSD gets low random_page_cost", + profile: profile.OLTP, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "OLTP favors index scans", + }, + { + name: "Web profile with SSD gets low random_page_cost", + profile: profile.Web, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "Web workload favors index scans", + }, + { + name: "Mixed profile with SSD gets low random_page_cost", + profile: profile.Mixed, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "Mixed workload uses general SSD value", + }, + { + name: "Desktop profile with SSD gets low random_page_cost", + profile: profile.Desktop, + diskType: "SSD", + expectedRandomPageCost: 1.1, + description: "Desktop uses general SSD value", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + in := input.Input{ + OS: "linux", + Arch: "amd64", + Profile: tt.profile, + DiskType: tt.diskType, + TotalRAM: 100 * bytes.GB, + MaxConnections: 100, + TotalCPU: 8, + PostgresVersion: 16.0, + } + + out, err := computeStorage(&in, category.NewExportCfg(in)) + if err != nil { + t.Fatalf("computeStorage failed: %v", err) + } + + if out.Storage.RandomPageCost != tt.expectedRandomPageCost { + t.Errorf("%s: expected random_page_cost = %v, got %v", + tt.description, tt.expectedRandomPageCost, out.Storage.RandomPageCost) + } + }) + } +} diff --git a/rules.yml b/rules.yml index 8a5ca46..e792062 100644 --- a/rules.yml +++ b/rules.yml @@ -1,72 +1,188 @@ categories: memory_related: shared_buffers: - abstract: This parameter allocate memory slots used by all process when the database starts. Mainly works as the disk cache and its similar to oracle's SGA buffer. + abstract: | + Allocates shared memory for caching data pages. Acts as PostgreSQL's main disk cache, similar to Oracle's SGA buffer. + + Start with **25% of RAM** as a baseline. For optimal tuning, use the **pg_buffercache extension** to analyze cache hit ratios for your specific workload. recomendations: Tuning Your PostgreSQL Server: https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server#shared_buffers - Tuning shared_buffers and wal_buffers: http://rhaas.blogspot.com.br/2012/03/tuning-sharedbuffers-and-walbuffers.html + Determining optimal shared_buffers using pg_buffercache: https://aws.amazon.com/blogs/database/determining-the-optimal-value-for-shared_buffers-using-the-pg_buffercache-extension-in-postgresql/ + Tuning shared_buffers for OLTP and data warehouse workloads: https://pganalyze.com/blog/5mins-postgres-tuning-shared-buffers-pgbench-TPROC-C-TPROC-H + "Introduction to Buffers in PostgreSQL": https://boringsql.com/posts/introduction-to-buffers/ + "PostgreSQL double buffering: understand the cache size": https://dev.to/franckpachot/postgresql-double-buffering-understand-the-cache-size-in-a-managed-service-oci-2oci + PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components effective_cache_size: - abstract: This parameter does not allocate any resource, just tells to the query planner how much of the operating system cache are avaliable to use. Remember that shared_buffers needs to smaller than 8GB, then the query planner will prefer read the disk because it will be on memory. + abstract: | + Estimates total memory available for disk caching (PostgreSQL + OS cache). Used by query planner for cost estimates. + + Does **NOT allocate memory** - only informs the planner about available cache. + + A good starting point: `effective_cache_size = RAM - shared_buffers` recomendations: "effective_cache_size: A practical example": https://www.cybertec-postgresql.com/en/effective_cache_size-a-practical-example/ "effective_cache_size: What it means in PostgreSQL": https://www.cybertec-postgresql.com/en/effective_cache_size-what-it-means-in-postgresql/ Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance work_mem: - abstract: This parameter defines how much a work_mem buffer can allocate. Each query can open many work_mem buffers when execute (normally one by subquery) if it uses any sort (or aggregate) operation. When work_mem its too small a temp file is created. + abstract: | + Memory per operation for sorts, hash joins, and aggregates. Each query can use **multiple work_mem buffers** simultaneously. + + **⚠️ Warning**: With high concurrency and large datasets, you can easily trigger **OOM kills** in Kubernetes pods or cloud instances. + + Maximum potential memory = `work_mem × operations × parallel_workers × connections` + + Example worst-case: 128MB × 3 operations × 2 workers × 100 connections = **102GB** + + Monitor temp file usage with `log_temp_files`. Consider **per-session** tuning (`SET work_mem`) for heavy queries instead of global settings. details: - Specifies the amount of memory to be used by internal sort operations and hash tables before writing to temporary disk files. The value defaults to four megabytes (4MB). Note that for a complex query, several sort or hash operations might be running in parallel; each operation will be allowed to use as much memory as this value specifies before it starts to write data into temporary files. Also, several running sessions could be doing such operations concurrently. Therefore, the total memory used could be many times the value of work_mem; it is necessary to keep this fact in mind when choosing the value. Sort operations are used for ORDER BY, DISTINCT, and merge joins. Hash tables are used in hash joins, hash-based aggregation, and hash-based processing of IN subqueries. recomendations: Configuring work_mem in Postgres: https://www.pgmustard.com/blog/work-mem - "Understaning postgresql.conf: WORK_MEM": https://www.depesz.com/2011/07/03/understanding-postgresql-conf-work_mem/ + The surprising logic of the Postgres work_mem setting: https://pganalyze.com/blog/5mins-postgres-work-mem-tuning + Temporary files in PostgreSQL - identify and fix: https://klouddb.io/temporary-files-in-postgresql-steps-to-identify-and-fix-temp-file-issues/ + PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components maintenance_work_mem: - abstract: This parameter defines how much a maintenance operation (ALTER TABLE, VACUUM, REINDEX, AutoVACUUM worker, etc) buffer can use. + abstract: | + Memory for maintenance operations: **VACUUM**, **CREATE INDEX**, **ALTER TABLE**, and autovacuum workers. + + Can be set higher than work_mem since fewer concurrent maintenance operations run. + + **Important**: Total usage = `maintenance_work_mem × autovacuum_max_workers`. Consider using `autovacuum_work_mem` separately. + + **PostgreSQL ≤16**: 1GB limit (~179M dead tuples per pass). **PostgreSQL 17+**: No limit (uses radix trees). recomendations: Adjusting maintenance_work_mem: https://www.cybertec-postgresql.com/en/adjusting-maintenance_work_mem/ How Much maintenance_work_mem Do I Need?: http://rhaas.blogspot.com/2019/01/how-much-maintenanceworkmem-do-i-need.html Don't give Postgres too much memory (even on busy systems): https://vondra.me/posts/dont-give-postgres-too-much-memory-even-on-busy-systems/ + Understanding autovacuum in Amazon RDS for PostgreSQL: https://aws.amazon.com/blogs/database/understanding-autovacuum-in-amazon-rds-for-postgresql-environments/ + PostgreSQL Performance Tuning Best Practices 2025: https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices Optimize PostgreSQL Server Performance Through Configuration: https://blog.crunchydata.com/blog/optimize-postgresql-server-performance Let's get back to basics - PostgreSQL Memory Components: https://www.postgresql.fastware.com/blog/back-to-basics-with-postgresql-memory-components checkpoint_related: min_wal_size: abstract: | - This parameter defines the minimum size of the pg_wal directory. pg_wal directory contains the WAL files. - Older versions refer to it as the pg_xlog directory. + Minimum size of pg_wal directory (pg_xlog in versions <10). WAL files are **recycled** rather than removed when below this threshold. + + Useful to handle **WAL spikes** during batch jobs or high write periods. recomendations: - "Configuration changes in 9.5: transaction log size": http://www.databasesoup.com/2016/01/configuration-changes-in-95-transaction.html + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html + "Checkpoint distance and amount of WAL": https://www.cybertec-postgresql.com/en/checkpoint-distance-and-amount-of-wal/ + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "Tuning Your Postgres Database for High Write Loads": https://www.crunchydata.com/blog/tuning-your-postgres-database-for-high-write-loads max_wal_size: abstract: | - This parameter defines the maximun size of the pg_wal directory. pg_wal directory contains the WAL files. - Older versions refer to it as the pg_xlog directory. + Triggers checkpoint when pg_wal exceeds this size. Larger values reduce checkpoint frequency but increase crash recovery time. + + **Recommendation**: Set to hold **1 hour of WAL**. Write-heavy systems may need significantly more. + + Monitor `pg_stat_bgwriter` to ensure most checkpoints are **timed** (not requested). + recomendations: + "Basics of Tuning Checkpoints": https://www.enterprisedb.com/blog/basics-tuning-checkpoints + "Tuning max_wal_size in PostgreSQL": https://www.enterprisedb.com/blog/tuning-maxwalsize-postgresql + "Checkpoint distance and amount of WAL": https://www.cybertec-postgresql.com/en/checkpoint-distance-and-amount-of-wal/ + "Tuning Your Postgres Database for High Write Loads": https://www.crunchydata.com/blog/tuning-your-postgres-database-for-high-write-loads + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices checkpoint_completion_target: - abstract: This parameter defines a percentual of checkpoint_timeout as a target to write the CHECKPOINT data on the disk. + abstract: | + Spreads checkpoint writes over this fraction of `checkpoint_timeout` to reduce I/O spikes. + + **Example**: `checkpoint_timeout = 5min` and `checkpoint_completion_target = 0.9` + → Checkpoint spreads writes over **270 seconds (4min 30s)**, leaving 30s buffer for sync overhead. + + Values higher than 0.9 risk checkpoint delays. Monitor via `pg_stat_bgwriter`. recomendations: - "Understaning postgresql.conf: CHECKPOINT_COMPLETION_TARGET": https://www.depesz.com/2010/11/03/checkpoint_completion_target/ + "Basics of Tuning Checkpoints": https://www.enterprisedb.com/blog/basics-tuning-checkpoints + "Measuring PostgreSQL Checkpoint Statistics": https://www.enterprisedb.com/blog/measuring-postgresql-checkpoint-statistics + "Checkpoints, Background Writer and monitoring": https://stormatics.tech/blogs/checkpoints-and-background-writer-and-how-to-monitor-it + "Deep dive into postgres stats: pg_stat_bgwriter": https://dataegret.com/2017/03/deep-dive-into-postgres-stats-pg_stat_bgwriter/ + "Understanding PostgreSQL Checkpoints": https://prateekcodes.com/understanding-postgres-checkpoints/ + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/wal-configuration.html wal_buffers: - abstract: This parameter defines a buffer to store WAL changes before write it in the WAL file. + abstract: | + Buffer for WAL data before flushing to disk. Default **-1** auto-tunes to 3% of `shared_buffers` (min 64kB, max 16MB). + + For **write-heavy workloads**, manual tuning can significantly improve performance: + - **DW profile**: 64MB (always, regardless of RAM) + - **OLTP profile**: 32MB (when shared_buffers > 8GB) + - **Others**: Auto-tune (default) + + WAL flushes at every commit, so extremely large values provide diminishing returns. recomendations: - Chapter 9 - Write Ahead Logging — WAL: http://www.interdb.jp/pg/pgsql09.html + "Tuning shared_buffers and wal_buffers": https://www.enterprisedb.com/blog/tuning-sharedbuffers-and-walbuffers + "Understanding shared_buffers, work_mem, and wal_buffers": https://www.postgresql.fastware.com/pzone/2024-06-understanding-shared-buffers-work-mem-and-wal-buffers-in-postgresql + "PostgreSQL WAL Configuration": https://www.postgresql.org/docs/current/runtime-config-wal.html + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "Tuning PostgreSQL for Write Heavy Workloads": https://www.cloudraft.io/blog/tuning-postgresql-for-write-heavy-workloads network_related: listen_addresses: - abstract: This parameter defines a network address to bind to. + abstract: | + Network interfaces PostgreSQL listens on for connections. + + **Security**: Default is `localhost` (local-only). Never use `*` or `0.0.0.0` exposed to internet. + + Use specific IPs with `pg_hba.conf` rules, or SSH tunnels/VPN for remote access. + recomendations: + "PostgreSQL Connections and Authentication": https://www.postgresql.org/docs/current/runtime-config-connection.html + "PostgreSQL Security: 12 rules for database hardening": https://www.cybertec-postgresql.com/en/postgresql-security-things-to-avoid-in-real-life/ + "Postgres security best practices": https://www.bytebase.com/reference/postgres/how-to/postgres-security-best-practices/ max_connections: - abstract: This parameter defines a max connections allowed. + abstract: | + Maximum concurrent database connections. Each connection consumes memory (~10MB + work_mem per operation). + + **Best practice**: Use **connection pooling** (PgBouncer, pgpool) instead of high max_connections. + + With pooling: 20-50 connections. Without pooling: 100-200 (but review memory impact). + + Formula: `(RAM - shared_buffers) / (work_mem × avg_operations_per_query)` for rough estimate. recomendations: - Tuning max_connections in PostgreSQL: https://www.cybertec-postgresql.com/en/tuning-max_connections-in-postgresql/ + "Tuning max_connections in PostgreSQL": https://www.cybertec-postgresql.com/en/tuning-max_connections-in-postgresql/ + "Why you should use Connection Pooling": https://www.enterprisedb.com/postgres-tutorials/why-you-should-use-connection-pooling-when-setting-maxconnections-postgres + "PgBouncer for PostgreSQL: How Connection Pooling Solves Enterprise Slowdowns": https://www.percona.com/blog/pgbouncer-for-postgresql-how-connection-pooling-solves-enterprise-slowdowns/ + "Complete Guide to Fixing PostgreSQL Performance with PgBouncer": https://opstree.com/blog/2025/10/07/postgresql-performance-with-pgbouncer/ + "How To Optimize PostgreSQL For High Traffic": https://www.nilebits.com/blog/2025/06/postgresql-high-connections/ storage_type: random_page_cost: - abstract: Sets the planner's estimate of the cost of a non-sequentially-fetched disk page. + abstract: | + Query planner's cost estimate for random disk access relative to sequential reads (`seq_page_cost = 1.0`). + + Lower values favor index scans, higher values favor sequential scans. Sequential scans become more efficient when queries return ~5-10% or more of table rows, common in analytical/DW workloads. + + **Debate (2025)**: Some experts advocate keeping higher values (4.0) for **plan stability** across cache states, while others recommend lower values (1.1-2.0) for SSD to favor index scans. + + Test with `EXPLAIN ANALYZE` to verify query plan choices for your workload. recomendations: - How a single PostgreSQL config change improved slow query performance by 50x: https://amplitude.engineering/how-a-single-postgresql-config-change-improved-slow-query-performance-by-50x-85593b8991b0 + "How a single PostgreSQL config change improved slow query performance by 50x": https://amplitude.engineering/how-a-single-postgresql-config-change-improved-slow-query-performance-by-50x-85593b8991b0 + "Better PostgreSQL performance on SSDs": https://www.cybertec-postgresql.com/en/better-postgresql-performance-on-ssds/ + "PostgreSQL with modern storage: what about a lower random_page_cost?": https://dev.to/aws-heroes/postgresql-with-modern-storage-what-about-a-lower-randompagecost-5b7f + "Postgres Scan Types in EXPLAIN Plans": https://www.crunchydata.com/blog/postgres-scan-types-in-explain-plans + "Tuning Your PostgreSQL Server": https://wiki.postgresql.org/wiki/Tuning_Your_PostgreSQL_Server effective_io_concurrency: - abstract: Sets the number of concurrent disk I/O operations that PostgreSQL expects can be executed simultaneously. + abstract: | + Controls concurrent disk I/O operations for **bitmap heap scans** only. + + Bitmap scans are used when queries need to fetch moderate result sets (too many rows for index scans, too few for sequential scans) or when combining multiple indexes. They're more common in analytical workloads. + + PostgreSQL 18 changes the default from 1 to 16. Values above 200 show diminishing returns in benchmarks. recomendations: "PostgreSQL: effective_io_concurrency benchmarked": https://portavita.github.io/2019-07-19-PostgreSQL_effective_io_concurrency_benchmarked/ + "Bitmap Heap Scan - pganalyze": https://pganalyze.com/docs/explain/scan-nodes/bitmap-heap-scan + "PostgreSQL indexing: Index scan vs. Bitmap scan vs. Sequential scan (basics)": https://www.cybertec-postgresql.com/en/postgresql-indexing-index-scan-vs-bitmap-scan-vs-sequential-scan-basics/ io_method: - abstract: Controls the asynchronous I/O implementation used by PostgreSQL. Options are "worker" (dedicated I/O worker processes), "io_uring" (Linux io_uring interface), and "sync" (traditional synchronous I/O). + abstract: | + Selects the async I/O implementation for read operations (PostgreSQL 18+). + + **worker** (default): Uses dedicated background processes. Best for most workloads, especially high-bandwidth sequential scans. Recommended as default. + + **io_uring** (Linux only): Kernel-level async I/O. Only switch after extensive testing proves benefit for your specific low-latency random-read patterns. Can hit file descriptor limits with high max_connections. + + **sync**: Traditional synchronous I/O. Slower than async methods - avoid unless debugging or testing. + + Note: Only affects reads. Writes, checkpoints, and WAL still use sync I/O. recomendations: "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ "Waiting for Postgres 18: Accelerating Disk Reads with Asynchronous I/O - pganalyze": https://pganalyze.com/blog/postgres-18-async-io @@ -74,21 +190,47 @@ categories: "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io "PostgreSQL 18: The AIO Revolution - dev.to": https://dev.to/mattleads/postgresql-18-the-aio-revolution-uuidv7-and-the-path-to-unprecedented-performance-415m io_workers: - abstract: Number of background I/O worker processes used when io_method is set to "worker". Determines how many concurrent I/O operations can be performed asynchronously. + abstract: | + Background worker processes for async I/O when `io_method = worker`. + + Default of 3 is too low for modern multi-core systems. Recommendation: **10-40% of CPU cores** depending on workload. + + Higher values benefit workloads with: + - Sequential scans (DW/analytical queries) + - High I/O latency (HDD vs SSD) + - High concurrent read operations + + Monitor `pg_stat_activity` for I/O wait events. If workers are saturated, increase this value. recomendations: "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ "Waiting for Postgres 18: Accelerating Disk Reads with Asynchronous I/O - pganalyze": https://pganalyze.com/blog/postgres-18-async-io "PostgreSQL 18: Better I/O performance with AIO - Cybertec": https://www.cybertec-postgresql.com/en/postgresql-18-better-i-o-performance-with-aio/ "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io maintenance_io_concurrency: - abstract: Sets the number of concurrent I/O operations that PostgreSQL expects can be executed simultaneously during maintenance operations such as VACUUM and CREATE INDEX. + abstract: | + Concurrent I/O operations for maintenance: **VACUUM**, **CREATE INDEX**, **ANALYZE**. + + Similar to `effective_io_concurrency` but for maintenance operations. PostgreSQL 18 changed default from 10 to 16. + + Typically set to match `effective_io_concurrency` based on storage capabilities. + recomendations: + "Quick Benchmark: ANALYZE vs. maintenance_io_concurrency": https://www.credativ.de/en/blog/postgresql-en/quick-benchmark-analyze-vs-maintenance_io_concurrency/ + "PostgreSQL 18 Asynchronous I/O: A Complete Guide": https://betterstack.com/community/guides/databases/postgresql-asynchronous-io/ + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "PostgreSQL Database Tuning for OLAP vs. OLTP Workloads": https://reintech.io/blog/postgresql-database-tuning-olap-vs-oltp io_combine_limit: - abstract: Maximum size of adjacent blocks combined into one I/O request. + abstract: | + Maximum adjacent blocks combined into a single I/O request (PostgreSQL 18+). + + Larger values benefit sequential scans and bitmap heap scans by reducing I/O operations. Limited by `io_max_combine_limit` server setting. recomendations: - The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist (With Examples): https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples - PostgreSQL 18 Asynchronous Disk I/O - Deep Dive Into Implementation: https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ + "The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist": https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples + "PostgreSQL 18 Asynchronous Disk I/O - Deep Dive": https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ io_max_combine_limit: - abstract: Server-wide limit that clamps io_combine_limit, controlling the largest I/O size in operations that combine I/O. + abstract: | + Server-wide limit that clamps `io_combine_limit` (PostgreSQL 18+). + + Set at server startup. Controls the maximum I/O size for operations that combine I/O. Increase for data warehouses with large sequential scans. recomendations: "Allow io_combine_limit up to 1MB": https://www.postgresql.org/message-id/CA+hUKGKd=U1zSFYNjwBBrXV3NsqR2U8dCUrCBVeB0DQ8Vb8Dwg@mail.gmail.com "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ @@ -98,28 +240,60 @@ categories: The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist (With Examples): https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples PostgreSQL 18 Asynchronous Disk I/O - Deep Dive Into Implementation: https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ io_max_concurrency: - abstract: Maximum number of concurrent I/O requests that can be in flight at once. + abstract: | + Hard limit on concurrent I/O operations per backend process (PostgreSQL 18+). + + Controls read-ahead with async I/O. Formula: `max read-ahead = effective_io_concurrency × io_combine_limit` + + Higher values benefit high-latency storage (cloud/EBS) with high IOPS. Watch memory usage - high concurrency increases memory pressure. recomendations: - "PostgreSQL 18 Beta 1 io_max_concurrency": https://www.postgresql.org/message-id/9673.1747250758@sss.pgh.pa.us + "PostgreSQL 18 Asynchronous I/O: A Complete Guide": https://betterstack.com/community/guides/databases/postgresql-asynchronous-io/ "Tuning AIO in PostgreSQL 18 - Tomas Vondra": https://vondra.me/posts/tuning-aio-in-postgresql-18/ "Waiting for Postgres 18: Accelerating Disk Reads with Asynchronous I/O - pganalyze": https://pganalyze.com/blog/postgres-18-async-io "PostgreSQL 18: Better I/O performance with AIO - Cybertec": https://www.cybertec-postgresql.com/en/postgresql-18-better-i-o-performance-with-aio/ "PostgreSQL 18 Asynchronous I/O - Neon": https://neon.com/postgresql/postgresql-18/asynchronous-io - The Ultimate PostgreSQL 18 Asynchronous I/O Tuning Checklist (With Examples): https://www.cybrosys.com/research-and-development/postgres/the-ultimate-postgresql-18-asynchronous-io-tuning-checklist-with-examples - PostgreSQL 18 Asynchronous Disk I/O - Deep Dive Into Implementation: https://www.credativ.de/en/blog/postgresql-en/postgresql-18-asynchronous-disk-i-o-deep-dive-into-implementation/ file_copy_method: - abstract: Selects the method used for copying files during CREATE DATABASE and ALTER DATABASE SET TABLESPACE operations. + abstract: | + Method for copying files during **CREATE DATABASE** and **ALTER DATABASE SET TABLESPACE** (PostgreSQL 18+). + + Recommendation: Use **clone** if your filesystem supports it - dramatically faster (200-600ms for 100s of GB) and initially consumes zero extra disk space. + recomendations: + "Instant database clones with PostgreSQL 18": https://boringsql.com/posts/instant-database-clones/ + "Instant Per-Branch Databases with PostgreSQL 18's clone": https://medium.com/axial-engineering/instant-per-branch-databases-with-postgresql-18s-clone-file-copy-and-copy-on-write-filesystems-1b1930bddbaa + "Git for Data: Instant PostgreSQL Database Cloning": https://vonng.com/en/pg/pg-clone/ worker_related: max_worker_processes: - abstract: Sets the maximum number of background processes that the system can support. + abstract: | + Maximum background worker processes (autovacuum, parallel query, replication, extensions). + + Pool from which all background workers are drawn. Must accommodate: + - Parallel query workers (`max_parallel_workers`) + - Logical replication workers + - Extensions (pg_stat_statements, etc.) + + Recommendation: Set to **CPU core count** or at least **25% of vCPUs**. Requires restart. recomendations: - "Comprehensive guide on how to tune database parameters and configuration in PostgreSQL": https://www.enterprisedb.com/postgres-tutorials/comprehensive-guide-how-tune-database-parameters-and-configuration-postgresql + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices + "PostgreSQL Performance Tuning: Key Parameters": https://www.tigerdata.com/learn/postgresql-performance-tuning-key-parameters + "Parallel Queries in Postgres - Crunchy Data": https://www.crunchydata.com/blog/parallel-queries-in-postgres max_parallel_workers_per_gather: - abstract: Sets the maximum number of parallel processes per executor node. + abstract: | + Maximum parallel workers per query executor node. + + Each worker consumes resources individually (work_mem, CPU, I/O). A query with 4 workers uses 5x resources (1 leader + 4 workers). recomendations: - "Comprehensive guide on how to tune database parameters and configuration in PostgreSQL": https://www.enterprisedb.com/postgres-tutorials/comprehensive-guide-how-tune-database-parameters-and-configuration-postgresql + "Increasing max parallel workers per gather in Postgres": https://www.pgmustard.com/blog/max-parallel-workers-per-gather + "Postgres Tuning & Performance for Analytics Data": https://www.crunchydata.com/blog/postgres-tuning-and-performance-for-analytics-data + "Parallel Queries in Postgres": https://www.crunchydata.com/blog/parallel-queries-in-postgres + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices max_parallel_workers: - abstract: Sets the maximum number of parallel workers that can be active at one time + abstract: | + Maximum parallel workers active system-wide across all queries. + + Limits total parallel workers from the `max_worker_processes` pool. Cannot exceed `max_worker_processes`. + + Recommendation: Set equal to **CPU core count** or `max_worker_processes`. recomendations: - "Comprehensive guide on how to tune database parameters and configuration in PostgreSQL": https://www.enterprisedb.com/postgres-tutorials/comprehensive-guide-how-tune-database-parameters-and-configuration-postgresql + "Parallel Queries in Postgres": https://www.crunchydata.com/blog/parallel-queries-in-postgres + "PostgreSQL Performance Tuning Best Practices 2025": https://www.mydbops.com/blog/postgresql-parameter-tuning-best-practices