diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..45e648e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,59 @@ +name: CI + +on: + push: + branches: + - main + - fix/** + - codex/** + pull_request: + workflow_dispatch: + +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - name: Format + run: cargo fmt --check + + - name: Lint + run: cargo clippy --all-targets --all-features -- -D warnings + + - name: Unit Tests + run: cargo test + + live-postgres: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + postgres_version: ["14", "18"] + env: + POSTGREAT_TEST_PG_VERSION: ${{ matrix.postgres_version }} + steps: + - uses: actions/checkout@v4 + + - uses: dtolnay/rust-toolchain@stable + + - uses: Swatinem/rust-cache@v2 + + - name: Live Analyze Test + run: cargo test --test it_analyze -- --ignored --test-threads=1 + + - name: Live Workload Test + run: cargo test --test it_workload -- --ignored --test-threads=1 + + - name: Live Workload Unavailable Test + run: cargo test --test it_workload_unavailable -- --ignored --test-threads=1 + + - name: Live Workload Visibility Test + run: cargo test --test it_workload_visibility -- --ignored --test-threads=1 + + - name: Live Workload Deallocation Test + run: cargo test --test it_workload_dealloc -- --ignored --test-threads=1 diff --git a/Cargo.lock b/Cargo.lock index 616aaf2..fd3881c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.6.21" @@ -73,6 +82,32 @@ version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" +[[package]] +name = "assert_cmd" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514" +dependencies = [ + "anstyle", + "bstr", + "libc", + "predicates", + "predicates-core", + "predicates-tree", + "wait-timeout", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "atoi" version = "2.0.0" @@ -82,12 +117,24 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -102,9 +149,15 @@ checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" [[package]] name = "bitflags" -version = "2.10.0" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" +checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" dependencies = [ "serde_core", ] @@ -118,6 +171,73 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "home", + "http", + "http-body-util", + "hyper", + "hyper-named-pipe", + "hyper-rustls", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + +[[package]] +name = "bstr" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" +dependencies = [ + "memchr", + "regex-automata", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d20789868f4b01b2f2caec9f5c4e0213b41e3e5702a50157d699ae31ced2fcb" + [[package]] name = "byteorder" version = "1.5.0" @@ -146,6 +266,18 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "serde", + "windows-link", +] + [[package]] name = "clap" version = "4.5.51" @@ -201,12 +333,40 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "windows-sys 0.59.0", +] + [[package]] name = "const-oid" version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -256,6 +416,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "der" version = "0.7.10" @@ -267,6 +462,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +dependencies = [ + "powerfmt", + "serde_core", +] + +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.10.7" @@ -290,12 +501,29 @@ dependencies = [ "syn", ] +[[package]] +name = "docker_credential" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d89dfcba45b4afad7450a99b39e751590463e45c04728cf555d36bb66940de8" +dependencies = [ + "base64 0.21.7", + "serde", + "serde_json", +] + [[package]] name = "dotenvy" version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "dyn-clone" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" + [[package]] name = "either" version = "1.15.0" @@ -305,12 +533,28 @@ dependencies = [ "serde", ] +[[package]] +name = "encode_unicode" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" + [[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "etcetera" version = "0.8.0" @@ -322,6 +566,17 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "etcetera" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26c7b13d0780cb82722fd59f6f57f925e143427e4a75313a6c77243bf5326ae6" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.59.0", +] + [[package]] name = "event-listener" version = "5.4.1" @@ -333,12 +588,38 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +[[package]] +name = "float-cmp" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09cf3155332e944990140d967ff5eceb70df778b34f77d8075db46e4704e6d8" +dependencies = [ + "num-traits", +] + [[package]] name = "flume" version = "0.11.1" @@ -350,6 +631,12 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" @@ -492,12 +779,31 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasip2", + "wasip3", +] + [[package]] name = "glob" version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.15.5" @@ -564,78 +870,235 @@ dependencies = [ ] [[package]] -name = "icu_collections" -version = "2.1.1" +name = "http" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" dependencies = [ - "displaydoc", - "potential_utf", - "yoke", - "zerofrom", - "zerovec", + "bytes", + "itoa", ] [[package]] -name = "icu_locale_core" -version = "2.1.1" +name = "http-body" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", + "bytes", + "http", ] [[package]] -name = "icu_normalizer" -version = "2.1.1" +name = "http-body-util" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", ] [[package]] -name = "icu_normalizer_data" -version = "2.1.1" +name = "httparse" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] -name = "icu_properties" -version = "2.1.1" +name = "httpdate" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", ] [[package]] -name = "icu_properties_data" -version = "2.1.1" +name = "hyper-named-pipe" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] [[package]] -name = "icu_provider" -version = "2.1.1" +name = "hyper-rustls" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "displaydoc", + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "libc", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" +dependencies = [ + "displaydoc", + "potential_utf", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" + +[[package]] +name = "icu_properties" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" + +[[package]] +name = "icu_provider" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" +dependencies = [ + "displaydoc", "icu_locale_core", "writeable", "yoke", @@ -644,6 +1107,18 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -665,6 +1140,17 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.12.0" @@ -673,6 +1159,21 @@ checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", "hashbrown 0.16.0", + "serde", + "serde_core", +] + +[[package]] +name = "insta" +version = "1.46.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e82db8c87c7f1ccecb34ce0c24399b8a73081427f3c7c50a5d597925356115e4" +dependencies = [ + "console", + "once_cell", + "serde", + "similar", + "tempfile", ] [[package]] @@ -696,6 +1197,16 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "js-sys" +version = "0.3.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49715b7073f385ba4bc528e5747d02e66cb39c6146efb66b781f131f0fb399c" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -705,11 +1216,17 @@ dependencies = [ "spin", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" -version = "0.2.177" +version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libm" @@ -723,9 +1240,9 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", - "redox_syscall", + "redox_syscall 0.5.18", ] [[package]] @@ -738,6 +1255,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.1" @@ -795,6 +1318,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -820,6 +1349,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf97ec579c3c42f953ef76dbf8d55ac91fb219dde70e49aa4a6b7d74e9919050" + [[package]] name = "num-integer" version = "0.1.46" @@ -862,6 +1397,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "parking" version = "2.2.1" @@ -886,11 +1427,36 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link", ] +[[package]] +name = "parse-display" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914a1c2265c98e2446911282c6ac86d8524f495792c38c5bd884f80499c7538a" +dependencies = [ + "parse-display-derive", + "regex", + "regex-syntax", +] + +[[package]] +name = "parse-display-derive" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae7800a4c974efd12df917266338e79a7a74415173caf7e70aa0a0707345281" +dependencies = [ + "proc-macro2", + "quote", + "regex", + "regex-syntax", + "structmeta", + "syn", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -950,8 +1516,11 @@ name = "postgreat" version = "0.1.0" dependencies = [ "anyhow", + "assert_cmd", "clap", + "insta", "itertools", + "predicates", "rstest", "serde", "serde_json", @@ -959,6 +1528,8 @@ dependencies = [ "snafu", "sqlparser", "sqlx", + "testcontainers 0.23.3", + "testcontainers-modules", "tokio", "tracing", "tracing-subscriber", @@ -973,6 +1544,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -982,6 +1559,46 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "predicates" +version = "3.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ada8f2932f28a27ee7b70dd6c1c39ea0675c55a36879ab92f3a715eaa1e63cfe" +dependencies = [ + "anstyle", + "difflib", + "float-cmp", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad38746f3166b4031b1a0d39ad9f954dd291e7854fcc0eed52ee41a0b50d144" + +[[package]] +name = "predicates-tree" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0de1b847b39c8131db0467e9df1ff60e6d0562ab8e9a16e568ad0fdb372e2f2" +dependencies = [ + "predicates-core", + "termtree", +] + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "3.4.0" @@ -1009,6 +1626,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.8.5" @@ -1036,7 +1659,16 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom", + "getrandom 0.2.16", +] + +[[package]] +name = "redox_syscall" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" +dependencies = [ + "bitflags 1.3.2", ] [[package]] @@ -1045,7 +1677,27 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.0", +] + +[[package]] +name = "ref-cast" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1091,7 +1743,7 @@ checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", "cfg-if", - "getrandom", + "getrandom 0.2.16", "libc", "untrusted", "windows-sys 0.52.0", @@ -1156,6 +1808,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.11.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + [[package]] name = "rustls" version = "0.23.35" @@ -1170,6 +1835,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "rustls-pki-types" version = "1.13.0" @@ -1190,18 +1876,80 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "ryu" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "schemars" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd191f9397d57d581cddd31014772520aa448f65ef991055d7f61582c65165f" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + +[[package]] +name = "schemars" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2b42f36aa1cd011945615b92222f6bf73c599a102a300334cd7f8dbeec726cc" +dependencies = [ + "dyn-clone", + "ref-cast", + "serde", + "serde_json", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags 2.11.0", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -1251,6 +1999,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_repr" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -1263,13 +2022,44 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "381b283ce7bc6b476d903296fb59d0d36633652b633b27f64db4fb46dcbfc3b9" +dependencies = [ + "base64 0.22.1", + "chrono", + "hex", + "indexmap 1.9.3", + "indexmap 2.12.0", + "schemars 0.9.0", + "schemars 1.2.1", + "serde_core", + "serde_json", + "serde_with_macros", + "time", +] + +[[package]] +name = "serde_with_macros" +version = "3.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6d4e30573c8cb306ed6ab1dca8423eec9a463ea0e155f45399455e0368b27e0" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "serde_yaml" version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap", + "indexmap 2.12.0", "itoa", "ryu", "serde", @@ -1332,6 +2122,12 @@ dependencies = [ "rand_core", ] +[[package]] +name = "similar" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" + [[package]] name = "slab" version = "0.4.11" @@ -1425,7 +2221,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "crc", "crossbeam-queue", @@ -1437,7 +2233,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink", - "indexmap", + "indexmap 2.12.0", "log", "memchr", "once_cell", @@ -1500,8 +2296,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", - "bitflags", + "base64 0.22.1", + "bitflags 2.11.0", "byteorder", "bytes", "crc", @@ -1542,12 +2338,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", - "bitflags", + "base64 0.22.1", + "bitflags 2.11.0", "byteorder", "crc", "dotenvy", - "etcetera", + "etcetera 0.8.0", "futures-channel", "futures-core", "futures-util", @@ -1619,6 +2415,29 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "structmeta" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" +dependencies = [ + "proc-macro2", + "quote", + "structmeta-derive", + "syn", +] + +[[package]] +name = "structmeta-derive" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1629,22 +2448,108 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" name = "syn" version = "2.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +checksum = "2f17c7e013e88258aa9543dcbe81aca68a667a9ac37cd69c9fbc07858bfe0e2f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +dependencies = [ + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + +[[package]] +name = "termtree" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f50febec83f5ee1df3015341d8bd429f2d1cc62bcba7ea2076759d315084683" + +[[package]] +name = "testcontainers" +version = "0.23.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59a4f01f39bb10fc2a5ab23eb0d888b1e2bb168c157f61a1b98e6c501c639c74" +dependencies = [ + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera 0.8.0", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", +] + +[[package]] +name = "testcontainers" +version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23bb7577dca13ad86a78e8271ef5d322f37229ec83b8d98da6d996c588a1ddb1" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "async-trait", + "bollard", + "bollard-stubs", + "bytes", + "docker_credential", + "either", + "etcetera 0.10.0", + "futures", + "log", + "memchr", + "parse-display", + "pin-project-lite", + "serde", + "serde_json", + "serde_with", + "thiserror", + "tokio", + "tokio-stream", + "tokio-tar", + "tokio-util", + "url", ] [[package]] -name = "synstructure" -version = "0.13.2" +name = "testcontainers-modules" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "eac95cde96549fc19c6bf19ef34cc42bd56e264c1cb97e700e21555be0ecf9e2" dependencies = [ - "proc-macro2", - "quote", - "syn", + "testcontainers 0.24.0", ] [[package]] @@ -1676,6 +2581,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "time" +version = "0.3.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" + +[[package]] +name = "time-macros" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" +dependencies = [ + "num-conv", + "time-core", +] + [[package]] name = "tinystr" version = "0.8.2" @@ -1729,6 +2665,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.17" @@ -1740,6 +2686,34 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-tar" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5714c010ca3e5c27114c1cdeb9d14641ace49874aa5626d7149e47aedace75" +dependencies = [ + "filetime", + "futures-core", + "libc", + "redox_syscall 0.3.5", + "tokio", + "tokio-stream", + "xattr", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml_datetime" version = "0.7.3" @@ -1755,7 +2729,7 @@ version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap", + "indexmap 2.12.0", "toml_datetime", "toml_parser", "winnow", @@ -1770,6 +2744,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.41" @@ -1832,6 +2812,12 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.19.0" @@ -1865,6 +2851,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + [[package]] name = "unsafe-libyaml" version = "0.2.11" @@ -1919,18 +2911,133 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasite" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" +[[package]] +name = "wasm-bindgen" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6532f9a5c1ece3798cb1c2cfdba640b9b3ba884f5db45973a6f442510a87d38e" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18a2d50fcf105fb33bb15f00e7a77b772945a2ee45dcf454961fd843e74c18e6" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03ce4caeaac547cdf713d280eda22a730824dd11e6b8c3ca9e42247b25c631e3" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75a326b8c223ee17883a4251907455a2431acc2791c98c26279376490c378c16" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap 2.12.0", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.11.0", + "hashbrown 0.15.5", + "indexmap 2.12.0", + "semver", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -1959,12 +3066,87 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -1983,6 +3165,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.60.2" @@ -2196,12 +3387,110 @@ dependencies = [ "memchr", ] +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap 2.12.0", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.11.0", + "indexmap 2.12.0", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap 2.12.0", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] + [[package]] name = "writeable" version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yoke" version = "0.8.1" diff --git a/Cargo.toml b/Cargo.toml index d8d0411..11c60d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,11 @@ sqlparser = "0.45" [dev-dependencies] rstest = "0.23" +assert_cmd = "2.0" +insta = { version = "1.42", features = ["json"] } +predicates = "3.1" +testcontainers = { version = "0.23", features = ["blocking"] } +testcontainers-modules = { version = "0.12", features = ["blocking", "postgres"] } [[bin]] name = "postgreat" diff --git a/PROGRESS.md b/PROGRESS.md index 98eaeff..b88d9bb 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -6,6 +6,18 @@ PostGreat is a Rust-based PostgreSQL configuration analyzer that provides eviden ## Work Log +### 2026-03-05 - Live PostgreSQL integration test harness +- Added a Docker-backed integration test layer using `testcontainers`, with PostgreSQL 14 and 18 as the supported live-test matrix. +- Reused the existing SQL fixtures under `tests/_data/` and added workload-specific setup for `pg_stat_statements`, seeded roles, live workload generation, and deallocation scenarios. +- Added ignored end-to-end tests that invoke the `postgreat` binary directly, assert JSON output, and snapshot stable scenario summaries for analyze, workload, limited visibility, unavailable `pg_stat_statements`, and statement eviction cases. +- Added a GitHub Actions workflow that runs the fast Rust checks plus the live PostgreSQL matrix, and documented the local commands in `README.md`. + +### 2026-03-05 - Workload reporting hardening +- Added workload metadata and coverage accounting so the `workload` report explains its scope (`pg_stat_statements` since reset), entry deallocations, query-text visibility, parse coverage, and candidate suppression counts. +- Enriched slow-query output with cumulative-share, cache-hit, temp-spill, and optional WAL-per-call context, while keeping the existing CLI interface stable. +- Expanded heuristic index candidates with structured evidence, confidence, and notes for ambiguous schema resolution, ignored partial/expression/invalid/non-B-tree indexes, and correlated table/index-health findings. +- Added reporter regression tests for Markdown/Text/JSON workload output and documented the workload-analysis review checklist in `README.md`. + ### 2026-02-03 - Workload analysis command - Added `workload` subcommand to analyze slow queries using `pg_stat_statements`, including heuristic index candidates derived from SQL parsing. - Implemented SQL parsing (WHERE/JOIN/ORDER BY) via `sqlparser` and reported parse failures and warnings. diff --git a/README.md b/README.md index 955b15d..3e655ed 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,16 @@ postgreat analyze --compute "8vCPU-64GB" ### Analyze Workload (Slow Queries & Index Candidates) -Requires `pg_stat_statements` to be installed and enabled on the target database. +Requires `pg_stat_statements` to be installed and usable on the target database. If the extension +exists but PostgreSQL was not restarted with `shared_preload_libraries = 'pg_stat_statements'`, +PostGreat will return a warning-only workload result instead of failing the command. +The current workload report uses cumulative `pg_stat_statements` counters, so it does not support +an exact historical lookback window like "last 1 hour". Results reflect activity since the +statement's stats were first collected or `pg_stat_statements` was last reset; true time-windowed +reporting requires external snapshots or a bucketed extension such as `pg_stat_monitor`. +The report now includes workload metadata and coverage notes so you can see the effective scope +(`pg_stat_statements` since reset), entry evictions, query-text visibility, parse coverage, and why +an index candidate was emitted or suppressed. ```bash postgreat workload \ @@ -259,10 +268,40 @@ src/ ### Running Tests +Fast test suite: + ```bash cargo test ``` +Live PostgreSQL integration tests: +- Require Docker and are ignored by default. +- Start a real PostgreSQL instance with `testcontainers`, seed it from `tests/_data/`, and invoke the `postgreat` binary end-to-end. +- Cover five scenarios: + - `it_analyze`: seeded `analyze --format json` run with table/index-health findings + - `it_workload`: happy-path `workload --format json` run with `pg_stat_statements` + - `it_workload_unavailable`: extension missing and installed-but-not-preloaded behavior + - `it_workload_visibility`: reduced query-text visibility without `pg_read_all_stats` + - `it_workload_dealloc`: `pg_stat_statements` entry eviction/deallocation warnings + +Run a single live test against PostgreSQL 18: + +```bash +POSTGREAT_TEST_PG_VERSION=18 cargo test --test it_workload -- --ignored --test-threads=1 +``` + +Run the full live suite against PostgreSQL 18: + +```bash +POSTGREAT_TEST_PG_VERSION=18 cargo test --test it_analyze -- --ignored --test-threads=1 +POSTGREAT_TEST_PG_VERSION=18 cargo test --test it_workload -- --ignored --test-threads=1 +POSTGREAT_TEST_PG_VERSION=18 cargo test --test it_workload_unavailable -- --ignored --test-threads=1 +POSTGREAT_TEST_PG_VERSION=18 cargo test --test it_workload_visibility -- --ignored --test-threads=1 +POSTGREAT_TEST_PG_VERSION=18 cargo test --test it_workload_dealloc -- --ignored --test-threads=1 +``` + +Swap `POSTGREAT_TEST_PG_VERSION=14` to run the same suite against PostgreSQL 14. + ### Code Formatting and Linting ```bash @@ -278,6 +317,13 @@ Contributions are welcome! Please ensure: 2. Add tests for new analysis logic 3. Update documentation as needed 4. Follow Rust naming and style conventions +5. For workload-analysis changes touching `pg_stat_statements`, SQL parsing, or index coverage: + - confirm cumulative vs real-time behavior explicitly + - document version-specific columns and fallback behavior + - check privilege-dependent visibility (`pg_read_all_stats`) + - verify index semantics for partial, expression, invalid, `INCLUDE`, and non-B-tree indexes + - cover ambiguous unqualified table names in tests +6. For PostgreSQL-semantic changes, include at least one official PostgreSQL doc link in the PR description and one regression test for the behavior being changed ## License diff --git a/src/analysis/query_parser.rs b/src/analysis/query_parser.rs index 5e567ae..c367aa9 100644 --- a/src/analysis/query_parser.rs +++ b/src/analysis/query_parser.rs @@ -23,8 +23,9 @@ impl TableRef { #[derive(Debug, Clone, Default)] pub struct TableColumnUsage { - pub filters: Vec, - pub joins: Vec, + pub equality_filters: Vec, + pub non_equality_filters: Vec, + pub equality_joins: Vec, pub orders: Vec, } @@ -43,8 +44,9 @@ struct PendingColumn { #[derive(Debug, Clone, Copy)] enum ColumnKind { - Filter, - Join, + EqualityFilter, + NonEqualityFilter, + EqualityJoin, Order, } @@ -240,14 +242,14 @@ impl QueryColumnCollector { self.pending.push(PendingColumn { relation: Some(table.clone()), name: normalize_ident(column), - kind: ColumnKind::Join, + kind: ColumnKind::EqualityJoin, }); } if let Some(table) = &right_table { self.pending.push(PendingColumn { relation: Some(table.clone()), name: normalize_ident(column), - kind: ColumnKind::Join, + kind: ColumnKind::EqualityJoin, }); } } @@ -268,15 +270,18 @@ impl QueryColumnCollector { self.collect_filter_expr(right); } BinaryOperator::Eq => { - self.push_column_if_applicable(left, ColumnKind::Filter); - self.push_column_if_applicable(right, ColumnKind::Filter); + self.collect_equality_predicate(left, right, ColumnKind::EqualityFilter); } - _ => {} + _ => self.collect_non_equality_predicate(left, right), }, - Expr::InList { expr, .. } => self.push_column_if_applicable(expr, ColumnKind::Filter), - Expr::Between { expr, .. } => self.push_column_if_applicable(expr, ColumnKind::Filter), + Expr::InList { expr, .. } => { + self.push_column_if_applicable(expr, ColumnKind::NonEqualityFilter) + } + Expr::Between { expr, .. } => { + self.push_column_if_applicable(expr, ColumnKind::NonEqualityFilter) + } Expr::IsNull(expr) | Expr::IsNotNull(expr) => { - self.push_column_if_applicable(expr, ColumnKind::Filter) + self.push_column_if_applicable(expr, ColumnKind::NonEqualityFilter) } Expr::Nested(expr) => self.collect_filter_expr(expr), _ => {} @@ -291,8 +296,7 @@ impl QueryColumnCollector { self.collect_join_expr(right); } BinaryOperator::Eq => { - self.push_column_if_applicable(left, ColumnKind::Join); - self.push_column_if_applicable(right, ColumnKind::Join); + self.collect_equality_predicate(left, right, ColumnKind::EqualityJoin); } _ => {} }, @@ -321,6 +325,32 @@ impl QueryColumnCollector { } } + fn collect_equality_predicate(&mut self, left: &Expr, right: &Expr, default_kind: ColumnKind) { + match (column_ref_from_expr(left), column_ref_from_expr(right)) { + (Some(left_column), Some(right_column)) => { + self.pending.push(PendingColumn { + relation: left_column.relation, + name: left_column.name, + kind: ColumnKind::EqualityJoin, + }); + self.pending.push(PendingColumn { + relation: right_column.relation, + name: right_column.name, + kind: ColumnKind::EqualityJoin, + }); + } + _ => { + self.push_column_if_applicable(left, default_kind); + self.push_column_if_applicable(right, default_kind); + } + } + } + + fn collect_non_equality_predicate(&mut self, left: &Expr, right: &Expr) { + self.push_column_if_applicable(left, ColumnKind::NonEqualityFilter); + self.push_column_if_applicable(right, ColumnKind::NonEqualityFilter); + } + fn merge_usage(&mut self, usage: QueryColumnUsage) { for table in usage.tables { if !self @@ -357,8 +387,13 @@ impl QueryColumnCollector { let Some(table_name) = table else { continue }; let entry = resolved_usage_by_table.entry(table_name).or_default(); match pending.kind { - ColumnKind::Filter => push_unique(&mut entry.filters, &pending.name), - ColumnKind::Join => push_unique(&mut entry.joins, &pending.name), + ColumnKind::EqualityFilter => { + push_unique(&mut entry.equality_filters, &pending.name) + } + ColumnKind::NonEqualityFilter => { + push_unique(&mut entry.non_equality_filters, &pending.name) + } + ColumnKind::EqualityJoin => push_unique(&mut entry.equality_joins, &pending.name), ColumnKind::Order => push_unique(&mut entry.orders, &pending.name), } } @@ -449,11 +484,14 @@ fn push_unique(values: &mut Vec, value: &str) { } fn merge_table_usage(target: &mut TableColumnUsage, source: &TableColumnUsage) { - for value in &source.filters { - push_unique(&mut target.filters, value); + for value in &source.equality_filters { + push_unique(&mut target.equality_filters, value); + } + for value in &source.non_equality_filters { + push_unique(&mut target.non_equality_filters, value); } - for value in &source.joins { - push_unique(&mut target.joins, value); + for value in &source.equality_joins { + push_unique(&mut target.equality_joins, value); } for value in &source.orders { push_unique(&mut target.orders, value); @@ -471,7 +509,10 @@ mod tests { assert_eq!(usage.tables.len(), 1); let table = usage.tables[0].full_name(); let table_usage = usage.usage_by_table.get(&table).expect("table usage"); - assert!(table_usage.filters.iter().any(|c| c == "customer_id")); + assert!(table_usage + .equality_filters + .iter() + .any(|c| c == "customer_id")); assert!(table_usage.orders.iter().any(|c| c == "created_at")); } @@ -486,8 +527,8 @@ mod tests { .find(|(k, _)| k.ends_with("orders")) .map(|(_, v)| v) .expect("orders"); - assert!(orders.joins.iter().any(|c| c == "customer_id")); - assert!(orders.filters.iter().any(|c| c == "status")); + assert!(orders.equality_joins.iter().any(|c| c == "customer_id")); + assert!(orders.equality_filters.iter().any(|c| c == "status")); } #[test] @@ -501,7 +542,7 @@ mod tests { .find(|(k, _)| k.ends_with("orders")) .map(|(_, v)| v) .expect("orders"); - assert!(!orders.filters.iter().any(|c| c == "status")); + assert!(!orders.equality_filters.iter().any(|c| c == "status")); } #[test] @@ -520,8 +561,8 @@ mod tests { .find(|(k, _)| k.ends_with("customers")) .map(|(_, v)| v) .expect("customers"); - assert!(orders.joins.iter().any(|c| c == "customer_id")); - assert!(customers.joins.iter().any(|c| c == "customer_id")); + assert!(orders.equality_joins.iter().any(|c| c == "customer_id")); + assert!(customers.equality_joins.iter().any(|c| c == "customer_id")); } #[test] @@ -534,7 +575,7 @@ mod tests { .find(|(k, _)| k.ends_with("orders")) .map(|(_, v)| v) .expect("orders"); - assert!(orders.filters.iter().any(|c| c == "customer_id")); + assert!(orders.equality_filters.iter().any(|c| c == "customer_id")); } #[test] @@ -553,9 +594,9 @@ mod tests { .find(|(k, _)| k.ends_with("customers")) .map(|(_, v)| v) .expect("customers"); - assert!(orders.filters.iter().any(|c| c == "customer_id")); - assert!(customers.filters.iter().any(|c| c == "id")); - assert!(customers.filters.iter().any(|c| c == "region")); + assert!(orders.equality_joins.iter().any(|c| c == "customer_id")); + assert!(customers.equality_joins.iter().any(|c| c == "id")); + assert!(customers.equality_filters.iter().any(|c| c == "region")); } #[test] @@ -568,7 +609,7 @@ mod tests { .find(|(k, _)| k.ends_with("orders")) .map(|(_, v)| v) .expect("orders"); - assert!(orders.filters.iter().any(|c| c == "customer_id")); + assert!(orders.equality_filters.iter().any(|c| c == "customer_id")); } #[test] @@ -588,9 +629,9 @@ mod tests { .find(|(k, _)| k.ends_with("customers")) .map(|(_, v)| v) .expect("customers"); - assert!(orders.filters.iter().any(|c| c == "customer_id")); - assert!(customers.filters.iter().any(|c| c == "id")); - assert!(customers.filters.iter().any(|c| c == "region")); + assert!(orders.equality_joins.iter().any(|c| c == "customer_id")); + assert!(customers.equality_joins.iter().any(|c| c == "id")); + assert!(customers.equality_filters.iter().any(|c| c == "region")); } #[test] @@ -609,8 +650,8 @@ mod tests { .find(|(k, _)| k.ends_with("customers")) .map(|(_, v)| v) .expect("customers"); - assert!(orders.filters.iter().any(|c| c == "status")); - assert!(!customers.filters.iter().any(|c| c == "status")); + assert!(orders.equality_filters.iter().any(|c| c == "status")); + assert!(!customers.equality_filters.iter().any(|c| c == "status")); } #[test] @@ -623,7 +664,7 @@ mod tests { .find(|(k, _)| k.ends_with("orders")) .map(|(_, v)| v) .expect("orders"); - assert!(orders.filters.iter().any(|c| c == "customer_id")); + assert!(orders.equality_filters.iter().any(|c| c == "customer_id")); } #[test] @@ -634,8 +675,34 @@ mod tests { .usage_by_table .iter() .find(|(k, _)| k.ends_with("orders")) - .map(|(_, table_usage)| table_usage.filters.iter().any(|c| c == "customer_id")) + .map(|(_, table_usage)| { + table_usage + .equality_filters + .iter() + .any(|c| c == "customer_id") + }) .unwrap_or(false); assert!(!has_customer_filter); } + + #[test] + fn classifies_non_equality_filters_separately() { + let query = + "SELECT * FROM orders WHERE created_at BETWEEN $1 AND $2 AND archived_at IS NULL"; + let usage = parse_query_columns(query).expect("parse"); + let orders = usage + .usage_by_table + .iter() + .find(|(k, _)| k.ends_with("orders")) + .map(|(_, v)| v) + .expect("orders"); + assert!(orders + .non_equality_filters + .iter() + .any(|c| c == "created_at")); + assert!(orders + .non_equality_filters + .iter() + .any(|c| c == "archived_at")); + } } diff --git a/src/analysis/table_index/indexes.rs b/src/analysis/table_index/indexes.rs index 40f43f0..b2341ab 100644 --- a/src/analysis/table_index/indexes.rs +++ b/src/analysis/table_index/indexes.rs @@ -15,6 +15,7 @@ struct IndexStatRow { schema: String, table_name: String, index_name: String, + key_columns: Vec, index_size_bytes: i64, index_size_pretty: String, idx_scan: i64, @@ -103,7 +104,7 @@ async fn fetch_soft_delete_candidates( tables_with_partial_idx AS ( SELECT DISTINCT indrelid FROM pg_index - WHERE indispartial = true + WHERE indpred IS NOT NULL ) SELECT s.nspname, @@ -142,6 +143,7 @@ fn identify_missing_partial_indexes(candidates: &[SoftDeleteCandidate]) -> Vec Vec schema: c.schema.clone(), table_name: c.table_name.clone(), index_name: c.column_name.clone(), // Use column name as proxy + key_columns: vec![c.column_name.clone()], index_size_bytes: 0, index_size_pretty: "0 B".to_string(), scans: 0, @@ -234,6 +237,10 @@ async fn fetch_index_stats(pool: &Pool) -> Result, C s.schemaname, s.relname, s.indexrelname, + COALESCE( + array_agg(a.attname ORDER BY arr.ord) FILTER (WHERE a.attname IS NOT NULL), + ARRAY[]::text[] + ) AS key_columns, s.idx_scan, s.idx_tup_read, s.idx_tup_fetch, @@ -241,14 +248,32 @@ async fn fetch_index_stats(pool: &Pool) -> Result, C pg_size_pretty(pg_relation_size(s.indexrelid)) AS index_size_pretty, t.n_live_tup, i.indisunique, - i.indispartial, + (i.indpred IS NOT NULL) AS is_partial, (i.indexprs IS NOT NULL) AS is_expression, EXISTS ( SELECT 1 FROM pg_constraint c WHERE c.conindid = s.indexrelid ) AS enforces_constraint FROM pg_stat_user_indexes s JOIN pg_index i ON s.indexrelid = i.indexrelid + LEFT JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS arr(attnum, ord) + ON arr.ord <= i.indnkeyatts + LEFT JOIN pg_attribute a + ON a.attrelid = s.relid + AND a.attnum = arr.attnum + AND arr.attnum > 0 LEFT JOIN pg_stat_user_tables t ON t.relid = s.relid + GROUP BY + s.schemaname, + s.relname, + s.indexrelname, + s.idx_scan, + s.idx_tup_read, + s.idx_tup_fetch, + s.indexrelid, + t.n_live_tup, + i.indisunique, + i.indpred, + i.indexprs "#; let rows = @@ -266,6 +291,7 @@ async fn fetch_index_stats(pool: &Pool) -> Result, C schema: row.get("schemaname"), table_name: row.get("relname"), index_name: row.get("indexrelname"), + key_columns: row.get("key_columns"), index_size_bytes: row.get("index_size_bytes"), index_size_pretty: row.get("index_size_pretty"), idx_scan: row.get("idx_scan"), @@ -275,7 +301,7 @@ async fn fetch_index_stats(pool: &Pool) -> Result, C is_unique: row.get("indisunique"), enforces_constraint: row.get("enforces_constraint"), is_expression: row.get("is_expression"), - is_partial: row.get("indispartial"), + is_partial: row.get("is_partial"), }); } @@ -298,6 +324,7 @@ fn identify_unused_indexes(rows: &[IndexStatRow]) -> Vec { schema: row.schema.clone(), table_name: row.table_name.clone(), index_name: row.index_name.clone(), + key_columns: row.key_columns.clone(), index_size_bytes: row.index_size_bytes, index_size_pretty: row.index_size_pretty.clone(), scans: row.idx_scan, @@ -334,6 +361,7 @@ fn identify_low_selectivity_indexes(rows: &[IndexStatRow]) -> Vec Vec, } #[derive(Debug, Clone, Copy)] struct TimeColumns { total: &'static str, - mean: &'static str, max: &'static str, } +#[derive(Debug, Clone, Default)] +struct WorkloadMetadataSnapshot { + server_version: Option, + stats_reset_at: Option, + seconds_since_reset: Option, + entry_deallocations: Option, + query_text_visible: bool, + has_wal_bytes: bool, +} + +#[derive(Debug)] +pub(crate) struct WorkloadAnalysis { + pub(crate) results: WorkloadResults, + pub(crate) available: bool, +} + +impl WorkloadAnalysis { + fn available(results: WorkloadResults) -> Self { + Self { + results, + available: true, + } + } + + fn unavailable(results: WorkloadResults) -> Self { + Self { + results, + available: false, + } + } +} + +#[derive(Debug)] +enum PgStatStatementsAvailability { + Available, + Unavailable { warning: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct StatementKey { + queryid: i64, + query: String, +} + +const RECENT_STATS_RESET_WARNING_WINDOW_SECS: f64 = 24.0 * 60.0 * 60.0; +const PARSE_FAILURE_WARNING_RATIO: f64 = 0.10; +const PARSE_FAILURE_WARNING_MIN: usize = 3; + +#[derive(Debug, Clone)] +struct IndexDefinition { + schema: String, + table: String, + access_method: String, + key_columns: Vec, + is_partial: bool, + is_expression: bool, + is_valid: bool, +} + #[derive(Debug, Default)] struct IndexCatalog { - indexes_by_table: HashMap>>, + indexes_by_table: HashMap>, schemas_by_table: HashMap>, } -pub async fn analyze( +#[derive(Debug, Clone, Default)] +struct SearchKey { + equality_columns: Vec, + ordered_columns: Vec, + display_columns: Vec, +} + +impl SearchKey { + fn from_usage(usage: &TableColumnUsage) -> Self { + let mut equality_columns = Vec::new(); + append_unique(&mut equality_columns, &usage.equality_filters); + append_unique(&mut equality_columns, &usage.equality_joins); + + let mut ordered_columns = Vec::new(); + append_unique(&mut ordered_columns, &usage.non_equality_filters); + append_unique(&mut ordered_columns, &usage.orders); + + let mut display_columns = Vec::new(); + append_unique(&mut display_columns, &usage.equality_filters); + append_unique(&mut display_columns, &usage.equality_joins); + append_unique(&mut display_columns, &usage.non_equality_filters); + append_unique(&mut display_columns, &usage.orders); + + Self { + equality_columns, + ordered_columns, + display_columns, + } + } + + fn is_empty(&self) -> bool { + self.display_columns.is_empty() + } +} + +pub(crate) async fn analyze( pool: &Pool, opts: &WorkloadOptions, -) -> Result { +) -> Result { let mut results = WorkloadResults::default(); - if !pg_stat_statements_installed(pool).await? { - results.warnings.push( - "pg_stat_statements extension is not installed; enable it to analyze slow queries." - .to_string(), - ); - return Ok(results); + match preflight_pg_stat_statements(pool).await? { + PgStatStatementsAvailability::Available => {} + PgStatStatementsAvailability::Unavailable { warning } => { + results.warnings.push(warning); + return Ok(WorkloadAnalysis::unavailable(results)); + } } - let version_num = match fetch_server_version(pool).await { - Ok(version) => version, - Err(err) => { - results - .warnings - .push(format!("Failed to detect server version: {err}")); - detect_pg_stat_statements_version(pool) - .await - .unwrap_or(130000) - } - }; + let metadata = collect_workload_metadata(pool, &mut results).await; + results.workload_metadata = build_workload_metadata(&metadata); + add_metadata_warnings(&metadata, &mut results); - let time_columns = if version_num >= 130000 { - TimeColumns { - total: "total_exec_time", - mean: "mean_exec_time", - max: "max_exec_time", - } - } else { - TimeColumns { - total: "total_time", - mean: "mean_time", - max: "max_time", - } - }; + let time_columns = resolve_time_columns(pool, &mut results, metadata.server_version).await; - let stats = fetch_statements(pool, opts, time_columns).await?; + let stats = fetch_statements(pool, opts, time_columns, metadata.has_wal_bytes).await?; if stats.is_empty() { results .warnings .push("No pg_stat_statements entries matched the filters.".to_string()); - return Ok(results); + return Ok(WorkloadAnalysis::available(results)); } results.slow_query_groups = build_slow_query_groups(&stats, opts); let index_catalog = fetch_index_catalog(pool).await?; - let mut candidates = build_index_candidates(&stats, &index_catalog, opts, &mut results); + let candidate_build = build_index_candidates(&stats, &index_catalog, opts); + let mut candidates = candidate_build.candidates; + results.parse_failures = candidate_build.coverage_stats.parser_errors; + results.coverage_stats = candidate_build.coverage_stats.clone(); + results.workload_metadata.parsed_queries = candidate_build.parsed_queries; + results.workload_metadata.parse_failures = candidate_build.coverage_stats.parser_errors; + results.workload_metadata.suppressed_candidates = + candidate_build.coverage_stats.suppressed_by_existing_index; + let workload_metadata = results.workload_metadata.clone(); + add_parse_failure_warning(stats.len(), &workload_metadata, &mut results); candidates.sort_by(|a, b| { b.total_time_ms .partial_cmp(&a.total_time_ms) @@ -115,7 +200,132 @@ pub async fn analyze( candidates.truncate(opts.limit); results.query_index_candidates = candidates; - Ok(results) + Ok(WorkloadAnalysis::available(results)) +} + +async fn collect_workload_metadata( + pool: &Pool, + results: &mut WorkloadResults, +) -> WorkloadMetadataSnapshot { + let server_version = match fetch_server_version(pool).await { + Ok(version) => Some(version), + Err(err) => { + results + .warnings + .push(format!("Failed to detect server version: {err}")); + None + } + }; + + let query_text_visible = match fetch_query_text_visibility(pool).await { + Ok(visible) => visible, + Err(err) => { + results + .warnings + .push(format!("Failed to detect query text visibility: {err}")); + true + } + }; + + let has_wal_bytes = match pg_stat_statements_has_column(pool, "wal_bytes").await { + Ok(has_column) => has_column, + Err(err) => { + results.warnings.push(format!( + "Failed to detect pg_stat_statements write metrics: {err}" + )); + false + } + }; + + let (stats_reset_at, seconds_since_reset, entry_deallocations) = + match fetch_pg_stat_statements_info(pool).await { + Ok(info) => info, + Err(err) => { + results + .warnings + .push(format!("Failed to read pg_stat_statements_info: {err}")); + (None, None, None) + } + }; + + WorkloadMetadataSnapshot { + server_version, + stats_reset_at, + seconds_since_reset, + entry_deallocations, + query_text_visible, + has_wal_bytes, + } +} + +fn build_workload_metadata(snapshot: &WorkloadMetadataSnapshot) -> WorkloadMetadata { + WorkloadMetadata { + server_version: snapshot.server_version, + stats_reset_at: snapshot.stats_reset_at.clone(), + entry_deallocations: snapshot.entry_deallocations, + query_text_visible: snapshot.query_text_visible, + ..WorkloadMetadata::default() + } +} + +fn add_metadata_warnings(snapshot: &WorkloadMetadataSnapshot, results: &mut WorkloadResults) { + if let (Some(stats_reset_at), Some(seconds_since_reset)) = + (&snapshot.stats_reset_at, snapshot.seconds_since_reset) + { + if seconds_since_reset <= RECENT_STATS_RESET_WARNING_WINDOW_SECS { + results.warnings.push(format!( + "Workload results are cumulative only since pg_stat_statements was last reset at {stats_reset_at}." + )); + } + } + + if snapshot.entry_deallocations.unwrap_or(0) > 0 { + results.warnings.push(format!( + "pg_stat_statements has evicted {} entries due to capacity pressure; low-frequency statements and derived findings may be incomplete.", + snapshot.entry_deallocations.unwrap_or(0) + )); + } + + if !snapshot.query_text_visible { + results.warnings.push( + "Query text visibility appears limited for the current role; grant pg_read_all_stats to avoid incomplete or anonymized workload findings." + .to_string(), + ); + } +} + +fn add_parse_failure_warning( + total_queries: usize, + metadata: &WorkloadMetadata, + results: &mut WorkloadResults, +) { + if metadata.parse_failures == 0 || total_queries == 0 { + return; + } + + let failure_ratio = metadata.parse_failures as f64 / total_queries as f64; + if metadata.parse_failures >= PARSE_FAILURE_WARNING_MIN + || failure_ratio >= PARSE_FAILURE_WARNING_RATIO + { + results.warnings.push(format!( + "Only {} of {} workload statements were parsed into index evidence; index candidate coverage is partial.", + metadata.parsed_queries, total_queries + )); + } +} + +async fn preflight_pg_stat_statements( + pool: &Pool, +) -> Result { + if !pg_stat_statements_installed(pool).await? { + return Ok(PgStatStatementsAvailability::Unavailable { + warning: + "pg_stat_statements extension is not installed; enable it to analyze slow queries." + .to_string(), + }); + } + + probe_pg_stat_statements(pool).await } async fn pg_stat_statements_installed(pool: &Pool) -> Result { @@ -129,6 +339,22 @@ async fn pg_stat_statements_installed(pool: &Pool) -> Result, +) -> Result { + let query = "SELECT 1 FROM pg_stat_statements LIMIT 1"; + match query_scalar::<_, i32>(query).fetch_optional(pool).await { + Ok(_) => Ok(PgStatStatementsAvailability::Available), + Err(source) => match pg_stat_statements_unavailable_warning(&source) { + Some(warning) => Ok(PgStatStatementsAvailability::Unavailable { warning }), + None => Err(CheckerError::QueryError { + query: query.into(), + source, + }), + }, + } +} + async fn fetch_server_version(pool: &Pool) -> Result { let query = "SELECT current_setting('server_version_num')::bigint"; query_scalar::<_, i64>(query) @@ -140,64 +366,124 @@ async fn fetch_server_version(pool: &Pool) -> Result) -> Result { + let query = r#" + SELECT current_setting('is_superuser')::boolean + OR pg_has_role(current_user, 'pg_read_all_stats', 'MEMBER') + "#; + query_scalar::<_, bool>(query) + .fetch_one(pool) + .await + .map_err(|source| CheckerError::QueryError { + query: query.into(), + source, + }) +} + async fn detect_pg_stat_statements_version(pool: &Pool) -> Option { + pg_stat_statements_has_column(pool, "total_exec_time") + .await + .ok() + .filter(|exists| *exists) + .map(|_| 130000) +} + +async fn pg_stat_statements_has_column( + pool: &Pool, + column_name: &str, +) -> Result { let query = r#" SELECT EXISTS( SELECT 1 FROM information_schema.columns WHERE table_name = 'pg_stat_statements' - AND column_name = 'total_exec_time' + AND column_name = $1 ) "#; query_scalar::<_, bool>(query) + .bind(column_name) .fetch_one(pool) .await - .ok() - .filter(|exists| *exists) - .map(|_| 130000) + .map_err(|source| CheckerError::QueryError { + query: query.into(), + source, + }) +} + +async fn fetch_pg_stat_statements_info( + pool: &Pool, +) -> Result<(Option, Option, Option), CheckerError> { + let query = r#" + SELECT + stats_reset::text AS stats_reset_at, + EXTRACT(EPOCH FROM now() - stats_reset)::double precision AS seconds_since_reset, + dealloc::bigint AS entry_deallocations + FROM pg_stat_statements_info + "#; + + sqlx::query(query) + .fetch_optional(pool) + .await + .map(|row| { + row.map_or((None, None, None), |row| { + ( + row.get("stats_reset_at"), + row.get("seconds_since_reset"), + row.get("entry_deallocations"), + ) + }) + }) + .map_err(|source| CheckerError::QueryError { + query: query.into(), + source, + }) +} + +async fn resolve_time_columns( + pool: &Pool, + results: &mut WorkloadResults, + server_version: Option, +) -> TimeColumns { + let version_num = server_version + .or(detect_pg_stat_statements_version(pool).await) + .unwrap_or_else(|| { + results.warnings.push( + "Falling back to PostgreSQL 13+ timing columns for pg_stat_statements.".to_string(), + ); + 130000 + }); + + if version_num >= 130000 { + TimeColumns { + total: "total_exec_time", + max: "max_exec_time", + } + } else { + TimeColumns { + total: "total_time", + max: "max_time", + } + } } async fn fetch_statements( pool: &Pool, opts: &WorkloadOptions, columns: TimeColumns, + has_wal_bytes: bool, ) -> Result, CheckerError> { let fetch_limit = (opts.limit.max(1) * 5).max(50) as i64; let metrics = [ - ("total_time_ms", columns.total), - ("mean_time_ms", columns.mean), - ("shared_blks_read", "shared_blks_read"), - ("temp_blks_written", "temp_blks_written"), + "total_time_ms", + "mean_time_ms", + "shared_blks_read", + "temp_blks_written", ]; - let mut map: HashMap = HashMap::new(); + let mut map: HashMap = HashMap::new(); - for (_, metric_column) in metrics { - let query = format!( - r#" - SELECT - s.queryid, - s.query, - s.calls, - s.rows, - s.shared_blks_read, - s.shared_blks_hit, - s.temp_blks_read, - s.temp_blks_written, - s.{total} AS total_time_ms, - s.{mean} AS mean_time_ms, - s.{max} AS max_time_ms - FROM pg_stat_statements s - WHERE s.dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) - AND s.calls >= $1 - ORDER BY s.{metric} DESC - LIMIT $2 - "#, - total = columns.total, - mean = columns.mean, - max = columns.max, - metric = metric_column - ); + for metric_column in metrics { + let query = build_statement_query(columns, metric_column, has_wal_bytes); let rows = sqlx::query(&query) .bind(opts.min_calls) @@ -210,34 +496,93 @@ async fn fetch_statements( })?; for row in rows { - let queryid: i64 = row.get("queryid"); - if map.contains_key(&queryid) { + let stat = StatementStat { + queryid: row.get("queryid"), + query: row.get("query"), + calls: row.get("calls"), + total_time_ms: row.get("total_time_ms"), + mean_time_ms: row.get("mean_time_ms"), + max_time_ms: row.get("max_time_ms"), + rows: row.get("rows"), + shared_blks_read: row.get("shared_blks_read"), + shared_blks_hit: row.get("shared_blks_hit"), + temp_blks_read: row.get("temp_blks_read"), + temp_blks_written: row.get("temp_blks_written"), + wal_bytes: row.get("wal_bytes"), + }; + let key = StatementKey { + queryid: stat.queryid, + query: stat.query.clone(), + }; + if map.contains_key(&key) { continue; } - map.insert( - queryid, - StatementStat { - queryid, - query: row.get("query"), - calls: row.get("calls"), - total_time_ms: row.get("total_time_ms"), - mean_time_ms: row.get("mean_time_ms"), - max_time_ms: row.get("max_time_ms"), - rows: row.get("rows"), - shared_blks_read: row.get("shared_blks_read"), - shared_blks_hit: row.get("shared_blks_hit"), - temp_blks_read: row.get("temp_blks_read"), - temp_blks_written: row.get("temp_blks_written"), - }, - ); + map.insert(key, stat); } } Ok(map.into_values().collect()) } +fn build_statement_query(columns: TimeColumns, metric_column: &str, has_wal_bytes: bool) -> String { + let wal_bytes_select = if has_wal_bytes { + "SUM(COALESCE(s.wal_bytes, 0))::bigint AS wal_bytes," + } else { + "NULL::bigint AS wal_bytes," + }; + + format!( + r#" + WITH aggregated AS ( + SELECT + COALESCE(s.queryid, 0)::bigint AS queryid, + COALESCE(s.query, '') AS query, + SUM(s.calls)::bigint AS calls, + SUM(s.rows)::bigint AS rows, + SUM(s.shared_blks_read)::bigint AS shared_blks_read, + SUM(s.shared_blks_hit)::bigint AS shared_blks_hit, + SUM(s.temp_blks_read)::bigint AS temp_blks_read, + SUM(s.temp_blks_written)::bigint AS temp_blks_written, + {wal_bytes} + SUM(s.{total}) AS total_time_ms, + CASE + WHEN SUM(s.calls) > 0 + THEN SUM(s.{total}) / SUM(s.calls)::double precision + ELSE 0 + END AS mean_time_ms, + MAX(s.{max}) AS max_time_ms + FROM pg_stat_statements s + WHERE s.dbid = (SELECT oid FROM pg_database WHERE datname = current_database()) + GROUP BY COALESCE(s.queryid, 0)::bigint, COALESCE(s.query, '') + HAVING SUM(s.calls) >= $1 + ) + SELECT + queryid, + query, + calls, + rows, + shared_blks_read, + shared_blks_hit, + temp_blks_read, + temp_blks_written, + wal_bytes, + total_time_ms, + mean_time_ms, + max_time_ms + FROM aggregated + ORDER BY {metric} DESC + LIMIT $2 + "#, + wal_bytes = wal_bytes_select, + total = columns.total, + max = columns.max, + metric = metric_column + ) +} + fn build_slow_query_groups(stats: &[StatementStat], opts: &WorkloadOptions) -> Vec { + let total_measured_time_ms: f64 = stats.iter().map(|stat| stat.total_time_ms).sum(); let groups = [ (SlowQueryKind::TotalTime, "total"), (SlowQueryKind::MeanTime, "mean"), @@ -282,6 +627,13 @@ fn build_slow_query_groups(stats: &[StatementStat], opts: &WorkloadOptions) -> V shared_blks_hit: stat.shared_blks_hit, temp_blks_read: stat.temp_blks_read, temp_blks_written: stat.temp_blks_written, + total_time_pct: total_time_pct(stat.total_time_ms, total_measured_time_ms), + cache_hit_ratio: cache_hit_ratio(stat.shared_blks_hit, stat.shared_blks_read), + temp_blks_written_per_call: per_call_i64(stat.temp_blks_written, stat.calls), + wal_bytes: stat.wal_bytes, + wal_bytes_per_call: stat + .wal_bytes + .and_then(|wal_bytes| per_call_i64(wal_bytes, stat.calls)), query_text: format_query_text(&stat.query, opts), }) .collect(); @@ -317,19 +669,70 @@ fn truncate_query(query: &str, max_len: usize) -> String { truncated } +fn total_time_pct(total_time_ms: f64, total_measured_time_ms: f64) -> f64 { + if total_measured_time_ms <= 0.0 { + 0.0 + } else { + (total_time_ms / total_measured_time_ms) * 100.0 + } +} + +fn cache_hit_ratio(shared_blks_hit: i64, shared_blks_read: i64) -> Option { + let total = shared_blks_hit + shared_blks_read; + if total <= 0 { + None + } else { + Some(shared_blks_hit as f64 / total as f64) + } +} + +fn per_call_i64(value: i64, calls: i64) -> Option { + if calls <= 0 { + None + } else { + Some(value as f64 / calls as f64) + } +} + +fn pg_stat_statements_unavailable_warning(source: &Error) -> Option { + let message = source.as_database_error()?.message(); + if is_pg_stat_statements_preload_error_message(message) { + Some(format!( + "pg_stat_statements is installed but not usable: {message}. Add it to shared_preload_libraries and restart PostgreSQL before running workload analysis." + )) + } else { + None + } +} + +fn is_pg_stat_statements_preload_error_message(message: &str) -> bool { + message.contains("pg_stat_statements must be loaded via shared_preload_libraries") + || message.contains("pg_stat_statements must be loaded via \"shared_preload_libraries\"") +} + +#[derive(Debug, Default)] +struct CandidateBuildResult { + candidates: Vec, + coverage_stats: WorkloadCoverageStats, + parsed_queries: usize, +} + fn build_index_candidates( stats: &[StatementStat], catalog: &IndexCatalog, opts: &WorkloadOptions, - results: &mut WorkloadResults, -) -> Vec { +) -> CandidateBuildResult { let mut deduped: HashMap = HashMap::new(); + let mut coverage_stats = WorkloadCoverageStats::default(); + let mut parsed_queries = 0; for stat in stats { match parse_query_columns(&stat.query) { Ok(usage) => { + parsed_queries += 1; let per_query = build_candidates_for_usage(stat, &usage, catalog); - for candidate in per_query { + merge_coverage_stats(&mut coverage_stats, &per_query.coverage_stats); + for candidate in per_query.candidates { let key = format!( "{}.{}:{}", candidate.schema, @@ -345,7 +748,7 @@ fn build_index_candidates( } } } - Err(_) => results.parse_failures += 1, + Err(_) => coverage_stats.parser_errors += 1, } } @@ -356,52 +759,75 @@ fn build_index_candidates( .unwrap_or(std::cmp::Ordering::Equal) }); candidates.truncate(opts.limit * 2); - candidates + CandidateBuildResult { + candidates, + coverage_stats, + parsed_queries, + } +} + +fn merge_coverage_stats(target: &mut WorkloadCoverageStats, source: &WorkloadCoverageStats) { + target.suppressed_by_existing_index += source.suppressed_by_existing_index; + target.skipped_internal_tables += source.skipped_internal_tables; + target.skipped_unresolved_schema += source.skipped_unresolved_schema; + target.skipped_unsupported_parse_shape += source.skipped_unsupported_parse_shape; + target.parser_errors += source.parser_errors; } fn build_candidates_for_usage( stat: &StatementStat, usage: &QueryColumnUsage, catalog: &IndexCatalog, -) -> Vec { +) -> CandidateBuildResult { let mut table_map = HashMap::new(); for table in &usage.tables { table_map.insert(table.full_name(), table.clone()); } + let mut coverage_stats = WorkloadCoverageStats::default(); let mut candidates = Vec::new(); for (table_name, usage) in &usage.usage_by_table { let table_ref = table_map.get(table_name); let Some(table_ref) = table_ref else { continue }; if is_internal_postgres_table(table_ref) { + coverage_stats.skipped_internal_tables += 1; continue; } - let mut columns = Vec::new(); - append_unique(&mut columns, &usage.filters); - append_unique(&mut columns, &usage.joins); - append_unique(&mut columns, &usage.orders); - - if columns.is_empty() { + let search_key = SearchKey::from_usage(usage); + if search_key.is_empty() { + coverage_stats.skipped_unsupported_parse_shape += 1; continue; } - if columns.len() > 3 { - columns.truncate(3); + let resolved = resolve_table_schema(table_ref, catalog); + if resolved.schema == "unknown" { + coverage_stats.skipped_unresolved_schema += 1; + continue; } - let resolved = resolve_table_schema(table_ref, catalog); - if resolved.schema != "unknown" && is_index_covered(&resolved.full_name, &columns, catalog) - { + let mut notes = collect_non_covering_index_notes(&resolved.full_name, &search_key, catalog); + if resolved.ambiguous_schema { + notes.push( + "table name resolved to public, but another schema may contain the same table" + .to_string(), + ); + } + if is_index_covered(&resolved.full_name, &search_key, catalog) { + coverage_stats.suppressed_by_existing_index += 1; continue; } + let columns = output_columns(&search_key); let reason = format_reason(usage, resolved.ambiguous_schema); candidates.push(QueryIndexCandidate { schema: resolved.schema, table: resolved.table, columns, reason, + confidence: candidate_confidence(usage, resolved.ambiguous_schema, ¬es), + evidence: evidence_from_usage(usage), + notes, queryid: stat.queryid, total_time_ms: stat.total_time_ms, mean_time_ms: stat.mean_time_ms, @@ -409,7 +835,11 @@ fn build_candidates_for_usage( }); } - candidates + CandidateBuildResult { + candidates, + coverage_stats, + parsed_queries: 0, + } } fn is_internal_postgres_table(table: &TableRef) -> bool { @@ -441,11 +871,14 @@ fn append_unique(target: &mut Vec, source: &[String]) { fn format_reason(usage: &TableColumnUsage, ambiguous_schema: bool) -> String { let mut parts = Vec::new(); - if !usage.filters.is_empty() { - parts.push(format!("WHERE {}", usage.filters.join(", "))); + let mut where_columns = Vec::new(); + append_unique(&mut where_columns, &usage.equality_filters); + append_unique(&mut where_columns, &usage.non_equality_filters); + if !where_columns.is_empty() { + parts.push(format!("WHERE {}", where_columns.join(", "))); } - if !usage.joins.is_empty() { - parts.push(format!("JOIN {}", usage.joins.join(", "))); + if !usage.equality_joins.is_empty() { + parts.push(format!("JOIN {}", usage.equality_joins.join(", "))); } if !usage.orders.is_empty() { parts.push(format!("ORDER BY {}", usage.orders.join(", "))); @@ -456,54 +889,218 @@ fn format_reason(usage: &TableColumnUsage, ambiguous_schema: bool) -> String { format!("heuristic from slow query: {}", parts.join("; ")) } -fn is_index_covered(table: &str, columns: &[String], catalog: &IndexCatalog) -> bool { - let Some(indexes) = catalog.indexes_by_table.get(table) else { - return false; - }; - - let target: Vec = columns.iter().map(|c| c.to_lowercase()).collect(); - for index_columns in indexes { - let index_lower: Vec = index_columns.iter().map(|c| c.to_lowercase()).collect(); - if index_lower.len() >= target.len() && index_lower[..target.len()] == target[..] { - return true; - } +fn evidence_from_usage(usage: &TableColumnUsage) -> QueryIndexEvidence { + QueryIndexEvidence { + equality_filters: usage.equality_filters.clone(), + non_equality_filters: usage.non_equality_filters.clone(), + equality_joins: usage.equality_joins.clone(), + order_by: usage.orders.clone(), } - - false } -struct ResolvedTable { - schema: String, - table: String, - full_name: String, +fn candidate_confidence( + usage: &TableColumnUsage, ambiguous_schema: bool, -} + notes: &[String], +) -> WorkloadFindingConfidence { + if ambiguous_schema || (usage.equality_filters.is_empty() && usage.equality_joins.is_empty()) { + return WorkloadFindingConfidence::Low; + } -fn resolve_table_schema(table: &TableRef, catalog: &IndexCatalog) -> ResolvedTable { - if let Some(schema) = &table.schema { - let full_name = format!("{}.{}", schema, table.name); - return ResolvedTable { - schema: schema.clone(), - table: table.name.clone(), - full_name, - ambiguous_schema: false, - }; + if !usage.non_equality_filters.is_empty() || !usage.orders.is_empty() || !notes.is_empty() { + return WorkloadFindingConfidence::Medium; } - let schemas = catalog - .schemas_by_table - .get(&table.name) - .cloned() - .unwrap_or_default(); + WorkloadFindingConfidence::High +} - if schemas.len() == 1 { - let schema = schemas[0].clone(); - return ResolvedTable { - schema: schema.clone(), - table: table.name.clone(), - full_name: format!("{}.{}", schema, table.name), - ambiguous_schema: false, - }; +fn output_columns(search_key: &SearchKey) -> Vec { + let mut columns = search_key.display_columns.clone(); + if columns.len() > 3 { + columns.truncate(3); + } + columns +} + +fn is_index_covered(table: &str, search_key: &SearchKey, catalog: &IndexCatalog) -> bool { + let Some(indexes) = catalog.indexes_by_table.get(table) else { + return false; + }; + + indexes + .iter() + .any(|index| index_covers_search_key(index, search_key)) +} + +fn collect_non_covering_index_notes( + table: &str, + search_key: &SearchKey, + catalog: &IndexCatalog, +) -> Vec { + let Some(indexes) = catalog.indexes_by_table.get(table) else { + return Vec::new(); + }; + + let mut notes = Vec::new(); + + if indexes + .iter() + .any(|index| index.is_partial && index_overlaps_search_key(index, search_key)) + { + notes.push( + "existing partial index ignored because this command cannot prove the query matches its predicate" + .to_string(), + ); + } + + if indexes + .iter() + .any(|index| index.is_expression && index_overlaps_search_key(index, search_key)) + { + notes.push( + "existing expression index ignored because expression equivalence is not proven" + .to_string(), + ); + } + + if indexes + .iter() + .any(|index| !index.is_valid && index_overlaps_search_key(index, search_key)) + { + notes.push("existing invalid index ignored".to_string()); + } + + if indexes.iter().any(|index| { + !index.access_method.eq_ignore_ascii_case("btree") + && index_overlaps_search_key(index, search_key) + }) { + notes.push("existing non-B-tree index not treated as generic coverage".to_string()); + } + + notes +} + +fn index_overlaps_search_key(index: &IndexDefinition, search_key: &SearchKey) -> bool { + index.key_columns.iter().any(|index_column| { + search_key + .display_columns + .iter() + .any(|candidate_column| index_column.eq_ignore_ascii_case(candidate_column)) + }) +} + +fn index_covers_search_key(index: &IndexDefinition, search_key: &SearchKey) -> bool { + if !index.is_valid || index.is_partial || index.is_expression { + return false; + } + + if index.access_method.eq_ignore_ascii_case("btree") { + btree_index_covers_search_key(index, search_key) + } else { + has_exact_prefix(&index.key_columns, &search_key.display_columns) + } +} + +fn btree_index_covers_search_key(index: &IndexDefinition, search_key: &SearchKey) -> bool { + let required_len = search_key.equality_columns.len() + search_key.ordered_columns.len(); + if required_len == 0 || index.key_columns.len() < required_len { + return false; + } + + let leading = &index.key_columns[..search_key.equality_columns.len()]; + if !same_column_set(leading, &search_key.equality_columns) { + return false; + } + + has_exact_prefix( + &index.key_columns[search_key.equality_columns.len()..], + &search_key.ordered_columns, + ) +} + +fn same_column_set(left: &[String], right: &[String]) -> bool { + left.len() == right.len() + && left.iter().all(|left_value| { + right + .iter() + .any(|right_value| left_value.eq_ignore_ascii_case(right_value)) + }) +} + +fn has_exact_prefix(index_columns: &[String], target_columns: &[String]) -> bool { + index_columns.len() >= target_columns.len() + && index_columns + .iter() + .zip(target_columns.iter()) + .all(|(index_column, target_column)| index_column.eq_ignore_ascii_case(target_column)) +} + +struct ResolvedTable { + schema: String, + table: String, + full_name: String, + ambiguous_schema: bool, +} + +const FETCH_INDEX_CATALOG_QUERY: &str = r#" + SELECT + n.nspname AS schema_name, + c.relname AS table_name, + am.amname AS access_method, + (i.indpred IS NOT NULL) AS is_partial, + (i.indexprs IS NOT NULL) AS is_expression, + i.indisvalid AS is_valid, + COALESCE( + array_agg(a.attname ORDER BY arr.ord) FILTER (WHERE a.attname IS NOT NULL), + ARRAY[]::text[] + ) AS key_columns + FROM pg_index i + JOIN pg_class c ON c.oid = i.indrelid + JOIN pg_class idx ON idx.oid = i.indexrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_am am ON am.oid = idx.relam + LEFT JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS arr(attnum, ord) + ON arr.ord <= i.indnkeyatts + LEFT JOIN pg_attribute a + ON a.attrelid = c.oid + AND a.attnum = arr.attnum + AND arr.attnum > 0 + WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') + GROUP BY + n.nspname, + c.relname, + idx.relname, + am.amname, + i.indpred, + i.indexprs, + i.indisvalid +"#; + +fn resolve_table_schema(table: &TableRef, catalog: &IndexCatalog) -> ResolvedTable { + if let Some(schema) = &table.schema { + let full_name = format!("{}.{}", schema, table.name); + return ResolvedTable { + schema: schema.clone(), + table: table.name.clone(), + full_name, + ambiguous_schema: false, + }; + } + + let schemas = catalog + .schemas_by_table + .get(&table.name) + .cloned() + .unwrap_or_default(); + + if schemas.len() == 1 { + let schema = schemas[0].clone(); + return ResolvedTable { + schema: schema.clone(), + table: table.name.clone(), + full_name: format!("{}.{}", schema, table.name), + ambiguous_schema: false, + }; } if schemas.contains(&"public".to_string()) { @@ -524,53 +1121,113 @@ fn resolve_table_schema(table: &TableRef, catalog: &IndexCatalog) -> ResolvedTab } async fn fetch_index_catalog(pool: &Pool) -> Result { - const QUERY: &str = r#" - SELECT - n.nspname AS schema_name, - c.relname AS table_name, - array_agg(a.attname ORDER BY arr.ord) AS columns - FROM pg_index i - JOIN pg_class c ON c.oid = i.indrelid - JOIN pg_class idx ON idx.oid = i.indexrelid - JOIN pg_namespace n ON n.oid = c.relnamespace - JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS arr(attnum, ord) - ON arr.attnum > 0 - JOIN pg_attribute a ON a.attrelid = c.oid AND a.attnum = arr.attnum - WHERE n.nspname NOT IN ('pg_catalog', 'information_schema') - GROUP BY n.nspname, c.relname, idx.relname - "#; - - let rows = - sqlx::query(QUERY) - .fetch_all(pool) - .await - .map_err(|source| CheckerError::QueryError { - query: QUERY.into(), - source, - })?; + let rows = sqlx::query(FETCH_INDEX_CATALOG_QUERY) + .fetch_all(pool) + .await + .map_err(|source| CheckerError::QueryError { + query: FETCH_INDEX_CATALOG_QUERY.into(), + source, + })?; let mut catalog = IndexCatalog::default(); for row in rows { - let schema: String = row.get("schema_name"); - let table: String = row.get("table_name"); - let columns: Vec = row.get("columns"); + let definition = IndexDefinition { + schema: row.get("schema_name"), + table: row.get("table_name"), + access_method: row.get("access_method"), + key_columns: row.get("key_columns"), + is_partial: row.get("is_partial"), + is_expression: row.get("is_expression"), + is_valid: row.get("is_valid"), + }; - let full_name = format!("{}.{}", schema, table); + let full_name = format!("{}.{}", definition.schema, definition.table); catalog .indexes_by_table .entry(full_name) .or_default() - .push(columns); - - let entry = catalog.schemas_by_table.entry(table).or_default(); - if !entry.contains(&schema) { - entry.push(schema); + .push(definition.clone()); + + let entry = catalog + .schemas_by_table + .entry(definition.table.clone()) + .or_default(); + if !entry.contains(&definition.schema) { + entry.push(definition.schema.clone()); } } Ok(catalog) } +pub(crate) fn correlate_table_health(results: &mut WorkloadResults) { + for candidate in &mut results.query_index_candidates { + if results.seq_scan_info.iter().any(|table| { + table.schema.eq_ignore_ascii_case(&candidate.schema) + && table.table_name.eq_ignore_ascii_case(&candidate.table) + }) { + push_unique_note( + &mut candidate.notes, + "table is also a sequential scan hotspot".to_string(), + ); + } + + if results.bloat_info.iter().any(|table| { + table.schema.eq_ignore_ascii_case(&candidate.schema) + && table.table_name.eq_ignore_ascii_case(&candidate.table) + }) { + push_unique_note( + &mut candidate.notes, + "table is also on the bloat watchlist".to_string(), + ); + } + + let overlapping_unused_indexes: Vec<_> = results + .index_usage_info + .iter() + .filter(|index| { + index.issue == IndexIssueKind::Unused + && index.schema.eq_ignore_ascii_case(&candidate.schema) + && index.table_name.eq_ignore_ascii_case(&candidate.table) + && index_usage_overlaps_candidate(index, candidate) + }) + .map(|index| { + let columns = if index.key_columns.is_empty() { + String::new() + } else { + format!(" ({})", index.key_columns.join(", ")) + }; + format!( + "unused overlapping index {}{} already exists on this table", + index.index_name, columns + ) + }) + .collect(); + for note in overlapping_unused_indexes { + push_unique_note(&mut candidate.notes, note); + } + } +} + +fn push_unique_note(notes: &mut Vec, note: String) { + if !notes.iter().any(|existing| existing == ¬e) { + notes.push(note); + } +} + +fn index_usage_overlaps_candidate( + index: &crate::models::IndexUsageInfo, + candidate: &QueryIndexCandidate, +) -> bool { + !index.key_columns.is_empty() + && index.key_columns.iter().any(|index_column| { + candidate + .columns + .iter() + .any(|candidate_column| index_column.eq_ignore_ascii_case(candidate_column)) + }) +} + #[cfg(test)] mod tests { use super::*; @@ -582,38 +1239,57 @@ mod tests { schema: Some("public".into()), name: "orders".into(), }); - let mut table_usage = TableColumnUsage::default(); - table_usage.filters = vec!["customer_id".into(), "status".into()]; - table_usage.joins = vec!["org_id".into()]; - table_usage.orders = vec!["created_at".into()]; + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into(), "status".into()], + equality_joins: vec!["org_id".into()], + orders: vec!["created_at".into()], + ..TableColumnUsage::default() + }; usage .usage_by_table .insert("public.orders".into(), table_usage); usage } - #[test] - fn candidate_orders_columns_by_filter_join_order() { - let usage = make_usage(); - let catalog = IndexCatalog::default(); - let stat = StatementStat { - queryid: 1, - query: "SELECT * FROM orders".into(), + fn make_stat(queryid: i64, query: &str, total_time_ms: f64) -> StatementStat { + StatementStat { + queryid, + query: query.into(), calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, + total_time_ms, + mean_time_ms: total_time_ms / 10.0, + max_time_ms: total_time_ms / 5.0, rows: 0, shared_blks_read: 0, shared_blks_hit: 0, temp_blks_read: 0, temp_blks_written: 0, - }; + wal_bytes: None, + } + } + + fn make_index_definition(columns: &[&str]) -> IndexDefinition { + IndexDefinition { + schema: "public".into(), + table: "orders".into(), + access_method: "btree".into(), + key_columns: columns.iter().map(|column| column.to_string()).collect(), + is_partial: false, + is_expression: false, + is_valid: true, + } + } + + #[test] + fn candidate_orders_columns_by_filter_join_order() { + let usage = make_usage(); + let catalog = IndexCatalog::default(); + let stat = make_stat(1, "SELECT * FROM orders", 1000.0); let candidates = build_candidates_for_usage(&stat, &usage, &catalog); - assert_eq!(candidates.len(), 1); + assert_eq!(candidates.candidates.len(), 1); assert_eq!( - candidates[0].columns, + candidates.candidates[0].columns, vec!["customer_id", "status", "org_id"] ); } @@ -625,98 +1301,63 @@ mod tests { schema: Some("public".into()), name: "orders".into(), }); - let mut table_usage = TableColumnUsage::default(); - table_usage.filters = vec!["customer_id".into(), "status".into()]; + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into(), "status".into()], + ..TableColumnUsage::default() + }; usage .usage_by_table .insert("public.orders".into(), table_usage); let mut catalog = IndexCatalog::default(); catalog.indexes_by_table.insert( "public.orders".into(), - vec![vec!["customer_id".into(), "status".into()]], + vec![make_index_definition(&["status", "customer_id"])], ); - let stat = StatementStat { - queryid: 1, - query: "SELECT * FROM orders".into(), - calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, - }; + let stat = make_stat(1, "SELECT * FROM orders", 1000.0); let candidates = build_candidates_for_usage(&stat, &usage, &catalog); - assert!(candidates.is_empty()); + assert!(candidates.candidates.is_empty()); } #[test] fn candidate_dedupes_by_columns() { - let catalog = IndexCatalog::default(); - let stat_one = StatementStat { - queryid: 1, - query: "SELECT * FROM orders WHERE customer_id = $1 AND status = 'open'".into(), - calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, - }; - let stat_two = StatementStat { - queryid: 2, - query: "SELECT * FROM orders WHERE customer_id = $1 AND status = 'open'".into(), - calls: 8, - total_time_ms: 500.0, - mean_time_ms: 120.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, - }; - - let mut results = WorkloadResults::default(); - let candidates = build_index_candidates( - &[stat_one, stat_two], - &catalog, - &WorkloadOptions::default(), - &mut results, + let mut catalog = IndexCatalog::default(); + catalog + .schemas_by_table + .insert("orders".into(), vec!["public".into()]); + let stat_one = make_stat( + 1, + "SELECT * FROM orders WHERE customer_id = $1 AND status = 'open'", + 1000.0, + ); + let stat_two = make_stat( + 2, + "SELECT * FROM orders WHERE customer_id = $1 AND status = 'open'", + 500.0, ); - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].queryid, 1); + + let build = + build_index_candidates(&[stat_one, stat_two], &catalog, &WorkloadOptions::default()); + assert_eq!(build.candidates.len(), 1); + assert_eq!(build.candidates[0].queryid, 1); } #[test] fn update_statement_produces_candidate_without_parse_failure() { - let catalog = IndexCatalog::default(); - let stat = StatementStat { - queryid: 1, - query: "UPDATE orders SET status = 'closed' WHERE customer_id = $1".into(), - calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, - }; + let mut catalog = IndexCatalog::default(); + catalog + .schemas_by_table + .insert("orders".into(), vec!["public".into()]); + let stat = make_stat( + 1, + "UPDATE orders SET status = 'closed' WHERE customer_id = $1", + 1000.0, + ); - let mut results = WorkloadResults::default(); - let candidates = - build_index_candidates(&[stat], &catalog, &WorkloadOptions::default(), &mut results); - assert_eq!(results.parse_failures, 0); - assert!(candidates.iter().any(|candidate| { + let build = build_index_candidates(&[stat], &catalog, &WorkloadOptions::default()); + assert_eq!(build.coverage_stats.parser_errors, 0); + assert!(build.candidates.iter().any(|candidate| { candidate.table == "orders" && candidate .columns @@ -727,26 +1368,15 @@ mod tests { #[test] fn delete_statement_produces_candidate_without_parse_failure() { - let catalog = IndexCatalog::default(); - let stat = StatementStat { - queryid: 1, - query: "DELETE FROM orders WHERE customer_id = $1".into(), - calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, - }; + let mut catalog = IndexCatalog::default(); + catalog + .schemas_by_table + .insert("orders".into(), vec!["public".into()]); + let stat = make_stat(1, "DELETE FROM orders WHERE customer_id = $1", 1000.0); - let mut results = WorkloadResults::default(); - let candidates = - build_index_candidates(&[stat], &catalog, &WorkloadOptions::default(), &mut results); - assert_eq!(results.parse_failures, 0); - assert!(candidates.iter().any(|candidate| { + let build = build_index_candidates(&[stat], &catalog, &WorkloadOptions::default()); + assert_eq!(build.coverage_stats.parser_errors, 0); + assert!(build.candidates.iter().any(|candidate| { candidate.table == "orders" && candidate .columns @@ -762,29 +1392,19 @@ mod tests { schema: None, name: "pg_stat_database".into(), }); - let mut table_usage = TableColumnUsage::default(); - table_usage.filters = vec!["datid".into()]; + let table_usage = TableColumnUsage { + equality_filters: vec!["datid".into()], + ..TableColumnUsage::default() + }; usage .usage_by_table .insert("pg_stat_database".into(), table_usage); let catalog = IndexCatalog::default(); - let stat = StatementStat { - queryid: 1, - query: "SELECT * FROM pg_stat_database WHERE datid = 1".into(), - calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, - }; + let stat = make_stat(1, "SELECT * FROM pg_stat_database WHERE datid = 1", 1000.0); let candidates = build_candidates_for_usage(&stat, &usage, &catalog); - assert!(candidates.is_empty()); + assert!(candidates.candidates.is_empty()); } #[test] @@ -794,29 +1414,385 @@ mod tests { schema: Some("public".into()), name: "pg_custom_events".into(), }); - let mut table_usage = TableColumnUsage::default(); - table_usage.filters = vec!["account_id".into()]; + let table_usage = TableColumnUsage { + equality_filters: vec!["account_id".into()], + ..TableColumnUsage::default() + }; usage .usage_by_table .insert("public.pg_custom_events".into(), table_usage); let catalog = IndexCatalog::default(); - let stat = StatementStat { - queryid: 1, - query: "SELECT * FROM public.pg_custom_events WHERE account_id = 42".into(), - calls: 10, - total_time_ms: 1000.0, - mean_time_ms: 100.0, - max_time_ms: 200.0, - rows: 0, - shared_blks_read: 0, - shared_blks_hit: 0, - temp_blks_read: 0, - temp_blks_written: 0, + let stat = make_stat( + 1, + "SELECT * FROM public.pg_custom_events WHERE account_id = 42", + 1000.0, + ); + + let candidates = build_candidates_for_usage(&stat, &usage, &catalog); + assert_eq!(candidates.candidates.len(), 1); + assert_eq!(candidates.candidates[0].table, "pg_custom_events"); + } + + #[test] + fn candidate_keeps_partial_index_candidates() { + let mut usage = QueryColumnUsage::default(); + usage.tables.push(TableRef { + schema: Some("public".into()), + name: "orders".into(), + }); + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into()], + ..TableColumnUsage::default() + }; + usage + .usage_by_table + .insert("public.orders".into(), table_usage); + + let mut partial_index = make_index_definition(&["customer_id"]); + partial_index.is_partial = true; + + let mut catalog = IndexCatalog::default(); + catalog + .indexes_by_table + .insert("public.orders".into(), vec![partial_index]); + + let stat = make_stat(1, "SELECT * FROM orders WHERE customer_id = $1", 1000.0); + let candidates = build_candidates_for_usage(&stat, &usage, &catalog); + assert_eq!(candidates.candidates.len(), 1); + } + + #[test] + fn candidate_keeps_when_include_columns_do_not_form_search_key() { + let mut usage = QueryColumnUsage::default(); + usage.tables.push(TableRef { + schema: Some("public".into()), + name: "orders".into(), + }); + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into(), "status".into()], + ..TableColumnUsage::default() + }; + usage + .usage_by_table + .insert("public.orders".into(), table_usage); + + let mut catalog = IndexCatalog::default(); + catalog.indexes_by_table.insert( + "public.orders".into(), + vec![make_index_definition(&["customer_id"])], + ); + + let stat = make_stat( + 1, + "SELECT * FROM orders WHERE customer_id = $1 AND status = $2", + 1000.0, + ); + let candidates = build_candidates_for_usage(&stat, &usage, &catalog); + assert_eq!(candidates.candidates.len(), 1); + } + + #[test] + fn candidate_keeps_when_order_sensitive_suffix_is_not_covered() { + let mut usage = QueryColumnUsage::default(); + usage.tables.push(TableRef { + schema: Some("public".into()), + name: "orders".into(), + }); + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into(), "status".into()], + orders: vec!["created_at".into()], + ..TableColumnUsage::default() + }; + usage + .usage_by_table + .insert("public.orders".into(), table_usage); + + let mut catalog = IndexCatalog::default(); + catalog.indexes_by_table.insert( + "public.orders".into(), + vec![make_index_definition(&[ + "status", + "created_at", + "customer_id", + ])], + ); + + let stat = make_stat( + 1, + "SELECT * FROM orders WHERE customer_id = $1 AND status = $2 ORDER BY created_at", + 1000.0, + ); + let candidates = build_candidates_for_usage(&stat, &usage, &catalog); + assert_eq!(candidates.candidates.len(), 1); + } + + #[test] + fn candidate_keeps_invalid_index_candidates() { + let mut usage = QueryColumnUsage::default(); + usage.tables.push(TableRef { + schema: Some("public".into()), + name: "orders".into(), + }); + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into()], + ..TableColumnUsage::default() + }; + usage + .usage_by_table + .insert("public.orders".into(), table_usage); + + let mut invalid_index = make_index_definition(&["customer_id"]); + invalid_index.is_valid = false; + + let mut catalog = IndexCatalog::default(); + catalog + .indexes_by_table + .insert("public.orders".into(), vec![invalid_index]); + + let stat = make_stat(1, "SELECT * FROM orders WHERE customer_id = $1", 1000.0); + let candidates = build_candidates_for_usage(&stat, &usage, &catalog); + assert_eq!(candidates.candidates.len(), 1); + } + + #[test] + fn candidate_keeps_expression_index_candidates() { + let mut usage = QueryColumnUsage::default(); + usage.tables.push(TableRef { + schema: Some("public".into()), + name: "orders".into(), + }); + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into()], + ..TableColumnUsage::default() }; + usage + .usage_by_table + .insert("public.orders".into(), table_usage); + + let mut expression_index = make_index_definition(&["customer_id"]); + expression_index.is_expression = true; + + let mut catalog = IndexCatalog::default(); + catalog + .indexes_by_table + .insert("public.orders".into(), vec![expression_index]); + let stat = make_stat(1, "SELECT * FROM orders WHERE customer_id = $1", 1000.0); let candidates = build_candidates_for_usage(&stat, &usage, &catalog); - assert_eq!(candidates.len(), 1); - assert_eq!(candidates[0].table, "pg_custom_events"); + assert_eq!(candidates.candidates.len(), 1); + } + + #[test] + fn statement_query_aggregates_calls_and_times() { + let query = build_statement_query( + TimeColumns { + total: "total_exec_time", + max: "max_exec_time", + }, + "total_time_ms", + true, + ); + assert!(query.contains("SUM(s.calls)::bigint AS calls")); + assert!(query.contains("SUM(s.total_exec_time) AS total_time_ms")); + assert!(query.contains("MAX(s.max_exec_time) AS max_time_ms")); + assert!(query.contains("SUM(COALESCE(s.wal_bytes, 0))::bigint AS wal_bytes")); + } + + #[test] + fn statement_query_groups_by_query_identity_and_aggregated_calls() { + let query = build_statement_query( + TimeColumns { + total: "total_exec_time", + max: "max_exec_time", + }, + "shared_blks_read", + false, + ); + assert!(query.contains("GROUP BY COALESCE(s.queryid, 0)::bigint, COALESCE(s.query, '')")); + assert!(query.contains("HAVING SUM(s.calls) >= $1")); + assert!(query.contains("ORDER BY shared_blks_read DESC")); + } + + #[test] + fn preload_error_message_is_classified_as_unavailable() { + assert!(is_pg_stat_statements_preload_error_message( + "pg_stat_statements must be loaded via shared_preload_libraries" + )); + } + + #[test] + fn fetch_index_catalog_query_limits_to_key_columns() { + assert!(FETCH_INDEX_CATALOG_QUERY.contains("arr.ord <= i.indnkeyatts")); + assert!(FETCH_INDEX_CATALOG_QUERY.contains("(i.indpred IS NOT NULL) AS is_partial")); + } + + #[test] + fn metadata_warnings_cover_recent_reset_deallocations_and_query_visibility() { + let snapshot = WorkloadMetadataSnapshot { + stats_reset_at: Some("2026-03-05 10:00:00+00".into()), + seconds_since_reset: Some(60.0), + entry_deallocations: Some(7), + query_text_visible: false, + ..WorkloadMetadataSnapshot::default() + }; + let mut results = WorkloadResults::default(); + add_metadata_warnings(&snapshot, &mut results); + + assert_eq!(results.warnings.len(), 3); + assert!(results + .warnings + .iter() + .any(|warning| warning.contains("last reset at"))); + assert!(results + .warnings + .iter() + .any(|warning| warning.contains("evicted 7 entries"))); + assert!(results + .warnings + .iter() + .any(|warning| warning.contains("pg_read_all_stats"))); + } + + #[test] + fn parse_failure_warning_triggers_for_non_trivial_failure_rate() { + let metadata = WorkloadMetadata { + parsed_queries: 7, + parse_failures: 3, + ..WorkloadMetadata::default() + }; + let mut results = WorkloadResults::default(); + add_parse_failure_warning(10, &metadata, &mut results); + + assert_eq!(results.warnings.len(), 1); + assert!(results.warnings[0].contains("index candidate coverage is partial")); + } + + #[test] + fn candidate_notes_include_ignored_index_types() { + let mut usage = QueryColumnUsage::default(); + usage.tables.push(TableRef { + schema: Some("public".into()), + name: "orders".into(), + }); + let table_usage = TableColumnUsage { + equality_filters: vec!["customer_id".into()], + ..TableColumnUsage::default() + }; + usage + .usage_by_table + .insert("public.orders".into(), table_usage); + + let mut partial_index = make_index_definition(&["customer_id"]); + partial_index.is_partial = true; + let mut expression_index = make_index_definition(&["customer_id"]); + expression_index.is_expression = true; + let mut invalid_index = make_index_definition(&["customer_id"]); + invalid_index.is_valid = false; + + let mut catalog = IndexCatalog::default(); + catalog.indexes_by_table.insert( + "public.orders".into(), + vec![partial_index, expression_index, invalid_index], + ); + + let stat = make_stat(1, "SELECT * FROM orders WHERE customer_id = $1", 1000.0); + let build = build_candidates_for_usage(&stat, &usage, &catalog); + + assert_eq!(build.candidates.len(), 1); + assert!(build.candidates[0] + .notes + .iter() + .any(|note| note.contains("partial index ignored"))); + assert!(build.candidates[0] + .notes + .iter() + .any(|note| note.contains("expression index ignored"))); + assert!(build.candidates[0] + .notes + .iter() + .any(|note| note.contains("invalid index ignored"))); + assert_eq!( + build.candidates[0].confidence, + WorkloadFindingConfidence::Medium + ); + } + + #[test] + fn correlate_table_health_adds_table_and_unused_index_notes() { + let mut results = WorkloadResults { + query_index_candidates: vec![QueryIndexCandidate { + schema: "public".into(), + table: "orders".into(), + columns: vec!["customer_id".into()], + reason: "heuristic".into(), + confidence: WorkloadFindingConfidence::High, + evidence: QueryIndexEvidence::default(), + notes: Vec::new(), + queryid: 1, + total_time_ms: 10.0, + mean_time_ms: 1.0, + calls: 10, + }], + seq_scan_info: vec![crate::models::TableSeqScanInfo { + schema: "public".into(), + table_name: "orders".into(), + seq_scan: 10, + idx_scan: 0, + live_tuples: 100, + table_size_bytes: 1024, + table_size_pretty: "1 kB".into(), + }], + bloat_info: vec![crate::models::TableBloatInfo { + schema: "public".into(), + table_name: "orders".into(), + live_tuples: 100, + dead_tuples: 10, + dead_tup_ratio: 0.1, + seq_scan: 10, + idx_scan: 0, + table_size_bytes: 1024, + table_size_pretty: "1 kB".into(), + last_autovacuum: None, + last_autoanalyze: None, + seconds_since_last_autovacuum: None, + seconds_since_last_autoanalyze: None, + }], + index_usage_info: vec![crate::models::IndexUsageInfo { + issue: IndexIssueKind::Unused, + schema: "public".into(), + table_name: "orders".into(), + index_name: "orders_customer_id_idx".into(), + key_columns: vec!["customer_id".into()], + index_size_bytes: 1024, + index_size_pretty: "1 kB".into(), + scans: 0, + tuples_read: 0, + tuples_fetched: 0, + avg_tuples_per_scan: 0.0, + heap_fetch_ratio: 0.0, + table_live_tup: Some(100), + is_unique: false, + enforces_constraint: false, + is_expression: false, + is_partial: false, + }], + ..WorkloadResults::default() + }; + + correlate_table_health(&mut results); + + assert!(results.query_index_candidates[0] + .notes + .iter() + .any(|note| note.contains("sequential scan hotspot"))); + assert!(results.query_index_candidates[0] + .notes + .iter() + .any(|note| note.contains("bloat watchlist"))); + assert!(results.query_index_candidates[0] + .notes + .iter() + .any(|note| note.contains("unused overlapping index"))); } } diff --git a/src/checker.rs b/src/checker.rs index db50f8c..e1659ee 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -89,7 +89,11 @@ impl ConfigChecker { } pub async fn analyze_workload(&mut self, opts: WorkloadOptions) -> Result { - let mut results = workload::analyze(&self.pool, &opts).await?; + let analysis = workload::analyze(&self.pool, &opts).await?; + let mut results = analysis.results; + if !analysis.available { + return Ok(results); + } info!("Running table and index health analysis..."); let mut table_results = AnalysisResults::default(); @@ -101,6 +105,7 @@ impl ConfigChecker { results.bloat_info = table_results.bloat_info; results.seq_scan_info = table_results.seq_scan_info; results.index_usage_info = table_results.index_usage_info; + workload::correlate_table_health(&mut results); } Ok(results) @@ -163,16 +168,31 @@ impl ConfigChecker { Err(err) => warn!("Failed to read pg_stat_activity for connection count: {err}"), } - // Fetch checkpoint stats for WAL analysis - match sqlx::query("SELECT checkpoints_timed, checkpoints_req FROM pg_stat_bgwriter") - .fetch_one(&self.pool) - .await + // PostgreSQL 17+ exposes checkpoint counters in pg_stat_checkpointer. + match sqlx::query( + "SELECT num_timed AS checkpoints_timed, num_requested AS checkpoints_req FROM pg_stat_checkpointer", + ) + .fetch_one(&self.pool) + .await { Ok(row) => { stats.checkpoints_timed = row.try_get("checkpoints_timed").ok(); stats.checkpoints_req = row.try_get("checkpoints_req").ok(); } - Err(err) => warn!("Failed to read pg_stat_bgwriter: {err}"), + Err(new_err) => match sqlx::query( + "SELECT checkpoints_timed, checkpoints_req FROM pg_stat_bgwriter", + ) + .fetch_one(&self.pool) + .await + { + Ok(row) => { + stats.checkpoints_timed = row.try_get("checkpoints_timed").ok(); + stats.checkpoints_req = row.try_get("checkpoints_req").ok(); + } + Err(old_err) => warn!( + "Failed to read checkpoint stats from pg_stat_checkpointer ({new_err}) or pg_stat_bgwriter ({old_err})" + ), + }, } // Use provided compute spec if available diff --git a/src/main.rs b/src/main.rs index 9c013f3..1520b36 100644 --- a/src/main.rs +++ b/src/main.rs @@ -74,7 +74,7 @@ enum Commands { #[arg(short = 'c', long = "config")] config_path: String, }, - /// Analyze workload performance using pg_stat_statements + /// Analyze workload performance using pg_stat_statements (must be installed and usable) Workload { /// Database host #[arg( @@ -135,7 +135,7 @@ async fn main() -> anyhow::Result<()> { tracing_subscriber::EnvFilter::try_from_default_env() .unwrap_or_else(|_| log_level.into()), ) - .with(tracing_subscriber::fmt::layer()) + .with(tracing_subscriber::fmt::layer().with_writer(std::io::stderr)) .init(); match cli.command { diff --git a/src/models.rs b/src/models.rs index 7d825dd..36b2bf8 100644 --- a/src/models.rs +++ b/src/models.rs @@ -109,6 +109,7 @@ pub struct IndexUsageInfo { pub schema: String, pub table_name: String, pub index_name: String, + pub key_columns: Vec, pub index_size_bytes: i64, pub index_size_pretty: String, pub scans: i64, @@ -224,9 +225,40 @@ pub struct SlowQueryInfo { pub shared_blks_hit: i64, pub temp_blks_read: i64, pub temp_blks_written: i64, + pub total_time_pct: f64, + pub cache_hit_ratio: Option, + pub temp_blks_written_per_call: Option, + pub wal_bytes: Option, + pub wal_bytes_per_call: Option, pub query_text: String, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WorkloadFindingConfidence { + High, + Medium, + Low, +} + +impl WorkloadFindingConfidence { + pub fn as_str(&self) -> &'static str { + match self { + WorkloadFindingConfidence::High => "high", + WorkloadFindingConfidence::Medium => "medium", + WorkloadFindingConfidence::Low => "low", + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct QueryIndexEvidence { + pub equality_filters: Vec, + pub non_equality_filters: Vec, + pub equality_joins: Vec, + pub order_by: Vec, +} + /// Represents a heuristic index candidate derived from slow queries. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct QueryIndexCandidate { @@ -234,15 +266,58 @@ pub struct QueryIndexCandidate { pub table: String, pub columns: Vec, pub reason: String, + pub confidence: WorkloadFindingConfidence, + pub evidence: QueryIndexEvidence, + pub notes: Vec, pub queryid: i64, pub total_time_ms: f64, pub mean_time_ms: f64, pub calls: i64, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WorkloadMetadata { + pub data_source: String, + pub scope: String, + pub stats_reset_at: Option, + pub entry_deallocations: Option, + pub server_version: Option, + pub query_text_visible: bool, + pub parsed_queries: usize, + pub parse_failures: usize, + pub suppressed_candidates: usize, +} + +impl Default for WorkloadMetadata { + fn default() -> Self { + Self { + data_source: "pg_stat_statements".into(), + scope: "cumulative_since_reset".into(), + stats_reset_at: None, + entry_deallocations: None, + server_version: None, + query_text_visible: true, + parsed_queries: 0, + parse_failures: 0, + suppressed_candidates: 0, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct WorkloadCoverageStats { + pub suppressed_by_existing_index: usize, + pub skipped_internal_tables: usize, + pub skipped_unresolved_schema: usize, + pub skipped_unsupported_parse_shape: usize, + pub parser_errors: usize, +} + /// Workload analysis results for slow query and index candidate reporting. #[derive(Debug, Clone, Serialize, Deserialize, Default)] pub struct WorkloadResults { + pub workload_metadata: WorkloadMetadata, + pub coverage_stats: WorkloadCoverageStats, pub slow_query_groups: Vec, pub query_index_candidates: Vec, pub index_usage_info: Vec, diff --git a/src/reporter.rs b/src/reporter.rs index b267509..402f5ea 100644 --- a/src/reporter.rs +++ b/src/reporter.rs @@ -603,24 +603,24 @@ impl WorkloadReporter { } fn report_markdown(&self, results: &WorkloadResults) -> Result<()> { - use std::io::Write; - let stdout = std::io::stdout(); let mut handle = stdout.lock(); + self.write_workload_markdown(&mut handle, results) + } + fn write_workload_markdown( + &self, + handle: &mut W, + results: &WorkloadResults, + ) -> Result<()> { writeln!(handle, "# PostgreSQL Workload Analysis Report\n").context(OutputSnafu)?; + self.write_workload_summary_markdown(handle, results)?; - writeln!(handle, "## Summary\n").context(OutputSnafu)?; - if results.warnings.is_empty() { - writeln!(handle, "- **Warnings**: None").context(OutputSnafu)?; - } else { - for warning in &results.warnings { - writeln!(handle, "- **Warning**: {}", warning).context(OutputSnafu)?; - } - } - writeln!(handle, "- **Parse failures**: {}", results.parse_failures) - .context(OutputSnafu)?; - writeln!(handle).context(OutputSnafu)?; + let show_wal = results + .slow_query_groups + .iter() + .flat_map(|group| group.queries.iter()) + .any(|query| query.wal_bytes_per_call.is_some()); for group in &results.slow_query_groups { writeln!(handle, "## {}\n", format_slow_query_kind(group.kind)).context(OutputSnafu)?; @@ -630,32 +630,70 @@ impl WorkloadReporter { continue; } - writeln!( - handle, - "| Query ID | Calls | Total ms | Mean ms | Max ms | Rows | Shared Read | Temp Written | Query |" - ) - .context(OutputSnafu)?; - writeln!( - handle, - "|---------|-------|----------|---------|--------|------|-------------|--------------|-------|" - ) - .context(OutputSnafu)?; - for query in &group.queries { + if show_wal { writeln!( handle, - "| {} | {} | {:.2} | {:.2} | {:.2} | {} | {} | {} | {} |", - query.queryid, - query.calls, - query.total_time_ms, - query.mean_time_ms, - query.max_time_ms, - query.rows, - query.shared_blks_read, - query.temp_blks_written, - query.query_text.replace('|', "\\|") + "| Query ID | Calls | Total ms | % Total | Mean ms | Max ms | Rows | Shared Read | Temp Written | Cache Hit % | Temp/call | WAL/call | Query |" + ) + .context(OutputSnafu)?; + writeln!( + handle, + "|---------|-------|----------|---------|---------|--------|------|-------------|--------------|-------------|-----------|----------|-------|" + ) + .context(OutputSnafu)?; + } else { + writeln!( + handle, + "| Query ID | Calls | Total ms | % Total | Mean ms | Max ms | Rows | Shared Read | Temp Written | Cache Hit % | Temp/call | Query |" + ) + .context(OutputSnafu)?; + writeln!( + handle, + "|---------|-------|----------|---------|---------|--------|------|-------------|--------------|-------------|-----------|-------|" ) .context(OutputSnafu)?; } + + for query in &group.queries { + if show_wal { + writeln!( + handle, + "| {} | {} | {:.2} | {:.1}% | {:.2} | {:.2} | {} | {} | {} | {} | {} | {} | {} |", + query.queryid, + query.calls, + query.total_time_ms, + query.total_time_pct, + query.mean_time_ms, + query.max_time_ms, + query.rows, + query.shared_blks_read, + query.temp_blks_written, + format_optional_pct(query.cache_hit_ratio), + format_optional_f64(query.temp_blks_written_per_call, " blocks"), + format_optional_i64_per_call(query.wal_bytes_per_call, " bytes"), + query.query_text.replace('|', "\\|") + ) + .context(OutputSnafu)?; + } else { + writeln!( + handle, + "| {} | {} | {:.2} | {:.1}% | {:.2} | {:.2} | {} | {} | {} | {} | {} | {} |", + query.queryid, + query.calls, + query.total_time_ms, + query.total_time_pct, + query.mean_time_ms, + query.max_time_ms, + query.rows, + query.shared_blks_read, + query.temp_blks_written, + format_optional_pct(query.cache_hit_ratio), + format_optional_f64(query.temp_blks_written_per_call, " blocks"), + query.query_text.replace('|', "\\|") + ) + .context(OutputSnafu)?; + } + } writeln!(handle).context(OutputSnafu)?; } @@ -663,25 +701,28 @@ impl WorkloadReporter { writeln!(handle, "## Index Candidates (Heuristic)\n").context(OutputSnafu)?; writeln!( handle, - "| Table | Columns | Calls | Total ms | Mean ms | Query ID | Reason |" + "| Table | Columns | Confidence | Calls | Total ms | Mean ms | Query ID | Evidence | Notes | Reason |" ) .context(OutputSnafu)?; writeln!( handle, - "|-------|---------|-------|----------|---------|----------|--------|" + "|-------|---------|------------|-------|----------|---------|----------|----------|-------|--------|" ) .context(OutputSnafu)?; for candidate in &results.query_index_candidates { writeln!( handle, - "| {}.{} | {} | {} | {:.2} | {:.2} | {} | {} |", + "| {}.{} | {} | {} | {} | {:.2} | {:.2} | {} | {} | {} | {} |", candidate.schema, candidate.table, candidate.columns.join(", "), + candidate.confidence.as_str(), candidate.calls, candidate.total_time_ms, candidate.mean_time_ms, candidate.queryid, + format_candidate_evidence(&candidate.evidence).replace('|', "\\|"), + format_notes(&candidate.notes).replace('|', "\\|"), candidate.reason.replace('|', "\\|") ) .context(OutputSnafu)?; @@ -699,13 +740,104 @@ impl WorkloadReporter { Ok(()) } - fn write_table_index_markdown( + fn write_workload_summary_markdown( &self, - mut handle: std::io::StdoutLock, + handle: &mut W, results: &WorkloadResults, ) -> Result<()> { - use std::io::Write; + writeln!(handle, "## Summary\n").context(OutputSnafu)?; + writeln!( + handle, + "- **Data source**: `{}`", + results.workload_metadata.data_source + ) + .context(OutputSnafu)?; + writeln!(handle, "- **Scope**: `{}`", results.workload_metadata.scope) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Stats reset at**: {}", + results + .workload_metadata + .stats_reset_at + .as_deref() + .unwrap_or("unknown") + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Entry deallocations**: {}", + results + .workload_metadata + .entry_deallocations + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Server version**: {}", + results + .workload_metadata + .server_version + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Query text visible**: {}", + if results.workload_metadata.query_text_visible { + "yes" + } else { + "no" + } + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Parsed queries**: {}", + results.workload_metadata.parsed_queries + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Parse failures**: {}", + results.workload_metadata.parse_failures + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Suppressed candidates**: {}", + results.workload_metadata.suppressed_candidates + ) + .context(OutputSnafu)?; + writeln!( + handle, + "- **Coverage summary**: {} suppressed by existing indexes, {} internal tables skipped, {} unresolved-schema tables skipped, {} unsupported parse shapes, {} parser errors", + results.coverage_stats.suppressed_by_existing_index, + results.coverage_stats.skipped_internal_tables, + results.coverage_stats.skipped_unresolved_schema, + results.coverage_stats.skipped_unsupported_parse_shape, + results.coverage_stats.parser_errors + ) + .context(OutputSnafu)?; + if results.warnings.is_empty() { + writeln!(handle, "- **Warnings**: None").context(OutputSnafu)?; + } else { + for warning in &results.warnings { + writeln!(handle, "- **Warning**: {}", warning).context(OutputSnafu)?; + } + } + writeln!(handle).context(OutputSnafu)?; + Ok(()) + } + fn write_table_index_markdown( + &self, + handle: &mut W, + results: &WorkloadResults, + ) -> Result<()> { writeln!(handle, "## Table & Index Health\n").context(OutputSnafu)?; if !results.bloat_info.is_empty() { @@ -830,9 +962,22 @@ impl WorkloadReporter { } fn report_json(&self, results: &WorkloadResults) -> Result<()> { - use std::io::Write; let stdout = std::io::stdout(); let mut handle = stdout.lock(); + self.write_workload_json(&mut handle, results) + } + + fn report_text(&self, results: &WorkloadResults) -> Result<()> { + let stdout = std::io::stdout(); + let mut handle = stdout.lock(); + self.write_workload_text(&mut handle, results) + } + + fn write_workload_json( + &self, + handle: &mut W, + results: &WorkloadResults, + ) -> Result<()> { let json = serde_json::to_string_pretty(results).map_err(|err| ReporterError::OutputError { source: std::io::Error::other(err), @@ -841,18 +986,71 @@ impl WorkloadReporter { Ok(()) } - fn report_text(&self, results: &WorkloadResults) -> Result<()> { - use std::io::Write; - let stdout = std::io::stdout(); - let mut handle = stdout.lock(); - + fn write_workload_text( + &self, + handle: &mut W, + results: &WorkloadResults, + ) -> Result<()> { writeln!(handle, "PostgreSQL Workload Analysis Report").context(OutputSnafu)?; + writeln!( + handle, + "Data source: {} ({})", + results.workload_metadata.data_source, results.workload_metadata.scope + ) + .context(OutputSnafu)?; + writeln!( + handle, + "Stats reset at: {}", + results + .workload_metadata + .stats_reset_at + .as_deref() + .unwrap_or("unknown") + ) + .context(OutputSnafu)?; + writeln!( + handle, + "Entry deallocations: {}", + results + .workload_metadata + .entry_deallocations + .map(|value| value.to_string()) + .unwrap_or_else(|| "unknown".to_string()) + ) + .context(OutputSnafu)?; + writeln!( + handle, + "Query text visible: {}", + if results.workload_metadata.query_text_visible { + "yes" + } else { + "no" + } + ) + .context(OutputSnafu)?; + writeln!( + handle, + "Parsed queries: {}, parse failures: {}, suppressed candidates: {}", + results.workload_metadata.parsed_queries, + results.workload_metadata.parse_failures, + results.workload_metadata.suppressed_candidates + ) + .context(OutputSnafu)?; + writeln!( + handle, + "Coverage summary: {} suppressed, {} internal, {} unresolved-schema, {} unsupported shapes, {} parser errors", + results.coverage_stats.suppressed_by_existing_index, + results.coverage_stats.skipped_internal_tables, + results.coverage_stats.skipped_unresolved_schema, + results.coverage_stats.skipped_unsupported_parse_shape, + results.coverage_stats.parser_errors + ) + .context(OutputSnafu)?; if !results.warnings.is_empty() { for warning in &results.warnings { writeln!(handle, "Warning: {warning}").context(OutputSnafu)?; } } - writeln!(handle, "Parse failures: {}", results.parse_failures).context(OutputSnafu)?; writeln!(handle).context(OutputSnafu)?; for group in &results.slow_query_groups { @@ -861,10 +1059,20 @@ impl WorkloadReporter { for query in &group.queries { writeln!( handle, - " - {} calls, total {:.2}ms, mean {:.2}ms, queryid {}", - query.calls, query.total_time_ms, query.mean_time_ms, query.queryid + " - {} calls, total {:.2}ms ({:.1}% of measured total), mean {:.2}ms, cache hit {}, temp/call {}, queryid {}", + query.calls, + query.total_time_ms, + query.total_time_pct, + query.mean_time_ms, + format_optional_pct(query.cache_hit_ratio), + format_optional_f64(query.temp_blks_written_per_call, " blocks"), + query.queryid ) .context(OutputSnafu)?; + if let Some(wal_bytes_per_call) = query.wal_bytes_per_call { + writeln!(handle, " WAL/call: {:.1} bytes", wal_bytes_per_call) + .context(OutputSnafu)?; + } } writeln!(handle).context(OutputSnafu)?; } @@ -874,12 +1082,23 @@ impl WorkloadReporter { for candidate in &results.query_index_candidates { writeln!( handle, - " - {}.{} ({})", + " - {}.{} ({}) [{}]", candidate.schema, candidate.table, - candidate.columns.join(", ") + candidate.columns.join(", "), + candidate.confidence.as_str() ) .context(OutputSnafu)?; + writeln!( + handle, + " evidence: {}", + format_candidate_evidence(&candidate.evidence) + ) + .context(OutputSnafu)?; + if !candidate.notes.is_empty() { + writeln!(handle, " notes: {}", format_notes(&candidate.notes)) + .context(OutputSnafu)?; + } } writeln!(handle).context(OutputSnafu)?; } @@ -915,6 +1134,56 @@ impl WorkloadReporter { } } +fn format_candidate_evidence(evidence: &crate::models::QueryIndexEvidence) -> String { + let mut parts = Vec::new(); + if !evidence.equality_filters.is_empty() { + parts.push(format!("WHERE = {}", evidence.equality_filters.join(", "))); + } + if !evidence.non_equality_filters.is_empty() { + parts.push(format!( + "WHERE range {}", + evidence.non_equality_filters.join(", ") + )); + } + if !evidence.equality_joins.is_empty() { + parts.push(format!("JOIN = {}", evidence.equality_joins.join(", "))); + } + if !evidence.order_by.is_empty() { + parts.push(format!("ORDER BY {}", evidence.order_by.join(", "))); + } + if parts.is_empty() { + "none".to_string() + } else { + parts.join("; ") + } +} + +fn format_notes(notes: &[String]) -> String { + if notes.is_empty() { + "none".to_string() + } else { + notes.join("; ") + } +} + +fn format_optional_pct(value: Option) -> String { + value + .map(|ratio| format!("{:.1}%", ratio * 100.0)) + .unwrap_or_else(|| "n/a".to_string()) +} + +fn format_optional_f64(value: Option, unit: &str) -> String { + value + .map(|value| format!("{value:.1}{unit}")) + .unwrap_or_else(|| "n/a".to_string()) +} + +fn format_optional_i64_per_call(value: Option, unit: &str) -> String { + value + .map(|value| format!("{value:.1}{unit}")) + .unwrap_or_else(|| "n/a".to_string()) +} + fn format_slow_query_kind(kind: SlowQueryKind) -> &'static str { match kind { SlowQueryKind::TotalTime => "Slow Queries by Total Time", @@ -927,16 +1196,16 @@ fn format_slow_query_kind(kind: SlowQueryKind) -> &'static str { fn describe_slow_query_kind(kind: SlowQueryKind) -> &'static str { match kind { SlowQueryKind::TotalTime => { - "Shows which statements consume the most cumulative execution time across all calls, useful for finding the biggest overall throughput drains." + "Shows which statements consume the most cumulative execution time since pg_stat_statements was last reset; this is not a time-windowed ranking." } SlowQueryKind::MeanTime => { - "Shows which statements are slow per execution, useful for reducing end-user latency and fixing expensive query plans." + "Shows which statements are slow per execution within the cumulative pg_stat_statements dataset, useful for reducing end-user latency and fixing expensive query plans." } SlowQueryKind::SharedBlksRead => { - "Highlights statements that perform the most disk-backed reads into shared buffers, useful for spotting I/O-heavy access patterns and missing indexes." + "Highlights statements that perform the most disk-backed reads in the cumulative pg_stat_statements dataset, useful for spotting I/O-heavy access patterns and missing indexes." } SlowQueryKind::TempBlksWritten => { - "Highlights statements that spill the most temporary blocks, useful for identifying costly sort/hash operations and memory pressure." + "Highlights statements that spill the most temporary blocks in the cumulative pg_stat_statements dataset, useful for identifying costly sort/hash operations and memory pressure." } } } @@ -959,3 +1228,156 @@ fn selectivity_ratio(index: &crate::models::IndexUsageInfo) -> f64 { index.avg_tuples_per_scan / table_rows } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::{ + QueryIndexCandidate, QueryIndexEvidence, SlowQueryGroup, SlowQueryInfo, + WorkloadCoverageStats, WorkloadFindingConfidence, WorkloadMetadata, + }; + + fn sample_workload_results() -> WorkloadResults { + WorkloadResults { + workload_metadata: WorkloadMetadata { + stats_reset_at: Some("2026-03-05 10:00:00+00".into()), + entry_deallocations: Some(7), + server_version: Some(160004), + query_text_visible: false, + parsed_queries: 7, + parse_failures: 3, + suppressed_candidates: 2, + ..WorkloadMetadata::default() + }, + coverage_stats: WorkloadCoverageStats { + suppressed_by_existing_index: 2, + skipped_internal_tables: 1, + skipped_unresolved_schema: 1, + skipped_unsupported_parse_shape: 2, + parser_errors: 3, + }, + slow_query_groups: vec![SlowQueryGroup { + kind: SlowQueryKind::TotalTime, + queries: vec![SlowQueryInfo { + queryid: 42, + calls: 10, + total_time_ms: 500.0, + mean_time_ms: 50.0, + max_time_ms: 100.0, + rows: 25, + shared_blks_read: 10, + shared_blks_hit: 90, + temp_blks_read: 0, + temp_blks_written: 20, + total_time_pct: 62.5, + cache_hit_ratio: Some(0.9), + temp_blks_written_per_call: Some(2.0), + wal_bytes: Some(2_048), + wal_bytes_per_call: Some(204.8), + query_text: "select * from orders where customer_id = $1".into(), + }], + }], + query_index_candidates: vec![QueryIndexCandidate { + schema: "public".into(), + table: "orders".into(), + columns: vec!["customer_id".into(), "created_at".into()], + reason: "heuristic from slow query: WHERE customer_id; ORDER BY created_at".into(), + confidence: WorkloadFindingConfidence::Low, + evidence: QueryIndexEvidence { + equality_filters: vec!["customer_id".into()], + non_equality_filters: Vec::new(), + equality_joins: Vec::new(), + order_by: vec!["created_at".into()], + }, + notes: vec![ + "table name resolved to public, but another schema may contain the same table" + .into(), + "table is also a sequential scan hotspot".into(), + ], + queryid: 42, + total_time_ms: 500.0, + mean_time_ms: 50.0, + calls: 10, + }], + warnings: vec![ + "Workload results are cumulative only since pg_stat_statements was last reset at 2026-03-05 10:00:00+00.".into(), + "pg_stat_statements has evicted 7 entries due to capacity pressure; low-frequency statements and derived findings may be incomplete.".into(), + "Query text visibility appears limited for the current role; grant pg_read_all_stats to avoid incomplete or anonymized workload findings.".into(), + "Only 7 of 10 workload statements were parsed into index evidence; index candidate coverage is partial.".into(), + ], + ..WorkloadResults::default() + } + } + + #[test] + fn workload_markdown_snapshot_includes_metadata_and_candidate_notes() { + let reporter = WorkloadReporter::new(ReportFormat::Markdown); + let results = sample_workload_results(); + let mut output = Vec::new(); + + reporter + .write_workload_markdown(&mut output, &results) + .expect("markdown workload report should render"); + + let rendered = String::from_utf8(output).expect("markdown should be utf8"); + assert!(rendered.contains("# PostgreSQL Workload Analysis Report")); + assert!(rendered.contains("- **Data source**: `pg_stat_statements`")); + assert!(rendered.contains("- **Entry deallocations**: 7")); + assert!(rendered.contains("- **Query text visible**: no")); + assert!( + rendered.contains("Only 7 of 10 workload statements were parsed into index evidence") + ); + assert!(rendered.contains("| public.orders | customer_id, created_at | low |")); + assert!(rendered.contains("table is also a sequential scan hotspot")); + } + + #[test] + fn workload_markdown_reports_none_when_warnings_absent() { + let reporter = WorkloadReporter::new(ReportFormat::Markdown); + let mut results = WorkloadResults::default(); + results.workload_metadata.parsed_queries = 1; + let mut output = Vec::new(); + + reporter + .write_workload_markdown(&mut output, &results) + .expect("markdown workload report should render"); + + let rendered = String::from_utf8(output).expect("markdown should be utf8"); + assert!(rendered.contains("- **Warnings**: None")); + } + + #[test] + fn workload_text_snapshot_includes_wal_and_coverage_summary() { + let reporter = WorkloadReporter::new(ReportFormat::Text); + let results = sample_workload_results(); + let mut output = Vec::new(); + + reporter + .write_workload_text(&mut output, &results) + .expect("text workload report should render"); + + let rendered = String::from_utf8(output).expect("text should be utf8"); + assert!(rendered.contains("Coverage summary: 2 suppressed, 1 internal, 1 unresolved-schema, 2 unsupported shapes, 3 parser errors")); + assert!(rendered.contains("WAL/call: 204.8 bytes")); + assert!(rendered.contains("evidence: WHERE = customer_id; ORDER BY created_at")); + } + + #[test] + fn workload_json_snapshot_includes_metadata_and_evidence_fields() { + let reporter = WorkloadReporter::new(ReportFormat::Json); + let results = sample_workload_results(); + let mut output = Vec::new(); + + reporter + .write_workload_json(&mut output, &results) + .expect("json workload report should render"); + + let rendered = String::from_utf8(output).expect("json should be utf8"); + assert!(rendered.contains("\"workload_metadata\"")); + assert!(rendered.contains("\"scope\": \"cumulative_since_reset\"")); + assert!(rendered.contains("\"query_text_visible\": false")); + assert!(rendered.contains("\"confidence\": \"low\"")); + assert!(rendered.contains("\"equality_filters\": [")); + assert!(rendered.contains("\"notes\": [")); + } +} diff --git a/tests/_data/00-extensions-and-roles.sql b/tests/_data/00-extensions-and-roles.sql new file mode 100644 index 0000000..832c61d --- /dev/null +++ b/tests/_data/00-extensions-and-roles.sql @@ -0,0 +1,17 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; + +DO $$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'app_user') THEN + CREATE ROLE app_user LOGIN PASSWORD 'app_password'; + END IF; + + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'reader_user') THEN + CREATE ROLE reader_user LOGIN PASSWORD 'reader_password'; + END IF; +END $$; + +GRANT USAGE ON SCHEMA public TO app_user, reader_user; +GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA public TO app_user; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public TO app_user; +GRANT SELECT ON ALL TABLES IN SCHEMA public TO reader_user; diff --git a/tests/_data/2-bloat-and-indexes.sql b/tests/_data/2-bloat-and-indexes.sql index cade82b..7bec86a 100644 --- a/tests/_data/2-bloat-and-indexes.sql +++ b/tests/_data/2-bloat-and-indexes.sql @@ -16,11 +16,6 @@ BEGIN UPDATE rental SET last_update = NOW() WHERE rental_id % 10 = 0; - - -- Delete some rows to create dead tuples that won't be re-used immediately - IF i % 10 = 0 THEN - DELETE FROM rental WHERE rental_id % 100 = 1; - END IF; END LOOP; END $$; diff --git a/tests/_data/3-workload.sql b/tests/_data/3-workload.sql new file mode 100644 index 0000000..59c60d9 --- /dev/null +++ b/tests/_data/3-workload.sql @@ -0,0 +1,66 @@ +-- 3-workload.sql +-- Deterministic workload used after pg_stat_statements_reset() in live integration tests. + +-- ================================================================ +-- 1. Repeated non-indexed filter on rental.return_date +-- ================================================================ +DO $$ +BEGIN + FOR i IN 1..20 LOOP + PERFORM rental_id + FROM rental + WHERE return_date > '2005-05-25 00:00:00'::timestamp; + END LOOP; +END $$; + +-- ================================================================ +-- 2. Equality filter + ORDER BY to exercise structured evidence +-- ================================================================ +DO $$ +BEGIN + FOR i IN 1..12 LOOP + PERFORM payment_id + FROM payment + WHERE customer_id = 42 + ORDER BY payment_date DESC + LIMIT 25; + END LOOP; +END $$; + +-- ================================================================ +-- 3. Temp-heavy join and aggregation workload +-- ================================================================ +DO $$ +BEGIN + FOR i IN 1..8 LOOP + PERFORM * + FROM ( + SELECT + c.customer_id, + SUM(p.amount) AS total_payment, + COUNT(r.rental_id) AS total_rentals + FROM customer c + JOIN payment p ON c.customer_id = p.customer_id + JOIN rental r ON p.rental_id = r.rental_id + JOIN inventory i ON r.inventory_id = i.inventory_id + JOIN film f ON i.film_id = f.film_id + WHERE f.description LIKE '%Action%' + OR f.description LIKE '%Drama%' + GROUP BY c.customer_id + HAVING SUM(p.amount) > 50 + ORDER BY total_payment DESC + ) AS temp_heavy_workload; + END LOOP; +END $$; + +-- ================================================================ +-- 4. Write-heavy updates to populate WAL metrics +-- ================================================================ +DO $$ +BEGIN + FOR i IN 1..12 LOOP + UPDATE rental + SET last_update = NOW() + make_interval(secs => i) + WHERE rental_id BETWEEN 1 AND 100; + END LOOP; +END $$; diff --git a/tests/it_analyze.rs b/tests/it_analyze.rs new file mode 100644 index 0000000..bf4cd28 --- /dev/null +++ b/tests/it_analyze.rs @@ -0,0 +1,39 @@ +mod support; + +use support::{analyze_snapshot_view, parse_json_output, ContainerProfile, TestPostgres, TestRole}; + +#[test] +#[ignore = "requires Docker"] +fn analyze_happy_path_json_snapshot() { + let server = TestPostgres::start(ContainerProfile::WorkloadEnabled); + let db = server.create_test_database("analyze"); + server.apply_table_index_fixture(&db); + + let assert = server + .analyze_command(&db, TestRole::Admin) + .assert() + .success(); + let json = parse_json_output(&assert.get_output().stdout); + + assert!( + json["bloat_info"] + .as_array() + .is_some_and(|entries| !entries.is_empty()), + "expected seeded bloat findings" + ); + assert!( + json["seq_scan_info"] + .as_array() + .is_some_and(|entries| !entries.is_empty()), + "expected seeded sequential scan findings" + ); + assert!( + json["index_usage_info"] + .as_array() + .is_some_and(|entries| !entries.is_empty()), + "expected seeded index findings" + ); + + let snapshot_name = format!("analyze_pg{}", server.version_tag()); + insta::assert_json_snapshot!(snapshot_name, analyze_snapshot_view(&json)); +} diff --git a/tests/it_workload.rs b/tests/it_workload.rs new file mode 100644 index 0000000..ce2f535 --- /dev/null +++ b/tests/it_workload.rs @@ -0,0 +1,73 @@ +mod support; + +use support::{ + parse_json_output, workload_happy_path_snapshot_view, ContainerProfile, TestPostgres, TestRole, +}; + +#[test] +#[ignore = "requires Docker"] +fn workload_happy_path_json_snapshot() { + let server = TestPostgres::start(ContainerProfile::WorkloadEnabled); + let db = server.create_test_database("workload"); + server.apply_table_index_fixture(&db); + server.reset_pg_stat_statements(&db); + server.run_workload_fixture_as_app(&db); + + let assert = server + .workload_command(&db, TestRole::Admin) + .assert() + .success(); + let json = parse_json_output(&assert.get_output().stdout); + + assert_eq!( + json["workload_metadata"]["data_source"], + "pg_stat_statements" + ); + assert_eq!(json["workload_metadata"]["scope"], "cumulative_since_reset"); + assert!( + json["coverage_stats"].is_object(), + "expected coverage stats" + ); + assert_eq!( + json["workload_metadata"]["query_text_visible"], + serde_json::Value::Bool(true) + ); + + let candidates = json["query_index_candidates"] + .as_array() + .expect("query_index_candidates should be an array"); + assert!(!candidates.is_empty(), "expected workload index candidates"); + assert!( + candidates.iter().any(|candidate| { + candidate["confidence"].is_string() + && candidate["evidence"].is_object() + && candidate["notes"].is_array() + }), + "expected candidate confidence, evidence, and notes" + ); + assert!( + candidates.iter().any(|candidate| { + candidate["notes"].as_array().is_some_and(|notes| { + notes.iter().any(|note| { + note.as_str().is_some_and(|note| { + note.contains("sequential scan hotspot") || note.contains("bloat watchlist") + }) + }) + }) + }), + "expected candidate correlation with seeded table/index health findings" + ); + assert!( + json["warnings"] + .as_array() + .is_some_and(|warnings| warnings.iter().any(|warning| { + warning + .as_str() + .is_some_and(|warning| warning.contains("last reset at")) + })), + "expected reset-scope warning" + ); + + let snapshot_name = format!("workload_pg{}", server.version_tag()); + insta::assert_json_snapshot!(snapshot_name, workload_happy_path_snapshot_view(&json)); +} diff --git a/tests/it_workload_dealloc.rs b/tests/it_workload_dealloc.rs new file mode 100644 index 0000000..e57ff50 --- /dev/null +++ b/tests/it_workload_dealloc.rs @@ -0,0 +1,42 @@ +mod support; + +use support::{ + parse_json_output, workload_dealloc_snapshot_view, ContainerProfile, TestPostgres, TestRole, +}; + +#[test] +#[ignore = "requires Docker"] +fn workload_deallocation_warning_json_snapshot() { + let server = TestPostgres::start(ContainerProfile::WorkloadEnabledLowMax); + let db = server.create_test_database("workload_dealloc"); + server.reset_pg_stat_statements(&db); + server.generate_distinct_queries_as_app(&db, 256); + + let assert = server + .workload_command(&db, TestRole::Admin) + .assert() + .success(); + let json = parse_json_output(&assert.get_output().stdout); + let diagnostics = server.pg_stat_statements_diagnostics(&db); + + let entry_deallocations = json["workload_metadata"]["entry_deallocations"] + .as_i64() + .expect("entry_deallocations should be reported"); + assert!( + entry_deallocations > 0, + "expected statement deallocations; diagnostics: {diagnostics}" + ); + assert!( + json["warnings"] + .as_array() + .is_some_and(|warnings| warnings.iter().any(|warning| { + warning + .as_str() + .is_some_and(|warning| warning.contains("evicted")) + })), + "expected deallocation warning" + ); + + let snapshot_name = format!("workload_dealloc_pg{}", server.version_tag()); + insta::assert_json_snapshot!(snapshot_name, workload_dealloc_snapshot_view(&json)); +} diff --git a/tests/it_workload_unavailable.rs b/tests/it_workload_unavailable.rs new file mode 100644 index 0000000..a968141 --- /dev/null +++ b/tests/it_workload_unavailable.rs @@ -0,0 +1,80 @@ +mod support; + +use support::{ + parse_json_output, workload_unavailable_snapshot_view, ContainerProfile, TestPostgres, TestRole, +}; + +#[test] +#[ignore = "requires Docker"] +fn workload_unavailable_scenarios() { + let enabled_server = TestPostgres::start(ContainerProfile::WorkloadEnabled); + let no_extension_db = enabled_server.create_test_database("workload_no_extension"); + enabled_server.drop_pg_stat_statements_extension(&no_extension_db); + + let no_extension_assert = enabled_server + .workload_command(&no_extension_db, TestRole::Admin) + .assert() + .success(); + let no_extension_json = parse_json_output(&no_extension_assert.get_output().stdout); + + assert!( + no_extension_json["warnings"] + .as_array() + .is_some_and(|warnings| warnings.iter().any(|warning| { + warning + .as_str() + .is_some_and(|warning| warning.contains("not installed")) + })), + "expected missing-extension warning" + ); + assert!( + no_extension_json["slow_query_groups"] + .as_array() + .is_some_and(|groups| groups.is_empty()), + "expected warning-only workload result when extension is missing" + ); + + let missing_snapshot = format!( + "workload_unavailable_missing_pg{}", + enabled_server.version_tag() + ); + insta::assert_json_snapshot!( + missing_snapshot, + workload_unavailable_snapshot_view(&no_extension_json) + ); + + let no_preload_server = TestPostgres::start(ContainerProfile::NoPreload); + let no_preload_db = no_preload_server.create_test_database("workload_no_preload"); + + let no_preload_assert = no_preload_server + .workload_command(&no_preload_db, TestRole::Admin) + .assert() + .success(); + let no_preload_json = parse_json_output(&no_preload_assert.get_output().stdout); + + assert!( + no_preload_json["warnings"] + .as_array() + .is_some_and(|warnings| warnings.iter().any(|warning| { + warning + .as_str() + .is_some_and(|warning| warning.contains("not usable")) + })), + "expected preload warning" + ); + assert!( + no_preload_json["slow_query_groups"] + .as_array() + .is_some_and(|groups| groups.is_empty()), + "expected warning-only workload result when preload is missing" + ); + + let preload_snapshot = format!( + "workload_unavailable_preload_pg{}", + no_preload_server.version_tag() + ); + insta::assert_json_snapshot!( + preload_snapshot, + workload_unavailable_snapshot_view(&no_preload_json) + ); +} diff --git a/tests/it_workload_visibility.rs b/tests/it_workload_visibility.rs new file mode 100644 index 0000000..efef0c6 --- /dev/null +++ b/tests/it_workload_visibility.rs @@ -0,0 +1,39 @@ +mod support; + +use support::{ + parse_json_output, workload_visibility_snapshot_view, ContainerProfile, TestPostgres, TestRole, +}; + +#[test] +#[ignore = "requires Docker"] +fn workload_visibility_warning_json_snapshot() { + let server = TestPostgres::start(ContainerProfile::WorkloadEnabled); + let db = server.create_test_database("workload_visibility"); + server.apply_table_index_fixture(&db); + server.reset_pg_stat_statements(&db); + server.run_workload_fixture_as_app(&db); + + let assert = server + .workload_command(&db, TestRole::Reader) + .assert() + .success(); + let json = parse_json_output(&assert.get_output().stdout); + + assert_eq!( + json["workload_metadata"]["query_text_visible"], + serde_json::Value::Bool(false) + ); + assert!( + json["warnings"] + .as_array() + .is_some_and(|warnings| warnings.iter().any(|warning| { + warning + .as_str() + .is_some_and(|warning| warning.contains("pg_read_all_stats")) + })), + "expected reduced visibility warning" + ); + + let snapshot_name = format!("workload_visibility_pg{}", server.version_tag()); + insta::assert_json_snapshot!(snapshot_name, workload_visibility_snapshot_view(&json)); +} diff --git a/tests/snapshots/it_analyze__analyze_pg14.snap b/tests/snapshots/it_analyze__analyze_pg14.snap new file mode 100644 index 0000000..73d17f7 --- /dev/null +++ b/tests/snapshots/it_analyze__analyze_pg14.snap @@ -0,0 +1,9 @@ +--- +source: tests/it_analyze.rs +expression: analyze_snapshot_view(&json) +--- +{ + "has_failed_index_only_finding": true, + "has_rental_bloat": true, + "has_rental_seq_scan": true +} diff --git a/tests/snapshots/it_analyze__analyze_pg18.snap b/tests/snapshots/it_analyze__analyze_pg18.snap new file mode 100644 index 0000000..73d17f7 --- /dev/null +++ b/tests/snapshots/it_analyze__analyze_pg18.snap @@ -0,0 +1,9 @@ +--- +source: tests/it_analyze.rs +expression: analyze_snapshot_view(&json) +--- +{ + "has_failed_index_only_finding": true, + "has_rental_bloat": true, + "has_rental_seq_scan": true +} diff --git a/tests/snapshots/it_workload__workload_pg14.snap b/tests/snapshots/it_workload__workload_pg14.snap new file mode 100644 index 0000000..7028039 --- /dev/null +++ b/tests/snapshots/it_workload__workload_pg14.snap @@ -0,0 +1,25 @@ +--- +source: tests/it_workload.rs +expression: workload_happy_path_snapshot_view(&json) +--- +{ + "has_non_equality_candidate": true, + "has_rental_candidate": true, + "has_table_health_note": true, + "metadata": { + "data_source": "pg_stat_statements", + "entry_deallocations_state": "zero", + "query_text_visible": true, + "scope": "cumulative_since_reset", + "stats_reset_state": "available" + }, + "slow_query_kinds": [ + "mean_time", + "shared_blks_read", + "temp_blks_written", + "total_time" + ], + "warning_categories": [ + "recent_reset" + ] +} diff --git a/tests/snapshots/it_workload__workload_pg18.snap b/tests/snapshots/it_workload__workload_pg18.snap new file mode 100644 index 0000000..7028039 --- /dev/null +++ b/tests/snapshots/it_workload__workload_pg18.snap @@ -0,0 +1,25 @@ +--- +source: tests/it_workload.rs +expression: workload_happy_path_snapshot_view(&json) +--- +{ + "has_non_equality_candidate": true, + "has_rental_candidate": true, + "has_table_health_note": true, + "metadata": { + "data_source": "pg_stat_statements", + "entry_deallocations_state": "zero", + "query_text_visible": true, + "scope": "cumulative_since_reset", + "stats_reset_state": "available" + }, + "slow_query_kinds": [ + "mean_time", + "shared_blks_read", + "temp_blks_written", + "total_time" + ], + "warning_categories": [ + "recent_reset" + ] +} diff --git a/tests/snapshots/it_workload_dealloc__workload_dealloc_pg14.snap b/tests/snapshots/it_workload_dealloc__workload_dealloc_pg14.snap new file mode 100644 index 0000000..58fc54b --- /dev/null +++ b/tests/snapshots/it_workload_dealloc__workload_dealloc_pg14.snap @@ -0,0 +1,23 @@ +--- +source: tests/it_workload_dealloc.rs +expression: workload_dealloc_snapshot_view(&json) +--- +{ + "metadata": { + "data_source": "pg_stat_statements", + "entry_deallocations_state": "nonzero", + "query_text_visible": true, + "scope": "cumulative_since_reset", + "stats_reset_state": "available" + }, + "slow_query_kinds": [ + "mean_time", + "shared_blks_read", + "temp_blks_written", + "total_time" + ], + "warning_categories": [ + "deallocations", + "recent_reset" + ] +} diff --git a/tests/snapshots/it_workload_dealloc__workload_dealloc_pg18.snap b/tests/snapshots/it_workload_dealloc__workload_dealloc_pg18.snap new file mode 100644 index 0000000..58fc54b --- /dev/null +++ b/tests/snapshots/it_workload_dealloc__workload_dealloc_pg18.snap @@ -0,0 +1,23 @@ +--- +source: tests/it_workload_dealloc.rs +expression: workload_dealloc_snapshot_view(&json) +--- +{ + "metadata": { + "data_source": "pg_stat_statements", + "entry_deallocations_state": "nonzero", + "query_text_visible": true, + "scope": "cumulative_since_reset", + "stats_reset_state": "available" + }, + "slow_query_kinds": [ + "mean_time", + "shared_blks_read", + "temp_blks_written", + "total_time" + ], + "warning_categories": [ + "deallocations", + "recent_reset" + ] +} diff --git a/tests/snapshots/it_workload_unavailable__workload_unavailable_missing_pg14.snap b/tests/snapshots/it_workload_unavailable__workload_unavailable_missing_pg14.snap new file mode 100644 index 0000000..cbc1404 --- /dev/null +++ b/tests/snapshots/it_workload_unavailable__workload_unavailable_missing_pg14.snap @@ -0,0 +1,12 @@ +--- +source: tests/it_workload_unavailable.rs +expression: workload_unavailable_snapshot_view(&no_extension_json) +--- +{ + "candidate_count": 0, + "slow_query_group_count": 0, + "stats_reset_state": "unavailable", + "warning_categories": [ + "extension_missing" + ] +} diff --git a/tests/snapshots/it_workload_unavailable__workload_unavailable_missing_pg18.snap b/tests/snapshots/it_workload_unavailable__workload_unavailable_missing_pg18.snap new file mode 100644 index 0000000..cbc1404 --- /dev/null +++ b/tests/snapshots/it_workload_unavailable__workload_unavailable_missing_pg18.snap @@ -0,0 +1,12 @@ +--- +source: tests/it_workload_unavailable.rs +expression: workload_unavailable_snapshot_view(&no_extension_json) +--- +{ + "candidate_count": 0, + "slow_query_group_count": 0, + "stats_reset_state": "unavailable", + "warning_categories": [ + "extension_missing" + ] +} diff --git a/tests/snapshots/it_workload_unavailable__workload_unavailable_preload_pg14.snap b/tests/snapshots/it_workload_unavailable__workload_unavailable_preload_pg14.snap new file mode 100644 index 0000000..01f2daf --- /dev/null +++ b/tests/snapshots/it_workload_unavailable__workload_unavailable_preload_pg14.snap @@ -0,0 +1,12 @@ +--- +source: tests/it_workload_unavailable.rs +expression: workload_unavailable_snapshot_view(&no_preload_json) +--- +{ + "candidate_count": 0, + "slow_query_group_count": 0, + "stats_reset_state": "unavailable", + "warning_categories": [ + "preload_unavailable" + ] +} diff --git a/tests/snapshots/it_workload_unavailable__workload_unavailable_preload_pg18.snap b/tests/snapshots/it_workload_unavailable__workload_unavailable_preload_pg18.snap new file mode 100644 index 0000000..01f2daf --- /dev/null +++ b/tests/snapshots/it_workload_unavailable__workload_unavailable_preload_pg18.snap @@ -0,0 +1,12 @@ +--- +source: tests/it_workload_unavailable.rs +expression: workload_unavailable_snapshot_view(&no_preload_json) +--- +{ + "candidate_count": 0, + "slow_query_group_count": 0, + "stats_reset_state": "unavailable", + "warning_categories": [ + "preload_unavailable" + ] +} diff --git a/tests/snapshots/it_workload_visibility__workload_visibility_pg14.snap b/tests/snapshots/it_workload_visibility__workload_visibility_pg14.snap new file mode 100644 index 0000000..ed359ba --- /dev/null +++ b/tests/snapshots/it_workload_visibility__workload_visibility_pg14.snap @@ -0,0 +1,23 @@ +--- +source: tests/it_workload_visibility.rs +expression: workload_visibility_snapshot_view(&json) +--- +{ + "metadata": { + "data_source": "pg_stat_statements", + "entry_deallocations_state": "zero", + "query_text_visible": false, + "scope": "cumulative_since_reset", + "stats_reset_state": "available" + }, + "slow_query_kinds": [ + "mean_time", + "shared_blks_read", + "temp_blks_written", + "total_time" + ], + "warning_categories": [ + "limited_visibility", + "recent_reset" + ] +} diff --git a/tests/snapshots/it_workload_visibility__workload_visibility_pg18.snap b/tests/snapshots/it_workload_visibility__workload_visibility_pg18.snap new file mode 100644 index 0000000..ed359ba --- /dev/null +++ b/tests/snapshots/it_workload_visibility__workload_visibility_pg18.snap @@ -0,0 +1,23 @@ +--- +source: tests/it_workload_visibility.rs +expression: workload_visibility_snapshot_view(&json) +--- +{ + "metadata": { + "data_source": "pg_stat_statements", + "entry_deallocations_state": "zero", + "query_text_visible": false, + "scope": "cumulative_since_reset", + "stats_reset_state": "available" + }, + "slow_query_kinds": [ + "mean_time", + "shared_blks_read", + "temp_blks_written", + "total_time" + ], + "warning_categories": [ + "limited_visibility", + "recent_reset" + ] +} diff --git a/tests/support/mod.rs b/tests/support/mod.rs new file mode 100644 index 0000000..e3b778c --- /dev/null +++ b/tests/support/mod.rs @@ -0,0 +1,546 @@ +#![allow(dead_code)] + +use assert_cmd::{cargo::cargo_bin_cmd, Command}; +use serde_json::{json, Value}; +use sqlx::{postgres::PgPoolOptions, raw_sql, Pool, Postgres}; +use std::env; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use testcontainers_modules::{ + postgres, + testcontainers::{runners::SyncRunner, Container, ImageExt}, +}; + +const TEMPLATE_DB: &str = "postgreat_template"; +const ADMIN_USER: &str = "postgres"; +const ADMIN_PASSWORD: &str = "postgres"; +const APP_USER: &str = "app_user"; +const APP_PASSWORD: &str = "app_password"; +const READER_USER: &str = "reader_user"; +const READER_PASSWORD: &str = "reader_password"; + +const FIXTURE_SCHEMA_SQL: &str = include_str!("../_data/0-schema.sql"); +const FIXTURE_DATA_SQL: &str = include_str!("../_data/1-data.sql"); +const FIXTURE_BLOAT_SQL: &str = include_str!("../_data/2-bloat-and-indexes.sql"); +const FIXTURE_ROLES_SQL: &str = include_str!("../_data/00-extensions-and-roles.sql"); +const FIXTURE_WORKLOAD_SQL: &str = include_str!("../_data/3-workload.sql"); + +#[derive(Debug, Clone, Copy)] +pub enum ContainerProfile { + WorkloadEnabled, + WorkloadEnabledLowMax, + NoPreload, +} + +#[derive(Debug, Clone, Copy)] +pub enum TestRole { + Admin, + App, + Reader, +} + +pub struct TestPostgres { + _container: Container, + runtime: tokio::runtime::Runtime, + host: String, + port: u16, + version_tag: String, +} + +#[derive(Debug, Clone)] +pub struct TestDatabase { + host: String, + port: u16, + name: String, +} + +impl TestPostgres { + pub fn start(profile: ContainerProfile) -> Self { + let version_tag = env::var("POSTGREAT_TEST_PG_VERSION").unwrap_or_else(|_| "18".into()); + let image = postgres::Postgres::default() + .with_init_sql(template_init_sql()) + .with_tag(version_tag.as_str()) + .with_cmd(container_cmd(profile)) + .with_startup_timeout(Duration::from_secs(120)); + let container = image.start().expect("postgres test container should start"); + let host = container + .get_host() + .expect("postgres host should resolve") + .to_string(); + let port = container + .get_host_port_ipv4(5432) + .expect("postgres port should resolve"); + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime should start"); + + let server = Self { + _container: container, + runtime, + host, + port, + version_tag, + }; + server.seed_template_database(); + server + } + + pub fn version_tag(&self) -> &str { + &self.version_tag + } + + pub fn create_test_database(&self, suffix: &str) -> TestDatabase { + let db_name = format!( + "pgtest_{}_{}_{}", + sanitize_ident(suffix), + sanitize_ident(self.version_tag()), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock should be after unix epoch") + .as_millis() + ); + let create_sql = format!( + "CREATE DATABASE {} TEMPLATE {}", + quote_ident(&db_name), + quote_ident(TEMPLATE_DB) + ); + let grant_sql = format!( + "GRANT CONNECT ON DATABASE {} TO {}, {}", + quote_ident(&db_name), + quote_ident(APP_USER), + quote_ident(READER_USER) + ); + + self.runtime.block_on(async { + let admin_pool = self.admin_pool("postgres").await; + sqlx::query(&create_sql) + .execute(&admin_pool) + .await + .expect("test database should be created from template"); + sqlx::query(&grant_sql) + .execute(&admin_pool) + .await + .expect("test roles should receive CONNECT on cloned database"); + }); + + TestDatabase { + host: self.host.clone(), + port: self.port, + name: db_name, + } + } + + pub fn apply_table_index_fixture(&self, db: &TestDatabase) { + self.execute_sql(db, TestRole::Admin, FIXTURE_BLOAT_SQL); + self.execute_sql(db, TestRole::Admin, "ANALYZE;"); + } + + pub fn run_workload_fixture_as_app(&self, db: &TestDatabase) { + self.execute_sql(db, TestRole::App, FIXTURE_WORKLOAD_SQL); + } + + pub fn reset_pg_stat_statements(&self, db: &TestDatabase) { + self.execute_sql(db, TestRole::Admin, "SELECT pg_stat_statements_reset();"); + } + + pub fn drop_pg_stat_statements_extension(&self, db: &TestDatabase) { + self.execute_sql( + db, + TestRole::Admin, + "DROP EXTENSION IF EXISTS pg_stat_statements;", + ); + } + + pub fn generate_distinct_queries_as_app(&self, db: &TestDatabase, count: usize) { + self.runtime.block_on(async { + let pool = self.pool_for_role(db, TestRole::App).await; + for query_shape in 1..=count { + let mut predicates = Vec::new(); + for value in 1..=query_shape { + predicates.push(format!("rental_id = {value}")); + } + let sql = format!( + "SELECT COUNT(*) FROM rental WHERE {};", + predicates.join(" OR ") + ); + sqlx::query(&sql) + .execute(&pool) + .await + .expect("distinct workload query should execute"); + } + }); + } + + pub fn pg_stat_statements_diagnostics(&self, db: &TestDatabase) -> Value { + self.runtime.block_on(async { + let pool = self.pool_for_role(db, TestRole::Admin).await; + let max_entries: i64 = sqlx::query_scalar( + "SELECT current_setting('pg_stat_statements.max')::bigint", + ) + .fetch_one(&pool) + .await + .expect("pg_stat_statements.max should be readable"); + let statement_count: i64 = sqlx::query_scalar("SELECT count(*)::bigint FROM pg_stat_statements") + .fetch_one(&pool) + .await + .expect("pg_stat_statements count should be readable"); + let deallocations: i64 = + sqlx::query_scalar("SELECT COALESCE(dealloc, 0)::bigint FROM pg_stat_statements_info") + .fetch_one(&pool) + .await + .expect("pg_stat_statements_info should be readable"); + json!({ + "pg_stat_statements_max": max_entries, + "statement_count": statement_count, + "deallocations": deallocations, + }) + }) + } + + pub fn analyze_command(&self, db: &TestDatabase, role: TestRole) -> Command { + let mut command = cargo_bin_cmd!("postgreat"); + let credentials = db.credentials(role); + command.args([ + "--format", + "json", + "analyze", + "--host", + db.host(), + "--port", + &db.port().to_string(), + "--database", + db.name(), + "--username", + credentials.0, + "--password", + credentials.1, + "--compute", + "8vCPU-64GB", + ]); + command + } + + pub fn workload_command(&self, db: &TestDatabase, role: TestRole) -> Command { + let mut command = cargo_bin_cmd!("postgreat"); + let credentials = db.credentials(role); + command.args([ + "--format", + "json", + "workload", + "--host", + db.host(), + "--port", + &db.port().to_string(), + "--database", + db.name(), + "--username", + credentials.0, + "--password", + credentials.1, + "--limit", + "10", + "--min-calls", + "1", + "--include-full-query", + ]); + command + } + + fn seed_template_database(&self) { + self.runtime.block_on(async { + let template_pool = self.admin_pool(TEMPLATE_DB).await; + sqlx::query("SELECT 1") + .execute(&template_pool) + .await + .expect("template database should be ready after init SQL"); + }); + } + + fn execute_sql(&self, db: &TestDatabase, role: TestRole, sql: &str) { + self.runtime.block_on(async { + let pool = self.pool_for_role(db, role).await; + raw_sql(sql) + .execute(&pool) + .await + .expect("fixture SQL should execute"); + }); + } + + async fn admin_pool(&self, database: &str) -> Pool { + let url = format!( + "postgres://{}:{}@{}:{}/{}", + ADMIN_USER, ADMIN_PASSWORD, self.host, self.port, database + ); + PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("admin pool should connect") + } + + async fn pool_for_role(&self, db: &TestDatabase, role: TestRole) -> Pool { + let credentials = db.credentials(role); + let url = format!( + "postgres://{}:{}@{}:{}/{}", + credentials.0, credentials.1, db.host, db.port, db.name + ); + PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .expect("role pool should connect") + } +} + +impl TestDatabase { + pub fn host(&self) -> &str { + &self.host + } + + pub fn port(&self) -> u16 { + self.port + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn credentials(&self, role: TestRole) -> (&'static str, &'static str) { + match role { + TestRole::Admin => (ADMIN_USER, ADMIN_PASSWORD), + TestRole::App => (APP_USER, APP_PASSWORD), + TestRole::Reader => (READER_USER, READER_PASSWORD), + } + } +} + +pub fn parse_json_output(output: &[u8]) -> Value { + serde_json::from_slice(output).unwrap_or_else(|err| { + panic!( + "CLI output should be valid JSON: {err}; stdout was: {}", + String::from_utf8_lossy(output) + ) + }) +} + +pub fn analyze_snapshot_view(value: &Value) -> Value { + json!({ + "has_rental_bloat": has_table_named(&value["bloat_info"], "rental"), + "has_rental_seq_scan": has_table_named(&value["seq_scan_info"], "rental"), + "has_failed_index_only_finding": has_issue_kind(&value["index_usage_info"], "failed_index_only"), + }) +} + +pub fn workload_happy_path_snapshot_view(value: &Value) -> Value { + json!({ + "metadata": workload_metadata_summary(value), + "warning_categories": warning_categories(&value["warnings"]), + "slow_query_kinds": slow_query_kinds(value), + "has_rental_candidate": has_candidate_for_table(value, "rental"), + "has_non_equality_candidate": has_non_equality_candidate(value), + "has_table_health_note": has_table_health_note(value), + }) +} + +pub fn workload_visibility_snapshot_view(value: &Value) -> Value { + json!({ + "metadata": workload_metadata_summary(value), + "warning_categories": warning_categories(&value["warnings"]), + "slow_query_kinds": slow_query_kinds(value), + }) +} + +pub fn workload_dealloc_snapshot_view(value: &Value) -> Value { + json!({ + "metadata": workload_metadata_summary(value), + "warning_categories": warning_categories(&value["warnings"]), + "slow_query_kinds": slow_query_kinds(value), + }) +} + +pub fn workload_unavailable_snapshot_view(value: &Value) -> Value { + json!({ + "warning_categories": warning_categories(&value["warnings"]), + "stats_reset_state": stats_reset_state(value), + "slow_query_group_count": value["slow_query_groups"] + .as_array() + .map_or(0, Vec::len), + "candidate_count": value["query_index_candidates"] + .as_array() + .map_or(0, Vec::len), + }) +} + +fn container_cmd(profile: ContainerProfile) -> Vec { + let mut args = vec!["-c".into()]; + match profile { + ContainerProfile::WorkloadEnabled => { + args.push("shared_preload_libraries=pg_stat_statements".into()); + } + ContainerProfile::WorkloadEnabledLowMax => { + args.push("shared_preload_libraries=pg_stat_statements".into()); + args.extend(["-c".into(), "pg_stat_statements.max=100".into()]); + } + ContainerProfile::NoPreload => { + args.push("log_min_messages=warning".into()); + } + } + + if !matches!(profile, ContainerProfile::NoPreload) { + args.extend([ + "-c".into(), + "compute_query_id=on".into(), + "-c".into(), + "pg_stat_statements.track=all".into(), + ]); + } + + args +} + +fn template_init_sql() -> Vec { + format!( + "CREATE DATABASE {template_db};\n\\connect {template_db}\n{schema}\n{data}\nSET search_path = public;\n{roles}\nANALYZE;\n", + template_db = TEMPLATE_DB, + schema = FIXTURE_SCHEMA_SQL, + data = FIXTURE_DATA_SQL, + roles = FIXTURE_ROLES_SQL, + ) + .into_bytes() +} + +fn has_table_named(value: &Value, table_name: &str) -> bool { + value + .as_array() + .into_iter() + .flatten() + .any(|entry| entry["table_name"].as_str() == Some(table_name)) +} + +fn has_issue_kind(value: &Value, issue: &str) -> bool { + value + .as_array() + .into_iter() + .flatten() + .any(|entry| entry["issue"].as_str() == Some(issue)) +} + +fn workload_metadata_summary(value: &Value) -> Value { + json!({ + "data_source": value["workload_metadata"]["data_source"], + "scope": value["workload_metadata"]["scope"], + "query_text_visible": value["workload_metadata"]["query_text_visible"], + "stats_reset_state": stats_reset_state(value), + "entry_deallocations_state": entry_deallocations_state(value), + }) +} + +fn stats_reset_state(value: &Value) -> &'static str { + if value["workload_metadata"]["stats_reset_at"].is_null() { + "unavailable" + } else { + "available" + } +} + +fn entry_deallocations_state(value: &Value) -> &'static str { + match value["workload_metadata"]["entry_deallocations"].as_i64() { + Some(0) => "zero", + Some(_) => "nonzero", + None => "unavailable", + } +} + +fn warning_categories(value: &Value) -> Vec { + let mut categories = value + .as_array() + .into_iter() + .flatten() + .filter_map(|warning| { + let warning = warning.as_str()?; + let category = if warning.contains("last reset at") { + Some("recent_reset") + } else if warning.contains("pg_read_all_stats") { + Some("limited_visibility") + } else if warning.contains("evicted") { + Some("deallocations") + } else if warning.contains("not installed") { + Some("extension_missing") + } else if warning.contains("not usable") { + Some("preload_unavailable") + } else { + None + }?; + Some(Value::String(category.to_string())) + }) + .collect::>(); + categories.sort_by(|left, right| left.as_str().cmp(&right.as_str())); + categories.dedup(); + categories +} + +fn slow_query_kinds(value: &Value) -> Vec { + let mut kinds = value["slow_query_groups"] + .as_array() + .into_iter() + .flatten() + .filter_map(|group| { + group["kind"] + .as_str() + .map(|kind| Value::String(kind.to_string())) + }) + .collect::>(); + kinds.sort_by(|left, right| left.as_str().cmp(&right.as_str())); + kinds.dedup(); + kinds +} + +fn has_candidate_for_table(value: &Value, table_name: &str) -> bool { + value["query_index_candidates"] + .as_array() + .into_iter() + .flatten() + .any(|candidate| candidate["table"].as_str() == Some(table_name)) +} + +fn has_non_equality_candidate(value: &Value) -> bool { + value["query_index_candidates"] + .as_array() + .into_iter() + .flatten() + .any(|candidate| { + candidate["evidence"]["non_equality_filters"] + .as_array() + .is_some_and(|filters| !filters.is_empty()) + }) +} + +fn has_table_health_note(value: &Value) -> bool { + value["query_index_candidates"] + .as_array() + .into_iter() + .flatten() + .any(|candidate| { + candidate["notes"].as_array().is_some_and(|notes| { + notes.iter().any(|note| { + note.as_str().is_some_and(|note| { + note.contains("sequential scan hotspot") || note.contains("bloat watchlist") + }) + }) + }) + }) +} + +fn quote_ident(value: &str) -> String { + format!("\"{}\"", value.replace('"', "\"\"")) +} + +fn sanitize_ident(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '_' + } + }) + .collect() +}