diff --git a/.rustfmt.toml b/.rustfmt.toml index 5207c2aa..d01043dc 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -7,3 +7,5 @@ use_small_heuristics = "Default" # Stable options only reorder_imports = true reorder_modules = true + +imports_granularity = "Module" diff --git a/Cargo.lock b/Cargo.lock index 5ce80744..31cc22ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -481,9 +481,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.58" +version = "1.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", "jobserver", @@ -616,9 +616,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" [[package]] name = "colorchoice" @@ -878,9 +878,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1005a6d4446f5120ef475ad3d2af2b30c49c2c9c6904258e3bb30219bebed5e4" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" dependencies = [ "cmov", ] @@ -1031,7 +1031,7 @@ dependencies = [ [[package]] name = "dojo-introspect" version = "0.1.0" -source = "git+https://github.com/dojoengine/dojo-introspect?rev=aadc3c9#aadc3c980706596a4a083413813a0a3ab01fded7" +source = "git+https://github.com/dojoengine/dojo-introspect?rev=c45d711#c45d711236c856bc60d567e43965b2043d5cb2b0" dependencies = [ "async-trait", "introspect-rust-macros", @@ -1476,7 +1476,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.13.0", + "indexmap 2.13.1", "slab", "tokio", "tokio-util", @@ -1972,9 +1972,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -1989,7 +1989,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.13.0", + "indexmap 2.13.1", "is-terminal", "itoa", "log", @@ -2012,7 +2012,7 @@ dependencies = [ [[package]] name = "introspect-events" version = "0.1.2" -source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" +source = "git+https://github.com/cartridge-gg/introspect?rev=b89ee9e#b89ee9ead02df170195bb4f67a48e4308edab855" dependencies = [ "cainome-cairo-serde", "introspect-types", @@ -2024,7 +2024,7 @@ dependencies = [ [[package]] name = "introspect-rust-macros" version = "0.1.0" -source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" +source = "git+https://github.com/cartridge-gg/introspect?rev=b89ee9e#b89ee9ead02df170195bb4f67a48e4308edab855" dependencies = [ "paste", "proc-macro2", @@ -2036,7 +2036,7 @@ dependencies = [ [[package]] name = "introspect-types" version = "0.1.2" -source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" +source = "git+https://github.com/cartridge-gg/introspect?rev=b89ee9e#b89ee9ead02df170195bb4f67a48e4308edab855" dependencies = [ "blake3", "convert_case", @@ -2332,7 +2332,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap 2.13.0", + "indexmap 2.13.1", "ipnet", "metrics", "metrics-util", @@ -2718,7 +2718,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.13.0", + "indexmap 2.13.1", ] [[package]] @@ -2728,7 +2728,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.13.0", + "indexmap 2.13.1", ] [[package]] @@ -3723,9 +3723,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -3814,7 +3814,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.0", + "indexmap 2.13.1", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -3988,8 +3988,7 @@ dependencies = [ [[package]] name = "sqlx" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "sqlx-core", "sqlx-macros", @@ -4001,8 +4000,7 @@ dependencies = [ [[package]] name = "sqlx-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "base64 0.22.1", "bigdecimal", @@ -4017,7 +4015,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink 0.10.0", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "memchr", "once_cell", @@ -4038,8 +4036,7 @@ dependencies = [ [[package]] name = "sqlx-macros" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "proc-macro2", "quote", @@ -4051,8 +4048,7 @@ dependencies = [ [[package]] name = "sqlx-macros-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "dotenvy", "either", @@ -4076,8 +4072,7 @@ dependencies = [ [[package]] name = "sqlx-mysql" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "atoi", "base64 0.22.1", @@ -4119,8 +4114,7 @@ dependencies = [ [[package]] name = "sqlx-postgres" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "atoi", "base64 0.22.1", @@ -4158,8 +4152,7 @@ dependencies = [ [[package]] name = "sqlx-sqlite" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "atoi", "flume", @@ -4242,7 +4235,7 @@ dependencies = [ "flate2", "foldhash 0.1.5", "hex", - "indexmap 2.13.0", + "indexmap 2.13.1", "num-traits", "serde", "serde_json", @@ -4614,9 +4607,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" dependencies = [ "bytes", "libc", @@ -4631,9 +4624,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -4738,7 +4731,7 @@ version = "0.25.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.13.1", "toml_datetime", "toml_parser", "winnow", @@ -4872,8 +4865,10 @@ dependencies = [ "torii-erc20", "torii-erc721", "torii-introspect", - "torii-introspect-postgres-sink", + "torii-introspect-sql-sink", "torii-log-sink", + "torii-pathfinder", + "torii-sql", "torii-sql-sink", "torii-test-utils", "tower 0.5.3", @@ -4901,15 +4896,13 @@ dependencies = [ "torii-config-common", "torii-dojo", "torii-ecs-sink", - "torii-entities-historical-sink", "torii-erc1155", "torii-erc20", "torii-erc721", - "torii-introspect-postgres-sink", - "torii-introspect-sqlite-sink", + "torii-introspect-sql-sink", "torii-pathfinder", "torii-runtime-common", - "torii-sqlite", + "torii-sql", "tracing", "tracing-subscriber", "url", @@ -4945,11 +4938,9 @@ dependencies = [ "anyhow", "async-trait", "base64 0.22.1", - "itertools 0.14.0", "reqwest", "serde", "serde_json", - "sqlx", "starknet", "tokio", "tracing", @@ -4979,6 +4970,7 @@ dependencies = [ "tokio", "torii", "torii-runtime-common", + "torii-sql", "tracing", ] @@ -5003,8 +4995,7 @@ dependencies = [ "torii", "torii-common", "torii-introspect", - "torii-postgres", - "torii-sqlite", + "torii-sql", "tracing", ] @@ -5037,25 +5028,7 @@ dependencies = [ "torii-dojo", "torii-introspect", "torii-runtime-common", - "tracing", -] - -[[package]] -name = "torii-entities-historical-sink" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "introspect-types", - "serde", - "serde_json", - "sqlx", - "starknet", - "thiserror 2.0.18", - "tokio", - "torii", - "torii-introspect", - "torii-runtime-common", + "torii-sql", "tracing", ] @@ -5126,6 +5099,7 @@ dependencies = [ "torii", "torii-erc20", "torii-runtime-common", + "torii-sql", "tracing", "tracing-subscriber", "url", @@ -5197,6 +5171,7 @@ dependencies = [ "tonic-build", "torii", "torii-common", + "torii-sql", ] [[package]] @@ -5217,46 +5192,19 @@ dependencies = [ "torii-controllers-sink", "torii-dojo", "torii-ecs-sink", - "torii-entities-historical-sink", "torii-erc1155", "torii-erc20", "torii-erc721", - "torii-introspect-postgres-sink", - "torii-introspect-sqlite-sink", + "torii-introspect-sql-sink", "torii-runtime-common", - "torii-sqlite", + "torii-sql", "tracing", "tracing-subscriber", "url", ] [[package]] -name = "torii-introspect-postgres-sink" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "hex", - "introspect-types", - "itertools 0.14.0", - "metrics", - "serde", - "serde_json", - "sqlx", - "starknet", - "starknet-types-core", - "thiserror 2.0.18", - "tokio", - "torii", - "torii-common", - "torii-introspect", - "torii-postgres", - "tracing", - "xxhash-rust", -] - -[[package]] -name = "torii-introspect-sqlite-sink" +name = "torii-introspect-sql-sink" version = "0.1.0" dependencies = [ "anyhow", @@ -5275,8 +5223,9 @@ dependencies = [ "tokio", "torii", "torii-introspect", - "torii-sqlite", + "torii-sql", "tracing", + "xxhash-rust", ] [[package]] @@ -5297,8 +5246,7 @@ dependencies = [ "tokio", "torii", "torii-dojo", - "torii-introspect-postgres-sink", - "torii-runtime-common", + "torii-introspect-sql-sink", "tracing", "tracing-subscriber", ] @@ -5341,35 +5289,28 @@ dependencies = [ ] [[package]] -name = "torii-postgres" +name = "torii-runtime-common" version = "0.1.0" dependencies = [ "anyhow", - "async-trait", - "crc", - "futures", - "hex", - "serde", - "serde_json", - "sqlx", - "starknet-types-core", - "thiserror 2.0.18", "tokio", + "tokio-postgres", "torii", - "torii-common", + "torii-sql", "tracing", - "xxhash-rust", ] [[package]] -name = "torii-runtime-common" +name = "torii-sql" version = "0.1.0" dependencies = [ - "anyhow", - "tokio", - "tokio-postgres", - "torii", - "tracing", + "async-trait", + "crc", + "futures", + "itertools 0.14.0", + "log", + "sqlx", + "starknet-types-core", ] [[package]] @@ -5396,16 +5337,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "torii-sqlite" -version = "0.1.0" -dependencies = [ - "async-trait", - "futures", - "sqlx", - "torii-common", -] - [[package]] name = "torii-starknet" version = "0.1.0" @@ -5452,6 +5383,7 @@ dependencies = [ "torii-erc20", "torii-erc721", "torii-runtime-common", + "torii-sql", "tracing", "tracing-subscriber", "url", @@ -5968,7 +5900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.13.1", "wasm-encoder", "wasmparser", ] @@ -5981,7 +5913,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.0", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap 2.13.1", "semver", ] @@ -6406,7 +6338,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.0", + "indexmap 2.13.1", "prettyplease", "syn", "wasm-metadata", @@ -6437,7 +6369,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.0", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "serde", "serde_derive", @@ -6456,7 +6388,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "semver", "serde", @@ -6468,9 +6400,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" diff --git a/Cargo.toml b/Cargo.toml index 7043f6e0..c574efb6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,17 +9,14 @@ members = [ "crates/torii-controllers-sink", "crates/arcade-sink", "crates/torii-ecs-sink", - "crates/torii-entities-historical-sink", "crates/torii-erc20", "crates/torii-erc721", "crates/torii-erc1155", + "crates/sql", "crates/introspect", "crates/dojo", - "crates/introspect-postgres-sink", - "crates/introspect-sqlite-sink", + "crates/introspect-sql-sink", "crates/testing", - "crates/postgres", - "crates/sqlite", "crates/pathfinder", "crates/starknet", "bins/torii-erc20", @@ -72,6 +69,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } metrics = "0.24" metrics-exporter-prometheus = "0.17" +log = "0.4.29" # CLI clap = { version = "4.5", features = ["derive"] } @@ -80,12 +78,9 @@ clap = { version = "4.5", features = ["derive"] } starknet = "0.17" # Database -sqlx = { version = "0.8.6", features = [ +sqlx = { git = "https://github.com/bengineer42/sqlx", version = "0.8.6", features = [ "runtime-tokio", - "sqlite", - "postgres", - "any", -] } +], rev = "22a01d3" } # Used patched version of sqlx with SqliteConnectOptions support (PR in progress: https://github.com/launchbadge/sqlx/pull/4209) crc = "3.4.0" # Hashing @@ -115,10 +110,10 @@ bigdecimal = "0.4.10" tonic-build = "0.12" # Introspect -introspect-events = { git = "https://github.com/cartridge-gg/introspect", rev = "34e93c1" } -introspect-types = { git = "https://github.com/cartridge-gg/introspect", rev = "34e93c1" } -introspect-rust-macros = { git = "https://github.com/cartridge-gg/introspect", rev = "34e93c1" } -dojo-introspect = { git = "https://github.com/dojoengine/dojo-introspect", rev = "aadc3c9" } +introspect-events = { git = "https://github.com/cartridge-gg/introspect", rev = "b89ee9e" } +introspect-types = { git = "https://github.com/cartridge-gg/introspect", rev = "b89ee9e" } +introspect-rust-macros = { git = "https://github.com/cartridge-gg/introspect", rev = "b89ee9e" } +dojo-introspect = { git = "https://github.com/dojoengine/dojo-introspect", rev = "c45d711" } # Utils @@ -133,26 +128,22 @@ torii-config-common = { path = "crates/torii-config-common" } torii-runtime-common = { path = "crates/torii-runtime-common" } # Internal crates +torii-sql = { path = "crates/sql" } torii-sql-sink.path = "crates/torii-sql-sink" torii-log-sink.path = "crates/torii-log-sink" torii-controllers-sink.path = "crates/torii-controllers-sink" torii-arcade-sink.path = "crates/arcade-sink" torii-ecs-sink.path = "crates/torii-ecs-sink" -torii-entities-historical-sink.path = "crates/torii-entities-historical-sink" torii-erc20.path = "crates/torii-erc20" torii-erc721.path = "crates/torii-erc721" torii-erc1155.path = "crates/torii-erc1155" torii-dojo.path = "./crates/dojo" torii-introspect.path = "crates/introspect" -torii-introspect-postgres-sink.path = "./crates/introspect-postgres-sink" -torii-introspect-sqlite-sink.path = "./crates/introspect-sqlite-sink" +torii-introspect-sql-sink.path = "./crates/introspect-sql-sink" torii-test-utils.path = "crates/testing" -torii-postgres.path = "crates/postgres" -torii-sqlite.path = "crates/sqlite" torii-pathfinder.path = "crates/pathfinder" torii-starknet.path = "crates/starknet" - [lib] name = "torii" path = "src/lib.rs" @@ -177,14 +168,14 @@ path = "examples/http_only_sink/main.rs" name = "test_block_extractor" path = "examples/test_block_extractor/main.rs" -[[example]] -name = "introspect_simple" -path = "examples/introspect/simple.rs" - [[example]] name = "introspect_restart" path = "examples/introspect/restart.rs" +[[example]] +name = "pathfinder" +path = "examples/pathfinder/main.rs" + [dependencies] anyhow.workspace = true async-trait.workspace = true @@ -200,7 +191,7 @@ prost-types.workspace = true rustls-pemfile = "2.0" serde_json.workspace = true serde.workspace = true -sqlx.workspace = true +sqlx = { workspace = true, features = ["sqlite"] } starknet.workspace = true tokio-stream.workspace = true tokio-rustls = { version = "0.26", default-features = false, features = [ @@ -235,18 +226,22 @@ resolve-path.workspace = true itertools.workspace = true dojo-introspect.workspace = true + +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } torii-sql-sink.workspace = true torii-log-sink.workspace = true torii-erc20.workspace = true torii-erc721.workspace = true torii-erc1155.workspace = true -torii-dojo.workspace = true +torii-dojo = { workspace = true, features = ["postgres", "sqlite"] } torii-introspect.workspace = true torii-common.workspace = true -torii-introspect-postgres-sink.workspace = true torii-test-utils.workspace = true - - +torii-introspect-sql-sink = { workspace = true, features = [ + "postgres", + "sqlite", +] } +torii-pathfinder.workspace = true # Example dependencies diff --git a/bins/torii-arcade/Cargo.toml b/bins/torii-arcade/Cargo.toml index 0ec65787..90ab6e4b 100644 --- a/bins/torii-arcade/Cargo.toml +++ b/bins/torii-arcade/Cargo.toml @@ -9,20 +9,21 @@ name = "torii-arcade" path = "src/main.rs" [dependencies] -torii = { path = "../../" } -torii-common = { path = "../../crates/torii-common" } +torii.workspace = true +torii-common.workspace = true torii-config-common.workspace = true -torii-arcade-sink = { path = "../../crates/arcade-sink" } -torii-dojo = { path = "../../crates/dojo" } -torii-ecs-sink = { path = "../../crates/torii-ecs-sink" } -torii-entities-historical-sink.workspace = true -torii-erc20 = { path = "../../crates/torii-erc20" } -torii-erc721 = { path = "../../crates/torii-erc721" } -torii-erc1155 = { path = "../../crates/torii-erc1155" } -torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } -torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } +torii-arcade-sink.workspace = true +torii-dojo.workspace = true +torii-ecs-sink.workspace = true +torii-erc20.workspace = true +torii-erc721.workspace = true +torii-erc1155.workspace = true +torii-introspect-sql-sink = { workspace = true, features = [ + "postgres", + "sqlite", +] } torii-runtime-common.workspace = true -torii-sqlite.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } torii-pathfinder = { workspace = true, features = ["etl"] } tokio = { version = "1", features = ["full"] } @@ -32,7 +33,7 @@ starknet = "0.17" url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" -sqlx = { version = "0.8", features = [ +sqlx = { workspace = true, features = [ "postgres", "sqlite", "runtime-tokio-rustls", diff --git a/bins/torii-arcade/src/main.rs b/bins/torii-arcade/src/main.rs index f2896b91..c263a6fb 100644 --- a/bins/torii-arcade/src/main.rs +++ b/bins/torii-arcade/src/main.rs @@ -3,11 +3,10 @@ mod config; use anyhow::{Error, Result}; use clap::Parser; use config::{Config, MetadataMode}; -use sqlx::postgres::PgPoolOptions; -use sqlx::sqlite::SqlitePoolOptions; use starknet::core::types::Felt; use std::collections::{HashMap, HashSet}; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use tonic::codec::CompressionEncoding; @@ -18,8 +17,7 @@ use torii::etl::extractor::{ ContractEventConfig, EventExtractor, EventExtractorConfig, Extractor, RetryPolicy, }; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; -use torii::etl::EngineDb; -use torii::etl::TypeId; +use torii::etl::{EngineDb, TypeId}; use torii::EtlConcurrencyConfig; use torii_arcade_sink::proto::arcade::arcade_server::ArcadeServer; use torii_arcade_sink::{ArcadeSink, FILE_DESCRIPTOR_SET as ARCADE_DESCRIPTOR_SET}; @@ -30,11 +28,9 @@ use torii_dojo::external_contract::{ contract_type_from_decoder_ids, RegisterExternalContractCommandHandler, RegisteredContractType, SharedContractTypeRegistry, SharedDecoderRegistry, }; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::store::sqlite::SqliteStore; +use torii_dojo::store::DojoStoreTrait; use torii_ecs_sink::proto::world::world_server::WorldServer; use torii_ecs_sink::{EcsSink, FILE_DESCRIPTOR_SET as ECS_DESCRIPTOR_SET}; -use torii_entities_historical_sink::EntitiesHistoricalSink; use torii_erc1155::proto::erc1155_server::Erc1155Server; use torii_erc1155::{ Erc1155Decoder, Erc1155MetadataCommandHandler, Erc1155Service, Erc1155Sink, Erc1155Storage, @@ -50,14 +46,11 @@ use torii_erc721::{ Erc721Decoder, Erc721MetadataCommandHandler, Erc721Service, Erc721Sink, Erc721Storage, FILE_DESCRIPTOR_SET as ERC721_DESCRIPTOR_SET, }; -use torii_introspect_postgres_sink::processor::IntrospectPgDb; -use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; +use torii_introspect_sql_sink::{IntrospectDb, NamespaceMode}; use torii_pathfinder::extractor::PathfinderCombinedExtractor; -use torii_runtime_common::database::{ - validate_uniform_backends, DatabaseBackend, DEFAULT_SQLITE_MAX_CONNECTIONS, -}; +use torii_runtime_common::database::{validate_uniform_backends, DEFAULT_SQLITE_MAX_CONNECTIONS}; use torii_runtime_common::token_support::{resolve_installed_token_support, InstalledTokenSupport}; -use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; +use torii_sql::{DbConnectionOptions, DbPool, DbPoolOptions, PoolExt}; type StarknetProvider = starknet::providers::jsonrpc::JsonRpcClient; @@ -257,7 +250,7 @@ async fn run_indexer(config: Config) -> Result<()> { ("erc1155", &erc1155_db_url), ], "torii-arcade does not support mixed storage backends in one runtime; configure all databases as either SQLite or PostgreSQL", - )?; + ).map_err(|err| anyhow::anyhow!(err))?; let provider = starknet::providers::jsonrpc::JsonRpcClient::new( starknet::providers::jsonrpc::HttpTransport::new( @@ -394,64 +387,32 @@ async fn run_indexer(config: Config) -> Result<()> { &erc1155_addresses, &config, )?; - - let (dojo_decoder, introspect_sink): ( - Arc, - Box, - ) = match backend { - DatabaseBackend::Postgres => { - let max_db_connections = config.max_db_connections.unwrap_or(5); - let pool = Arc::new( - PgPoolOptions::new() - .max_connections(max_db_connections) - .connect(&storage_database_url) - .await?, - ); - - let decoder = DojoDecoder::, _>::new(pool.clone(), (*provider).clone()); - let sink = IntrospectPgDb::new(pool.clone(), ()); - decoder.store.initialize().await?; - decoder.load_tables(&[]).await?; - - ( - Arc::new(decoder) as Arc, - Box::new(sink), - ) - } - DatabaseBackend::Sqlite => { - let options = sqlite_connect_options(&storage_database_url)?; - let max_db_connections = match config.max_db_connections { - Some(limit) => limit.max(1), - None if is_sqlite_memory_path(&storage_database_url) => 1, - None => DEFAULT_SQLITE_MAX_CONNECTIONS, - }; - let pool = Arc::new( - SqlitePoolOptions::new() - .max_connections(max_db_connections) - .connect_with(options) - .await?, - ); - - sqlx::query("PRAGMA journal_mode=WAL") - .execute(pool.as_ref()) - .await?; - sqlx::query("PRAGMA synchronous=NORMAL") - .execute(pool.as_ref()) - .await?; - sqlx::query("PRAGMA foreign_keys=ON") - .execute(pool.as_ref()) - .await?; - - let decoder = DojoDecoder::, _>::new(pool.clone(), (*provider).clone()); - decoder.store.initialize().await?; - decoder.load_tables(&[]).await?; - - ( - Arc::new(decoder) as Arc, - Box::new(IntrospectSqliteDb::new(pool.clone(), ())), - ) - } + let conn_options = + DbConnectionOptions::from_str(&engine_database_url).map_err(anyhow::Error::msg)?; + let max_connections = match config.max_db_connections { + Some(n) => n, + None => match &conn_options { + DbConnectionOptions::Postgres(_) => 10, + DbConnectionOptions::Sqlite(ops) if ops.is_in_memory() => 1, + DbConnectionOptions::Sqlite(_) => DEFAULT_SQLITE_MAX_CONNECTIONS, + }, }; + let pool_options = DbPoolOptions::new().max_connections(max_connections); + let pool = pool_options.connect_any_with(conn_options).await?; + if let DbPool::Sqlite(pool) = &pool { + pool.execute_queries([ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=NORMAL", + "PRAGMA foreign_keys=ON", + ]) + .await?; + } + let dojo_decoder = DojoDecoder::new(pool.clone(), provider.clone()); + let introspect_sink = IntrospectDb::new(pool, NamespaceMode::Address); + + dojo_decoder.initialize().await?; + dojo_decoder.load_tables(&[]).await?; + introspect_sink.initialize_introspect_sql_sink().await?; let ecs_sink = EcsSink::new( &storage_database_url, @@ -479,20 +440,8 @@ async fn run_indexer(config: Config) -> Result<()> { .register_encoded_file_descriptor_set(ECS_DESCRIPTOR_SET) .register_encoded_file_descriptor_set(ARCADE_DESCRIPTOR_SET); - let historical_sink = Box::new( - EntitiesHistoricalSink::new( - &storage_database_url, - config.max_db_connections, - (), - historical_models, - ) - .await?, - ); - let arcade_projection_pipeline = ArcadeProjectionPipeline::new(vec![ - introspect_sink, - historical_sink, - Box::new(arcade_sink), - ]); + let arcade_projection_pipeline = + ArcadeProjectionPipeline::new(vec![Box::new(introspect_sink), Box::new(arcade_sink)]); let mut torii_config = torii::ToriiConfig::builder() .port(config.port) @@ -504,7 +453,7 @@ async fn run_indexer(config: Config) -> Result<()> { }) .engine_database_url(engine_database_url) .with_extractor(extractor) - .add_decoder(dojo_decoder) + .add_decoder(Arc::new(dojo_decoder)) .add_sink_boxed(Box::new(ecs_sink)) .add_sink_boxed(Box::new(arcade_projection_pipeline)); diff --git a/bins/torii-erc20/Cargo.toml b/bins/torii-erc20/Cargo.toml index 9dd48af1..b69a7d7b 100644 --- a/bins/torii-erc20/Cargo.toml +++ b/bins/torii-erc20/Cargo.toml @@ -9,10 +9,12 @@ path = "src/main.rs" [dependencies] # Core dependencies -torii = { path = "../../" } -torii-erc20 = { path = "../../crates/torii-erc20" } +torii.workspace = true +torii-erc20.workspace = true torii-runtime-common.workspace = true +torii-sql = { workspace = true, optional = true } + # Async runtime tokio = { version = "1", features = ["full"] } @@ -41,7 +43,7 @@ features = ["flamegraph", "prost-codec"] optional = true [features] -profiling = ["pprof"] +profiling = ["pprof", "torii-sql"] [lints] workspace = true diff --git a/bins/torii-erc20/src/main.rs b/bins/torii-erc20/src/main.rs index 82cc1bb3..66240c38 100644 --- a/bins/torii-erc20/src/main.rs +++ b/bins/torii-erc20/src/main.rs @@ -38,7 +38,7 @@ use torii::etl::decoder::DecoderId; use torii::etl::extractor::{BlockRangeConfig, BlockRangeExtractor}; use torii_runtime_common::database::resolve_single_db_setup; #[cfg(feature = "profiling")] -use torii_runtime_common::database::{backend_from_url_or_path, DatabaseBackend}; +use torii_sql::DbBackend; // Import from the library crate use torii_erc20::proto::erc20_server::Erc20Server; @@ -219,10 +219,10 @@ async fn main() -> Result<()> { .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - let db_backend = match backend_from_url_or_path(&db_setup.storage_url) { - DatabaseBackend::Postgres => "postgres", - DatabaseBackend::Sqlite => "sqlite", - }; + let db_backend = db_setup + .storage_url + .parse::() + .map_err(anyhow::Error::new)?; let filename = format!("flamegraph-torii-erc20-block-range-{db_backend}-{ts}.svg"); let file = std::fs::File::create(&filename).unwrap(); report.flamegraph(file).unwrap(); diff --git a/bins/torii-introspect-bin/Cargo.toml b/bins/torii-introspect-bin/Cargo.toml index bf3d6261..62049cc2 100644 --- a/bins/torii-introspect-bin/Cargo.toml +++ b/bins/torii-introspect-bin/Cargo.toml @@ -9,20 +9,21 @@ name = "torii-server" path = "src/main.rs" [dependencies] -torii = { path = "../../" } -torii-common = { path = "../../crates/torii-common" } +torii.workspace = true +torii-common.workspace = true torii-config-common.workspace = true torii-controllers-sink.workspace = true -torii-dojo = { path = "../../crates/dojo" } -torii-erc20 = { path = "../../crates/torii-erc20" } -torii-erc721 = { path = "../../crates/torii-erc721" } -torii-erc1155 = { path = "../../crates/torii-erc1155" } -torii-ecs-sink = { path = "../../crates/torii-ecs-sink" } -torii-entities-historical-sink.workspace = true -torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } -torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } +torii-dojo.workspace = true +torii-erc20.workspace = true +torii-erc721.workspace = true +torii-erc1155.workspace = true +torii-ecs-sink.workspace = true +torii-introspect-sql-sink = { workspace = true, features = [ + "postgres", + "sqlite", +] } torii-runtime-common.workspace = true -torii-sqlite = { path = "../../crates/sqlite" } +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } tokio = { version = "1", features = ["full"] } tracing = "0.1" @@ -31,7 +32,12 @@ starknet = "0.17" url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" -sqlx = { version = "0.8", features = ["postgres", "sqlite", "runtime-tokio-rustls", "any"] } +sqlx = { workspace = true, features = [ + "postgres", + "sqlite", + "runtime-tokio-rustls", + "any", +] } tonic.workspace = true tonic-reflection.workspace = true tonic-web.workspace = true diff --git a/bins/torii-introspect-bin/src/config.rs b/bins/torii-introspect-bin/src/config.rs index 2af7c2bf..58bda6c3 100644 --- a/bins/torii-introspect-bin/src/config.rs +++ b/bins/torii-introspect-bin/src/config.rs @@ -1,13 +1,9 @@ use anyhow::{bail, Result}; use clap::{ArgGroup, Parser}; use starknet::core::types::Felt; +use std::collections::HashSet; use std::path::{Path, PathBuf}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum StorageBackend { - Postgres, - Sqlite, -} +use torii_sql::DbBackend; /// Dojo introspect indexer backed by PostgreSQL or SQLite. /// @@ -177,21 +173,15 @@ impl Config { Self::parse_addresses("ERC1155", &self.erc1155) } - pub fn historical_models(&self) -> Vec { - let mut models = Vec::with_capacity(self.historical.len()); - for model in &self.historical { - if !model.is_empty() && !models.contains(model) { - models.push(model.clone()); - } - } - models + pub fn historical_models(&self) -> &[String] { + &self.historical } - pub fn storage_backend(&self) -> StorageBackend { + pub fn storage_backend(&self) -> DbBackend { if self.storage_database_url.is_some() { - StorageBackend::Postgres + DbBackend::Postgres } else { - StorageBackend::Sqlite + DbBackend::Sqlite } } @@ -210,7 +200,10 @@ impl Config { Ok(url.clone()) } Some(_) => bail!("--storage-database-url must be a PostgreSQL URL"), - None => Ok(db_dir.join("introspect.db").to_string_lossy().to_string()), + None => Ok(format!( + "sqlite://{}", + db_dir.join("introspect.db").to_string_lossy() + )), } } @@ -225,6 +218,21 @@ impl Config { } } +pub fn parse_historical_models( + historical: &[String], + contracts: &[Felt], +) -> Result> { + let mut models = HashSet::with_capacity(historical.len()); + for model in historical { + let parts: Vec<&str> = model.splitn(2, ':').collect(); + match parts.len() { + 1 => contracts.iter().for_each(|&addr| {models.insert((addr, parts[0].to_string()));}), + 2 => {models.insert((Felt::from_hex(parts[0])?, parts[1].to_string()));}, + _ => bail!("Invalid historical model format: {model}. Expected format is either `ModelName` or `0xContractAddress:ModelName`"), + } + } + Ok(models) +} #[cfg(test)] mod tests { use super::*; @@ -272,7 +280,7 @@ mod tests { fn sqlite_is_default_when_storage_database_url_is_omitted() { let cfg = Config::parse_from(["torii-server", "--contract", "0x1"]); - assert_eq!(cfg.storage_backend(), StorageBackend::Sqlite); + assert_eq!(cfg.storage_backend(), DbBackend::Sqlite); assert!(cfg .storage_database_url(Path::new("./torii-data")) .unwrap() diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index ea102237..e4f014c0 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -4,12 +4,13 @@ mod config; use anyhow::Result; use clap::Parser; -use config::{Config, StorageBackend}; +use config::Config; use sqlx::postgres::PgPoolOptions; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use starknet::core::types::Felt; use std::collections::{HashMap, HashSet}; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use tonic::codec::CompressionEncoding; @@ -27,11 +28,9 @@ use torii_dojo::external_contract::{ contract_type_from_decoder_ids, RegisterExternalContractCommandHandler, RegisteredContractType, SharedContractTypeRegistry, SharedDecoderRegistry, }; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::store::sqlite::SqliteStore; +use torii_dojo::store::DojoStoreTrait; use torii_ecs_sink::proto::world::world_server::WorldServer; use torii_ecs_sink::{EcsSink, FILE_DESCRIPTOR_SET as ECS_DESCRIPTOR_SET}; -use torii_entities_historical_sink::EntitiesHistoricalSink; use torii_erc1155::proto::erc1155_server::Erc1155Server; use torii_erc1155::{ Erc1155Decoder, Erc1155MetadataCommandHandler, Erc1155Service, Erc1155Sink, Erc1155Storage, @@ -47,13 +46,14 @@ use torii_erc721::{ Erc721Decoder, Erc721MetadataCommandHandler, Erc721Service, Erc721Sink, Erc721Storage, FILE_DESCRIPTOR_SET as ERC721_DESCRIPTOR_SET, }; -use torii_introspect_postgres_sink::processor::IntrospectPgDb; -use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; +use torii_introspect_sql_sink::{IntrospectPgDb, IntrospectSqliteDb, NamespaceMode}; use torii_runtime_common::database::{ resolve_token_db_setup, TokenDbSetup, DEFAULT_SQLITE_MAX_CONNECTIONS, }; use torii_runtime_common::token_support::{resolve_installed_token_support, InstalledTokenSupport}; -use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; +use torii_sql::DbBackend; + +use crate::config::parse_historical_models; type StarknetProvider = starknet::providers::jsonrpc::JsonRpcClient; @@ -492,7 +492,7 @@ async fn run_indexer(config: Config) -> Result<()> { erc1155: !token_targets.erc1155.is_empty(), }, ); - let historical_models = config.historical_models(); + let historical_models = parse_historical_models(config.historical_models(), &contracts)?; let token_db_setup = if installed_token_support.any() { Some(resolve_token_db_setup( db_dir, @@ -638,12 +638,12 @@ async fn run_indexer(config: Config) -> Result<()> { }, )); - if matches!(backend, StorageBackend::Sqlite) { + if matches!(backend, DbBackend::Sqlite) { tokio::fs::create_dir_all(db_dir).await?; } match backend { - StorageBackend::Postgres => { + DbBackend::Postgres => { run_with_postgres( &config, &storage_database_url, @@ -656,13 +656,13 @@ async fn run_indexer(config: Config) -> Result<()> { decoder_registry.clone(), contract_type_registry.clone(), installed_external_decoders.clone(), - historical_models.clone(), + historical_models, provider, extractor, ) .await?; } - StorageBackend::Sqlite => { + DbBackend::Sqlite => { run_with_sqlite( &config, &storage_database_url, @@ -675,7 +675,7 @@ async fn run_indexer(config: Config) -> Result<()> { decoder_registry.clone(), contract_type_registry.clone(), installed_external_decoders.clone(), - historical_models.clone(), + historical_models, provider, extractor, ) @@ -699,22 +699,23 @@ async fn run_with_postgres( decoder_registry: SharedDecoderRegistry, contract_type_registry: SharedContractTypeRegistry, installed_external_decoders: HashSet, - historical_models: Vec, + historical_models: HashSet<(Felt, String)>, provider: StarknetProvider, extractor: Box, ) -> Result<()> { let token_provider = Arc::new(provider.clone()); let max_db_connections = config.max_db_connections.unwrap_or(5); - let pool = Arc::new( - PgPoolOptions::new() - .max_connections(max_db_connections) - .connect(storage_database_url) - .await?, - ); + let pool = PgPoolOptions::new() + .max_connections(max_db_connections) + .connect(storage_database_url) + .await?; + + let mut decoder = DojoDecoder::new(pool.clone(), provider); + let introspect_sink = IntrospectPgDb::new(pool.clone(), NamespaceMode::Address); - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - let introspect_sink = IntrospectPgDb::new(pool.clone(), ()); - decoder.store.initialize().await?; + decoder.append_historical(historical_models); + decoder.initialize().await?; + introspect_sink.initialize_introspect_sql_sink().await?; decoder.load_tables(&[]).await?; let decoder: Arc = Arc::new(decoder); @@ -736,16 +737,7 @@ async fn run_with_postgres( .add_decoder(decoder) .add_sink_boxed(Box::new( OrderedSinkPipeline::new("introspect-projection-pipeline") - .push(Box::new(introspect_sink)) - .push(Box::new( - EntitiesHistoricalSink::new( - storage_database_url, - config.max_db_connections, - (), - historical_models, - ) - .await?, - )), + .push(Box::new(introspect_sink)), )); if let Some(tls) = config.tls_config()? { torii_config = torii_config.with_tls(tls); @@ -879,36 +871,33 @@ async fn run_with_sqlite( decoder_registry: SharedDecoderRegistry, contract_type_registry: SharedContractTypeRegistry, installed_external_decoders: HashSet, - historical_models: Vec, + historical_models: HashSet<(Felt, String)>, provider: StarknetProvider, extractor: Box, ) -> Result<()> { let token_provider = Arc::new(provider.clone()); - let options = sqlite_connect_options(storage_database_url)?; + let options = SqliteConnectOptions::from_str(storage_database_url)?.create_if_missing(true); let max_db_connections = match config.max_db_connections { Some(limit) => limit.max(1), - None if is_sqlite_memory_path(storage_database_url) => 1, + None if options.is_in_memory() => 1, None => DEFAULT_SQLITE_MAX_CONNECTIONS, }; - let pool = Arc::new( - SqlitePoolOptions::new() - .max_connections(max_db_connections) - .connect_with(options) - .await?, - ); + let pool = SqlitePoolOptions::new() + .max_connections(max_db_connections) + .connect_with(options) + .await?; sqlx::query("PRAGMA journal_mode=WAL") - .execute(pool.as_ref()) + .execute(&pool) .await?; sqlx::query("PRAGMA synchronous=NORMAL") - .execute(pool.as_ref()) - .await?; - sqlx::query("PRAGMA foreign_keys=ON") - .execute(pool.as_ref()) + .execute(&pool) .await?; + sqlx::query("PRAGMA foreign_keys=ON").execute(&pool).await?; - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - decoder.store.initialize().await?; + let mut decoder = DojoDecoder::new(pool.clone(), provider); + decoder.append_historical(historical_models); + decoder.initialize().await?; decoder.load_tables(&[]).await?; let decoder: Arc = Arc::new(decoder); @@ -929,17 +918,9 @@ async fn run_with_sqlite( .with_extractor(extractor) .add_decoder(decoder) .add_sink_boxed(Box::new( - OrderedSinkPipeline::new("introspect-projection-pipeline") - .push(Box::new(IntrospectSqliteDb::new(pool.clone(), ()))) - .push(Box::new( - EntitiesHistoricalSink::new( - storage_database_url, - config.max_db_connections, - (), - historical_models, - ) - .await?, - )), + OrderedSinkPipeline::new("introspect-projection-pipeline").push(Box::new( + IntrospectSqliteDb::new(pool.clone(), NamespaceMode::Address), + )), )); if let Some(tls) = config.tls_config()? { torii_config = torii_config.with_tls(tls); @@ -1064,7 +1045,8 @@ async fn run_with_sqlite( #[cfg(test)] mod tests { use super::{ecs_token_storage_urls, InstalledTokenSupport}; - use torii_runtime_common::database::{DatabaseBackend, TokenDbSetup}; + use torii_runtime_common::database::TokenDbSetup; + use torii_sql::DbBackend; fn token_db_setup() -> TokenDbSetup { TokenDbSetup { @@ -1072,10 +1054,10 @@ mod tests { erc20_url: "./torii-data/erc20.db".to_string(), erc721_url: "./torii-data/erc721.db".to_string(), erc1155_url: "./torii-data/erc1155.db".to_string(), - engine_backend: DatabaseBackend::Sqlite, - erc20_backend: DatabaseBackend::Sqlite, - erc721_backend: DatabaseBackend::Sqlite, - erc1155_backend: DatabaseBackend::Sqlite, + engine_backend: DbBackend::Sqlite, + erc20_backend: DbBackend::Sqlite, + erc721_backend: DbBackend::Sqlite, + erc1155_backend: DbBackend::Sqlite, } } diff --git a/bins/torii-introspect-synth/Cargo.toml b/bins/torii-introspect-synth/Cargo.toml index 8caedd45..fbac5244 100644 --- a/bins/torii-introspect-synth/Cargo.toml +++ b/bins/torii-introspect-synth/Cargo.toml @@ -8,10 +8,9 @@ name = "torii-introspect-synth" path = "src/main.rs" [dependencies] -torii = { path = "../../" } -torii-runtime-common.workspace = true -torii-dojo = { path = "../../crates/dojo" } -torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } +torii.workspace = true +torii-dojo = { workspace = true, features = ["postgres"] } +torii-introspect-sql-sink = { workspace = true, features = ["postgres"] } tokio = { version = "1", features = ["full"] } tracing = "0.1" @@ -22,7 +21,7 @@ async-trait = "0.1" chrono = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres"] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } starknet = "0.17" starknet-types-core.workspace = true dojo-introspect.workspace = true diff --git a/bins/torii-introspect-synth/src/main.rs b/bins/torii-introspect-synth/src/main.rs index 9f71e3e6..7c07d3a3 100644 --- a/bins/torii-introspect-synth/src/main.rs +++ b/bins/torii-introspect-synth/src/main.rs @@ -26,8 +26,8 @@ use torii::etl::sink::{EventBus, Sink, SinkContext}; use torii::etl::Decoder; use torii::grpc::SubscriptionManager; use torii_dojo::decoder::DojoDecoder; -use torii_dojo::store::postgres::PgStore; -use torii_introspect_postgres_sink::IntrospectPgDb; +use torii_dojo::store::DojoStoreTrait; +use torii_introspect_sql_sink::IntrospectPgDb; const EXTRACTOR_TYPE: &str = "synthetic_introspect"; const STATE_KEY: &str = "last_block"; @@ -418,14 +418,12 @@ async fn main() -> Result<()> { fs::create_dir_all(&output_dir) .with_context(|| format!("failed to create output dir {}", output_dir.display()))?; - let pool = Arc::new( - PgPoolOptions::new() - .max_connections(5) - .connect(&config.db_url) - .await?, - ); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&config.db_url) + .await?; if config.reset_schema { - reset_schema(pool.as_ref()).await?; + reset_schema(&pool).await?; } let started = Instant::now(); @@ -447,8 +445,8 @@ async fn main() -> Result<()> { .await?, ); - let decoder = DojoDecoder::, _>::new(pool.clone(), NeverFetchSchema); - decoder.store.initialize().await?; + let decoder = DojoDecoder::new(pool.clone(), NeverFetchSchema); + decoder.initialize().await?; decoder.load_tables(&[]).await?; let decoder: Arc = Arc::new(decoder); let decoder_context = @@ -486,7 +484,7 @@ async fn main() -> Result<()> { } } - let verification = verify_run(pool.as_ref(), &config).await?; + let verification = verify_run(&pool, &config).await?; let summary = Summary { run_id: run_id.clone(), duration_ms: started.elapsed().as_millis(), diff --git a/bins/torii-tokens/Cargo.toml b/bins/torii-tokens/Cargo.toml index b6a98319..627c87f0 100644 --- a/bins/torii-tokens/Cargo.toml +++ b/bins/torii-tokens/Cargo.toml @@ -10,13 +10,14 @@ path = "src/main.rs" [dependencies] # Core dependencies -torii = { path = "../../" } -torii-common = { path = "../../crates/torii-common" } +torii.workspace = true +torii-common.workspace = true torii-config-common.workspace = true torii-runtime-common.workspace = true -torii-erc20 = { path = "../../crates/torii-erc20" } -torii-erc721 = { path = "../../crates/torii-erc721" } -torii-erc1155 = { path = "../../crates/torii-erc1155" } +torii-erc20.workspace = true +torii-erc721.workspace = true +torii-erc1155.workspace = true +torii-sql = { workspace = true, optional = true } # Async runtime tokio = { version = "1", features = ["full"] } @@ -47,7 +48,7 @@ features = ["flamegraph", "prost-codec"] optional = true [features] -profiling = ["pprof"] +profiling = ["pprof", "torii-sql"] [lints] workspace = true diff --git a/bins/torii-tokens/src/main.rs b/bins/torii-tokens/src/main.rs index 3d49c1ff..44ee9c98 100644 --- a/bins/torii-tokens/src/main.rs +++ b/bins/torii-tokens/src/main.rs @@ -61,7 +61,7 @@ use torii_common::{MetadataFetcher, TokenUriService}; use torii_config_common::apply_observability_env; use torii_runtime_common::database::resolve_token_db_setup; #[cfg(feature = "profiling")] -use torii_runtime_common::database::DatabaseBackend; +use torii_sql::DbBackend; // Import from ERC20 library crate use torii_erc20::proto::erc20_server::Erc20Server; @@ -782,7 +782,7 @@ async fn run_indexer(config: Config) -> Result<()> { .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - let db_backend = if db_setup.erc20_backend == DatabaseBackend::Postgres { + let db_backend = if db_setup.erc20_backend == DbBackend::Postgres { "postgres" } else { "sqlite" diff --git a/crates/dojo/Cargo.toml b/crates/dojo/Cargo.toml index 0e09b633..ccf2cf22 100644 --- a/crates/dojo/Cargo.toml +++ b/crates/dojo/Cargo.toml @@ -18,20 +18,23 @@ tracing.workspace = true thiserror.workspace = true hex.workspace = true itertools.workspace = true - -dojo-introspect.workspace = true -introspect-types.workspace = true -torii-introspect.workspace = true -torii-postgres.workspace = true -torii-sqlite.workspace = true sqlx = { workspace = true, features = [ - "postgres", - "sqlite", "runtime-tokio-rustls", "macros", "migrate", ] } +dojo-introspect.workspace = true +introspect-types.workspace = true + +torii-introspect.workspace = true +torii-sql.workspace = true + + +[features] +postgres = ["sqlx/postgres"] +sqlite = ["sqlx/sqlite"] + [build-dependencies] tonic-build.workspace = true diff --git a/crates/dojo/migrations/001_dojo_store.sql b/crates/dojo/migrations/postgres/001_dojo_store.sql similarity index 100% rename from crates/dojo/migrations/001_dojo_store.sql rename to crates/dojo/migrations/postgres/001_dojo_store.sql diff --git a/crates/dojo/src/decoder.rs b/crates/dojo/src/decoder.rs index 1b95122d..9afb1627 100644 --- a/crates/dojo/src/decoder.rs +++ b/crates/dojo/src/decoder.rs @@ -9,6 +9,7 @@ use dojo_introspect::events::{ ModelWithSchemaRegistered, StoreDelRecord, StoreSetRecord, StoreUpdateMember, StoreUpdateRecord, }; +use dojo_introspect::selector::compute_selector_from_dojo_tag; use dojo_introspect::serde::dojo_primary_def; use dojo_introspect::{DojoSchema, DojoSchemaFetcher}; use introspect_types::{ @@ -18,14 +19,14 @@ use introspect_types::{ use itertools::Itertools; use starknet::core::types::EmittedEvent; use starknet_types_core::felt::Felt; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::sync::RwLock; use torii::etl::event::EmittedEventExt; use torii::etl::{Decoder, Envelope, EventMsg}; use torii_introspect::events::{IntrospectBody, IntrospectMsg}; use torii_introspect::schema::{TableMetadata, TableSchema}; -use torii_introspect::EventId; +use torii_introspect::{CreateTable, EventId}; pub const DOJO_ID_FIELD_NAME: &str = "entity_id"; @@ -33,6 +34,7 @@ pub struct DojoDecoder { pub tables: RwLock>, pub store: Store, pub fetcher: F, + pub append_only: HashSet<(Felt, Felt)>, } fn deserialize_data<'a, T>(keys: &[Felt], data: &'a [Felt]) -> DojoToriiResult @@ -65,12 +67,12 @@ pub trait DojoRecordEvent: Sized + CairoEventInfo + Debug { #[async_trait] impl DojoStoreTrait for DojoDecoder where - Store: DojoStoreTrait + Send + Sync, - Store::Error: ToString, - F: Send + Sync + 'static, + Store: DojoStoreTrait + Sync, + F: Sync, { - type Error = DojoToriiError; - + async fn initialize(&self) -> DojoToriiResult<()> { + self.store.initialize().await + } async fn save_table( &self, owner: &Felt, @@ -81,14 +83,10 @@ where self.store .save_table(owner, table, tx_hash, block_number) .await - .map_err(DojoToriiError::store_error) } async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { - self.store - .read_tables(owners) - .await - .map_err(DojoToriiError::store_error) + self.store.read_tables(owners).await } } @@ -116,18 +114,23 @@ impl DojoDecoder { impl DojoDecoder where - Store: DojoStoreTrait + Sync, + Store: DojoStoreTrait + Sync + Send, F: DojoSchemaFetcher + Send + Sync + 'static, { - pub fn new>(store: S, fetcher: F) -> Self { - let store = store.into(); + pub fn new(store: Store, fetcher: F) -> Self { Self { tables: Default::default(), store, fetcher, + append_only: HashSet::new(), + } + } + pub fn append_historical(&mut self, models: HashSet<(Felt, String)>) { + for (address, name) in models { + let table_id = compute_selector_from_dojo_tag(&name).unwrap(); + self.append_only.insert((address, table_id)); } } - pub async fn load_tables(&self, owners: &[Felt]) -> DojoToriiResult<()> { let new = self.read_tables(owners).await?; let mut tables = self.tables.write()?; @@ -155,6 +158,7 @@ where tables: RwLock::new(tables), store, fetcher, + append_only: HashSet::new(), } } @@ -165,7 +169,7 @@ where name: &str, schema: DojoSchema, metadata: &impl TableMetadata, - ) -> DojoToriiResult { + ) -> DojoToriiResult { let full_table = DojoTable::from_schema(schema, namespace, name, dojo_primary_def()); self.save_table( owner, @@ -185,7 +189,10 @@ where } } self.tables.write()?.insert(id, table); - Ok(full_table.into()) + Ok(CreateTable::from_schema( + full_table.into(), + self.append_only.contains(&(*owner, id)), + )) } pub async fn update_table( @@ -334,10 +341,14 @@ where .err_into(); } - self.decode_event_data(event, selector, keys, &event.data) + match self + .decode_event_data(event, selector, keys, &event.data) .await - .map(|msg| vec![msg.to_body(event).into()]) - .err_into() + { + Ok(msg) => Ok(vec![msg.to_envelope(event)]), + Err(DojoToriiError::UnknownDojoEventSelector(_)) => Ok(Vec::new()), + Err(e) => Err(e.into()), + } } } @@ -347,15 +358,10 @@ mod tests { use crate::ExternalContractRegisteredBody; use async_trait::async_trait; use dojo_introspect::DojoIntrospectError; - use introspect_types::{ - utils::string_to_cairo_serialize_byte_array, Attribute, ColumnDef, TypeDef, - }; + use introspect_types::utils::string_to_cairo_serialize_byte_array; + use introspect_types::{Attribute, ColumnDef, TypeDef}; use std::sync::Mutex; - #[derive(Debug, thiserror::Error)] - #[error("{0}")] - struct FakeStoreError(String); - #[derive(Default)] struct FakeStore { saved_blocks: Mutex>, @@ -363,20 +369,21 @@ mod tests { #[async_trait] impl DojoStoreTrait for FakeStore { - type Error = FakeStoreError; - + async fn initialize(&self) -> DojoToriiResult { + Ok(()) + } async fn save_table( &self, _owner: &Felt, _table: &DojoTable, _tx_hash: &Felt, block_number: u64, - ) -> Result<(), Self::Error> { + ) -> DojoToriiResult { self.saved_blocks.lock().unwrap().push(block_number); Ok(()) } - async fn read_tables(&self, _owners: &[Felt]) -> Result, Self::Error> { + async fn read_tables(&self, _owners: &[Felt]) -> DojoToriiResult> { Ok(Vec::new()) } } diff --git a/crates/dojo/src/error.rs b/crates/dojo/src/error.rs index 911f7550..e9d4cc78 100644 --- a/crates/dojo/src/error.rs +++ b/crates/dojo/src/error.rs @@ -1,10 +1,10 @@ -use std::sync::PoisonError; - use dojo_introspect::DojoIntrospectError; use introspect_types::transcode::TranscodeError; use introspect_types::DecodeError; use starknet::core::utils::NonAsciiNameError; use starknet_types_core::felt::Felt; +use std::error::Error as StdError; +use std::sync::PoisonError; #[derive(Debug, thiserror::Error)] pub enum DojoToriiError { @@ -12,8 +12,8 @@ pub enum DojoToriiError { UnknownDojoEventSelector(Felt), #[error("Missing event selector")] MissingEventSelector, - #[error("Column {0:#066x} not found in table {1}")] - ColumnNotFound(Felt, String), + #[error("Column {1:#066x} not found in table {0}")] + ColumnNotFound(String, Felt), #[error("Failed to parse field {0:#066x} in table {1}")] FieldParseError(Felt, String), #[error("Too many values provided for field {0:#066x}")] @@ -26,8 +26,6 @@ pub enum DojoToriiError { TableNotFoundById(Felt), #[error("Failed to acquire lock: {0}")] LockError(String), - #[error("Store error: {0}")] - StoreError(String), #[error("Starknet selector error: {0}")] StarknetSelectorError(#[from] NonAsciiNameError), #[error("Lock poisoned: {0}")] @@ -40,9 +38,13 @@ pub enum DojoToriiError { DojoIntrospectError(#[from] DojoIntrospectError), #[error("Transcode error: {0:?}")] TranscodeError(TranscodeError), + #[error("Store error: {0}")] + StoreError(#[source] Box), + #[error(transparent)] + JsonError(#[from] serde_json::Error), } -pub type DojoToriiResult = std::result::Result; +pub type DojoToriiResult = std::result::Result; impl From> for DojoToriiError { fn from(err: TranscodeError) -> Self { @@ -51,8 +53,8 @@ impl From> for DojoToriiError { } impl DojoToriiError { - pub fn store_error(err: T) -> Self { - Self::StoreError(err.to_string()) + pub fn store_error(err: T) -> Self { + Self::StoreError(Box::new(err)) } } diff --git a/crates/dojo/src/event.rs b/crates/dojo/src/event.rs index 3ae7b61b..a3034ccf 100644 --- a/crates/dojo/src/event.rs +++ b/crates/dojo/src/event.rs @@ -35,7 +35,6 @@ where raw, ) .await - .map(Into::into) } } @@ -55,7 +54,6 @@ where decoder .register_table(&raw.from_address, &self.namespace, &self.name, schema, raw) .await - .map(Into::into) } } @@ -75,7 +73,6 @@ where decoder .register_table(&raw.from_address, &self.namespace, &self.name, schema, raw) .await - .map(Into::into) } } diff --git a/crates/dojo/src/store/json.rs b/crates/dojo/src/store/json.rs index a3349671..d3988b1d 100644 --- a/crates/dojo/src/store/json.rs +++ b/crates/dojo/src/store/json.rs @@ -1,6 +1,7 @@ use super::DojoStoreTrait; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiResult}; use async_trait::async_trait; +use introspect_types::ResultInto; use serde_json::Error as JsonError; use starknet_types_core::felt::Felt; use std::fs; @@ -68,19 +69,24 @@ impl JsonStore { #[async_trait] impl DojoStoreTrait for JsonStore { - type Error = JsonError; - + async fn initialize(&self) -> DojoToriiResult { + // No initialization needed for JSON store, but we can check if the path is accessible. + if !self.path.exists() { + std::fs::create_dir_all(&self.path).map_err(JsonError::io)?; + } + Ok(()) + } async fn save_table( &self, _owner: &Felt, table: &DojoTable, _tx_hash: &Felt, _block_number: u64, - ) -> Result<(), Self::Error> { - self.dump_table(table) + ) -> DojoToriiResult { + self.dump_table(table).err_into() } - async fn read_tables(&self, _owners: &[Felt]) -> Result, Self::Error> { - self.load_all_tables() + async fn read_tables(&self, _owners: &[Felt]) -> DojoToriiResult> { + self.load_all_tables().err_into() } } diff --git a/crates/dojo/src/store/mod.rs b/crates/dojo/src/store/mod.rs index 148ad129..946ffc60 100644 --- a/crates/dojo/src/store/mod.rs +++ b/crates/dojo/src/store/mod.rs @@ -1,31 +1,35 @@ pub mod json; + +#[cfg(feature = "postgres")] pub mod postgres; +#[cfg(feature = "sqlite")] pub mod sqlite; +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub mod sql; + use crate::table::DojoTableInfo; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiResult}; use async_trait::async_trait; use starknet_types_core::felt::Felt; use std::collections::HashMap; #[async_trait] -pub trait DojoStoreTrait -where - Self: Send + Sync + 'static + Sized, -{ - type Error: std::error::Error; +pub trait DojoStoreTrait { + async fn initialize(&self) -> DojoToriiResult; async fn save_table( &self, owner: &Felt, table: &DojoTable, tx_hash: &Felt, block_number: u64, - ) -> Result<(), Self::Error>; - async fn read_tables(&self, owners: &[Felt]) -> Result, Self::Error>; + ) -> DojoToriiResult; + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult>; async fn read_table_map( &self, owners: &[Felt], - ) -> Result, Self::Error> { + ) -> DojoToriiResult> { Ok(self .read_tables(owners) .await? diff --git a/crates/dojo/src/store/postgres.rs b/crates/dojo/src/store/postgres.rs index 995aa859..3462183c 100644 --- a/crates/dojo/src/store/postgres.rs +++ b/crates/dojo/src/store/postgres.rs @@ -1,23 +1,20 @@ use super::DojoStoreTrait; use crate::decoder::primary_field_def; use crate::table::DojoTableInfo; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiError, DojoToriiResult}; use async_trait::async_trait; -use introspect_types::{Attribute, ColumnInfo, ResultInto, TypeDef}; +use introspect_types::{Attribute, ColumnInfo, TypeDef}; use itertools::Itertools; use sqlx::migrate::Migrator; -use sqlx::postgres::PgArguments; use sqlx::query::Query; use sqlx::types::Json; -use sqlx::{FromRow, Postgres}; +use sqlx::FromRow; use starknet_types_core::felt::Felt; use std::collections::HashMap; -use std::ops::Deref; -use torii_common::sql::SqlxResult; use torii_introspect::postgres::owned::PgTypeDef; use torii_introspect::postgres::PgFelt; use torii_introspect::schema::ColumnKeyTrait; -use torii_postgres::db::PostgresConnection; +use torii_sql::{PgArguments, PgPool, PoolExt, Postgres, SqlxResult}; pub const FETCH_TABLES_QUERY: &str = r#" SELECT DISTINCT ON (owner, id) @@ -87,7 +84,7 @@ pub const INSERT_COLUMN_QUERY: &str = r#" updated_at = NOW(), updated_tx = EXCLUDED.updated_tx"#; -pub const DOJO_STORE_MIGRATIONS: Migrator = sqlx::migrate!(); +pub const DOJO_STORE_MIGRATIONS: Migrator = sqlx::migrate!("./migrations/postgres"); #[derive(Debug, thiserror::Error)] pub enum DojoPgStoreError { @@ -101,8 +98,6 @@ pub enum DojoPgStoreError { table_id: Felt, column_id: Felt, }, - #[error("Duplicate tables found for owner {owner:?} and table id {table_id}")] - DuplicateTables { owner: Felt, table_id: Felt }, } impl DojoPgStoreError { @@ -191,34 +186,6 @@ where } } -// #[async_trait] -// impl PgTypeDef for DojoTableInfo { -// type Row = DojoTableRow; -// async fn get_rows( -// pool: &PgPool, -// query: &'static str, -// owners: &[Felt], -// ) -> SqlxResult> { -// Self::get_pg_rows(pool, query, owners) -// .await -// .map(|rows| rows.into_iter().map_into().collect_vec()) -// } -// } - -// #[async_trait] -// impl PgTypeDef<()> for DojoTable { -// type Row = DojoTableRow; -// async fn get_rows( -// pool: &PgPool, -// query: &'static str, -// owners: &[Felt], -// ) -> SqlxResult> { -// Self::get_pg_rows(pool, query, owners) -// .await -// .map(|rows| rows.into_iter().map_into().collect_vec()) -// } -// } - pub fn table_insert_query( owner: &Felt, id: &Felt, @@ -309,42 +276,44 @@ impl DojoTable { pub struct PgStore(pub T); -impl Deref for PgStore { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.0 +impl> PoolExt for PgStore { + fn pool(&self) -> &PgPool { + self.0.pool() } } -impl PgStore { +impl + Send + Sync> PgStore { pub async fn initialize(&self) -> SqlxResult<()> { self.migrate(Some("dojo"), DOJO_STORE_MIGRATIONS).await } } -impl From for PgStore { +impl> From for PgStore { fn from(pool: T) -> Self { PgStore(pool) } } #[async_trait] -impl DojoStoreTrait for PgStore { - type Error = DojoPgStoreError; - +impl DojoStoreTrait for PgPool { + async fn initialize(&self) -> DojoToriiResult { + self.migrate(Some("dojo"), DOJO_STORE_MIGRATIONS) + .await + .map_err(DojoToriiError::store_error) + } async fn save_table( &self, owner: &Felt, table: &DojoTable, tx_hash: &Felt, block_number: u64, - ) -> Result<(), Self::Error> { - let mut transaction = self.begin().await?; + ) -> DojoToriiResult { + let mut transaction = self.begin().await.map_err(DojoToriiError::store_error)?; table .insert_query(owner, tx_hash, block_number) .execute(&mut *transaction) - .await?; + .await + .map_err(DojoToriiError::store_error)?; for (id, info) in &table.columns { column_info_insert_query( INSERT_COLUMN_QUERY, @@ -356,27 +325,33 @@ impl DojoStoreTrait for PgStore Result, Self::Error> { + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { let mut tables = PgTypeDef::get_rows::(self.pool(), FETCH_TABLES_QUERY, owners) - .await? + .await + .map_err(DojoToriiError::store_error)? .into_iter() .map(|row: ((), DojoTable)| row.1) .collect_vec(); let mut columns: HashMap<(Felt, Felt), _> = ColumnInfo::get_hash_map::(self.pool(), FETCH_COLUMNS_QUERY, owners) - .await?; + .await + .map_err(DojoToriiError::store_error)?; for table in &mut tables { for key in table.key_fields.iter().chain(table.value_fields.iter()) { - let column = columns.remove(&(table.id, *key)).ok_or_else(|| { - DojoPgStoreError::column_not_found(table.name.clone(), &(table.id, *key)) - })?; + let column = columns + .remove(&(table.id, *key)) + .ok_or_else(|| DojoToriiError::ColumnNotFound(table.name.clone(), *key))?; table.columns.insert(*key, column); } } diff --git a/crates/dojo/src/store/sql.rs b/crates/dojo/src/store/sql.rs new file mode 100644 index 00000000..14cb676b --- /dev/null +++ b/crates/dojo/src/store/sql.rs @@ -0,0 +1,35 @@ +use crate::store::DojoStoreTrait; +use crate::{DojoTable, DojoToriiResult}; +use async_trait::async_trait; +use starknet_types_core::felt::Felt; +use torii_sql::DbPool; + +#[async_trait] +impl DojoStoreTrait for DbPool { + async fn initialize(&self) -> DojoToriiResult { + match self { + DbPool::Postgres(pool) => pool.initialize().await, + DbPool::Sqlite(pool) => pool.initialize().await, + } + } + + async fn save_table( + &self, + owner: &Felt, + table: &DojoTable, + tx_hash: &Felt, + block_number: u64, + ) -> DojoToriiResult { + match self { + DbPool::Postgres(pool) => pool.save_table(owner, table, tx_hash, block_number).await, + DbPool::Sqlite(pool) => pool.save_table(owner, table, tx_hash, block_number).await, + } + } + + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { + match self { + DbPool::Postgres(pool) => pool.read_tables(owners).await, + DbPool::Sqlite(pool) => pool.read_tables(owners).await, + } + } +} diff --git a/crates/dojo/src/store/sqlite.rs b/crates/dojo/src/store/sqlite.rs index fd348d5e..4bd41068 100644 --- a/crates/dojo/src/store/sqlite.rs +++ b/crates/dojo/src/store/sqlite.rs @@ -1,21 +1,20 @@ use super::DojoStoreTrait; use crate::decoder::primary_field_def; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiError, DojoToriiResult}; use async_trait::async_trait; use introspect_types::ColumnInfo; use serde::{Deserialize, Serialize}; use sqlx::migrate::Migrator; use sqlx::sqlite::SqliteArguments; -use sqlx::Arguments; -use sqlx::{FromRow, Sqlite}; +use sqlx::{Arguments, FromRow, Sqlite, SqlitePool}; use starknet_types_core::felt::Felt; use std::collections::HashMap; use std::io; use std::ops::Deref; -use torii_common::sql::SqlxResult; use torii_common::{blob_to_felt, felt_to_blob}; use torii_introspect::schema::ColumnKeyTrait; -use torii_sqlite::SqliteConnection; +use torii_sql::sqlite::SqliteDbConnection; +use torii_sql::{PoolExt, SqlxResult}; pub const DOJO_SQLITE_STORE_MIGRATIONS: Migrator = sqlx::migrate!("./migrations/sqlite"); @@ -162,31 +161,34 @@ impl Deref for SqliteStore { } } -impl SqliteStore { +impl SqliteStore { pub async fn initialize(&self) -> SqlxResult<()> { self.migrate(Some("dojo"), DOJO_SQLITE_STORE_MIGRATIONS) .await } } -impl From for SqliteStore { +impl From for SqliteStore { fn from(pool: T) -> Self { Self(pool) } } #[async_trait] -impl DojoStoreTrait for SqliteStore { - type Error = DojoSqliteStoreError; - +impl DojoStoreTrait for SqlitePool { + async fn initialize(&self) -> DojoToriiResult { + self.migrate(Some("dojo"), DOJO_SQLITE_STORE_MIGRATIONS) + .await + .map_err(DojoToriiError::store_error) + } async fn save_table( &self, owner: &Felt, table: &DojoTable, tx_hash: &Felt, block_number: u64, - ) -> Result<(), Self::Error> { - let mut transaction = self.begin().await?; + ) -> DojoToriiResult { + let mut transaction = self.begin().await.map_err(DojoToriiError::store_error)?; sqlx::query( r" @@ -227,7 +229,8 @@ impl DojoStoreTrait for SqliteStore .bind(felt_to_blob(*tx_hash)) .bind(felt_to_blob(*tx_hash)) .execute(&mut *transaction) - .await?; + .await + .map_err(DojoToriiError::store_error)?; for (id, info) in &table.columns { let payload = StoredColumnInfo { @@ -248,37 +251,45 @@ impl DojoStoreTrait for SqliteStore .bind(felt_to_blob(*id)) .bind(serde_json::to_string(&payload)?) .execute(&mut *transaction) - .await?; + .await + .map_err(DojoToriiError::store_error)?; } - transaction.commit().await?; + transaction + .commit() + .await + .map_err(DojoToriiError::store_error)?; Ok(()) } - async fn read_tables(&self, owners: &[Felt]) -> Result, Self::Error> { + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { let (table_query, table_args) = select_table_query(owners); let rows = sqlx::query_as_with::(&table_query, table_args) .fetch_all(self.pool()) - .await?; + .await + .map_err(DojoToriiError::store_error)?; let mut tables = rows .into_iter() .map(table_row_into_table) - .collect::, _>>()?; + .collect::, _>>() + .map_err(DojoToriiError::store_error)?; let (column_query, column_args) = select_column_query(owners); let mut columns: HashMap<(Felt, Felt), ColumnInfo> = sqlx::query_as_with::(&column_query, column_args) .fetch_all(self.pool()) - .await? + .await + .map_err(DojoToriiError::store_error)? .into_iter() .map(column_row_into_entry::<(Felt, Felt)>) - .collect::, _>>()?; + .collect::, _>>() + .map_err(DojoToriiError::store_error)?; for table in &mut tables { for key in table.key_fields.iter().chain(table.value_fields.iter()) { - let column = columns.remove(&(table.id, *key)).ok_or_else(|| { - DojoSqliteStoreError::column_not_found(table.name.clone(), &(table.id, *key)) - })?; + let column = columns + .remove(&(table.id, *key)) + .ok_or_else(|| DojoToriiError::ColumnNotFound(table.name.clone(), *key))?; table.columns.insert(*key, column); } } diff --git a/crates/dojo/src/table.rs b/crates/dojo/src/table.rs index 05048a88..674f4808 100644 --- a/crates/dojo/src/table.rs +++ b/crates/dojo/src/table.rs @@ -178,7 +178,7 @@ impl DojoTable { pub fn get_column(&self, selector: &Felt) -> DojoToriiResult<&ColumnInfo> { self.columns .get(selector) - .ok_or_else(|| DojoToriiError::ColumnNotFound(*selector, self.name.clone())) + .ok_or_else(|| DojoToriiError::ColumnNotFound(self.name.clone(), *selector)) } pub fn selectors(&self) -> impl Iterator + '_ { @@ -270,7 +270,7 @@ impl DojoTableInfo { pub fn get_column(&self, selector: &Felt) -> DojoToriiResult<&ColumnInfo> { self.columns .get(selector) - .ok_or_else(|| DojoToriiError::ColumnNotFound(*selector, self.name.clone())) + .ok_or_else(|| DojoToriiError::ColumnNotFound(self.name.clone(), *selector)) } pub fn selectors(&self) -> impl Iterator + '_ { diff --git a/crates/introspect-postgres-sink/Cargo.toml b/crates/introspect-postgres-sink/Cargo.toml deleted file mode 100644 index 816f88dd..00000000 --- a/crates/introspect-postgres-sink/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "torii-introspect-postgres-sink" -version = "0.1.0" -edition = "2021" -description = "PostgreSQL sink implementation for Torii runtime" -authors = ["Torii Runtime "] -license = "Apache-2.0" - -[dependencies] -sqlx = { workspace = true, features = [ - "postgres", - "sqlite", - "runtime-tokio-rustls", - "macros", - "migrate", -] } -anyhow.workspace = true -async-trait.workspace = true -metrics.workspace = true -xxhash-rust.workspace = true -hex.workspace = true -serde.workspace = true -serde_json.workspace = true -starknet.workspace = true -tokio.workspace = true -torii.workspace = true -tracing.workspace = true -starknet-types-core.workspace = true -thiserror.workspace = true -introspect-types.workspace = true -itertools.workspace = true - -# Local crates -torii-introspect.workspace = true -torii-postgres.workspace = true -torii-common.workspace = true diff --git a/crates/introspect-postgres-sink/src/error.rs b/crates/introspect-postgres-sink/src/error.rs deleted file mode 100644 index 0dda8017..00000000 --- a/crates/introspect-postgres-sink/src/error.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::sync::PoisonError; - -use introspect_types::{PrimaryTypeDef, TypeDef}; -use sqlx::Error as SqlxError; -use starknet_types_core::felt::Felt; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum PgTypeError { - #[error("Unsupported type for {0}")] - UnsupportedType(String), - #[error("Nested arrays are not supported")] - NestedArrays, -} - -pub type PgTypeResult = std::result::Result; - -#[derive(Debug, Error)] -pub enum PgTableError { - #[error("Column with id: {0} not found in table {1}")] - ColumnNotFound(Felt, String), - #[error(transparent)] - TypeError(#[from] PgTypeError), - #[error("Current type mismatch error")] - TypeMismatch, - #[error("Unsupported upgrade for table {table} column {column}: {reason}")] - UnsupportedUpgrade { - table: String, - column: String, - reason: UpgradeError, - }, -} - -pub type TableResult = std::result::Result; - -#[derive(Debug, thiserror::Error)] -pub enum UpgradeError { - #[error("Failed to upgrade type from {old} to {new}")] - TypeUpgradeError { - old: &'static str, - new: &'static str, - }, - #[error("Failed to upgrade primary from {old} to {new}")] - PrimaryUpgradeError { - old: &'static str, - new: &'static str, - }, - #[error(transparent)] - TypeCreationError(#[from] PgTypeError), - #[error("Array length cannot be decreased from {old} to {new}")] - ArrayLengthDecreaseError { old: u32, new: u32 }, - #[error("Cannot reduce element in tuple")] - TupleReductionError, -} - -pub type UpgradeResult = Result; - -impl UpgradeError { - pub fn type_upgrade_err(old: &TypeDef, new: &TypeDef) -> UpgradeResult { - Err(Self::TypeUpgradeError { - old: old.item_name(), - new: new.item_name(), - }) - } - pub fn type_upgrade_to_err(old: &TypeDef, new: &'static str) -> UpgradeResult { - Err(Self::TypeUpgradeError { - old: old.item_name(), - new, - }) - } - pub fn type_cast_err(old: &TypeDef, new: &'static str) -> UpgradeResult { - Err(Self::TypeUpgradeError { - old: old.item_name(), - new, - }) - } - pub fn array_shorten_err(old: u32, new: u32) -> UpgradeResult { - Err(Self::ArrayLengthDecreaseError { old, new }) - } - pub fn primary_upgrade_err(old: &PrimaryTypeDef, new: &PrimaryTypeDef) -> UpgradeResult { - Err(Self::PrimaryUpgradeError { - old: old.item_name(), - new: new.item_name(), - }) - } -} - -pub trait UpgradeResultExt { - fn to_table_result(self, table: &str, column: &str) -> TableResult; -} - -impl UpgradeResultExt for UpgradeResult { - fn to_table_result(self, table: &str, column: &str) -> TableResult { - self.map_err(|err| PgTableError::UnsupportedUpgrade { - table: table.to_string(), - column: column.to_string(), - reason: err, - }) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum PgDbError { - #[error(transparent)] - DatabaseError(#[from] SqlxError), - #[error("Invalid event format: {0}")] - InvalidEventFormat(String), - #[error(transparent)] - JsonError(#[from] serde_json::Error), - #[error(transparent)] - IoError(#[from] std::io::Error), - #[error(transparent)] - TableError(#[from] PgTableError), - #[error(transparent)] - TypeError(#[from] PgTypeError), - #[error("Table with id: {0} already exists, incoming name: {1}, existing name: {2}")] - TableAlreadyExists(Felt, String, String), - #[error("Table not found with id: {0}")] - TableNotFound(Felt), - #[error("Table not alive - id: {0}, name: {1}")] - TableNotAlive(Felt, String), - #[error("Manager does not support updating")] - UpdateNotSupported, - #[error("Table poison error: {0}")] - PoisonError(String), -} - -pub type PgDbResult = std::result::Result; - -impl From> for PgDbError { - fn from(err: PoisonError) -> Self { - Self::PoisonError(err.to_string()) - } -} diff --git a/crates/introspect-postgres-sink/src/lib.rs b/crates/introspect-postgres-sink/src/lib.rs deleted file mode 100644 index 83d42122..00000000 --- a/crates/introspect-postgres-sink/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod create; -pub mod error; -pub mod json; -pub mod processor; -pub mod query; -pub mod sink; -pub mod table; -pub mod types; -pub mod upgrade; -pub mod utils; - -pub use error::{ - PgDbError, PgDbResult, PgTableError, PgTypeError, PgTypeResult, TableResult, UpgradeError, - UpgradeResult, UpgradeResultExt, -}; -pub use processor::IntrospectPgDb; -pub use types::{ - PgSchema, PostgresArray, PostgresField, PostgresScalar, PostgresType, PrimaryKey, SchemaName, -}; -pub use utils::{truncate, HasherExt}; - -pub const INTROSPECT_PG_SINK_MIGRATIONS: sqlx::migrate::Migrator = sqlx::migrate!("./migrations"); diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs deleted file mode 100644 index 196a7ef1..00000000 --- a/crates/introspect-postgres-sink/src/processor.rs +++ /dev/null @@ -1,334 +0,0 @@ -use crate::json::PostgresJsonSerializer; -use crate::query::{fetch_columns, fetch_dead_fields, fetch_tables, CreatePgTable}; -use crate::table::{DeadField, PgTable}; -use crate::{PgDbError, PgDbResult, PgSchema, INTROSPECT_PG_SINK_MIGRATIONS}; -use introspect_types::ColumnInfo; -use serde_json::Serializer as JsonSerializer; -use sqlx::PgPool; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use std::io::Write; -use std::ops::Deref; -use std::rc::Rc; -use std::sync::RwLock; -use torii::etl::envelope::MetaData; -use torii_common::sql::{PgQuery, Queries}; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_introspect::schema::TableSchema; -use torii_introspect::InsertsFields; -use torii_postgres::PostgresConnection; - -pub const COMMIT_CMD: &str = "--COMMIT"; -pub const DEAD_MEMBERS_TABLE: &str = "__introspect_dead_fields"; -pub const TABLES_TABLE: &str = "__introspect_tables"; -pub const COLUMNS_TABLE: &str = "__introspect_columns"; -pub const METADATA_CONFLICTS: &str = "__updated_at = NOW(), __updated_block = EXCLUDED.__updated_block, __updated_tx = EXCLUDED.__updated_tx"; - -#[derive(Debug, Default)] -pub struct PostgresTables(pub RwLock>); - -#[derive(Debug, Default)] -pub struct DeadFields(pub RwLock>>); - -impl Deref for PostgresTables { - type Target = RwLock>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Deref for DeadFields { - type Target = RwLock>>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl From> for PgSchema { - fn from(value: Option) -> Self { - match value { - Some(s) => Self::Custom(s), - None => PgSchema::Public, - } - } -} - -impl From<()> for PgSchema { - fn from(_: ()) -> Self { - PgSchema::Public - } -} - -impl From for PgSchema { - fn from(value: String) -> Self { - Self::Custom(value) - } -} - -impl From<&str> for PgSchema { - fn from(value: &str) -> Self { - Self::Custom(value.to_string()) - } -} - -impl From> for PgSchema { - fn from(value: Option<&str>) -> Self { - match value { - Some(s) => Self::Custom(s.to_string()), - None => PgSchema::Public, - } - } -} - -impl PostgresTables { - pub fn create_table( - &self, - schema: &Rc, - to_table: impl Into, - metadata: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - let (id, table) = Into::::into(to_table).into(); - self.assert_table_not_exists(&id, &table.name)?; - CreatePgTable::new(schema, &id, &table)?.make_queries(queries); - let table = PgTable::new(schema, table, None); - table.insert_queries( - &id, - None, - metadata.block_number.unwrap_or_default(), - metadata.transaction_hash, - queries, - )?; - let mut tables: std::sync::RwLockWriteGuard<'_, HashMap> = self.write()?; - tables.insert(id, table); - Ok(()) - } - - pub fn update_table( - &self, - to_table: impl Into, - metadata: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - let tx_hash = &metadata.transaction_hash; - let block_number = metadata.block_number.unwrap_or_default(); - let (id, table) = Into::::into(to_table).into(); - let mut tables = self.write()?; - let existing = tables - .get_mut(&id) - .ok_or_else(|| PgDbError::TableNotFound(id))?; - let upgrades = existing.update_from_info(&id, &table)?; - upgrades.to_queries(&id, block_number, tx_hash, queries)?; - existing.insert_queries( - &id, - Some(&upgrades.columns_upgraded), - block_number, - metadata.transaction_hash, - queries, - ) - } - - pub fn assert_table_not_exists(&self, id: &Felt, name: &str) -> PgDbResult<()> { - match self.read()?.get(id) { - Some(existing) => Err(PgDbError::TableAlreadyExists( - *id, - name.to_string(), - existing.name.to_string(), - )), - None => Ok(()), - } - } - - pub fn set_table_dead(&self, id: &Felt) -> PgDbResult<()> { - let mut tables = self.write()?; - match tables.get_mut(id) { - Some(table) => { - table.alive = false; - Ok(()) - } - None => Err(PgDbError::TableNotFound(*id)), - } - } - - pub fn insert_fields( - &self, - event: &InsertsFields, - context: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - let tables = self.read().unwrap(); - let table = match tables.get(&event.table) { - Some(table) => Ok(table), - None => Err(PgDbError::TableNotFound(event.table)), - }?; - if !table.alive { - return Ok(()); - } - let record = table.get_record_schema(&event.columns)?; - let table_name = &table.name; - let mut writer = Vec::new(); - let schema = &table.schema; - write!( - writer, - r#"INSERT INTO "{schema}"."{table_name}" SELECT * FROM jsonb_populate_recordset(NULL::"{schema}"."{table_name}", $$"# - ) - .unwrap(); - record.parse_records_with_metadata( - &event.records, - context, - &mut JsonSerializer::new(&mut writer), - &PostgresJsonSerializer, - )?; - write!( - writer, - r#"$$) ON CONFLICT ("{}") DO UPDATE SET {METADATA_CONFLICTS}"#, - record.primary().name - ) - .unwrap(); - for ColumnInfo { name, .. } in record.columns() { - write!( - writer, - r#", "{name}" = COALESCE(EXCLUDED."{name}", "{table_name}"."{name}")"#, - name = name - ) - .unwrap(); - } - let string = unsafe { String::from_utf8_unchecked(writer) }; - queries.add(string); - Ok(()) - } - - pub fn handle_message( - &self, - schema: &Rc, - msg: &IntrospectMsg, - metadata: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - match msg { - IntrospectMsg::CreateTable(event) => { - self.create_table(schema, event.clone(), metadata, queries) - } - IntrospectMsg::UpdateTable(event) => { - self.update_table(event.clone(), metadata, queries) - } - IntrospectMsg::AddColumns(event) => self.set_table_dead(&event.table), - IntrospectMsg::DropColumns(event) => self.set_table_dead(&event.table), - IntrospectMsg::RetypeColumns(event) => self.set_table_dead(&event.table), - IntrospectMsg::RetypePrimary(event) => self.set_table_dead(&event.table), - IntrospectMsg::RenameTable(_) - | IntrospectMsg::DropTable(_) - | IntrospectMsg::RenameColumns(_) - | IntrospectMsg::RenamePrimary(_) => Ok(()), - IntrospectMsg::InsertsFields(event) => self.insert_fields(event, metadata, queries), - IntrospectMsg::DeleteRecords(_) | IntrospectMsg::DeletesFields(_) => Ok(()), - } - } -} - -fn make_schema_query(schema: &PgSchema) -> String { - format!(r#"CREATE SCHEMA IF NOT EXISTS "{schema}""#) -} - -pub struct IntrospectPgDb { - tables: PostgresTables, - schema: PgSchema, - pool: T, -} - -impl PostgresConnection for IntrospectPgDb { - fn pool(&self) -> &PgPool { - self.pool.pool() - } -} - -impl IntrospectPgDb { - pub fn new(pool: T, schema: impl Into) -> Self { - Self { - tables: PostgresTables::default(), - schema: schema.into(), - pool, - } - } - - pub async fn load_store_data(&self) -> PgDbResult<()> { - let mut tables = fetch_tables(self.pool(), &self.schema) - .await? - .into_iter() - .map(|t| t.to_table(&self.schema)) - .collect::>(); - for (table_id, id, column_info) in fetch_columns(self.pool(), &self.schema).await? { - if let Some(table) = tables.get_mut(&table_id) { - table.columns.insert(id, column_info); - } - } - for (table_id, id, field) in fetch_dead_fields(self.pool(), &self.schema).await? { - if let Some(table) = tables.get_mut(&table_id) { - table.dead.insert(id, field); - } - } - let mut tables_map = self.tables.write()?; - tables_map.extend(tables); - Ok(()) - } - - pub async fn initialize_introspect_pg_sink(&self) -> PgDbResult<()> { - self.migrate(Some("introspect"), INTROSPECT_PG_SINK_MIGRATIONS) - .await?; - self.execute_queries(make_schema_query(&self.schema)) - .await?; - self.load_store_data().await - } - - pub async fn process_message( - &self, - msg: &IntrospectMsg, - metadata: &MetaData, - ) -> PgDbResult<()> { - let mut queries = Vec::new(); - { - let schema = Rc::new(self.schema.clone()); - self.tables - .handle_message(&schema, msg, metadata, &mut queries)?; - } - self.execute_queries(queries).await?; - Ok(()) - } - - pub async fn process_messages( - &self, - msgs: Vec<&IntrospectBody>, - ) -> PgDbResult>> { - let mut queries = Vec::new(); - let mut results = Vec::with_capacity(msgs.len()); - { - let schema = Rc::new(self.schema.clone()); - for body in msgs { - let (msg, metadata) = body.into(); - results.push( - self.tables - .handle_message(&schema, msg, metadata, &mut queries), - ); - } - } - let mut batch = Vec::new(); - for query in queries { - if query == *COMMIT_CMD { - self.execute_queries(std::mem::take(&mut batch)).await?; - } else { - batch.push(query); - } - } - if !batch.is_empty() { - self.execute_queries(batch).await?; - } - Ok(results) - } -} - -pub struct MessageWithContext<'a, M> { - pub msg: &'a M, - pub context: &'a MetaData, -} diff --git a/crates/introspect-postgres-sink/src/sink.rs b/crates/introspect-postgres-sink/src/sink.rs deleted file mode 100644 index d33b260b..00000000 --- a/crates/introspect-postgres-sink/src/sink.rs +++ /dev/null @@ -1,102 +0,0 @@ -use crate::processor::IntrospectPgDb; -use anyhow::Result; -use async_trait::async_trait; -use std::sync::Arc; -use torii::axum::Router; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, SinkContext, TopicInfo}, -}; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_postgres::PostgresConnection; - -pub const LOGGING_TARGET: &str = "torii::sinks::introspect::postgres"; -const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); - -#[async_trait] -impl Sink for IntrospectPgDb { - fn name(&self) -> &'static str { - "introspect-postgres" - } - - fn interested_types(&self) -> Vec { - vec![INTROSPECT_TYPE] - } - - async fn process(&self, envelopes: &[Envelope], _batch: &ExtractionBatch) -> Result<()> { - let mut processed = 0usize; - let mut create_tables: usize = 0usize; - let mut update_tables = 0usize; - let mut inserts_fields = 0usize; - let mut inserted_records = 0usize; - let mut delete_records = 0usize; - let mut msgs = Vec::with_capacity(envelopes.len()); - for envelope in envelopes { - if envelope.type_id == INTROSPECT_TYPE { - if let Some(body) = envelope.downcast_ref::() { - match &body.msg { - IntrospectMsg::CreateTable(_) => create_tables += 1, - IntrospectMsg::UpdateTable(_) => update_tables += 1, - IntrospectMsg::InsertsFields(event) => { - inserts_fields += 1; - inserted_records += event.records.len(); - } - IntrospectMsg::DeleteRecords(event) => { - delete_records += event.rows.len(); - } - _ => {} - } - processed += 1; - msgs.push(body); - } - } - } - self.process_messages(msgs).await?; - if processed > 0 { - tracing::info!( - target: LOGGING_TARGET, - processed, - create_tables, - update_tables, - inserts_fields, - inserted_records, - delete_records, - "Processed introspect envelopes" - ); - ::metrics::counter!("torii_introspect_sink_messages_total", "message" => "create_table") - .increment(create_tables as u64); - ::metrics::counter!("torii_introspect_sink_messages_total", "message" => "update_table") - .increment(update_tables as u64); - ::metrics::counter!("torii_introspect_sink_messages_total", "message" => "inserts_fields") - .increment(inserts_fields as u64); - ::metrics::counter!("torii_introspect_sink_records_total", "message" => "inserts_fields") - .increment(inserted_records as u64); - ::metrics::counter!("torii_introspect_sink_records_total", "message" => "delete_records") - .increment(delete_records as u64); - } - - Ok(()) - } - - fn topics(&self) -> Vec { - Vec::new() - } - - fn build_routes(&self) -> Router { - Router::new() - } - - async fn initialize( - &mut self, - _event_bus: Arc, - _context: &SinkContext, - ) -> Result<()> { - self.initialize_introspect_pg_sink().await?; - tracing::info!( - target: LOGGING_TARGET, - "Initialized introspect Postgres sink" - ); - Ok(()) - } -} diff --git a/crates/introspect-postgres-sink/src/table.rs b/crates/introspect-postgres-sink/src/table.rs deleted file mode 100644 index eeef1a9c..00000000 --- a/crates/introspect-postgres-sink/src/table.rs +++ /dev/null @@ -1,104 +0,0 @@ -use introspect_types::{ColumnInfo, MemberDef, PrimaryDef, TypeDef}; -use itertools::Itertools; -use sqlx::Error::Encode as EncodeError; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use torii_common::sql::PgQuery; -use torii_introspect::{schema::TableInfo, tables::RecordSchema}; - -use crate::{ - query::{insert_columns_query, insert_table_query}, - PgDbResult, PgSchema, PgTableError, TableResult, -}; - -#[derive(Debug)] -pub struct PgTable { - pub schema: PgSchema, - pub name: String, - pub primary: PrimaryDef, - pub columns: HashMap, - pub alive: bool, - pub dead: HashMap, -} - -#[derive(Debug)] -pub struct DeadField { - pub name: String, - pub type_def: TypeDef, -} - -impl From for DeadField { - fn from(value: MemberDef) -> Self { - DeadField { - name: value.name, - type_def: value.type_def, - } - } -} - -impl From for MemberDef { - fn from(value: DeadField) -> Self { - MemberDef { - name: value.name, - attributes: Vec::new(), - type_def: value.type_def, - } - } -} - -impl PgTable { - pub fn column(&self, id: &Felt) -> TableResult<&ColumnInfo> { - self.columns - .get(id) - .ok_or_else(|| PgTableError::ColumnNotFound(*id, self.name.clone())) - } - - pub fn columns(&self, ids: &[Felt]) -> TableResult> { - ids.iter() - .map(|id| self.column(id)) - .collect::>>() - } - - pub fn new(schema: &PgSchema, info: TableInfo, dead: Option>) -> Self { - PgTable { - schema: schema.clone(), - name: info.name, - primary: info.primary, - columns: info.columns.into_iter().map_into().collect(), - alive: true, - dead: dead.unwrap_or_default().into_iter().collect(), - } - } - pub fn get_record_schema(&self, columns: &[Felt]) -> TableResult> { - Ok(RecordSchema::new(&self.primary, self.columns(columns)?)) - } - pub fn insert_queries( - &self, - id: &Felt, - column_ids: Option<&[Felt]>, - block_number: u64, - transaction_hash: Felt, - queries: &mut Vec, - ) -> PgDbResult<()> { - queries.push( - insert_table_query( - &self.schema, - id, - &self.name, - &self.primary, - block_number, - &transaction_hash, - ) - .map_err(EncodeError)?, - ); - let columns = match column_ids { - Some(ids) => ids.iter().zip(self.columns(ids)?).collect_vec(), - None => self.columns.iter().collect_vec(), - }; - queries.push( - insert_columns_query(&self.schema, id, columns, block_number, &transaction_hash) - .map_err(EncodeError)?, - ); - Ok(()) - } -} diff --git a/crates/introspect-sqlite-sink/Cargo.toml b/crates/introspect-sql-sink/Cargo.toml similarity index 57% rename from crates/introspect-sqlite-sink/Cargo.toml rename to crates/introspect-sql-sink/Cargo.toml index 6113d553..9f4275fb 100644 --- a/crates/introspect-sqlite-sink/Cargo.toml +++ b/crates/introspect-sql-sink/Cargo.toml @@ -1,18 +1,13 @@ [package] -name = "torii-introspect-sqlite-sink" +name = "torii-introspect-sql-sink" version = "0.1.0" edition = "2021" -description = "SQLite sink implementation for Torii runtime" +description = "Database sink implementation for Torii runtime" authors = ["Torii Runtime "] license = "Apache-2.0" [dependencies] -sqlx = { workspace = true, features = [ - "sqlite", - "runtime-tokio-rustls", - "macros", - "migrate", -] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "macros"] } anyhow.workspace = true async-trait.workspace = true metrics.workspace = true @@ -29,8 +24,14 @@ introspect-types.workspace = true itertools.workspace = true primitive-types.workspace = true +# Local crates torii-introspect.workspace = true -torii-sqlite.workspace = true +torii-sql.workspace = true -[lints] -workspace = true +# Postgres features +xxhash-rust = { workspace = true, optional = true } + + +[features] +postgres = ["sqlx/postgres", "dep:xxhash-rust", "torii-sql/postgres"] +sqlite = ["sqlx/sqlite", "torii-sql/sqlite"] diff --git a/crates/introspect-postgres-sink/migrations/001_domains.sql b/crates/introspect-sql-sink/migrations/postgres/001_domains.sql similarity index 100% rename from crates/introspect-postgres-sink/migrations/001_domains.sql rename to crates/introspect-sql-sink/migrations/postgres/001_domains.sql diff --git a/crates/introspect-postgres-sink/migrations/002_metadata_function.sql b/crates/introspect-sql-sink/migrations/postgres/002_metadata_function.sql similarity index 100% rename from crates/introspect-postgres-sink/migrations/002_metadata_function.sql rename to crates/introspect-sql-sink/migrations/postgres/002_metadata_function.sql diff --git a/crates/introspect-postgres-sink/migrations/003_store.sql b/crates/introspect-sql-sink/migrations/postgres/003_store.sql similarity index 94% rename from crates/introspect-postgres-sink/migrations/003_store.sql rename to crates/introspect-sql-sink/migrations/postgres/003_store.sql index d58e2145..461fd11f 100644 --- a/crates/introspect-postgres-sink/migrations/003_store.sql +++ b/crates/introspect-sql-sink/migrations/postgres/003_store.sql @@ -22,7 +22,10 @@ CREATE TABLE IF NOT EXISTS introspect.db_tables ( "schema" TEXT NOT NULL, id felt252 NOT NULL, name TEXT NOT NULL, + "owner" felt252 NOT NULL, primary_def introspect.primary_def NOT NULL, + append_only BOOLEAN NOT NULL DEFAULT FALSE, + alive BOOLEAN NOT NULL DEFAULT TRUE, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), created_block uint64 NOT NULL, diff --git a/crates/introspect-sqlite-sink/migrations/001_init.sql b/crates/introspect-sql-sink/migrations/sqlite/001_init.sql similarity index 100% rename from crates/introspect-sqlite-sink/migrations/001_init.sql rename to crates/introspect-sql-sink/migrations/sqlite/001_init.sql diff --git a/crates/introspect-sql-sink/migrations/sqlite/002_schema_state.sql b/crates/introspect-sql-sink/migrations/sqlite/002_schema_state.sql new file mode 100644 index 00000000..1c345fb2 --- /dev/null +++ b/crates/introspect-sql-sink/migrations/sqlite/002_schema_state.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS introspect_db_tables ( + namespace TEXT NOT NULL, + id TEXT NOT NULL, + owner TEXT NOT NULL, + name TEXT NOT NULL, + "primary" TEXT NOT NULL, + columns TEXT NOT NULL, + append_only INTEGER NOT NULL DEFAULT 0, + alive INTEGER NOT NULL DEFAULT 1, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()), + PRIMARY KEY (namespace, id) +); diff --git a/crates/introspect-sql-sink/src/backend.rs b/crates/introspect-sql-sink/src/backend.rs new file mode 100644 index 00000000..563e2e5e --- /dev/null +++ b/crates/introspect-sql-sink/src/backend.rs @@ -0,0 +1,114 @@ +use crate::processor::{messages_to_queries, DbColumn, DbDeadField, DbTable, COMMIT_CMD}; +use crate::table::Table; +use crate::tables::Tables; +use crate::{DbResult, NamespaceMode, RecordResult, TableResult}; +use async_trait::async_trait; +use introspect_types::{ColumnDef, PrimaryDef}; +use sqlx::{Database, Pool}; +use starknet_types_core::felt::Felt; +use std::fmt::Debug; +use torii_introspect::events::IntrospectBody; +use torii_introspect::Record; +use torii_sql::{Executable, FlexQuery, PoolExt}; + +#[async_trait] +pub trait IntrospectProcessor { + async fn process_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>>; +} + +#[async_trait] +pub trait IntrospectInitialize { + async fn initialize(&self) -> DbResult<()>; + async fn load_tables(&self, namespaces: &Option>) -> DbResult>; + async fn load_columns(&self, namespaces: &Option>) -> DbResult>; + async fn load_dead_fields( + &self, + namespaces: &Option>, + ) -> DbResult>; +} + +#[allow(clippy::too_many_arguments)] +pub trait IntrospectQueryMaker: Database { + fn create_table_queries( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> TableResult<()>; + fn update_table_queries( + table: &mut Table, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> TableResult<()>; + fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> RecordResult<()>; +} + +#[async_trait] +pub trait IntrospectPool { + async fn commit_queries(&self, queries: Vec>) -> DbResult<()>; + async fn execute_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + let mut queries = Vec::new(); + let results = messages_to_queries::(namespaces, tables, msgs, &mut queries)?; + self.commit_queries(queries).await?; + Ok(results) + } +} + +#[async_trait] +impl IntrospectPool for Pool +where + Vec>: Executable, + FlexQuery: Debug + Clone, +{ + async fn commit_queries(&self, queries: Vec>) -> DbResult<()> { + let mut batch = Vec::new(); + for query in queries { + if query == *COMMIT_CMD { + self.execute_queries(std::mem::take(&mut batch)).await?; + } else { + batch.push(query); + } + } + if !batch.is_empty() { + match self.execute_queries(batch.clone()).await { + Ok(_) => (), + Err(e) => { + for query in batch { + eprintln!("Failed query: {query:?}"); + } + return Err(e.into()); + } + } + } + Ok(()) + } +} diff --git a/crates/introspect-sql-sink/src/error.rs b/crates/introspect-sql-sink/src/error.rs new file mode 100644 index 00000000..bacc8750 --- /dev/null +++ b/crates/introspect-sql-sink/src/error.rs @@ -0,0 +1,247 @@ +use crate::TableKey; +use introspect_types::{DecodeError, PrimaryTypeDef, TypeDef}; +use sqlx::error::BoxDynError; +use sqlx::Error as SqlxError; +use starknet_types_core::felt::{Felt, FromStrError}; +use std::fmt::Display; +use std::sync::PoisonError; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TypeError { + #[error("Unsupported type for {0}")] + UnsupportedType(String), + #[error("Nested arrays are not supported")] + NestedArrays, +} + +pub type TypeResult = std::result::Result; + +#[derive(Debug, Error)] +pub enum TableError { + #[error("Table {0} has not got columns: {1:?}")] + ColumnsNotFound(String, ColumnsNotFoundError), + #[error(transparent)] + TypeError(#[from] TypeError), + #[error("Current type mismatch error")] + TypeMismatch, + #[error("Unsupported upgrade for table {table} column {column}: {reason}")] + UnsupportedUpgrade { + table: String, + column: String, + reason: UpgradeError, + }, + #[error(transparent)] + JsonError(#[from] serde_json::Error), + #[error("error occurred while encoding a value: {0}")] + Encode(#[from] BoxDynError), +} + +#[derive(Debug)] +pub struct ColumnNotFoundError(pub Felt); + +#[derive(Debug, Default, Error)] +pub struct ColumnsNotFoundError(pub Vec); + +impl ColumnsNotFoundError { + pub fn to_table_error(self, table: &str) -> TableError { + TableError::ColumnsNotFound(table.to_string(), self) + } +} + +impl Display for ColumnsNotFoundError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some((first, rest)) = self.0.split_first() { + write!(f, "0x{first:#063x}")?; + for col in rest { + write!(f, ", 0x{col:#063x}")?; + } + } + Ok(()) + } +} + +pub trait CollectColumnResults { + fn collect_columns(self) -> Result, ColumnsNotFoundError>; +} + +impl CollectColumnResults for I +where + I: Iterator>, +{ + fn collect_columns(self) -> Result, ColumnsNotFoundError> { + let mut columns = Vec::new(); + let mut not_found = Vec::new(); + for result in self { + match result { + Ok(col) => columns.push(col), + Err(e) => not_found.push(e.0), + } + } + if not_found.is_empty() { + Ok(columns) + } else { + Err(ColumnsNotFoundError(not_found)) + } + } +} + +pub type TableResult = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum UpgradeError { + #[error("Failed to upgrade type from {old} to {new}")] + TypeUpgradeError { + old: &'static str, + new: &'static str, + }, + #[error("Failed to upgrade primary from {old} to {new}")] + PrimaryUpgradeError { + old: &'static str, + new: &'static str, + }, + #[error(transparent)] + TypeCreationError(#[from] TypeError), + #[error("Array length cannot be decreased from {old} to {new}")] + ArrayLengthDecreaseError { old: u32, new: u32 }, + #[error("Cannot reduce element in tuple")] + TupleReductionError, +} + +pub type UpgradeResult = Result; + +impl UpgradeError { + pub fn type_upgrade_err(old: &TypeDef, new: &TypeDef) -> UpgradeResult { + Err(Self::TypeUpgradeError { + old: old.item_name(), + new: new.item_name(), + }) + } + pub fn type_upgrade_to_err(old: &TypeDef, new: &'static str) -> UpgradeResult { + Err(Self::TypeUpgradeError { + old: old.item_name(), + new, + }) + } + pub fn type_cast_err(old: &TypeDef, new: &'static str) -> UpgradeResult { + Err(Self::TypeUpgradeError { + old: old.item_name(), + new, + }) + } + pub fn array_shorten_err(old: u32, new: u32) -> UpgradeResult { + Err(Self::ArrayLengthDecreaseError { old, new }) + } + pub fn primary_upgrade_err(old: &TypeDef, new: &PrimaryTypeDef) -> UpgradeResult { + Err(Self::PrimaryUpgradeError { + old: old.item_name(), + new: new.item_name(), + }) + } +} + +pub trait UpgradeResultExt { + fn to_table_result(self, table: &str, column: &str) -> TableResult; +} + +impl UpgradeResultExt for UpgradeResult { + fn to_table_result(self, table: &str, column: &str) -> TableResult { + self.map_err(|err| TableError::UnsupportedUpgrade { + table: table.to_string(), + column: column.to_string(), + reason: err, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RecordError { + #[error(transparent)] + JsonError(#[from] serde_json::Error), + #[error(transparent)] + TypeError(#[from] TypeError), + #[error("Record does not match table schema")] + SchemaMismatch, + #[error(transparent)] + DecodeError(#[from] DecodeError), + #[error(transparent)] + SqlxError(#[from] SqlxError), + #[error("Columns with ids: {0} not found")] + ColumnsNotFound(#[from] ColumnsNotFoundError), +} + +pub type RecordResult = std::result::Result; + +pub trait RecordResultExt { + fn to_db_result(self, table: &str) -> DbResult; +} + +impl RecordResultExt for RecordResult { + fn to_db_result(self, table: &str) -> DbResult { + self.map_err(|err| DbError::RecordError(table.to_string(), err)) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum DbError { + #[error(transparent)] + DatabaseError(#[from] SqlxError), + #[error("Invalid event format: {0}")] + InvalidEventFormat(String), + #[error(transparent)] + JsonError(#[from] serde_json::Error), + #[error(transparent)] + IoError(#[from] std::io::Error), + #[error(transparent)] + TableError(#[from] TableError), + #[error(transparent)] + TypeError(#[from] TypeError), + #[error("Table with id: {0} already exists, incoming name: {1}, existing name: {2}")] + TableAlreadyExists(TableKey, String, String), + #[error("Table not found with id: {0}")] + TableNotFound(TableKey), + #[error("Table not alive - id: {0}, name: {1}")] + TableNotAlive(Felt, String), + #[error("Manager does not support updating")] + UpdateNotSupported, + #[error("Table poison error: {0}")] + PoisonError(String), + #[error("Namespace not found for address: {0:#063x}")] + NamespaceNotFound(Felt), + #[error("Could not parse record for table {0}: {1}")] + RecordError(String, RecordError), + #[error("Failed to pass string to felt")] + FeltFromStrError(#[from] FromStrError), +} + +pub type DbResult = std::result::Result; + +impl From> for DbError { + fn from(err: PoisonError) -> Self { + Self::PoisonError(err.to_string()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum NamespaceError { + #[error("Invalid address length for address: {0} should be 63 characters long")] + InvalidAddressLength(String), + #[error(transparent)] + AddressFromStrError(#[from] FromStrError), + #[error("Namespace {0} does not match expected namespace {1}")] + NamespaceMismatch(String, String), + #[error("Namespace {1} not found for address: {0:#063x}")] + AddressNotFound(Felt, String), +} + +pub type NamespaceResult = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum TableLoadError { + #[error(transparent)] + NamespaceError(#[from] NamespaceError), + #[error("Table {0} {1:#063x} not found for column {2} with id: {3:#063x}")] + ColumnTableNotFound(String, Felt, String, Felt), + #[error("Table {0} {1:#063x} not found for dead field {2} with id: {3}")] + TableDeadNotFound(String, Felt, String, u128), +} diff --git a/crates/introspect-sql-sink/src/lib.rs b/crates/introspect-sql-sink/src/lib.rs new file mode 100644 index 00000000..249898a0 --- /dev/null +++ b/crates/introspect-sql-sink/src/lib.rs @@ -0,0 +1,31 @@ +pub mod backend; +pub mod error; +pub mod namespace; +pub mod processor; +pub mod sink; +pub mod table; +pub mod tables; + +pub use backend::{IntrospectInitialize, IntrospectProcessor, IntrospectQueryMaker}; +pub use error::{ + DbError, DbResult, RecordError, RecordResult, TableError, TableResult, TypeError, TypeResult, + UpgradeError, UpgradeResult, UpgradeResultExt, +}; +pub use namespace::{NamespaceKey, NamespaceMode, TableKey}; +pub use processor::{DbColumn, DbDeadField, DbTable, IntrospectDb}; +pub use sink::IntrospectSqlSink; +pub use table::{DeadField, DeadFieldDef, Table}; + +#[cfg(feature = "postgres")] +pub mod postgres; +#[cfg(feature = "postgres")] +pub use postgres::IntrospectPgDb; + +#[cfg(feature = "sqlite")] +pub mod sqlite; +#[cfg(feature = "sqlite")] +pub use sqlite::IntrospectSqliteDb; + +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub mod runtime; diff --git a/crates/introspect-sql-sink/src/namespace.rs b/crates/introspect-sql-sink/src/namespace.rs new file mode 100644 index 00000000..695a3049 --- /dev/null +++ b/crates/introspect-sql-sink/src/namespace.rs @@ -0,0 +1,206 @@ +use crate::error::{NamespaceError, NamespaceResult}; +use crate::{DbError, DbResult}; +use introspect_types::ResultInto; +use itertools::Itertools; +use starknet_types_core::felt::Felt; +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +pub enum NamespaceMode { + None, + Single(Arc), + Address, + Named(HashMap>), + Addresses(HashSet), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableKey { + namespace: NamespaceKey, + id: Felt, +} + +impl TableKey { + pub fn new(namespace: NamespaceKey, id: Felt) -> Self { + Self { namespace, id } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NamespaceKey { + None, + Single(Arc), + Address(Felt), + Named(Arc), +} + +pub fn felt_to_namespace(address: &Felt) -> String { + format!("{address:063x}") +} + +impl Hash for TableKey { + fn hash(&self, state: &mut H) { + self.namespace.hash(state); + self.id.hash(state); + } +} + +impl Hash for NamespaceKey { + fn hash(&self, state: &mut H) { + match self { + NamespaceKey::Address(addr) => addr.hash(state), + NamespaceKey::Named(name) => name.hash(state), + NamespaceKey::Single(_) => {} + NamespaceKey::None => {} + } + } +} + +impl Display for NamespaceKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NamespaceKey::Address(addr) => write!(f, "{addr:063x}"), + NamespaceKey::Named(name) | NamespaceKey::Single(name) => name.fmt(f), + NamespaceKey::None => Ok(()), + } + } +} + +impl Display for TableKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !matches!(self.namespace, NamespaceKey::Single(_) | NamespaceKey::None) { + write!(f, "{} ", self.namespace)?; + } + write!(f, "{:#063x}", self.id) + } +} + +impl From for NamespaceMode { + fn from(value: String) -> Self { + NamespaceMode::Single(value.into()) + } +} + +impl From<&str> for NamespaceMode { + fn from(value: &str) -> Self { + NamespaceMode::Single(value.into()) + } +} + +impl From> for NamespaceMode { + fn from(value: HashMap) -> Self { + NamespaceMode::Named( + value + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect::>(), + ) + } +} + +impl From<[(Felt, &str); N]> for NamespaceMode { + fn from(value: [(Felt, &str); N]) -> Self { + NamespaceMode::Named( + value + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect::>(), + ) + } +} + +impl From<[(Felt, String); N]> for NamespaceMode { + fn from(value: [(Felt, String); N]) -> Self { + NamespaceMode::Named( + value + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect::>(), + ) + } +} + +impl From<[Felt; N]> for NamespaceMode { + fn from(value: [Felt; N]) -> Self { + NamespaceMode::Addresses(value.into_iter().collect()) + } +} + +impl From> for NamespaceMode { + fn from(value: Vec) -> Self { + NamespaceMode::Addresses(value.into_iter().collect()) + } +} + +fn felt_try_from_namespace(namespace: &str) -> NamespaceResult { + match namespace.len() == 63 { + true => Felt::from_hex(namespace).err_into(), + false => Err(NamespaceError::InvalidAddressLength(namespace.to_string())), + } +} + +impl NamespaceMode { + pub fn namespaces(&self) -> Option> { + match self { + NamespaceMode::None => Some(vec!["".to_string()]), + NamespaceMode::Single(name) => Some(vec![name.to_string()]), + NamespaceMode::Address => None, + NamespaceMode::Named(map) => { + Some(map.values().unique().map(ToString::to_string).collect()) + } + NamespaceMode::Addresses(set) => Some(set.iter().map(felt_to_namespace).collect()), + } + } + + pub fn get_namespace_key( + &self, + namespace: String, + owner: &Felt, + ) -> NamespaceResult { + match self { + NamespaceMode::None => Ok(NamespaceKey::None), + NamespaceMode::Single(s) => match **s == *namespace { + true => Ok(NamespaceKey::Single(s.clone())), + false => Err(NamespaceError::NamespaceMismatch(namespace, s.to_string())), + }, + NamespaceMode::Address => { + felt_try_from_namespace(&namespace).map(NamespaceKey::Address) + } + NamespaceMode::Named(map) => match map.get(owner) { + Some(s) if **s == *namespace => Ok(NamespaceKey::Named(s.clone())), + Some(s) => Err(NamespaceError::NamespaceMismatch(namespace, s.to_string())), + None => Err(NamespaceError::AddressNotFound(*owner, namespace)), + }, + NamespaceMode::Addresses(set) => { + let address = felt_try_from_namespace(&namespace)?; + match set.contains(&address) { + true => Ok(NamespaceKey::Address(address)), + false => Err(NamespaceError::AddressNotFound(address, namespace)), + } + } + } + } + + pub fn get_key(&self, namespace: String, id: Felt, owner: &Felt) -> NamespaceResult { + self.get_namespace_key(namespace, owner) + .map(|k| TableKey::new(k, id)) + } + + pub fn to_namespace(&self, from_address: &Felt) -> DbResult { + match self { + NamespaceMode::None => Ok(NamespaceKey::None), + NamespaceMode::Single(name) => Ok(NamespaceKey::Single(name.clone())), + NamespaceMode::Address => Ok(NamespaceKey::Address(*from_address)), + NamespaceMode::Named(map) => match map.get(from_address) { + Some(namespace) => Ok(NamespaceKey::Named(namespace.clone())), + None => Err(DbError::NamespaceNotFound(*from_address)), + }, + NamespaceMode::Addresses(set) => match set.contains(from_address) { + true => Ok(NamespaceKey::Address(*from_address)), + false => Err(DbError::NamespaceNotFound(*from_address)), + }, + } + } +} diff --git a/crates/introspect-sql-sink/src/postgres/append_only.rs b/crates/introspect-sql-sink/src/postgres/append_only.rs new file mode 100644 index 00000000..9d505c64 --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/append_only.rs @@ -0,0 +1,106 @@ +use super::json::PostgresJsonSerializer; +use crate::postgres::insert::pg_json_felt252; +use crate::{RecordResult, Table}; +use introspect_types::ColumnInfo; +use serde::ser::SerializeMap; +use serde_json::Serializer as JsonSerializer; +use starknet_types_core::felt::Felt; +use std::io::Write; +use torii_introspect::tables::SerializeEntries; +use torii_introspect::Record; +use torii_sql::postgres::PgQuery; +use torii_sql::Queries; + +struct MetaData<'a> { + pub block_number: u64, + pub transaction_hash: &'a Felt, +} + +impl<'a> MetaData<'a> { + pub fn new(block_number: u64, transaction_hash: &'a Felt) -> Self { + Self { + block_number, + transaction_hash, + } + } +} + +impl SerializeEntries for MetaData<'_> { + fn entry_count(&self) -> usize { + 2 + } + fn serialize_entries( + &self, + map: &mut ::SerializeMap, + ) -> Result<(), S::Error> { + let tx_hash = pg_json_felt252(self.transaction_hash); + map.serialize_entry("__created_block", &self.block_number)?; + map.serialize_entry("__created_tx", &tx_hash) + } +} + +#[allow(clippy::too_many_arguments)] +pub fn append_only_record_queries( + table: &Table, + column_ids: &[Felt], + records: &[Record], + _from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let schema = table.get_record_schema(column_ids)?; + let namespace = &table.namespace; + let table_name = &table.name; + let mut writer = Vec::new(); + let metadata = MetaData::new(block_number, transaction_hash); + let primary = schema.primary_name(); + let columns: Vec<(&Felt, &ColumnInfo)> = table.columns.iter().collect(); + + write!( + writer, + r#"WITH latest AS (SELECT DISTINCT ON ("{primary}") * FROM "{namespace}"."{table_name}" ORDER BY "{primary}", "__revision" DESC), + input AS ( SELECT * FROM jsonb_populate_recordset(NULL::"{namespace}"."{table_name}", $$"# + ) + .unwrap(); + + schema.parse_records_with_metadata( + records, + &metadata, + &mut JsonSerializer::new(&mut writer), + &PostgresJsonSerializer, + )?; + + write!( + writer, + r#"$$)) INSERT INTO "{namespace}"."{table_name}" ("{primary}", "__revision", "__created_block", "__created_tx""# + ) + .unwrap(); + + for (_, ColumnInfo { name, .. }) in &columns { + write!(writer, r#", "{name}""#).unwrap(); + } + + write!( + writer, + r#") SELECT i."{primary}", (SELECT COALESCE(MAX("__revision"), 0) + 1 FROM "{namespace}"."{table_name}" WHERE "{primary}" = i."{primary}") AS "__revision", i."__created_block", i."__created_tx""#, + ).unwrap(); + + for (id, ColumnInfo { name, .. }) in &columns { + if column_ids.contains(id) { + write!(writer, r#", i."{name}""#).unwrap(); + } else { + write!(writer, r#", COALESCE(i."{name}", l."{name}")"#).unwrap(); + } + } + + write!( + writer, + r#" FROM input i LEFT JOIN latest l USING ("{primary}")"# + ) + .unwrap(); + + let string = unsafe { String::from_utf8_unchecked(writer) }; + queries.add(string); + Ok(()) +} diff --git a/crates/introspect-sql-sink/src/postgres/backend.rs b/crates/introspect-sql-sink/src/postgres/backend.rs new file mode 100644 index 00000000..31a5c5dd --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/backend.rs @@ -0,0 +1,147 @@ +use super::insert::insert_record_queries; +use super::query::{insert_columns_query, insert_table_query, CreatePgTable}; +use super::upgrade::PgTableUpgrade; +use crate::postgres::append_only::append_only_record_queries; +use crate::processor::IntrospectDb; +use crate::{ + IntrospectQueryMaker, IntrospectSqlSink, RecordResult, Table, TableError, TableResult, +}; +use async_trait::async_trait; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::{ColumnDef, FeltIds, PrimaryDef}; +use starknet_types_core::felt::Felt; +use torii_introspect::Record; +use torii_sql::{PgPool, PgQuery, Postgres}; + +pub type IntrospectPgDb = IntrospectDb; + +#[async_trait] +impl IntrospectQueryMaker for Postgres { + fn create_table_queries( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + let ns = namespace.into(); + CreatePgTable::new(&ns, id, name, primary, columns, append_only)?.make_queries(queries); + store_table_queries( + namespace, + id, + name, + primary, + append_only, + columns, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn update_table_queries( + table: &mut Table, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + let upgrades = table.upgrade_table(name, primary, columns)?; + upgrades.to_queries(block_number, transaction_hash, queries)?; + let columns: Vec<(&Felt, &introspect_types::ColumnInfo)> = table + .columns_with_ids(&upgrades.columns_upgraded) + .map_err(|e| e.to_table_error(&table.name))?; + store_table_queries( + &table.namespace, + &table.id, + name, + primary, + table.append_only, + columns, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> RecordResult<()> { + if table.append_only { + append_only_record_queries( + table, + columns, + records, + from_address, + block_number, + transaction_hash, + queries, + ) + } else { + insert_record_queries( + table, + columns, + records, + from_address, + block_number, + transaction_hash, + queries, + ) + } + } +} + +#[allow(clippy::too_many_arguments)] +fn store_table_queries( + schema: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + append_only: bool, + columns: CS, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, +) -> TableResult<()> +where + CS: Names + FeltIds + TypeDefs, +{ + queries.push( + insert_table_query( + schema, + id, + name, + primary, + from_address, + append_only, + block_number, + transaction_hash, + ) + .map_err(TableError::Encode)?, + ); + + queries.push( + insert_columns_query(schema, id, columns, block_number, transaction_hash) + .map_err(TableError::Encode)?, + ); + Ok(()) +} + +impl IntrospectSqlSink for PgPool { + const NAME: &'static str = "Introspect Postgres"; +} diff --git a/crates/introspect-postgres-sink/src/create.rs b/crates/introspect-sql-sink/src/postgres/create.rs similarity index 84% rename from crates/introspect-postgres-sink/src/create.rs rename to crates/introspect-sql-sink/src/postgres/create.rs index 41bf4d85..34df03c8 100644 --- a/crates/introspect-postgres-sink/src/create.rs +++ b/crates/introspect-sql-sink/src/postgres/create.rs @@ -1,9 +1,7 @@ -use crate::{ - query::{CreatePgTable, CreatesType}, - utils::{AsBytes, HasherExt}, - PgSchema, PgTypeError, PgTypeResult, PostgresField, PostgresScalar, PostgresType, PrimaryKey, - SchemaName, -}; +use super::query::{make_schema_query, CreatePgTable, CreatesType}; +use super::utils::{AsBytes, HasherExt}; +use super::{PostgresField, PostgresScalar, PostgresType, PrimaryKey, SchemaName}; +use crate::{TypeError, TypeResult}; use introspect_types::{ ArrayDef, ColumnDef, EnumDef, FixedArrayDef, MemberDef, OptionDef, PrimaryDef, PrimaryTypeDef, StructDef, TupleDef, TypeDef, VariantDef, @@ -11,17 +9,17 @@ use introspect_types::{ use itertools::Itertools; use starknet_types_core::felt::Felt; use std::rc::Rc; -use torii_common::sql::{PgQuery, Queries}; -use torii_introspect::schema::TableInfo; +use torii_sql::postgres::PgQuery; +use torii_sql::Queries; use xxhash_rust::xxh3::Xxh3; pub trait PostgresTypeExtractor { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult; + ) -> TypeResult; } pub trait PostgresFieldExtractor { @@ -34,10 +32,10 @@ pub trait PostgresFieldExtractor { } fn extract_field( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { Ok(PostgresField::new( self.name().to_string(), self.type_def() @@ -54,7 +52,7 @@ impl PostgresField { } } - pub fn new_composite(name: S, schema: &Rc, type_name: T) -> Self + pub fn new_composite(name: S, schema: &Rc, type_name: T) -> Self where S: Into, T: Into, @@ -111,10 +109,10 @@ impl PostgresFieldExtractor for (&Felt, &VariantDef) { impl PostgresTypeExtractor for TypeDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { match self { TypeDef::None => Ok(PostgresScalar::None.into()), TypeDef::Bool => Ok(PostgresScalar::Boolean.into()), @@ -149,7 +147,7 @@ impl PostgresTypeExtractor for TypeDef { TypeDef::Option(def) => def.type_def.extract_type(schema, branch, creates), TypeDef::Nullable(def) => def.type_def.extract_type(schema, branch, creates), TypeDef::Felt252Dict(_) | TypeDef::Result(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { - Err(PgTypeError::UnsupportedType(format!("{self:?}"))) + Err(TypeError::UnsupportedType(format!("{self:?}"))) } } } @@ -187,10 +185,10 @@ impl From<&PrimaryDef> for PrimaryKey { impl PostgresTypeExtractor for ArrayDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { self.type_def .extract_type(schema, branch, creates)? .to_array(None) @@ -200,10 +198,10 @@ impl PostgresTypeExtractor for ArrayDef { impl PostgresTypeExtractor for FixedArrayDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { self.type_def .extract_type(schema, branch, creates)? .to_array(Some(self.size)) @@ -213,15 +211,15 @@ impl PostgresTypeExtractor for FixedArrayDef { impl PostgresTypeExtractor for StructDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { let members = self .members .iter() .map(|f| f.extract_field(schema, branch, creates)) - .collect::>>()?; + .collect::>>()?; let name = branch.type_name(&self.name); creates.push(CreatesType::new_struct(schema, &name, members).into()); Ok(PostgresType::composite(schema, name)) @@ -231,10 +229,10 @@ impl PostgresTypeExtractor for StructDef { impl PostgresTypeExtractor for EnumDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { let name = branch.type_name(&self.name); let variants_type = branch.type_name(&format!("v_{}", self.name)); let variant_names = self.variants.values().map(|v| v.name.clone()).collect_vec(); @@ -259,10 +257,10 @@ impl PostgresTypeExtractor for EnumDef { impl PostgresTypeExtractor for TupleDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { let mut variants = Vec::with_capacity(self.elements.len()); for (i, element) in self.elements.iter().enumerate() { variants.push( @@ -280,44 +278,51 @@ impl PostgresTypeExtractor for TupleDef { impl PostgresTypeExtractor for OptionDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { self.type_def.extract_type(schema, branch, creates) } } impl CreatePgTable { - pub fn new(schema: &Rc, id: &Felt, table: &TableInfo) -> PgTypeResult { - let TableInfo { - name, - attributes: _, - primary, - columns, - } = table; - let mut creates = Vec::new(); + pub fn new( + schema: &Rc, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + ) -> TypeResult { + let mut creates: Vec = Vec::new(); let branch = Xxh3::new_based(id); let primary = primary.into(); let columns = columns .iter() .map(|col| col.extract_field(schema, &branch, &mut creates)) - .collect::>>()?; + .collect::>>()?; Ok(Self { name: SchemaName::new(schema, name), primary, columns, pg_types: creates, + append_only, }) } pub fn make_queries(&self, queries: &mut Vec) { + if !self.name.1.is_empty() { + queries.add(make_schema_query(&self.name.0)); + } for pg_type in &self.pg_types { queries.add(pg_type.to_string()); } queries.add(self.to_string()); - queries.add(format!( - r#"CREATE TRIGGER set_timestamps BEFORE INSERT ON {} FOR EACH ROW EXECUTE FUNCTION introspect.set_default_timestamps();"#, - self.name - )); + if !self.append_only { + queries.add(format!( + r#"CREATE TRIGGER set_timestamps BEFORE INSERT ON {} FOR EACH ROW EXECUTE FUNCTION introspect.set_default_timestamps();"#, + self.name + )); + } } } diff --git a/crates/introspect-sql-sink/src/postgres/handler.rs b/crates/introspect-sql-sink/src/postgres/handler.rs new file mode 100644 index 00000000..c445a50c --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/handler.rs @@ -0,0 +1,28 @@ +use super::query::{fetch_columns, fetch_dead_fields, fetch_tables}; +use crate::backend::IntrospectInitialize; +use crate::{DbColumn, DbDeadField, DbResult, DbTable}; +use async_trait::async_trait; +use introspect_types::ResultInto; +use sqlx::PgPool; +use torii_sql::PoolExt; + +pub const INTROSPECT_PG_SINK_MIGRATIONS: sqlx::migrate::Migrator = + sqlx::migrate!("./migrations/postgres"); + +#[async_trait] +impl IntrospectInitialize for PgPool { + async fn load_tables(&self, schemas: &Option>) -> DbResult> { + fetch_tables(self.pool(), schemas).await.err_into() + } + async fn load_columns(&self, schemas: &Option>) -> DbResult> { + fetch_columns(self.pool(), schemas).await.err_into() + } + async fn load_dead_fields(&self, schemas: &Option>) -> DbResult> { + fetch_dead_fields(self.pool(), schemas).await.err_into() + } + async fn initialize(&self) -> DbResult<()> { + self.migrate(Some("introspect"), INTROSPECT_PG_SINK_MIGRATIONS) + .await + .err_into() + } +} diff --git a/crates/introspect-sql-sink/src/postgres/insert.rs b/crates/introspect-sql-sink/src/postgres/insert.rs new file mode 100644 index 00000000..a49fe6f4 --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/insert.rs @@ -0,0 +1,92 @@ +use super::json::PostgresJsonSerializer; +use crate::{RecordResult, Table}; +use introspect_types::ColumnInfo; +use serde::ser::SerializeMap; +use serde_json::Serializer as JsonSerializer; +use starknet_types_core::felt::Felt; +use std::io::Write; +use torii_introspect::tables::SerializeEntries; +use torii_introspect::Record; +use torii_sql::postgres::PgQuery; +use torii_sql::Queries; + +pub const METADATA_CONFLICTS: &str = "__updated_at = NOW(), __updated_block = EXCLUDED.__updated_block, __updated_tx = EXCLUDED.__updated_tx"; + +struct MetaData<'a> { + pub block_number: u64, + pub transaction_hash: &'a Felt, +} + +impl<'a> MetaData<'a> { + pub fn new(block_number: u64, transaction_hash: &'a Felt) -> Self { + Self { + block_number, + transaction_hash, + } + } +} + +pub fn pg_json_felt252(value: &Felt) -> String { + format!("\\x{}", hex::encode(value.to_bytes_be())) +} + +impl SerializeEntries for MetaData<'_> { + fn entry_count(&self) -> usize { + 4 + } + fn serialize_entries( + &self, + map: &mut ::SerializeMap, + ) -> Result<(), S::Error> { + let tx_hash = pg_json_felt252(self.transaction_hash); + map.serialize_entry("__created_block", &self.block_number)?; + map.serialize_entry("__updated_block", &self.block_number)?; + map.serialize_entry("__created_tx", &tx_hash)?; + map.serialize_entry("__updated_tx", &tx_hash) + } +} + +#[allow(clippy::too_many_arguments)] +pub fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + _from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let schema = table.get_record_schema(columns)?; + let namespace = &table.namespace; + let table_name = &table.name; + let mut writer = Vec::new(); + let metadata = MetaData::new(block_number, transaction_hash); + write!( + writer, + r#"INSERT INTO "{namespace}"."{table_name}" SELECT * FROM jsonb_populate_recordset(NULL::"{namespace}"."{table_name}", $$"# + ) + .unwrap(); + schema.parse_records_with_metadata( + records, + &metadata, + &mut JsonSerializer::new(&mut writer), + &PostgresJsonSerializer, + )?; + write!( + writer, + r#"$$) ON CONFLICT ("{}") DO UPDATE SET {METADATA_CONFLICTS}"#, + schema.primary().name + ) + .unwrap(); + for ColumnInfo { name, .. } in schema.columns() { + write!( + writer, + r#", "{name}" = COALESCE(EXCLUDED."{name}", "{table_name}"."{name}")"#, + name = name + ) + .unwrap(); + } + let string = unsafe { String::from_utf8_unchecked(writer) }; + queries.add(string); + Ok(()) +} diff --git a/crates/introspect-postgres-sink/src/json.rs b/crates/introspect-sql-sink/src/postgres/json.rs similarity index 64% rename from crates/introspect-postgres-sink/src/json.rs rename to crates/introspect-sql-sink/src/postgres/json.rs index 7e8cc911..02d78188 100644 --- a/crates/introspect-postgres-sink/src/json.rs +++ b/crates/introspect-sql-sink/src/postgres/json.rs @@ -1,8 +1,9 @@ use introspect_types::serialize::ToCairoDeSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; -use introspect_types::{CairoDeserializer, ResultDef, TupleDef, TypeDef}; -use serde::ser::SerializeMap; +use introspect_types::{CairoDeserializer, EnumDef, ResultDef, TupleDef, TypeDef, VariantDef}; +use serde::ser::{Error as SerError, SerializeMap}; use serde::Serializer; +use starknet_types_core::felt::Felt; pub struct PostgresJsonSerializer; @@ -48,27 +49,40 @@ impl CairoTypeSerialization for PostgresJsonSerializer { seq.end() } - fn serialize_variant<'a, S: Serializer>( + fn serialize_enum<'a, S: Serializer>( &'a self, data: &mut impl CairoDeserializer, serializer: S, - name: &str, - type_def: &'a TypeDef, + enum_def: &'a EnumDef, + variant: Felt, ) -> Result { - match type_def { - TypeDef::None => { - let mut map = serializer.serialize_map(Some(1))?; - map.serialize_entry("_variant", name)?; - map - } - _ => { - let mut map = serializer.serialize_map(Some(2))?; - map.serialize_entry("_variant", name)?; - map.serialize_entry(name, &type_def.to_de_se(data, self))?; - map + let VariantDef { name, type_def, .. } = + enum_def.get_variant(&variant).map_err(S::Error::custom)?; + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("_variant", name)?; + if type_def != &TypeDef::None { + map.serialize_entry(name, &type_def.to_de_se(data, self))?; + } + for v in &enum_def.order { + if v != &variant { + let VariantDef { name, type_def, .. } = + enum_def.get_variant(v).map_err(S::Error::custom)?; + if type_def != &TypeDef::None { + map.serialize_entry(name, &())?; + } } } - .end() + map.end() + } + + fn serialize_variant<'a, S: Serializer>( + &'a self, + _data: &mut impl CairoDeserializer, + _serializer: S, + _name: &str, + _type_def: &'a TypeDef, + ) -> Result { + unimplemented!("variant serialization is only supported within enums, and should not be called directly") } fn serialize_result<'a, S: Serializer>( diff --git a/crates/introspect-sql-sink/src/postgres/mod.rs b/crates/introspect-sql-sink/src/postgres/mod.rs new file mode 100644 index 00000000..5bdfab78 --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/mod.rs @@ -0,0 +1,16 @@ +pub mod append_only; +pub mod backend; +pub mod create; +pub mod handler; +pub mod insert; +pub mod json; +pub mod query; +pub mod types; +pub mod upgrade; +pub mod utils; + +pub use backend::IntrospectPgDb; +pub use types::{ + PostgresArray, PostgresField, PostgresScalar, PostgresType, PrimaryKey, SchemaName, +}; +pub use utils::{truncate, HasherExt}; diff --git a/crates/introspect-postgres-sink/src/query.rs b/crates/introspect-sql-sink/src/postgres/query.rs similarity index 74% rename from crates/introspect-postgres-sink/src/query.rs rename to crates/introspect-sql-sink/src/postgres/query.rs index 63b96983..9a1adc1b 100644 --- a/crates/introspect-postgres-sink/src/query.rs +++ b/crates/introspect-sql-sink/src/postgres/query.rs @@ -1,59 +1,62 @@ -use introspect_types::{ColumnDef, ColumnInfo, MemberDef, PrimaryDef, TypeDef}; +use super::{PostgresField, PostgresType, PrimaryKey, SchemaName}; +use crate::{DbColumn, DbDeadField, DbTable, DeadFieldDef, TableError, TableResult}; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::{ColumnDef, FeltIds, MemberDef, PrimaryDef, TypeDef}; use itertools::Itertools; use sqlx::error::BoxDynError; +use sqlx::postgres::{PgArguments, PgRow}; use sqlx::prelude::FromRow; -use sqlx::Error::Encode as EncodeError; -use sqlx::{postgres::PgArguments, types::Json}; +use sqlx::query::QueryAs; +use sqlx::types::Json; use sqlx::{Arguments, Executor, Postgres}; use starknet_types_core::felt::Felt; -use torii_common::sql::{PgQuery, Queries, SqlxResult}; +use std::collections::HashMap; +use std::fmt::{Display, Formatter, Result as FmtResult, Write}; +use std::rc::Rc; use torii_introspect::postgres::types::{PgPrimary, Uint128}; use torii_introspect::postgres::PgFelt; +use torii_sql::postgres::PgQuery; +use torii_sql::{Queries, SqlxResult}; -use crate::table::PgTable; -use crate::{ - processor::COMMIT_CMD, table::DeadField, PgSchema, PostgresField, PostgresType, PrimaryKey, - SchemaName, -}; -use std::collections::HashMap; -use std::{ - fmt::{Display, Formatter, Result as FmtResult, Write}, - rc::Rc, -}; +pub const COMMIT_CMD: &str = "--COMMIT"; -const CREATE_METADATA_COLUMNS: &str = "__created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __created_block public.uint64 NOT NULL, __updated_block public.uint64 NOT NULL, __created_tx public.felt252 NOT NULL, __updated_tx public.felt252 NOT NULL);"; +const CREATE_METADATA_COLUMNS: &str = "__created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __created_block public.uint64 NOT NULL, __updated_block public.uint64 NOT NULL, __created_tx public.felt252 NOT NULL, __updated_tx public.felt252 NOT NULL"; +const CREATE_APPEND_ONLY_METADATA_COLUMNS: &str = "__created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __created_block public.uint64 NOT NULL, __created_tx public.felt252 NOT NULL"; +const APPEND_ONLY_REVISION_COLUMN: &str = r#""__revision" bigint NOT NULL, "#; +const INSERT_TABLE_QUERY: &str = r#"INSERT INTO introspect.db_tables + ("schema", id, owner, name, primary_def, append_only, updated_at, created_block, updated_block, created_tx, updated_tx) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), $7::uint64, $7::uint64, $8, $8) + ON CONFLICT ("schema", id) DO UPDATE SET + name = EXCLUDED.name, append_only = EXCLUDED.append_only, primary_def = EXCLUDED.primary_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; const INSERT_DEAD_MEMBER_QUERY: &str = r#"INSERT INTO introspect.db_dead_fields ("schema", "table", id, name, type_def, updated_at, created_block, updated_block, created_tx, updated_tx) SELECT $1, $2, unnest($3::bigint[]), unnest($4::text[]), unnest($5::jsonb[]), NOW(), $6::uint64, $6::uint64, $7, $7 ON CONFLICT ("schema", "table", id) DO UPDATE SET name = EXCLUDED.name, type_def = EXCLUDED.type_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; -const INSERT_TABLE_QUERY: &str = r#"INSERT INTO introspect.db_tables - ("schema", id, name, primary_def, updated_at, created_block, updated_block, created_tx, updated_tx) - VALUES ($1, $2, $3, $4, NOW(), $5::uint64, $5::uint64, $6, $6) - ON CONFLICT ("schema", id) DO UPDATE SET - name = EXCLUDED.name, primary_def = EXCLUDED.primary_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; + const INSERT_COLUMN_QUERY: &str = r#"INSERT INTO introspect.db_columns ("schema", "table", id, name, type_def, updated_at, created_block, updated_block, created_tx, updated_tx) SELECT $1, $2, unnest($3::felt252[]), unnest($4::text[]), unnest($5::jsonb[]), NOW(), $6::uint64, $6::uint64, $7, $7 ON CONFLICT ("schema", "table", id) DO UPDATE SET name = EXCLUDED.name, type_def = EXCLUDED.type_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; -const FETCH_TABLES_QUERY: &str = - r#"SELECT id, name, primary_def FROM introspect.db_tables WHERE "schema" = $1"#; -const FETCH_COLUMNS_QUERY: &str = - r#"SELECT "table", id, name, type_def FROM introspect.db_columns WHERE "schema" = $1"#; -const FETCH_DEAD_FIELDS_QUERY: &str = - r#"SELECT "table", id, name, type_def FROM introspect.db_dead_fields WHERE "schema" = $1"#; +const FETCH_TABLES_QUERY: &str = r#"SELECT "schema", id, name, primary_def, owner, append_only FROM introspect.db_tables WHERE $1::text[] = '{}'::text[] OR "schema" = ANY($1::text[])"#; +const FETCH_COLUMNS_QUERY: &str = r#"SELECT "schema", "table", id, name, type_def FROM introspect.db_columns WHERE $1::text[] = '{}'::text[] OR "schema" = ANY($1::text[])"#; +const FETCH_DEAD_FIELDS_QUERY: &str = r#"SELECT "schema", "table", id, name, type_def FROM introspect.db_dead_fields WHERE $1::text[] = '{}'::text[] OR "schema" = ANY($1::text[])"#; #[derive(FromRow)] pub struct TableRow { + pub schema: String, pub id: PgFelt, pub name: String, pub primary_def: PgPrimary, + pub owner: PgFelt, + pub append_only: bool, } #[derive(FromRow)] pub struct ColumnRow { + pub schema: String, pub table: PgFelt, pub id: PgFelt, pub name: String, @@ -62,36 +65,33 @@ pub struct ColumnRow { #[derive(FromRow)] pub struct DeadFieldRow { + pub schema: String, pub table: PgFelt, pub id: Uint128, pub name: String, pub type_def: Json, } -pub struct PgTableHead { - id: Felt, - name: String, - primary: PrimaryDef, -} - #[derive(Debug)] pub struct CreatePgTable { pub name: SchemaName, pub primary: PrimaryKey, pub columns: Vec, pub pg_types: Vec, + pub append_only: bool, } #[derive(Debug)] pub struct TableUpgrade { - pub schema: Rc, + pub schema: Rc, + pub id: Felt, pub name: String, pub old_name: Option, pub atomic: Vec, pub alters: Vec, pub columns: Vec, pub columns_upgraded: Vec, - pub dead: Vec, + pub dead: Vec, pub col_alters: Vec, } @@ -99,7 +99,7 @@ pub struct TableUpgrade { pub struct ColumnUpgrade { pub atomic: Vec, pub alters: Vec, - pub dead: Vec, + pub dead: Vec, pub altered: bool, pub upgraded: bool, } @@ -156,35 +156,6 @@ pub struct EnumUpgrade { add: Vec, } -#[derive(Debug)] -pub struct DeadFieldWithId { - pub id: u128, - pub name: String, - pub type_def: TypeDef, -} - -impl From for (u128, DeadField) { - fn from(value: DeadFieldWithId) -> Self { - ( - value.id, - DeadField { - name: value.name, - type_def: value.type_def, - }, - ) - } -} - -impl From<(u128, DeadField)> for DeadFieldWithId { - fn from(value: (u128, DeadField)) -> Self { - DeadFieldWithId { - id: value.0, - name: value.1.name, - type_def: value.1.type_def, - } - } -} - #[derive(Debug)] pub enum CreatesType { Struct(CreateStruct), @@ -198,10 +169,25 @@ impl Display for CreatePgTable { r#"CREATE TABLE IF NOT EXISTS {} ({}, "#, self.name, self.primary )?; + if self.append_only { + APPEND_ONLY_REVISION_COLUMN.fmt(f)?; + } for column in &self.columns { write!(f, "{column}, ")?; } - CREATE_METADATA_COLUMNS.fmt(f) + if self.append_only { + write!( + f, + r#"{CREATE_APPEND_ONLY_METADATA_COLUMNS}, PRIMARY KEY ("{}", "__revision"));"#, + self.primary.name + ) + } else { + write!( + f, + r#"{CREATE_METADATA_COLUMNS}, PRIMARY KEY ("{}"));"#, + self.primary.name + ) + } } } @@ -242,7 +228,7 @@ impl Display for CreatesType { impl CreatesType { pub fn new_struct>( - schema: &Rc, + schema: &Rc, name: S, fields: Vec, ) -> Self { @@ -252,11 +238,7 @@ impl CreatesType { }) } - pub fn new_enum>( - schema: &Rc, - name: S, - variants: Vec, - ) -> Self { + pub fn new_enum>(schema: &Rc, name: S, variants: Vec) -> Self { Self::Enum(CreateEnum { name: SchemaName::new(schema, name), variants, @@ -265,9 +247,10 @@ impl CreatesType { } impl TableUpgrade { - pub fn new>(schema: &Rc, name: S) -> Self { + pub fn new>(schema: &Rc, id: Felt, name: S) -> Self { Self { schema: schema.clone(), + id, name: name.into(), old_name: None, columns: Vec::new(), @@ -338,31 +321,31 @@ impl TableUpgrade { pub fn to_queries( &self, - table_id: &Felt, block_number: u64, transaction_hash: &Felt, queries: &mut Vec, - ) -> SqlxResult<()> { - let schema = &self.schema; - let name = &self.name; + ) -> TableResult<()> { queries.add( insert_dead_member_query( &self.schema, - table_id, + &self.id, &self.dead, block_number, transaction_hash, ) - .map_err(EncodeError)?, + .map_err(TableError::Encode)?, ); if let Some(old_name) = &self.old_name { queries.add(format!( - r#"ALTER TABLE "{schema}"."{old_name}" RENAME TO "{name}";"# + r#"ALTER TABLE {} RENAME TO "{}";"#, + SchemaName::new(&self.schema, old_name), + self.name )); } self.atomic.iter().for_each(|m| m.to_queries(queries)); + let name = SchemaName::new(&self.schema, &self.name); if let Some((last, columns)) = self.columns.split_last() { - let mut alterations = format!(r#"ALTER TABLE "{schema}"."{name}" "#); + let mut alterations = format!(r#"ALTER TABLE {name} "#); columns .iter() .for_each(|m| write!(alterations, "{m}, ").unwrap()); @@ -375,8 +358,8 @@ impl TableUpgrade { fn alter_queries(&self, queries: &mut Vec) { if let Some((last, others)) = self.col_alters.split_last() { - let (schema, name) = (&self.schema, &self.name); - let mut forward = format!(r#"ALTER TABLE "{schema}"."{name}" "#); + let table_name = SchemaName::new(&self.schema, &self.name); + let mut forward = format!(r#"ALTER TABLE {table_name} "#); let mut reverse = forward.clone(); for PostgresField { name: col, pg_type } in others { write!( @@ -496,7 +479,7 @@ impl TypeMods for Vec { impl ColumnUpgrade { pub fn maybe_alter( &mut self, - schema: &Rc, + schema: &Rc, name: &str, field: &str, pg_type: Option, @@ -513,7 +496,7 @@ impl ColumnUpgrade { pub fn add_struct_mod>( &mut self, - schema: &Rc, + schema: &Rc, name: S, mods: Vec, ) { @@ -527,7 +510,7 @@ impl ColumnUpgrade { } pub fn add_enum_mod>( &mut self, - schema: &Rc, + schema: &Rc, name: S, rename: Vec<(String, String)>, add: Vec, @@ -544,7 +527,7 @@ impl ColumnUpgrade { } pub fn add_dead_member(&mut self, id: u128, member: &MemberDef) { self.upgraded = true; - self.dead.push(DeadFieldWithId { + self.dead.push(DeadFieldDef { id, name: member.name.clone(), type_def: member.type_def.clone(), @@ -577,64 +560,50 @@ impl From for TypeMod { } } -impl From for (Felt, Felt, ColumnInfo) { +impl From for DbColumn { fn from(value: ColumnRow) -> Self { - ( - value.table.into(), - value.id.into(), - ColumnInfo { - name: value.name, - attributes: Vec::new(), - type_def: value.type_def.0, - }, - ) + DbColumn { + namespace: value.schema, + table: value.table.into(), + id: value.id.into(), + name: value.name, + type_def: value.type_def.0, + } } } -impl From for (Felt, u128, DeadField) { +impl From for DbDeadField { fn from(value: DeadFieldRow) -> Self { - ( - value.table.into(), - value.id.into(), - DeadField { - name: value.name, - type_def: value.type_def.0, - }, - ) + DbDeadField { + namespace: value.schema, + table: value.table.into(), + id: value.id.into(), + name: value.name, + type_def: value.type_def.0, + } } } -impl From for PgTableHead { +impl From for DbTable { fn from(value: TableRow) -> Self { - let row = value; - PgTableHead { - id: row.id.into(), - name: row.name, - primary: row.primary_def.into(), + DbTable { + namespace: value.schema, + id: value.id.into(), + owner: value.owner.into(), + name: value.name, + primary: value.primary_def.into(), + columns: HashMap::new(), + dead: HashMap::new(), + append_only: value.append_only, + alive: true, } } } -impl PgTableHead { - pub fn to_table(self, schema: &PgSchema) -> (Felt, PgTable) { - ( - self.id, - PgTable { - schema: schema.clone(), - name: self.name, - primary: self.primary, - columns: HashMap::new(), - alive: true, - dead: HashMap::new(), - }, - ) - } -} - fn insert_dead_member_query( - schema: &PgSchema, + schema: &str, table: &Felt, - fields: &[DeadFieldWithId], + fields: &[DeadFieldDef], block_number: u64, transaction_hash: &Felt, ) -> Result { @@ -650,55 +619,71 @@ fn insert_dead_member_query( Ok(PgQuery::new(INSERT_DEAD_MEMBER_QUERY, args)) } -pub fn insert_columns_query( - schema: &PgSchema, +pub fn insert_columns_query( + schema: &str, table: &Felt, - columns: Vec<(&Felt, &ColumnInfo)>, + columns: CS, block_number: u64, transaction_hash: &Felt, -) -> Result { +) -> Result +where + CS: Names + FeltIds + TypeDefs, +{ let mut args = PgArguments::default(); args.add(schema.to_string())?; args.add(PgFelt::from(*table))?; - args.add( - columns - .iter() - .map(|(id, _)| PgFelt::from(*id)) - .collect_vec(), - )?; - args.add(columns.iter().map(|(_, c)| c.name.clone()).collect_vec())?; - args.add(columns.iter().map(|(_, c)| Json(&c.type_def)).collect_vec())?; + args.add(columns.ids().iter().map_into::().collect_vec())?; + args.add(columns.names())?; + args.add(columns.type_defs().into_iter().map(Json).collect_vec())?; args.add(block_number.to_string())?; args.add(PgFelt::from(*transaction_hash))?; Ok(PgQuery::new(INSERT_COLUMN_QUERY, args)) } +#[allow(clippy::too_many_arguments)] pub fn insert_table_query( - schema: &PgSchema, + schema: &str, id: &Felt, name: &str, primary_def: &PrimaryDef, + from_address: &Felt, + append_only: bool, block_number: u64, transaction_hash: &Felt, ) -> Result { let mut args = PgArguments::default(); args.add(schema.to_string())?; args.add(PgFelt::from(*id))?; + args.add(PgFelt::from(*from_address))?; args.add(name.to_owned())?; args.add(PgPrimary::from(primary_def))?; + args.add(append_only)?; args.add(block_number.to_string())?; args.add(PgFelt::from(*transaction_hash))?; Ok(PgQuery::new(INSERT_TABLE_QUERY, args)) } +pub fn schema_query<'a, R>( + query: &'a str, + schemas: &'a Option>, +) -> QueryAs<'a, Postgres, R, PgArguments> +where + R: for<'r> FromRow<'r, PgRow>, +{ + let query = sqlx::query_as::<_, R>(query); + match schemas { + Some(schemas) => query.bind(schemas), + None => query.bind("{}".to_string()), + } +} + pub async fn fetch_tables<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( conn: E, - schema: &PgSchema, -) -> SqlxResult> { - sqlx::query_as::<_, TableRow>(FETCH_TABLES_QUERY) - .bind(schema.to_string()) + schemas: &Option>, +) -> SqlxResult> { + schema_query::(FETCH_TABLES_QUERY, schemas) .fetch_all(conn) .await .map(|rows| rows.into_iter().map_into().collect()) @@ -706,10 +691,9 @@ pub async fn fetch_tables<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( pub async fn fetch_columns<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( conn: E, - schema: &PgSchema, -) -> SqlxResult> { - sqlx::query_as::<_, ColumnRow>(FETCH_COLUMNS_QUERY) - .bind(schema.to_string()) + schemas: &Option>, +) -> SqlxResult> { + schema_query::(FETCH_COLUMNS_QUERY, schemas) .fetch_all(conn) .await .map(|rows| rows.into_iter().map_into().collect()) @@ -717,11 +701,14 @@ pub async fn fetch_columns<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( pub async fn fetch_dead_fields<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( conn: E, - schema: &PgSchema, -) -> SqlxResult> { - sqlx::query_as::<_, DeadFieldRow>(FETCH_DEAD_FIELDS_QUERY) - .bind(schema.to_string()) + schemas: &Option>, +) -> SqlxResult> { + schema_query::(FETCH_DEAD_FIELDS_QUERY, schemas) .fetch_all(conn) .await .map(|rows| rows.into_iter().map_into().collect()) } + +pub fn make_schema_query(schema: &str) -> String { + format!(r#"CREATE SCHEMA IF NOT EXISTS "{schema}""#) +} diff --git a/crates/introspect-postgres-sink/src/types.rs b/crates/introspect-sql-sink/src/postgres/types.rs similarity index 83% rename from crates/introspect-postgres-sink/src/types.rs rename to crates/introspect-sql-sink/src/postgres/types.rs index 7354b8b4..57f97de6 100644 --- a/crates/introspect-postgres-sink/src/types.rs +++ b/crates/introspect-sql-sink/src/postgres/types.rs @@ -1,20 +1,11 @@ +use crate::{TypeError, TypeResult}; use serde::{Deserialize, Serialize}; -use std::{ - collections::VecDeque, - fmt::{Display, Formatter, Result as FmtResult}, - rc::Rc, -}; - -use crate::{PgTypeError, PgTypeResult}; - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -pub enum PgSchema { - Public, - Custom(String), -} +use std::collections::VecDeque; +use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::rc::Rc; #[derive(Clone, Deserialize, Serialize, PartialEq, Debug)] -pub struct SchemaName(pub Rc, pub String); +pub struct SchemaName(pub Rc, pub String); #[derive(Clone, Deserialize, Serialize, PartialEq, Debug)] pub enum PostgresScalar { @@ -75,18 +66,12 @@ impl From for PostgresType { } } -impl Display for PgSchema { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PgSchema::Custom(namespace) => write!(f, "{namespace}",), - PgSchema::Public => write!(f, "public"), - } - } -} - impl Display for SchemaName { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - write!(f, r#""{}"."{}""#, self.0, self.1) + match self.0.is_empty() { + true => write!(f, r#""{}""#, self.1), + false => write!(f, r#""{}"."{}""#, self.0, self.1), + } } } @@ -136,7 +121,7 @@ impl Display for PostgresType { impl Display for PrimaryKey { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - write!(f, r#""{}" {} PRIMARY KEY"#, self.name, self.pg_type) + write!(f, r#""{}" {}"#, self.name, self.pg_type) } } @@ -156,7 +141,7 @@ impl From for PostgresType { } impl SchemaName { - pub fn new>(schema: &Rc, name: T) -> Self { + pub fn new>(schema: &Rc, name: T) -> Self { Self(schema.clone(), name.into()) } pub fn replace>(&mut self, name: S) -> String { @@ -168,7 +153,7 @@ impl PostgresType { pub fn is_composite(&self) -> bool { matches!(self.scalar, PostgresScalar::Composite(_)) } - pub fn to_array(self, size: Option) -> PgTypeResult { + pub fn to_array(self, size: Option) -> TypeResult { let arr = match (self.array, size) { (PostgresArray::None, None) => PostgresArray::Dynamic, (PostgresArray::None, Some(size)) => PostgresArray::Fixed(VecDeque::from([size])), @@ -176,7 +161,7 @@ impl PostgresType { sizes.push_back(size); PostgresArray::Fixed(sizes) } - _ => return Err(PgTypeError::NestedArrays), + _ => return Err(TypeError::NestedArrays), }; Ok(Self { scalar: self.scalar, @@ -184,7 +169,7 @@ impl PostgresType { }) } - pub fn composite>(schema: &Rc, name: S) -> Self { + pub fn composite>(schema: &Rc, name: S) -> Self { Self { scalar: PostgresScalar::Composite(SchemaName::new(schema, name)), array: PostgresArray::None, diff --git a/crates/introspect-postgres-sink/src/upgrade.rs b/crates/introspect-sql-sink/src/postgres/upgrade.rs similarity index 83% rename from crates/introspect-postgres-sink/src/upgrade.rs rename to crates/introspect-sql-sink/src/postgres/upgrade.rs index f565d01d..ac6681ca 100644 --- a/crates/introspect-postgres-sink/src/upgrade.rs +++ b/crates/introspect-sql-sink/src/postgres/upgrade.rs @@ -1,33 +1,38 @@ +use super::create::PostgresTypeExtractor; +use super::query::{ColumnUpgrade, StructMod, StructMods, TableUpgrade}; +use super::{HasherExt, PostgresScalar, PostgresType}; use crate::{ - create::PostgresTypeExtractor, - query::{ColumnUpgrade, StructMod, StructMods, TableUpgrade}, - table::{DeadField, PgTable}, - HasherExt, PgSchema, PgTypeError, PgTypeResult, PostgresScalar, PostgresType, TableResult, - UpgradeError, UpgradeResult, UpgradeResultExt, + DeadField, Table, TableResult, TypeError, TypeResult, UpgradeError, UpgradeResult, + UpgradeResultExt, }; use introspect_types::{ ArrayDef, ColumnDef, EnumDef, FixedArrayDef, MemberDef, OptionDef, PrimaryDef, PrimaryTypeDef, ResultInto, StructDef, TupleDef, TypeDef, VariantDef, }; -use starknet_types_core::felt::Felt; -use std::{collections::HashMap, rc::Rc}; -use torii_introspect::schema::TableInfo; +use std::collections::HashMap; +use std::rc::Rc; use xxhash_rust::xxh3::Xxh3; -impl PgTable { - pub fn update_from_info(&mut self, id: &Felt, info: &TableInfo) -> TableResult { - self.update(id, &info.name, &info.primary, &info.columns) - } - pub fn update( +pub trait PgTableUpgrade { + fn upgrade_table( + &mut self, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + ) -> TableResult; + fn retype_primary(&mut self, new: &PrimaryTypeDef) -> UpgradeResult>; +} + +impl PgTableUpgrade for Table { + fn upgrade_table( &mut self, - id: &Felt, name: &str, primary: &PrimaryDef, columns: &[ColumnDef], ) -> TableResult { - let branch = Xxh3::new_based(id); - let schema = Rc::new(self.schema.clone()); - let mut table_mod = TableUpgrade::new(&schema, self.name.clone()); + let branch = Xxh3::new_based(&self.id); + let schema: Rc = self.namespace(); + let mut table_mod = TableUpgrade::new(&schema, self.id, self.name.clone()); table_mod.rename_table(name); table_mod.rename_column(&mut self.primary.name, &primary.name); let pg_type = self @@ -64,58 +69,66 @@ impl PgTable { Ok(table_mod) } fn retype_primary(&mut self, new: &PrimaryTypeDef) -> UpgradeResult> { - use crate::PostgresScalar::{ + use super::PostgresScalar::{ BigInt, Felt252 as PgFelt252, Int, Int128, SmallInt, Uint128, Uint16, Uint32, Uint64, Uint8, }; use introspect_types::PrimaryTypeDef::{ + Bool as PBool, Bytes31 as PBytes31, Bytes31Encoded as PBytes31Encoded, + ClassHash as PClassHash, ContractAddress as PContractAddress, + EthAddress as PEthAddress, Felt252 as PFelt252, ShortUtf8 as PShortUtf8, + StorageAddress as PStorageAddress, StorageBaseAddress as PStorageBaseAddress, + I128 as PI128, I16 as PI16, I32 as PI32, I64 as PI64, I8 as PI8, U128 as PU128, + U16 as PU16, U32 as PU32, U64 as PU64, U8 as PU8, + }; + use introspect_types::TypeDef::{ Bool, Bytes31, Bytes31Encoded, ClassHash, ContractAddress, EthAddress, Felt252, ShortUtf8, StorageAddress, StorageBaseAddress, I128, I16, I32, I64, I8, U128, U16, U32, U64, U8, }; match (&self.primary.type_def, new) { - (Bool, Bool) - | (U8, U8) - | (U16, U16) - | (U32, U32) - | (U64, U64) - | (U128, U128) - | (I8, I8) - | (I16, I16) - | (I32, I32) - | (I64, I64) - | (I128, I128) - | (ShortUtf8, ShortUtf8) - | (EthAddress, EthAddress) - | (ClassHash, ClassHash) - | (ContractAddress, ContractAddress) - | (StorageAddress, StorageAddress) - | (StorageBaseAddress, StorageBaseAddress) - | (Bytes31, Bytes31) - | (Bytes31Encoded(_), Bytes31Encoded(_)) - | (Felt252, Felt252) => Ok(None), - (Bool, U8) => self.primary.type_def.update_as(U8, Uint8), - (Bool | U8, U16) => self.primary.type_def.update_as(U16, Uint16), - (Bool | U8 | U16, U32) => self.primary.type_def.update_as(U32, Uint32), - (Bool | U8 | U16 | U32, U64) => self.primary.type_def.update_as(U64, Uint64), - (Bool | U8 | U16 | U32 | U64, U128) => self.primary.type_def.update_as(U128, Uint128), - (Bool | U8, I8) => self.primary.type_def.update_as(I8, SmallInt), - (Bool | U8 | I8, I16) => self.primary.type_def.update_as(I16, SmallInt), - (Bool | U8 | U16 | I8 | I16, I32) => self.primary.type_def.update_as(I32, Int), - (Bool | U8 | U16 | U32 | I8 | I16 | I32, I64) => { + (Bool, PBool) + | (U8, PU8) + | (U16, PU16) + | (U32, PU32) + | (U64, PU64) + | (U128, PU128) + | (I8, PI8) + | (I16, PI16) + | (I32, PI32) + | (I64, PI64) + | (I128, PI128) + | (ShortUtf8, PShortUtf8) + | (EthAddress, PEthAddress) + | (ClassHash, PClassHash) + | (ContractAddress, PContractAddress) + | (StorageAddress, PStorageAddress) + | (StorageBaseAddress, PStorageBaseAddress) + | (Bytes31, PBytes31) + | (Bytes31Encoded(_), PBytes31Encoded(_)) + | (Felt252, PFelt252) => Ok(None), + (Bool, PU8) => self.primary.type_def.update_as(U8, Uint8), + (Bool | U8, PU16) => self.primary.type_def.update_as(U16, Uint16), + (Bool | U8 | U16, PU32) => self.primary.type_def.update_as(U32, Uint32), + (Bool | U8 | U16 | U32, PU64) => self.primary.type_def.update_as(U64, Uint64), + (Bool | U8 | U16 | U32 | U64, PU128) => self.primary.type_def.update_as(U128, Uint128), + (Bool | U8, PI8) => self.primary.type_def.update_as(I8, SmallInt), + (Bool | U8 | I8, PI16) => self.primary.type_def.update_as(I16, SmallInt), + (Bool | U8 | U16 | I8 | I16, PI32) => self.primary.type_def.update_as(I32, Int), + (Bool | U8 | U16 | U32 | I8 | I16 | I32, PI64) => { self.primary.type_def.update_as(I64, BigInt) } - (Bool | U8 | U16 | U32 | U64 | I8 | I16 | I32 | I64, I128) => { + (Bool | U8 | U16 | U32 | U64 | I8 | I16 | I32 | I64, PI128) => { self.primary.type_def.update_as(I128, Int128) } ( EthAddress, - ClassHash | ContractAddress | StorageAddress | StorageBaseAddress | Felt252, - ) => self.primary.type_def.update_to(new, PgFelt252), + PClassHash | PContractAddress | PStorageAddress | PStorageBaseAddress | PFelt252, + ) => self.primary.type_def.update_to(&new.into(), PgFelt252), ( ClassHash | ContractAddress | StorageAddress | StorageBaseAddress | Felt252, - ClassHash | ContractAddress | StorageAddress | StorageBaseAddress | Felt252, - ) => self.primary.type_def.update_no_change(new), + PClassHash | PContractAddress | PStorageAddress | PStorageBaseAddress | PFelt252, + ) => self.primary.type_def.update_no_change(&new.into()), _ => UpgradeError::primary_upgrade_err(&self.primary.type_def, new), } } @@ -129,7 +142,7 @@ pub trait ExtractItem { fn as_tuple(&mut self) -> UpgradeResult<&mut TupleDef>; fn update_as_array( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &ArrayDef, dead: &mut HashMap, @@ -137,13 +150,13 @@ pub trait ExtractItem { ) -> UpgradeResult>; fn update_as_option( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &TypeDef, dead: &mut HashMap, queries: &mut ColumnUpgrade, ) -> UpgradeResult>; - fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> PgTypeResult; + fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> TypeResult; } impl ExtractItem for TypeDef { @@ -181,7 +194,7 @@ impl ExtractItem for TypeDef { fn update_as_array( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &ArrayDef, dead: &mut HashMap, @@ -195,7 +208,7 @@ impl ExtractItem for TypeDef { } fn update_as_option( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &TypeDef, dead: &mut HashMap, @@ -210,7 +223,7 @@ impl ExtractItem for TypeDef { } } } - fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> PgTypeResult { + fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> TypeResult { match self { TypeDef::None => Ok(PostgresScalar::None.into()), TypeDef::Bool => Ok(PostgresScalar::Boolean.into()), @@ -248,7 +261,7 @@ impl ExtractItem for TypeDef { TypeDef::Option(def) => def.get_pg_type(schema, branch), TypeDef::Nullable(def) => def.get_pg_type(schema, branch), TypeDef::Felt252Dict(_) | TypeDef::Result(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { - Err(PgTypeError::UnsupportedType(format!("{self:?}"))) + Err(TypeError::UnsupportedType(format!("{self:?}"))) } } } @@ -257,7 +270,7 @@ impl ExtractItem for TypeDef { pub trait CompareType { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &Self, dead: &mut HashMap, @@ -297,7 +310,7 @@ pub trait UpgradeField { fn name(&self) -> &str; fn upgrade_field( &mut self, - schema: &Rc, + schema: &Rc, name: &str, branch: &Xxh3, new: &Self, @@ -312,7 +325,7 @@ pub trait UpgradeField { } fn add_field( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, queries: &mut ColumnUpgrade, ) -> UpgradeResult { @@ -350,7 +363,7 @@ impl UpgradeField for VariantDef { impl CompareType for TypeDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &TypeDef, dead: &mut HashMap, @@ -439,7 +452,7 @@ impl CompareType for TypeDef { impl CompareType for StructDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &StructDef, dead: &mut HashMap, @@ -477,7 +490,7 @@ impl CompareType for StructDef { impl CompareType for EnumDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &EnumDef, dead: &mut HashMap, @@ -517,7 +530,7 @@ impl CompareType for EnumDef { impl CompareType for FixedArrayDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &Self, dead: &mut HashMap, @@ -540,7 +553,7 @@ impl CompareType for FixedArrayDef { impl CompareType for TupleDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &Self, dead: &mut HashMap, diff --git a/crates/introspect-postgres-sink/src/utils.rs b/crates/introspect-sql-sink/src/postgres/utils.rs similarity index 100% rename from crates/introspect-postgres-sink/src/utils.rs rename to crates/introspect-sql-sink/src/postgres/utils.rs diff --git a/crates/introspect-sql-sink/src/processor.rs b/crates/introspect-sql-sink/src/processor.rs new file mode 100644 index 00000000..23c64b16 --- /dev/null +++ b/crates/introspect-sql-sink/src/processor.rs @@ -0,0 +1,275 @@ +use crate::backend::{IntrospectInitialize, IntrospectPool, IntrospectProcessor}; +use crate::error::TableLoadError; +use crate::table::{DeadField, Table}; +use crate::tables::Tables; +use crate::{DbResult, IntrospectQueryMaker, NamespaceKey, NamespaceMode}; +use async_trait::async_trait; +use introspect_types::{ColumnInfo, PrimaryDef, TypeDef}; +use itertools::Itertools; +use sqlx::{Database, Pool}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::fmt::Debug; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; +use torii_sql::{Executable, FlexQuery, PoolExt}; + +pub const COMMIT_CMD: &str = "--COMMIT"; + +pub struct IntrospectDb { + tables: Tables, + namespaces: NamespaceMode, + db: Backend, +} + +pub struct DbTable { + pub namespace: String, + pub id: Felt, + pub owner: Felt, + pub name: String, + pub primary: PrimaryDef, + pub columns: HashMap, + pub dead: HashMap, + pub append_only: bool, + pub alive: bool, +} + +pub struct DbColumn { + pub namespace: String, + pub table: Felt, + pub id: Felt, + pub name: String, + pub type_def: TypeDef, +} + +pub struct DbDeadField { + pub namespace: String, + pub table: Felt, + pub id: u128, + pub name: String, + pub type_def: TypeDef, +} + +impl PoolExt for IntrospectDb> { + fn pool(&self) -> &Pool { + &self.db + } +} + +pub trait IntoHashMap { + fn into_hash_map(self) -> HashMap; +} + +impl IntoHashMap for Vec +where + T: Into<(K, V)>, + K: std::hash::Hash + Eq, +{ + fn into_hash_map(self) -> HashMap { + self.into_iter().map_into().collect() + } +} + +#[async_trait] +impl IntrospectProcessor for Pool +where + Vec>: Executable, + FlexQuery: Debug + Clone, +{ + async fn process_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + self.execute_msgs(namespaces, tables, msgs).await + } +} + +impl IntrospectDb { + pub fn new(pool: Backend, namespaces: impl Into) -> Self { + Self { + tables: Tables::default(), + namespaces: namespaces.into(), + db: pool, + } + } +} +impl IntrospectDb { + pub async fn initialize_introspect_sql_sink(&self) -> DbResult> { + self.db.initialize().await?; + self.load_store_data().await + } + + pub async fn process_messages( + &self, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + self.db + .process_msgs(&self.namespaces, &self.tables, msgs) + .await + } + + pub async fn load_store_data(&self) -> DbResult> { + let mut errors = Vec::new(); + let namespaces = self.namespaces.namespaces(); + let mut tables: HashMap<(String, Felt), Table> = + self.db.load_tables(&namespaces).await?.into_hash_map(); + for column in self.db.load_columns(&namespaces).await? { + let (namespace, table_id, id, column_info) = column.into(); + if let Some(table) = tables.get_mut(&(namespace.clone(), table_id)) { + table.columns.insert(id, column_info); + } else { + errors.push(TableLoadError::ColumnTableNotFound( + namespace, + table_id, + column_info.name, + id, + )); + } + } + for dead_field in self.db.load_dead_fields(&namespaces).await? { + let (namespace, table_id, id, field) = dead_field.into(); + if let Some(table) = tables.get_mut(&(namespace.clone(), table_id)) { + table.dead.insert(id, field); + } else { + errors.push(TableLoadError::TableDeadNotFound( + namespace, table_id, field.name, id, + )); + } + } + let mut map = self.tables.write()?; + for ((namespace, id), table) in tables { + match self.namespaces.get_key(namespace, id, &table.owner) { + Ok(key) => { + map.insert(key, table); + } + Err(err) => errors.push(TableLoadError::NamespaceError(err)), + } + } + Ok(errors) + } +} + +impl From for ((String, Felt), Table) { + fn from(value: DbTable) -> Self { + ( + (value.namespace.clone(), value.id), + Table { + id: value.id, + namespace: value.namespace, + name: value.name, + owner: value.owner, + primary: value.primary.into(), + columns: value.columns, + dead: value.dead, + append_only: value.append_only, + alive: value.alive, + }, + ) + } +} + +impl From for (String, Felt, Felt, ColumnInfo) { + fn from(value: DbColumn) -> Self { + ( + value.namespace, + value.table, + value.id, + ColumnInfo { + name: value.name, + attributes: Vec::new(), + type_def: value.type_def, + }, + ) + } +} + +impl From for (String, Felt, u128, DeadField) { + fn from(value: DbDeadField) -> Self { + ( + value.namespace, + value.table, + value.id, + DeadField { + name: value.name, + type_def: value.type_def, + }, + ) + } +} + +pub fn messages_to_queries( + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + queries: &mut Vec>, +) -> DbResult>> { + let mut results = Vec::with_capacity(msgs.len()); + for body in msgs { + let (msg, metadata) = body.into(); + let namespace = namespaces.to_namespace(&metadata.from_address)?; + results.push(handle_message::( + namespace, + tables, + msg, + &metadata.from_address, + metadata.block_number.unwrap_or(u64::MAX), + &metadata.transaction_hash, + queries, + )); + } + Ok(results) +} + +pub fn handle_message( + namespace: NamespaceKey, + tables: &Tables, + msg: &IntrospectMsg, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, +) -> DbResult<()> { + match msg { + IntrospectMsg::CreateTable(event) => tables.create_table::( + namespace, + &event.id, + &event.name, + &event.primary, + &event.columns, + event.append_only, + from_address, + block_number, + transaction_hash, + queries, + ), + IntrospectMsg::UpdateTable(event) => tables.update_table::( + namespace, + &event.id, + &event.name, + &event.primary, + &event.columns, + from_address, + block_number, + transaction_hash, + queries, + ), + IntrospectMsg::AddColumns(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::DropColumns(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::RetypeColumns(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::RetypePrimary(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::RenameTable(_) + | IntrospectMsg::DropTable(_) + | IntrospectMsg::RenameColumns(_) + | IntrospectMsg::RenamePrimary(_) => Ok(()), + IntrospectMsg::InsertsFields(event) => tables.insert_fields::( + namespace, + event, + from_address, + block_number, + transaction_hash, + queries, + ), + IntrospectMsg::DeleteRecords(_) | IntrospectMsg::DeletesFields(_) => Ok(()), + } +} diff --git a/crates/introspect-sql-sink/src/runtime.rs b/crates/introspect-sql-sink/src/runtime.rs new file mode 100644 index 00000000..1c3667eb --- /dev/null +++ b/crates/introspect-sql-sink/src/runtime.rs @@ -0,0 +1,58 @@ +use crate::tables::Tables; +use crate::{ + DbColumn, DbDeadField, DbResult, DbTable, IntrospectInitialize, IntrospectProcessor, + IntrospectSqlSink, NamespaceMode, +}; +use async_trait::async_trait; +use torii_introspect::events::IntrospectBody; +use torii_sql::DbPool; + +impl IntrospectSqlSink for DbPool { + const NAME: &'static str = "introspect-sql"; +} + +#[async_trait] +impl IntrospectInitialize for DbPool { + async fn initialize(&self) -> DbResult { + match self { + DbPool::Postgres(pg) => pg.initialize().await, + DbPool::Sqlite(site) => site.initialize().await, + } + } + async fn load_tables(&self, namespaces: &Option>) -> DbResult> { + match self { + DbPool::Postgres(pg) => pg.load_tables(namespaces).await, + DbPool::Sqlite(site) => site.load_tables(namespaces).await, + } + } + async fn load_columns(&self, namespaces: &Option>) -> DbResult> { + match self { + DbPool::Postgres(pg) => pg.load_columns(namespaces).await, + DbPool::Sqlite(site) => site.load_columns(namespaces).await, + } + } + async fn load_dead_fields( + &self, + namespaces: &Option>, + ) -> DbResult> { + match self { + DbPool::Postgres(pg) => pg.load_dead_fields(namespaces).await, + DbPool::Sqlite(site) => site.load_dead_fields(namespaces).await, + } + } +} + +#[async_trait] +impl IntrospectProcessor for DbPool { + async fn process_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + match self { + DbPool::Postgres(pg) => pg.process_msgs(namespaces, tables, msgs).await, + DbPool::Sqlite(site) => site.process_msgs(namespaces, tables, msgs).await, + } + } +} diff --git a/crates/introspect-sqlite-sink/src/sink.rs b/crates/introspect-sql-sink/src/sink.rs similarity index 82% rename from crates/introspect-sqlite-sink/src/sink.rs rename to crates/introspect-sql-sink/src/sink.rs index 86706aed..8f133ed5 100644 --- a/crates/introspect-sqlite-sink/src/sink.rs +++ b/crates/introspect-sql-sink/src/sink.rs @@ -1,32 +1,36 @@ -use crate::processor::IntrospectSqliteDb; use anyhow::Result; use async_trait::async_trait; use std::sync::Arc; use torii::axum::Router; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, SinkContext, TopicInfo}, -}; +use torii::etl::envelope::{Envelope, TypeId}; +use torii::etl::extractor::ExtractionBatch; +use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_sqlite::SqliteConnection; -pub const LOGGING_TARGET: &str = "torii::sinks::introspect::sqlite"; +use crate::{IntrospectDb, IntrospectInitialize, IntrospectProcessor}; + +pub const LOGGING_TARGET: &str = "torii::sinks::introspect-sql"; const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); +pub trait IntrospectSqlSink { + const NAME: &'static str; +} + #[async_trait] -impl Sink for IntrospectSqliteDb { +impl Sink + for IntrospectDb +{ fn name(&self) -> &'static str { - "introspect-sqlite" + Backend::NAME } fn interested_types(&self) -> Vec { - vec![TypeId::new("introspect")] + vec![INTROSPECT_TYPE] } async fn process(&self, envelopes: &[Envelope], _batch: &ExtractionBatch) -> Result<()> { let mut processed = 0usize; - let mut create_tables = 0usize; + let mut create_tables: usize = 0usize; let mut update_tables = 0usize; let mut inserts_fields = 0usize; let mut inserted_records = 0usize; @@ -102,10 +106,11 @@ impl Sink for IntrospectSqliteDb { _event_bus: Arc, _context: &SinkContext, ) -> Result<()> { - self.initialize_introspect_sqlite_sink().await?; + self.initialize_introspect_sql_sink().await?; tracing::info!( target: LOGGING_TARGET, - "Initialized introspect SQLite sink" + "Connected to introspect SQL sink with database: {}", + Backend::NAME ); Ok(()) } diff --git a/crates/introspect-sql-sink/src/sqlite/append_only.rs b/crates/introspect-sql-sink/src/sqlite/append_only.rs new file mode 100644 index 00000000..5607c0a2 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/append_only.rs @@ -0,0 +1,71 @@ +use crate::sqlite::record::SqliteDeserializer; +use crate::sqlite::table::qualified_table_name; +use crate::sqlite::types::{SqliteType, TypeDefSqliteExt}; +use crate::{RecordResult, Table}; +use introspect_types::bytes::IntoByteSource; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::ColumnInfo; +use itertools::Itertools; +use sqlx::Arguments; +use sqlx::Error::Encode as EncodeError; +use starknet_types_core::felt::Felt; +use std::fmt::Write as FmtWrite; +use std::sync::Arc; +use torii_introspect::Record; +use torii_sql::{Queries, SqliteArguments, SqliteQuery}; + +pub fn append_only_record_queries( + table: &Table, + column_ids: &[Felt], + records: &[Record], + _from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let table_name = qualified_table_name(&table.namespace, &table.name); + let primary = &table.primary.name; + let columns = table.columns.iter().collect_vec(); + let mut sql = format!(r#"INSERT INTO "{table_name}" ("{primary}", "__revision""#,); + for name in columns.names() { + write!(sql, r#", "{name}""#).unwrap(); + } + write!(sql, r#") VALUES (?, (SELECT COALESCE(MAX("__revision"), 0) + 1 FROM "{table_name}" WHERE "{primary}" = ?1)"#).unwrap(); + for (id, ColumnInfo { name, type_def, .. }) in &columns { + match column_ids.iter().position(|c| &c == id) { + Some(index) => write!(sql, r#", {}"#, type_def.index_placeholder(index + 2)?).unwrap(), + None => match type_def.try_into()? { + SqliteType::Json => write!( + sql, + r#", (SELECT jsonb("{name}") FROM "{table_name}" WHERE "{primary}" = ?1 ORDER BY "__revision" DESC LIMIT 1)"# + ).unwrap(), + _ => write!( + sql, + r#", (SELECT "{name}" FROM "{table_name}" WHERE "{primary}" = ?1 ORDER BY "__revision" DESC LIMIT 1)"# + ).unwrap(), + } + } + } + let schema = table.get_record_schema(column_ids)?; + sql.push(')'); + let sql: Arc = sql.into(); + for record in records { + let mut arguments: SqliteArguments<'static> = SqliteArguments::default(); + let mut primary_data = record.id.as_slice().into_source(); + let mut data = record.values.as_slice().into_source(); + arguments + .add( + schema + .primary_type_def() + .deserialize_column(&mut primary_data)?, + ) + .map_err(EncodeError)?; + for type_def in schema.columns().type_defs() { + arguments + .add(type_def.deserialize_column(&mut data)?) + .map_err(EncodeError)?; + } + queries.add((sql.clone(), arguments)); + } + Ok(()) +} diff --git a/crates/introspect-sql-sink/src/sqlite/backend.rs b/crates/introspect-sql-sink/src/sqlite/backend.rs new file mode 100644 index 00000000..6f0d4999 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/backend.rs @@ -0,0 +1,190 @@ +use crate::sqlite::append_only::append_only_record_queries; +use crate::sqlite::record::insert_record_queries; +use crate::sqlite::table::{ + create_table_query, persist_table_state_query, qualified_table_name, update_column, + update_columns, FETCH_TABLES_QUERY, +}; +use crate::{ + DbColumn, DbDeadField, DbResult, DbTable, IntrospectDb, IntrospectInitialize, + IntrospectQueryMaker, IntrospectSqlSink, RecordResult, Table, TableResult, UpgradeResultExt, +}; +use async_trait::async_trait; +use introspect_types::{ColumnDef, ColumnInfo, PrimaryDef, ResultInto}; +use itertools::Itertools; +use sqlx::prelude::FromRow; +use sqlx::types::Json; +use starknet_types_core::felt::{Felt, FromStrError}; +use std::collections::HashMap; +use torii_introspect::Record; +use torii_sql::{PoolExt, Queries, Sqlite, SqlitePool, SqliteQuery}; + +pub const INTROSPECT_SQLITE_SINK_MIGRATIONS: sqlx::migrate::Migrator = + sqlx::migrate!("./migrations/sqlite"); + +pub type IntrospectSqliteDb = IntrospectDb; + +#[derive(FromRow)] +pub struct SqliteTableRow { + namespace: String, + id: String, + owner: String, + name: String, + primary: Json, + columns: Json>, + append_only: bool, + alive: bool, +} + +impl TryFrom for DbTable { + type Error = FromStrError; + fn try_from(value: SqliteTableRow) -> Result { + Ok(DbTable { + namespace: value.namespace, + id: Felt::from_hex(&value.id)?, + owner: Felt::from_hex(&value.owner)?, + name: value.name, + primary: value.primary.0, + columns: value.columns.0.into_iter().map_into().collect(), + dead: HashMap::new(), + append_only: value.append_only, + alive: value.alive, + }) + } +} + +#[async_trait] +impl IntrospectQueryMaker for Sqlite { + fn create_table_queries( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + queries.add(create_table_query( + namespace, + name, + primary, + columns, + append_only, + )?); + persist_table_state_query( + namespace, + id, + name, + primary, + columns, + append_only, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn update_table_queries( + table: &mut Table, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + let table_name = qualified_table_name(&table.namespace, name); + if table.name != name { + queries.add(format!( + r#"ALTER TABLE "{}" RENAME TO "{table_name}""#, + qualified_table_name(&table.namespace, &table.name), + )); + table.name = name.to_string(); + } + update_column( + &table_name, + &mut table.primary, + &primary.name, + &((&primary.type_def).into()), + queries, + ) + .to_table_result(&table_name, "primary")?; + update_columns(&mut table.columns, &table_name, columns, queries)?; + persist_table_state_query( + &table.namespace, + &table.id, + &table.name, + primary, + &table.columns.iter().collect_vec(), + table.append_only, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + _from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, + ) -> RecordResult<()> { + if table.append_only { + append_only_record_queries( + table, + columns, + records, + _from_address, + _block_number, + _transaction_hash, + queries, + ) + } else { + insert_record_queries( + table, + columns, + records, + _from_address, + _block_number, + _transaction_hash, + queries, + ) + } + } +} + +impl IntrospectSqlSink for SqlitePool { + const NAME: &'static str = "Introspect Sqlite"; +} + +#[async_trait] +impl IntrospectInitialize for SqlitePool { + async fn load_tables(&self, _schemas: &Option>) -> DbResult> { + let rows: Vec = sqlx::query_as(FETCH_TABLES_QUERY) + .fetch_all(self.pool()) + .await?; + + let tables: Vec = rows + .into_iter() + .map(|row| row.try_into()) + .collect::>()?; + Ok(tables) + } + async fn load_columns(&self, _schemas: &Option>) -> DbResult> { + Ok(Vec::new()) + } + async fn load_dead_fields(&self, _schemas: &Option>) -> DbResult> { + Ok(Vec::new()) + } + async fn initialize(&self) -> DbResult<()> { + self.migrate(Some("introspect"), INTROSPECT_SQLITE_SINK_MIGRATIONS) + .await + .err_into() + } +} diff --git a/crates/introspect-sqlite-sink/src/json.rs b/crates/introspect-sql-sink/src/sqlite/json.rs similarity index 92% rename from crates/introspect-sqlite-sink/src/json.rs rename to crates/introspect-sql-sink/src/sqlite/json.rs index 3b7c28d8..7cf2f75b 100644 --- a/crates/introspect-sqlite-sink/src/json.rs +++ b/crates/introspect-sql-sink/src/sqlite/json.rs @@ -2,7 +2,7 @@ use introspect_types::serialize::ToCairoDeSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; use introspect_types::{CairoDeserializer, ResultDef, TupleDef, TypeDef}; use primitive_types::{U256, U512}; -use serde::ser::SerializeMap; +use serde::ser::{SerializeMap, SerializeTuple}; use serde::Serializer; pub struct SqliteJsonSerializer; @@ -54,9 +54,9 @@ impl CairoTypeSerialization for SqliteJsonSerializer { serializer: S, tuple: &'a TupleDef, ) -> Result { - let mut seq = serializer.serialize_map(Some(tuple.elements.len()))?; - for (index, element) in tuple.elements.iter().enumerate() { - seq.serialize_entry(&format!("_{index}"), &element.to_de_se(data, self))?; + let mut seq = serializer.serialize_tuple(tuple.elements.len())?; + for element in tuple.elements.iter() { + seq.serialize_element(&element.to_de_se(data, self))?; } seq.end() } diff --git a/crates/introspect-sql-sink/src/sqlite/mod.rs b/crates/introspect-sql-sink/src/sqlite/mod.rs new file mode 100644 index 00000000..3f8c4ef1 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/mod.rs @@ -0,0 +1,12 @@ +pub mod append_only; +pub mod backend; +pub mod json; +pub mod record; +pub mod table; +pub mod types; + +use sqlx::migrate::Migrator; + +pub use backend::IntrospectSqliteDb; + +pub const INTROSPECT_SQLITE_SINK_MIGRATIONS: Migrator = sqlx::migrate!("./migrations/sqlite"); diff --git a/crates/introspect-sql-sink/src/sqlite/record.rs b/crates/introspect-sql-sink/src/sqlite/record.rs new file mode 100644 index 00000000..ab7d5d48 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/record.rs @@ -0,0 +1,230 @@ +use crate::sqlite::json::SqliteJsonSerializer; +use crate::sqlite::table::qualified_table_name; +use crate::sqlite::types::{SqliteColumn, SqliteType}; +use crate::{RecordResult, Table, TypeResult}; +use introspect_types::bytes::IntoByteSource; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::serialize::CairoSeFrom; +use introspect_types::{CairoDeserializer, DecodeError, EthAddress, ResultInto, TypeDef}; +use itertools::Itertools; +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::sqlite::SqliteArgumentValue; +use sqlx::Error::Encode as EncodeError; +use sqlx::{Arguments, Encode, Sqlite, Type}; +use starknet_types_core::felt::Felt; +use std::sync::Arc; +use torii_introspect::Record; +use torii_sql::{Queries, SqliteArguments, SqliteQuery}; + +pub fn coalesce_sql<'a>(table_name: &str, column: &SqliteColumn<'a>) -> String { + let column_name = column.name; + match column.sql_type { + SqliteType::Json => { + format!( + r#""{column_name}" = COALESCE(jsonb(excluded."{column_name}"), "{table_name}"."{column_name}")"# + ) + } + _ => format!( + r#""{column_name}" = COALESCE(excluded."{column_name}", "{table_name}"."{column_name}")"# + ), + } +} + +pub enum SqliteValue { + Null, + Integer(i64), + Text(String), + Blob(Vec), +} + +impl SqliteValue { + fn integer(n: impl Into) -> Self { + SqliteValue::Integer(n.into()) + } + fn text(s: impl ToString) -> Self { + SqliteValue::Text(s.to_string()) + } +} + +impl From for SqliteValue { + fn from(value: String) -> Self { + SqliteValue::Text(value) + } +} + +impl From> for SqliteValue { + fn from(value: Vec) -> Self { + SqliteValue::Blob(value) + } +} + +impl From<[u8; N]> for SqliteValue { + fn from(value: [u8; N]) -> Self { + SqliteValue::Blob(value.to_vec()) + } +} + +impl From for SqliteValue { + fn from(value: Felt) -> Self { + SqliteValue::Text(format!("{value:#064x}")) + } +} + +impl From for SqliteValue { + fn from(value: EthAddress) -> Self { + SqliteValue::Text(format!("0x{}", hex::encode(value.0))) + } +} + +impl From for SqliteValue { + fn from(value: bool) -> Self { + SqliteValue::Integer(if value { 1 } else { 0 }) + } +} + +pub trait SqliteDeserializer { + fn deserialize_column(&self, data: &mut impl CairoDeserializer) -> RecordResult; + fn deserialize_json(&self, data: &mut impl CairoDeserializer) -> RecordResult; +} + +impl SqliteDeserializer for TypeDef { + fn deserialize_column(&self, data: &mut impl CairoDeserializer) -> RecordResult { + match self { + TypeDef::None => Ok(SqliteValue::Null), + TypeDef::Felt252 + | TypeDef::ClassHash + | TypeDef::ContractAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress => data.next_felt().result_into(), + TypeDef::ShortUtf8 => data.next_short_string().result_into(), + TypeDef::Bytes31 | TypeDef::Bytes31Encoded(_) => data.next_bytes::<31>().result_into(), + TypeDef::Bool => data.next_bool().result_into(), + TypeDef::U8 => data.next_u8().map(SqliteValue::integer).err_into(), + TypeDef::U16 => data.next_u16().map(SqliteValue::integer).err_into(), + TypeDef::U32 => data.next_u32().map(SqliteValue::integer).err_into(), + TypeDef::U64 => data.next_u64().map(SqliteValue::text).err_into(), + TypeDef::U128 => data.next_u128().map(SqliteValue::text).err_into(), + TypeDef::U256 => data.next_u256().map(SqliteValue::text).err_into(), + TypeDef::U512 => data.next_u512().map(SqliteValue::text).err_into(), + TypeDef::I8 => data.next_i8().map(SqliteValue::integer).err_into(), + TypeDef::I16 => data.next_i16().map(SqliteValue::integer).err_into(), + TypeDef::I32 => data.next_i32().map(SqliteValue::integer).err_into(), + TypeDef::I64 => data.next_i64().map(SqliteValue::integer).err_into(), + TypeDef::I128 => data.next_i128().map(SqliteValue::text).err_into(), + TypeDef::EthAddress => data.next_eth_address().result_into(), + TypeDef::Utf8String => data.next_string().result_into(), + TypeDef::ByteArray | TypeDef::ByteArrayEncoded(_) | TypeDef::Custom(_) => { + data.next_byte_array_bytes().result_into() + } + TypeDef::Tuple(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Felt252Dict(_) + | TypeDef::Struct(_) + | TypeDef::Enum(_) + | TypeDef::Option(_) + | TypeDef::Result(_) + | TypeDef::Nullable(_) => self.deserialize_json(data), + TypeDef::Ref(_) => Err(DecodeError::message( + "TypeDef Ref needs to be expanded before transoding", + )) + .err_into(), + } + } + fn deserialize_json(&self, data: &mut impl CairoDeserializer) -> RecordResult { + let se = CairoSeFrom::new(self, data, &SqliteJsonSerializer); + serde_json::to_string(&se).result_into() + } +} + +impl From for SqliteArgumentValue<'_> { + fn from(value: SqliteValue) -> Self { + match value { + SqliteValue::Null => SqliteArgumentValue::Null, + SqliteValue::Integer(n) => SqliteArgumentValue::Int64(n), + SqliteValue::Text(s) => SqliteArgumentValue::Text(s.into()), + SqliteValue::Blob(b) => SqliteArgumentValue::Blob(b.into()), + } + } +} + +impl Type for SqliteValue { + fn type_info() -> ::TypeInfo { + // SqliteValue is dynamically typed; report as Text since SQLite is flexible with types. + >::type_info() + } + + fn compatible(ty: &::TypeInfo) -> bool { + >::compatible(ty) + || >::compatible(ty) + || as Type>::compatible(ty) + } +} + +impl<'q> Encode<'q, Sqlite> for SqliteValue { + fn encode_by_ref(&self, buf: &mut Vec>) -> Result { + match self { + SqliteValue::Null => Ok(IsNull::Yes), + SqliteValue::Integer(n) => >::encode_by_ref(n, buf), + SqliteValue::Text(s) => >::encode_by_ref(s, buf), + SqliteValue::Blob(b) => as Encode>::encode_by_ref(b, buf), + } + } +} + +pub fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + _from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let schema = table.get_record_schema(columns)?; + let table_name = qualified_table_name(&table.namespace, &table.name); + let all_columns = schema.all_columns(); + let sql_columns = all_columns + .iter() + .map(|c| (*c).try_into()) + .collect::>>()?; + let column_names = all_columns.names(); + let placeholders = sql_columns.iter().map(SqliteColumn::placeholder).join(", "); + let coalesce = sql_columns[1..] + .iter() + .map(|col| coalesce_sql(&table_name, col)) + .join(", "); + let sql: Arc = format!( + r#"INSERT INTO "{table_name}" ({}) VALUES ({}) ON CONFLICT("{}") DO UPDATE SET {}"#, + column_names + .iter() + .map(|name| format!(r#""{name}""#)) + .collect::>() + .join(", "), + placeholders, + schema.primary_name(), + coalesce + ) + .into(); + + for record in records { + let mut arguments: SqliteArguments<'static> = SqliteArguments::default(); + let mut primary_data = record.id.as_slice().into_source(); + let mut data = record.values.as_slice().into_source(); + arguments + .add( + schema + .primary_type_def() + .deserialize_column(&mut primary_data)?, + ) + .map_err(EncodeError)?; + for type_def in schema.columns().type_defs() { + arguments + .add(type_def.deserialize_column(&mut data)?) + .map_err(EncodeError)?; + } + queries.add((sql.clone(), arguments)); + } + Ok(()) +} diff --git a/crates/introspect-sql-sink/src/sqlite/table.rs b/crates/introspect-sql-sink/src/sqlite/table.rs new file mode 100644 index 00000000..eb48b10c --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/table.rs @@ -0,0 +1,216 @@ +use crate::sqlite::types::SqliteType; +use crate::{TableResult, UpgradeError, UpgradeResult, UpgradeResultExt}; +use introspect_types::schema::AsColumnRef; +use introspect_types::{ColumnDef, ColumnInfo, PrimaryDef, TypeDef}; +use serde::ser::SerializeMap; +use serde::Serializer; +use serde_json::{Result as JsonResult, Serializer as JsonSerializer}; +use sqlx::Arguments; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::fmt::{Display, Write}; +use torii_sql::types::SqlFelt; +use torii_sql::{Queries, SqliteArguments, SqliteQuery}; + +pub const FETCH_TABLES_QUERY: &str = r#" + SELECT namespace, id, owner, name, "primary", columns, append_only, alive + FROM introspect_db_tables + ORDER BY updated_at ASC +"#; + +const INSERT_TABLE_QUERY: &str = r#" INSERT INTO introspect_db_tables + (namespace, id, owner, name, "primary", columns, append_only, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, unixepoch()) + ON CONFLICT (namespace, id) DO UPDATE SET + owner = excluded.owner, name = excluded.name, "primary" = excluded."primary", columns = excluded.columns, append_only = excluded.append_only, updated_at = unixepoch() +"#; + +struct TableName<'a>(&'a str, &'a str); + +impl Display for TableName<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !self.0.is_empty() { + write!(f, "{}__", self.0)?; + } + self.1.fmt(f) + } +} + +pub fn qualified_table_name(namespace: &str, table_name: &str) -> String { + if namespace.is_empty() { + table_name.to_string() + } else { + format!("{}__{}", namespace, table_name) + } +} + +pub fn serialize_columns<'a>(columns: &'a [impl AsColumnRef<'a>]) -> JsonResult { + let mut data = Vec::new(); + let mut serializer = JsonSerializer::new(&mut data); + let mut array = serializer.serialize_map(Some(columns.len()))?; + for column in columns { + let (id, info) = column.as_entry(); + array.serialize_entry(id, &info)?; + } + array.end()?; + Ok(unsafe { String::from_utf8_unchecked(data) }) +} + +#[allow(clippy::too_many_arguments)] +pub fn persist_table_state_query<'a>( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &'a [impl AsColumnRef<'a>], + append_only: bool, + from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, +) -> TableResult<()> { + let mut args = SqliteArguments::default(); + args.add(namespace.to_string())?; + args.add(Into::::into(*id))?; + args.add(Into::::into(*from_address))?; + args.add(name.to_string())?; + args.add(serde_json::to_string(primary)?)?; + args.add(serialize_columns(columns)?)?; + args.add(append_only)?; + queries.add((INSERT_TABLE_QUERY, args)); + Ok(()) +} + +pub fn create_table_query( + namespace: &str, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, +) -> TableResult { + let table_name = TableName(namespace, name); + let mut query = format!( + r#"CREATE TABLE IF NOT EXISTS "{table_name}" ("{}" {}"#, + primary.name, + TryInto::::try_into(primary)? + ); + if append_only { + query.push_str(r#", "__revision" INTEGER NOT NULL"#); + } + for column in columns { + let sql_type: SqliteType = column.try_into()?; + write!(query, r#", "{}" {sql_type}"#, column.name).unwrap(); + } + if append_only { + write!( + query, + r#", PRIMARY KEY ("{}", "__revision"));"#, + primary.name + ) + .unwrap(); + } else { + write!(query, r#", PRIMARY KEY ("{}"));"#, primary.name).unwrap(); + } + Ok(query) +} + +pub fn update_columns( + columns: &mut HashMap, + table_name: &str, + new: &[ColumnDef], + queries: &mut Vec, +) -> TableResult<()> { + for column in new { + let result = match columns.get_mut(&column.id) { + Some(existing) => update_column( + table_name, + existing, + &column.name, + &column.type_def, + queries, + ), + None => { + columns.insert( + column.id, + ColumnInfo { + name: column.name.clone(), + type_def: column.type_def.clone(), + attributes: column.attributes.clone(), + }, + ); + create_column_query(table_name, column).map(|query| queries.add(query)) + } + }; + result.to_table_result(table_name, &column.name)?; + } + Ok(()) +} + +pub fn update_column( + table_name: &str, + column: &mut ColumnInfo, + new_name: &str, + new_type: &TypeDef, + queries: &mut Vec, +) -> UpgradeResult { + use introspect_types::TypeDef::{ + Array, Bool, ByteArray, ByteArrayEncoded, Bytes31, Bytes31Encoded, ClassHash, + ContractAddress, Custom, Enum, EthAddress, Felt252, FixedArray, Nullable, + Option as TDOption, Result as TDResult, ShortUtf8, StorageAddress, StorageBaseAddress, + Struct, Tuple, Utf8String, I128, I16, I32, I64, I8, U128, U16, U256, U32, U512, U64, U8, + }; + if column.name != new_name { + queries.add(format!( + r#"ALTER TABLE "{table_name}" RENAME COLUMN "{}" TO "{new_name}";"#, + column.name + )); + column.name = new_name.to_string(); + } + let cast = match (&column.type_def, &new_type) { + (Bool | U8 | U16 | U32, Bool | U8 | U16 | U32 | I8 | I16 | I32 | I64) + | (I8 | I16 | I32 | I64, I8 | I16 | I32 | I64) + | (U64 | U128 | U256 | U512, U64 | U128 | U256 | U512 | I128) + | (I128, I128) + | ( + Felt252 | ClassHash | ContractAddress | EthAddress | StorageAddress + | StorageBaseAddress, + Felt252 | ClassHash | ContractAddress | EthAddress | StorageAddress + | StorageBaseAddress, + ) + | (ShortUtf8 | Utf8String, ShortUtf8 | Utf8String) + | ( + Bytes31 | Bytes31Encoded(_) | ByteArray | ByteArrayEncoded(_) | Custom(_), + Bytes31 | Bytes31Encoded(_) | ByteArray | ByteArrayEncoded(_) | Custom(_), + ) + | ( + Tuple(_) | Array(_) | FixedArray(_) | Struct(_) | Enum(_) | TDOption(_) | TDResult(_) + | Nullable(_), + Tuple(_) | Array(_) | FixedArray(_) | Struct(_) | Enum(_) | TDOption(_) | TDResult(_) + | Nullable(_), + ) => None, + (Bool | U8 | U16 | U32, U64 | U128 | U256 | U512 | I128) | (I8 | I16 | I32 | I64, I128) => { + Some(format!(r#"CAST("{new_name}" AS TEXT)"#)) + } + ( + Bool | U8 | U16 | U32, + Felt252 | ClassHash | ContractAddress | EthAddress | StorageAddress + | StorageBaseAddress, + ) => Some(format!(r#"printf('0x%064x', "{new_name}")"#)), + _ => return UpgradeError::type_upgrade_err(&column.type_def, new_type), + }; + column.type_def = new_type.clone(); + if let Some(cast) = cast { + queries.add(format!( + r#"UPDATE "{table_name}" SET "{new_name}" = {cast};"# + )); + } + Ok(()) +} + +pub fn create_column_query(table_name: &str, column: &ColumnDef) -> UpgradeResult { + let sql_type: SqliteType = column.try_into()?; + Ok(format!( + r#"ALTER TABLE "{table_name}" ADD COLUMN "{}" {sql_type};"#, + column.name + )) +} diff --git a/crates/introspect-sql-sink/src/sqlite/types.rs b/crates/introspect-sql-sink/src/sqlite/types.rs new file mode 100644 index 00000000..749e69eb --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/types.rs @@ -0,0 +1,161 @@ +use std::fmt::Display; + +use introspect_types::{ColumnDef, ColumnInfo, PrimaryDef, PrimaryTypeDef, TypeDef}; + +use crate::{TypeError, TypeResult}; + +pub enum SqliteType { + Null, + Text, + Integer, + Real, + Blob, + Json, +} + +impl Display for SqliteType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SqliteType::Null => write!(f, "NULL"), + SqliteType::Text => write!(f, "TEXT"), + SqliteType::Integer => write!(f, "INTEGER"), + SqliteType::Real => write!(f, "REAL"), + SqliteType::Blob => write!(f, "BLOB"), + SqliteType::Json => write!(f, "TEXT"), + } + } +} + +pub struct SqliteColumn<'a> { + pub name: &'a str, + pub sql_type: SqliteType, +} + +impl TryFrom<&TypeDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &TypeDef) -> Result { + match value { + TypeDef::None => Ok(SqliteType::Null), + TypeDef::Bool + | TypeDef::U8 + | TypeDef::U16 + | TypeDef::U32 + | TypeDef::I8 + | TypeDef::I16 + | TypeDef::I32 + | TypeDef::I64 => Ok(SqliteType::Integer), + TypeDef::U64 | TypeDef::U128 | TypeDef::U256 | TypeDef::U512 | TypeDef::I128 => { + Ok(SqliteType::Text) + } + TypeDef::Felt252 + | TypeDef::ClassHash + | TypeDef::ContractAddress + | TypeDef::EthAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress => Ok(SqliteType::Text), + TypeDef::ShortUtf8 | TypeDef::Utf8String => Ok(SqliteType::Text), + TypeDef::Bytes31 + | TypeDef::Bytes31Encoded(_) + | TypeDef::ByteArray + | TypeDef::ByteArrayEncoded(_) => Ok(SqliteType::Blob), + TypeDef::Struct(_) + | TypeDef::Enum(_) + | TypeDef::Tuple(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Option(_) + | TypeDef::Nullable(_) + | TypeDef::Result(_) => Ok(SqliteType::Json), + TypeDef::Custom(_) => Ok(SqliteType::Blob), + _ => Err(TypeError::UnsupportedType(value.item_name().to_string())), + } + } +} + +impl TryFrom<&ColumnDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &ColumnDef) -> Result { + (&value.type_def).try_into() + } +} + +impl TryFrom<&ColumnInfo> for SqliteType { + type Error = TypeError; + + fn try_from(value: &ColumnInfo) -> Result { + (&value.type_def).try_into() + } +} + +impl TryFrom<&PrimaryTypeDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &PrimaryTypeDef) -> Result { + (&Into::::into(value)).try_into() + } +} + +impl TryFrom<&PrimaryDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &PrimaryDef) -> Result { + (&value.type_def).try_into() + } +} + +impl<'a> TryFrom<&'a ColumnInfo> for SqliteColumn<'a> { + type Error = TypeError; + + fn try_from(value: &'a ColumnInfo) -> Result { + let sql_type = (&value.type_def).try_into()?; + Ok(SqliteColumn { + name: &value.name, + sql_type, + }) + } +} + +impl SqliteType { + pub fn placeholder(&self) -> &'static str { + match self { + SqliteType::Null => "NULL", + SqliteType::Text | SqliteType::Integer | SqliteType::Real | SqliteType::Blob => "?", + SqliteType::Json => "jsonb(?)", + } + } + pub fn index_placeholder(&self, index: usize) -> String { + match self { + SqliteType::Null => "NULL".to_string(), + SqliteType::Text | SqliteType::Integer | SqliteType::Real | SqliteType::Blob => { + format!("?{index}") + } + SqliteType::Json => format!("jsonb(?{index})"), + } + } +} + +impl<'a> SqliteColumn<'a> { + pub fn placeholder(&self) -> &'static str { + self.sql_type.placeholder() + } + pub fn index_placeholder(&self, index: usize) -> String { + self.sql_type.index_placeholder(index) + } +} + +pub trait TypeDefSqliteExt { + fn placeholder(&self) -> TypeResult<&'static str>; + fn index_placeholder(&self, index: usize) -> TypeResult; +} + +impl TypeDefSqliteExt for TypeDef { + fn placeholder(&self) -> TypeResult<&'static str> { + self.try_into().map(|t: SqliteType| t.placeholder()) + } + fn index_placeholder(&self, index: usize) -> TypeResult { + self.try_into() + .map(|t: SqliteType| t.index_placeholder(index)) + } +} diff --git a/crates/introspect-sql-sink/src/table.rs b/crates/introspect-sql-sink/src/table.rs new file mode 100644 index 00000000..7d9242ee --- /dev/null +++ b/crates/introspect-sql-sink/src/table.rs @@ -0,0 +1,137 @@ +use crate::error::{CollectColumnResults, ColumnNotFoundError, ColumnsNotFoundError}; +use introspect_types::{ColumnDef, ColumnInfo, MemberDef, PrimaryDef, TypeDef}; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::rc::Rc; +use torii_introspect::tables::RecordSchema; + +#[derive(Debug)] +pub struct Table { + pub id: Felt, + pub namespace: String, + pub name: String, + pub owner: Felt, + pub primary: ColumnInfo, + pub columns: HashMap, + pub dead: HashMap, + pub append_only: bool, + pub alive: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeadField { + pub name: String, + pub type_def: TypeDef, +} + +#[derive(Debug)] +pub struct DeadFieldDef { + pub id: u128, + pub name: String, + pub type_def: TypeDef, +} + +impl From for DeadField { + fn from(value: MemberDef) -> Self { + DeadField { + name: value.name, + type_def: value.type_def, + } + } +} + +impl From for MemberDef { + fn from(value: DeadField) -> Self { + MemberDef { + name: value.name, + attributes: Vec::new(), + type_def: value.type_def, + } + } +} + +impl From for (u128, DeadField) { + fn from(value: DeadFieldDef) -> Self { + ( + value.id, + DeadField { + name: value.name, + type_def: value.type_def, + }, + ) + } +} + +impl From<(u128, DeadField)> for DeadFieldDef { + fn from(value: (u128, DeadField)) -> Self { + DeadFieldDef { + id: value.0, + name: value.1.name, + type_def: value.1.type_def, + } + } +} + +impl Table { + pub fn column(&self, id: &Felt) -> Result<&ColumnInfo, ColumnNotFoundError> { + self.columns.get(id).ok_or(ColumnNotFoundError(*id)) + } + + pub fn namespace(&self) -> Rc { + self.namespace.as_str().into() + } + + pub fn columns(&self, ids: &[Felt]) -> Result, ColumnsNotFoundError> { + ids.iter().map(|id| self.column(id)).collect_columns() + } + + pub fn all_columns(&self) -> Vec<&ColumnInfo> { + self.columns.values().collect() + } + + pub fn columns_with_ids<'a>( + &'a self, + ids: &'a [Felt], + ) -> Result, ColumnsNotFoundError> { + ids.iter() + .map(|id| self.column(id).map(|col| (id, col))) + .collect_columns() + } + + pub fn all_columns_with_ids(&self) -> Vec<(&Felt, &ColumnInfo)> { + self.columns.iter().collect() + } + + #[allow(clippy::too_many_arguments)] + pub fn new( + namespace: String, + id: Felt, + owner: Felt, + name: String, + primary: PrimaryDef, + columns: &[ColumnDef], + dead: Option>, + append_only: bool, + ) -> Self { + Table { + id, + namespace, + owner, + name, + primary: primary.into(), + columns: columns.iter().cloned().map_into().collect(), + dead: dead.unwrap_or_default().into_iter().collect(), + append_only, + alive: true, + } + } + + pub fn get_record_schema( + &self, + columns: &[Felt], + ) -> Result, ColumnsNotFoundError> { + Ok(RecordSchema::new(&self.primary, self.columns(columns)?)) + } +} diff --git a/crates/introspect-sql-sink/src/tables.rs b/crates/introspect-sql-sink/src/tables.rs new file mode 100644 index 00000000..95693cdb --- /dev/null +++ b/crates/introspect-sql-sink/src/tables.rs @@ -0,0 +1,155 @@ +use crate::backend::IntrospectQueryMaker; +use crate::error::RecordResultExt; +use crate::namespace::{NamespaceKey, TableKey}; +use crate::table::Table; +use crate::{DbError, DbResult}; +use introspect_types::{ColumnDef, PrimaryDef, ResultInto}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::ops::Deref; +use std::sync::RwLock; +use torii_introspect::InsertsFields; +use torii_sql::FlexQuery; + +#[derive(Debug, Default)] +pub struct Tables(pub RwLock>); + +impl Deref for Tables { + type Target = RwLock>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[allow(clippy::too_many_arguments)] +impl Tables { + pub fn create_table( + &self, + namespace_key: NamespaceKey, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> DbResult<()> { + let namespace = namespace_key.to_string(); + + let key = TableKey::new(namespace_key, *id); + self.assert_table_not_exists(&key, name)?; + DB::create_table_queries( + &namespace, + id, + name, + primary, + columns, + append_only, + from_address, + block_number, + transaction_hash, + queries, + )?; + let mut tables = self.write()?; + tables.insert( + key, + Table::new( + namespace, + *id, + *from_address, + name.to_string(), + primary.clone(), + columns, + None, + append_only, + ), + ); + Ok(()) + } + + pub fn update_table( + &self, + namespace_key: NamespaceKey, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> DbResult<()> { + let mut tables = self.write()?; + let key = TableKey::new(namespace_key, *id); + let table = tables + .get_mut(&key) + .ok_or_else(|| DbError::TableNotFound(key.clone()))?; + DB::update_table_queries( + table, + name, + primary, + columns, + from_address, + block_number, + transaction_hash, + queries, + ) + .err_into() + } + + pub fn assert_table_not_exists(&self, id: &TableKey, name: &str) -> DbResult<()> { + match self.read()?.get(id) { + Some(existing) => Err(DbError::TableAlreadyExists( + id.clone(), + name.to_string(), + existing.name.to_string(), + )), + None => Ok(()), + } + } + + pub fn set_table_dead(&self, namespace: NamespaceKey, id: Felt) -> DbResult<()> { + let mut tables = self.write()?; + let key = TableKey::new(namespace, id); + match tables.get_mut(&key) { + Some(table) => { + table.alive = false; + Ok(()) + } + None => Err(DbError::TableNotFound(key)), + } + } + + pub fn insert_fields( + &self, + namespace: NamespaceKey, + event: &InsertsFields, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> DbResult<()> { + let tables = self.read().unwrap(); + let key = TableKey::new(namespace, event.table); + let table = match tables.get(&key) { + Some(table) => Ok(table), + None => Err(DbError::TableNotFound(key)), + }?; + if !table.alive { + return Ok(()); + } + DB::insert_record_queries( + table, + &event.columns, + &event.records, + from_address, + block_number, + transaction_hash, + queries, + ) + .to_db_result(&table.name) + } +} diff --git a/crates/introspect-sqlite-sink/migrations/002_schema_state.sql b/crates/introspect-sqlite-sink/migrations/002_schema_state.sql deleted file mode 100644 index caa564d4..00000000 --- a/crates/introspect-sqlite-sink/migrations/002_schema_state.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE IF NOT EXISTS introspect_sink_schema_state ( - table_id TEXT PRIMARY KEY, - table_schema_json TEXT NOT NULL, - alive INTEGER NOT NULL DEFAULT 1, - updated_at INTEGER NOT NULL DEFAULT (unixepoch()) -); diff --git a/crates/introspect-sqlite-sink/src/lib.rs b/crates/introspect-sqlite-sink/src/lib.rs deleted file mode 100644 index ac41c665..00000000 --- a/crates/introspect-sqlite-sink/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod json; -pub mod processor; -pub mod sink; -pub mod table; - -use sqlx::migrate::Migrator; - -pub const INTROSPECT_SQLITE_SINK_MIGRATIONS: Migrator = sqlx::migrate!("./migrations"); diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs deleted file mode 100644 index d3a00a93..00000000 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ /dev/null @@ -1,650 +0,0 @@ -use crate::json::SqliteJsonSerializer; -use crate::table::{SqliteTable, SqliteTableError}; -use crate::INTROSPECT_SQLITE_SINK_MIGRATIONS; -use introspect_types::{PrimaryTypeDef, TypeDef}; -use serde_json::{Serializer as JsonSerializer, Value}; -use sqlx::Error as SqlxError; -use sqlx::Row; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use std::fmt::Display; -use std::ops::Deref; -use std::sync::{PoisonError, RwLock}; -use torii::etl::envelope::MetaData; -use torii::etl::EventMsg; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_introspect::schema::TableSchema; -use torii_introspect::InsertsFields; -use torii_sqlite::SqliteConnection; - -#[derive(Debug, thiserror::Error)] -pub enum SqliteDbError { - #[error(transparent)] - DatabaseError(#[from] SqlxError), - #[error(transparent)] - JsonError(#[from] serde_json::Error), - #[error(transparent)] - TableError(#[from] SqliteTableError), - #[error("record frame must serialize to an object")] - InvalidRecordFrame, - #[error("Table with id: {0} already exists, incoming name: {1}, existing name: {2}")] - TableAlreadyExists(Felt, String, String), - #[error("Table not found with id: {0}")] - TableNotFound(Felt), - #[error("Table poison error: {0}")] - PoisonError(String), -} - -type SqliteDbResult = std::result::Result; - -impl From> for SqliteDbError { - fn from(err: PoisonError) -> Self { - Self::PoisonError(err.to_string()) - } -} - -#[derive(Debug, Default)] -pub struct SqliteTables(pub RwLock>); - -impl Deref for SqliteTables { - type Target = RwLock>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[derive(Debug, Clone, Default)] -pub enum SqliteNamespace { - #[default] - None, - Custom(String), -} - -impl SqliteNamespace { - pub fn prefix(&self) -> &str { - match self { - Self::None => "", - Self::Custom(prefix) => prefix, - } - } -} - -impl From<()> for SqliteNamespace { - fn from((): ()) -> Self { - Self::None - } -} - -impl From for SqliteNamespace { - fn from(value: String) -> Self { - if value.is_empty() { - Self::None - } else { - Self::Custom(value) - } - } -} - -impl From<&str> for SqliteNamespace { - fn from(value: &str) -> Self { - if value.is_empty() { - Self::None - } else { - Self::Custom(value.to_string()) - } - } -} - -impl Display for SqliteNamespace { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::None => f.write_str("main"), - Self::Custom(prefix) => f.write_str(prefix), - } - } -} - -impl SqliteTables { - pub fn assert_table_not_exists(&self, id: &Felt, name: &str) -> SqliteDbResult<()> { - match self.read()?.get(id) { - Some(existing) => Err(SqliteDbError::TableAlreadyExists( - *id, - name.to_string(), - existing.name.clone(), - )), - None => Ok(()), - } - } - - pub fn create_table( - &self, - namespace: &SqliteNamespace, - to_table: impl Into, - ) -> SqliteDbResult<(Felt, String)> { - let table = to_table.into(); - self.assert_table_not_exists(&table.id, &table.name)?; - let (id, sqlite_table) = SqliteTable::new_from_table(namespace.prefix(), table); - let create_query = create_table_query(&sqlite_table); - self.write()?.insert(id, sqlite_table); - Ok((id, create_query)) - } - - pub fn set_table_dead(&self, id: &Felt) -> SqliteDbResult<()> { - if let Some(table) = self.write()?.get_mut(id) { - table.alive = false; - return Ok(()); - } - Err(SqliteDbError::TableNotFound(*id)) - } -} - -fn sqlite_column_type(type_def: &TypeDef) -> &'static str { - if is_json_type(type_def) { - "JSONB" - } else if matches!( - type_def, - TypeDef::Bool - | TypeDef::I8 - | TypeDef::I16 - | TypeDef::I32 - | TypeDef::U8 - | TypeDef::U16 - | TypeDef::U32 - ) { - "INTEGER" - } else { - "TEXT" - } -} - -fn sqlite_primary_type(type_def: &PrimaryTypeDef) -> &'static str { - if matches!( - type_def, - PrimaryTypeDef::Bool - | PrimaryTypeDef::I8 - | PrimaryTypeDef::I16 - | PrimaryTypeDef::I32 - | PrimaryTypeDef::U8 - | PrimaryTypeDef::U16 - | PrimaryTypeDef::U32 - ) { - "INTEGER" - } else { - "TEXT" - } -} - -fn is_json_type(type_def: &TypeDef) -> bool { - matches!( - type_def, - TypeDef::Struct(_) - | TypeDef::Enum(_) - | TypeDef::Tuple(_) - | TypeDef::Array(_) - | TypeDef::FixedArray(_) - | TypeDef::Option(_) - | TypeDef::Nullable(_) - | TypeDef::Result(_) - ) -} - -fn create_table_query(table: &SqliteTable) -> String { - let primary_type = sqlite_primary_type(&table.primary.type_def); - let mut columns = Vec::with_capacity(table.columns.len() + 1); - columns.push(format!( - r#""{}" {primary_type} PRIMARY KEY"#, - table.primary.name - )); - for column_id in &table.order { - let column = &table.columns[column_id]; - let col_type = sqlite_column_type(&column.type_def); - columns.push(format!(r#""{}" {col_type}"#, column.name)); - } - format!( - r#"CREATE TABLE IF NOT EXISTS "{}" ({});"#, - table.storage_name, - columns.join(", ") - ) -} - -enum SqliteBindValue { - Null, - Integer(i64), - Text(String), -} - -fn to_bind_value(value: &Value, type_def: &TypeDef) -> SqliteBindValue { - if value.is_null() { - return SqliteBindValue::Null; - } - - match type_def { - TypeDef::Bool => match value.as_bool() { - Some(b) => SqliteBindValue::Integer(i64::from(b)), - None => SqliteBindValue::Null, - }, - TypeDef::I8 | TypeDef::I16 | TypeDef::I32 | TypeDef::U8 | TypeDef::U16 | TypeDef::U32 => { - match value.as_i64() { - Some(n) => SqliteBindValue::Integer(n), - None => SqliteBindValue::Null, - } - } - TypeDef::U64 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - TypeDef::I64 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - TypeDef::Felt252 - | TypeDef::ClassHash - | TypeDef::ContractAddress - | TypeDef::StorageAddress - | TypeDef::StorageBaseAddress - | TypeDef::EthAddress - | TypeDef::U256 - | TypeDef::U512 => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - - TypeDef::U128 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{:032x}", n as u128)), - None => match value.as_str() { - Some(s) => match s.parse::() { - Ok(n) => SqliteBindValue::Text(format!("0x{n:032x}")), - Err(_) => SqliteBindValue::Text(s.to_string()), - }, - None => SqliteBindValue::Null, - }, - }, - TypeDef::I128 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("{}", n as i128)), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - TypeDef::Struct(_) - | TypeDef::Enum(_) - | TypeDef::Tuple(_) - | TypeDef::Array(_) - | TypeDef::FixedArray(_) - | TypeDef::Option(_) - | TypeDef::Nullable(_) - | TypeDef::Result(_) => SqliteBindValue::Text(value.to_string()), - - _ => match value { - Value::String(s) => SqliteBindValue::Text(s.clone()), - _ => SqliteBindValue::Text(value.to_string()), - }, - } -} - -fn primary_to_bind_value(value: &Value, type_def: &PrimaryTypeDef) -> SqliteBindValue { - if value.is_null() { - return SqliteBindValue::Null; - } - - match type_def { - PrimaryTypeDef::Bool => match value.as_bool() { - Some(b) => SqliteBindValue::Integer(i64::from(b)), - None => SqliteBindValue::Null, - }, - PrimaryTypeDef::I8 - | PrimaryTypeDef::I16 - | PrimaryTypeDef::I32 - | PrimaryTypeDef::U8 - | PrimaryTypeDef::U16 - | PrimaryTypeDef::U32 => match value.as_i64() { - Some(n) => SqliteBindValue::Integer(n), - None => SqliteBindValue::Null, - }, - PrimaryTypeDef::U64 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - PrimaryTypeDef::I64 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - PrimaryTypeDef::Felt252 - | PrimaryTypeDef::ClassHash - | PrimaryTypeDef::ContractAddress - | PrimaryTypeDef::StorageAddress - | PrimaryTypeDef::StorageBaseAddress - | PrimaryTypeDef::EthAddress => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - - PrimaryTypeDef::U128 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{:032x}", n as u128)), - None => match value.as_str() { - Some(s) => match s.parse::() { - Ok(n) => SqliteBindValue::Text(format!("0x{n:032x}")), - Err(_) => SqliteBindValue::Text(s.to_string()), - }, - None => SqliteBindValue::Null, - }, - }, - PrimaryTypeDef::I128 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("{}", n as i128)), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - _ => match value { - Value::String(s) => SqliteBindValue::Text(s.clone()), - _ => SqliteBindValue::Text(value.to_string()), - }, - } -} - -pub struct IntrospectSqliteDb { - tables: SqliteTables, - namespace: SqliteNamespace, - pool: T, -} - -impl SqliteConnection for IntrospectSqliteDb { - fn pool(&self) -> &sqlx::SqlitePool { - self.pool.pool() - } -} - -impl IntrospectSqliteDb { - pub fn new(pool: T, namespace: impl Into) -> Self { - Self { - tables: SqliteTables::default(), - namespace: namespace.into(), - pool, - } - } - - pub async fn initialize_introspect_sqlite_sink(&self) -> SqliteDbResult<()> { - self.migrate(Some("introspect"), INTROSPECT_SQLITE_SINK_MIGRATIONS) - .await?; - self.load_persisted_state().await?; - Ok(()) - } - - async fn load_persisted_state(&self) -> SqliteDbResult<()> { - let rows = sqlx::query( - r" - SELECT table_schema_json, alive - FROM introspect_sink_schema_state - ORDER BY updated_at ASC - ", - ) - .fetch_all(self.pool()) - .await?; - - let mut tables = self.tables.write()?; - for row in rows { - let schema_json: String = row.try_get("table_schema_json")?; - let alive: i64 = row.try_get("alive")?; - let table_schema: TableSchema = serde_json::from_str(&schema_json)?; - let (id, mut table) = - SqliteTable::new_from_table(self.namespace.prefix(), table_schema); - table.alive = alive != 0; - tables.insert(id, table); - } - - Ok(()) - } - - async fn persist_table_state(&self, table: &TableSchema, alive: bool) -> SqliteDbResult<()> { - let schema_json = serde_json::to_string(table)?; - let alive = i64::from(alive); - sqlx::query( - r" - INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) - VALUES (?1, ?2, ?3, unixepoch()) - ON CONFLICT (table_id) - DO UPDATE SET - table_schema_json = excluded.table_schema_json, - alive = excluded.alive, - updated_at = unixepoch() - ", - ) - .bind(format!("{:#x}", table.id)) - .bind(schema_json) - .bind(alive) - .execute(self.pool()) - .await?; - Ok(()) - } - - async fn update_table(&self, event: impl Into) -> SqliteDbResult<()> { - let table_schema: TableSchema = event.into(); - let id = table_schema.id; - let exists_in_memory = self.tables.read()?.contains_key(&id); - - if !exists_in_memory { - let (_, query) = self - .tables - .create_table(&self.namespace, table_schema.clone())?; - self.execute_queries(&[query]).await?; - self.persist_table_state(&table_schema, true).await?; - return Ok(()); - } - - let (old_columns, storage_name) = { - let tables = self.tables.read()?; - let old = tables.get(&id).unwrap(); - (old.columns.clone(), old.storage_name.clone()) - }; - - let (_, new_table) = - SqliteTable::new_from_table(self.namespace.prefix(), table_schema.clone()); - - let mut alter_queries = Vec::new(); - for (col_id, col_info) in &new_table.columns { - if !old_columns.contains_key(col_id) { - let col_type = sqlite_column_type(&col_info.type_def); - alter_queries.push(format!( - r#"ALTER TABLE "{storage_name}" ADD COLUMN "{}" {col_type}"#, - col_info.name - )); - } - } - - if !alter_queries.is_empty() { - self.execute_queries(&alter_queries).await?; - } - - self.tables.write()?.insert(id, new_table); - self.persist_table_state(&table_schema, true).await?; - Ok(()) - } - - pub fn load_tables_no_commit(&self, table_schemas: Vec) -> SqliteDbResult<()> { - let mut tables = self.tables.write()?; - for table in table_schemas { - let (id, sqlite_table) = SqliteTable::new_from_table(self.namespace.prefix(), table); - tables.insert(id, sqlite_table); - } - Ok(()) - } - - pub async fn process_message( - &self, - msg: &IntrospectMsg, - metadata: &MetaData, - ) -> SqliteDbResult<()> { - match msg { - IntrospectMsg::CreateTable(event) => { - let (_, query) = self.tables.create_table(&self.namespace, event.clone())?; - self.execute_queries(&[query]).await?; - self.persist_table_state(&event.clone().into(), true) - .await?; - Ok(()) - } - IntrospectMsg::UpdateTable(event) => self.update_table(event.clone()).await, - IntrospectMsg::AddColumns(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "AddColumns received — table kept alive, new columns ignored until next UpdateTable" - ); - Ok(()) - } - IntrospectMsg::DropColumns(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "DropColumns received — table kept alive, columns left in place" - ); - Ok(()) - } - IntrospectMsg::RetypeColumns(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "RetypeColumns received — table kept alive, types unchanged" - ); - Ok(()) - } - IntrospectMsg::RetypePrimary(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "RetypePrimary received — table kept alive, primary type unchanged" - ); - Ok(()) - } - IntrospectMsg::RenameTable(_) - | IntrospectMsg::DropTable(_) - | IntrospectMsg::RenameColumns(_) - | IntrospectMsg::RenamePrimary(_) - | IntrospectMsg::DeleteRecords(_) - | IntrospectMsg::DeletesFields(_) => Ok(()), - IntrospectMsg::InsertsFields(event) => self.insert_fields(event, metadata).await, - } - } - - pub async fn process_messages( - &self, - msgs: Vec<&IntrospectBody>, - ) -> SqliteDbResult>> { - let mut results = Vec::with_capacity(msgs.len()); - for body in msgs { - let (msg, metadata) = body.into(); - let result = self.process_message(msg, metadata).await; - if let Err(ref err) = result { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - event_id = msg.event_id(), - error = %err, - "Failed to process introspect message" - ); - } - results.push(result); - } - Ok(results) - } - - async fn insert_fields( - &self, - event: &InsertsFields, - _metadata: &MetaData, - ) -> SqliteDbResult<()> { - let table = self - .tables - .read()? - .get(&event.table) - .ok_or(SqliteDbError::TableNotFound(event.table))? - .clone(); - if !table.alive { - return Ok(()); - } - - let record_schema = table.get_schema(&event.columns)?; - let column_names = std::iter::once(table.primary.name.as_str()) - .chain( - event - .columns - .iter() - .map(|id| table.columns[id].name.as_str()), - ) - .collect::>(); - - let column_type_defs: Vec<&TypeDef> = event - .columns - .iter() - .map(|id| &table.columns[id].type_def) - .collect(); - - let mut bytes = Vec::new(); - let mut serializer = JsonSerializer::new(&mut bytes); - record_schema.parse_records_with_metadata( - &event.records, - &(), - &mut serializer, - &SqliteJsonSerializer, - )?; - let rows = serde_json::from_slice::>(&bytes)?; - - let mut tx = self.begin().await?; - for value in rows { - let object = value.as_object().ok_or(SqliteDbError::InvalidRecordFrame)?; - - let mut query = sqlx::query(&table.upsert_sql); - - let primary_value = object - .get(table.primary.name.as_str()) - .cloned() - .unwrap_or(Value::Null); - match primary_to_bind_value(&primary_value, &table.primary.type_def) { - SqliteBindValue::Null => { - query = query.bind(None::>); - } - SqliteBindValue::Integer(n) => { - query = query.bind(n); - } - SqliteBindValue::Text(s) => { - query = query.bind(s); - } - } - - for (column_name, type_def) in column_names.iter().skip(1).zip(column_type_defs.iter()) - { - let val = object.get(*column_name).cloned().unwrap_or(Value::Null); - match to_bind_value(&val, type_def) { - SqliteBindValue::Null => { - query = query.bind(None::); - } - SqliteBindValue::Integer(n) => { - query = query.bind(n); - } - SqliteBindValue::Text(s) => { - query = query.bind(s); - } - } - } - query.execute(&mut *tx).await?; - } - tx.commit().await?; - Ok(()) - } -} diff --git a/crates/introspect-sqlite-sink/src/table.rs b/crates/introspect-sqlite-sink/src/table.rs deleted file mode 100644 index 8bbb653e..00000000 --- a/crates/introspect-sqlite-sink/src/table.rs +++ /dev/null @@ -1,151 +0,0 @@ -use introspect_types::{ColumnDef, ColumnInfo, FeltIds, PrimaryDef}; -use itertools::Itertools; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use thiserror::Error; -use torii_introspect::schema::TableSchema; -use torii_introspect::tables::RecordSchema; - -#[derive(Debug, Error)] -pub enum SqliteTableError { - #[error("Column with id: {0} not found in table {1}")] - ColumnNotFound(Felt, String), -} - -pub type TableResult = std::result::Result; - -#[derive(Debug, Clone)] -pub struct SqliteTable { - pub name: String, - pub storage_name: String, - pub primary: PrimaryDef, - pub columns: HashMap, - pub order: Vec, - pub upsert_sql: String, - pub alive: bool, -} - -impl SqliteTable { - pub fn new( - storage_name: String, - name: String, - primary: PrimaryDef, - columns: Vec, - ) -> Self { - Self { - name, - storage_name, - primary, - order: columns.ids(), - columns: columns.into_iter().map_into().collect(), - upsert_sql: String::new(), - alive: true, - } - .with_upsert_sql() - } - - pub fn new_from_table(namespace: &str, table: impl Into) -> (Felt, Self) { - let table = table.into(); - let storage_name = if namespace.is_empty() { - table.name.clone() - } else { - format!("{namespace}__{}", table.name) - }; - ( - table.id, - Self::new(storage_name, table.name, table.primary, table.columns), - ) - } - - pub fn get_column(&self, selector: &Felt) -> TableResult<&ColumnInfo> { - self.columns - .get(selector) - .ok_or_else(|| SqliteTableError::ColumnNotFound(*selector, self.name.clone())) - } - - pub fn get_schema(&self, column_ids: &[Felt]) -> TableResult> { - let columns = column_ids - .iter() - .map(|selector| self.get_column(selector)) - .collect::, _>>()?; - Ok(RecordSchema::new(&self.primary, columns)) - } - - fn with_upsert_sql(mut self) -> Self { - self.upsert_sql = build_upsert_sql(&self); - self - } -} - -fn sqlite_column_type(type_def: &introspect_types::TypeDef) -> &'static str { - if matches!( - type_def, - introspect_types::TypeDef::Struct(_) - | introspect_types::TypeDef::Enum(_) - | introspect_types::TypeDef::Tuple(_) - | introspect_types::TypeDef::Array(_) - | introspect_types::TypeDef::FixedArray(_) - | introspect_types::TypeDef::Option(_) - | introspect_types::TypeDef::Nullable(_) - | introspect_types::TypeDef::Result(_) - ) { - "JSONB" - } else { - "" - } -} - -fn build_upsert_sql(table: &SqliteTable) -> String { - let column_names = std::iter::once(table.primary.name.as_str()) - .chain(table.order.iter().map(|id| table.columns[id].name.as_str())) - .collect::>(); - let column_type_defs = table - .order - .iter() - .map(|id| &table.columns[id].type_def) - .collect::>(); - - let placeholders = std::iter::once("?".to_string()) - .chain(column_type_defs.iter().map(|td| { - if sqlite_column_type(td) == "JSONB" { - "jsonb(?)".to_string() - } else { - "?".to_string() - } - })) - .collect::>() - .join(", "); - - let update_columns = column_names - .iter() - .skip(1) - .zip(column_type_defs.iter()) - .map(|(name, td)| { - if sqlite_column_type(td) == "JSONB" { - format!( - r#""{name}" = COALESCE(jsonb(excluded."{name}"), "{table_name}"."{name}")"#, - table_name = table.storage_name - ) - } else { - format!( - r#""{name}" = COALESCE(excluded."{name}", "{table_name}"."{name}")"#, - table_name = table.storage_name - ) - } - }) - .collect::>() - .join(", "); - - format!( - r#"INSERT INTO "{}" ({}) VALUES ({}) ON CONFLICT("{}") DO UPDATE SET {}"#, - table.storage_name, - column_names - .iter() - .map(|name| format!(r#""{name}""#)) - .collect::>() - .join(", "), - placeholders, - table.primary.name, - update_columns - ) -} diff --git a/crates/introspect/Cargo.toml b/crates/introspect/Cargo.toml index 4e00c48e..729e4483 100644 --- a/crates/introspect/Cargo.toml +++ b/crates/introspect/Cargo.toml @@ -28,6 +28,7 @@ async-trait.workspace = true bigdecimal.workspace = true torii-common.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } [build-dependencies] tonic-build = "0.12" diff --git a/crates/introspect/src/events.rs b/crates/introspect/src/events.rs index c4797559..1a3be3cc 100644 --- a/crates/introspect/src/events.rs +++ b/crates/introspect/src/events.rs @@ -72,6 +72,7 @@ pub struct CreateTable { pub attributes: Vec, pub primary: PrimaryDef, pub columns: Vec, + pub append_only: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -288,6 +289,7 @@ impl From for CreateTable { attributes: schema.attributes, primary: schema.primary, columns: schema.columns, + append_only: false, } } } @@ -351,6 +353,19 @@ impl InsertsFields { } } +impl CreateTable { + pub fn from_schema(schema: TableSchema, append_only: bool) -> Self { + Self { + id: schema.id, + name: schema.name, + attributes: schema.attributes, + primary: schema.primary, + columns: schema.columns, + append_only, + } + } +} + impl DeleteRecords { pub fn new(table: Felt, rows: Vec) -> Self { Self { table, rows } diff --git a/crates/introspect/src/postgres/global.rs b/crates/introspect/src/postgres/global.rs index 41f23449..d651ce22 100644 --- a/crates/introspect/src/postgres/global.rs +++ b/crates/introspect/src/postgres/global.rs @@ -6,7 +6,7 @@ use itertools::Itertools; use sqlx::types::Json; use sqlx::{FromRow, PgPool}; use starknet_types_core::felt::Felt; -use torii_common::sql::SqlxResult; +use torii_sql::SqlxResult; use crate::postgres::{attribute_type, felt252_type, string_type, PgAttribute, PgFelt}; diff --git a/crates/introspect/src/postgres/owned.rs b/crates/introspect/src/postgres/owned.rs index c781c520..bc0c2521 100644 --- a/crates/introspect/src/postgres/owned.rs +++ b/crates/introspect/src/postgres/owned.rs @@ -10,7 +10,7 @@ use sqlx::types::Json; use sqlx::{FromRow, PgPool, Postgres}; use starknet_types_core::felt::Felt; use std::collections::HashMap; -use torii_common::sql::SqlxResult; +use torii_sql::SqlxResult; pub const TABLE_INSERT_QUERY: &str = " INSERT INTO introspect.tables (owner, id, name, attributes, primary_def, column_ids, updated_at, created_block, updated_block, created_tx, updated_tx) diff --git a/crates/introspect/src/tables.rs b/crates/introspect/src/tables.rs index 43775d93..be081a9c 100644 --- a/crates/introspect/src/tables.rs +++ b/crates/introspect/src/tables.rs @@ -1,34 +1,40 @@ use crate::Record; use introspect_types::bytes::IntoByteSource; +use introspect_types::schema::Names; use introspect_types::serialize::CairoSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; use introspect_types::{CairoDeserializer, ColumnInfo, PrimaryDef, PrimaryTypeDef, TypeDef}; use serde::ser::{SerializeMap, SerializeSeq}; use serde::{Serialize, Serializer}; -use starknet_types_core::felt::Felt; use std::ops::Deref; -use torii::etl::envelope::MetaData; pub struct RecordSchema<'a> { - primary: &'a PrimaryDef, + primary: &'a ColumnInfo, columns: Vec<&'a ColumnInfo>, } impl<'a> RecordSchema<'a> { - pub fn new(primary: &'a PrimaryDef, columns: Vec<&'a ColumnInfo>) -> Self { + pub fn new(primary: &'a ColumnInfo, columns: Vec<&'a ColumnInfo>) -> Self { Self { primary, columns } } pub fn columns(&self) -> &[&'a ColumnInfo] { &self.columns } - + pub fn all_columns(&self) -> Vec<&'a ColumnInfo> { + std::iter::once(self.primary) + .chain(self.columns.iter().copied()) + .collect() + } pub fn column_names(&self) -> Vec<&str> { - self.columns.iter().map(|col| col.name.as_str()).collect() + self.columns.names() } - pub fn primary(&self) -> &PrimaryDef { + pub fn primary(&self) -> &ColumnInfo { self.primary } + pub fn primary_type_def(&self) -> &TypeDef { + &self.primary.type_def + } pub fn primary_name(&self) -> &str { &self.primary.name @@ -132,7 +138,7 @@ impl<'a, M: SerializeEntries> RecordWithMetadata<'a, M> { } pub struct RecordFrame<'a, C: CairoTypeSerialization, M: SerializeEntries> { - primary: &'a PrimaryDef, + primary: &'a ColumnInfo, columns: &'a [&'a ColumnInfo], id: &'a [u8; 32], values: &'a [u8], @@ -161,7 +167,7 @@ impl<'a, C: CairoTypeSerialization, M: SerializeEntries> RecordFrame<'a, C, M> { &self, map: &mut ::SerializeMap, ) -> Result<(), S::Error> { - let mut id: introspect_types::bytes::DerefBytesSource<&[u8]> = self.id.into_source(); + let mut id = self.id.into_source(); map.serialize_entry( &self.primary.name, &CairoSeFrom::new(&self.primary.type_def, &mut id, self.cairo_se), @@ -215,23 +221,3 @@ impl SerializeEntries for () { Ok(()) } } - -pub fn pg_json_felt252(value: &Felt) -> String { - format!("\\x{}", hex::encode(value.to_bytes_be())) -} - -impl SerializeEntries for MetaData { - fn entry_count(&self) -> usize { - 4 - } - fn serialize_entries( - &self, - map: &mut ::SerializeMap, - ) -> Result<(), S::Error> { - let tx_hash = pg_json_felt252(&self.transaction_hash); - map.serialize_entry("__created_block", &self.block_number)?; - map.serialize_entry("__updated_block", &self.block_number)?; - map.serialize_entry("__created_tx", &tx_hash)?; - map.serialize_entry("__updated_tx", &tx_hash) - } -} diff --git a/crates/pathfinder/Cargo.toml b/crates/pathfinder/Cargo.toml index 92bad707..35a1306b 100644 --- a/crates/pathfinder/Cargo.toml +++ b/crates/pathfinder/Cargo.toml @@ -23,5 +23,4 @@ anyhow = { workspace = true, optional = true } workspace = true [features] -default = ["etl"] etl = ["torii", "async-trait", "anyhow", "torii-starknet/starknet"] diff --git a/crates/pathfinder/src/fetcher.rs b/crates/pathfinder/src/fetcher.rs index 8f64f919..e3aebabf 100644 --- a/crates/pathfinder/src/fetcher.rs +++ b/crates/pathfinder/src/fetcher.rs @@ -104,10 +104,18 @@ impl EventFetcher for Connection { for row in self.get_block_events_rows(from_block, to_block)? { let block: BlockEvents = row.try_into()?; - let block_hash = hash_rows - .next() - .map(|r| r.hash) - .ok_or_else(|| PFError::block_hash_missing(block.block_number))?; + let block_hash = loop { + match hash_rows.next() { + Some(hash_row) => { + if hash_row.number == block.block_number { + break hash_row.hash; + } else if hash_row.number > block.block_number { + return Err(PFError::block_hash_missing(block.block_number)); + } + } + None => return Err(PFError::block_hash_missing(block.block_number)), + } + }; for transaction in block.transactions { let transaction_hash = tx_hashes .next() @@ -144,12 +152,18 @@ impl EventFetcher for Connection { for row in self.get_block_events_rows(from_block, to_block)? { let block = BlockEvents::try_from(row)?; - let ctx = ctx_iter - .next() - .ok_or_else(|| PFError::block_context_missing(block.block_number))?; - if ctx.number != block.block_number { - return Err(PFError::block_context_missing(block.block_number)); - } + let ctx = loop { + match ctx_iter.next() { + Some(ctx) => match ctx.number.cmp(&block.block_number) { + std::cmp::Ordering::Equal => break ctx, + std::cmp::Ordering::Less => contexts.push(ctx.into()), + std::cmp::Ordering::Greater => { + return Err(PFError::block_context_missing(block.block_number)) + } + }, + None => return Err(PFError::block_context_missing(block.block_number)), + } + }; for transaction in block.transactions { let transaction_hash = tx_hashes .next() diff --git a/crates/postgres/Cargo.toml b/crates/postgres/Cargo.toml deleted file mode 100644 index 9624b2f6..00000000 --- a/crates/postgres/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "torii-postgres" -version = "0.1.0" -edition = "2021" -description = "PostgreSQL utils for Torii runtime" -authors = ["Torii Runtime "] -license = "Apache-2.0" - -[dependencies] -sqlx = { workspace = true, features = [ - "postgres", - "runtime-tokio-rustls", - "macros", - "migrate", -] } -anyhow.workspace = true -async-trait.workspace = true -xxhash-rust.workspace = true -hex.workspace = true -serde.workspace = true -serde_json.workspace = true -tokio.workspace = true -torii.workspace = true -tracing.workspace = true -starknet-types-core.workspace = true -thiserror.workspace = true -futures.workspace = true -crc.workspace = true - - -torii-common.workspace = true diff --git a/crates/postgres/src/db.rs b/crates/postgres/src/db.rs deleted file mode 100644 index ef5d608d..00000000 --- a/crates/postgres/src/db.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::migration::SchemaMigrator; -use async_trait::async_trait; -use sqlx::{migrate::Migrator, Postgres}; -pub use sqlx::{PgPool, Transaction}; -use std::ops::Deref; -use torii_common::sql::{Executable, SqlxResult}; - -#[async_trait] -pub trait PostgresConnection { - fn pool(&self) -> &PgPool; - - async fn begin(&self) -> SqlxResult> { - Ok(self.pool().begin().await?) - } - async fn migrate(&self, schema: Option<&'static str>, migrator: Migrator) -> SqlxResult<()> { - let result = match schema { - Some(schema) => SchemaMigrator::new(schema, migrator).run(self.pool()).await, - None => migrator.run(self.pool()).await, - }; - Ok(result?) - } - async fn execute_queries(&self, queries: impl Executable + Send) -> SqlxResult<()> { - let mut transaction = self.begin().await?; - queries.execute(&mut transaction).await?; - transaction.commit().await - } -} - -#[allow(clippy::explicit_auto_deref)] -#[async_trait] -impl + Send + Sync + 'static> PostgresConnection for T { - fn pool(&self) -> &PgPool { - &**self - } -} diff --git a/crates/postgres/src/lib.rs b/crates/postgres/src/lib.rs deleted file mode 100644 index a2da4928..00000000 --- a/crates/postgres/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod db; -pub mod metadata; -pub mod migration; -pub use db::PostgresConnection; diff --git a/crates/postgres/src/metadata.rs b/crates/postgres/src/metadata.rs deleted file mode 100644 index 3368b76c..00000000 --- a/crates/postgres/src/metadata.rs +++ /dev/null @@ -1,72 +0,0 @@ -use sqlx::{postgres::PgArguments, query::Query, Postgres}; -use std::fmt::Write; -use torii::etl::EventContext; - -pub const INSERTS: &str = "__created_block, __updated_block, __created_tx, __updated_tx"; -pub const CONFLICTS: &str = "__updated_at = NOW(), __updated_block = EXCLUDED.__updated_block, __updated_tx = EXCLUDED.__updated_tx"; - -pub trait PgMetadata { - fn insert_string( - &self, - schema: &str, - table: &str, - primary_name: &str, - primary_value: String, - ) -> String; - fn static_insert_query(schema: &str, table: &str, primary_name: &str) -> String { - format!( - r#" - INSERT INTO "{schema}"."{table}" ("{primary_name}", {INSERTS}) - VALUES ($1, $2, $2, $3, $3) - ON CONFLICT ({primary_name}) DO UPDATE SET {CONFLICTS}"# - ) - } - fn insert_values(&self, string: &mut String); - fn bind_query<'a>( - &self, - query: &'a str, - primary_value: String, - ) -> Query<'a, Postgres, PgArguments>; -} - -impl PgMetadata for EventContext { - fn insert_string( - &self, - schema: &str, - table: &str, - primary_name: &str, - primary_value: String, - ) -> String { - let mut string = format!( - r#"INSERT INTO "{schema}"."{table}" ("{primary_name}", {INSERTS}) VALUES ({primary_value}, "# - ); - self.insert_values(&mut string); - write!( - string, - ") ON CONFLICT ({primary_name}) DO UPDATE SET {CONFLICTS}" - ) - .unwrap(); - string - } - fn bind_query<'a>( - &self, - query: &'a str, - primary_value: String, - ) -> Query<'a, Postgres, PgArguments> { - let block_number = self.block.number.to_string(); - let tx_hash = self.transaction.hash.to_bytes_be(); - sqlx::query::(query) - .bind(primary_value.clone()) - .bind(block_number) - .bind(tx_hash) - } - fn insert_values(&self, string: &mut String) { - let block_number = self.block.number; - let tx_hash = hex::encode(self.transaction.hash.to_bytes_be()); - write!( - string, - "{block_number}, {block_number}, '\\x{tx_hash}', '\\x{tx_hash}'" - ) - .unwrap(); - } -} diff --git a/crates/postgres/src/migration.rs b/crates/postgres/src/migration.rs deleted file mode 100644 index 9be9fce7..00000000 --- a/crates/postgres/src/migration.rs +++ /dev/null @@ -1,443 +0,0 @@ -use futures::future::BoxFuture; -use sqlx::{ - migrate::{AppliedMigration, Migrate, MigrateError, Migration, MigrationSource, Migrator}, - query, query_scalar, Acquire, PgConnection, Postgres, -}; -use sqlx::{query_as, Executor}; -use std::{ - collections::{HashMap, HashSet}, - ops::{Deref, DerefMut}, - slice, - time::{Duration, Instant}, -}; - -pub struct SchemaMigrator { - pub migrator: Migrator, - pub schema: &'static str, -} - -pub struct PgAcquiredSchema<'a, A> -where - A: Acquire<'a>, -{ - pub connection: >::Connection, - pub schema: &'static str, -} - -impl<'a, A> Deref for PgAcquiredSchema<'a, A> -where - A: Acquire<'a, Database = Postgres>, -{ - type Target = PgConnection; - fn deref(&self) -> &Self::Target { - &self.connection - } -} - -impl<'a, A> DerefMut for PgAcquiredSchema<'a, A> -where - A: Acquire<'a, Database = Postgres>, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.connection - } -} - -impl<'a, A> Migrate for PgAcquiredSchema<'a, A> -where - A: Acquire<'a, Database = Postgres>, - A: Executor<'a, Database = Postgres>, -{ - fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - // language=SQL - self.connection - .execute( - format!( - r#" -CREATE SCHEMA IF NOT EXISTS {schema}; -CREATE TABLE IF NOT EXISTS {schema}._sqlx_migrations ( - version BIGINT PRIMARY KEY, - description TEXT NOT NULL, - installed_on TIMESTAMPTZ NOT NULL DEFAULT now(), - success BOOLEAN NOT NULL, - checksum BYTEA NOT NULL, - execution_time BIGINT NOT NULL -); - "#, - schema = self.schema - ) - .as_str(), - ) - .await?; - - Ok(()) - }) - } - - fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - // language=SQL - - let row: Option<(i64,)> = query_as( - format!("SELECT version FROM {schema}._sqlx_migrations WHERE success = false ORDER BY version LIMIT 1", schema = self.schema).as_str() - ) - .fetch_optional(&mut *self.connection) - .await?; - - Ok(row.map(|r: (i64,)| r.0)) - }) - } - - fn list_applied_migrations( - &mut self, - ) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - // language=SQL - let rows: Vec<(i64, Vec)> = query_as( - format!( - "SELECT version, checksum FROM {schema}._sqlx_migrations ORDER BY version", - schema = self.schema - ) - .as_str(), - ) - .fetch_all(&mut *self.connection) - .await?; - - let migrations = rows - .into_iter() - .map(|(version, checksum)| AppliedMigration { - version, - checksum: checksum.into(), - }) - .collect(); - - Ok(migrations) - }) - } - - fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - let database_name = current_database(&mut self.connection).await?; - let lock_id = generate_lock_id(&database_name); - - // create an application lock over the database - // this function will not return until the lock is acquired - - // https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS - // https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS-TABLE - - // language=SQL - let _ = query("SELECT pg_advisory_lock($1)") - .bind(lock_id) - .execute(&mut *self.connection) - .await?; - - Ok(()) - }) - } - - fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - let database_name = current_database(self).await?; - let lock_id = generate_lock_id(&database_name); - - // language=SQL - let _ = query("SELECT pg_advisory_unlock($1)") - .bind(lock_id) - .execute(&mut *self.connection) - .await?; - - Ok(()) - }) - } - - fn apply<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let start = Instant::now(); - let schema = self.schema; - // execute migration queries - if migration.no_tx { - execute_migration(self, schema, migration).await?; - } else { - // Use a single transaction for the actual migration script and the essential bookeeping so we never - // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. - // The `execution_time` however can only be measured for the whole transaction. This value _only_ exists for - // data lineage and debugging reasons, so it is not super important if it is lost. So we initialize it to -1 - // and update it once the actual transaction completed. - let mut tx = self.begin().await?; - execute_migration(&mut tx, schema, migration).await?; - tx.commit().await?; - } - - // Update `elapsed_time`. - // NOTE: The process may disconnect/die at this point, so the elapsed time value might be lost. We accept - // this small risk since this value is not super important. - let elapsed = start.elapsed(); - - // language=SQL - #[allow(clippy::cast_possible_truncation)] - let _ = query(&format!( - r#" - UPDATE {schema}._sqlx_migrations - SET execution_time = $1 - WHERE version = $2 - "#, - schema = self.schema - )) - .bind(elapsed.as_nanos() as i64) - .bind(migration.version) - .execute(&mut *self.connection) - .await?; - - Ok(elapsed) - }) - } - - fn revert<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let start = Instant::now(); - let schema = self.schema; - // execute migration queries - if migration.no_tx { - revert_migration(&mut self.connection, schema, migration).await?; - } else { - // Use a single transaction for the actual migration script and the essential bookeeping so we never - // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. - let mut tx = self.begin().await?; - revert_migration(&mut tx, schema, migration).await?; - tx.commit().await?; - } - - let elapsed = start.elapsed(); - - Ok(elapsed) - }) - } -} - -async fn current_database(conn: &mut PgConnection) -> Result { - // language=SQL - Ok(query_scalar("SELECT current_database()") - .fetch_one(conn) - .await?) -} - -fn generate_lock_id(database_name: &str) -> i64 { - const CRC_IEEE: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); - // 0x3d32ad9e chosen by fair dice roll - 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) -} - -async fn execute_migration( - conn: &mut PgConnection, - schema: &'static str, - migration: &Migration, -) -> Result<(), MigrateError> { - let _ = conn - .execute(&*migration.sql) - .await - .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; - - // language=SQL - let _ = query( - format!(r#" - INSERT INTO {schema}._sqlx_migrations ( version, description, success, checksum, execution_time ) - VALUES ( $1, $2, TRUE, $3, -1 ) - "#).as_str() - ) - .bind(migration.version) - .bind(&*migration.description) - .bind(&*migration.checksum) - .execute(conn) - .await?; - - Ok(()) -} - -async fn revert_migration( - conn: &mut PgConnection, - schema: &'static str, - migration: &Migration, -) -> Result<(), MigrateError> { - let _ = conn - .execute(&*migration.sql) - .await - .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; - - // language=SQL - let _ = query(format!(r#"DELETE FROM {schema}._sqlx_migrations WHERE version = $1"#).as_str()) - .bind(migration.version) - .execute(conn) - .await?; - - Ok(()) -} - -impl SchemaMigrator { - pub const fn new(schema: &'static str, migrator: Migrator) -> Self { - Self { migrator, schema } - } - pub async fn new_from_source<'s, S>( - schema: &'static str, - source: S, - ) -> Result - where - S: MigrationSource<'s>, - { - Migrator::new(source) - .await - .map(|migrator| Self { migrator, schema }) - } - - /// Specify whether applied migrations that are missing from the resolved migrations should be ignored. - pub fn set_ignore_missing(&mut self, ignore_missing: bool) -> &Self { - self.migrator.ignore_missing = ignore_missing; - self - } - - /// Specify whether or not to lock the database during migration. Defaults to `true`. - /// - /// ### Warning - /// Disabling locking can lead to errors or data loss if multiple clients attempt to apply migrations simultaneously - /// without some sort of mutual exclusion. - /// - /// This should only be used if the database does not support locking, e.g. CockroachDB which talks the Postgres - /// protocol but does not support advisory locks used by SQLx's migrations support for Postgres. - pub fn set_locking(&mut self, locking: bool) -> &Self { - self.migrator.locking = locking; - self - } - - /// Get an iterator over all known migrations. - pub fn iter(&self) -> slice::Iter<'_, Migration> { - self.migrator.migrations.iter() - } - - /// Check if a migration version exists. - pub fn version_exists(&self, version: i64) -> bool { - self.iter().any(|m| m.version == version) - } - - /// Run any pending migrations against the database; and, validate previously applied migrations - /// against the current migration source to detect accidental changes in previously-applied migrations. - /// - /// # Examples - /// - /// ```rust,no_run - /// # use sqlx::migrate::MigrateError; - /// # fn main() -> Result<(), MigrateError> { - /// # sqlx::__rt::test_block_on(async move { - /// use sqlx::migrate::Migrator; - /// use sqlx::sqlite::SqlitePoolOptions; - /// - /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; - /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; - /// m.run(&pool).await - /// # }) - /// # } - /// ``` - pub async fn run<'a, A>(&self, migrator: A) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Postgres>, - PgAcquiredSchema<'a, A>: Migrate, - { - let mut conn = PgAcquiredSchema { - connection: migrator.acquire().await?, - schema: self.schema, - }; - self.migrator.run_direct(&mut conn).await - } - - /// Run down migrations against the database until a specific version. - /// - /// # Examples - /// - /// ```rust,no_run - /// # use sqlx::migrate::MigrateError; - /// # fn main() -> Result<(), MigrateError> { - /// # sqlx::__rt::test_block_on(async move { - /// use sqlx::migrate::Migrator; - /// use sqlx::sqlite::SqlitePoolOptions; - /// - /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; - /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; - /// m.undo(&pool, 4).await - /// # }) - /// # } - /// ``` - pub async fn undo<'a, A>(&self, migrator: A, target: i64) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Postgres>, - PgAcquiredSchema<'a, A>: Migrate, - { - let mut conn = PgAcquiredSchema { - connection: migrator.acquire().await?, - schema: self.schema, - }; - // lock the database for exclusive access by the migrator - if self.migrator.locking { - conn.lock().await?; - } - - // creates [_migrations] table only if needed - // eventually this will likely migrate previous versions of the table - conn.ensure_migrations_table().await?; - - let version = conn.dirty_version().await?; - if let Some(version) = version { - return Err(MigrateError::Dirty(version)); - } - - let applied_migrations = conn.list_applied_migrations().await?; - validate_applied_migrations(&applied_migrations, &self.migrator)?; - - let applied_migrations: HashMap<_, _> = applied_migrations - .into_iter() - .map(|m| (m.version, m)) - .collect(); - - for migration in self - .iter() - .rev() - .filter(|m| m.migration_type.is_down_migration()) - .filter(|m| applied_migrations.contains_key(&m.version)) - .filter(|m| m.version > target) - { - conn.revert(migration).await?; - } - - // unlock the migrator to allow other migrators to run - // but do nothing as we already migrated - if self.migrator.locking { - conn.unlock().await?; - } - - Ok(()) - } -} - -fn validate_applied_migrations( - applied_migrations: &[AppliedMigration], - migrator: &Migrator, -) -> Result<(), MigrateError> { - if migrator.ignore_missing { - return Ok(()); - } - - let migrations: HashSet<_> = migrator.iter().map(|m| m.version).collect(); - - for applied_migration in applied_migrations { - if !migrations.contains(&applied_migration.version) { - return Err(MigrateError::VersionMissing(applied_migration.version)); - } - } - - Ok(()) -} diff --git a/crates/sql/Cargo.toml b/crates/sql/Cargo.toml new file mode 100644 index 00000000..6197320b --- /dev/null +++ b/crates/sql/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "torii-sql" +version = "0.1.0" +edition = "2021" +description = "Common utilities for Torii token indexers" + +[dependencies] +async-trait = "0.1" +sqlx = { workspace = true, features = ["runtime-tokio-rustls"] } +itertools.workspace = true +futures.workspace = true +crc.workspace = true +starknet-types-core.workspace = true +log.workspace = true + +[features] +postgres = ["sqlx/postgres"] +sqlite = ["sqlx/sqlite"] +mysql = ["sqlx/mysql"] + +[lints] +workspace = true diff --git a/crates/sql/src/connection.rs b/crates/sql/src/connection.rs new file mode 100644 index 00000000..9e544687 --- /dev/null +++ b/crates/sql/src/connection.rs @@ -0,0 +1,76 @@ +use crate::SqlxError; +use std::fmt::Display; +use std::str::FromStr; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DbBackend { + Postgres, + Sqlite, +} + +pub struct DbOption { + postgres: T, + sqlite: T, +} + +pub const POSTGRES_URL_SCHEMES: [&str; 2] = ["postgres", "postgresql"]; +pub const SQLITE_URL_SCHEMES: [&str; 1] = ["sqlite"]; + +impl FromStr for DbBackend { + type Err = SqlxError; + + fn from_str(s: &str) -> Result { + if s.starts_with("postgres") || s.starts_with("postgresql") { + Ok(DbBackend::Postgres) + } else if s.starts_with("sqlite") || s == ":memory:" || s == "memory" { + Ok(DbBackend::Sqlite) + } else { + Err(SqlxError::Configuration( + format!("Unsupported database url: {s}").into(), + )) + } + } +} + +impl Display for DbBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_str().fmt(f) + } +} + +impl DbBackend { + pub fn as_str(&self) -> &'static str { + match self { + DbBackend::Postgres => "postgres", + DbBackend::Sqlite => "sqlite", + } + } +} + +impl TryFrom<&str> for DbBackend { + type Error = SqlxError; + + fn try_from(value: &str) -> Result { + DbBackend::from_str(value) + } +} + +impl TryFrom for DbBackend { + type Error = SqlxError; + + fn try_from(value: String) -> Result { + DbBackend::try_from(value.as_str()) + } +} + +impl DbOption { + pub fn new(postgres: T, sqlite: T) -> Self { + Self { postgres, sqlite } + } + + pub fn value(self, db_type: &DbBackend) -> T { + match db_type { + DbBackend::Postgres => self.postgres, + DbBackend::Sqlite => self.sqlite, + } + } +} diff --git a/crates/sql/src/lib.rs b/crates/sql/src/lib.rs new file mode 100644 index 00000000..13e98d24 --- /dev/null +++ b/crates/sql/src/lib.rs @@ -0,0 +1,31 @@ +pub mod connection; +pub mod migrate; +pub mod pool; +pub mod query; +pub mod types; + +pub use connection::DbBackend; +pub use migrate::{AcquiredSchema, SchemaMigrator}; +pub use pool::{DbPoolOptions, PoolConfig, PoolExt}; +pub use query::{Executable, FlexQuery, Queries}; + +pub use sqlx::Error as SqlxError; +pub type SqlxResult = std::result::Result; + +#[cfg(feature = "postgres")] +pub mod postgres; +#[cfg(feature = "postgres")] +pub use postgres::{PgArguments, PgDbConnection, PgPool, PgQuery, Postgres}; + +#[cfg(feature = "sqlite")] +pub mod sqlite; +#[cfg(feature = "sqlite")] +pub use sqlite::{Sqlite, SqliteArguments, SqliteDbConnection, SqlitePool, SqliteQuery}; + +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub mod runtime; + +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub use runtime::{DbConnectionOptions, DbPool}; diff --git a/crates/sql/src/migrate.rs b/crates/sql/src/migrate.rs new file mode 100644 index 00000000..42093cdc --- /dev/null +++ b/crates/sql/src/migrate.rs @@ -0,0 +1,218 @@ +use sqlx::{ + migrate::{AppliedMigration, Migrate, MigrateError, Migration, MigrationSource, Migrator}, + Acquire, Connection, Database, +}; +use std::{ + collections::HashMap, + ops::{Deref, DerefMut}, +}; +use std::{collections::HashSet, slice}; + +pub struct SchemaMigrator { + pub migrator: Migrator, + pub schema: &'static str, +} + +pub struct AcquiredSchema +where + DB: Database, + C: Connection, +{ + pub connection: C, + pub schema: &'static str, +} + +impl Deref for AcquiredSchema +where + DB: Database, + C: Connection, +{ + type Target = C; + fn deref(&self) -> &Self::Target { + &self.connection + } +} + +impl DerefMut for AcquiredSchema +where + DB: Database, + C: Connection, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.connection + } +} + +impl AcquiredSchema +where + DB: Database, + C: Connection, +{ + pub fn new(connection: C, schema: &'static str) -> Self { + Self { connection, schema } + } +} + +impl SchemaMigrator { + pub const fn new(schema: &'static str, migrator: Migrator) -> Self { + Self { migrator, schema } + } + pub async fn new_from_source<'s, S>( + schema: &'static str, + source: S, + ) -> Result + where + S: MigrationSource<'s>, + { + Migrator::new(source) + .await + .map(|migrator| Self { migrator, schema }) + } + + /// Specify whether applied migrations that are missing from the resolved migrations should be ignored. + pub fn set_ignore_missing(&mut self, ignore_missing: bool) -> &Self { + self.migrator.ignore_missing = ignore_missing; + self + } + + /// Specify whether or not to lock the database during migration. Defaults to `true`. + /// + /// ### Warning + /// Disabling locking can lead to errors or data loss if multiple clients attempt to apply migrations simultaneously + /// without some sort of mutual exclusion. + /// + /// This should only be used if the database does not support locking, e.g. CockroachDB which talks the Postgres + /// protocol but does not support advisory locks used by SQLx's migrations support for Postgres. + pub fn set_locking(&mut self, locking: bool) -> &Self { + self.migrator.locking = locking; + self + } + + /// Get an iterator over all known migrations. + pub fn iter(&self) -> slice::Iter<'_, Migration> { + self.migrator.migrations.iter() + } + + /// Check if a migration version exists. + pub fn version_exists(&self, version: i64) -> bool { + self.iter().any(|m| m.version == version) + } + + /// Run any pending migrations against the database; and, validate previously applied migrations + /// against the current migration source to detect accidental changes in previously-applied migrations. + /// + /// # Examples + /// + /// ```rust,no_run + /// # use sqlx::migrate::MigrateError; + /// # fn main() -> Result<(), MigrateError> { + /// # sqlx::__rt::test_block_on(async move { + /// use sqlx::migrate::Migrator; + /// use sqlx::sqlite::SqlitePoolOptions; + /// + /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; + /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; + /// m.run(&pool).await + /// # }) + /// # } + /// ``` + pub async fn run<'a, A>(&self, migrator: A) -> Result<(), MigrateError> + where + A: Acquire<'a>, + >::Connection: Connection, + AcquiredSchema: Migrate, + { + let mut conn = AcquiredSchema { + connection: migrator.acquire().await?, + schema: self.schema, + }; + self.migrator.run_direct(&mut conn).await + } + + /// Run down migrations against the database until a specific version. + /// + /// # Examples + /// + /// ```rust,no_run + /// # use sqlx::migrate::MigrateError; + /// # fn main() -> Result<(), MigrateError> { + /// # sqlx::__rt::test_block_on(async move { + /// use sqlx::migrate::Migrator; + /// use sqlx::sqlite::SqlitePoolOptions; + /// + /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; + /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; + /// m.undo(&pool, 4).await + /// # }) + /// # } + /// ``` + pub async fn undo<'a, A>(&self, migrator: A, target: i64) -> Result<(), MigrateError> + where + A: Acquire<'a>, + >::Connection: Connection, + AcquiredSchema: Migrate, + { + let mut conn = AcquiredSchema:: { + connection: migrator.acquire().await?, + schema: self.schema, + }; + // lock the database for exclusive access by the migrator + if self.migrator.locking { + conn.lock().await?; + } + + // creates [_migrations] table only if needed + // eventually this will likely migrate previous versions of the table + conn.ensure_migrations_table().await?; + + let version = conn.dirty_version().await?; + if let Some(version) = version { + return Err(MigrateError::Dirty(version)); + } + + let applied_migrations = conn.list_applied_migrations().await?; + validate_applied_migrations(&applied_migrations, &self.migrator)?; + + let applied_migrations: HashMap<_, _> = applied_migrations + .into_iter() + .map(|m| (m.version, m)) + .collect(); + + for migration in self + .iter() + .rev() + .filter(|m| m.migration_type.is_down_migration()) + .filter(|m| applied_migrations.contains_key(&m.version)) + .filter(|m| m.version > target) + { + conn.revert(migration).await?; + } + + // unlock the migrator to allow other migrators to run + // but do nothing as we already migrated + if self.migrator.locking { + conn.unlock().await?; + } + + Ok(()) + } +} + +fn validate_applied_migrations( + applied_migrations: &[AppliedMigration], + migrator: &Migrator, +) -> Result<(), MigrateError> { + if migrator.ignore_missing { + return Ok(()); + } + + let migrations: HashSet<_> = migrator.iter().map(|m| m.version).collect(); + + for applied_migration in applied_migrations { + if !migrations.contains(&applied_migration.version) { + return Err(MigrateError::VersionMissing(applied_migration.version)); + } + } + + Ok(()) +} diff --git a/crates/sql/src/pool.rs b/crates/sql/src/pool.rs new file mode 100644 index 00000000..73387198 --- /dev/null +++ b/crates/sql/src/pool.rs @@ -0,0 +1,302 @@ +use std::time::Duration; + +use crate::query::Executable; +use crate::{AcquiredSchema, SqlxResult}; +use async_trait::async_trait; +use log::LevelFilter; +use sqlx::migrate::{Migrate, Migrator}; +use sqlx::pool::PoolOptions; +use sqlx::{Connection, Database, Pool, Transaction}; + +#[async_trait] +pub trait PoolExt { + fn pool(&self) -> &Pool; + async fn begin(&self) -> SqlxResult> { + Ok(self.pool().begin().await?) + } + async fn migrate(&self, schema: Option<&'static str>, migrator: Migrator) -> SqlxResult<()> + where + ::Connection: Migrate, + AcquiredSchema::Connection>: Migrate, + { + let result = match schema { + Some(schema) => { + let mut conn: AcquiredSchema::Connection> = AcquiredSchema { + connection: self.pool().acquire().await?.detach(), + schema, + }; + migrator.run_direct(&mut conn).await + } + None => migrator.run(self.pool()).await, + }; + Ok(result?) + } + async fn execute_queries + Send>(&self, queries: E) -> SqlxResult<()> { + let mut transaction: Transaction<'_, DB> = self.begin().await?; + queries.execute(&mut transaction).await?; + transaction.commit().await + } +} + +impl PoolExt for Pool { + fn pool(&self) -> &Pool { + self + } +} + +const DEFAULT_TEST_BEFORE_ACQUIRE: bool = true; +const DEFAULT_MAX_CONNECTIONS: u32 = 10; +const DEFAULT_MIN_CONNECTIONS: u32 = 0; +const DEFAULT_ACQUIRE_TIME_LEVEL: LevelFilter = LevelFilter::Off; +const DEFAULT_ACQUIRE_SLOW_LEVEL: LevelFilter = LevelFilter::Warn; +const DEFAULT_ACQUIRE_SLOW_THRESHOLD: Duration = Duration::from_secs(2); +const DEFAULT_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_IDLE_TIMEOUT: Option = Some(Duration::from_secs(10 * 60)); +const DEFAULT_MAX_LIFETIME: Option = Some(Duration::from_secs(30 * 60)); +const DEFAULT_FAIR: bool = true; + +#[derive(Debug, Clone, Copy)] +pub struct DbPoolOptions { + pub test_before_acquire: bool, + pub max_connections: u32, + pub acquire_time_level: LevelFilter, + pub acquire_slow_level: LevelFilter, + pub acquire_slow_threshold: Duration, + pub acquire_timeout: Duration, + pub min_connections: u32, + pub max_lifetime: Option, + pub idle_timeout: Option, + pub fair: bool, +} + +#[derive(Debug, Clone)] +pub struct PoolConfig { + pub url: String, + pub options: DbPoolOptions, +} + +impl PoolConfig { + pub fn new(url: String) -> Self { + Self { + url, + options: DbPoolOptions::new(), + } + } + pub async fn connect(&self) -> SqlxResult> { + self.options.connect(&self.url).await + } + pub fn options(&self) -> PoolOptions { + self.options.options() + } + pub fn max_connections(mut self, max: u32) -> Self { + self.options.max_connections = max; + self + } + + pub fn get_max_connections(&self) -> u32 { + self.options.max_connections + } + + pub fn min_connections(mut self, min: u32) -> Self { + self.options.min_connections = min; + self + } + + pub fn get_min_connections(&self) -> u32 { + self.options.min_connections + } + + pub fn acquire_time_level(mut self, level: LevelFilter) -> Self { + self.options.acquire_time_level = level; + self + } + + pub fn acquire_slow_level(mut self, level: LevelFilter) -> Self { + self.options.acquire_slow_level = level; + self + } + + pub fn acquire_slow_threshold(mut self, threshold: Duration) -> Self { + self.options.acquire_slow_threshold = threshold; + self + } + + pub fn get_acquire_slow_threshold(&self) -> Duration { + self.options.acquire_slow_threshold + } + + pub fn acquire_timeout(mut self, timeout: Duration) -> Self { + self.options.acquire_timeout = timeout; + self + } + + pub fn get_acquire_timeout(&self) -> Duration { + self.options.acquire_timeout + } + + pub fn max_lifetime(mut self, lifetime: impl Into>) -> Self { + self.options.max_lifetime = lifetime.into(); + self + } + + pub fn get_max_lifetime(&self) -> Option { + self.options.max_lifetime + } + + pub fn idle_timeout(mut self, timeout: impl Into>) -> Self { + self.options.idle_timeout = timeout.into(); + self + } + + pub fn get_idle_timeout(&self) -> Option { + self.options.idle_timeout + } + + pub fn test_before_acquire(mut self, test: bool) -> Self { + self.options.test_before_acquire = test; + self + } + + pub fn get_test_before_acquire(&self) -> bool { + self.options.test_before_acquire + } + + pub fn fair(mut self, fair: bool) -> Self { + self.options.fair = fair; + self + } + + pub fn get_fair(&self) -> bool { + self.options.fair + } +} + +impl Default for DbPoolOptions { + fn default() -> Self { + DbPoolOptions::new() + } +} + +impl DbPoolOptions { + pub fn new() -> Self { + Self { + test_before_acquire: DEFAULT_TEST_BEFORE_ACQUIRE, + max_connections: DEFAULT_MAX_CONNECTIONS, + acquire_time_level: DEFAULT_ACQUIRE_TIME_LEVEL, + acquire_slow_level: DEFAULT_ACQUIRE_SLOW_LEVEL, + acquire_slow_threshold: DEFAULT_ACQUIRE_SLOW_THRESHOLD, + acquire_timeout: DEFAULT_ACQUIRE_TIMEOUT, + min_connections: DEFAULT_MIN_CONNECTIONS, + max_lifetime: DEFAULT_MAX_LIFETIME, + idle_timeout: DEFAULT_IDLE_TIMEOUT, + fair: DEFAULT_FAIR, + } + } + + pub async fn connect(&self, url: &str) -> SqlxResult> { + self.options::().connect(url).await + } + + pub async fn connect_with( + &self, + options: ::Options, + ) -> SqlxResult> { + self.options::().connect_with(options).await + } + + pub fn options(&self) -> PoolOptions { + PoolOptions::::new() + .test_before_acquire(self.test_before_acquire) + .max_connections(self.max_connections) + .acquire_time_level(self.acquire_time_level) + .acquire_slow_level(self.acquire_slow_level) + .acquire_slow_threshold(self.acquire_slow_threshold) + .acquire_timeout(self.acquire_timeout) + .min_connections(self.min_connections) + .max_lifetime(self.max_lifetime) + .idle_timeout(self.idle_timeout) + .__fair(self.fair) + } + + pub fn max_connections(mut self, max: u32) -> Self { + self.max_connections = max; + self + } + + pub fn get_max_connections(&self) -> u32 { + self.max_connections + } + + pub fn min_connections(mut self, min: u32) -> Self { + self.min_connections = min; + self + } + + pub fn get_min_connections(&self) -> u32 { + self.min_connections + } + + pub fn acquire_time_level(mut self, level: LevelFilter) -> Self { + self.acquire_time_level = level; + self + } + + pub fn acquire_slow_level(mut self, level: LevelFilter) -> Self { + self.acquire_slow_level = level; + self + } + + pub fn acquire_slow_threshold(mut self, threshold: Duration) -> Self { + self.acquire_slow_threshold = threshold; + self + } + + pub fn get_acquire_slow_threshold(&self) -> Duration { + self.acquire_slow_threshold + } + + pub fn acquire_timeout(mut self, timeout: Duration) -> Self { + self.acquire_timeout = timeout; + self + } + + pub fn get_acquire_timeout(&self) -> Duration { + self.acquire_timeout + } + + pub fn max_lifetime(mut self, lifetime: impl Into>) -> Self { + self.max_lifetime = lifetime.into(); + self + } + + pub fn get_max_lifetime(&self) -> Option { + self.max_lifetime + } + + pub fn idle_timeout(mut self, timeout: impl Into>) -> Self { + self.idle_timeout = timeout.into(); + self + } + + pub fn get_idle_timeout(&self) -> Option { + self.idle_timeout + } + + pub fn test_before_acquire(mut self, test: bool) -> Self { + self.test_before_acquire = test; + self + } + + pub fn get_test_before_acquire(&self) -> bool { + self.test_before_acquire + } + + pub fn fair(mut self, fair: bool) -> Self { + self.fair = fair; + self + } + + pub fn get_fair(&self) -> bool { + self.fair + } +} diff --git a/crates/sql/src/postgres/migrate.rs b/crates/sql/src/postgres/migrate.rs new file mode 100644 index 00000000..02982cc6 --- /dev/null +++ b/crates/sql/src/postgres/migrate.rs @@ -0,0 +1,227 @@ +use crate::AcquiredSchema; +use futures::future::BoxFuture; +use sqlx::migrate::{AppliedMigration, Migrate, MigrateError, Migration}; +use sqlx::{query, query_as, query_scalar, Connection, Executor, PgConnection, Postgres}; +use std::time::{Duration, Instant}; + +impl Migrate for AcquiredSchema { + fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + // language=SQL + self.connection + .execute( + format!( + "CREATE SCHEMA IF NOT EXISTS {schema}; + CREATE TABLE IF NOT EXISTS {schema}._sqlx_migrations ( + version BIGINT PRIMARY KEY, + description TEXT NOT NULL, + installed_on TIMESTAMPTZ NOT NULL DEFAULT now(), + success BOOLEAN NOT NULL, + checksum BYTEA NOT NULL, + execution_time BIGINT NOT NULL + );", + schema = self.schema + ) + .as_str(), + ) + .await?; + + Ok(()) + }) + } + + fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + // language=SQL + + let row: Option<(i64,)> = query_as( + format!("SELECT version FROM {schema}._sqlx_migrations WHERE success = false ORDER BY version LIMIT 1", schema = self.schema).as_str() + ) + .fetch_optional(&mut self.connection) + .await?; + + Ok(row.map(|r: (i64,)| r.0)) + }) + } + + fn list_applied_migrations( + &mut self, + ) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + // language=SQL + let rows: Vec<(i64, Vec)> = query_as( + format!( + "SELECT version, checksum FROM {schema}._sqlx_migrations ORDER BY version", + schema = self.schema + ) + .as_str(), + ) + .fetch_all(&mut self.connection) + .await?; + + let migrations = rows + .into_iter() + .map(|(version, checksum)| AppliedMigration { + version, + checksum: checksum.into(), + }) + .collect(); + + Ok(migrations) + }) + } + + fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + let database_name = current_database(&mut self.connection).await?; + let lock_id = generate_lock_id(&database_name); + + // create an application lock over the database + // this function will not return until the lock is acquired + + // https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS + // https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS-TABLE + + // language=SQL + let _ = query("SELECT pg_advisory_lock($1)") + .bind(lock_id) + .execute(&mut self.connection) + .await?; + + Ok(()) + }) + } + + fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + let database_name = current_database(self).await?; + let lock_id = generate_lock_id(&database_name); + + // language=SQL + let _ = query("SELECT pg_advisory_unlock($1)") + .bind(lock_id) + .execute(&mut self.connection) + .await?; + + Ok(()) + }) + } + + fn apply<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let start = Instant::now(); + let schema = self.schema; + // execute migration queries + if migration.no_tx { + execute_migration(self, schema, migration).await?; + } else { + // Use a single transaction for the actual migration script and the essential bookeeping so we never + // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. + // The `execution_time` however can only be measured for the whole transaction. This value _only_ exists for + // data lineage and debugging reasons, so it is not super important if it is lost. So we initialize it to -1 + // and update it once the actual transaction completed. + let mut tx = Connection::begin(&mut self.connection).await?; + execute_migration(&mut tx, schema, migration).await?; + tx.commit().await?; + } + + // Update `elapsed_time`. + // NOTE: The process may disconnect/die at this point, so the elapsed time value might be lost. We accept + // this small risk since this value is not super important. + let elapsed = start.elapsed(); + + // language=SQL + #[allow(clippy::cast_possible_truncation)] + let _ = query(&format!( + "UPDATE {schema}._sqlx_migrations SET execution_time = $1 WHERE version = $2", + schema = self.schema + )) + .bind(elapsed.as_nanos() as i64) + .bind(migration.version) + .execute(&mut self.connection) + .await?; + + Ok(elapsed) + }) + } + + fn revert<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let start = Instant::now(); + let schema = self.schema; + // execute migration queries + if migration.no_tx { + revert_migration(&mut self.connection, schema, migration).await?; + } else { + // Use a single transaction for the actual migration script and the essential bookeeping so we never + // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. + let mut tx = Connection::begin(&mut self.connection).await?; + revert_migration(&mut tx, schema, migration).await?; + tx.commit().await?; + } + + let elapsed = start.elapsed(); + + Ok(elapsed) + }) + } +} + +async fn current_database(conn: &mut PgConnection) -> Result { + // language=SQL + Ok(query_scalar("SELECT current_database()") + .fetch_one(conn) + .await?) +} + +fn generate_lock_id(database_name: &str) -> i64 { + const CRC_IEEE: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); + // 0x3d32ad9e chosen by fair dice roll + 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) +} + +async fn execute_migration( + conn: &mut PgConnection, + schema: &'static str, + migration: &Migration, +) -> Result<(), MigrateError> { + let _ = conn + .execute(&*migration.sql) + .await + .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; + + // language=SQL + query( + format!(r#"INSERT INTO "{schema}"._sqlx_migrations ( version, description, success, checksum, execution_time ) VALUES ( $1, $2, TRUE, $3, -1 )"#).as_str() + ) + .bind(migration.version) + .bind(&*migration.description) + .bind(&*migration.checksum) + .execute(conn) + .await?; + Ok(()) +} + +async fn revert_migration( + conn: &mut PgConnection, + schema: &'static str, + migration: &Migration, +) -> Result<(), MigrateError> { + let _ = conn + .execute(&*migration.sql) + .await + .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; + + query(format!(r#"DELETE FROM "{schema}"._sqlx_migrations WHERE version = $1"#).as_str()) + .bind(migration.version) + .execute(conn) + .await?; + + Ok(()) +} diff --git a/crates/sql/src/postgres/mod.rs b/crates/sql/src/postgres/mod.rs new file mode 100644 index 00000000..379307b3 --- /dev/null +++ b/crates/sql/src/postgres/mod.rs @@ -0,0 +1,35 @@ +pub mod migrate; +pub mod types; + +pub use sqlx::postgres::PgArguments; +pub use sqlx::{PgPool, Postgres}; + +use crate::{Executable, FlexQuery, SqlxResult}; +use futures::future::BoxFuture; +use sqlx::{Executor, PgTransaction}; + +pub type PgQuery = crate::FlexQuery; + +pub trait PgDbConnection: crate::PoolExt {} +impl crate::PgDbConnection for T {} + +impl Executable for FlexQuery { + fn execute<'t>(self, transaction: &'t mut PgTransaction) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + match self.args { + Some(args) => { + transaction + .execute(sqlx::query_with(self.sql.as_ref(), args)) + .await?; + } + None => { + transaction.execute(self.sql.as_ref()).await?; + } + } + Ok(()) + }) + } +} diff --git a/crates/sql/src/postgres/types.rs b/crates/sql/src/postgres/types.rs new file mode 100644 index 00000000..3ed6d57e --- /dev/null +++ b/crates/sql/src/postgres/types.rs @@ -0,0 +1,38 @@ +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::postgres::{PgArgumentBuffer, PgHasArrayType, PgTypeInfo, PgValueRef}; +use sqlx::{Decode, Encode, Postgres, Type}; + +use crate::types::SqlFelt; + +impl Type for SqlFelt { + fn type_info() -> PgTypeInfo { + PgTypeInfo::with_name("felt252") + } + + fn compatible(ty: &PgTypeInfo) -> bool { + *ty == PgTypeInfo::with_name("felt252") || <[u8] as Type>::compatible(ty) + } +} + +impl PgHasArrayType for SqlFelt { + fn array_type_info() -> PgTypeInfo { + PgTypeInfo::with_name("_felt252") + } +} + +impl Encode<'_, Postgres> for SqlFelt { + fn encode_by_ref(&self, buf: &mut PgArgumentBuffer) -> Result { + <&[u8] as Encode>::encode(self.0.as_slice(), buf) + } +} + +impl Decode<'_, Postgres> for SqlFelt { + fn decode(value: PgValueRef<'_>) -> Result { + let bytes = <&[u8] as Decode>::decode(value)?; + let arr: [u8; 32] = bytes + .try_into() + .map_err(|_| format!("expected 32 bytes for felt252, got {}", bytes.len()))?; + Ok(SqlFelt(arr)) + } +} diff --git a/crates/sql/src/query.rs b/crates/sql/src/query.rs new file mode 100644 index 00000000..fe916a37 --- /dev/null +++ b/crates/sql/src/query.rs @@ -0,0 +1,315 @@ +use crate::SqlxResult; +use futures::future::BoxFuture; +use itertools::Itertools; +use sqlx::{Database, Executor, Transaction}; +use std::fmt::Display; +use std::sync::Arc; + +pub trait Executable { + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't; +} + +impl Executable for &str +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self).await?; + Ok(()) + }) + } +} + +impl Executable for &String +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self.as_str()).await?; + Ok(()) + }) + } +} + +impl Executable for String +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self.as_str()).await?; + Ok(()) + }) + } +} + +impl Executable for FlexStr +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self.as_ref()).await?; + Ok(()) + }) + } +} + +#[derive(Debug, Clone)] +pub enum FlexStr { + Owned(String), + Static(&'static str), + Shared(Arc), +} + +impl Display for FlexStr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_ref().fmt(f) + } +} + +impl FlexStr { + pub fn as_str(&self) -> &str { + match self { + FlexStr::Owned(s) => s.as_str(), + FlexStr::Shared(s) => s, + FlexStr::Static(s) => s, + } + } +} + +impl AsRef for FlexStr { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl PartialEq for FlexStr { + fn eq(&self, other: &str) -> bool { + self.as_ref() == other + } +} + +/// A query with optional bind arguments. +/// +/// SQL can be any `FlexStr` (String, Arc, &'static str). +/// The `Bound` variant carries SQL + pre-built arguments. +/// The per-database `Executable` impls handle the lifetime requirements: +/// Postgres needs no special treatment; SQLite uses an unsafe lifetime extension +/// that is sound because the `FlexStr` outlives the `.await` point. +pub struct FlexQuery { + pub(crate) sql: FlexStr, + pub(crate) args: Option<::Arguments<'static>>, +} + +impl Clone for FlexQuery +where + DB::Arguments<'static>: Clone, +{ + fn clone(&self) -> Self { + Self { + sql: self.sql.clone(), + args: self.args.clone(), + } + } +} + +impl std::fmt::Debug for FlexQuery +where + DB::Arguments<'static>: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FlexQuery") + .field("sql", &self.sql) + .field("args", &self.args) + .finish() + } +} + +impl FlexQuery { + pub fn new(sql: impl Into, args: ::Arguments<'static>) -> Self { + FlexQuery { + sql: sql.into(), + args: Some(args), + } + } + + pub fn from_sql(sql: impl Into) -> Self { + FlexQuery { + sql: sql.into(), + args: None, + } + } +} + +impl PartialEq for FlexQuery { + fn eq(&self, other: &str) -> bool { + self.sql.as_ref() == other + } +} + +impl From for FlexStr { + fn from(s: String) -> Self { + FlexStr::Owned(s) + } +} + +impl From<&'static str> for FlexStr { + fn from(s: &'static str) -> Self { + FlexStr::Static(s) + } +} + +impl From> for FlexStr { + fn from(s: Arc) -> Self { + FlexStr::Shared(s) + } +} + +impl From for FlexQuery { + fn from(sql: FlexStr) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From<&'static str> for FlexQuery { + fn from(sql: &'static str) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From for FlexQuery { + fn from(sql: String) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From> for FlexQuery { + fn from(sql: Arc) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From<(S, A)> for FlexQuery +where + S: Into, + A: Into<::Arguments<'static>>, +{ + fn from((sql, args): (S, A)) -> Self { + FlexQuery::new(sql, args.into()) + } +} + +pub trait Queries { + fn add(&mut self, query: impl Into>); + fn adds(&mut self, queries: impl IntoIterator>>); +} + +impl Queries for Vec> { + fn add(&mut self, query: impl Into>) { + self.push(query.into()); + } + fn adds(&mut self, queries: impl IntoIterator>>) { + self.extend(queries.into_iter().map_into()); + } +} + +impl Executable for &[String; N] +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for query in self { + transaction.execute(query.as_str()).await?; + } + Ok(()) + }) + } +} + +impl + Send> Executable for Vec { + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} + +impl<'a, DB: Database, T> Executable for &'a Vec +where + &'a T: Executable + Send, + T: Send + Sync, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} + +impl<'a, DB: Database, T> Executable for &'a [T] +where + &'a T: Executable + Send, + T: Send + Sync, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} + +impl Executable for [T; N] +where + T: Executable + Send, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} diff --git a/crates/sql/src/runtime.rs b/crates/sql/src/runtime.rs new file mode 100644 index 00000000..e9c16bec --- /dev/null +++ b/crates/sql/src/runtime.rs @@ -0,0 +1,58 @@ +use std::str::FromStr; + +use crate::connection::DbBackend; +use crate::{DbPoolOptions, PoolConfig, SqlxError, SqlxResult}; +use sqlx::postgres::PgConnectOptions; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{Pool, Postgres, Sqlite}; + +#[derive(Clone)] +pub enum DbPool { + Postgres(Pool), + Sqlite(Pool), +} + +#[derive(Debug, Clone)] +pub enum DbConnectionOptions { + Postgres(PgConnectOptions), + Sqlite(SqliteConnectOptions), +} + +impl FromStr for DbConnectionOptions { + type Err = SqlxError; + fn from_str(s: &str) -> Result { + match DbBackend::from_str(s)? { + DbBackend::Postgres => PgConnectOptions::from_str(s).map(DbConnectionOptions::Postgres), + DbBackend::Sqlite => SqliteConnectOptions::from_str(s).map(DbConnectionOptions::Sqlite), + } + } +} + +impl PoolConfig { + pub async fn connect_any(&self) -> SqlxResult { + match DbBackend::try_from(self.url.as_str()) { + Ok(DbBackend::Postgres) => self.connect::().await.map(DbPool::Postgres), + Ok(DbBackend::Sqlite) => self.connect::().await.map(DbPool::Sqlite), + Err(err) => Err(SqlxError::Configuration(err.into())), + } + } +} + +impl DbPoolOptions { + pub async fn connect_any(&self, url: &str) -> SqlxResult { + match DbBackend::try_from(url) { + Ok(DbBackend::Postgres) => self.connect::(url).await.map(DbPool::Postgres), + Ok(DbBackend::Sqlite) => self.connect::(url).await.map(DbPool::Sqlite), + Err(err) => Err(SqlxError::Configuration(err.into())), + } + } + + pub async fn connect_any_with(&self, options: DbConnectionOptions) -> SqlxResult { + match options { + DbConnectionOptions::Postgres(opts) => { + self.connect_with(opts).await.map(DbPool::Postgres) + } + DbConnectionOptions::Sqlite(opts) => self.connect_with(opts).await.map(DbPool::Sqlite), + } + } +} diff --git a/crates/sql/src/sqlite/migrate.rs b/crates/sql/src/sqlite/migrate.rs new file mode 100644 index 00000000..11c1ca22 --- /dev/null +++ b/crates/sql/src/sqlite/migrate.rs @@ -0,0 +1,160 @@ +use crate::AcquiredSchema; +use futures::future::BoxFuture; +use sqlx::migrate::{AppliedMigration, Migrate, MigrateError, Migration}; +use sqlx::{query, query_as, Acquire, Executor, Sqlite, SqliteConnection}; +use std::borrow::Cow; +use std::time::{Duration, Instant}; + +impl AcquiredSchema { + fn table_name(&self) -> Cow<'static, str> { + Cow::Owned(format!("_sqlx_migrations_{}", self.schema)) + } +} + +impl Migrate for AcquiredSchema { + fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + let table_name = self.table_name(); + self.connection + .execute( + format!( + r#" +CREATE TABLE IF NOT EXISTS "{table_name}" ( + version BIGINT PRIMARY KEY, + description TEXT NOT NULL, + installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + success BOOLEAN NOT NULL, + checksum BLOB NOT NULL, + execution_time BIGINT NOT NULL +); + "# + ) + .as_str(), + ) + .await?; + + Ok(()) + }) + } + + fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + let table_name = self.table_name(); + let row: Option<(i64,)> = query_as( + format!( + r#"SELECT version FROM "{table_name}" WHERE success = false ORDER BY version LIMIT 1"# + ) + .as_str(), + ) + .fetch_optional(&mut self.connection) + .await?; + + Ok(row.map(|row| row.0)) + }) + } + + fn list_applied_migrations( + &mut self, + ) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + let table_name = self.table_name(); + let rows: Vec<(i64, Vec)> = query_as( + format!(r#"SELECT version, checksum FROM "{table_name}" ORDER BY version"#) + .as_str(), + ) + .fetch_all(&mut self.connection) + .await?; + + Ok(rows + .into_iter() + .map(|(version, checksum)| AppliedMigration { + version, + checksum: checksum.into(), + }) + .collect()) + }) + } + + fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { Ok(()) }) + } + + fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { Ok(()) }) + } + + fn apply<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let table_name = self.table_name(); + let mut tx = self.begin().await?; + let start = Instant::now(); + + let _ = tx + .execute(&*migration.sql) + .await + .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; + + let _ = query( + format!( + r#" +INSERT INTO "{table_name}" (version, description, success, checksum, execution_time) +VALUES (?1, ?2, TRUE, ?3, -1) + "# + ) + .as_str(), + ) + .bind(migration.version) + .bind(&*migration.description) + .bind(&*migration.checksum) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + let elapsed = start.elapsed(); + + #[allow(clippy::cast_possible_truncation)] + let _ = query( + format!( + r#" +UPDATE "{table_name}" +SET execution_time = ?1 +WHERE version = ?2 + "# + ) + .as_str(), + ) + .bind(elapsed.as_nanos() as i64) + .bind(migration.version) + .execute(&mut self.connection) + .await?; + + Ok(elapsed) + }) + } + + fn revert<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let table_name = self.table_name(); + let mut tx = self.begin().await?; + let start = Instant::now(); + + let _ = tx.execute(&*migration.sql).await?; + + let _ = query(format!(r#"DELETE FROM "{table_name}" WHERE version = ?1"#).as_str()) + .bind(migration.version) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(start.elapsed()) + }) + } +} diff --git a/crates/sql/src/sqlite/mod.rs b/crates/sql/src/sqlite/mod.rs new file mode 100644 index 00000000..d797fb48 --- /dev/null +++ b/crates/sql/src/sqlite/mod.rs @@ -0,0 +1,68 @@ +pub mod migrate; +pub mod types; + +use futures::future::BoxFuture; +pub use sqlx::sqlite::SqliteArguments; +use sqlx::{Executor, SqliteTransaction}; +pub use sqlx::{Sqlite, SqlitePool}; + +use sqlx::sqlite::SqliteConnectOptions; +use std::str::FromStr; + +use crate::{Executable, SqlxResult}; + +pub type SqliteQuery = super::FlexQuery; + +pub trait SqliteDbConnection: super::PoolExt {} +impl> SqliteDbConnection for T {} + +pub fn is_sqlite_memory_path(path: &str) -> bool { + path == ":memory:" + || path == "sqlite::memory:" + || path == "sqlite://:memory:" + || path.contains("mode=memory") +} + +pub fn sqlite_connect_options(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return SqliteConnectOptions::from_str("sqlite::memory:"); + } + + let options = if path.starts_with("sqlite:") { + SqliteConnectOptions::from_str(path)? + } else { + SqliteConnectOptions::new().filename(path) + }; + + if path.starts_with("sqlite:") && path.contains("mode=") { + Ok(options) + } else { + Ok(options.create_if_missing(true)) + } +} + +#[allow(unsafe_code)] +impl Executable for SqliteQuery { + fn execute<'t>(self, transaction: &'t mut SqliteTransaction) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + match self.args { + Some(args) => { + // SAFETY: `self.sql` is moved into this async block and lives for + // its entire duration. The extended reference is only used by + // `query_with` which is immediately awaited, so it cannot outlive + // the backing data in `FlexStr`. + let sql_ref: &'static str = + unsafe { std::mem::transmute::<&str, &'static str>(self.sql.as_ref()) }; + transaction.execute(sqlx::query_with(sql_ref, args)).await?; + } + None => { + transaction.execute(self.sql.as_ref()).await?; + } + } + Ok(()) + }) + } +} diff --git a/crates/sql/src/sqlite/types.rs b/crates/sql/src/sqlite/types.rs new file mode 100644 index 00000000..8d3ad9ab --- /dev/null +++ b/crates/sql/src/sqlite/types.rs @@ -0,0 +1,51 @@ +use crate::types::SqlFelt; +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::{Decode, Encode, Sqlite, Type}; + +impl Type for SqlFelt { + fn type_info() -> ::TypeInfo { + >::type_info() + } +} + +impl<'q> Encode<'q, Sqlite> for SqlFelt { + fn encode_by_ref( + &self, + buf: &mut ::ArgumentBuffer<'q>, + ) -> Result { + let mut hex = String::with_capacity(66); + hex.push_str("0x"); + for byte in &self.0 { + use std::fmt::Write; + write!(hex, "{byte:02x}").unwrap(); + } + Encode::::encode(hex, buf) + } +} + +impl Decode<'_, Sqlite> for SqlFelt { + fn decode(value: ::ValueRef<'_>) -> Result { + let s = >::decode(value)?; + let s = s.strip_prefix("0x").unwrap_or(&s); + if s.len() != 64 { + return Err(format!("expected 64 hex chars for felt252, got {}", s.len()).into()); + } + let mut arr = [0u8; 32]; + for (i, chunk) in s.as_bytes().chunks(2).enumerate() { + let hi = hex_nibble(chunk[0])?; + let lo = hex_nibble(chunk[1])?; + arr[i] = (hi << 4) | lo; + } + Ok(SqlFelt(arr)) + } +} + +fn hex_nibble(c: u8) -> Result { + match c { + b'0'..=b'9' => Ok(c - b'0'), + b'a'..=b'f' => Ok(c - b'a' + 10), + b'A'..=b'F' => Ok(c - b'A' + 10), + _ => Err(format!("invalid hex char: {}", c as char).into()), + } +} diff --git a/crates/sql/src/types.rs b/crates/sql/src/types.rs new file mode 100644 index 00000000..74837070 --- /dev/null +++ b/crates/sql/src/types.rs @@ -0,0 +1,22 @@ +use starknet_types_core::felt::Felt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SqlFelt(pub [u8; 32]); + +impl From for Felt { + fn from(value: SqlFelt) -> Self { + Felt::from_bytes_be(&value.0) + } +} + +impl From for SqlFelt { + fn from(value: Felt) -> Self { + SqlFelt(value.to_bytes_be()) + } +} + +impl From<&Felt> for SqlFelt { + fn from(value: &Felt) -> Self { + SqlFelt(value.to_bytes_be()) + } +} diff --git a/crates/sqlite/Cargo.toml b/crates/sqlite/Cargo.toml deleted file mode 100644 index 08f62825..00000000 --- a/crates/sqlite/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "torii-sqlite" -version = "0.1.0" -edition = "2021" -description = "SQLite connection helpers for Torii storage crates" - -[dependencies] -async-trait.workspace = true -futures.workspace = true -sqlx = { workspace = true, features = [ - "sqlite", - "runtime-tokio-rustls", - "migrate", -] } - -torii-common.workspace = true - -[lints] -workspace = true diff --git a/crates/sqlite/src/db.rs b/crates/sqlite/src/db.rs deleted file mode 100644 index c9f4a200..00000000 --- a/crates/sqlite/src/db.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::ops::Deref; -use std::str::FromStr; - -pub use async_trait::async_trait; -use sqlx::migrate::Migrator; -use sqlx::sqlite::SqliteConnectOptions; -use sqlx::Sqlite; -pub use sqlx::{SqlitePool, Transaction}; -use torii_common::sql::SqlxResult; - -use crate::migration::NamespaceMigrator; - -#[async_trait] -pub trait SqliteConnection { - fn pool(&self) -> &SqlitePool; - - async fn begin(&self) -> SqlxResult> { - Ok(self.pool().begin().await?) - } - - async fn migrate(&self, namespace: Option<&'static str>, migrator: Migrator) -> SqlxResult<()> { - let result = match namespace { - Some(namespace) => { - NamespaceMigrator::new(namespace, migrator) - .run(self.pool()) - .await - } - None => migrator.run(self.pool()).await, - }; - Ok(result?) - } - - async fn execute_queries(&self, queries: &[String]) -> SqlxResult<()> { - let mut transaction = self.begin().await?; - for query in queries { - sqlx::query(query).execute(&mut *transaction).await?; - } - transaction.commit().await - } -} - -#[allow(clippy::explicit_auto_deref)] -#[async_trait] -impl + Send + Sync + 'static> SqliteConnection for T { - fn pool(&self) -> &SqlitePool { - &**self - } -} - -pub fn is_sqlite_memory_path(path: &str) -> bool { - path == ":memory:" - || path == "sqlite::memory:" - || path == "sqlite://:memory:" - || path.contains("mode=memory") -} - -pub fn sqlite_connect_options(path: &str) -> Result { - if path == ":memory:" || path == "sqlite::memory:" { - return SqliteConnectOptions::from_str("sqlite::memory:"); - } - - let options = if path.starts_with("sqlite:") { - SqliteConnectOptions::from_str(path)? - } else { - SqliteConnectOptions::new().filename(path) - }; - - if path.starts_with("sqlite:") && path.contains("mode=") { - Ok(options) - } else { - Ok(options.create_if_missing(true)) - } -} diff --git a/crates/sqlite/src/lib.rs b/crates/sqlite/src/lib.rs deleted file mode 100644 index e0e1dc65..00000000 --- a/crates/sqlite/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod db; -pub mod migration; - -pub use db::{is_sqlite_memory_path, sqlite_connect_options, SqliteConnection}; diff --git a/crates/sqlite/src/migration.rs b/crates/sqlite/src/migration.rs deleted file mode 100644 index 8b9e3eef..00000000 --- a/crates/sqlite/src/migration.rs +++ /dev/null @@ -1,320 +0,0 @@ -use futures::future::BoxFuture; -use sqlx::migrate::{ - AppliedMigration, Migrate, MigrateError, Migration, MigrationSource, Migrator, -}; -use sqlx::sqlite::SqliteConnection; -use sqlx::{query, query_as, Acquire, Executor, Sqlite}; -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; -use std::ops::{Deref, DerefMut}; -use std::slice; -use std::time::{Duration, Instant}; - -pub struct NamespaceMigrator { - pub migrator: Migrator, - pub namespace: &'static str, -} - -pub struct SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a>, -{ - pub connection: >::Connection, - pub namespace: &'static str, -} - -impl<'a, A> Deref for SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite>, -{ - type Target = SqliteConnection; - - fn deref(&self) -> &Self::Target { - &self.connection - } -} - -impl<'a, A> DerefMut for SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite>, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.connection - } -} - -impl<'a, A> SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite>, -{ - fn table_name(&self) -> Cow<'static, str> { - Cow::Owned(format!("_sqlx_migrations_{}", self.namespace)) - } -} - -impl<'a, A> Migrate for SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite> + Executor<'a, Database = Sqlite>, -{ - fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - let table_name = self.table_name(); - self.connection - .execute( - format!( - r#" -CREATE TABLE IF NOT EXISTS "{table_name}" ( - version BIGINT PRIMARY KEY, - description TEXT NOT NULL, - installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - success BOOLEAN NOT NULL, - checksum BLOB NOT NULL, - execution_time BIGINT NOT NULL -); - "# - ) - .as_str(), - ) - .await?; - - Ok(()) - }) - } - - fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - let table_name = self.table_name(); - let row: Option<(i64,)> = query_as( - format!( - r#"SELECT version FROM "{table_name}" WHERE success = false ORDER BY version LIMIT 1"# - ) - .as_str(), - ) - .fetch_optional(&mut *self.connection) - .await?; - - Ok(row.map(|row| row.0)) - }) - } - - fn list_applied_migrations( - &mut self, - ) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - let table_name = self.table_name(); - let rows: Vec<(i64, Vec)> = query_as( - format!(r#"SELECT version, checksum FROM "{table_name}" ORDER BY version"#) - .as_str(), - ) - .fetch_all(&mut *self.connection) - .await?; - - Ok(rows - .into_iter() - .map(|(version, checksum)| AppliedMigration { - version, - checksum: checksum.into(), - }) - .collect()) - }) - } - - fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { Ok(()) }) - } - - fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { Ok(()) }) - } - - fn apply<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let table_name = self.table_name(); - let mut tx = self.begin().await?; - let start = Instant::now(); - - let _ = tx - .execute(&*migration.sql) - .await - .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; - - let _ = query( - format!( - r#" -INSERT INTO "{table_name}" (version, description, success, checksum, execution_time) -VALUES (?1, ?2, TRUE, ?3, -1) - "# - ) - .as_str(), - ) - .bind(migration.version) - .bind(&*migration.description) - .bind(&*migration.checksum) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - let elapsed = start.elapsed(); - - #[allow(clippy::cast_possible_truncation)] - let _ = query( - format!( - r#" -UPDATE "{table_name}" -SET execution_time = ?1 -WHERE version = ?2 - "# - ) - .as_str(), - ) - .bind(elapsed.as_nanos() as i64) - .bind(migration.version) - .execute(&mut *self.connection) - .await?; - - Ok(elapsed) - }) - } - - fn revert<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let table_name = self.table_name(); - let mut tx = self.begin().await?; - let start = Instant::now(); - - let _ = tx.execute(&*migration.sql).await?; - - let _ = query(format!(r#"DELETE FROM "{table_name}" WHERE version = ?1"#).as_str()) - .bind(migration.version) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(start.elapsed()) - }) - } -} - -impl NamespaceMigrator { - pub const fn new(namespace: &'static str, migrator: Migrator) -> Self { - Self { - migrator, - namespace, - } - } - - pub async fn new_from_source<'s, S>( - namespace: &'static str, - source: S, - ) -> Result - where - S: MigrationSource<'s>, - { - Migrator::new(source).await.map(|migrator| Self { - migrator, - namespace, - }) - } - - pub fn set_ignore_missing(&mut self, ignore_missing: bool) -> &Self { - self.migrator.ignore_missing = ignore_missing; - self - } - - pub fn set_locking(&mut self, locking: bool) -> &Self { - self.migrator.locking = locking; - self - } - - pub fn iter(&self) -> slice::Iter<'_, Migration> { - self.migrator.migrations.iter() - } - - pub fn version_exists(&self, version: i64) -> bool { - self.iter().any(|migration| migration.version == version) - } - - pub async fn run<'a, A>(&self, migrator: A) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Sqlite>, - SqliteAcquiredNamespace<'a, A>: Migrate, - { - let mut conn = SqliteAcquiredNamespace { - connection: migrator.acquire().await?, - namespace: self.namespace, - }; - self.migrator.run_direct(&mut conn).await - } - - pub async fn undo<'a, A>(&self, migrator: A, target: i64) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Sqlite>, - SqliteAcquiredNamespace<'a, A>: Migrate, - { - let mut conn = SqliteAcquiredNamespace { - connection: migrator.acquire().await?, - namespace: self.namespace, - }; - - if self.migrator.locking { - conn.lock().await?; - } - - conn.ensure_migrations_table().await?; - - if let Some(version) = conn.dirty_version().await? { - return Err(MigrateError::Dirty(version)); - } - - let applied_migrations = conn.list_applied_migrations().await?; - validate_applied_migrations(&applied_migrations, &self.migrator)?; - - let applied_migrations: HashMap<_, _> = applied_migrations - .into_iter() - .map(|migration| (migration.version, migration)) - .collect(); - - for migration in self - .iter() - .rev() - .filter(|migration| migration.migration_type.is_down_migration()) - .filter(|migration| applied_migrations.contains_key(&migration.version)) - .filter(|migration| migration.version > target) - { - conn.revert(migration).await?; - } - - if self.migrator.locking { - conn.unlock().await?; - } - - Ok(()) - } -} - -fn validate_applied_migrations( - applied_migrations: &[AppliedMigration], - migrator: &Migrator, -) -> Result<(), MigrateError> { - if migrator.ignore_missing { - return Ok(()); - } - - let migrations: HashSet<_> = migrator.iter().map(|migration| migration.version).collect(); - - for applied_migration in applied_migrations { - if !migrations.contains(&applied_migration.version) { - return Err(MigrateError::VersionMissing(applied_migration.version)); - } - } - - Ok(()) -} diff --git a/crates/testing/src/event_reader.rs b/crates/testing/src/event_reader.rs index 9b79a376..98ac85e2 100644 --- a/crates/testing/src/event_reader.rs +++ b/crates/testing/src/event_reader.rs @@ -45,6 +45,10 @@ pub struct EventIterator { pub event: usize, } +pub struct MultiContractEventIterator { + iterators: VecDeque, +} + impl EventIterator { pub fn new>(path: P) -> Self { let path = resolve_path_like(path); @@ -63,6 +67,16 @@ impl EventIterator { } } +impl MultiContractEventIterator { + pub fn new>(paths: Vec

) -> Self { + let iterators = paths + .into_iter() + .map(|p| EventIterator::new(p)) + .collect::>(); + Self { iterators } + } +} + impl Iterator for EventIterator { type Item = EmittedEvent; @@ -81,3 +95,18 @@ impl Iterator for EventIterator { } } } + +impl Iterator for MultiContractEventIterator { + type Item = EmittedEvent; + + fn next(&mut self) -> Option { + while let Some(iterator) = self.iterators.front_mut() { + if let Some(event) = iterator.next() { + self.iterators.rotate_left(1); + return Some(event); + } + self.iterators.pop_front(); + } + None + } +} diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 1635a8e8..70afaa2d 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -2,5 +2,5 @@ mod dojo; mod event_reader; pub mod utils; pub use dojo::FakeProvider; -pub use event_reader::EventIterator; +pub use event_reader::{EventIterator, MultiContractEventIterator}; pub use utils::{read_json_file, resolve_path_like}; diff --git a/crates/torii-common/Cargo.toml b/crates/torii-common/Cargo.toml index 9112e39f..a10b65f1 100644 --- a/crates/torii-common/Cargo.toml +++ b/crates/torii-common/Cargo.toml @@ -15,13 +15,6 @@ urlencoding = "2" async-trait = "0.1" serde.workspace = true serde_json.workspace = true -sqlx = { workspace = true, features = [ - "postgres", - "runtime-tokio-rustls", - "mysql", - "sqlite", -] } -itertools.workspace = true [lints] workspace = true diff --git a/crates/torii-common/src/lib.rs b/crates/torii-common/src/lib.rs index 5a96b8dc..497cbb9a 100644 --- a/crates/torii-common/src/lib.rs +++ b/crates/torii-common/src/lib.rs @@ -5,7 +5,6 @@ pub mod json; pub mod metadata; -pub mod sql; pub mod token_uri; pub mod utils; diff --git a/crates/torii-common/src/sql.rs b/crates/torii-common/src/sql.rs deleted file mode 100644 index 531adaf0..00000000 --- a/crates/torii-common/src/sql.rs +++ /dev/null @@ -1,241 +0,0 @@ -use async_trait::async_trait; -use itertools::Itertools; -use sqlx::{Database, Executor, Postgres}; -pub use sqlx::{PgPool, Transaction}; -use std::borrow::Cow; -use std::sync::Arc; - -pub use sqlx::Error as SqlxError; - -pub type SqlxResult = std::result::Result; - -#[async_trait] -pub trait Executable { - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()>; -} - -#[async_trait] -impl Executable for &str -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self).await?; - Ok(()) - } -} - -#[async_trait] -impl Executable for &String -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_str()).await?; - Ok(()) - } -} - -#[async_trait] -impl Executable for String -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_str()).await?; - Ok(()) - } -} - -#[async_trait] -impl Executable for Cow<'_, str> -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_ref()).await?; - Ok(()) - } -} - -pub struct QueryLike, DB: Database> { - sql: S, - args: Option<::Arguments<'static>>, -} - -impl, DB: Database> QueryLike { - pub fn new(sql: impl Into, args: ::Arguments<'static>) -> Self { - Self { - sql: sql.into(), - args: Some(args), - } - } - - pub fn from_sql(sql: S) -> Self { - Self { sql, args: None } - } -} - -#[async_trait] -impl Executable for FlexStr -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_ref()).await?; - Ok(()) - } -} - -pub enum FlexStr { - Owned(String), - Static(&'static str), - Shared(Arc), -} - -impl AsRef for FlexStr { - fn as_ref(&self) -> &str { - match self { - FlexStr::Owned(s) => s.as_str(), - FlexStr::Static(s) => s, - FlexStr::Shared(s) => s, - } - } -} - -impl PartialEq for FlexStr { - fn eq(&self, other: &str) -> bool { - self.as_ref() == other - } -} - -impl, DB: Database> PartialEq for QueryLike { - fn eq(&self, other: &str) -> bool { - self.sql.as_ref() == other - } -} - -impl From for FlexStr { - fn from(s: String) -> Self { - FlexStr::Owned(s) - } -} - -impl From<&'static str> for FlexStr { - fn from(s: &'static str) -> Self { - FlexStr::Static(s) - } -} - -impl From> for FlexStr { - fn from(s: Arc) -> Self { - FlexStr::Shared(s) - } -} - -impl From for QueryLike { - fn from(sql: FlexStr) -> Self { - QueryLike::from_sql(sql) - } -} - -impl From<&'static str> for QueryLike { - fn from(sql: &'static str) -> Self { - QueryLike::from_sql(sql.into()) - } -} - -impl From for QueryLike { - fn from(sql: String) -> Self { - QueryLike::from_sql(sql.into()) - } -} - -impl From> for QueryLike { - fn from(sql: Arc) -> Self { - QueryLike::from_sql(sql.into()) - } -} - -pub trait Queries { - fn add(&mut self, query: impl Into>); - fn adds(&mut self, queries: impl IntoIterator>>); -} - -impl Queries for Vec> { - fn add(&mut self, query: impl Into>) { - self.push(query.into()); - } - fn adds(&mut self, queries: impl IntoIterator>>) { - self.extend(queries.into_iter().map_into()); - } -} - -#[async_trait] -impl Executable for &[String; N] -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for query in self { - transaction.execute(query.as_str()).await?; - } - Ok(()) - } -} - -#[async_trait] -impl + Send> Executable for Vec { - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for item in self { - item.execute(transaction).await?; - } - Ok(()) - } -} - -#[async_trait] -impl<'a, DB: sqlx::Database, T> Executable for &'a Vec -where - &'a T: Executable + Send, - T: Send + Sync, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for item in self { - item.execute(transaction).await?; - } - Ok(()) - } -} - -#[async_trait] -impl<'a, DB: sqlx::Database, T> Executable for &'a [T] -where - &'a T: Executable + Send, - T: Send + Sync, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for item in self { - item.execute(transaction).await?; - } - Ok(()) - } -} - -#[async_trait] -impl + Send, DB: sqlx::Database> Executable for QueryLike -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, - for<'q> ::Arguments<'static>: sqlx::IntoArguments<'q, DB>, -{ - async fn execute(mut self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - let sql = self.sql.as_ref(); - let args = self.args.take().unwrap_or_default(); - transaction.execute(sqlx::query_with(sql, args)).await?; - Ok(()) - } -} - -pub type PgQuery = QueryLike; -pub type MySqlQuery = QueryLike; -pub type SqliteQuery = QueryLike; diff --git a/crates/torii-controllers-sink/Cargo.toml b/crates/torii-controllers-sink/Cargo.toml index 4f5c07a6..bf170517 100644 --- a/crates/torii-controllers-sink/Cargo.toml +++ b/crates/torii-controllers-sink/Cargo.toml @@ -4,8 +4,9 @@ version = "0.1.0" edition = "2021" [dependencies] -torii = { path = "../.." } +torii.workspace = true torii-runtime-common.workspace = true +torii-sql.workspace = true anyhow.workspace = true async-trait.workspace = true @@ -14,7 +15,12 @@ metrics.workspace = true reqwest = { version = "0.12", features = ["json", "rustls-tls"] } serde.workspace = true serde_json.workspace = true -sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "postgres", "any"] } +sqlx = { workspace = true, features = [ + "runtime-tokio", + "sqlite", + "postgres", + "any", +] } starknet.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/torii-controllers-sink/src/lib.rs b/crates/torii-controllers-sink/src/lib.rs index 1bdfad76..f576c90d 100644 --- a/crates/torii-controllers-sink/src/lib.rs +++ b/crates/torii-controllers-sink/src/lib.rs @@ -1,22 +1,22 @@ -use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; - use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use chrono::{DateTime, TimeZone, Utc}; use reqwest::Client; use serde::Deserialize; use serde_json::json; -use sqlx::{ - any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, -}; +use sqlx::any::AnyPoolOptions; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{Any, ConnectOptions, Pool, QueryBuilder, Row}; use starknet::core::types::Felt; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; use torii::axum::Router; use torii::etl::extractor::ExtractionBatch; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; use torii::etl::TypeId; use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; +use torii_sql::DbBackend; pub const DEFAULT_API_QUERY_URL: &str = "https://api.cartridge.gg/query"; pub const CONTROLLERS_TABLE: &str = "controllers"; @@ -28,23 +28,6 @@ const CONTROLLERS_TYPE: TypeId = TypeId::new("controllers.sync"); const CONTROLLER_PROCESSING_BATCH_SIZE: usize = 10_000; const SQLITE_CONTROLLER_UPSERT_BATCH_SIZE: usize = 199; const POSTGRES_CONTROLLER_UPSERT_BATCH_SIZE: usize = 10_000; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Sqlite, - Postgres, -} - -impl DbBackend { - fn detect(database_url: &str) -> Self { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - Self::Postgres - } else { - Self::Sqlite - } - } -} - #[derive(Debug, Clone, Deserialize)] struct ControllerAccount { username: String, @@ -131,7 +114,7 @@ impl ControllersStore { async fn new(database_url: &str, max_connections: Option) -> Result { sqlx::any::install_default_drivers(); - let backend = DbBackend::detect(database_url); + let backend = DbBackend::from_str(database_url).map_err(|e| anyhow!(e))?; let database_url = match backend { DbBackend::Postgres => database_url.to_string(), DbBackend::Sqlite => sqlite_url(database_url)?, @@ -555,7 +538,9 @@ mod tests { use serde_json::Value; use sqlx::query_scalar; use tokio::net::TcpListener; - use torii::axum::{extract::State, routing::post, Json, Router}; + use torii::axum::extract::State; + use torii::axum::routing::post; + use torii::axum::{Json, Router}; use torii::command::CommandBus; use torii::grpc::SubscriptionManager; diff --git a/crates/torii-ecs-sink/Cargo.toml b/crates/torii-ecs-sink/Cargo.toml index 3598c8c9..c06930c5 100644 --- a/crates/torii-ecs-sink/Cargo.toml +++ b/crates/torii-ecs-sink/Cargo.toml @@ -4,10 +4,12 @@ version = "0.1.0" edition = "2021" [dependencies] -torii = { path = "../.." } -torii-dojo = { path = "../dojo" } -torii-introspect = { path = "../introspect" } +torii.workspace = true +torii-dojo = { workspace = true, features = ["postgres", "sqlite"] } +torii-introspect.workspace = true torii-runtime-common.workspace = true +torii-sql.workspace = true + dojo-introspect.workspace = true introspect-types.workspace = true @@ -20,7 +22,12 @@ prost.workspace = true prost-types.workspace = true serde.workspace = true serde_json.workspace = true -sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "postgres", "any"] } +sqlx = { workspace = true, features = [ + "runtime-tokio", + "sqlite", + "postgres", + "any", +] } starknet.workspace = true tokio.workspace = true tokio-stream.workspace = true diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index 565828cf..64060a60 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -1,10 +1,24 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::pin::Pin; -use std::str::FromStr; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::Instant; - +use crate::proto::types::clause::ClauseType; +use crate::proto::types::member_value::ValueType; +use crate::proto::types::{ + self, ComparisonOperator, ContractType, LogicalOperator, PaginationDirection, PatternMatching, +}; +use crate::proto::world::world_server::World; +use crate::proto::world::{ + RetrieveContractsRequest, RetrieveContractsResponse, RetrieveControllersRequest, + RetrieveControllersResponse, RetrieveEntitiesRequest, RetrieveEntitiesResponse, + RetrieveEventsRequest, RetrieveEventsResponse, RetrieveTokenBalancesRequest, + RetrieveTokenBalancesResponse, RetrieveTokenContractsRequest, RetrieveTokenContractsResponse, + RetrieveTokenTransfersRequest, RetrieveTokenTransfersResponse, RetrieveTokensRequest, + RetrieveTokensResponse, RetrieveTransactionsRequest, RetrieveTransactionsResponse, + SubscribeContractsRequest, SubscribeContractsResponse, SubscribeEntitiesRequest, + SubscribeEntityResponse, SubscribeEventsRequest, SubscribeEventsResponse, + SubscribeTokenBalancesRequest, SubscribeTokenBalancesResponse, SubscribeTokenTransfersRequest, + SubscribeTokenTransfersResponse, SubscribeTokensRequest, SubscribeTokensResponse, + SubscribeTransactionsRequest, SubscribeTransactionsResponse, UpdateEntitiesSubscriptionRequest, + UpdateTokenBalancesSubscriptionRequest, UpdateTokenSubscriptionRequest, + UpdateTokenTransfersSubscriptionRequest, WorldsRequest, WorldsResponse, +}; use anyhow::{anyhow, Result}; use chrono::Utc; use introspect_types::serialize::ToCairoDeSeFrom; @@ -19,46 +33,32 @@ use prost::Message; use serde::ser::SerializeMap; use serde::Serializer; use serde_json::{Map, Serializer as JsonSerializer, Value}; -use sqlx::AnyConnection; -use sqlx::{ - any::AnyPoolOptions, pool::PoolConnection, postgres::PgPoolOptions, - sqlite::SqliteConnectOptions, sqlite::SqlitePoolOptions, Any, Column, ConnectOptions, Pool, - QueryBuilder, Row, -}; +use sqlx::any::AnyPoolOptions; +use sqlx::pool::PoolConnection; +use sqlx::postgres::PgPoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use sqlx::{Any, AnyConnection, Column, ConnectOptions, Pool, QueryBuilder, Row}; use starknet::core::types::Felt; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::fs; +use std::pin::Pin; +use std::str::FromStr; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Instant; use tokio::sync::mpsc::error::TrySendError; use tokio::sync::{mpsc, Mutex, RwLock}; use tokio::time::{sleep, Duration}; -use tokio_stream::{wrappers::ReceiverStream, Stream}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::Stream; use tonic::{Request, Response, Status}; -use torii_dojo::store::postgres::PgStore; use torii_dojo::store::sqlite::SqliteStore; use torii_dojo::store::DojoStoreTrait; use torii_dojo::DojoTable; use torii_introspect::events::{CreateTable, Record, UpdateTable}; use torii_introspect::schema::TableSchema; use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; - -use crate::proto::types::{ - self, clause::ClauseType, member_value::ValueType, ComparisonOperator, ContractType, - LogicalOperator, PaginationDirection, PatternMatching, -}; -use crate::proto::world::{ - world_server::World, RetrieveContractsRequest, RetrieveContractsResponse, - RetrieveControllersRequest, RetrieveControllersResponse, RetrieveEntitiesRequest, - RetrieveEntitiesResponse, RetrieveEventsRequest, RetrieveEventsResponse, - RetrieveTokenBalancesRequest, RetrieveTokenBalancesResponse, RetrieveTokenContractsRequest, - RetrieveTokenContractsResponse, RetrieveTokenTransfersRequest, RetrieveTokenTransfersResponse, - RetrieveTokensRequest, RetrieveTokensResponse, RetrieveTransactionsRequest, - RetrieveTransactionsResponse, SubscribeContractsRequest, SubscribeContractsResponse, - SubscribeEntitiesRequest, SubscribeEntityResponse, SubscribeEventsRequest, - SubscribeEventsResponse, SubscribeTokenBalancesRequest, SubscribeTokenBalancesResponse, - SubscribeTokenTransfersRequest, SubscribeTokenTransfersResponse, SubscribeTokensRequest, - SubscribeTokensResponse, SubscribeTransactionsRequest, SubscribeTransactionsResponse, - UpdateEntitiesSubscriptionRequest, UpdateTokenBalancesSubscriptionRequest, - UpdateTokenSubscriptionRequest, UpdateTokenTransfersSubscriptionRequest, WorldsRequest, - WorldsResponse, -}; +use torii_sql::DbBackend; const SUBSCRIPTION_SEEN_CACHE_CAPACITY: usize = 4096; @@ -85,29 +85,6 @@ impl TableKind { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Sqlite, - Postgres, -} - -impl DbBackend { - fn detect(database_url: &str) -> Self { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - Self::Postgres - } else { - Self::Sqlite - } - } - - fn as_str(self) -> &'static str { - match self { - Self::Sqlite => "sqlite", - Self::Postgres => "postgres", - } - } -} - #[derive(Clone)] pub struct EcsService { state: Arc, @@ -646,12 +623,11 @@ impl EcsService { ) -> Result { sqlx::any::install_default_drivers(); - let backend = DbBackend::detect(database_url); + let backend = get_db_backend(database_url); let database_url = match backend { DbBackend::Postgres => database_url.to_string(), DbBackend::Sqlite => sqlite_url(database_url)?, }; - let has_erc20 = erc20_url.is_some(); let has_erc721 = erc721_url.is_some(); let has_erc1155 = erc1155_url.is_some(); @@ -912,15 +888,23 @@ impl EcsService { ("erc1155", &self.state.erc1155_url), ] { if let Some(url) = url { - let path = sqlite_db_path(url); - let file_exists = std::path::Path::new(&path).exists(); - tracing::info!( - schema, - path = %path, - file_exists, - "Attaching ERC database" - ); - attach_sqlite_database(&mut conn, schema, url).await?; + let options = SqliteConnectOptions::from_str(url)?; + #[allow(clippy::match_bool)] + #[allow(clippy::single_match_else)] + match options.is_in_memory() { + true => tracing::info!(schema, "Attaching in-memory ERC database"), + false => { + let path = options.get_filename(); + let file_exists = path.exists(); + tracing::info!( + schema, + path = %path.display(), + file_exists, + "Attaching ERC database" + ); + } + } + attach_sqlite_database(&mut conn, schema, &options).await?; match sqlx::query(sqlite_master_preview_sql(schema)) .fetch_all(&mut *conn) .await @@ -3369,7 +3353,7 @@ impl EcsService { .max_connections(1) .connect_with(SqliteConnectOptions::from_str(&self.state.database_url)?) .await?; - let store = SqliteStore(Arc::new(pool)); + let store = SqliteStore(pool); Ok(store.read_tables(&[]).await?) } DbBackend::Postgres => { @@ -3377,8 +3361,7 @@ impl EcsService { .max_connections(1) .connect(&self.state.database_url) .await?; - let store = PgStore(Arc::new(pool)); - Ok(store.read_tables(&[]).await?) + Ok(pool.read_tables(&[]).await?) } } } @@ -4922,7 +4905,8 @@ async fn attach_sqlite_databases( ("erc1155", erc1155_url), ] { if let Some(url) = url { - attach_sqlite_database(conn, schema, url).await?; + let options = SqliteConnectOptions::from_str(url)?; + attach_sqlite_database(conn, schema, &options).await?; } } Ok(()) @@ -4931,7 +4915,7 @@ async fn attach_sqlite_databases( async fn attach_sqlite_database( conn: &mut AnyConnection, schema: &str, - url: &str, + options: &SqliteConnectOptions, ) -> sqlx::Result<()> { let attached = sqlx::query_scalar::("SELECT 1 FROM pragma_database_list WHERE name = ? LIMIT 1") @@ -4942,35 +4926,22 @@ async fn attach_sqlite_database( if attached { return Ok(()); } + let path = options.get_filename(); - let db_path = sqlite_db_path(url); - if !is_sqlite_memory_url(url) { - let path = std::path::Path::new(&db_path); - if let Some(parent) = path.parent().filter(|path| !path.as_os_str().is_empty()) { - std::fs::create_dir_all(parent)?; - } - if !path.exists() { - std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(false) - .open(path)?; - } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + if !path.exists() { + fs::File::create(path)?; } - - let path = db_path.replace('\'', "''"); // sqlite-dynamic-ok: ATTACH requires the database path and schema identifier in SQL text. + let path = path.to_string_lossy().replace('\'', "''"); sqlx::query(&format!("ATTACH DATABASE '{path}' AS {schema}")) .execute(&mut *conn) .await?; Ok(()) } -fn is_sqlite_memory_url(url: &str) -> bool { - matches!(url, ":memory:" | "sqlite::memory:") - || (url.starts_with("sqlite:file:") && url.contains("mode=memory")) -} - fn sqlite_master_preview_sql(schema: &str) -> &'static str { match schema { "erc20" => "SELECT name FROM erc20.sqlite_master WHERE type='table' LIMIT 5", @@ -4980,18 +4951,12 @@ fn sqlite_master_preview_sql(schema: &str) -> &'static str { } } -fn sqlite_db_path(url: &str) -> String { - let path = url - .strip_prefix("sqlite://") - .or_else(|| url.strip_prefix("sqlite:")) - .unwrap_or(url); - let p = std::path::Path::new(path); - if let (Some(parent), Some(file_name)) = (p.parent(), p.file_name()) { - if let Ok(abs_parent) = parent.canonicalize() { - return abs_parent.join(file_name).to_string_lossy().into_owned(); - } +fn get_db_backend(url: &str) -> DbBackend { + if url.starts_with("postgres") { + DbBackend::Postgres + } else { + DbBackend::Sqlite } - path.to_string() } fn sqlite_url(path: &str) -> Result { @@ -5178,10 +5143,9 @@ fn record_to_json_map( info }) .collect::>(); - let schema = torii_introspect::tables::RecordSchema::new( - &schema_table.primary, - schema_columns.iter().collect(), - ); + let primary = schema_table.primary.into(); + let schema = + torii_introspect::tables::RecordSchema::new(&primary, schema_columns.iter().collect()); let mut bytes = Vec::new(); let mut serializer = JsonSerializer::new(&mut bytes); @@ -6644,6 +6608,7 @@ mod tests { attributes: vec![], type_def: TypeDef::Bool, }], + append_only: false, }; service.cache_created_table(world_address, &table).await; @@ -6739,6 +6704,7 @@ mod tests { attributes: vec![], type_def: TypeDef::Bool, }], + append_only: false, }; service.cache_created_table(world_address, &table).await; @@ -6848,6 +6814,7 @@ mod tests { attributes: vec![Attribute::new_empty("key".to_string())], type_def: TypeDef::Felt252, }], + append_only: false, }; service.cache_created_table(world_address, &table).await; diff --git a/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs b/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs index 58487346..c6a1c590 100644 --- a/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs +++ b/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs @@ -23,8 +23,7 @@ fn runtime_sqlite_queries_avoid_uncached_prepare_and_inline_format_sql() { let root = workspace_root(); let files = [ "crates/torii-ecs-sink/src/grpc_service.rs", - "crates/torii-entities-historical-sink/src/lib.rs", - "crates/introspect-sqlite-sink/src/processor.rs", + "crates/introspect-sql-sink/src/sqlite/table.rs", "crates/torii-erc20/src/storage.rs", "crates/torii-erc721/src/storage.rs", "crates/torii-erc1155/src/storage.rs", diff --git a/crates/torii-entities-historical-sink/Cargo.toml b/crates/torii-entities-historical-sink/Cargo.toml deleted file mode 100644 index 710230ef..00000000 --- a/crates/torii-entities-historical-sink/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "torii-entities-historical-sink" -version = "0.1.0" -edition = "2021" -description = "Append-only historical entity sink for Torii introspect models" -authors = ["Torii Runtime "] -license = "Apache-2.0" - -[dependencies] -anyhow.workspace = true -async-trait.workspace = true -serde.workspace = true -serde_json.workspace = true -sqlx = { workspace = true, features = ["any", "postgres", "sqlite", "runtime-tokio-rustls"] } -starknet.workspace = true -tokio.workspace = true -torii.workspace = true -torii-runtime-common.workspace = true -tracing.workspace = true -thiserror.workspace = true -introspect-types.workspace = true - -torii-introspect.workspace = true - -[lints] -workspace = true diff --git a/crates/torii-entities-historical-sink/src/lib.rs b/crates/torii-entities-historical-sink/src/lib.rs deleted file mode 100644 index 025d2182..00000000 --- a/crates/torii-entities-historical-sink/src/lib.rs +++ /dev/null @@ -1,1199 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{anyhow, Context, Result}; -use async_trait::async_trait; -use sqlx::any::AnyPoolOptions; -use sqlx::{Any, Pool, Row}; -use starknet::core::types::Felt; -use tokio::sync::RwLock; -use torii::axum::Router; -use torii::etl::envelope::{Envelope, TypeId}; -use torii::etl::extractor::ExtractionBatch; -use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; -use torii_introspect::events::{CreateTable, IntrospectBody, IntrospectMsg, UpdateTable}; -use torii_introspect::schema::TableSchema; -use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; - -const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Sqlite, - Postgres, -} - -impl DbBackend { - fn detect(database_url: &str) -> Self { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - Self::Postgres - } else { - Self::Sqlite - } - } -} - -#[derive(Clone, Debug, Default)] -pub enum HistoricalNamespace { - #[default] - Default, - Custom(String), -} - -impl HistoricalNamespace { - fn sqlite_prefix(&self) -> &str { - match self { - Self::Default => "", - Self::Custom(prefix) => prefix, - } - } - - fn postgres_schema(&self) -> &str { - match self { - Self::Default => "public", - Self::Custom(schema) => schema, - } - } -} - -impl From<()> for HistoricalNamespace { - fn from((): ()) -> Self { - Self::Default - } -} - -impl From for HistoricalNamespace { - fn from(value: String) -> Self { - if value.is_empty() { - Self::Default - } else { - Self::Custom(value) - } - } -} - -impl From<&str> for HistoricalNamespace { - fn from(value: &str) -> Self { - if value.is_empty() { - Self::Default - } else { - Self::Custom(value.to_string()) - } - } -} - -#[derive(Clone, Debug)] -struct HistoryColumn { - name: String, - type_sql: String, -} - -#[derive(Clone, Debug)] -struct TrackedTable { - logical_name: String, - base_name: String, - history_name: String, - columns: Vec, - sqlite_queries: Option, -} - -#[derive(Clone, Debug)] -struct TrackedTableSqliteQueries { - next_revision: String, - copy_current_row: String, - insert_tombstone: String, -} - -#[derive(Clone)] -pub struct EntitiesHistoricalSink { - pool: Pool, - backend: DbBackend, - namespace: HistoricalNamespace, - tracked_names: HashSet, - tracked_tables: Arc>>, -} - -impl EntitiesHistoricalSink { - pub async fn new( - database_url: &str, - max_connections: Option, - namespace: impl Into, - tracked_models: Vec, - ) -> Result { - sqlx::any::install_default_drivers(); - - let backend = DbBackend::detect(database_url); - let database_url = match backend { - DbBackend::Postgres => database_url.to_string(), - DbBackend::Sqlite => sqlite_url(database_url)?, - }; - let pool = AnyPoolOptions::new() - .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { - DEFAULT_SQLITE_MAX_CONNECTIONS - } else { - 5 - })) - .connect(&database_url) - .await?; - - Ok(Self { - pool, - backend, - namespace: namespace.into(), - tracked_names: tracked_models - .into_iter() - .filter(|name| !name.is_empty()) - .collect(), - tracked_tables: Arc::new(RwLock::new(HashMap::new())), - }) - } - - async fn bootstrap(&self) -> Result<()> { - if self.tracked_names.is_empty() { - return Ok(()); - } - - if self.backend == DbBackend::Sqlite { - sqlx::query("PRAGMA journal_mode=WAL") - .execute(&self.pool) - .await - .ok(); - } - - match self.backend { - DbBackend::Sqlite => { - let rows = sqlx::query( - "SELECT table_schema_json - FROM introspect_sink_schema_state - WHERE alive != 0", - ) - .fetch_all(&self.pool) - .await?; - for row in rows { - let schema_json: String = row.try_get("table_schema_json")?; - let schema: TableSchema = serde_json::from_str(&schema_json)?; - if self.tracked_names.contains(&schema.name) { - self.sync_tracked_table(schema.id, &schema.name).await?; - } - } - } - DbBackend::Postgres => { - let rows = sqlx::query( - "SELECT id, name - FROM introspect.db_tables - WHERE \"schema\" = $1", - ) - .bind(self.namespace.postgres_schema()) - .fetch_all(&self.pool) - .await?; - for row in rows { - let table_name: String = row.try_get("name")?; - if !self.tracked_names.contains(&table_name) { - continue; - } - let table_id = felt_from_row_hex(&row, "id")?; - self.sync_tracked_table(table_id, &table_name).await?; - } - } - } - - Ok(()) - } - - async fn resolve_tracked_table(&self, table_id: Felt) -> Result> { - let tracked = { - let tracked_tables = self.tracked_tables.read().await; - tracked_tables.get(&table_id).cloned() - }; - - if let Some(table) = tracked { - return Ok(Some(table)); - } - - let resolved = self.lookup_table_name(table_id).await?; - let Some(table_name) = resolved else { - return Ok(None); - }; - if !self.tracked_names.contains(&table_name) { - return Ok(None); - } - - let tracked = self.sync_tracked_table(table_id, &table_name).await?; - Ok(Some(tracked)) - } - - async fn lookup_table_name(&self, table_id: Felt) -> Result> { - let (canonical_table_id, compact_table_id) = felt_hex_variants(table_id); - match self.backend { - DbBackend::Sqlite => { - let row = sqlx::query( - "SELECT table_schema_json - FROM introspect_sink_schema_state - WHERE table_id = ?1 OR table_id = ?2 - LIMIT 1", - ) - .bind(canonical_table_id) - .bind(compact_table_id) - .fetch_optional(&self.pool) - .await?; - row.map(|row| { - let schema_json: String = row.try_get("table_schema_json")?; - let schema: TableSchema = serde_json::from_str(&schema_json)?; - Ok::<_, anyhow::Error>(schema.name) - }) - .transpose() - } - DbBackend::Postgres => { - let row = sqlx::query( - "SELECT name - FROM introspect.db_tables - WHERE \"schema\" = $1 AND (id::text = $2 OR id::text = $3) - LIMIT 1", - ) - .bind(self.namespace.postgres_schema()) - .bind(canonical_table_id) - .bind(compact_table_id) - .fetch_optional(&self.pool) - .await?; - row.map(|row| row.try_get("name").map_err(Into::into)) - .transpose() - } - } - } - - async fn sync_tracked_table(&self, table_id: Felt, logical_name: &str) -> Result { - let base_name = match self.backend { - DbBackend::Sqlite => sqlite_storage_name(self.namespace.sqlite_prefix(), logical_name), - DbBackend::Postgres => logical_name.to_string(), - }; - let history_name = match self.backend { - DbBackend::Sqlite => sqlite_storage_name( - self.namespace.sqlite_prefix(), - &format!("{logical_name}_historical"), - ), - DbBackend::Postgres => format!("{logical_name}_historical"), - }; - let columns = self.load_source_columns(&base_name).await?; - if !columns.iter().any(|column| column.name == "entity_id") { - return Err(anyhow!( - "tracked table '{logical_name}' does not expose entity_id column" - )); - } - let sqlite_queries = (self.backend == DbBackend::Sqlite) - .then(|| build_tracked_table_sqlite_queries(&base_name, &history_name, &columns)); - let tracked = TrackedTable { - logical_name: logical_name.to_string(), - base_name, - history_name, - sqlite_queries, - columns, - }; - self.ensure_history_table(&tracked).await?; - self.tracked_tables - .write() - .await - .insert(table_id, tracked.clone()); - Ok(tracked) - } - - async fn load_source_columns(&self, base_name: &str) -> Result> { - match self.backend { - DbBackend::Sqlite => { - // sqlite-dynamic-ok: PRAGMA table_info requires the table identifier in SQL text. - let rows = sqlx::query(&format!( - "PRAGMA table_info({})", - quote_sqlite_identifier(base_name) - )) - .fetch_all(&self.pool) - .await?; - let mut columns = Vec::with_capacity(rows.len()); - for row in rows { - let name: String = row.try_get("name")?; - let type_sql: String = row.try_get("type")?; - columns.push(HistoryColumn { name, type_sql }); - } - Ok(columns) - } - DbBackend::Postgres => { - let rows = sqlx::query( - "SELECT a.attname AS column_name, - pg_catalog.format_type(a.atttypid, a.atttypmod) AS column_type - FROM pg_catalog.pg_attribute a - JOIN pg_catalog.pg_class c ON c.oid = a.attrelid - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 - AND c.relname = $2 - AND a.attnum > 0 - AND NOT a.attisdropped - ORDER BY a.attnum", - ) - .bind(self.namespace.postgres_schema()) - .bind(base_name) - .fetch_all(&self.pool) - .await?; - - let mut columns = Vec::new(); - for row in rows { - let name: String = row.try_get("column_name")?; - if name.starts_with("__") { - continue; - } - let type_sql: String = row.try_get("column_type")?; - columns.push(HistoryColumn { name, type_sql }); - } - Ok(columns) - } - } - } - - async fn ensure_history_table(&self, tracked: &TrackedTable) -> Result<()> { - let create_sql = match self.backend { - DbBackend::Sqlite => self.create_history_table_sqlite(tracked), - DbBackend::Postgres => self.create_history_table_postgres(tracked), - }; - sqlx::query(&create_sql).execute(&self.pool).await?; - - let existing_columns = self.load_existing_history_columns(tracked).await?; - for column in &tracked.columns { - if existing_columns.contains(&column.name) { - continue; - } - let add_sql = match self.backend { - DbBackend::Sqlite => format!( - "ALTER TABLE {} ADD COLUMN {} {}", - quote_sqlite_identifier(&tracked.history_name), - quote_ident(&column.name), - column.type_sql - ), - DbBackend::Postgres => format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS {} {}", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name), - quote_ident(&column.name), - column.type_sql - ), - }; - sqlx::query(&add_sql).execute(&self.pool).await?; - } - - for meta_sql in self.ensure_history_meta_columns_sql(tracked, &existing_columns) { - sqlx::query(&meta_sql).execute(&self.pool).await?; - } - - for index_sql in self.ensure_history_indexes_sql(tracked) { - sqlx::query(&index_sql).execute(&self.pool).await?; - } - - Ok(()) - } - - async fn load_existing_history_columns( - &self, - tracked: &TrackedTable, - ) -> Result> { - let rows = match self.backend { - DbBackend::Sqlite => { - // sqlite-dynamic-ok: PRAGMA table_info requires the table identifier in SQL text. - sqlx::query(&format!( - "PRAGMA table_info({})", - quote_sqlite_identifier(&tracked.history_name) - )) - .fetch_all(&self.pool) - .await? - } - DbBackend::Postgres => { - sqlx::query( - "SELECT a.attname AS column_name - FROM pg_catalog.pg_attribute a - JOIN pg_catalog.pg_class c ON c.oid = a.attrelid - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 - AND c.relname = $2 - AND a.attnum > 0 - AND NOT a.attisdropped", - ) - .bind(self.namespace.postgres_schema()) - .bind(&tracked.history_name) - .fetch_all(&self.pool) - .await? - } - }; - let mut columns = HashSet::with_capacity(rows.len()); - for row in rows { - let name = if self.backend == DbBackend::Sqlite { - row.try_get("name")? - } else { - row.try_get("column_name")? - }; - columns.insert(name); - } - Ok(columns) - } - - fn create_history_table_sqlite(&self, tracked: &TrackedTable) -> String { - let model_columns = tracked - .columns - .iter() - .map(|column| { - if column.name == "entity_id" { - format!("{} {} NOT NULL", quote_ident(&column.name), column.type_sql) - } else { - format!("{} {}", quote_ident(&column.name), column.type_sql) - } - }) - .collect::>() - .join(", "); - format!( - "CREATE TABLE IF NOT EXISTS {} ({model_columns}, \ - \"revision\" INTEGER NOT NULL DEFAULT 0, \ - \"historical_deleted\" INTEGER NOT NULL DEFAULT 0, \ - \"historical_block_number\" INTEGER, \ - \"historical_tx_hash\" TEXT NOT NULL DEFAULT '', \ - \"historical_executed_at\" INTEGER NOT NULL DEFAULT 0)", - quote_sqlite_identifier(&tracked.history_name) - ) - } - - fn create_history_table_postgres(&self, tracked: &TrackedTable) -> String { - let model_columns = tracked - .columns - .iter() - .map(|column| { - if column.name == "entity_id" { - format!("{} {} NOT NULL", quote_ident(&column.name), column.type_sql) - } else { - format!("{} {}", quote_ident(&column.name), column.type_sql) - } - }) - .collect::>() - .join(", "); - format!( - "CREATE TABLE IF NOT EXISTS {} ({model_columns}, \ - \"revision\" BIGINT NOT NULL DEFAULT 0, \ - \"historical_deleted\" BOOLEAN NOT NULL DEFAULT FALSE, \ - \"historical_block_number\" BIGINT, \ - \"historical_tx_hash\" TEXT NOT NULL DEFAULT '', \ - \"historical_executed_at\" BIGINT NOT NULL DEFAULT 0)", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ) - } - - fn ensure_history_meta_columns_sql( - &self, - tracked: &TrackedTable, - existing_columns: &HashSet, - ) -> Vec { - match self.backend { - DbBackend::Sqlite => { - let mut sql = Vec::new(); - let target = quote_sqlite_identifier(&tracked.history_name); - if !existing_columns.contains("revision") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"revision\" INTEGER NOT NULL DEFAULT 0" - )); - } - if !existing_columns.contains("historical_deleted") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_deleted\" INTEGER NOT NULL DEFAULT 0" - )); - } - if !existing_columns.contains("historical_block_number") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_block_number\" INTEGER" - )); - } - if !existing_columns.contains("historical_tx_hash") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_tx_hash\" TEXT NOT NULL DEFAULT ''" - )); - } - if !existing_columns.contains("historical_executed_at") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_executed_at\" INTEGER NOT NULL DEFAULT 0" - )); - } - sql - } - DbBackend::Postgres => vec![ - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"revision\" BIGINT NOT NULL DEFAULT 0", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_deleted\" BOOLEAN NOT NULL DEFAULT FALSE", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_block_number\" BIGINT", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_tx_hash\" TEXT NOT NULL DEFAULT ''", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_executed_at\" BIGINT NOT NULL DEFAULT 0", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - ], - } - } - - fn ensure_history_indexes_sql(&self, tracked: &TrackedTable) -> Vec { - let unique_index = format!("{}_entity_revision_idx", tracked.history_name); - let entity_index = format!("{}_entity_idx", tracked.history_name); - match self.backend { - DbBackend::Sqlite => vec![ - format!( - "CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} (\"entity_id\", \"revision\")", - quote_ident(&unique_index), - quote_sqlite_identifier(&tracked.history_name) - ), - format!( - "CREATE INDEX IF NOT EXISTS {} ON {} (\"entity_id\")", - quote_ident(&entity_index), - quote_sqlite_identifier(&tracked.history_name) - ), - ], - DbBackend::Postgres => vec![ - format!( - "CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} (\"entity_id\", \"revision\")", - quote_ident(&unique_index), - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "CREATE INDEX IF NOT EXISTS {} ON {} (\"entity_id\")", - quote_ident(&entity_index), - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - ], - } - } - - async fn next_revision( - &self, - tracked: &TrackedTable, - canonical_entity_id_hex: &str, - compact_entity_id_hex: &str, - ) -> Result { - let sql = match self.backend { - DbBackend::Sqlite => tracked - .sqlite_queries - .as_ref() - .expect("sqlite queries available for sqlite backend") - .next_revision - .clone(), - DbBackend::Postgres => format!( - "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ - FROM {} WHERE \"entity_id\"::text = $1 OR \"entity_id\"::text = $2", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - }; - let row = sqlx::query(&sql) - .bind(canonical_entity_id_hex) - .bind(compact_entity_id_hex) - .fetch_one(&self.pool) - .await?; - Ok(row.try_get::("next_revision")?) - } - - async fn persist_snapshot( - &self, - tracked: &TrackedTable, - entity_id: Felt, - block_number: Option, - tx_hash: Felt, - executed_at: u64, - deleted: bool, - ) -> Result<()> { - let (canonical_entity_id_hex, compact_entity_id_hex) = felt_hex_variants(entity_id); - let revision = self - .next_revision(tracked, &canonical_entity_id_hex, &compact_entity_id_hex) - .await?; - let copied = self - .copy_current_row_into_history( - tracked, - &canonical_entity_id_hex, - &compact_entity_id_hex, - revision, - block_number, - tx_hash, - executed_at, - deleted, - ) - .await?; - if copied == 0 { - if deleted { - self.insert_tombstone_only( - tracked, - &canonical_entity_id_hex, - revision, - block_number, - tx_hash, - executed_at, - ) - .await?; - return Ok(()); - } - return Err(anyhow!( - "unable to load latest row for tracked model '{}' entity {}", - tracked.logical_name, - canonical_entity_id_hex - )); - } - Ok(()) - } - - async fn copy_current_row_into_history( - &self, - tracked: &TrackedTable, - canonical_entity_id_hex: &str, - compact_entity_id_hex: &str, - revision: i64, - block_number: Option, - tx_hash: Felt, - executed_at: u64, - deleted: bool, - ) -> Result { - let sql = match self.backend { - DbBackend::Sqlite => tracked - .sqlite_queries - .as_ref() - .expect("sqlite queries available for sqlite backend") - .copy_current_row - .clone(), - DbBackend::Postgres => { - let source_columns = tracked - .columns - .iter() - .map(|column| quote_ident(&column.name)) - .collect::>() - .join(", "); - let insert_columns = format!( - "{source_columns}, \"revision\", \"historical_deleted\", \ - \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" - ); - let history_target = - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name); - let source_target = - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.base_name); - format!( - "INSERT INTO {history_target} ({insert_columns}) \ - SELECT {source_columns}, $1, $2, $3, $4, $5 \ - FROM {source_target} WHERE (\"entity_id\"::text = $6 OR \"entity_id\"::text = $7) LIMIT 1" - ) - } - }; - let mut query = sqlx::query(&sql).bind(revision); - query = match self.backend { - DbBackend::Sqlite => query.bind(i64::from(deleted)), - DbBackend::Postgres => query.bind(deleted), - }; - let result = query - .bind(block_number.map(|value| value as i64)) - .bind(felt_hex(tx_hash)) - .bind(executed_at as i64) - .bind(canonical_entity_id_hex) - .bind(compact_entity_id_hex) - .execute(&self.pool) - .await?; - Ok(result.rows_affected()) - } - - async fn insert_tombstone_only( - &self, - tracked: &TrackedTable, - entity_id_hex: &str, - revision: i64, - block_number: Option, - tx_hash: Felt, - executed_at: u64, - ) -> Result<()> { - let sql = match self.backend { - DbBackend::Sqlite => tracked - .sqlite_queries - .as_ref() - .expect("sqlite queries available for sqlite backend") - .insert_tombstone - .clone(), - DbBackend::Postgres => { - let mut columns = Vec::with_capacity(tracked.columns.len() + 5); - let mut values_sql = Vec::with_capacity(tracked.columns.len() + 5); - - let mut bind_index = 1_usize; - for column in &tracked.columns { - columns.push(quote_ident(&column.name)); - if column.name == "entity_id" { - values_sql.push(self.entity_cast_placeholder(bind_index, &column.type_sql)); - bind_index += 1; - } else { - values_sql.push("NULL".to_string()); - } - } - - columns.extend([ - "\"revision\"".to_string(), - "\"historical_deleted\"".to_string(), - "\"historical_block_number\"".to_string(), - "\"historical_tx_hash\"".to_string(), - "\"historical_executed_at\"".to_string(), - ]); - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - - let target = - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name); - format!( - "INSERT INTO {target} ({}) VALUES ({})", - columns.join(", "), - values_sql.join(", ") - ) - } - }; - - let mut query = sqlx::query(&sql).bind(entity_id_hex).bind(revision); - query = match self.backend { - DbBackend::Sqlite => query.bind(1_i64), - DbBackend::Postgres => query.bind(true), - }; - query - .bind(block_number.map(|value| value as i64)) - .bind(felt_hex(tx_hash)) - .bind(executed_at as i64) - .execute(&self.pool) - .await?; - - Ok(()) - } - - fn value_placeholder(&self, index: usize) -> String { - match self.backend { - DbBackend::Sqlite => "?".to_string(), - DbBackend::Postgres => format!("${index}"), - } - } - - fn entity_cast_placeholder(&self, index: usize, type_sql: &str) -> String { - match self.backend { - DbBackend::Sqlite => "?".to_string(), - DbBackend::Postgres => format!("CAST(${index} AS {type_sql})"), - } - } - - async fn process_table_schema(&self, table: &CreateTable) -> Result<()> { - if self.tracked_names.contains(&table.name) { - self.sync_tracked_table(table.id, &table.name).await?; - } - Ok(()) - } - - async fn process_table_update(&self, table: &UpdateTable) -> Result<()> { - if self.tracked_names.contains(&table.name) { - self.sync_tracked_table(table.id, &table.name).await?; - } - Ok(()) - } -} - -#[async_trait] -impl Sink for EntitiesHistoricalSink { - fn name(&self) -> &'static str { - "entities-historical" - } - - fn interested_types(&self) -> Vec { - vec![INTROSPECT_TYPE] - } - - async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> Result<()> { - for envelope in envelopes { - if envelope.type_id != INTROSPECT_TYPE { - continue; - } - let Some(body) = envelope.downcast_ref::() else { - continue; - }; - let context = batch - .get_event_context(&body.metadata.transaction_hash, body.metadata.from_address) - .unwrap_or_default(); - let block_number = body - .metadata - .block_number - .or(Some(context.transaction.block_number)); - let executed_at = context.block.timestamp; - - match &body.msg { - IntrospectMsg::CreateTable(table) => { - self.process_table_schema(table).await?; - } - IntrospectMsg::UpdateTable(table) => { - self.process_table_update(table).await?; - } - IntrospectMsg::InsertsFields(insert) => { - let Some(tracked) = self.resolve_tracked_table(insert.table).await? else { - continue; - }; - for record in &insert.records { - self.persist_snapshot( - &tracked, - Felt::from_bytes_be(&record.id), - block_number, - body.metadata.transaction_hash, - executed_at, - false, - ) - .await - .with_context(|| { - format!( - "failed to persist historical snapshot for '{}' insert", - tracked.logical_name - ) - })?; - } - } - IntrospectMsg::DeleteRecords(delete) => { - let Some(tracked) = self.resolve_tracked_table(delete.table).await? else { - continue; - }; - for row in &delete.rows { - self.persist_snapshot( - &tracked, - row.to_felt(), - block_number, - body.metadata.transaction_hash, - executed_at, - true, - ) - .await - .with_context(|| { - format!( - "failed to persist historical tombstone for '{}'", - tracked.logical_name - ) - })?; - } - } - _ => {} - } - } - - Ok(()) - } - - fn topics(&self) -> Vec { - Vec::new() - } - - fn build_routes(&self) -> Router { - Router::new() - } - - async fn initialize( - &mut self, - _event_bus: Arc, - _context: &SinkContext, - ) -> Result<()> { - self.bootstrap().await?; - Ok(()) - } -} - -fn sqlite_storage_name(prefix: &str, logical_name: &str) -> String { - if prefix.is_empty() { - logical_name.to_string() - } else { - format!("{prefix}__{logical_name}") - } -} - -fn sqlite_url(database_url: &str) -> Result { - if database_url == ":memory:" { - return Ok("sqlite::memory:".to_string()); - } - if database_url.starts_with("sqlite:") { - return Ok(database_url.to_string()); - } - Ok(format!("sqlite://{database_url}")) -} - -fn quote_ident(identifier: &str) -> String { - format!("\"{}\"", identifier.replace('"', "\"\"")) -} - -fn quote_sqlite_identifier(table: &str) -> String { - quote_ident(table) -} - -fn quote_pg_qualified(schema: &str, table: &str) -> String { - format!("{}.{}", quote_ident(schema), quote_ident(table)) -} - -fn build_tracked_table_sqlite_queries( - base_name: &str, - history_name: &str, - columns: &[HistoryColumn], -) -> TrackedTableSqliteQueries { - let history_target = quote_sqlite_identifier(history_name); - let source_target = quote_sqlite_identifier(base_name); - let source_columns = columns - .iter() - .map(|column| quote_ident(&column.name)) - .collect::>() - .join(", "); - let insert_columns = format!( - "{source_columns}, \"revision\", \"historical_deleted\", \ - \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" - ); - - let mut tombstone_columns = Vec::with_capacity(columns.len() + 5); - let mut tombstone_values = Vec::with_capacity(columns.len() + 5); - for column in columns { - tombstone_columns.push(quote_ident(&column.name)); - if column.name == "entity_id" { - tombstone_values.push("?".to_string()); - } else { - tombstone_values.push("NULL".to_string()); - } - } - tombstone_columns.extend([ - "\"revision\"".to_string(), - "\"historical_deleted\"".to_string(), - "\"historical_block_number\"".to_string(), - "\"historical_tx_hash\"".to_string(), - "\"historical_executed_at\"".to_string(), - ]); - tombstone_values.extend([ - "?".to_string(), - "?".to_string(), - "?".to_string(), - "?".to_string(), - "?".to_string(), - ]); - - TrackedTableSqliteQueries { - next_revision: format!( - "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ - FROM {history_target} WHERE \"entity_id\" = ?1 OR \"entity_id\" = ?2" - ), - copy_current_row: format!( - "INSERT INTO {history_target} ({insert_columns}) \ - SELECT {source_columns}, ?1, ?2, ?3, ?4, ?5 \ - FROM {source_target} \ - WHERE (\"entity_id\" = ?6 OR \"entity_id\" = ?7) \ - LIMIT 1" - ), - insert_tombstone: format!( - "INSERT INTO {history_target} ({}) VALUES ({})", - tombstone_columns.join(", "), - tombstone_values.join(", ") - ), - } -} - -fn felt_hex(value: Felt) -> String { - format!("{value:#x}") -} - -fn canonical_felt_hex(value: Felt) -> String { - format!("{value:#066x}") -} - -fn compact_hex_str(value: &str) -> String { - let value = value.trim(); - let Some(hex) = value.strip_prefix("0x") else { - return value.to_string(); - }; - let trimmed = hex.trim_start_matches('0'); - if trimmed.is_empty() { - "0x0".to_string() - } else { - format!("0x{trimmed}") - } -} - -fn felt_hex_variants(value: Felt) -> (String, String) { - let canonical = canonical_felt_hex(value); - let compact = compact_hex_str(&canonical); - (canonical, compact) -} - -fn felt_from_row_hex(row: &sqlx::any::AnyRow, column: &str) -> Result { - let value: String = row.try_get(column)?; - Felt::from_hex(&value).map_err(Into::into) -} - -#[cfg(test)] -mod tests { - use super::*; - use introspect_types::{ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; - use torii::etl::envelope::{EventBody, MetaData}; - use torii::etl::extractor::ExtractionBatch; - use torii_introspect::events::{DeleteRecords, InsertsFields, IntrospectMsg, Record}; - - fn sqlite_table_schema(name: &str, id: Felt) -> TableSchema { - TableSchema { - id, - name: name.to_string(), - attributes: Vec::new(), - primary: PrimaryDef { - name: "entity_id".to_string(), - attributes: Vec::new(), - type_def: PrimaryTypeDef::Felt252, - }, - columns: vec![ - ColumnDef { - id: Felt::from(1_u8), - name: "name".to_string(), - attributes: Vec::new(), - type_def: TypeDef::ByteArray, - }, - ColumnDef { - id: Felt::from(2_u8), - name: "score".to_string(), - attributes: Vec::new(), - type_def: TypeDef::U32, - }, - ], - } - } - - async fn sqlite_sink() -> Result { - let sink = EntitiesHistoricalSink::new( - "sqlite::memory:", - Some(1), - (), - vec!["NUMS-Game".to_string()], - ) - .await?; - sqlx::query( - "CREATE TABLE introspect_sink_schema_state ( - table_id TEXT PRIMARY KEY, - table_schema_json TEXT NOT NULL, - alive INTEGER NOT NULL DEFAULT 1, - updated_at INTEGER NOT NULL DEFAULT (unixepoch()) - )", - ) - .execute(&sink.pool) - .await?; - sqlx::query( - "CREATE TABLE \"NUMS-Game\" ( - \"entity_id\" TEXT PRIMARY KEY, - \"name\" TEXT, - \"score\" INTEGER - )", - ) - .execute(&sink.pool) - .await?; - let schema = sqlite_table_schema("NUMS-Game", Felt::from(9_u8)); - sqlx::query( - "INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) - VALUES (?1, ?2, 1, unixepoch())", - ) - .bind(canonical_felt_hex(schema.id)) - .bind(serde_json::to_string(&schema)?) - .execute(&sink.pool) - .await?; - Ok(sink) - } - - #[tokio::test] - async fn initializes_history_table_from_sqlite_schema_state() -> Result<()> { - let sink = sqlite_sink().await?; - sink.bootstrap().await?; - - let row = sqlx::query( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'NUMS-Game_historical'", - ) - .fetch_one(&sink.pool) - .await?; - let name: String = row.try_get("name")?; - assert_eq!(name, "NUMS-Game_historical"); - Ok(()) - } - - #[tokio::test] - async fn appends_revisions_and_delete_tombstones_in_sqlite() -> Result<()> { - let sink = sqlite_sink().await?; - sink.bootstrap().await?; - - let entity_id = Felt::from(77_u8); - sqlx::query( - "INSERT INTO \"NUMS-Game\" (\"entity_id\", \"name\", \"score\") VALUES (?1, ?2, ?3)", - ) - .bind(canonical_felt_hex(entity_id)) - .bind("first") - .bind(10_i64) - .execute(&sink.pool) - .await?; - - let insert = EventBody { - metadata: MetaData { - block_number: Some(1), - transaction_hash: Felt::from(100_u16), - from_address: Felt::ZERO, - }, - msg: IntrospectMsg::InsertsFields(InsertsFields::new( - Felt::from(9_u8), - Vec::new(), - vec![Record::new(entity_id, Vec::new())], - )), - }; - sink.process(&[Envelope::from(insert)], &ExtractionBatch::empty()) - .await?; - - sqlx::query("UPDATE \"NUMS-Game\" SET \"score\" = 25 WHERE \"entity_id\" = ?1") - .bind(canonical_felt_hex(entity_id)) - .execute(&sink.pool) - .await?; - let update = EventBody { - metadata: MetaData { - block_number: Some(2), - transaction_hash: Felt::from(101_u16), - from_address: Felt::ZERO, - }, - msg: IntrospectMsg::InsertsFields(InsertsFields::new( - Felt::from(9_u8), - Vec::new(), - vec![Record::new(entity_id, Vec::new())], - )), - }; - sink.process(&[Envelope::from(update)], &ExtractionBatch::empty()) - .await?; - - let delete = EventBody { - metadata: MetaData { - block_number: Some(3), - transaction_hash: Felt::from(102_u16), - from_address: Felt::ZERO, - }, - msg: IntrospectMsg::DeleteRecords(DeleteRecords::new( - Felt::from(9_u8), - vec![entity_id.into()], - )), - }; - sink.process(&[Envelope::from(delete)], &ExtractionBatch::empty()) - .await?; - - let rows = sqlx::query( - "SELECT revision, historical_deleted, score FROM \"NUMS-Game_historical\" ORDER BY revision", - ) - .fetch_all(&sink.pool) - .await?; - - assert_eq!(rows.len(), 3); - assert_eq!(rows[0].try_get::("revision")?, 1); - assert_eq!(rows[0].try_get::("historical_deleted")?, 0); - assert_eq!(rows[0].try_get::("score")?, 10); - assert_eq!(rows[1].try_get::("revision")?, 2); - assert_eq!(rows[1].try_get::("score")?, 25); - assert_eq!(rows[2].try_get::("revision")?, 3); - assert_eq!(rows[2].try_get::("historical_deleted")?, 1); - assert_eq!(rows[2].try_get::("score")?, 25); - Ok(()) - } -} diff --git a/crates/torii-erc20/src/storage.rs b/crates/torii-erc20/src/storage.rs index 82b0296e..fa8fac88 100644 --- a/crates/torii-erc20/src/storage.rs +++ b/crates/torii-erc20/src/storage.rs @@ -173,18 +173,8 @@ pub enum TransferDirection { } /// Storage for ERC20 transfers and approvals -pub struct Erc20Storage { - backend: StorageBackend, - conn: Arc>, - balance_cache: Arc>, - pg_conns: Option>>>, - pg_rr: AtomicUsize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StorageBackend { - Sqlite, - Postgres, +pub struct Erc20Storage { + backend: Backend, } /// Transfer data for batch insertion diff --git a/crates/torii-runtime-common/Cargo.toml b/crates/torii-runtime-common/Cargo.toml index 423d5e70..9cbc5ffd 100644 --- a/crates/torii-runtime-common/Cargo.toml +++ b/crates/torii-runtime-common/Cargo.toml @@ -10,6 +10,7 @@ tokio.workspace = true tokio-postgres = "0.7" tracing.workspace = true torii.workspace = true +torii-sql.workspace = true [lints] workspace = true diff --git a/crates/torii-runtime-common/src/database.rs b/crates/torii-runtime-common/src/database.rs index ef7aeec0..ef409403 100644 --- a/crates/torii-runtime-common/src/database.rs +++ b/crates/torii-runtime-common/src/database.rs @@ -1,26 +1,21 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, Result as AnyResult}; use std::path::{Path, PathBuf}; +use torii_sql::connection::DbBackend; pub const DEFAULT_SQLITE_MAX_CONNECTIONS: u32 = 500; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DatabaseBackend { - Postgres, - Sqlite, -} - -pub fn backend_from_url_or_path(value: &str) -> DatabaseBackend { +pub fn backend_from_url_or_path(value: &str) -> DbBackend { if value.starts_with("postgres://") || value.starts_with("postgresql://") { - DatabaseBackend::Postgres + DbBackend::Postgres } else { - DatabaseBackend::Sqlite + DbBackend::Sqlite } } pub fn validate_uniform_backends( named_urls: &[(&str, &str)], mixed_backend_message: &str, -) -> Result { +) -> AnyResult { let Some((_, first_url)) = named_urls.first() else { bail!("at least one database URL must be provided"); }; @@ -78,17 +73,17 @@ pub struct TokenDbSetup { pub erc20_url: String, pub erc721_url: String, pub erc1155_url: String, - pub engine_backend: DatabaseBackend, - pub erc20_backend: DatabaseBackend, - pub erc721_backend: DatabaseBackend, - pub erc1155_backend: DatabaseBackend, + pub engine_backend: DbBackend, + pub erc20_backend: DbBackend, + pub erc721_backend: DbBackend, + pub erc1155_backend: DbBackend, } pub fn resolve_token_db_setup( db_dir: &Path, engine_database_url: Option<&str>, storage_database_url: Option<&str>, -) -> Result { +) -> AnyResult { let engine_url = engine_database_url.map_or_else( || db_dir.join("engine.db").to_string_lossy().to_string(), ToOwned::to_owned, @@ -119,10 +114,10 @@ pub fn resolve_token_db_setup( if engine_database_url .map(backend_from_url_or_path) - .is_some_and(|backend| backend == DatabaseBackend::Postgres) - && (erc20_backend != DatabaseBackend::Postgres - || erc721_backend != DatabaseBackend::Postgres - || erc1155_backend != DatabaseBackend::Postgres) + .is_some_and(|backend| backend == DbBackend::Postgres) + && (erc20_backend != DbBackend::Postgres + || erc721_backend != DbBackend::Postgres + || erc1155_backend != DbBackend::Postgres) { bail!( "Engine is configured for Postgres but one or more token storages resolved to SQLite. Set --storage-database-url to the same Postgres URL." @@ -161,8 +156,8 @@ mod tests { fn resolves_sqlite_defaults() { let db_dir = Path::new("./torii-data"); let setup = resolve_token_db_setup(db_dir, None, None).unwrap(); - assert_eq!(setup.engine_backend, DatabaseBackend::Sqlite); - assert_eq!(setup.erc20_backend, DatabaseBackend::Sqlite); + assert_eq!(setup.engine_backend, DbBackend::Sqlite); + assert_eq!(setup.erc20_backend, DbBackend::Sqlite); assert!(setup.engine_url.ends_with("engine.db")); assert!(setup.erc20_url.ends_with("erc20.db")); } @@ -174,8 +169,9 @@ mod tests { db_dir, Some("postgres://localhost/torii"), Some("./torii-data"), - ) - .expect_err("expected mixed backend validation error"); + ); + println!("{err:?}"); + let err = err.expect_err("expected mixed backend validation error"); assert!(err .to_string() .contains("Engine is configured for Postgres")); @@ -190,8 +186,8 @@ mod tests { Some("postgres://localhost/torii"), ) .unwrap(); - assert_eq!(setup.engine_backend, DatabaseBackend::Postgres); - assert_eq!(setup.erc721_backend, DatabaseBackend::Postgres); + assert_eq!(setup.engine_backend, DbBackend::Postgres); + assert_eq!(setup.erc721_backend, DbBackend::Postgres); } #[test] @@ -204,7 +200,7 @@ mod tests { "mixed backends", ) .unwrap(); - assert_eq!(backend, DatabaseBackend::Postgres); + assert_eq!(backend, DbBackend::Postgres); } #[test] diff --git a/crates/torii-sql-sink/Cargo.toml b/crates/torii-sql-sink/Cargo.toml index 97443de8..24c8b41e 100644 --- a/crates/torii-sql-sink/Cargo.toml +++ b/crates/torii-sql-sink/Cargo.toml @@ -12,7 +12,12 @@ tokio = { version = "1.35", features = ["full"] } async-trait = "0.1" # Database -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite", "postgres", "any"] } +sqlx = { workspace = true, features = [ + "runtime-tokio-rustls", + "sqlite", + "postgres", + "any", +] } # gRPC tonic = "0.12" diff --git a/examples/introspect/restart.rs b/examples/introspect/restart.rs index 1221bd2d..e02f1b53 100644 --- a/examples/introspect/restart.rs +++ b/examples/introspect/restart.rs @@ -1,32 +1,50 @@ use itertools::Itertools; -use sqlx::{postgres::PgPoolOptions, PgPool}; -use std::sync::Arc; +use starknet::core::types::Felt; use torii_dojo::decoder::DojoDecoder; -use torii_dojo::store::postgres::PgStore; +use torii_dojo::store::DojoStoreTrait; use torii_dojo::DojoToriiError; -use torii_introspect_postgres_sink::IntrospectPgDb; -use torii_test_utils::{resolve_path_like, EventIterator, FakeProvider}; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; +use torii_introspect_sql_sink::IntrospectDb; +use torii_sql::{DbPool, PoolConfig}; +use torii_test_utils::{resolve_path_like, FakeProvider, MultiContractEventIterator}; -const DB_URL: &str = "postgres://torii:torii@localhost:5432/torii"; -// const CHAIN_DATA_PATH: &str = "~/tc-tests/pistols-2"; -// const SCHEMA_NAME: &str = "pistols"; -const CHAIN_DATA_PATH: &str = "~/tc-tests/blob-arena-2"; -const SCHEMA_NAME: &str = "blob_arena"; -const BATCH_SIZE: usize = 10000; +// const DB_URL: &str = "postgres://torii:torii@localhost:5432/torii"; +const DB_URL: &str = "sqlite://sqlite-data.db?mode=rwc"; +const EVENT_PATHS: [&str; 2] = ["~/tc-tests/blob-arena/events", "~/tc-tests/pistols/events"]; +const MODEL_CONTRACTS_PATH: &str = "~/tc-tests/model-contracts"; +const BATCH_SIZE: usize = 2000; +const PISTOLS_ADDRESS: Felt = + Felt::from_hex_unchecked("08b4838140a3cbd36ebe64d4b5aaf56a30cc3753c928a79338bf56c53f506c5"); +const BLOB_ARENA_ADDRESS: Felt = + Felt::from_hex_unchecked("2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb"); + +// const SCHEMA_MAP: [(Felt, &str); 2] = [ +// (PISTOLS_ADDRESS, "pistols"), +// (BLOB_ARENA_ADDRESS, "blob_arena"), +// ]; + +const ADDRESSES: [Felt; 2] = [PISTOLS_ADDRESS, BLOB_ARENA_ADDRESS]; async fn run_events( - events: &mut EventIterator, + events: &mut MultiContractEventIterator, provider: FakeProvider, - pool: Arc, + pool: DbPool, end: Option, event_n: &mut u32, success: &mut u32, ) -> bool { - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - let db = IntrospectPgDb::new(pool.clone(), SCHEMA_NAME); - decoder.store.initialize().await.unwrap(); + println!("Starting event processing run"); + let decoder = DojoDecoder::new(pool.clone(), provider); + let db = IntrospectDb::new(pool, ADDRESSES); + decoder.initialize().await.unwrap(); decoder.load_tables(&[]).await.unwrap(); - db.initialize_introspect_pg_sink().await.unwrap(); + let errors = db.initialize_introspect_sql_sink().await.unwrap(); + if !errors.is_empty() { + for err in errors { + println!("Error loading table: {err}"); + } + panic!(""); + } let mut running = true; let mut this_run = 0; while running { @@ -39,6 +57,13 @@ async fn run_events( *event_n += 1; this_run += 1; match decoder.decode_raw_event(&event).await { + Ok(IntrospectBody { + metadata, + msg: IntrospectMsg::CreateTable(mut msg), + }) => { + msg.append_only = true; + msgs.push((IntrospectMsg::CreateTable(msg), metadata).into()); + } Ok(msg) => { msgs.push(msg); } @@ -50,6 +75,7 @@ async fn run_events( } }; } + let msgs_ref = msgs.iter().collect_vec(); for res in db.process_messages(msgs_ref).await.unwrap() { match res { @@ -72,19 +98,21 @@ async fn run_events( #[tokio::main] async fn main() { - let chain_path = resolve_path_like(CHAIN_DATA_PATH); - let events_path = chain_path.join("events"); - let contracts_path = chain_path.join("model-contracts"); - let provider = FakeProvider::new(contracts_path); - let mut event_iterator = EventIterator::new(events_path); - let pool = Arc::new(PgPoolOptions::new().connect(DB_URL).await.unwrap()); + let event_paths = EVENT_PATHS.map(resolve_path_like).to_vec(); + let provider = FakeProvider::new(resolve_path_like(MODEL_CONTRACTS_PATH)); + let mut event_iterator = MultiContractEventIterator::new(event_paths); + let pool = PoolConfig::new(DB_URL.to_string()) + .max_connections(5) + .connect_any() + .await + .unwrap(); let mut event_n = 0; let mut success = 0; while run_events( &mut event_iterator, provider.clone(), pool.clone(), - Some(500000), + Some(20000), &mut event_n, &mut success, ) diff --git a/examples/introspect/simple.rs b/examples/introspect/simple.rs deleted file mode 100644 index cbe2b205..00000000 --- a/examples/introspect/simple.rs +++ /dev/null @@ -1,65 +0,0 @@ -use itertools::Itertools; -use sqlx::postgres::PgPoolOptions; -use std::sync::Arc; -use torii_dojo::decoder::DojoDecoder; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::DojoToriiError; -use torii_introspect_postgres_sink::IntrospectPgDb; -use torii_test_utils::{resolve_path_like, EventIterator, FakeProvider}; - -const DB_URL: &str = "postgres://torii:torii@localhost:5432/torii"; -const CHAIN_DATA_PATH: &str = "~/tc-tests/pistols"; -const SCHEMA_NAME: &str = "pistols"; -// const CHAIN_DATA_PATH: &str = "~/tc-tests/blob-arena"; -// const SCHEMA_NAME: &str = "blob_arena"; -const BATCH_SIZE: usize = 1000; - -#[tokio::main] -async fn main() { - let chain_path = resolve_path_like(CHAIN_DATA_PATH); - let events_path = chain_path.join("events"); - let contracts_path = chain_path.join("model-contracts"); - let provider = FakeProvider::new(contracts_path); - let mut event_iterator = EventIterator::new(events_path); - - let pool = Arc::new(PgPoolOptions::new().connect(DB_URL).await.unwrap()); - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - let db = IntrospectPgDb::new(pool.clone(), SCHEMA_NAME); - decoder.store.initialize().await.unwrap(); - db.initialize_introspect_pg_sink().await.unwrap(); - - let mut event_n = 0; - let mut success = 0; - let mut running = true; - while running { - let mut msgs = Vec::with_capacity(BATCH_SIZE); - for _ in 0..BATCH_SIZE { - let Some(event) = event_iterator.next() else { - running = false; - break; - }; - event_n += 1; - match decoder.decode_raw_event(&event).await { - Ok(msg) => { - msgs.push(msg); - } - Err(DojoToriiError::UnknownDojoEventSelector(_)) => { - println!("Unknown event selector, skipping event"); - } - Err(err) => { - println!("Failed to decode event: {err:?}"); - } - }; - } - let msgs_ref = msgs.iter().collect_vec(); - for res in db.process_messages(msgs_ref).await.unwrap() { - match res { - Err(err) => println!("Failed to process message: {err:?}"), - Ok(()) => success += 1, - } - } - println!( - "Processed batch of events, total events processed: {event_n}, successful: {success}" - ); - } -} diff --git a/examples/pathfinder/main.rs b/examples/pathfinder/main.rs new file mode 100644 index 00000000..57bf5370 --- /dev/null +++ b/examples/pathfinder/main.rs @@ -0,0 +1,23 @@ +use torii_pathfinder::{connect, EventFetcher}; + +const DB_PATH: &str = "/mnt/store/mainnet.sqlite"; + +const BATCH_SIZE: u64 = 10000; + +fn main() { + let conn = connect(DB_PATH).unwrap(); + let mut current_block = 6000000; + for _ in 0..100 { + let (blocks, events) = conn + .get_emitted_events_with_context(current_block, current_block + BATCH_SIZE - 1) + .expect("failed to fetch events with context"); + println!( + "Fetched {} blocks and {} events for blocks {} to {}", + blocks.len(), + events.len(), + current_block, + current_block + BATCH_SIZE - 1 + ); + current_block += BATCH_SIZE; + } +} diff --git a/sqlite-data.db-journal b/sqlite-data.db-journal new file mode 100644 index 00000000..51827734 Binary files /dev/null and b/sqlite-data.db-journal differ diff --git a/src/etl/engine_db.rs b/src/etl/engine_db.rs index 94215a7d..d3e9e93f 100644 --- a/src/etl/engine_db.rs +++ b/src/etl/engine_db.rs @@ -4,7 +4,9 @@ //! This will be enhanced with actual Torii features in the future. use anyhow::{Context, Result}; -use sqlx::{any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, Row}; +use sqlx::any::AnyPoolOptions; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{Any, ConnectOptions, Pool, Row}; use starknet::core::types::Felt; use std::collections::HashMap; use std::str::FromStr;