diff --git a/.agents/skills/torii-maintainer/SKILL.md b/.agents/skills/torii-maintainer/SKILL.md new file mode 100644 index 00000000..354aaab7 --- /dev/null +++ b/.agents/skills/torii-maintainer/SKILL.md @@ -0,0 +1,393 @@ +--- +name: torii-maintainer +description: "Use when working on Torii (Starknet indexer): adding or debugging a sink / decoder / extractor / identification rule / command handler, tracing the ETL loop, cursor commit, EventBus / CommandBus / SubscriptionManager flow, gRPC / HTTP surface, contract identification, or introspect / dojo / token pipelines. Keywords: torii, ETL, sink, decoder, extractor, envelope, event bus, command bus, engine db, contract identifier, identification rule, introspect, dojo, erc20, erc721, erc1155, arcade, starknet indexer, cursor commit, prefetch queue." +globs: ["**/Cargo.toml", "src/etl/**", "crates/**"] +user-invocable: true +--- + +# torii-maintainer + +Deep architectural knowledge for the Torii Starknet indexer. Every +per-crate README exists to fill in the details; this skill is the map. + +## 1. When to use this skill + +Trigger on any of: + +- "writing a new sink / decoder / extractor / identification rule / command handler" +- "events lost after restart" / "cursor seems wrong" / "duplicate processing" +- "sink never receives envelope" / "decoder returns nothing" / "decoder fallback is slow" +- "auto-identification cache miss" / "`torii_registry_identify_jobs_total{status='dropped'}` is increasing" +- "gRPC 404 after adding a service" / "reflection does not see my service" +- "adding TLS / a topic / a REST route / a Prometheus metric" +- anything touching `src/lib.rs::run`, `src/etl/...`, `crates/torii-*-sink/`, `crates/dojo/`, `crates/introspect*/`, `crates/torii-erc{20,721,1155}/`. + +Do **not** use for: generic Rust questions (prefer `m01-ownership`, `m06-error-handling`, `m07-concurrency`, `m11-ecosystem`), CLI UX (`domain-cli`), or web design (`domain-web`). + +## 2. System map + +```text + ┌─────────────────────┐ + │ bins/*/main.rs │ + │ (build ToriiConfig │ + │ via builder) │ + └──────────┬──────────┘ + │ + v + ┌────────────────────────────────────────────────────────────────────┐ + │ torii::run(config) (src/lib.rs:613) │ + │ │ + │ Task 1 — producer Task 2 — identifier Task 3 — consumer│ + │ extractor.extract() identify_contracts prefetch_rx │ + │ → mpsc (optional) → decoder │ + │ capacity = on contract_addresses context │ + │ max_prefetch_batches .decode_events│ + │ → multi_sink │ + │ .process │ + │ → commit_cursor + │ │ + │ side channels: │ + │ EventBus ─ SubscriptionManager ─ gRPC clients │ + │ CommandBus ─ CommandHandlers (async, fire-and-forget) │ + │ │ + │ persistence: │ + │ EngineDb (head, cursor, contract→decoder cache) │ + │ │ + │ surface: │ + │ axum HTTP /health /metrics + sink-merged routes │ + │ tonic gRPC torii.Torii + sink services + reflection │ + │ optional TLS listener │ + └────────────────────────────────────────────────────────────────────┘ +``` + +**Cursor-commit invariant** (`src/lib.rs:1113-1127`): the extractor's +cursor is committed **only after** `multi_sink.process(...)` returns +`Ok`. A crash mid-batch replays the last window; it never loses it. + +## 3. Trait contracts + +All four traits live in the root `torii` crate's `src/etl/`. Each sink or +decoder you ship impls one or more of these. + +### `Sink` — `src/etl/sink/mod.rs:74` + +```rust +#[async_trait] +pub trait Sink: Send + Sync { + fn name(&self) -> &str; + fn interested_types(&self) -> Vec; + async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> anyhow::Result<()>; + fn topics(&self) -> Vec; + fn build_routes(&self) -> Router; + async fn initialize(&mut self, event_bus: Arc, context: &SinkContext) -> anyhow::Result<()>; +} +``` + +Invariants: + +- `interested_types` is **advisory** today — the `Sink::process` call receives *every* envelope, not just matching ones. Filter in `process` by `envelope.type_id`. +- `topics` is consumed at startup by `GrpcState` (`src/grpc.rs:134`) and published via `ListTopics`. +- `build_routes` output is `.merge`d into the main HTTP router by `MultiSink::build_routes` (`src/etl/sink/multi.rs:87`). +- `initialize` sees the `EventBus` + `SinkContext`. **This is your only chance to capture them by `Arc` for later use** — after `MultiSink` wraps the sink in `Arc` you lose mutable access. See `src/lib.rs:645` for the call site. +- `SinkContext { database_root, command_bus }` (`src/etl/sink/mod.rs:23`) — use `context.command_bus` to dispatch work to `CommandHandler`s. + +### `Decoder` — `src/etl/decoder/mod.rs:96` + +```rust +#[async_trait] +pub trait Decoder: Send + Sync { + fn decoder_name(&self) -> &str; + async fn decode(&self, keys: &[Felt], data: &[Felt], context: EventContext) -> anyhow::Result>; + async fn decode_event(&self, event: &StarknetEvent) -> anyhow::Result> { /* default */ } + async fn decode_events(&self, events: &[StarknetEvent]) -> anyhow::Result> { /* default */ } +} +``` + +Invariants: + +- `decoder_name` produces a stable `DecoderId` via `xxh3_64`. **Never** change the name of a decoder in production — the registry cache is keyed on it and changing it leaves stale mappings referencing nothing (the routing logic in `DecoderContext::decode` evicts them, but you pay a full re-identification round per contract). +- Each decoder should return *only* envelopes it wants to produce. Returning an empty `Vec` is the "not interested" signal. +- `EventContext` = `{ from_address, block_number, transaction_hash }`; add more via `envelope.metadata` if a sink needs it. + +### `Extractor` — `src/etl/extractor/mod.rs:299` + +```rust +#[async_trait] +pub trait Extractor: Send + Sync { + fn set_start_block(&mut self, start_block: u64); + async fn extract(&mut self, cursor: Option, engine_db: &EngineDb) -> Result; + fn is_finished(&self) -> bool; + async fn commit_cursor(&mut self, _cursor: &str, _engine_db: &EngineDb) -> Result<()> { Ok(()) } + fn as_any(&self) -> &dyn std::any::Any; +} +``` + +Return-value semantics: + +- Non-empty batch: process and call again. +- Empty batch + `is_finished() = false`: at the head; producer sleeps `cycle_interval` (see `src/lib.rs:976`) before retrying. +- Empty batch + `is_finished() = true`: shutdown ETL loop (`src/lib.rs:1035`). + +### `ContractIdentifier` — `src/etl/identification/registry.rs:32` + +```rust +#[async_trait] +pub trait ContractIdentifier: Send + Sync { + async fn identify_contracts(&self, contract_addresses: &[Felt]) -> Result>>; + fn shared_cache(&self) -> Arc>>>; +} +``` + +The only production impl is `ContractRegistry` (`src/etl/identification/registry.rs:64`), which: +- Batches RPC calls (`MAX_BATCH_SIZE = 500`) via `JsonRpcClient::batch_requests`. +- Keeps two caches: positive (`Arc>>>`) and bounded negative (`NegativeCache`, capacity `100_000`). +- Persists positives to `EngineDb` via `set_contract_decoders_batch`; loads them on startup via `load_from_db`. + +### `IdentificationRule` — `src/etl/identification/rule.rs:52` + +```rust +pub trait IdentificationRule: Send + Sync { + fn name(&self) -> &str; + fn decoder_ids(&self) -> Vec; + fn identify_by_abi(&self, contract_address: Felt, class_hash: Felt, abi: &ContractAbi) -> Result>; +} +``` + +One rule per contract shape. See `crates/torii-erc20/src/identification.rs` for the reference impl. + +### `CommandHandler` — `src/command.rs:35` + +```rust +#[async_trait] +pub trait CommandHandler: Send + Sync { + fn supports(&self, command: &dyn Command) -> bool; + fn attach_event_bus(&self, _event_bus: Arc) {} + async fn handle_command(&self, command: Box) -> Result<()>; +} +``` + +The `CommandBus` (`src/command.rs:148`) is a bounded mpsc that dispatches `Box` to the **single** matching handler. Ambiguous matches are rejected (`CommandDispatchError::Ambiguous`). + +## 4. ETL loop anatomy + +`src/lib.rs::run` (line 613 → 1293) spawns three tokio tasks: + +1. **Producer** (`src/lib.rs:856-980`) — loops `extractor.extract(cursor, engine_db)`, pushes `PrefetchedBatch { batch, cursor, extractor_finished }` into an `mpsc` of capacity `etl_concurrency.max_prefetch_batches`. Also enqueues unique `contract_addresses` onto the identifier's channel (non-blocking `try_send`; full queue → `torii_registry_identify_jobs_total{status=dropped}`). +2. **Identifier** (`src/lib.rs:824-848`) — optional; present only if `ToriiConfigBuilder::with_contract_identifier` was called. Consumes addresses and calls `ContractIdentifier::identify_contracts`. Populates the registry cache; `DecoderContext::decode` reads through it on every event. +3. **Consumer** (`src/lib.rs:985-1141`) — awaits a `PrefetchedBatch`, decodes via `DecoderContext.decode_events`, processes via `MultiSink.process`, then **only on success** calls `extractor.commit_cursor`. + +**Graceful shutdown** (`src/lib.rs:1156-1269`): +- `CancellationToken` flipped by SIGINT/SIGTERM. +- Server gets 15 s to drain (`SERVER_SHUTDOWN_TIMEOUT_SECS = 15`). +- ETL loop gets `config.shutdown_timeout` s (default 30) to finish the in-flight batch. + +## 5. EventBus, CommandBus, SubscriptionManager + +| Component | File | Purpose | +|---|---|---| +| `SubscriptionManager` | `src/grpc.rs:42` | `Arc>>` — one entry per connected gRPC client | +| `ClientSubscription` | `src/grpc.rs:31` | `{ topics: HashMap, tx: mpsc::Sender }` | +| `EventBus` | `src/etl/sink/mod.rs:188` | Sink-facing façade over `SubscriptionManager`; exposes `publish_protobuf(topic, type_id, data, decoded, update_type, filter_fn)` | +| `GrpcState` | `src/grpc.rs:134` | Carries `subscription_manager` + `topics: Vec` | +| `ToriiService` | `src/grpc.rs:156` | `torii.Torii` service (`GetVersion`, `ListTopics`, `SubscribeToTopicsStream`, `SubscribeToTopics`) | +| `CommandBus` / `CommandBusSender` | `src/command.rs:148 / :71` | Bounded mpsc, routes `Box` to matching `CommandHandler` | + +Key routing rules: + +- `EventBus::publish_protobuf` takes both `&Any` and `&T decoded`; the `filter_fn: Fn(&T, &filters) -> bool` runs *once per connected client* — keep it cheap. This avoids re-decoding the payload per client. +- A topic is visible to `ListTopics` only if a registered sink includes it in `topics()`. +- `SubscriptionManager::update_subscriptions` is additive for new topics and honours `unsubscribe_topics` for removal. +- `CommandBus::dispatch` fails if zero or >1 handlers match (`CommandDispatchError::Unsupported | Ambiguous`). + +## 6. Storage stack + +Three layers; don't confuse them: + +- **`EngineDb`** (`src/etl/engine_db.rs:33`) — cursor, head, extractor state, contract→decoder cache. One per binary. `sqlx::Pool` (not `torii-sql`). Embedded schema under `sql/engine_schema*.sql`. +- **`torii-sql`** (`crates/sql/`) — shared abstraction for every *sink's* DB. `DbBackend::{Postgres, Sqlite}`, `PoolExt`, `SchemaMigrator`. Token sinks + `introspect-sql-sink` + `torii-controllers-sink` + `torii-ecs-sink` all route through it. +- **Sink-private storage** — `Erc20Storage`, `Erc721Storage`, `Erc1155Storage`, `ControllersStore`, `IntrospectDb`. Each owns its own schema and connection pool. + +Binary plumbing to resolve URLs: `torii-runtime-common::database::{resolve_single_db_setup, resolve_token_db_setup}` and `backend_from_url_or_path`. **Never mix** a Postgres engine with SQLite storage — `resolve_token_db_setup` rejects that combination. + +## 7. Sink authoring recipe + +1. Define a body type implementing `TypedBody` (or use the `typed_body_impl!` macro from `src/etl/envelope.rs`). +2. Decide the `TypeId` — call `TypeId::new("your_sink")`; make it a `const` at the top of the file. +3. Implement `Decoder` if your sink needs one (see `crates/torii-erc20/src/decoder.rs`). +4. Implement `Sink`: + - `name()` returns a short stable name. + - `interested_types()` lists your `TypeId`(s). + - `process(envelopes, batch)` — downcast matching envelopes, persist, optionally `EventBus::publish_protobuf` and/or `grpc_service.broadcast`. Use `batch.is_live(threshold)` to skip broadcasting during historical backfill. + - `topics()` returns the topic descriptors (`TopicInfo::new`). Keep filter keys stable. + - `build_routes()` returns an `axum::Router`. + - `initialize()` captures `event_bus` + `context.command_bus` into `self`. +5. If you have a gRPC service, expose it via `get_grpc_service_impl(&self) -> Arc` so binaries can mount it via `ToriiConfigBuilder::with_grpc_router(...)` before calling `torii::run`. Reference: `crates/torii-sql-sink/README.md`. +6. If you have custom reflection, set `with_custom_reflection(true)` and register both `TORII_DESCRIPTOR_SET` and your `FILE_DESCRIPTOR_SET` bytes in a `tonic_reflection::server::Builder`. + +## 8. Decoder authoring recipe + +1. Pick a stable `decoder_name` (e.g. `"erc20"`); the `DecoderId` is `xxh3_64(name)` — keep the name **forever**. +2. In `decode(keys, data, ctx)`: + - If the event's first key isn't one you care about, `return Ok(Vec::new())`. + - Deserialise; return one or more `Envelope`s via `EventMsg::to_envelope(ctx)` (see `src/etl/envelope.rs:118`). +3. Register with `ToriiConfigBuilder::add_decoder(Arc::new(MyDecoder::new()))`. +4. **For auto-discovery**: implement `IdentificationRule` returning the same `DecoderId::new("erc20")` and register the rule on a `ContractRegistry`. +5. **For performance**: encourage binaries to populate `ContractFilter::map_contract(addr, vec![decoder_id])` for known contracts. Routing priority in `DecoderContext::decode` (`src/etl/decoder/context.rs:313`): blacklist → explicit mapping → registry cache → fallback (try all decoders). + +## 9. Binary authoring recipe + +Every binary follows this skeleton — see `bins/torii-erc20/src/main.rs` for the minimal form: + +1. `clap::Parser` config → validate via `torii-config-common`. +2. Resolve DB setup via `torii-runtime-common::database::resolve_{single,token}_db_setup`. +3. Construct provider (`JsonRpcClient`) and extractor. +4. Create storage + sink + gRPC service; capture `Arc` for router assembly. +5. Create decoder(s) + `IdentificationRule`(s); build a `ContractRegistry` and call `load_from_db().await`. +6. Build `tonic::transport::server::Router` with all sink gRPC services + optional reflection. +7. Build `ToriiConfig`: + - `.add_sink_boxed(Box::new(sink))` + - `.add_decoder(Arc::new(decoder))` + - `.with_grpc_router(grpc_router)` + `.with_custom_reflection(true)` if you composed reflection yourself + - `.with_contract_identifier(registry)` for auto-discovery + - `.with_command_handler(Box::new(handler))` for async work + - `.with_tls(ToriiTlsConfig::new(cert, key))` if needed + - `.etl_concurrency(EtlConcurrencyConfig { max_prefetch_batches })` +8. `torii::run(config).await`. + +**Synthetic pair**: for every binary that backs a production workload there is usually a `-synth` sibling (`bins/torii-erc20-synth`, `bins/torii-tokens-synth`, `bins/torii-introspect-synth`) that swaps the RPC extractor for a deterministic `SyntheticExtractor` and writes a `RunReport` JSON. Use those for perf regression testing; do **not** add new knobs to the production binary for profiling purposes. + +## 10. Bug classes & runbooks + +### 10.1 "Events missing after restart / duplicate processing" +- Check `torii_cursor_commit_failures_total`. If non-zero, look at the `extractor.commit_cursor` error in logs. +- Re-read the invariant: cursor is committed *after* `multi_sink.process` succeeds. If a sink panics, the cursor is *not* advanced → batch replays. +- If a sink has side-effects (e.g. sending to an external API) and you require exactly-once, add idempotency on the sink side — the pipeline gives at-least-once for side-effects between extract and cursor-commit. + +### 10.2 "Sink never receives an envelope" +- Confirm the decoder's `decode()` actually returns an envelope for the event. Start at `src/etl/decoder/context.rs:313` and trace which branch is taken (blacklist / mapping / registry / fallback). +- Match `TypeId` — the `envelope.type_id` must match one of `sink.interested_types()`. Mismatched `TypeId::new("…")` strings are the most common cause. +- If the contract isn't in an explicit mapping and no registry is configured, only the **fallback all-decoders** path runs. See `has_registry` in `DecoderContext` (`context.rs:49`). + +### 10.3 "Decoder fallback is slow" +- With many decoders, fallback tries all of them serially per event. Fix by registering a `ContractRegistry` + `IdentificationRule` to cache the mapping after first contact, or by adding an explicit `ContractFilter::map_contract`. +- `torii_registry_identify_duration_seconds` histogram shows how long identification itself takes; `torii_rpc_chunk_duration_seconds{extractor="registry"}` shows the RPC portion. + +### 10.4 "Identification queue full / dropped" +- `torii_registry_identify_jobs_total{status="dropped"}` increasing means the producer's `identify_tx.try_send` is failing on a full channel. +- Increase `etl_concurrency.max_prefetch_batches` (the identify queue is sized at `prefetch_capacity.saturating_mul(2).max(8)`) or reduce batch size. + +### 10.5 "gRPC 404 after adding a service" +- You must pass the router to Torii via `.with_grpc_router(router)`; Torii will mount the core service into it. If you also composed reflection yourself, set `.with_custom_reflection(true)` or you'll get duplicate-service errors. +- Confirm the service descriptor set is fed to `tonic_reflection` — otherwise `grpcurl list` won't show it. + +### 10.6 "Sink panics abort the pipeline" +- They don't. `MultiSink::process` (`src/etl/sink/multi.rs:51-66`) logs the error, increments `torii_sink_failures_total{sink=""}`, but continues. The cursor is **not** advanced because the multi-sink returns its own error to the ETL loop if *any* sink failed. +- If you need all-or-nothing semantics, wrap your sink to convert transient errors into retries, or sequence multiple sinks behind an ordered façade (see `bins/torii-arcade` / `bins/torii-introspect-bin`). + +### 10.7 "EngineDb locked / PG pool exhausted" +- Engine DB is a single pool with `max_connections = 5` for real SQLite files and `1` for `:memory:` (`src/etl/engine_db.rs:80-88`). Do not hold an `EngineDb` transaction across `extract` / `decode` / `process`. +- Postgres sinks pool separately; check `max_db_connections` in binary configs. + +### 10.8 "TLS listener won't start" +- Both `--tls-cert` and `--tls-key` must be present (enforced in `torii-introspect-bin/src/config.rs` and `torii-arcade/src/config.rs`). The PEM files are re-loaded at startup only; hot reload is not supported. + +### 10.9 "Can't change decoder name without breaking clients" +- The `DecoderId` hash is persisted in the engine DB. If you change `decoder_name`, on next startup `DecoderContext::decode` will see stale cache entries pointing at an unknown `DecoderId`, evict them (`context.rs:341-360`), and fall back to all-decoders for the affected contracts until re-identified. This is safe but slow — renaming a decoder costs a full re-identification pass. + +## 11. Observability reference + +Emitted from `src/lib.rs::run` unless otherwise noted. All use the `metrics` facade backed by `metrics-exporter-prometheus`. + +| Metric | Type | Labels | Meaning | +|---|---|---|---| +| `torii_etl_cycle_total` | counter | `status={ok,empty,decode_error,sink_error,extract_error}` | One increment per ETL cycle | +| `torii_etl_cycle_duration_seconds` | histogram | — | End-to-end cycle latency | +| `torii_etl_cycle_gap_blocks` | gauge | — | `chain_head - last_block` | +| `torii_etl_last_success_timestamp_seconds` | gauge | — | Last successful batch wall time | +| `torii_etl_inflight_cycles` | gauge | — | 0 or 1 | +| `torii_etl_prefetch_queue_depth` | gauge | — | Prefetched batches waiting | +| `torii_etl_prefetch_stall_seconds` | histogram | — | Producer vs consumer pressure | +| `torii_events_extracted_total` | counter | — | Raw events out of the extractor | +| `torii_events_decoded_total` | counter | — | Events that reached decode | +| `torii_decode_envelopes_total` | counter | — | Envelopes produced | +| `torii_events_processed_total` | counter | — | Events after sink processing | +| `torii_transactions_processed_total` | counter | — | Transactions after sink processing | +| `torii_extract_batch_size_total` | counter | `unit={events,blocks,transactions}` | Batch fan-out | +| `torii_decode_failures_total` | counter | `stage` | Decode errors | +| `torii_cursor_commit_failures_total` | counter | — | Commit failures (data-safety canary) | +| `torii_sink_process_duration_seconds` | histogram | `sink` | Per-sink latency | +| `torii_sink_failures_total` | counter | `sink` | Per-sink failure count | +| `torii_registry_identify_duration_seconds` | histogram | — | Identification worker cycle | +| `torii_registry_identify_jobs_total` | counter | `status={enqueued,dropped,closed}` | Identify-queue health | +| `torii_rpc_chunk_duration_seconds` | histogram | `extractor`, `method` | Chunked RPC latency | +| `torii_rpc_parallelism` | gauge | — | Configured concurrency | +| `torii_command_dispatch_total` | counter | `command`, `status={enqueued,unsupported,ambiguous,dropped_full,closed}` | CommandBus dispatch outcomes | +| `torii_command_handle_total` | counter | `command`, `status={ok,error}` | Handler outcomes | +| `torii_command_handle_duration_seconds` | histogram | `command` | Handler latency | +| `torii_observability_enabled` | gauge | — | Seeded to 1 when metrics are on | +| `torii_uptime_seconds` | gauge | — | Set from `/health` | +| `torii_build_info` | gauge | `version` | Build info beacon | + +Enable / disable via `TORII_METRICS_ENABLED` (`src/metrics.rs:20`). + +## 12. Crate map + +Root library: + +- [`src/README.md`](../../../src/README.md) — `torii` (root library). Owns `ToriiConfig`, `run()`, `EngineDb`, `DecoderContext`, `MultiSink`, the three tokio tasks, CommandBus, SubscriptionManager. + +Shared / core: + +- [`crates/torii-common/README.md`](../../../crates/torii-common/README.md) — U256/Felt blob codecs, metadata + token-URI helpers. +- [`crates/types/README.md`](../../../crates/types/README.md) — `StarknetEvent`, `EventContext`, `BlockContext`. +- [`crates/torii-config-common/README.md`](../../../crates/torii-config-common/README.md) — CLI validation helpers. +- [`crates/torii-runtime-common/README.md`](../../../crates/torii-runtime-common/README.md) — DB setup + sink init helpers. +- [`crates/sql/README.md`](../../../crates/sql/README.md) — `torii-sql` (SQLite + Postgres abstraction). +- [`crates/testing/README.md`](../../../crates/testing/README.md) — `torii-test-utils` (event fixtures + FakeProvider). +- [`crates/pathfinder/README.md`](../../../crates/pathfinder/README.md) — Pathfinder SQLite reader + optional `etl` extractor. + +Sinks: + +- [`crates/torii-sql-sink/README.md`](../../../crates/torii-sql-sink/README.md) — **reference sink**; SQL operations + gRPC + HTTP + EventBus. +- [`crates/torii-log-sink/README.md`](../../../crates/torii-log-sink/README.md) — in-memory log ring, the minimum viable sink. +- [`crates/torii-controllers-sink/README.md`](../../../crates/torii-controllers-sink/README.md) — GraphQL-driven Cartridge controller sync. +- [`crates/arcade-sink/README.md`](../../../crates/arcade-sink/README.md) — Arcade projections from Dojo introspect events. +- [`crates/torii-ecs-sink/README.md`](../../../crates/torii-ecs-sink/README.md) — legacy `world.World` gRPC facade. + +Tokens: + +- [`crates/torii-erc20/README.md`](../../../crates/torii-erc20/README.md) — full ERC20 stack (decoder + rule + sink + service). +- [`crates/torii-erc721/README.md`](../../../crates/torii-erc721/README.md) — full ERC721 stack; owner-of tracking + EIP-4906. +- [`crates/torii-erc1155/README.md`](../../../crates/torii-erc1155/README.md) — full ERC1155 stack; transfer-history-first. + +Domain / introspect: + +- [`crates/dojo/README.md`](../../../crates/dojo/README.md) — `DojoDecoder`, `DojoEvent`, external-contract registration. +- [`crates/introspect/README.md`](../../../crates/introspect/README.md) — `IntrospectMsg` (13 schema-mutation variants). +- [`crates/introspect-sql-sink/README.md`](../../../crates/introspect-sql-sink/README.md) — Postgres / SQLite materializer for introspect events. + +Binaries: + +- [`bins/torii-erc20/README.md`](../../../bins/torii-erc20/README.md) — canonical minimal binary. +- [`bins/torii-erc20-synth/README.md`](../../../bins/torii-erc20-synth/README.md) — ERC20 perf harness (Postgres only). +- [`bins/torii-tokens/README.md`](../../../bins/torii-tokens/README.md) — unified ERC20 + ERC721 + ERC1155. +- [`bins/torii-tokens-synth/README.md`](../../../bins/torii-tokens-synth/README.md) — multi-standard perf harness with subcommands. +- [`bins/torii-introspect-bin/README.md`](../../../bins/torii-introspect-bin/README.md) — full-stack Dojo + tokens (`torii-server`). +- [`bins/torii-introspect-synth/README.md`](../../../bins/torii-introspect-synth/README.md) — Dojo introspect perf harness. +- [`bins/torii-arcade/README.md`](../../../bins/torii-arcade/README.md) — Cartridge Arcade backend. + +## Verification cues before acting on memory + +Before recommending a file:line referenced above, verify it still exists: + +```bash +rg -n "^pub trait Sink\\b" src/etl/sink/mod.rs +rg -n "^pub trait Decoder\\b" src/etl/decoder/mod.rs +rg -n "^pub trait Extractor\\b" src/etl/extractor/mod.rs +rg -n "^pub trait ContractIdentifier" src/etl/identification/registry.rs +rg -n "^pub trait IdentificationRule" src/etl/identification/rule.rs +rg -n "pub async fn run" src/lib.rs +``` + +If any of these no longer match, the skill is drifting — open the +per-crate README to re-anchor before editing. diff --git a/.rustfmt.toml b/.rustfmt.toml index 5207c2aa..d01043dc 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -7,3 +7,5 @@ use_small_heuristics = "Default" # Stable options only reorder_imports = true reorder_modules = true + +imports_granularity = "Module" diff --git a/AGENTS.md b/AGENTS.md index e828a058..388800db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,5 @@ -[Agent Skills Index]|root: ./agents|IMPORTANT: Prefer retrieval-led reasoning over pre-training for any tasks covered by skills.|skills|create-a-plan:{create-a-plan.md},create-pr:{create-pr.md} +[Agent Skills Index]|root: ./agents|IMPORTANT: Prefer retrieval-led reasoning over pre-training for any tasks covered by skills.|skills|create-a-plan:{create-a-plan.md},create-pr:{create-pr.md},torii-maintainer:{SKILL.md} # Agent Instructions diff --git a/Cargo.lock b/Cargo.lock index 16dbbecd..0e4c899f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -58,9 +58,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alphanumeric-sort" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ffdfeac16e9b4d75e41225dc2545d9c2082a0634b5d7f6f70e168546eecb1" +checksum = "d7789d0c06f5a946e0d860f8a3cd055dd714d44657c16116cac8ee47da03f280" [[package]] name = "android_system_properties" @@ -145,12 +145,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" -[[package]] -name = "assert_matches" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" - [[package]] name = "async-stream" version = "0.3.6" @@ -481,9 +475,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.58" +version = "1.2.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1e928d4b69e3077709075a938a05ffbedfa53a84c8f766efbf8220bb1ff60e1" +checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" dependencies = [ "find-msvc-tools", "jobserver", @@ -616,9 +610,9 @@ dependencies = [ [[package]] name = "cmov" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de0758edba32d61d1fd9f4d69491b47604b91ee2f7e6b33de7e54ca4ebe55dc3" +checksum = "3f88a43d011fc4a6876cb7344703e297c71dda42494fee094d5f7c76bf13f746" [[package]] name = "colorchoice" @@ -878,9 +872,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1005a6d4446f5120ef475ad3d2af2b30c49c2c9c6904258e3bb30219bebed5e4" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" dependencies = [ "cmov", ] @@ -919,18 +913,6 @@ dependencies = [ "syn", ] -[[package]] -name = "darrentsung_debug_parser" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf488eca7807ce3c8e64bee95c3fbf8f1935c905b3b73835e75db16fc458fdc4" -dependencies = [ - "anyhow", - "html-escape", - "nom", - "ordered-float", -] - [[package]] name = "data-encoding" version = "2.10.0" @@ -967,12 +949,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "diff" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" - [[package]] name = "digest" version = "0.10.7" @@ -1031,7 +1007,7 @@ dependencies = [ [[package]] name = "dojo-introspect" version = "0.1.0" -source = "git+https://github.com/dojoengine/dojo-introspect?rev=aadc3c9#aadc3c980706596a4a083413813a0a3ab01fded7" +source = "git+https://github.com/dojoengine/dojo-introspect?rev=c45d711#c45d711236c856bc60d567e43965b2043d5cb2b0" dependencies = [ "async-trait", "introspect-rust-macros", @@ -1187,9 +1163,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.3.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" [[package]] name = "find-msvc-tools" @@ -1476,7 +1452,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.13.0", + "indexmap 2.13.1", "slab", "tokio", "tokio-util", @@ -1601,15 +1577,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "html-escape" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476" -dependencies = [ - "utf8-width", -] - [[package]] name = "http" version = "1.4.0" @@ -1972,9 +1939,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" dependencies = [ "equivalent", "hashbrown 0.16.1", @@ -1989,7 +1956,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.13.0", + "indexmap 2.13.1", "is-terminal", "itoa", "log", @@ -2012,7 +1979,7 @@ dependencies = [ [[package]] name = "introspect-events" version = "0.1.2" -source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" +source = "git+https://github.com/cartridge-gg/introspect?rev=b89ee9e#b89ee9ead02df170195bb4f67a48e4308edab855" dependencies = [ "cainome-cairo-serde", "introspect-types", @@ -2024,7 +1991,7 @@ dependencies = [ [[package]] name = "introspect-rust-macros" version = "0.1.0" -source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" +source = "git+https://github.com/cartridge-gg/introspect?rev=b89ee9e#b89ee9ead02df170195bb4f67a48e4308edab855" dependencies = [ "paste", "proc-macro2", @@ -2036,7 +2003,7 @@ dependencies = [ [[package]] name = "introspect-types" version = "0.1.2" -source = "git+https://github.com/cartridge-gg/introspect?rev=34e93c1#34e93c10c867c53c622cce03abb6431c9dae0ef5" +source = "git+https://github.com/cartridge-gg/introspect?rev=b89ee9e#b89ee9ead02df170195bb4f67a48e4308edab855" dependencies = [ "blake3", "convert_case", @@ -2332,7 +2299,7 @@ dependencies = [ "hyper", "hyper-rustls", "hyper-util", - "indexmap 2.13.0", + "indexmap 2.13.1", "ipnet", "metrics", "metrics-util", @@ -2374,12 +2341,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2435,16 +2396,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2615,15 +2566,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "parity-scale-codec" version = "3.7.5" @@ -2718,7 +2660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset 0.4.2", - "indexmap 2.13.0", + "indexmap 2.13.1", ] [[package]] @@ -2728,7 +2670,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772" dependencies = [ "fixedbitset 0.5.7", - "indexmap 2.13.0", + "indexmap 2.13.1", ] [[package]] @@ -2921,26 +2863,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "pretty_assertions" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae130e2f271fbc2ac3a40fb1d07180839cdbbe443c7a27e1e3c13c5cac0116d" -dependencies = [ - "diff", - "yansi", -] - -[[package]] -name = "pretty_assertions_sorted" -version = "1.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa95d32882f2adbdfd30312733271b83c527ee8007bf78dc21afe510463ac6a0" -dependencies = [ - "darrentsung_debug_parser", - "pretty_assertions", -] - [[package]] name = "prettyplease" version = "0.2.37" @@ -3723,9 +3645,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.27" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" [[package]] name = "serde" @@ -3814,7 +3736,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.0", + "indexmap 2.13.1", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -3988,8 +3910,7 @@ dependencies = [ [[package]] name = "sqlx" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fefb893899429669dcdd979aff487bd78f4064e5e7907e4269081e0ef7d97dc" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "sqlx-core", "sqlx-macros", @@ -4001,8 +3922,7 @@ dependencies = [ [[package]] name = "sqlx-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "base64 0.22.1", "bigdecimal", @@ -4017,7 +3937,7 @@ dependencies = [ "futures-util", "hashbrown 0.15.5", "hashlink 0.10.0", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "memchr", "once_cell", @@ -4038,8 +3958,7 @@ dependencies = [ [[package]] name = "sqlx-macros" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2d452988ccaacfbf5e0bdbc348fb91d7c8af5bee192173ac3636b5fb6e6715d" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "proc-macro2", "quote", @@ -4051,8 +3970,7 @@ dependencies = [ [[package]] name = "sqlx-macros-core" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a9c1841124ac5a61741f96e1d9e2ec77424bf323962dd894bdb93f37d5219b" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "dotenvy", "either", @@ -4076,8 +3994,7 @@ dependencies = [ [[package]] name = "sqlx-mysql" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "atoi", "base64 0.22.1", @@ -4119,8 +4036,7 @@ dependencies = [ [[package]] name = "sqlx-postgres" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "atoi", "base64 0.22.1", @@ -4158,8 +4074,7 @@ dependencies = [ [[package]] name = "sqlx-sqlite" version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2d12fe70b2c1b4401038055f90f151b78208de1f9f89a7dbfd41587a10c3eea" +source = "git+https://github.com/bengineer42/sqlx?rev=22a01d3#22a01d31a28ea4adf5d14d8c773df881feda5e86" dependencies = [ "atoi", "flume", @@ -4242,7 +4157,7 @@ dependencies = [ "flate2", "foldhash 0.1.5", "hex", - "indexmap 2.13.0", + "indexmap 2.13.1", "num-traits", "serde", "serde_json", @@ -4359,6 +4274,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "starknet-types-raw" +version = "0.1.0" +source = "git+https://github.com/cartridge-gg/starknet-types-raw?branch=main#79e192ffe34aad18e737c6630eb69ee295a49027" +dependencies = [ + "num-traits", + "serde", + "starknet", + "starknet-types-core", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -4614,9 +4540,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.50.0" +version = "1.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27ad5e34374e03cfffefc301becb44e9dc3c17584f414349ebe29ed26661822d" +checksum = "2bd1c4c0fc4a7ab90fc15ef6daaa3ec3b893f004f915f2392557ed23237820cd" dependencies = [ "bytes", "libc", @@ -4631,9 +4557,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.6.1" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c55a2eff8b69ce66c84f85e1da1c233edc36ceb85a2058d11b0d6a3c7e7569c" +checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", @@ -4738,7 +4664,7 @@ version = "0.25.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a82418ca169e235e6c399a84e395ab6debeb3bc90edc959bf0f48647c6a32d1b" dependencies = [ - "indexmap 2.13.0", + "indexmap 2.13.1", "toml_datetime", "toml_parser", "winnow", @@ -4849,6 +4775,7 @@ dependencies = [ "itertools 0.14.0", "metrics", "metrics-exporter-prometheus", + "primitive-types 0.14.0", "prost 0.13.5", "prost-types 0.13.5", "resolve-path", @@ -4857,6 +4784,7 @@ dependencies = [ "serde_json", "sqlx", "starknet", + "starknet-types-raw", "tempfile", "tokio", "tokio-rustls", @@ -4872,11 +4800,13 @@ dependencies = [ "torii-erc20", "torii-erc721", "torii-introspect", - "torii-introspect-postgres-sink", + "torii-introspect-sql-sink", "torii-log-sink", "torii-pathfinder", + "torii-sql", "torii-sql-sink", "torii-test-utils", + "torii-types", "tower 0.5.3", "tower-http 0.5.2", "tracing", @@ -4900,17 +4830,16 @@ dependencies = [ "torii-arcade-sink", "torii-common", "torii-config-common", + "torii-controllers-sink", "torii-dojo", "torii-ecs-sink", - "torii-entities-historical-sink", "torii-erc1155", "torii-erc20", "torii-erc721", - "torii-introspect-postgres-sink", - "torii-introspect-sqlite-sink", + "torii-introspect-sql-sink", "torii-pathfinder", "torii-runtime-common", - "torii-sqlite", + "torii-sql", "tracing", "tracing-subscriber", "url", @@ -4933,9 +4862,11 @@ dependencies = [ "tonic-build", "torii", "torii-common", + "torii-dojo", "torii-erc721", "torii-introspect", "torii-runtime-common", + "torii-sql", "tracing", ] @@ -4946,12 +4877,12 @@ dependencies = [ "anyhow", "async-trait", "base64 0.22.1", - "itertools 0.14.0", + "primitive-types 0.14.0", "reqwest", "serde", "serde_json", - "sqlx", "starknet", + "starknet-types-raw", "tokio", "tracing", "urlencoding", @@ -4971,15 +4902,17 @@ dependencies = [ "anyhow", "async-trait", "chrono", + "hex", "metrics", "reqwest", "serde", "serde_json", "sqlx", - "starknet", + "starknet-types-raw", "tokio", "torii", "torii-runtime-common", + "torii-sql", "tracing", ] @@ -4998,14 +4931,15 @@ dependencies = [ "sqlx", "starknet", "starknet-types-core", + "starknet-types-raw", "thiserror 2.0.18", "tokio", "tonic-build", "torii", "torii-common", "torii-introspect", - "torii-postgres", - "torii-sqlite", + "torii-sql", + "torii-types", "tracing", ] @@ -5029,7 +4963,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "starknet", + "starknet-types-raw", "tokio", "tokio-stream", "tonic", @@ -5038,25 +4972,7 @@ dependencies = [ "torii-dojo", "torii-introspect", "torii-runtime-common", - "tracing", -] - -[[package]] -name = "torii-entities-historical-sink" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "introspect-types", - "serde", - "serde_json", - "sqlx", - "starknet", - "thiserror 2.0.18", - "tokio", - "torii", - "torii-introspect", - "torii-runtime-common", + "torii-sql", "tracing", ] @@ -5071,18 +4987,23 @@ dependencies = [ "chrono", "futures", "hex", + "primitive-types 0.14.0", "prost 0.13.5", "prost-types 0.13.5", "rusqlite", "serde", "serde_json", "starknet", + "starknet-types-raw", + "thiserror 2.0.18", "tokio", "tokio-postgres", "tonic", "tonic-build", "torii", "torii-common", + "torii-sql", + "torii-types", "tracing", ] @@ -5098,18 +5019,22 @@ dependencies = [ "futures", "hex", "metrics", + "primitive-types 0.14.0", "prost 0.13.5", "prost-types 0.13.5", "rusqlite", "serde", "serde_json", "starknet", + "starknet-types-raw", "tokio", "tokio-postgres", "tonic", "tonic-build", "torii", "torii-common", + "torii-sql", + "torii-types", "tracing", ] @@ -5127,6 +5052,7 @@ dependencies = [ "torii", "torii-erc20", "torii-runtime-common", + "torii-sql", "tracing", "tracing-subscriber", "url", @@ -5147,6 +5073,7 @@ dependencies = [ "torii", "torii-erc20", "torii-runtime-common", + "torii-sql", "tracing", "tracing-subscriber", ] @@ -5162,18 +5089,21 @@ dependencies = [ "chrono", "futures", "hex", + "primitive-types 0.14.0", "prost 0.13.5", "prost-types 0.13.5", "rusqlite", "serde", "serde_json", "starknet", + "starknet-types-raw", "tokio", "tokio-postgres", "tonic", "tonic-build", "torii", "torii-common", + "torii-sql", "tracing", ] @@ -5192,12 +5122,13 @@ dependencies = [ "serde", "serde_json", "sqlx", - "starknet", "starknet-types-core", + "starknet-types-raw", "thiserror 2.0.18", "tonic-build", "torii", "torii-common", + "torii-sql", ] [[package]] @@ -5218,46 +5149,19 @@ dependencies = [ "torii-controllers-sink", "torii-dojo", "torii-ecs-sink", - "torii-entities-historical-sink", "torii-erc1155", "torii-erc20", "torii-erc721", - "torii-introspect-postgres-sink", - "torii-introspect-sqlite-sink", + "torii-introspect-sql-sink", "torii-runtime-common", - "torii-sqlite", + "torii-sql", "tracing", "tracing-subscriber", "url", ] [[package]] -name = "torii-introspect-postgres-sink" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "hex", - "introspect-types", - "itertools 0.14.0", - "metrics", - "serde", - "serde_json", - "sqlx", - "starknet", - "starknet-types-core", - "thiserror 2.0.18", - "tokio", - "torii", - "torii-common", - "torii-introspect", - "torii-postgres", - "tracing", - "xxhash-rust", -] - -[[package]] -name = "torii-introspect-sqlite-sink" +name = "torii-introspect-sql-sink" version = "0.1.0" dependencies = [ "anyhow", @@ -5270,14 +5174,16 @@ dependencies = [ "serde", "serde_json", "sqlx", - "starknet", "starknet-types-core", + "starknet-types-raw", "thiserror 2.0.18", "tokio", "torii", + "torii-dojo", "torii-introspect", - "torii-sqlite", + "torii-sql", "tracing", + "xxhash-rust", ] [[package]] @@ -5294,12 +5200,12 @@ dependencies = [ "serde_json", "sqlx", "starknet", - "starknet-types-core", + "starknet-types-raw", "tokio", "torii", "torii-dojo", - "torii-introspect-postgres-sink", - "torii-runtime-common", + "torii-introspect-sql-sink", + "torii-types", "tracing", "tracing-subscriber", ] @@ -5315,7 +5221,7 @@ dependencies = [ "prost-types 0.13.5", "serde", "serde_json", - "starknet", + "starknet-types-raw", "tokio", "tokio-stream", "tonic", @@ -5334,43 +5240,36 @@ dependencies = [ "rand 0.8.5", "rusqlite", "serde", - "starknet", + "starknet-types-raw", "thiserror 2.0.18", "torii", - "torii-starknet", + "torii-types", "zstd", ] [[package]] -name = "torii-postgres" +name = "torii-runtime-common" version = "0.1.0" dependencies = [ "anyhow", - "async-trait", - "crc", - "futures", - "hex", - "serde", - "serde_json", - "sqlx", - "starknet-types-core", - "thiserror 2.0.18", "tokio", + "tokio-postgres", "torii", - "torii-common", + "torii-sql", "tracing", - "xxhash-rust", ] [[package]] -name = "torii-runtime-common" +name = "torii-sql" version = "0.1.0" dependencies = [ - "anyhow", - "tokio", - "tokio-postgres", - "torii", - "tracing", + "async-trait", + "crc", + "futures", + "itertools 0.14.0", + "log", + "sqlx", + "starknet-types-raw", ] [[package]] @@ -5388,7 +5287,7 @@ dependencies = [ "serde", "serde_json", "sqlx", - "starknet", + "starknet-types-raw", "tokio", "tonic", "tonic-build", @@ -5397,28 +5296,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "torii-sqlite" -version = "0.1.0" -dependencies = [ - "async-trait", - "futures", - "sqlx", - "torii-common", -] - -[[package]] -name = "torii-starknet" -version = "0.1.0" -dependencies = [ - "assert_matches", - "bincode", - "pretty_assertions_sorted", - "serde", - "serde_json", - "starknet", -] - [[package]] name = "torii-test-utils" version = "0.1.0" @@ -5430,8 +5307,8 @@ dependencies = [ "resolve-path", "serde", "serde_json", - "starknet", "starknet-types-core", + "starknet-types-raw", ] [[package]] @@ -5453,6 +5330,7 @@ dependencies = [ "torii-erc20", "torii-erc721", "torii-runtime-common", + "torii-sql", "tracing", "tracing-subscriber", "url", @@ -5476,10 +5354,20 @@ dependencies = [ "torii-erc20", "torii-erc721", "torii-runtime-common", + "torii-sql", "tracing", "tracing-subscriber", ] +[[package]] +name = "torii-types" +version = "0.1.0" +dependencies = [ + "starknet", + "starknet-types-raw", + "thiserror 2.0.18", +] + [[package]] name = "tower" version = "0.4.13" @@ -5768,12 +5656,6 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" -[[package]] -name = "utf8-width" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1292c0d970b54115d14f2492fe0170adf21d68a1de108eebc51c1df4f346a091" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -5969,7 +5851,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.0", + "indexmap 2.13.1", "wasm-encoder", "wasmparser", ] @@ -5982,7 +5864,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.0", "hashbrown 0.15.5", - "indexmap 2.13.0", + "indexmap 2.13.1", "semver", ] @@ -6407,7 +6289,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.0", + "indexmap 2.13.1", "prettyplease", "syn", "wasm-metadata", @@ -6438,7 +6320,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.0", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "serde", "serde_derive", @@ -6457,7 +6339,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.0", + "indexmap 2.13.1", "log", "semver", "serde", @@ -6469,9 +6351,9 @@ dependencies = [ [[package]] name = "writeable" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" [[package]] name = "wyz" @@ -6488,12 +6370,6 @@ version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" -[[package]] -name = "yansi" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" - [[package]] name = "yoke" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 4c14aeb6..1a198d3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,19 +9,16 @@ members = [ "crates/torii-controllers-sink", "crates/arcade-sink", "crates/torii-ecs-sink", - "crates/torii-entities-historical-sink", "crates/torii-erc20", "crates/torii-erc721", "crates/torii-erc1155", + "crates/sql", "crates/introspect", "crates/dojo", - "crates/introspect-postgres-sink", - "crates/introspect-sqlite-sink", + "crates/introspect-sql-sink", "crates/testing", - "crates/postgres", - "crates/sqlite", "crates/pathfinder", - "crates/starknet", + "crates/types", "bins/torii-erc20", "bins/torii-erc20-synth", "bins/torii-introspect-bin", @@ -72,6 +69,7 @@ tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } metrics = "0.24" metrics-exporter-prometheus = "0.17" +log = "0.4.29" # CLI clap = { version = "4.5", features = ["derive"] } @@ -80,13 +78,11 @@ clap = { version = "4.5", features = ["derive"] } starknet = "0.17" # Database -sqlx = { version = "0.8.6", features = [ +sqlx = { git = "https://github.com/bengineer42/sqlx", version = "0.8.6", features = [ "runtime-tokio", - "sqlite", - "postgres", - "any", -] } +], rev = "22a01d3" } # Used patched version of sqlx with SqliteConnectOptions support (PR in progress: https://github.com/launchbadge/sqlx/pull/4209) crc = "3.4.0" +rusqlite = { version = "0.32.1", features = ["bundled"] } # Hashing blake3 = "1.8.3" @@ -110,15 +106,16 @@ hex = "0.4" primitive-types = { version = "0.14.0", features = ["serde"] } starknet-types-core = "0.2.0" bigdecimal = "0.4.10" +starknet-types-raw = { git = "https://github.com/cartridge-gg/starknet-types-raw", branch = "main" } # Build tonic-build = "0.12" # Introspect -introspect-events = { git = "https://github.com/cartridge-gg/introspect", rev = "34e93c1" } -introspect-types = { git = "https://github.com/cartridge-gg/introspect", rev = "34e93c1" } -introspect-rust-macros = { git = "https://github.com/cartridge-gg/introspect", rev = "34e93c1" } -dojo-introspect = { git = "https://github.com/dojoengine/dojo-introspect", rev = "aadc3c9" } +introspect-events = { git = "https://github.com/cartridge-gg/introspect", rev = "b89ee9e" } +introspect-types = { git = "https://github.com/cartridge-gg/introspect", rev = "b89ee9e" } +introspect-rust-macros = { git = "https://github.com/cartridge-gg/introspect", rev = "b89ee9e" } +dojo-introspect = { git = "https://github.com/dojoengine/dojo-introspect", rev = "c45d711" } # Utils @@ -133,25 +130,21 @@ torii-config-common = { path = "crates/torii-config-common" } torii-runtime-common = { path = "crates/torii-runtime-common" } # Internal crates +torii-sql = { path = "crates/sql" } torii-sql-sink.path = "crates/torii-sql-sink" torii-log-sink.path = "crates/torii-log-sink" torii-controllers-sink.path = "crates/torii-controllers-sink" torii-arcade-sink.path = "crates/arcade-sink" torii-ecs-sink.path = "crates/torii-ecs-sink" -torii-entities-historical-sink.path = "crates/torii-entities-historical-sink" torii-erc20.path = "crates/torii-erc20" torii-erc721.path = "crates/torii-erc721" torii-erc1155.path = "crates/torii-erc1155" torii-dojo.path = "./crates/dojo" torii-introspect.path = "crates/introspect" -torii-introspect-postgres-sink.path = "./crates/introspect-postgres-sink" -torii-introspect-sqlite-sink.path = "./crates/introspect-sqlite-sink" +torii-introspect-sql-sink.path = "./crates/introspect-sql-sink" torii-test-utils.path = "crates/testing" -torii-postgres.path = "crates/postgres" -torii-sqlite.path = "crates/sqlite" torii-pathfinder.path = "crates/pathfinder" -torii-starknet.path = "crates/starknet" - +torii-types.path = "crates/types" [lib] name = "torii" @@ -177,10 +170,6 @@ path = "examples/http_only_sink/main.rs" name = "test_block_extractor" path = "examples/test_block_extractor/main.rs" -[[example]] -name = "introspect_simple" -path = "examples/introspect/simple.rs" - [[example]] name = "introspect_restart" path = "examples/introspect/restart.rs" @@ -204,8 +193,9 @@ prost-types.workspace = true rustls-pemfile = "2.0" serde_json.workspace = true serde.workspace = true -sqlx.workspace = true +sqlx = { workspace = true, features = ["sqlite"] } starknet.workspace = true +starknet-types-raw = { workspace = true, features = ["events"] } tokio-stream.workspace = true tokio-rustls = { version = "0.26", default-features = false, features = [ "logging", @@ -218,6 +208,7 @@ tonic-reflection.workspace = true tonic-web.workspace = true tonic.workspace = true torii-common.workspace = true +torii-types = { workspace = true, features = ["field"] } tower.workspace = true tower-http.workspace = true tracing.workspace = true @@ -238,20 +229,24 @@ tower.workspace = true resolve-path.workspace = true itertools.workspace = true dojo-introspect.workspace = true +primitive-types.workspace = true + +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } torii-sql-sink.workspace = true torii-log-sink.workspace = true torii-erc20.workspace = true torii-erc721.workspace = true torii-erc1155.workspace = true -torii-dojo.workspace = true +torii-dojo = { workspace = true, features = ["postgres", "sqlite"] } torii-introspect.workspace = true torii-common.workspace = true -torii-introspect-postgres-sink.workspace = true torii-test-utils.workspace = true +torii-introspect-sql-sink = { workspace = true, features = [ + "postgres", + "sqlite", +] } torii-pathfinder.workspace = true - - # Example dependencies diff --git a/benches/perf_harness.rs b/benches/perf_harness.rs index e0ffbdf7..7d38713a 100644 --- a/benches/perf_harness.rs +++ b/benches/perf_harness.rs @@ -1,8 +1,10 @@ use async_trait::async_trait; -use axum::{body::Body, http::Request, Router}; +use axum::body::Body; +use axum::http::Request; +use axum::Router; use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; -use starknet::core::types::{EmittedEvent, Felt, U256}; -use starknet::macros::selector; +use primitive_types::U256; +use starknet_types_raw::Felt; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -16,17 +18,18 @@ use torii::etl::engine_db::{EngineDb, EngineDbConfig}; use torii::etl::envelope::{Envelope, TypeId, TypedBody}; use torii::etl::extractor::{ExtractionBatch, RetryPolicy}; use torii::etl::sink::{EventBus, MultiSink, Sink, SinkContext, TopicInfo}; -use torii::etl::Decoder; -use torii::grpc::{proto::TopicSubscription, SubscriptionManager}; +use torii::etl::{Decoder, StarknetEvent}; +use torii::grpc::proto::TopicSubscription; +use torii::grpc::SubscriptionManager; use torii::http::create_http_router; use torii_common::{blob_to_felt, blob_to_u256, felt_to_blob, u256_to_blob}; use torii_erc1155::decoder::Erc1155Decoder; -use torii_erc20::{ - decoder::Erc20Decoder, - storage::{Erc20Storage, TransferData, TransferDirection}, -}; +use torii_erc20::decoder::Erc20Decoder; +use torii_erc20::storage::{Erc20Storage, TransferData, TransferDirection}; use torii_erc721::decoder::Erc721Decoder; +const TRANSFER_SELECTOR: Felt = Felt::selector("Transfer"); + #[derive(Debug)] struct BenchBody { value: u64, @@ -57,15 +60,20 @@ impl Decoder for MockDecoder { self.name } - async fn decode_event(&self, event: &EmittedEvent) -> anyhow::Result> { - if event.from_address != self.contract { + async fn decode( + &self, + _keys: &[Felt], + _data: &[Felt], + context: torii::etl::EventContext, + ) -> anyhow::Result> { + if context.from_address != self.contract { return Ok(Vec::new()); } Ok(vec![Envelope::new( - format!("{}:{}", self.name, event.block_number.unwrap_or_default()), + format!("{}:{}", self.name, context.block_number), Box::new(BenchBody { - value: event.block_number.unwrap_or_default(), + value: context.block_number, }), HashMap::new(), )]) @@ -131,43 +139,41 @@ fn make_engine_db(rt: &Runtime) -> EngineDb { }) } -fn make_erc20_transfer_event(seed: u64) -> EmittedEvent { - EmittedEvent { +fn make_erc20_transfer_event(seed: u64) -> StarknetEvent { + StarknetEvent { from_address: Felt::from(0x1000 + seed), keys: vec![ - selector!("Transfer"), + TRANSFER_SELECTOR, Felt::from(0x2000 + seed), Felt::from(0x3000 + seed), ], data: vec![Felt::from(10_000 + seed), Felt::ZERO], - block_hash: None, - block_number: Some(1_000_000 + seed), + block_number: 1_000_000 + seed, transaction_hash: Felt::from(0x4000 + seed), } } -fn make_erc721_transfer_event(seed: u64) -> EmittedEvent { - EmittedEvent { +fn make_erc721_transfer_event(seed: u64) -> StarknetEvent { + StarknetEvent { from_address: Felt::from(0x1100 + seed), keys: vec![ - selector!("Transfer"), + TRANSFER_SELECTOR, Felt::from(0x2200 + seed), Felt::from(0x3300 + seed), Felt::from(seed), Felt::ZERO, ], data: vec![], - block_hash: None, - block_number: Some(2_000_000 + seed), + block_number: 2_000_000 + seed, transaction_hash: Felt::from(0x4400 + seed), } } -fn make_erc1155_transfer_single_event(seed: u64) -> EmittedEvent { - EmittedEvent { +fn make_erc1155_transfer_single_event(seed: u64) -> StarknetEvent { + StarknetEvent { from_address: Felt::from(0x1200 + seed), keys: vec![ - selector!("TransferSingle"), + TRANSFER_SELECTOR, Felt::from(0x2300 + seed), Felt::from(0x3400 + seed), Felt::from(0x4500 + seed), @@ -178,23 +184,28 @@ fn make_erc1155_transfer_single_event(seed: u64) -> EmittedEvent { Felt::from(100 + seed), Felt::ZERO, ], - block_hash: None, - block_number: Some(3_000_000 + seed), + block_number: 3_000_000 + seed, transaction_hash: Felt::from(0x4600 + seed), } } -fn make_context_event(contract: Felt, seed: u64) -> EmittedEvent { - EmittedEvent { +fn make_context_event(contract: Felt, seed: u64) -> StarknetEvent { + StarknetEvent { from_address: contract, - keys: vec![selector!("Transfer")], + keys: vec![TRANSFER_SELECTOR], data: Vec::new(), - block_hash: None, - block_number: Some(10_000 + seed), + block_number: 10_000 + seed, transaction_hash: Felt::from(0x5000 + seed), } } +fn primitive_u256_from_words(low: u128, high: u128) -> U256 { + let mut bytes = [0_u8; 32]; + bytes[..16].copy_from_slice(&low.to_le_bytes()); + bytes[16..].copy_from_slice(&high.to_le_bytes()); + U256::from_little_endian(&bytes) +} + fn make_transfer_batch(size: usize, offset: u64) -> Vec { (0..size) .map(|i| { @@ -204,7 +215,7 @@ fn make_transfer_batch(size: usize, offset: u64) -> Vec { token: Felt::from(0x5000 + ((i + offset) % 4)), from: Felt::from(0x6000 + ((i + offset) % 1024)), to: Felt::from(0x7000 + ((i + offset + 7) % 1024)), - amount: U256::from_words((i + offset) as u128, 0), + amount: primitive_u256_from_words((i + offset) as u128, 0), block_number: 4_000_000 + i + offset, tx_hash: Felt::from(0x8000 + i + offset), timestamp: None, @@ -253,7 +264,7 @@ fn benchmark_common_conversions(c: &mut Criterion) { group.bench_function("felt_roundtrip", |b| { b.iter(|| { let blob = felt_to_blob(black_box(felt)); - black_box(blob_to_felt(black_box(&blob))) + black_box(blob_to_felt::(black_box(&blob))) }); }); @@ -261,15 +272,15 @@ fn benchmark_common_conversions(c: &mut Criterion) { group.bench_function("u256_roundtrip_small", |b| { b.iter(|| { let blob = u256_to_blob(black_box(small)); - black_box(blob_to_u256(black_box(&blob))) + black_box(blob_to_u256::(black_box(&blob))) }); }); - let large = U256::from_words(u128::MAX - 7, u128::MAX - 11); + let large = primitive_u256_from_words(u128::MAX - 7, u128::MAX - 11); group.bench_function("u256_roundtrip_large", |b| { b.iter(|| { let blob = u256_to_blob(black_box(large)); - black_box(blob_to_u256(black_box(&blob))) + black_box(blob_to_u256::(black_box(&blob))) }); }); @@ -325,7 +336,7 @@ fn benchmark_decoders(c: &mut Criterion) { b.to_async(&rt).iter(|| async { black_box( erc20_decoder - .decode(black_box(&batch_events)) + .decode_events(black_box(&batch_events)) .await .expect("erc20 batch decode failed"), ) @@ -435,7 +446,7 @@ fn benchmark_decoder_context(c: &mut Criterion) { b.to_async(&rt).iter(|| async { black_box( context_registry - .decode(black_box(&batch_events)) + .decode_events(black_box(&batch_events)) .await .expect("context batch decode failed"), ) diff --git a/bins/torii-arcade/Cargo.toml b/bins/torii-arcade/Cargo.toml index 0ec65787..4ed729d1 100644 --- a/bins/torii-arcade/Cargo.toml +++ b/bins/torii-arcade/Cargo.toml @@ -9,20 +9,22 @@ name = "torii-arcade" path = "src/main.rs" [dependencies] -torii = { path = "../../" } -torii-common = { path = "../../crates/torii-common" } +torii.workspace = true +torii-common.workspace = true torii-config-common.workspace = true -torii-arcade-sink = { path = "../../crates/arcade-sink" } -torii-dojo = { path = "../../crates/dojo" } -torii-ecs-sink = { path = "../../crates/torii-ecs-sink" } -torii-entities-historical-sink.workspace = true -torii-erc20 = { path = "../../crates/torii-erc20" } -torii-erc721 = { path = "../../crates/torii-erc721" } -torii-erc1155 = { path = "../../crates/torii-erc1155" } -torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } -torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } +torii-controllers-sink.workspace = true +torii-arcade-sink.workspace = true +torii-dojo.workspace = true +torii-ecs-sink.workspace = true +torii-erc20.workspace = true +torii-erc721.workspace = true +torii-erc1155.workspace = true +torii-introspect-sql-sink = { workspace = true, features = [ + "postgres", + "sqlite", +] } torii-runtime-common.workspace = true -torii-sqlite.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } torii-pathfinder = { workspace = true, features = ["etl"] } tokio = { version = "1", features = ["full"] } @@ -32,7 +34,7 @@ starknet = "0.17" url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" -sqlx = { version = "0.8", features = [ +sqlx = { workspace = true, features = [ "postgres", "sqlite", "runtime-tokio-rustls", diff --git a/bins/torii-arcade/README.md b/bins/torii-arcade/README.md index e2474858..b02d5536 100644 --- a/bins/torii-arcade/README.md +++ b/bins/torii-arcade/README.md @@ -1,155 +1,122 @@ -# torii-arcade - -`torii-arcade` is a Cartridge Arcade metatorii backend that runs the following in one process: - -- Dojo introspect indexing for the Arcade world -- `world.World` ECS gRPC reads -- `arcade.v1.Arcade` low-latency marketplace and inventory reads -- ERC20 / ERC721 / ERC1155 token indexing and gRPC reads -- Core `torii.Torii` subscriptions and metrics endpoint - -## Defaults - -- RPC: `https://api.cartridge.gg/x/starknet/mainnet` -- Primary world: `0x2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb` -- Default indexing seed set: upstream `torii-arcade-metatorii.toml` worlds and token contracts -- Default Dojo introspect set: upstream `WORLD:*` contracts except the primary world contract - `0x2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb`, which is excluded from - introspect decoding by default because it currently emits incompatible historical Dojo record events -- Metadata mode: `inline` -- Well-known ERC20s: enabled by default -- Token URI + image cache: enabled by default in inline metadata mode, with images cached under - `./data/image-cache` -- DB directory: `./torii-data` - -SQLite local defaults: - -- engine: `./torii-data/arcade-engine.db` -- dojo / ecs: `./torii-data/arcade-introspect.db` -- erc20: `./torii-data/arcade-erc20.db` -- erc721: `./torii-data/arcade-erc721.db` -- erc1155: `./torii-data/arcade-erc1155.db` - -If `--database-url` or `--storage-database-url` is PostgreSQL and `--token-storage-database-url` is omitted, token storage defaults to the same PostgreSQL database. - -`torii-arcade` requires one backend per runtime. Mixed SQLite/PostgreSQL configurations are rejected at startup. - -## Start - -From the repository root: - -```bash -cargo run --bin torii-arcade -- --from-block 0 -``` - -Useful overrides: - -```bash -cargo run --bin torii-arcade -- \ - --from-block 0 \ - --port 3000 \ - --erc721 0x1e1c477f2ef896fd638b50caa31e3aa8f504d5c6cb3c09c99cd0b72523f07f7 \ - --erc20 0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7,0x04718f5a0Fc34cC1AF16A1cdee98fFB20C31f5cD61D6Ab07201858f4287c938D -``` - -If you need to opt a world back into Dojo introspect explicitly for investigation: - -```bash -cargo run --bin torii-arcade -- \ - --from-block 0 \ - --introspect-contracts 0x2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb,0x8b4838140a3cbd36ebe64d4b5aaf56a30cc3753c928a79338bf56c53f506c5 -``` - -PostgreSQL: - -```bash -cargo run --bin torii-arcade -- \ - --database-url postgres://torii:torii@localhost:5432/torii \ - --storage-database-url postgres://torii:torii@localhost:5432/torii \ - --from-block 0 +# torii-arcade (binary) + +**Cartridge Arcade backend.** Combines Dojo introspect indexing, the +`world.World` ECS gRPC, the `arcade.v1.Arcade` projection gRPC, token +indexers (ERC20/721/1155), controller sync, and core `torii.Torii` +subscriptions into one process with sensible Arcade-specific defaults. + +## Role in Torii + +Production binary for the Cartridge Arcade UI. Pre-ships a curated list +of world contracts + token contracts in its CLI defaults, so operators +usually run it with `cargo run --bin torii-arcade` and override only the +backend URLs. Compared to `bins/torii-introspect-bin` it adds the +`arcade-sink` projection layer. + +## Architecture + +```text ++-----------------------------+ +| clap::Parser Config | defaults: Arcade world +| (src/config.rs) | 0x07a0…04ff7 + 9 more Dojo contracts, +| | 17 ERC721, 46 ERC20, 1 ERC1155 +| | --introspect-contracts override possible +| | --controllers / --observability +| | --pathfinder-path for bulk backfill ++-------------+---------------+ + | + v ++------------------------------------------------------+ +| main() | +| | +| Backend URL resolution (config.engine_database_url, | +| storage_database_url, token_storage_urls): | +| - single --database-url (postgres): shared pool | +| - --storage-database-url overrides just storage | +| - SQLite defaults under --db-dir: | +| arcade-engine.db, arcade-introspect.db, | +| arcade-erc20.db, arcade-erc721.db, | +| arcade-erc1155.db | +| | +| Extractor: EventExtractor (optionally backed by | +| torii-pathfinder for backfill when | +| --pathfinder-path is set) | +| | +| Decoders: | +| DojoDecoder, Erc20Decoder, Erc721Decoder, | +| Erc1155Decoder | +| | +| Sinks (ordered — arcade projections depend on | +| introspect tables already being up to date): | +| 1. IntrospectDb (introspect-sql-sink) | +| 2. EcsSink (legacy world.World) | +| 3. ArcadeSink (Arcade projections + | +| arcade.v1 gRPC) | +| 4. ControllersSink (--controllers) | +| 5. Erc20Sink / Erc721Sink / Erc1155Sink | +| | +| Contract identification: | +| explicit mappings for every contract in defaults | +| ContractRegistry + Erc*Rule (load_from_db on | +| startup) | +| | +| EtlConcurrencyConfig | +| max_prefetch_batches = --max-prefetch-batches | +| | +| grpc_router: WorldServer + ArcadeServer + | +| Erc20Server + Erc721Server + Erc1155Server + | +| composed reflection over five descriptor sets | +| | +| optional TLS | +| torii::run(config) | ++------------------------------------------------------+ ``` -## Local TLS + ALPN - -For browser-compatible local HTTPS, use `mkcert` instead of a raw self-signed certificate. -Modern browsers require a trusted local CA and `subjectAltName` entries for `localhost`. - -Install and trust the local development CA: - -```bash -brew install mkcert nss -mkcert -install -mkdir -p certs -mkcert -cert-file certs/dev-cert.pem -key-file certs/dev-key.pem localhost 127.0.0.1 ::1 -``` - -Start `torii-arcade` with TLS enabled: - -```bash -cargo run --bin torii-arcade -- \ - --from-block 0 \ - --tls-cert ./certs/dev-cert.pem \ - --tls-key ./certs/dev-key.pem -``` - -The local listener advertises ALPN for `h2` and `http/1.1`. Native gRPC clients will negotiate -HTTP/2 automatically; HTTPS health and metrics endpoints remain available on the same port. - -Verify the local HTTPS listener: - -```bash -curl https://localhost:3000/health -grpcurl -insecure localhost:3000 list -``` - -## Operational validation - -The mixed backend can be checked end-to-end from the repository root: - -```bash -./scripts/test-arcade-backend-e2e.sh -``` - -The script verifies: - -- gRPC reflection -- `world.World` -- `arcade.v1.Arcade` -- `torii.sinks.erc20.Erc20` -- `torii.sinks.erc721.Erc721` -- `Worlds`, `RetrieveEntities`, `RetrieveEventMessages`, `RetrieveEvents` -- `ListGames`, `ListEditions`, `ListCollections`, `ListListings`, `ListSales` -- token service stats for the mixed runtime - -## Exposed gRPC services - -- `torii.Torii` -- `world.World` -- `arcade.v1.Arcade` -- `torii.sinks.erc20.Erc20` when ERC20 contracts are configured -- `torii.sinks.erc721.Erc721` when ERC721 contracts are configured -- `torii.sinks.erc1155.Erc1155` when ERC1155 contracts are configured - -## Arcade service surface - -`arcade.v1.Arcade` is backed by projection tables maintained in the introspect storage database: - -- `ListGames` -- `ListEditions` -- `ListCollections` -- `ListListings` -- `ListSales` -- `GetPlayerInventory` - -The service bootstraps its projections from existing `ARCADE-*` introspect tables on startup and then incrementally refreshes them from live introspect envelopes, so it works both on a warm database and during active indexing. - -## Current scope - -This binary is the mixed indexing/runtime entrypoint. It intentionally keeps the extraction model explicit and event-based for throughput: - -- Dojo indexing defaults to the upstream `torii-arcade-metatorii.toml` `WORLD:*` contracts -- Dojo introspect decoding defaults to the compatible subset of those contracts; excluded contracts are blacklisted before decode so they do not trigger decoder fallback -- token indexing defaults to the upstream `torii-arcade-metatorii.toml` ERC20 / ERC721 / ERC1155 contracts -- `--include-well-known` is opt-in and disabled by default so the runtime matches the upstream seed config unless explicitly extended - -Dynamic token enrollment is not part of this first cut. +## Deep Dive + +### CLI highlights (most flags have Arcade-specific defaults) + +| Flag | Default | Purpose | +|---|---|---| +| `--rpc-url` (env `STARKNET_RPC_URL`) | Cartridge mainnet | RPC | +| `--world-address` (env `ARCADE_WORLD_ADDRESS`) | `0x07a079…04ff7` | Primary Arcade world | +| `--dojo-contracts` | 10 curated Arcade-related worlds | Event source Dojo contracts | +| `--introspect-contracts` | falls back to `--dojo-contracts` | Override which contracts feed introspect decoding | +| `--erc20` / `--erc721` / `--erc1155` | baked-in defaults (46 / 17 / 1) | Token contracts | +| `--include-well-known` | `true` | Pre-map ETH + STRK | +| `--from-block`, `--to-block` | `0` / none | Range | +| `--db-dir` | `./torii-data` | SQLite root; files prefixed with `arcade-` | +| `--database-url` (env `DATABASE_URL`) | — | Engine DB URL (Postgres or SQLite) | +| `--storage-database-url` (env `STORAGE_DATABASE_URL`) | — | Introspect storage URL | +| `--token-storage-database-url` (env `TOKEN_STORAGE_DATABASE_URL`) | — | Per-token URL override | +| `--port` | `3000` | gRPC + HTTP | +| `--tls-cert`, `--tls-key` | — | PEM pair | +| `--controllers`, `--controllers-api-url` | off / `https://api.cartridge.gg/query` | Opt-in Cartridge controller sync | +| `--observability` | off | Metrics | +| `--event-chunk-size`, `--event-block-batch-size` | `1000` / `10000` | `starknet_getEvents` tuning | +| `--max-prefetch-batches` | `2` | ETL prefetch queue depth | +| `--rpc-parallelism` | `0` (auto) | Chunked-RPC parallelism | +| `--max-db-connections` | auto | Pool ceiling | +| `--ignore-saved-state` | off | Force reprocessing | +| `--index-external-contracts` | `true` | React to Dojo `ExternalContractRegistered` events | +| `--allow-unsafe-latest-schema-bootstrap` | off | Dev-only schema bootstrap | +| `--metadata-mode` | `inline` | `inline` or `deferred` | +| `--historical` | `[]` | `ModelName` or `0xAddr:ModelName` for `_historical` tables | +| `--pathfinder-path` | — | Optional Pathfinder SQLite DB for bulk backfill | + +### Sink ordering + +Arcade relies on the introspect tables being populated before the +projection step runs, so the sink pipeline is ordered (see +`src/main.rs`). A failure in an earlier sink aborts the batch; the +cursor is not committed and the batch replays on the next cycle. + +### Workspace dependencies + +`torii`, `torii-dojo`, `torii-introspect`, `torii-introspect-sql-sink`, `torii-ecs-sink`, `torii-arcade-sink`, `torii-erc20`, `torii-erc721`, `torii-erc1155`, `torii-controllers-sink`, `torii-common`, `torii-runtime-common`, `torii-config-common`, `torii-sql`, `torii-pathfinder` (feature `etl`), `torii-types`, plus `clap`, `tonic`, `tonic-reflection`, `tokio`, `tracing`, `starknet`, `anyhow`, `url`. + +### Extension Points + +- Add a new Arcade projection → extend `ArcadeSink` / `ArcadeService`, not this binary. +- Swap to Pathfinder backfill → pass `--pathfinder-path`; the extractor is swapped to `PathfinderCombinedExtractor` so it streams archival data then flips to the live RPC extractor. +- Private Arcade namespace → point `--database-url` at a dedicated Postgres + keep storage URL at the same target; `resolve_token_db_setup` will keep everything on one backend. diff --git a/bins/torii-arcade/src/config.rs b/bins/torii-arcade/src/config.rs index 661d5b1b..bbc2e13a 100644 --- a/bins/torii-arcade/src/config.rs +++ b/bins/torii-arcade/src/config.rs @@ -2,6 +2,7 @@ use anyhow::{bail, Result}; use clap::{Parser, ValueEnum}; use starknet::core::types::Felt; use std::path::{Path, PathBuf}; +use torii_controllers_sink::DEFAULT_API_QUERY_URL as DEFAULT_CONTROLLERS_API_URL; const DEFAULT_RPC_URL: &str = "https://api.cartridge.gg/x/starknet/mainnet"; const DEFAULT_WORLD_ADDRESS: &str = @@ -147,6 +148,14 @@ pub struct Config { #[arg(long, env = "TORII_TLS_KEY")] pub tls_key: Option, + /// Cartridge-compatible GraphQL API used to fetch controller usernames. + #[arg(long, default_value = DEFAULT_CONTROLLERS_API_URL)] + pub controllers_api_url: String, + + /// Enable controller synchronization into the arcade storage database. + #[arg(long)] + pub controllers: bool, + #[arg(long)] pub observability: bool, @@ -471,6 +480,32 @@ mod tests { assert!(cfg.tls_config().is_err()); } + #[test] + fn controllers_sync_defaults_to_disabled() { + let cfg = Config::parse_from(["torii-arcade"]); + + assert_eq!(cfg.controllers_api_url, DEFAULT_CONTROLLERS_API_URL); + assert!(!cfg.controllers); + } + + #[test] + fn controllers_flag_enables_sync() { + let cfg = Config::parse_from(["torii-arcade", "--controllers"]); + + assert!(cfg.controllers); + } + + #[test] + fn controllers_api_url_overrides_default() { + let cfg = Config::parse_from([ + "torii-arcade", + "--controllers-api-url", + "https://example.com/query", + ]); + + assert_eq!(cfg.controllers_api_url, "https://example.com/query"); + } + #[test] fn postgres_storage_defaults_to_shared_postgres_for_tokens() { let cfg = Config::parse_from([ diff --git a/bins/torii-arcade/src/main.rs b/bins/torii-arcade/src/main.rs index f2896b91..6e920bea 100644 --- a/bins/torii-arcade/src/main.rs +++ b/bins/torii-arcade/src/main.rs @@ -3,11 +3,11 @@ mod config; use anyhow::{Error, Result}; use clap::Parser; use config::{Config, MetadataMode}; -use sqlx::postgres::PgPoolOptions; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::{sqlite::SqliteConnectOptions, Executor}; use starknet::core::types::Felt; use std::collections::{HashMap, HashSet}; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use tonic::codec::CompressionEncoding; @@ -18,23 +18,21 @@ use torii::etl::extractor::{ ContractEventConfig, EventExtractor, EventExtractorConfig, Extractor, RetryPolicy, }; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; -use torii::etl::EngineDb; -use torii::etl::TypeId; +use torii::etl::{EngineDb, TypeId}; use torii::EtlConcurrencyConfig; use torii_arcade_sink::proto::arcade::arcade_server::ArcadeServer; use torii_arcade_sink::{ArcadeSink, FILE_DESCRIPTOR_SET as ARCADE_DESCRIPTOR_SET}; use torii_common::{MetadataFetcher, TokenUriService}; use torii_config_common::apply_observability_env; +use torii_controllers_sink::ControllersSink; use torii_dojo::decoder::DojoDecoder; use torii_dojo::external_contract::{ contract_type_from_decoder_ids, RegisterExternalContractCommandHandler, RegisteredContractType, SharedContractTypeRegistry, SharedDecoderRegistry, }; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::store::sqlite::SqliteStore; +use torii_dojo::store::DojoStoreTrait; use torii_ecs_sink::proto::world::world_server::WorldServer; use torii_ecs_sink::{EcsSink, FILE_DESCRIPTOR_SET as ECS_DESCRIPTOR_SET}; -use torii_entities_historical_sink::EntitiesHistoricalSink; use torii_erc1155::proto::erc1155_server::Erc1155Server; use torii_erc1155::{ Erc1155Decoder, Erc1155MetadataCommandHandler, Erc1155Service, Erc1155Sink, Erc1155Storage, @@ -50,14 +48,11 @@ use torii_erc721::{ Erc721Decoder, Erc721MetadataCommandHandler, Erc721Service, Erc721Sink, Erc721Storage, FILE_DESCRIPTOR_SET as ERC721_DESCRIPTOR_SET, }; -use torii_introspect_postgres_sink::processor::IntrospectPgDb; -use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; +use torii_introspect_sql_sink::{IntrospectDb, NamespaceMode}; use torii_pathfinder::extractor::PathfinderCombinedExtractor; -use torii_runtime_common::database::{ - validate_uniform_backends, DatabaseBackend, DEFAULT_SQLITE_MAX_CONNECTIONS, -}; +use torii_runtime_common::database::{validate_uniform_backends, DEFAULT_SQLITE_MAX_CONNECTIONS}; use torii_runtime_common::token_support::{resolve_installed_token_support, InstalledTokenSupport}; -use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; +use torii_sql::{DbConnectionOptions, DbPool, DbPoolOptions, SqlitePool}; type StarknetProvider = starknet::providers::jsonrpc::JsonRpcClient; @@ -99,9 +94,9 @@ async fn load_persisted_contract_registries( let mut contract_types = contract_type_registry.write().await; for (contract, decoder_ids, _) in mappings { - decoders.insert(contract, decoder_ids.clone()); + decoders.insert(contract.into(), decoder_ids.clone()); if let Some(contract_type) = contract_type_from_decoder_ids(&decoder_ids) { - contract_types.insert(contract, contract_type); + contract_types.insert(contract.into(), contract_type); } } @@ -225,6 +220,37 @@ fn advertised_token_services(installed_token_support: InstalledTokenSupport) -> services } +fn database_connection_options(database_url: &str) -> Result { + if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { + return DbConnectionOptions::from_str(database_url).map_err(anyhow::Error::msg); + } + + if database_url == ":memory:" || database_url == "sqlite::memory:" { + return Ok(DbConnectionOptions::Sqlite(SqliteConnectOptions::from_str( + "sqlite::memory:", + )?)); + } + + let options = if database_url.starts_with("sqlite:") { + SqliteConnectOptions::from_str(database_url)? + } else { + SqliteConnectOptions::new().filename(database_url) + }; + + if database_url.starts_with("sqlite:") && database_url.contains("mode=") { + Ok(DbConnectionOptions::Sqlite(options)) + } else { + Ok(DbConnectionOptions::Sqlite(options.create_if_missing(true))) + } +} + +async fn configure_sqlite_pool(pool: &SqlitePool) -> Result<()> { + pool.execute("PRAGMA journal_mode=WAL").await?; + pool.execute("PRAGMA synchronous=NORMAL").await?; + pool.execute("PRAGMA foreign_keys=ON").await?; + Ok(()) +} + #[tokio::main] async fn main() -> Result<()> { let env_filter = tracing_subscriber::EnvFilter::try_from_default_env() @@ -257,7 +283,7 @@ async fn run_indexer(config: Config) -> Result<()> { ("erc1155", &erc1155_db_url), ], "torii-arcade does not support mixed storage backends in one runtime; configure all databases as either SQLite or PostgreSQL", - )?; + ).map_err(|err| anyhow::anyhow!(err))?; let provider = starknet::providers::jsonrpc::JsonRpcClient::new( starknet::providers::jsonrpc::HttpTransport::new( @@ -332,6 +358,17 @@ async fn run_indexer(config: Config) -> Result<()> { "disabled" } ); + tracing::info!( + "Controllers sync: {}", + if config.controllers { + "enabled" + } else { + "disabled" + } + ); + if config.controllers { + tracing::info!("Controllers API URL: {}", config.controllers_api_url); + } tracing::info!( "External contract indexing: {}", if config.index_external_contracts { @@ -394,64 +431,26 @@ async fn run_indexer(config: Config) -> Result<()> { &erc1155_addresses, &config, )?; - - let (dojo_decoder, introspect_sink): ( - Arc, - Box, - ) = match backend { - DatabaseBackend::Postgres => { - let max_db_connections = config.max_db_connections.unwrap_or(5); - let pool = Arc::new( - PgPoolOptions::new() - .max_connections(max_db_connections) - .connect(&storage_database_url) - .await?, - ); - - let decoder = DojoDecoder::, _>::new(pool.clone(), (*provider).clone()); - let sink = IntrospectPgDb::new(pool.clone(), ()); - decoder.store.initialize().await?; - decoder.load_tables(&[]).await?; - - ( - Arc::new(decoder) as Arc, - Box::new(sink), - ) - } - DatabaseBackend::Sqlite => { - let options = sqlite_connect_options(&storage_database_url)?; - let max_db_connections = match config.max_db_connections { - Some(limit) => limit.max(1), - None if is_sqlite_memory_path(&storage_database_url) => 1, - None => DEFAULT_SQLITE_MAX_CONNECTIONS, - }; - let pool = Arc::new( - SqlitePoolOptions::new() - .max_connections(max_db_connections) - .connect_with(options) - .await?, - ); - - sqlx::query("PRAGMA journal_mode=WAL") - .execute(pool.as_ref()) - .await?; - sqlx::query("PRAGMA synchronous=NORMAL") - .execute(pool.as_ref()) - .await?; - sqlx::query("PRAGMA foreign_keys=ON") - .execute(pool.as_ref()) - .await?; - - let decoder = DojoDecoder::, _>::new(pool.clone(), (*provider).clone()); - decoder.store.initialize().await?; - decoder.load_tables(&[]).await?; - - ( - Arc::new(decoder) as Arc, - Box::new(IntrospectSqliteDb::new(pool.clone(), ())), - ) - } + let conn_options = database_connection_options(&engine_database_url)?; + let max_connections = match config.max_db_connections { + Some(n) => n, + None => match &conn_options { + DbConnectionOptions::Postgres(_) => 10, + DbConnectionOptions::Sqlite(ops) if ops.is_in_memory() => 1, + DbConnectionOptions::Sqlite(_) => DEFAULT_SQLITE_MAX_CONNECTIONS, + }, }; + let pool_options = DbPoolOptions::new().max_connections(max_connections); + let pool = pool_options.connect_any_with(conn_options).await?; + if let DbPool::Sqlite(pool) = &pool { + configure_sqlite_pool(pool).await?; + } + let dojo_decoder = DojoDecoder::new(pool.clone(), provider.clone()); + let introspect_sink = IntrospectDb::new(pool, NamespaceMode::Address); + + dojo_decoder.initialize().await?; + dojo_decoder.load_tables(&[]).await?; + introspect_sink.initialize_introspect_sql_sink().await?; let ecs_sink = EcsSink::new( &storage_database_url, @@ -479,20 +478,8 @@ async fn run_indexer(config: Config) -> Result<()> { .register_encoded_file_descriptor_set(ECS_DESCRIPTOR_SET) .register_encoded_file_descriptor_set(ARCADE_DESCRIPTOR_SET); - let historical_sink = Box::new( - EntitiesHistoricalSink::new( - &storage_database_url, - config.max_db_connections, - (), - historical_models, - ) - .await?, - ); - let arcade_projection_pipeline = ArcadeProjectionPipeline::new(vec![ - introspect_sink, - historical_sink, - Box::new(arcade_sink), - ]); + let arcade_projection_pipeline = + ArcadeProjectionPipeline::new(vec![Box::new(introspect_sink), Box::new(arcade_sink)]); let mut torii_config = torii::ToriiConfig::builder() .port(config.port) @@ -504,7 +491,7 @@ async fn run_indexer(config: Config) -> Result<()> { }) .engine_database_url(engine_database_url) .with_extractor(extractor) - .add_decoder(dojo_decoder) + .add_decoder(Arc::new(dojo_decoder)) .add_sink_boxed(Box::new(ecs_sink)) .add_sink_boxed(Box::new(arcade_projection_pipeline)); @@ -514,7 +501,14 @@ async fn run_indexer(config: Config) -> Result<()> { if config.index_external_contracts { torii_config = torii_config - .with_registry_cache(decoder_registry.clone()) + .with_registry_cache(Arc::new(RwLock::new( + decoder_registry + .read() + .await + .iter() + .map(|(contract, decoder_ids)| ((*contract).into(), decoder_ids.clone())) + .collect(), + ))) .with_command_handler(Box::new(RegisterExternalContractCommandHandler::new( registry_engine_db.clone(), decoder_registry.clone(), @@ -522,13 +516,30 @@ async fn run_indexer(config: Config) -> Result<()> { ))); } + if config.controllers { + torii_config = torii_config.add_sink_boxed(Box::new( + ControllersSink::new( + &storage_database_url, + config.max_db_connections, + Some(config.controllers_api_url.clone()), + ) + .await?, + )); + } + if !excluded_dojo_contracts.is_empty() { - torii_config = torii_config.blacklist_contracts(excluded_dojo_contracts.clone()); + torii_config = torii_config.blacklist_contracts( + excluded_dojo_contracts + .iter() + .copied() + .map(Into::into) + .collect(), + ); } let dojo_decoder_id = DecoderId::new("dojo-introspect"); for contract in &introspect_contracts { - torii_config = torii_config.map_contract(*contract, vec![dojo_decoder_id]); + torii_config = torii_config.map_contract((*contract).into(), vec![dojo_decoder_id]); } for contract in &excluded_dojo_contracts { @@ -545,7 +556,8 @@ async fn run_indexer(config: Config) -> Result<()> { let mut token_uri_services = Vec::new(); if install_erc20 { - let storage = Arc::new(Erc20Storage::new(&erc20_db_url).await?); + let erc20_pool = Erc20Storage::connect_pool(&erc20_db_url).await?; + let storage = Arc::new(Erc20Storage::from_pool(&erc20_db_url, erc20_pool).await?); let grpc_service = Erc20Service::new(storage.clone()); let sink = Box::new( Erc20Sink::new(storage.clone()) @@ -569,13 +581,14 @@ async fn run_indexer(config: Config) -> Result<()> { reflection_builder.register_encoded_file_descriptor_set(ERC20_DESCRIPTOR_SET); let decoder_id = DecoderId::new("erc20"); for address in &erc20_addresses { - torii_config = torii_config.map_contract(*address, vec![decoder_id]); + torii_config = torii_config.map_contract((*address).into(), vec![decoder_id]); } erc20_grpc_service = Some(grpc_service); } if install_erc721 { - let storage = Arc::new(Erc721Storage::new(&erc721_db_url).await?); + let erc721_pool = Erc721Storage::connect_pool(&erc721_db_url).await?; + let storage = Arc::new(Erc721Storage::from_pool(&erc721_db_url, erc721_pool).await?); let grpc_service = Erc721Service::new(storage.clone()); let mut sink = Erc721Sink::new(storage.clone()).with_grpc_service(grpc_service.clone()); if config.metadata_mode == MetadataMode::Inline { @@ -604,13 +617,16 @@ async fn run_indexer(config: Config) -> Result<()> { reflection_builder.register_encoded_file_descriptor_set(ERC721_DESCRIPTOR_SET); let decoder_id = DecoderId::new("erc721"); for address in &erc721_addresses { - torii_config = torii_config.map_contract(*address, vec![decoder_id]); + torii_config = torii_config.map_contract((*address).into(), vec![decoder_id]); } erc721_grpc_service = Some(grpc_service); } if install_erc1155 { - let storage = Arc::new(Erc1155Storage::new(&erc1155_db_url).await?); + let erc1155_pool = DbPoolOptions::new() + .connect_any_with(database_connection_options(&erc1155_db_url)?) + .await?; + let storage = Arc::new(Erc1155Storage::new(erc1155_pool, &erc1155_db_url).await?); let grpc_service = Erc1155Service::new(storage.clone()); let mut sink = Erc1155Sink::new(storage.clone()) .with_grpc_service(grpc_service.clone()) @@ -640,7 +656,7 @@ async fn run_indexer(config: Config) -> Result<()> { reflection_builder.register_encoded_file_descriptor_set(ERC1155_DESCRIPTOR_SET); let decoder_id = DecoderId::new("erc1155"); for address in &erc1155_addresses { - torii_config = torii_config.map_contract(*address, vec![decoder_id]); + torii_config = torii_config.map_contract((*address).into(), vec![decoder_id]); } erc1155_grpc_service = Some(grpc_service); } @@ -817,7 +833,7 @@ fn append_unique_contract_configs( for &address in addresses { if seen.insert(address) { configs.push(ContractEventConfig { - address, + address: address.into(), from_block, to_block, }); @@ -827,8 +843,10 @@ fn append_unique_contract_configs( #[cfg(test)] mod tests { - use super::advertised_token_services; + use super::{advertised_token_services, configure_sqlite_pool, database_connection_options}; + use std::path::Path; use torii_runtime_common::token_support::InstalledTokenSupport; + use torii_sql::{DbConnectionOptions, SqlitePool}; #[test] fn advertised_token_services_include_installed_services_without_explicit_targets() { @@ -858,4 +876,42 @@ mod tests { assert_eq!(services, vec!["torii.sinks.erc721.Erc721"]); } + + #[test] + fn database_connection_options_accepts_plain_sqlite_path() { + let options = database_connection_options("./torii-data/arcade-engine.db").unwrap(); + + let DbConnectionOptions::Sqlite(options) = options else { + panic!("expected sqlite connection options"); + }; + assert_eq!( + options.get_filename(), + Path::new("./torii-data/arcade-engine.db") + ); + } + + #[test] + fn database_connection_options_accepts_in_memory_sqlite() { + let options = database_connection_options(":memory:").unwrap(); + + let DbConnectionOptions::Sqlite(options) = options else { + panic!("expected sqlite connection options"); + }; + assert!(options.is_in_memory()); + } + + #[test] + fn database_connection_options_preserves_postgres_urls() { + let options = + database_connection_options("postgres://torii:torii@localhost:5432/torii").unwrap(); + + assert!(matches!(options, DbConnectionOptions::Postgres(_))); + } + + #[tokio::test] + async fn configure_sqlite_pool_applies_pragmas_outside_transaction() { + let pool = SqlitePool::connect("sqlite::memory:").await.unwrap(); + + configure_sqlite_pool(&pool).await.unwrap(); + } } diff --git a/bins/torii-erc20-synth/Cargo.toml b/bins/torii-erc20-synth/Cargo.toml index 662f622b..a7f8c85e 100644 --- a/bins/torii-erc20-synth/Cargo.toml +++ b/bins/torii-erc20-synth/Cargo.toml @@ -11,6 +11,7 @@ path = "src/main.rs" torii = { path = "../../" } torii-erc20 = { path = "../../crates/torii-erc20" } torii-runtime-common.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } # Async runtime tokio = { version = "1", features = ["full"] } diff --git a/bins/torii-erc20-synth/README.md b/bins/torii-erc20-synth/README.md index f3806145..ab89c9f0 100644 --- a/bins/torii-erc20-synth/README.md +++ b/bins/torii-erc20-synth/README.md @@ -1,34 +1,104 @@ -# torii-erc20-synth +# torii-erc20-synth (binary) -Deterministic end-to-end ERC20 ingestion workload for profiling extractor -> decode -> sink -> DB write. +**Deterministic end-to-end ERC20 ingestion profiler.** Drives the ERC20 +pipeline with a hand-rolled `SyntheticErc20Extractor` so the extract step +runs at local-memory speed and every other stage — decode, sink, cursor +commit — is measured cleanly. Always Postgres: the profiler assumes a +single uniform backend so results are comparable between runs. -## Quick start (local Postgres) +## Role in Torii + +Performance regression harness for `crates/torii-erc20`. The live binary +(`bins/torii-erc20`) has the full auto-identification + RPC path; the +synth variant replaces both with a deterministic generator so the numbers +reflect the DB path, not the RPC. Output is a machine-readable JSON +report plus (with `--features profiling`) a flamegraph, both under +`perf/runs//`. + +## Architecture + +```text ++--------------------------+ +| clap::Parser Config | --db-url postgres://… +| (block_count, tx_per_ | --from-block, --block-count +| block, approval_ratio, | --tx-per-block, --blocks-per-batch +| token_count, …) | --approval-ratio-bps, --token-count ++------------+-------------+ --wallet-count, --seed + | --reset-schema (drops erc20 schema) + v ++--------------------------+ +| EngineDb | postgres: DROP SCHEMA CASCADE for engine + +| + Erc20Storage | erc20 (when --reset-schema), then re-init +| + Erc20Sink | +| + SyntheticErc20Metadata| names "Synthetic ERC20 AB12CD" etc. +| CommandHandler | → CommandBus (parallelism=1, retries=1) ++------------+-------------+ + | + v ++--------------------------+ +| SyntheticErc20Extractor | deterministic RNG from --seed +| (crates/torii-erc20: | emits ExtractionBatch{events, blocks, tx} +| src/synthetic.rs) | ++------------+-------------+ + | + v + Tight loop (no spawn): extractor.extract(...) → + DecoderContext.decode_events → sink.process → + extractor.commit_cursor → record StageSample + (extract_ms, decode_ms, sink_ms, commit_ms, loop_total_ms) + | + v ++--------------------------+ +| RunReport (JSON) | stage p50/p95/p99/max, slow cycles, +| perf/runs// | blocking-suspect heuristic, DB summary +| | Also emits flamegraph if `profiling` feat ++--------------------------+ +``` + +## Deep Dive + +### CLI + +| Flag | Default | Purpose | +|---|---|---| +| `--db-url` (env `DATABASE_URL`) | `postgres://torii:torii@localhost:5432/torii` | **Postgres only** — required for comparability | +| `--from-block` | `1_000_000` | Synthetic starting block | +| `--block-count` | `200` | Total blocks generated in this run | +| `--tx-per-block` | `1_000` | Transactions (≈ events) per block | +| `--blocks-per-batch` | `1` | Blocks in each `extract()` batch | +| `--approval-ratio-bps` | `2_000` (=20%) | Fraction of events that are approvals vs transfers | +| `--token-count` | `16` | Distinct token contracts in the workload | +| `--wallet-count` | `20_000` | Unique wallet addresses | +| `--seed` | `42` | RNG seed — same seed → same events | +| `--output-root` | `perf/runs` | Per-run output dir | +| `--reset-schema` | `true` | `DROP SCHEMA IF EXISTS engine CASCADE` + `erc20 CASCADE` before the run | + +### Quick start ```bash docker compose up -d postgres cargo run -p torii-erc20-synth --release ``` -Artifacts are written to `perf/runs//`: +Artifacts land in `perf/runs//`: -- `report.json` -- `report.md` -- `context.env` -- `flamegraph.svg` (when built with `--features profiling`) +- `report.json` — canonical stage / throughput / slow-cycle report +- `report.md` — human-readable rendering +- `context.env` — the exact env + CLI used +- `flamegraph.svg` — only when built with `--features profiling` -## Useful flags +### Internal shape -```bash -cargo run -p torii-erc20-synth --release -- \ - --block-count 200 \ - --tx-per-block 1000 \ - --approval-ratio-bps 2000 \ - --blocks-per-batch 1 \ - --db-url postgres://torii:torii@localhost:5432/torii -``` +- Uses `torii_runtime_common::sink::initialize_sink_with_command_handlers` + `drop_postgres_schemas` (this binary is the reason the schema-drop helper exists). +- `SyntheticErc20MetadataCommandHandler` generates names like `Synthetic ERC20 AB12CD` / `SAB12CD` / `decimals = 18` directly into storage, bypassing metadata fetching so measurements aren't polluted by network I/O. +- Records a `StageSample` per loop iteration; writes a `RunReport` JSON + (opt-in) flamegraph at exit. -Disable schema reset for append-mode experiments: +### Workspace dependencies -```bash -cargo run -p torii-erc20-synth -- --reset-schema false -``` +`torii`, `torii-erc20`, `torii-runtime-common`, plus `clap`, `tokio`, `tracing`, `serde`, `serde_json`, `anyhow`. Feature `profiling` pulls `pprof`. + +### Extension Points + +- New workload shape → extend `SyntheticErc20Config` in `crates/torii-erc20/src/synthetic.rs` and add CLI flags here. +- New report field → add it to `RunReport` in `main.rs`; the JSON is the spec. +- Switch backend → not supported; use `bins/torii-erc20` for SQLite. diff --git a/bins/torii-erc20-synth/src/main.rs b/bins/torii-erc20-synth/src/main.rs index fbd4eb2c..71c6e6f4 100644 --- a/bins/torii-erc20-synth/src/main.rs +++ b/bins/torii-erc20-synth/src/main.rs @@ -154,7 +154,6 @@ struct ConfigSnapshot { seed: u64, db_url_redacted: String, reset_schema: bool, - pg_pool_size_env: Option, } #[derive(Debug, Serialize)] @@ -246,7 +245,8 @@ async fn main() -> Result<()> { fs::create_dir_all(&output_dir) .with_context(|| format!("failed to create output dir {}", output_dir.display()))?; - let storage = Arc::new(Erc20Storage::new(&cfg.db_url).await?); + let erc20_pool = Erc20Storage::connect_pool(&cfg.db_url).await?; + let storage = Arc::new(Erc20Storage::from_pool(&cfg.db_url, erc20_pool).await?); let mut sink = Erc20Sink::new(storage.clone()).with_metadata_pipeline( SYNTH_METADATA_COMMAND_PARALLELISM, @@ -304,7 +304,7 @@ async fn main() -> Result<()> { } let decode_start = Instant::now(); - let envelopes = decoder_context.decode(&batch.events).await?; + let envelopes = decoder_context.decode_events(&batch.events).await?; let decode_ms = ms(decode_start.elapsed()); let sink_start = Instant::now(); @@ -486,7 +486,6 @@ async fn build_report( seed: cfg.seed, db_url_redacted: redact_db_url(&cfg.db_url), reset_schema: cfg.reset_schema, - pg_pool_size_env: std::env::var("TORII_ERC20_PG_POOL_SIZE").ok(), }, totals: Totals { cycles: samples.len(), diff --git a/bins/torii-erc20/Cargo.toml b/bins/torii-erc20/Cargo.toml index 9dd48af1..261cd729 100644 --- a/bins/torii-erc20/Cargo.toml +++ b/bins/torii-erc20/Cargo.toml @@ -9,10 +9,12 @@ path = "src/main.rs" [dependencies] # Core dependencies -torii = { path = "../../" } -torii-erc20 = { path = "../../crates/torii-erc20" } +torii.workspace = true +torii-erc20.workspace = true torii-runtime-common.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } + # Async runtime tokio = { version = "1", features = ["full"] } diff --git a/bins/torii-erc20/README.md b/bins/torii-erc20/README.md index cb0f5bb3..ac276cbd 100644 --- a/bins/torii-erc20/README.md +++ b/bins/torii-erc20/README.md @@ -1,265 +1,88 @@ -# Torii ERC20 - Starknet ERC20 Token Indexer +# torii-erc20 (binary) -A production-ready indexer for ERC20 tokens on Starknet. +Stand-alone **ERC20 indexer**. The simplest canonical wiring of the Torii +stack: one decoder, one sink, one gRPC service, one extractor. Start here +if you are learning how binaries are assembled. -## Features +## Role in Torii -- ✅ **Explicit Contract Mapping**: Pre-configure known tokens (ETH, STRK, custom) -- ✅ **Auto-Discovery**: Automatically detect and index new ERC20 contracts -- ✅ **Strict Mode**: Disable auto-discovery for controlled production deployments -- ✅ **Real-time Balance Tracking**: Maintain up-to-date balances for all addresses -- ✅ **Transfer History**: Complete audit trail of all ERC20 transfers -- ✅ **SQLite Storage**: Efficient local database with full history -- ✅ **gRPC Subscriptions**: Real-time updates via Torii's event bus - -## Installation - -```bash -# From workspace root -cargo build --release --bin torii-erc20 - -# Binary will be at: -# target/release/torii-erc20 -``` - -## Usage - -### Basic Usage (Auto-Discovery Enabled) - -```bash -# Start indexing from block 100,000 -torii-erc20 --from-block 100000 -``` - -This will: -- Index ETH and STRK tokens (explicit mappings) -- Auto-discover other ERC20 contracts via ABI heuristics -- Store data in `./erc20-data.db` -- Start gRPC server on port 3000 - -### Strict Mode (Production) - -```bash -# Only index explicitly configured contracts -torii-erc20 --no-auto-discovery -``` - -This mode: -- **Disables** auto-discovery -- Only indexes ETH and STRK -- Useful for production where you want strict control - -### Custom Contracts - -```bash -# Index specific contracts only -torii-erc20 \ - --no-auto-discovery \ - --contracts 0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7,0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d -``` - -### Network Selection - -```bash -# Mainnet -torii-erc20 --rpc-url https://api.cartridge.gg/x/starknet/mainnet - -# Sepolia (default) -torii-erc20 --rpc-url https://api.cartridge.gg/x/starknet/sepolia -``` - -### All Options - -```bash -torii-erc20 \ - --rpc-url \ - --from-block \ - --to-block \ - --db-path ./my-data.db \ - --database-url postgres://torii:torii@localhost:5432/torii \ - --port 3000 \ - --no-auto-discovery \ - --contracts , -``` - -`--database-url` (or `DATABASE_URL`) enables PostgreSQL-backed storage for both engine state and ERC20 data. If unset, local SQLite files are used. - -Runtime bootstrap in this binary is shared through `torii-runtime-common` (`resolve_single_db_setup`) so the database wiring matches other Torii binaries. +Replaces a per-contract subgraph for ERC20 tokens. Runs a +`BlockRangeExtractor` over the configured RPC, feeds events through +`Erc20Decoder` / `Erc20Sink`, serves `torii.sinks.erc20.Erc20` and core +`torii.Torii` gRPC + HTTP on the same port. Uses the same `torii::run` +orchestrator as every other binary. ## Architecture -``` -┌─────────────────────────────────────────────────────────────┐ -│ Torii ERC20 Binary │ -├─────────────────────────────────────────────────────────────┤ -│ │ -│ ┌──────────────┐ ┌────────────────────────────────┐ │ -│ │ BlockRange │─────▶│ ContractRegistry │ │ -│ │ Extractor │ │ - ETH (explicit) │ │ -│ └──────────────┘ │ - STRK (explicit) │ │ -│ │ │ - ERC20Rule (auto-discovery) │ │ -│ │ └────────────────────────────────┘ │ -│ │ │ │ -│ ▼ ▼ │ -│ ┌──────────────┐ ┌────────────────────────────────┐ │ -│ │ Events │─────▶│ MultiDecoder + Registry │ │ -│ │ │ │ (lazy identification) │ │ -│ └──────────────┘ └────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌────────────────────────────────┐ │ -│ │ ERC20 Decoder │ │ -│ │ (Transfer events) │ │ -│ └────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌────────────────────────────────┐ │ -│ │ ERC20 Sink │ │ -│ │ - Store transfers │ │ -│ │ - Update balances │ │ -│ └────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌────────────────────────────────┐ │ -│ │ SQLite Storage │ │ -│ │ - transfers table │ │ -│ │ - balances table │ │ -│ └────────────────────────────────┘ │ -└─────────────────────────────────────────────────────────────┘ +```text ++------------------------+ +--------------------------+ +| clap::Parser Config | -----> | resolve_single_db_setup | +| (src/config.rs) | | (runtime-common) | ++------------------------+ +-------------+------------+ + | + v ++-----------------------------------------------------------+ +| main() — build ToriiConfig | +| | +| - Erc20Storage::connect_pool(storage_url) | +| - JsonRpcClient over --rpc-url | +| - BlockRangeExtractor (from_block → to_block, batch=50) | +| - Erc20Decoder + Erc20Sink + Erc20Service | +| - Erc20MetadataCommandHandler | +| parallelism=1, queue=4096, max_retries=3 | +| - auto-discovery (unless --no-auto-discovery): | +| ContractRegistry::new(provider, engine_db) | +| .with_rule(Erc20Rule) → registry.load_from_db() | +| - explicit mappings for well-known ETH / STRK / SURVIVOR | +| - grpc_router = Server::builder() | +| .add_service(Erc20Server::new(service)) | +| + reflection (TORII_DESCRIPTOR_SET + | +| erc20 FILE_DESCRIPTOR_SET) | +| | +| torii::run(config).await | ++-----------------------------------------------------------+ ``` -## Database Schema +## Deep Dive -### Transfers Table +### CLI (source: `src/config.rs`) -```sql -CREATE TABLE transfers ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - token TEXT NOT NULL, - from_addr TEXT NOT NULL, - to_addr TEXT NOT NULL, - amount TEXT NOT NULL, - block_number INTEGER NOT NULL, - tx_hash TEXT NOT NULL, - timestamp INTEGER DEFAULT (strftime('%s', 'now')), - UNIQUE(token, tx_hash, from_addr, to_addr) -); -``` +| Flag | Env | Default | Purpose | +|---|---|---|---| +| `--rpc-url` | `STARKNET_RPC_URL` | Cartridge mainnet | Starknet JSON-RPC endpoint | +| `--from-block` | — | `0` | Backfill starting block | +| `--to-block` | — | none (follow head) | Stop block for finite runs | +| `--db-path` | — | `./erc20-data.db` | SQLite storage; engine.db derives next to it | +| `--database-url` | `DATABASE_URL` | — | Postgres URL / SQLite URL override | +| `--no-auto-discovery` | — | off | Strict mode: only explicitly mapped contracts | +| `--contracts` | — | `[]` | Comma-separated hex addresses | +| `--port` | — | `3000` | TCP port for gRPC + HTTP | +| `--cycle-interval` | — | `3` | Seconds between extractor idle retries | -### Balances Table +Pre-mapped contracts (added unless overridden): +ETH (`0x049D36…004dC7`), STRK (`0x04718f…c938D`), SURVIVOR +(`0x042DD7…2Ec86B`). See `Config::well_known_contracts` in `src/config.rs`. -```sql -CREATE TABLE balances ( - token TEXT NOT NULL, - address TEXT NOT NULL, - balance TEXT NOT NULL, - updated_at INTEGER DEFAULT (strftime('%s', 'now')), - PRIMARY KEY (token, address) -); -``` +### Internal modules -## Contract Identification +- `config.rs` — `Config` struct + the well-known contract list. +- `main.rs` — canonical build order: logging → config → storage → provider → decoder → rule → (optional) `ContractRegistry.load_from_db` → explicit mappings → sink → metadata command handler → gRPC router → `torii::run`. -### Explicit Mapping (Highest Priority) - -Well-known contracts are explicitly mapped: -- **Mainnet ETH**: `0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7` -- **Mainnet STRK**: `0x04718f5a0fc34cc1af16a1cdee98ffb20c31f5cd61d6ab07201858f4287c938d` -- **Sepolia ETH**: Same as mainnet -- **Sepolia STRK**: Same as mainnet - -### Auto-Discovery (When Enabled) - -The ERC20Rule automatically identifies contracts with: -- `transfer()`, `balance_of()`, `total_supply()` functions -- `Transfer` event signature - -### Performance - -**Lazy Identification:** -- Contracts identified on first event (network call) -- Subsequent events from same contract use cached result -- No pre-iteration of events needed -- Lock-per-event approach (see performance notes in code for optimization path) - -## Example Queries - -```bash -# Connect to database -sqlite3 erc20-data.db - -# Get total transfers -SELECT COUNT(*) FROM transfers; - -# Get transfers for ETH -SELECT * FROM transfers -WHERE token = '0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7' -LIMIT 10; - -# Get top 10 addresses by balance for a token -SELECT address, balance -FROM balances -WHERE token = '0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7' -ORDER BY CAST(balance AS INTEGER) DESC -LIMIT 10; - -# Get unique tokens indexed -SELECT COUNT(DISTINCT token) FROM transfers; -``` - -## Development - -### Running Tests - -```bash -cargo test --bin torii-erc20 -``` - -### Adding Custom Tokens - -Edit `src/config.rs` to add more well-known contracts: - -```rust -pub fn well_known_contracts(&self) -> Vec<(Felt, &'static str)> { - vec![ - (Felt::from_hex_unchecked("0x..."), "MY_TOKEN"), - // ... - ] -} -``` - -## Performance Considerations - -- **Lock overhead**: Currently locks mutex per event. See code comments for optimization path if this becomes a bottleneck. -- **Balance calculation**: Current implementation uses simple string storage. Production should use proper u256 arithmetic. -- **Database**: SQLite performs well for moderate loads. For high-throughput production, consider PostgreSQL. - -## Performance Profiling - -The binary includes built-in timing instrumentation that measures each phase of the ETL loop: -- Extract time (RPC calls) -- Decode time (includes contract identification) -- Sink processing time (database writes) -- Total loop time - -Example output: -``` -📦 Batch #1: Extracted 1234 events from 10 blocks (extract_time: 450.23ms) - ✓ Decoded into 156 envelopes (decode_time: 89.45ms) - ✓ Processed through sink (sink_time: 23.12ms) | Total loop: 562.80ms -``` +### Workspace dependencies -For detailed CPU profiling with flamegraphs, see **[PROFILING.md](PROFILING.md)**. +`torii`, `torii-erc20`, `torii-runtime-common`, `torii-sql` (features: +`postgres`, `sqlite`), `starknet`, `tonic`, `tonic-reflection`, `clap`, +`tokio`, `tracing`, `url`, `anyhow`. -## Future Improvements +Opt-in feature: `profiling` — enables `pprof`; writes a flamegraph on +shutdown (see `PROFILING.md` in this directory for details). -- [ ] Proper u256 balance arithmetic -- [ ] Token metadata (name, symbol, decimals) -- [ ] HTTP API for querying balances -- [ ] WebSocket subscriptions for real-time updates -- [ ] PostgreSQL support -- [ ] Dashboard UI +### Extension Points -## License +- **Historical only**: `--from-block N --to-block M`; extractor sets `is_finished() = true` and the ETL loop exits cleanly. +- **Live only**: start near the chain head without `--to-block`. +- **Custom tokens**: add to `Config::well_known_contracts` (pre-map) or pass `--contracts 0xaaa,0xbbb`. Combine with `--no-auto-discovery` to pin indexing. +- **Extra auto-identification**: register another `IdentificationRule` on the `ContractRegistry` before `torii::run`. +- **Skip metadata fetching**: drop the `with_metadata_pipeline` call on the sink; only the metadata enrichment is affected. -MIT +See also `bins/torii-erc20-synth` for the paired profiling harness. diff --git a/bins/torii-erc20/src/main.rs b/bins/torii-erc20/src/main.rs index 82cc1bb3..110effca 100644 --- a/bins/torii-erc20/src/main.rs +++ b/bins/torii-erc20/src/main.rs @@ -38,7 +38,7 @@ use torii::etl::decoder::DecoderId; use torii::etl::extractor::{BlockRangeConfig, BlockRangeExtractor}; use torii_runtime_common::database::resolve_single_db_setup; #[cfg(feature = "profiling")] -use torii_runtime_common::database::{backend_from_url_or_path, DatabaseBackend}; +use torii_sql::DbBackend; // Import from the library crate use torii_erc20::proto::erc20_server::Erc20Server; @@ -89,7 +89,8 @@ async fn main() -> Result<()> { // Create storage let db_setup = resolve_single_db_setup(&config.db_path, config.database_url.as_deref()); - let storage = Arc::new(Erc20Storage::new(&db_setup.storage_url).await?); + let erc20_pool = Erc20Storage::connect_pool(&db_setup.storage_url).await?; + let storage = Arc::new(Erc20Storage::from_pool(&db_setup.storage_url, erc20_pool).await?); tracing::info!("Database initialized"); // Create Starknet provider @@ -164,14 +165,14 @@ async fn main() -> Result<()> { let erc20_decoder_id = DecoderId::new("erc20"); for (address, name) in config.well_known_contracts() { tracing::info!("Mapping {} at {:#x} to ERC20 decoder", name, address); - torii_config = torii_config.map_contract(address, vec![erc20_decoder_id]); + torii_config = torii_config.map_contract(address.into(), vec![erc20_decoder_id]); } // Add custom contracts with explicit mappings for contract_str in &config.contracts { let address = Felt::from_hex(contract_str)?; tracing::info!("Mapping custom contract {:#x} to ERC20 decoder", address); - torii_config = torii_config.map_contract(address, vec![erc20_decoder_id]); + torii_config = torii_config.map_contract(address.into(), vec![erc20_decoder_id]); } // Log mode: no_auto_discovery affects whether unmapped contracts are tried @@ -219,10 +220,10 @@ async fn main() -> Result<()> { .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - let db_backend = match backend_from_url_or_path(&db_setup.storage_url) { - DatabaseBackend::Postgres => "postgres", - DatabaseBackend::Sqlite => "sqlite", - }; + let db_backend = db_setup + .storage_url + .parse::() + .map_err(anyhow::Error::new)?; let filename = format!("flamegraph-torii-erc20-block-range-{db_backend}-{ts}.svg"); let file = std::fs::File::create(&filename).unwrap(); report.flamegraph(file).unwrap(); diff --git a/bins/torii-introspect-bin/Cargo.toml b/bins/torii-introspect-bin/Cargo.toml index c11c32b8..17b36331 100644 --- a/bins/torii-introspect-bin/Cargo.toml +++ b/bins/torii-introspect-bin/Cargo.toml @@ -9,20 +9,21 @@ name = "torii-server" path = "src/main.rs" [dependencies] -torii = { path = "../../" } -torii-common = { path = "../../crates/torii-common" } +torii.workspace = true +torii-common.workspace = true torii-config-common.workspace = true torii-controllers-sink.workspace = true -torii-dojo = { path = "../../crates/dojo" } -torii-erc20 = { path = "../../crates/torii-erc20" } -torii-erc721 = { path = "../../crates/torii-erc721" } -torii-erc1155 = { path = "../../crates/torii-erc1155" } -torii-ecs-sink = { path = "../../crates/torii-ecs-sink" } -torii-entities-historical-sink.workspace = true -torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } -torii-introspect-sqlite-sink = { path = "../../crates/introspect-sqlite-sink" } +torii-dojo.workspace = true +torii-erc20.workspace = true +torii-erc721.workspace = true +torii-erc1155.workspace = true +torii-ecs-sink.workspace = true +torii-introspect-sql-sink = { workspace = true, features = [ + "postgres", + "sqlite", +] } torii-runtime-common.workspace = true -torii-sqlite = { path = "../../crates/sqlite" } +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } tokio = { version = "1", features = ["full"] } tracing = "0.1" @@ -31,7 +32,12 @@ starknet = "0.17" url = "2.5" clap = { version = "4.5", features = ["derive", "env"] } anyhow = "1.0" -sqlx = { version = "0.8", features = ["postgres", "sqlite", "runtime-tokio-rustls", "any"] } +sqlx = { workspace = true, features = [ + "postgres", + "sqlite", + "runtime-tokio-rustls", + "any", +] } tonic.workspace = true tonic-reflection.workspace = true tonic-web.workspace = true diff --git a/bins/torii-introspect-bin/README.md b/bins/torii-introspect-bin/README.md index ef6cdc14..ea3ce50d 100644 --- a/bins/torii-introspect-bin/README.md +++ b/bins/torii-introspect-bin/README.md @@ -1,86 +1,126 @@ -# Torii Server - -Dojo introspect and token indexer backed by PostgreSQL or SQLite. - -## Run - -```bash -cargo run --bin torii-server -- \ - --contract 0x123...,0x456... \ - --storage-database-url postgres://torii:torii@localhost:5432/torii -``` - -## Mixed Dojo + Token Indexing - -```bash -cargo run --bin torii-server -- \ - --from-block 0 \ - --contract 0x0000 \ - --erc20 0x001,0x002 \ - --erc721 0x003,0x0045 \ - --chunk-size 1000 \ - --batch-size 10000 -``` - -Use `--contract`/`--contracts` for Dojo introspect targets and `--erc20`, `--erc721`, `--erc1155` -for token contracts. The extractor deduplicates overlapping addresses and runs them in one ETL -pipeline. - -## Throughput Tuning - -The introspect indexer now exposes the same core ETL parallelism knobs as `torii-tokens`. - -```bash -cargo run --bin torii-server -- \ - --contract 0x123...,0x456... \ - --storage-database-url postgres://torii:torii@localhost:5432/torii \ - --rpc-parallelism 4 \ - --max-prefetch-batches 8 +# torii-introspect-bin (binary `torii-server`) + +**Canonical full-stack Torii binary.** Indexes Dojo introspect events, +optional token transfers (ERC20/721/1155), and optional controller +usernames into a shared SQL backend. Exposes the legacy `world.World` +gRPC API plus every token gRPC service on one port. The most complete +example of how the pipeline fits together. + +## Role in Torii + +Production deployments of Dojo worlds point at this binary. It combines: + +1. `torii-dojo::DojoDecoder` consuming the world contract's events, +2. `introspect-sql-sink` materialising schema + records, +3. `torii-ecs-sink` serving the legacy `world.World` gRPC, +4. Optional `torii-erc20/721/1155` sinks + decoders + rules for token + data referenced by models, +5. Optional `torii-controllers-sink` for Cartridge controller metadata, +6. Optional TLS for the listener. + +Read `bins/torii-erc20` first — this binary is the same pattern scaled +up. + +## Architecture + +```text ++-----------------------------+ +| clap::Parser Config | argument group: --contract + --erc20 + +| (src/config.rs) | --erc721 + --erc1155 (at least one) +| | --database-url / --storage-database-url +| | --historical (for _historical tables) +| | --tls-cert / --tls-key +| | --controllers (enable sync) +| | --observability / --ignore-saved-state +| | --index-external-contracts (default on) ++-------------+---------------+ + | + v ++--------------------------------------------------------+ +| main() | +| | +| resolve engine + storage URLs | +| (Postgres uniform or SQLite under --db-dir) | +| | +| EventExtractor over --rpc-url | +| chunk=1000, block-batch=10000 | +| cursor-resume unless --ignore-saved-state | +| | +| Decoders: | +| DojoDecoder (over dojo-introspect schema fetcher) | +| + optional Erc20Decoder / Erc721Decoder / | +| Erc1155Decoder | +| | +| Sinks (installed by config): | +| - IntrospectDb | +| (IntrospectPgDb or IntrospectSqliteDb) | +| - EcsSink (reads the introspect tables; serves | +| legacy world.World gRPC) | +| - ControllersSink (--controllers) | +| - Erc20Sink/Erc721Sink/Erc1155Sink + services | +| | +| ContractRegistry with Erc*Rule registered; | +| load_from_db() on startup; | +| handles Dojo ExternalContractRegistered via | +| RegisterExternalContractCommand (index_external_ | +| contracts flag) | +| | +| EtlConcurrencyConfig | +| max_prefetch_batches = --max-prefetch-batches | +| | +| grpc_router adds all token servers + WorldServer (ECS) | +| + reflection composing TORII + token + ecs + arcade | +| descriptor sets | +| | +| optional TLS listener (both --tls-cert and --tls-key) | +| | +| torii::run(config).await | ++--------------------------------------------------------+ ``` -Notes: - -- `--rpc-parallelism`: concurrent chunked RPC requests (`0` = auto). -- `--chunk-size`: events per `starknet_getEvents` request. -- `--batch-size`: block range queried per iteration. -- `--max-prefetch-batches`: extracted batches buffered ahead of decode/store. +## Deep Dive -## Local TLS + ALPN +### CLI highlights -For browser-compatible local HTTPS, use `mkcert` instead of a raw self-signed certificate. -Modern browsers require a trusted local CA and `subjectAltName` entries for `localhost`. +| Flag | Default | Purpose | +|---|---|---| +| `--contract` / `--contracts` | required* | Dojo world contract addresses (`*` required if no token list) | +| `--erc20`, `--erc721`, `--erc1155` | `[]` | Token contracts to pin | +| `--rpc-url` (env `STARKNET_RPC_URL`) | Cartridge mainnet | JSON-RPC endpoint | +| `--from-block`, `--to-block` | `0` / none | Range | +| `--db-dir` | `./torii-data` | SQLite root | +| `--database-url` (env `DATABASE_URL`) | — | Engine DB URL | +| `--storage-database-url` (env `STORAGE_DATABASE_URL`) | — | Storage DB URL (**Postgres only** for introspect-pg path) | +| `--port` | `3000` | gRPC + HTTP | +| `--tls-cert`, `--tls-key` | — | PEM files; must come as a pair | +| `--controllers` | off | Install `torii-controllers-sink` | +| `--controllers-api-url` | `https://api.cartridge.gg/query` | GraphQL URL for controllers sink | +| `--observability` | off | Sets `TORII_METRICS_ENABLED` | +| `--event-chunk-size` / `--event-block-batch-size` | `1000` / `10000` | `starknet_getEvents` tuning | +| `--max-prefetch-batches` | `2` | ETL prefetch queue depth | +| `--cycle-interval` | `3` | Idle retry seconds | +| `--rpc-parallelism` | `0` (auto) | Chunked-RPC parallelism | +| `--max-db-connections` | auto | Pool size ceiling | +| `--ignore-saved-state` | off | Force reprocessing from `--from-block` | +| `--index-external-contracts` | `true` | React to Dojo `ExternalContractRegistered` events at runtime | +| `--historical` | `[]` | `ModelName` or `0xAddr:ModelName` entries to mirror into append-only `_historical` tables | -Install and trust the local development CA: +Argument group enforces that **at least one** of `--contract`, +`--erc20`, `--erc721`, `--erc1155` is provided. -```bash -brew install mkcert nss -mkcert -install -mkdir -p certs -mkcert -cert-file certs/dev-cert.pem -key-file certs/dev-key.pem localhost 127.0.0.1 ::1 -``` - -Start `torii-server` with TLS enabled: +### Internal modules -```bash -cargo run --bin torii-server -- \ - --contract 0x123...,0x456... \ - --tls-cert ./certs/dev-cert.pem \ - --tls-key ./certs/dev-key.pem -``` +- `config.rs` — `Config`, `parse_historical_models`, backend resolution (`storage_backend`, `engine_database_url`, `storage_database_url`), TLS pair validation. +- `main.rs` — sink assembly, registry wiring, optional TLS, gRPC reflection composition, cursor-resume logic. -The local listener advertises ALPN for `h2` and `http/1.1`. Native gRPC clients will negotiate -HTTP/2 automatically; HTTPS health and metrics endpoints remain available on the same port. +### Workspace dependencies -Verify the local HTTPS listener: +`torii`, `torii-dojo`, `torii-introspect`, `torii-introspect-sql-sink` (postgres + sqlite), `torii-ecs-sink`, `torii-controllers-sink`, `torii-erc20` / `torii-erc721` / `torii-erc1155`, `torii-arcade-sink` (for its descriptor set — the `ArcadeServer` is only mounted if enabled in custom builds), `torii-common`, `torii-runtime-common`, `torii-config-common`, `torii-sql` (postgres + sqlite). Plus `clap`, `tonic`, `tonic-reflection`, `tokio`, `tracing`, `starknet`, `anyhow`, `url`. -```bash -curl https://localhost:3000/health -grpcurl -insecure localhost:3000 list -``` +### Extension Points -## Notes +- Add another sink → build it, push `Box` into the builder, add its descriptor set to the reflection composition. +- Change extractor → replace `EventExtractor` with `BlockRangeExtractor` or a custom one; the rest of the pipeline doesn't care. +- Add Postgres schemas → `introspect-sql-sink::NamespaceMode` decides the layout; flip at binary level. -- `--storage-database-url` must be PostgreSQL when provided. -- If `--storage-database-url` is omitted, SQLite files are created under `--db-dir` for introspect and token storages. -- `--observability` controls `TORII_METRICS_ENABLED` through shared helpers. -- Config validation and observability wiring are shared through `torii-config-common`. +Paired profiler: `bins/torii-introspect-synth`. diff --git a/bins/torii-introspect-bin/src/config.rs b/bins/torii-introspect-bin/src/config.rs index 2af7c2bf..dd14b5c5 100644 --- a/bins/torii-introspect-bin/src/config.rs +++ b/bins/torii-introspect-bin/src/config.rs @@ -1,13 +1,9 @@ use anyhow::{bail, Result}; use clap::{ArgGroup, Parser}; use starknet::core::types::Felt; +use std::collections::HashSet; use std::path::{Path, PathBuf}; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum StorageBackend { - Postgres, - Sqlite, -} +use torii_sql::DbBackend; /// Dojo introspect indexer backed by PostgreSQL or SQLite. /// @@ -187,11 +183,11 @@ impl Config { models } - pub fn storage_backend(&self) -> StorageBackend { + pub fn storage_backend(&self) -> DbBackend { if self.storage_database_url.is_some() { - StorageBackend::Postgres + DbBackend::Postgres } else { - StorageBackend::Sqlite + DbBackend::Sqlite } } @@ -210,7 +206,10 @@ impl Config { Ok(url.clone()) } Some(_) => bail!("--storage-database-url must be a PostgreSQL URL"), - None => Ok(db_dir.join("introspect.db").to_string_lossy().to_string()), + None => Ok(format!( + "sqlite://{}", + db_dir.join("introspect.db").to_string_lossy() + )), } } @@ -225,6 +224,21 @@ impl Config { } } +pub fn parse_historical_models( + historical: &[String], + contracts: &[Felt], +) -> Result> { + let mut models = HashSet::with_capacity(historical.len()); + for model in historical { + let parts: Vec<&str> = model.splitn(2, ':').collect(); + match parts.len() { + 1 => contracts.iter().for_each(|&addr| {models.insert((addr, parts[0].to_string()));}), + 2 => {models.insert((Felt::from_hex(parts[0])?, parts[1].to_string()));}, + _ => bail!("Invalid historical model format: {model}. Expected format is either `ModelName` or `0xContractAddress:ModelName`"), + } + } + Ok(models) +} #[cfg(test)] mod tests { use super::*; @@ -272,7 +286,7 @@ mod tests { fn sqlite_is_default_when_storage_database_url_is_omitted() { let cfg = Config::parse_from(["torii-server", "--contract", "0x1"]); - assert_eq!(cfg.storage_backend(), StorageBackend::Sqlite); + assert_eq!(cfg.storage_backend(), DbBackend::Sqlite); assert!(cfg .storage_database_url(Path::new("./torii-data")) .unwrap() diff --git a/bins/torii-introspect-bin/src/main.rs b/bins/torii-introspect-bin/src/main.rs index ea102237..72e51d77 100644 --- a/bins/torii-introspect-bin/src/main.rs +++ b/bins/torii-introspect-bin/src/main.rs @@ -4,12 +4,13 @@ mod config; use anyhow::Result; use clap::Parser; -use config::{Config, StorageBackend}; +use config::Config; use sqlx::postgres::PgPoolOptions; -use sqlx::sqlite::SqlitePoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use starknet::core::types::Felt; use std::collections::{HashMap, HashSet}; use std::path::Path; +use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use tonic::codec::CompressionEncoding; @@ -27,11 +28,9 @@ use torii_dojo::external_contract::{ contract_type_from_decoder_ids, RegisterExternalContractCommandHandler, RegisteredContractType, SharedContractTypeRegistry, SharedDecoderRegistry, }; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::store::sqlite::SqliteStore; +use torii_dojo::store::DojoStoreTrait; use torii_ecs_sink::proto::world::world_server::WorldServer; use torii_ecs_sink::{EcsSink, FILE_DESCRIPTOR_SET as ECS_DESCRIPTOR_SET}; -use torii_entities_historical_sink::EntitiesHistoricalSink; use torii_erc1155::proto::erc1155_server::Erc1155Server; use torii_erc1155::{ Erc1155Decoder, Erc1155MetadataCommandHandler, Erc1155Service, Erc1155Sink, Erc1155Storage, @@ -47,13 +46,14 @@ use torii_erc721::{ Erc721Decoder, Erc721MetadataCommandHandler, Erc721Service, Erc721Sink, Erc721Storage, FILE_DESCRIPTOR_SET as ERC721_DESCRIPTOR_SET, }; -use torii_introspect_postgres_sink::processor::IntrospectPgDb; -use torii_introspect_sqlite_sink::processor::IntrospectSqliteDb; +use torii_introspect_sql_sink::{IntrospectPgDb, IntrospectSqliteDb, NamespaceMode}; use torii_runtime_common::database::{ resolve_token_db_setup, TokenDbSetup, DEFAULT_SQLITE_MAX_CONNECTIONS, }; use torii_runtime_common::token_support::{resolve_installed_token_support, InstalledTokenSupport}; -use torii_sqlite::{is_sqlite_memory_path, sqlite_connect_options}; +use torii_sql::{DbBackend, DbPoolOptions}; + +use crate::config::parse_historical_models; type StarknetProvider = starknet::providers::jsonrpc::JsonRpcClient; @@ -201,9 +201,9 @@ async fn load_persisted_contract_registries( let mut contract_types = contract_type_registry.write().await; for (contract, decoder_ids, _) in mappings { - decoders.insert(contract, decoder_ids.clone()); + decoders.insert(contract.into(), decoder_ids.clone()); if let Some(contract_type) = contract_type_from_decoder_ids(&decoder_ids) { - contract_types.insert(contract, contract_type); + contract_types.insert(contract.into(), contract_type); } } @@ -233,7 +233,7 @@ fn append_unique_contract_configs( for &address in addresses { if seen.insert(address) { configs.push(ContractEventConfig { - address, + address: address.into(), from_block, to_block, }); @@ -283,7 +283,7 @@ fn apply_contract_mappings( contract, decoder_ids ); - torii_config = torii_config.map_contract(contract, decoder_ids); + torii_config = torii_config.map_contract(contract.into(), decoder_ids); } torii_config @@ -353,7 +353,8 @@ async fn configure_token_support( token_db_setup.expect("token DB setup must exist when token support is configured"); if installed_token_support.erc20 { - let storage = Arc::new(Erc20Storage::new(&db_setup.erc20_url).await?); + let erc20_pool = Erc20Storage::connect_pool(&db_setup.erc20_url).await?; + let storage = Arc::new(Erc20Storage::from_pool(&db_setup.erc20_url, erc20_pool).await?); tracing::info!("ERC20 database initialized: {}", db_setup.erc20_url); let decoder: Arc = Arc::new(Erc20Decoder::new()); @@ -383,7 +384,8 @@ async fn configure_token_support( } if installed_token_support.erc721 { - let storage = Arc::new(Erc721Storage::new(&db_setup.erc721_url).await?); + let erc721_pool = Erc721Storage::connect_pool(&db_setup.erc721_url).await?; + let storage = Arc::new(Erc721Storage::from_pool(&db_setup.erc721_url, erc721_pool).await?); tracing::info!("ERC721 database initialized: {}", db_setup.erc721_url); let decoder: Arc = Arc::new(Erc721Decoder::new()); @@ -418,7 +420,10 @@ async fn configure_token_support( } if installed_token_support.erc1155 { - let storage = Arc::new(Erc1155Storage::new(&db_setup.erc1155_url).await?); + let erc1155_pool = DbPoolOptions::new() + .connect_any(&db_setup.erc1155_url) + .await?; + let storage = Arc::new(Erc1155Storage::new(erc1155_pool, &db_setup.erc1155_url).await?); tracing::info!("ERC1155 database initialized: {}", db_setup.erc1155_url); let decoder: Arc = Arc::new(Erc1155Decoder::new()); @@ -492,7 +497,8 @@ async fn run_indexer(config: Config) -> Result<()> { erc1155: !token_targets.erc1155.is_empty(), }, ); - let historical_models = config.historical_models(); + let historical_model_names = config.historical_models(); + let historical_models = parse_historical_models(&historical_model_names, &contracts)?; let token_db_setup = if installed_token_support.any() { Some(resolve_token_db_setup( db_dir, @@ -638,12 +644,12 @@ async fn run_indexer(config: Config) -> Result<()> { }, )); - if matches!(backend, StorageBackend::Sqlite) { + if matches!(backend, DbBackend::Sqlite) { tokio::fs::create_dir_all(db_dir).await?; } match backend { - StorageBackend::Postgres => { + DbBackend::Postgres => { run_with_postgres( &config, &storage_database_url, @@ -656,13 +662,13 @@ async fn run_indexer(config: Config) -> Result<()> { decoder_registry.clone(), contract_type_registry.clone(), installed_external_decoders.clone(), - historical_models.clone(), + historical_models, provider, extractor, ) .await?; } - StorageBackend::Sqlite => { + DbBackend::Sqlite => { run_with_sqlite( &config, &storage_database_url, @@ -675,7 +681,7 @@ async fn run_indexer(config: Config) -> Result<()> { decoder_registry.clone(), contract_type_registry.clone(), installed_external_decoders.clone(), - historical_models.clone(), + historical_models, provider, extractor, ) @@ -699,22 +705,23 @@ async fn run_with_postgres( decoder_registry: SharedDecoderRegistry, contract_type_registry: SharedContractTypeRegistry, installed_external_decoders: HashSet, - historical_models: Vec, + historical_models: HashSet<(Felt, String)>, provider: StarknetProvider, extractor: Box, ) -> Result<()> { let token_provider = Arc::new(provider.clone()); let max_db_connections = config.max_db_connections.unwrap_or(5); - let pool = Arc::new( - PgPoolOptions::new() - .max_connections(max_db_connections) - .connect(storage_database_url) - .await?, - ); + let pool = PgPoolOptions::new() + .max_connections(max_db_connections) + .connect(storage_database_url) + .await?; + + let mut decoder = DojoDecoder::new(pool.clone(), provider); + let introspect_sink = IntrospectPgDb::new(pool.clone(), NamespaceMode::Address); - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - let introspect_sink = IntrospectPgDb::new(pool.clone(), ()); - decoder.store.initialize().await?; + decoder.append_historical(historical_models); + decoder.initialize().await?; + introspect_sink.initialize_introspect_sql_sink().await?; decoder.load_tables(&[]).await?; let decoder: Arc = Arc::new(decoder); @@ -736,23 +743,21 @@ async fn run_with_postgres( .add_decoder(decoder) .add_sink_boxed(Box::new( OrderedSinkPipeline::new("introspect-projection-pipeline") - .push(Box::new(introspect_sink)) - .push(Box::new( - EntitiesHistoricalSink::new( - storage_database_url, - config.max_db_connections, - (), - historical_models, - ) - .await?, - )), + .push(Box::new(introspect_sink)), )); if let Some(tls) = config.tls_config()? { torii_config = torii_config.with_tls(tls); } if config.index_external_contracts { torii_config = torii_config - .with_registry_cache(decoder_registry.clone()) + .with_registry_cache(Arc::new(RwLock::new( + decoder_registry + .read() + .await + .iter() + .map(|(contract, decoder_ids)| ((*contract).into(), decoder_ids.clone())) + .collect(), + ))) .with_command_handler(Box::new(RegisterExternalContractCommandHandler::new( registry_engine_db.clone(), decoder_registry.clone(), @@ -879,36 +884,33 @@ async fn run_with_sqlite( decoder_registry: SharedDecoderRegistry, contract_type_registry: SharedContractTypeRegistry, installed_external_decoders: HashSet, - historical_models: Vec, + historical_models: HashSet<(Felt, String)>, provider: StarknetProvider, extractor: Box, ) -> Result<()> { let token_provider = Arc::new(provider.clone()); - let options = sqlite_connect_options(storage_database_url)?; + let options = SqliteConnectOptions::from_str(storage_database_url)?.create_if_missing(true); let max_db_connections = match config.max_db_connections { Some(limit) => limit.max(1), - None if is_sqlite_memory_path(storage_database_url) => 1, + None if options.is_in_memory() => 1, None => DEFAULT_SQLITE_MAX_CONNECTIONS, }; - let pool = Arc::new( - SqlitePoolOptions::new() - .max_connections(max_db_connections) - .connect_with(options) - .await?, - ); + let pool = SqlitePoolOptions::new() + .max_connections(max_db_connections) + .connect_with(options) + .await?; sqlx::query("PRAGMA journal_mode=WAL") - .execute(pool.as_ref()) + .execute(&pool) .await?; sqlx::query("PRAGMA synchronous=NORMAL") - .execute(pool.as_ref()) - .await?; - sqlx::query("PRAGMA foreign_keys=ON") - .execute(pool.as_ref()) + .execute(&pool) .await?; + sqlx::query("PRAGMA foreign_keys=ON").execute(&pool).await?; - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - decoder.store.initialize().await?; + let mut decoder = DojoDecoder::new(pool.clone(), provider); + decoder.append_historical(historical_models); + decoder.initialize().await?; decoder.load_tables(&[]).await?; let decoder: Arc = Arc::new(decoder); @@ -929,24 +931,23 @@ async fn run_with_sqlite( .with_extractor(extractor) .add_decoder(decoder) .add_sink_boxed(Box::new( - OrderedSinkPipeline::new("introspect-projection-pipeline") - .push(Box::new(IntrospectSqliteDb::new(pool.clone(), ()))) - .push(Box::new( - EntitiesHistoricalSink::new( - storage_database_url, - config.max_db_connections, - (), - historical_models, - ) - .await?, - )), + OrderedSinkPipeline::new("introspect-projection-pipeline").push(Box::new( + IntrospectSqliteDb::new(pool.clone(), NamespaceMode::Address), + )), )); if let Some(tls) = config.tls_config()? { torii_config = torii_config.with_tls(tls); } if config.index_external_contracts { torii_config = torii_config - .with_registry_cache(decoder_registry.clone()) + .with_registry_cache(Arc::new(RwLock::new( + decoder_registry + .read() + .await + .iter() + .map(|(contract, decoder_ids)| ((*contract).into(), decoder_ids.clone())) + .collect(), + ))) .with_command_handler(Box::new(RegisterExternalContractCommandHandler::new( registry_engine_db.clone(), decoder_registry.clone(), @@ -1064,7 +1065,8 @@ async fn run_with_sqlite( #[cfg(test)] mod tests { use super::{ecs_token_storage_urls, InstalledTokenSupport}; - use torii_runtime_common::database::{DatabaseBackend, TokenDbSetup}; + use torii_runtime_common::database::TokenDbSetup; + use torii_sql::DbBackend; fn token_db_setup() -> TokenDbSetup { TokenDbSetup { @@ -1072,10 +1074,10 @@ mod tests { erc20_url: "./torii-data/erc20.db".to_string(), erc721_url: "./torii-data/erc721.db".to_string(), erc1155_url: "./torii-data/erc1155.db".to_string(), - engine_backend: DatabaseBackend::Sqlite, - erc20_backend: DatabaseBackend::Sqlite, - erc721_backend: DatabaseBackend::Sqlite, - erc1155_backend: DatabaseBackend::Sqlite, + engine_backend: DbBackend::Sqlite, + erc20_backend: DbBackend::Sqlite, + erc721_backend: DbBackend::Sqlite, + erc1155_backend: DbBackend::Sqlite, } } diff --git a/bins/torii-introspect-synth/Cargo.toml b/bins/torii-introspect-synth/Cargo.toml index 8caedd45..174444e5 100644 --- a/bins/torii-introspect-synth/Cargo.toml +++ b/bins/torii-introspect-synth/Cargo.toml @@ -8,10 +8,9 @@ name = "torii-introspect-synth" path = "src/main.rs" [dependencies] -torii = { path = "../../" } -torii-runtime-common.workspace = true -torii-dojo = { path = "../../crates/dojo" } -torii-introspect-postgres-sink = { path = "../../crates/introspect-postgres-sink" } +torii.workspace = true +torii-dojo = { workspace = true, features = ["postgres"] } +torii-introspect-sql-sink = { workspace = true, features = ["postgres"] } tokio = { version = "1", features = ["full"] } tracing = "0.1" @@ -22,11 +21,13 @@ async-trait = "0.1" chrono = "0.4" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "postgres"] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "postgres"] } starknet = "0.17" -starknet-types-core.workspace = true dojo-introspect.workspace = true introspect-types.workspace = true +starknet-types-raw.workspace = true +torii-types.workspace = true + [lints] workspace = true diff --git a/bins/torii-introspect-synth/README.md b/bins/torii-introspect-synth/README.md new file mode 100644 index 00000000..1b0c4d52 --- /dev/null +++ b/bins/torii-introspect-synth/README.md @@ -0,0 +1,86 @@ +# torii-introspect-synth (binary) + +**Deterministic Dojo introspect profiler.** Generates a stream of +synthetic `ModelWithSchemaRegistered` + `StoreSetRecord` / +`StoreUpdateMember` events, pumps them through `DojoDecoder` and +`introspect-sql-sink`, and writes a timing report. Postgres-only, same +family as `torii-erc20-synth` / `torii-tokens-synth`. + +## Role in Torii + +Perf regression harness for the introspect path. Keeps the decoder + +`IntrospectPgDb` hot without requiring a live Dojo world — useful when +changing schema generation, JSON serialisation, or the per-record write +path in `introspect-sql-sink`. + +## Architecture + +```text ++--------------------------+ +| clap::Parser Config | --db-url (Postgres) +| | --from-block, --block-count +| | --records-per-block +| | --blocks-per-batch, --seed +| | --output-root, --reset-schema ++------------+-------------+ + | + v ++--------------------------+ +| Synthetic extractor | bakes a single model: +| (in main.rs) | NAMESPACE="synthetic" +| | MODEL_NAME="position" +| | TABLE_NAME="synthetic-position" +| | FROM_ADDRESS=0x100 +| | emits ModelWithSchemaRegistered first, +| | then --records-per-block StoreSetRecord / +| | StoreUpdateMember events per block, +| | deterministic via --seed ++------------+-------------+ + | + v ++--------------------------+ +| DojoDecoder + | fetcher = in-process DojoSchemaFetcher that +| IntrospectPgDb | returns the synthetic schema (no RPC); +| | DecoderContext routes to DojoDecoder ++------------+-------------+ + | + v + tight loop: extract → decode → sink.process → commit_cursor + record StageSample (extract/decode/sink/commit/loop_total) + | + v ++--------------------------+ +| RunReport JSON | perf/synthetic-runs// ++--------------------------+ +``` + +## Deep Dive + +### CLI + +| Flag | Default | Purpose | +|---|---|---| +| `--db-url` (env `DATABASE_URL`) | `postgres://torii:torii@localhost:5432/torii` | **Postgres only** | +| `--from-block` | `1_000_000` | Synthetic starting block | +| `--block-count` | `50` | Blocks produced in the run | +| `--records-per-block` | `250` | `StoreSetRecord` / `StoreUpdateMember` events per block | +| `--blocks-per-batch` | `1` | Blocks per extract cycle | +| `--seed` | `42` | Deterministic RNG seed | +| `--output-root` | `perf/synthetic-runs` | Artifact root | +| `--reset-schema` | `true` | Drop and recreate the synthetic schema before the run | + +### Internal shape + +- Hand-rolls a `DojoSchemaFetcher` that returns a static schema for the synthetic model — no RPC is called. +- Uses `IntrospectPgDb` directly (not `introspect-sql-sink::runtime::IntrospectDb` dispatch) to keep the critical path narrow. +- Persists extractor state under `EXTRACTOR_TYPE = "synthetic_introspect"` / `STATE_KEY = "last_block"` in `EngineDb` so restarts pick up where they stopped (unless `--reset-schema`). + +### Workspace dependencies + +`torii`, `torii-dojo`, `torii-introspect`, `torii-introspect-sql-sink` (feature `postgres`), `torii-types`, `dojo-introspect`, `introspect-types`, `sqlx` (postgres), `clap`, `serde`, `async-trait`, `tokio`, `starknet-types-raw`, `tracing`, `anyhow`. + +### Extension Points + +- New event mix → widen the synthetic extractor's per-block generator (add more `StoreUpdateRecord` variants, more models, etc.). +- Additional model → bump `NAMESPACE` / `MODEL_NAME` constants and the fetcher's schema map. +- Switch to SQLite → not supported; the binary targets `IntrospectPgDb` explicitly. diff --git a/bins/torii-introspect-synth/src/main.rs b/bins/torii-introspect-synth/src/main.rs index 9f71e3e6..bf59961f 100644 --- a/bins/torii-introspect-synth/src/main.rs +++ b/bins/torii-introspect-synth/src/main.rs @@ -5,14 +5,12 @@ use dojo_introspect::events::{ModelWithSchemaRegistered, StoreSetRecord, StoreUp use dojo_introspect::selector::compute_selector_from_namespace_and_name; use dojo_introspect::serde::primitive; use dojo_introspect::{DojoIntrospectResult, DojoSchema, DojoSchemaFetcher}; -use introspect_types::utils::{ascii_str_to_felt, string_to_cairo_serialize_byte_array}; +use introspect_types::utils::string_to_cairo_serialize_byte_array; use introspect_types::CairoEventInfo; use serde::Serialize; use sqlx::postgres::PgPoolOptions; use sqlx::{PgPool, Row}; -use starknet::core::types::EmittedEvent; -use starknet::core::utils::get_selector_from_name; -use starknet_types_core::felt::Felt; +use starknet_types_raw::Felt; use std::collections::HashMap; use std::fs; use std::path::PathBuf; @@ -26,8 +24,9 @@ use torii::etl::sink::{EventBus, Sink, SinkContext}; use torii::etl::Decoder; use torii::grpc::SubscriptionManager; use torii_dojo::decoder::DojoDecoder; -use torii_dojo::store::postgres::PgStore; -use torii_introspect_postgres_sink::IntrospectPgDb; +use torii_dojo::store::DojoStoreTrait; +use torii_introspect_sql_sink::IntrospectPgDb; +use torii_types::event::StarknetEvent; const EXTRACTOR_TYPE: &str = "synthetic_introspect"; const STATE_KEY: &str = "last_block"; @@ -173,24 +172,31 @@ impl SyntheticIntrospectExtractor { } fn table_id(&self) -> Felt { - compute_selector_from_namespace_and_name(NAMESPACE, MODEL_NAME) + compute_selector_from_namespace_and_name(NAMESPACE, MODEL_NAME).into() } fn score_selector(&self) -> Felt { - get_selector_from_name("score").expect("valid selector") + Felt::selector("score") } - fn model_registration_event(&self, block_number: u64) -> EmittedEvent { - let mut keys = vec![ModelWithSchemaRegistered::SELECTOR]; - keys.extend(string_to_cairo_serialize_byte_array(MODEL_NAME)); - keys.extend(string_to_cairo_serialize_byte_array(NAMESPACE)); - - EmittedEvent { + fn model_registration_event(&self, block_number: u64) -> StarknetEvent { + let mut keys = vec![ModelWithSchemaRegistered::SELECTOR.into()]; + keys.extend( + string_to_cairo_serialize_byte_array(MODEL_NAME) + .into_iter() + .map(Felt::from), + ); + keys.extend( + string_to_cairo_serialize_byte_array(NAMESPACE) + .into_iter() + .map(Felt::from), + ); + + StarknetEvent { from_address: FROM_ADDRESS, keys, data: encode_legacy_schema(), - block_hash: Some(block_hash_for(block_number)), - block_number: Some(block_number), + block_number, transaction_hash: tx_hash_for(block_number, 0), } } @@ -221,16 +227,15 @@ impl SyntheticIntrospectExtractor { entity_id: Felt, owner: Felt, initial_score: Felt, - ) -> EmittedEvent { + ) -> StarknetEvent { let mut data = encode_array([owner]); data.extend(encode_array([initial_score])); - EmittedEvent { + StarknetEvent { from_address: FROM_ADDRESS, - keys: vec![StoreSetRecord::SELECTOR, self.table_id(), entity_id], + keys: vec![StoreSetRecord::SELECTOR.into(), self.table_id(), entity_id], data, - block_hash: Some(block_hash_for(block_number)), - block_number: Some(block_number), + block_number, transaction_hash: tx_hash_for(block_number, record_tx_index(record_index, false)), } } @@ -241,18 +246,17 @@ impl SyntheticIntrospectExtractor { record_index: usize, entity_id: Felt, final_score: Felt, - ) -> EmittedEvent { - EmittedEvent { + ) -> StarknetEvent { + StarknetEvent { from_address: FROM_ADDRESS, keys: vec![ - StoreUpdateMember::SELECTOR, + StoreUpdateMember::SELECTOR.into(), self.table_id(), entity_id, self.score_selector(), ], data: encode_array([final_score]), - block_hash: Some(block_hash_for(block_number)), - block_number: Some(block_number), + block_number, transaction_hash: tx_hash_for(block_number, record_tx_index(record_index, true)), } } @@ -398,7 +402,10 @@ struct NeverFetchSchema; #[async_trait] impl DojoSchemaFetcher for NeverFetchSchema { - async fn schema(&self, contract_address: Felt) -> DojoIntrospectResult { + async fn schema( + &self, + contract_address: starknet::core::types::Felt, + ) -> DojoIntrospectResult { panic!("provider fetch should not be called in synthetic introspect run: {contract_address:#x}"); } } @@ -418,14 +425,12 @@ async fn main() -> Result<()> { fs::create_dir_all(&output_dir) .with_context(|| format!("failed to create output dir {}", output_dir.display()))?; - let pool = Arc::new( - PgPoolOptions::new() - .max_connections(5) - .connect(&config.db_url) - .await?, - ); + let pool = PgPoolOptions::new() + .max_connections(5) + .connect(&config.db_url) + .await?; if config.reset_schema { - reset_schema(pool.as_ref()).await?; + reset_schema(&pool).await?; } let started = Instant::now(); @@ -447,8 +452,8 @@ async fn main() -> Result<()> { .await?, ); - let decoder = DojoDecoder::, _>::new(pool.clone(), NeverFetchSchema); - decoder.store.initialize().await?; + let decoder = DojoDecoder::new(pool.clone(), NeverFetchSchema); + decoder.initialize().await?; decoder.load_tables(&[]).await?; let decoder: Arc = Arc::new(decoder); let decoder_context = @@ -476,7 +481,7 @@ async fn main() -> Result<()> { } total_events += batch.events.len(); - let envelopes = decoder_context.decode(&batch.events).await?; + let envelopes = decoder_context.decode_events(&batch.events).await?; total_envelopes += envelopes.len(); sink.process(&envelopes, &batch).await?; @@ -486,7 +491,7 @@ async fn main() -> Result<()> { } } - let verification = verify_run(pool.as_ref(), &config).await?; + let verification = verify_run(&pool, &config).await?; let summary = Summary { run_id: run_id.clone(), duration_ms: started.elapsed().as_millis(), @@ -638,7 +643,7 @@ async fn verify_run(pool: &PgPool, config: &Config) -> Result { WHERE "entity_id" = $1 "# )) - .bind(verified_entity_id.to_bytes_be().to_vec()) + .bind(verified_entity_id.to_be_bytes().to_vec()) .fetch_one(pool) .await?; @@ -657,7 +662,7 @@ async fn verify_run(pool: &PgPool, config: &Config) -> Result { } fn encode_legacy_schema() -> Vec { - let mut schema = vec![ascii_str_to_felt(MODEL_NAME)]; + let mut schema = vec![Felt::from_short_ascii_str_unchecked(MODEL_NAME)]; schema.extend(encode_attributes(&[])); schema.extend(encode_columns()); schema @@ -668,14 +673,14 @@ fn encode_columns() -> Vec { columns.extend(encode_column( "owner", &["key"], - primitive::CONTRACT_ADDRESS_FELT, + primitive::CONTRACT_ADDRESS_FELT.into(), )); - columns.extend(encode_column("score", &[], primitive::U32_FELT)); + columns.extend(encode_column("score", &[], primitive::U32_FELT.into())); columns } fn encode_column(name: &str, attributes: &[&str], primitive_type: Felt) -> Vec { - let mut column = vec![ascii_str_to_felt(name)]; + let mut column = vec![Felt::from_short_ascii_str_unchecked(name)]; column.extend(encode_attributes(attributes)); column.extend(vec![Felt::ZERO, primitive_type]); column @@ -686,7 +691,7 @@ fn encode_attributes(attributes: &[&str]) -> Vec { encoded.extend( attributes .iter() - .map(|attribute| ascii_str_to_felt(attribute)), + .map(|attribute| Felt::from_short_ascii_str_unchecked(attribute)), ); encoded } diff --git a/bins/torii-tokens-synth/Cargo.toml b/bins/torii-tokens-synth/Cargo.toml index 00a651f9..297b9d1c 100644 --- a/bins/torii-tokens-synth/Cargo.toml +++ b/bins/torii-tokens-synth/Cargo.toml @@ -11,6 +11,7 @@ path = "src/main.rs" torii = { path = "../../" } torii-runtime-common.workspace = true torii-common.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } torii-erc20 = { path = "../../crates/torii-erc20" } torii-erc721 = { path = "../../crates/torii-erc721" } torii-erc1155 = { path = "../../crates/torii-erc1155" } diff --git a/bins/torii-tokens-synth/README.md b/bins/torii-tokens-synth/README.md new file mode 100644 index 00000000..4e02b605 --- /dev/null +++ b/bins/torii-tokens-synth/README.md @@ -0,0 +1,92 @@ +# torii-tokens-synth (binary) + +**Deterministic multi-standard token profiler.** Drives the ERC20, +ERC721, and ERC1155 pipelines with hand-rolled synthetic extractors and +produces a uniform `RunReport` JSON, so the three sinks (and the +combination) can be benchmarked side by side. Postgres-only, like +`torii-erc20-synth`. + +## Role in Torii + +One-stop perf harness for all three token sinks. Same shape as +`torii-erc20-synth` but with clap subcommands so you pick which sink(s) +to exercise. + +## Architecture + +```text ++--------------------------+ +| clap::Subcommand Cli | subcommands: +| | erc20 Erc20ProfileConfig +| | erc721 Erc721ProfileConfig +| | erc1155 Erc1155ProfileConfig +| | all AllProfileConfig ++------------+-------------+ + | + v ++----------------------------------------+ +| CommonConfig (flattened into each) | +| --db-url (Postgres only) | +| --from-block, --block-count | +| --tx-per-block, --blocks-per-batch | +| --token-count, --wallet-count | +| --seed, --output-root | +| --reset-schema | ++------------+---------------------------+ + | + +--- Erc20SpecificConfig (--approval-ratio-bps) + +--- Erc721SpecificConfig (--approval-ratio-bps, + | --approval-for-all-ratio-bps, + | --metadata-update-ratio-bps, + | --batch-metadata-update-ratio-bps, + | --max-token-id) + +--- Erc1155SpecificConfig (--transfer-single-ratio-bps, + --transfer-batch-ratio-bps, + --min-batch-size, --max-batch-size, + --approval-for-all-ratio-bps, + --uri-ratio-bps, --token-id-count) + +subcommand.run(): + drop_postgres_schemas(engine, erc20, erc721, erc1155) when --reset-schema + open storages for installed standards + register SyntheticErc*MetadataCommandHandler + + SyntheticErc*TokenUriCommandHandler (hand-rolled, no HTTP) + extractor = Synthetic{Erc20|Erc721|Erc1155}Extractor (deterministic) + tight loop: extract → decode → sink.process → commit_cursor → StageSample + write RunReport JSON to perf/runs// +``` + +## Deep Dive + +### Subcommands & flags + +| Subcommand | Exercises | Extra flags | +|---|---|---| +| `erc20` | Erc20 sink alone | `--approval-ratio-bps` | +| `erc721` | Erc721 sink alone | `--approval-ratio-bps`, `--approval-for-all-ratio-bps`, `--metadata-update-ratio-bps`, `--batch-metadata-update-ratio-bps`, `--max-token-id` | +| `erc1155` | Erc1155 sink alone | `--transfer-single-ratio-bps`, `--transfer-batch-ratio-bps`, `--min-batch-size`, `--max-batch-size`, `--approval-for-all-ratio-bps`, `--uri-ratio-bps`, `--token-id-count` | +| `all` | All three concurrently | Combines all of the above | + +Common flags on every subcommand: `--db-url` (default +`postgres://torii:torii@localhost:5432/torii`), `--from-block`, +`--block-count`, `--tx-per-block`, `--blocks-per-batch`, `--token-count`, +`--wallet-count`, `--seed`, `--output-root`, `--reset-schema`. + +### Internal shape + +- Hand-rolled `CommandHandler` implementations for metadata + token-URI so the perf numbers reflect storage only (no reqwest, no JSON parsing): + - `SyntheticErc20MetadataCommandHandler` + - `SyntheticErc721MetadataCommandHandler` / `SyntheticErc721TokenUriCommandHandler` + - `SyntheticErc1155MetadataCommandHandler` / `SyntheticErc1155TokenUriCommandHandler` +- Reuses `drop_postgres_schemas` + `initialize_sink_with_command_handlers` from `torii-runtime-common`. +- Output shape is identical to `torii-erc20-synth` (`RunReport` JSON: `config`, `totals`, `throughput`, `stage_latency_ms`, `stage_share_percent`, `slowest_cycles`, `blocking_suspects`, `db`). + +### Workspace dependencies + +`torii`, `torii-erc20`, `torii-erc721`, `torii-erc1155`, `torii-common`, `torii-runtime-common`, `torii-sql`, plus `clap` (with `derive`), `tokio`, `tracing`, `serde`, `serde_json`, `anyhow`. Feature `profiling` pulls `pprof`. + +### Extension Points + +- New rate-shape for a sink → add fields to the `*SpecificConfig` + propagate through the matching `Synthetic*Config` in the token crate. +- Cross-sink scenarios → the `all` subcommand already runs all three; for more specific combinations add a new subcommand variant. +- This binary is Postgres-only by design. For SQLite perf, fork one; do not parameterise here. diff --git a/bins/torii-tokens-synth/src/main.rs b/bins/torii-tokens-synth/src/main.rs index 792c4a14..c11d1c11 100644 --- a/bins/torii-tokens-synth/src/main.rs +++ b/bins/torii-tokens-synth/src/main.rs @@ -16,7 +16,7 @@ use torii::etl::engine_db::{EngineDb, EngineDbConfig}; use torii::etl::extractor::SyntheticExtractor; use torii::etl::sink::Sink; use torii::etl::{Decoder, DecoderContext}; -use torii_common::{TokenUriResult, TokenUriStore}; +use torii_common::{bytes_to_u256, u256_to_bytes, TokenUriResult, TokenUriStore}; use torii_erc1155::handlers::{FetchErc1155MetadataCommand, RefreshErc1155TokenUriCommand}; use torii_erc1155::{ Erc1155Decoder, Erc1155Sink, Erc1155Storage, SyntheticErc1155Config, SyntheticErc1155Extractor, @@ -30,6 +30,7 @@ use torii_erc721::{ Erc721Decoder, Erc721Sink, Erc721Storage, SyntheticErc721Config, SyntheticErc721Extractor, }; use torii_runtime_common::sink::{drop_postgres_schemas, initialize_sink_with_command_handlers}; +use torii_sql::DbPoolOptions; const SYNTH_COMMAND_QUEUE_SIZE: usize = 4096; const SYNTH_METADATA_COMMAND_PARALLELISM: usize = 1; @@ -162,8 +163,8 @@ impl CommandHandler for SyntheticErc721TokenUriCommandHandler { self.storage .store_token_uri(&TokenUriResult { - contract: command.contract, - token_id: command.token_id, + contract: command.contract.into(), + token_id: bytes_to_u256(&u256_to_bytes(command.token_id)), uri: Some(format!( "synthetic://erc721/{:#x}/{}", command.contract, command.token_id @@ -220,7 +221,7 @@ impl CommandHandler for SyntheticErc1155TokenUriCommandHandler { self.storage .store_token_uri(&TokenUriResult { contract: command.contract, - token_id: command.token_id, + token_id: bytes_to_u256(&u256_to_bytes(command.token_id)), uri: Some(format!( "synthetic://erc1155/{:#x}/{}", command.contract, command.token_id @@ -507,7 +508,8 @@ async fn run_erc20_profile(config: Erc20ProfileConfig) -> Result<()> { fs::create_dir_all(&output_dir) .with_context(|| format!("failed to create output dir {}", output_dir.display()))?; - let storage = Arc::new(Erc20Storage::new(&config.common.db_url).await?); + let erc20_pool = Erc20Storage::connect_pool(&config.common.db_url).await?; + let storage = Arc::new(Erc20Storage::from_pool(&config.common.db_url, erc20_pool).await?); let mut sink = Erc20Sink::new(storage.clone()).with_metadata_pipeline( SYNTH_METADATA_COMMAND_PARALLELISM, @@ -565,7 +567,7 @@ async fn run_erc20_profile(config: Erc20ProfileConfig) -> Result<()> { } let decode_start = Instant::now(); - let envelopes = decoder_context.decode(&batch.events).await?; + let envelopes = decoder_context.decode_events(&batch.events).await?; let decode_ms = ms(decode_start.elapsed()); let sink_start = Instant::now(); @@ -703,7 +705,7 @@ async fn run_erc721_profile(config: Erc721ProfileConfig) -> Result<()> { } let decode_start = Instant::now(); - let envelopes = decoder_context.decode(&batch.events).await?; + let envelopes = decoder_context.decode_events(&batch.events).await?; let decode_ms = ms(decode_start.elapsed()); let sink_start = Instant::now(); @@ -780,7 +782,10 @@ async fn run_erc1155_profile(config: Erc1155ProfileConfig) -> Result<()> { fs::create_dir_all(&output_dir) .with_context(|| format!("failed to create output dir {}", output_dir.display()))?; - let storage = Arc::new(Erc1155Storage::new(&config.common.db_url).await?); + let erc1155_pool = DbPoolOptions::new() + .connect_any(&config.common.db_url) + .await?; + let storage = Arc::new(Erc1155Storage::new(erc1155_pool, &config.common.db_url).await?); let mut sink = Erc1155Sink::new(storage.clone()) .with_metadata_commands() @@ -843,7 +848,7 @@ async fn run_erc1155_profile(config: Erc1155ProfileConfig) -> Result<()> { } let decode_start = Instant::now(); - let envelopes = decoder_context.decode(&batch.events).await?; + let envelopes = decoder_context.decode_events(&batch.events).await?; let decode_ms = ms(decode_start.elapsed()); let sink_start = Instant::now(); diff --git a/bins/torii-tokens/Cargo.toml b/bins/torii-tokens/Cargo.toml index b6a98319..8ae30c72 100644 --- a/bins/torii-tokens/Cargo.toml +++ b/bins/torii-tokens/Cargo.toml @@ -10,13 +10,14 @@ path = "src/main.rs" [dependencies] # Core dependencies -torii = { path = "../../" } -torii-common = { path = "../../crates/torii-common" } +torii.workspace = true +torii-common.workspace = true torii-config-common.workspace = true torii-runtime-common.workspace = true -torii-erc20 = { path = "../../crates/torii-erc20" } -torii-erc721 = { path = "../../crates/torii-erc721" } -torii-erc1155 = { path = "../../crates/torii-erc1155" } +torii-erc20.workspace = true +torii-erc721.workspace = true +torii-erc1155.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } # Async runtime tokio = { version = "1", features = ["full"] } diff --git a/bins/torii-tokens/README.md b/bins/torii-tokens/README.md index a4c4361c..04a1389d 100644 --- a/bins/torii-tokens/README.md +++ b/bins/torii-tokens/README.md @@ -1,486 +1,119 @@ -# Torii Tokens - -Unified Starknet token indexer for ERC20, ERC721, and ERC1155 tokens. - -## Quick Start - -```bash -# Block-range mode: auto-discovers all token contracts -torii-tokens --from-block 100000 - -# Enable observability (metrics endpoint + collection) -torii-tokens --observability --from-block 100000 - -# Event mode: index specific contracts only -torii-tokens --mode event --erc20 0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7 --from-block 100000 -``` - -## Features - -- **Multi-token support**: ERC20, ERC721, and ERC1155 in a single indexer -- **Balance tracking**: Real-time balance computation with on-chain verification -- **Auto-discovery**: Automatically identifies token contracts by ABI inspection (block-range mode) -- **gRPC subscriptions**: Real-time streaming of token events -- **Historical queries**: Paginated queries for transfers, approvals, and ownership -- **Flexible extraction**: Block-range mode for full indexing, event mode for targeted contracts - -## Installation - -```bash -# From the repository root -cargo build --release -p torii-tokens - -# Binary will be at target/release/torii-tokens -``` - -## Usage - -### Block Range Mode (default) - -Fetches ALL events from each block. Best for full chain indexing with auto-discovery. - -```bash -# Auto-discover all token contracts from block 100000 -torii-tokens --from-block 100000 - -# Index with explicit contracts (will also auto-discover others) -torii-tokens --erc20 0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7 --from-block 0 - -# Index multiple token types -torii-tokens \ - --erc20 0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7 \ - --erc721 0x...nft \ - --erc1155 0x...game_items \ - --from-block 0 -``` - -### Event Mode - -Uses `starknet_getEvents` with per-contract cursors. Best for indexing specific contracts. - -```bash -# Index specific ERC20 contracts -torii-tokens --mode event \ - --erc20 0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7 \ - --from-block 0 - -# Index multiple contracts -torii-tokens --mode event \ - --erc20 0x049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7,0x04718f5a0Fc34cC1AF16A1cdee98fFB20C31f5cD61D6Ab07201858f4287c938D \ - --from-block 0 -``` - -### Custom Configuration - -```bash -# Custom RPC and port -torii-tokens \ - --rpc-url https://your-rpc.example.com \ - --port 8080 \ - --from-block 0 - -# Custom database directory -torii-tokens --db-dir /path/to/data --from-block 0 -``` - -### Throughput Tuning - -For large backfills, tune concurrency in this order: `--rpc-parallelism`, then queue depth. - -```bash -# Conservative baseline -torii-tokens --from-block 0 \ - --batch-size 1000 \ - --rpc-parallelism 4 \ - --max-prefetch-batches 8 \ - --metadata-mode deferred - -# More aggressive (if RPC endpoint can sustain it) -torii-tokens --from-block 0 \ - --batch-size 1000 \ - --rpc-parallelism 6 \ - --max-prefetch-batches 8 \ - --metadata-mode deferred -``` - -Notes: - -- `--rpc-parallelism`: concurrent chunked RPC requests (`0` = auto). -- `--max-prefetch-batches`: extracted batches buffered ahead of decode/store. -- `--metadata-mode deferred`: reduce metadata-side RPC/load during backfill. -- `--metadata-parallelism`, `--metadata-queue-capacity`, `--metadata-max-retries` control async metadata workers (ERC20), queue depth, and capped retry attempts. -- `--metadata-queue-capacity` also controls the token-URI request queue for ERC721/ERC1155 in `inline` mode (increase this if you see `Dropping token URI requests: queue is full`). - -### CLI Options - -| Option | Default | Description | -|--------|---------|-------------| -| `--mode` | `block-range` | Extraction mode (`block-range` or `event`) | -| `--rpc-url` | Cartridge mainnet | Starknet RPC endpoint | -| `--from-block` | `0` | Starting block number | -| `--to-block` | None | Ending block (None = follow chain head) | -| `--db-dir` | `./torii-data` | Directory for database files | -| `--database-url` | None | Engine DB URL/path (e.g. `postgres://...`) | -| `--port` | `3000` | HTTP/gRPC server port | -| `--observability` | `false` | Enable observability (Prometheus metrics endpoint + metric collection) | -| `--erc20` | None | ERC20 contract addresses (comma-separated) | -| `--erc721` | None | ERC721 contract addresses (comma-separated) | -| `--erc1155` | None | ERC1155 contract addresses (comma-separated) | -| `--batch-size` | `50` | Blocks per batch (block-range mode) | -| `--event-chunk-size` | `1000` | Events per RPC request (event mode) | -| `--event-block-batch-size` | `10000` | Block range per iteration (event mode) | -| `--max-prefetch-batches` | `2` | Number of extracted batches prefetched ahead | -| `--rpc-parallelism` | `0` | Concurrent chunked RPC requests (`0` = auto) | -| `--metadata-mode` | `inline` | Metadata behavior (`inline` or `deferred`) | -| `--metadata-parallelism` | `8` | Async metadata workers (ERC20 metadata pipeline) | -| `--metadata-queue-capacity` | `2048` | Metadata queue size (ERC20 metadata jobs + ERC721/ERC1155 token-URI request queue in `inline` mode) | -| `--metadata-max-retries` | `5` | Max metadata retry attempts (capped backoff) | - -### Metadata Mode - -- `inline` (default): metadata jobs run while indexing. -- `deferred`: disables ERC721 metadata/token-URI fetch setup to maximize ingest throughput. - -Current behavior details: - -- ERC20 metadata is fetched asynchronously in background workers and is configured by metadata pipeline flags. -- ERC721/ERC1155 token-URI fetch requests use `--metadata-queue-capacity` for queue depth in inline mode. -- ERC1155 token-URI service is disabled in deferred mode; contract metadata fetch can still occur via sink metadata fetcher. - -## Extraction Modes - -### Block Range Mode - -- Fetches complete block data in batches -- Inspects contract ABIs to identify token types automatically -- Single cursor tracks overall progress -- Best for full chain indexing - -### Event Mode - -- Each contract has its own cursor -- Adding a new contract starts fresh from `--from-block` -- Existing contracts resume from their saved position -- Best for targeted indexing with lower resource usage - -**Adding contracts in event mode:** -```bash -# Initial run -torii-tokens --mode event --erc20 0x...ETH,0x...STRK --from-block 0 - -# Add USDC later - ETH and STRK resume from their cursors, USDC starts from block 0 -torii-tokens --mode event --erc20 0x...ETH,0x...STRK,0x...USDC --from-block 0 -``` - -## Database and Cursors - -The indexer maintains internal cursors to track indexing progress. On restart, indexing resumes from the last processed position. - -**To reset and re-index from scratch, delete the database directory:** - -```bash -rm -rf ./torii-data -torii-tokens --from-block 0 -``` - -The indexer creates separate SQLite databases for each component: - -``` - -Set `--database-url` (or `DATABASE_URL`) to run engine + token storages on PostgreSQL. If unset, the local SQLite files below are used. -./torii-data/ - engine.db # ETL state, cursors, statistics - erc20.db # ERC20 transfers, approvals, balances - erc721.db # ERC721 transfers, ownership - erc1155.db # ERC1155 transfers, balances -``` - -Database resolution logic here is shared via `torii-runtime-common` (`resolve_token_db_setup`) to keep SQLite/PostgreSQL behavior consistent across binaries. - -Each database uses WAL mode for performance and crash safety. - -## gRPC API Reference - -### Available Services - -```bash -# List all services -grpcurl -plaintext localhost:3000 list - -# Services: -# - grpc.reflection.v1.ServerReflection -# - torii.Torii (EventBus subscriptions) -# - torii.sinks.erc20.Erc20 -# - torii.sinks.erc721.Erc721 -# - torii.sinks.erc1155.Erc1155 -``` - -### Address Encoding - -All addresses and U256 values are encoded as **base64-encoded big-endian bytes**. - -```bash -# Convert hex address to base64 -printf '%s' "049D36570D4e46f48e99674bd3fcc84644DdD6b96F7C741B1562B82f9e004dC7" | xxd -r -p | base64 -# Output: BJ02Vw1ORvSOmWdL0/zIRkTd1rlvfHQbFWK4L54ATcc= - -# Convert base64 back to hex -echo "BJ02Vw1ORvSOmWdL0/zIRkTd1rlvfHQbFWK4L54ATcc=" | base64 -d | xxd -p -``` - ---- - -### ERC20 Service - -**Service:** `torii.sinks.erc20.Erc20` - -#### GetStats - -```bash -grpcurl -plaintext localhost:3000 torii.sinks.erc20.Erc20/GetStats -``` - -#### GetTransfers - -```bash -# Get all transfers (first 100) -grpcurl -plaintext -d '{}' localhost:3000 torii.sinks.erc20.Erc20/GetTransfers - -# Filter by wallet (matches from OR to) -grpcurl -plaintext -d '{ - "filter": {"wallet": "BJ02Vw1ORvSOmWdL0/zIRkTd1rlvfHQbFWK4L54ATcc="}, - "limit": 10 -}' localhost:3000 torii.sinks.erc20.Erc20/GetTransfers - -# Filter by token contract -grpcurl -plaintext -d '{ - "filter": {"tokens": ["BJ02Vw1ORvSOmWdL0/zIRkTd1rlvfHQbFWK4L54ATcc="]}, - "limit": 10 -}' localhost:3000 torii.sinks.erc20.Erc20/GetTransfers - -# Filter by block range -grpcurl -plaintext -d '{ - "filter": {"blockFrom": "100000", "blockTo": "200000"}, - "limit": 10 -}' localhost:3000 torii.sinks.erc20.Erc20/GetTransfers - -# Pagination -grpcurl -plaintext -d '{ - "cursor": {"blockNumber": "150000", "id": "42"}, - "limit": 10 -}' localhost:3000 torii.sinks.erc20.Erc20/GetTransfers -``` - -#### GetApprovals - -```bash -# Filter by account (matches owner OR spender) -grpcurl -plaintext -d '{ - "filter": {"account": "BJ02Vw1ORvSOmWdL0/zIRkTd1rlvfHQbFWK4L54ATcc="}, - "limit": 10 -}' localhost:3000 torii.sinks.erc20.Erc20/GetApprovals -``` - -#### SubscribeTransfers - -```bash -# Subscribe to all transfers -grpcurl -plaintext -d '{"clientId": "my-client"}' \ - localhost:3000 torii.sinks.erc20.Erc20/SubscribeTransfers - -# Subscribe with wallet filter -grpcurl -plaintext -d '{ - "clientId": "my-client", - "filter": {"wallet": "BJ02Vw1ORvSOmWdL0/zIRkTd1rlvfHQbFWK4L54ATcc="} -}' localhost:3000 torii.sinks.erc20.Erc20/SubscribeTransfers -``` - -#### SubscribeApprovals - -```bash -grpcurl -plaintext -d '{"clientId": "my-client"}' \ - localhost:3000 torii.sinks.erc20.Erc20/SubscribeApprovals -``` - -**ERC20 Filter Fields:** - -| Field | Type | Description | -|-------|------|-------------| -| `wallet` | bytes | Matches `from` OR `to` | -| `from` | bytes | Exact sender address | -| `to` | bytes | Exact receiver address | -| `tokens` | bytes[] | Token contract whitelist | -| `direction` | enum | `DIRECTION_ALL`, `DIRECTION_SENT`, `DIRECTION_RECEIVED` | -| `blockFrom` | uint64 | Minimum block number | -| `blockTo` | uint64 | Maximum block number | - ---- - -### ERC721 Service - -**Service:** `torii.sinks.erc721.Erc721` - -#### GetStats - -```bash -grpcurl -plaintext localhost:3000 torii.sinks.erc721.Erc721/GetStats -``` - -#### GetTransfers - -```bash -# Filter by wallet -grpcurl -plaintext -d '{ - "filter": {"wallet": "...base64..."}, - "limit": 10 -}' localhost:3000 torii.sinks.erc721.Erc721/GetTransfers - -# Filter by specific NFT token IDs -grpcurl -plaintext -d '{ - "filter": {"tokenIds": ["AQ==", "Ag==", "Aw=="]}, - "limit": 10 -}' localhost:3000 torii.sinks.erc721.Erc721/GetTransfers -``` - -#### GetOwnership - -```bash -# Get all NFTs owned by an address -grpcurl -plaintext -d '{ - "filter": {"owner": "...base64..."}, - "limit": 100 -}' localhost:3000 torii.sinks.erc721.Erc721/GetOwnership -``` - -#### GetOwner - -```bash -# Get owner of a specific NFT -grpcurl -plaintext -d '{ - "token": "...nft_contract_base64...", - "tokenId": "AQ==" -}' localhost:3000 torii.sinks.erc721.Erc721/GetOwner -``` - -#### SubscribeTransfers - -```bash -grpcurl -plaintext -d '{ - "clientId": "my-client", - "filter": {"tokenIds": ["AQ==", "Ag=="]} -}' localhost:3000 torii.sinks.erc721.Erc721/SubscribeTransfers -``` - -**ERC721 Filter Fields:** - -| Field | Type | Description | -|-------|------|-------------| -| `wallet` | bytes | Matches `from` OR `to` | -| `from` | bytes | Exact sender address | -| `to` | bytes | Exact receiver address | -| `tokens` | bytes[] | NFT contract whitelist | -| `tokenIds` | bytes[] | Specific NFT token IDs | -| `blockFrom` | uint64 | Minimum block number | -| `blockTo` | uint64 | Maximum block number | - ---- - -### ERC1155 Service - -**Service:** `torii.sinks.erc1155.Erc1155` - -#### GetStats - -```bash -grpcurl -plaintext localhost:3000 torii.sinks.erc1155.Erc1155/GetStats -``` - -#### GetTransfers - -```bash -# Filter by wallet -grpcurl -plaintext -d '{ - "filter": {"wallet": "...base64..."}, - "limit": 10 -}' localhost:3000 torii.sinks.erc1155.Erc1155/GetTransfers - -# Filter by operator -grpcurl -plaintext -d '{ - "filter": {"operator": "...base64..."}, - "limit": 10 -}' localhost:3000 torii.sinks.erc1155.Erc1155/GetTransfers - -# Filter by specific token IDs -grpcurl -plaintext -d '{ - "filter": {"tokenIds": ["AQ==", "Ag=="]}, - "limit": 10 -}' localhost:3000 torii.sinks.erc1155.Erc1155/GetTransfers -``` - -#### SubscribeTransfers - -```bash -grpcurl -plaintext -d '{ - "clientId": "my-client", - "filter": {"tokens": ["...game_items_contract..."]} -}' localhost:3000 torii.sinks.erc1155.Erc1155/SubscribeTransfers -``` - -**ERC1155 Filter Fields:** - -| Field | Type | Description | -|-------|------|-------------| -| `wallet` | bytes | Matches `from` OR `to` | -| `from` | bytes | Exact sender address | -| `to` | bytes | Exact receiver address | -| `operator` | bytes | Exact operator address | -| `tokens` | bytes[] | Token contract whitelist | -| `tokenIds` | bytes[] | Specific token IDs | -| `blockFrom` | uint64 | Minimum block number | -| `blockTo` | uint64 | Maximum block number | - ---- - -### Core Torii Service - -**Service:** `torii.Torii` - -EventBus-based subscriptions that work across all sinks. - -#### ListTopics - -```bash -grpcurl -plaintext localhost:3000 torii.Torii/ListTopics -``` - -#### SubscribeToTopicsStream - -```bash -# Subscribe to ERC20 transfers via EventBus -grpcurl -plaintext -d '{ - "clientId": "my-client", - "topics": [{"topic": "erc20.transfer"}] -}' localhost:3000 torii.Torii/SubscribeToTopicsStream -``` - -## Environment Variables - -| Variable | Description | -|----------|-------------| -| `STARKNET_RPC_URL` | Default RPC URL (overridden by `--rpc-url`) | -| `RUST_LOG` | Log level (e.g., `info`, `debug`, `torii=debug`) | - -## Logging - -Control log verbosity with `RUST_LOG`: - -```bash -# Default info level -torii-tokens --from-block 0 - -# Debug logging for torii components -RUST_LOG=torii=debug torii-tokens --from-block 0 - -# Trace all SQL queries -RUST_LOG=torii=debug,sqlx=trace torii-tokens --from-block 0 -``` +# torii-tokens (binary) + +Unified **ERC20 + ERC721 + ERC1155 indexer**. Combines all three token +sinks behind one process, one port, one engine DB. Supports three +extraction modes (`block-range`, `event`, `global-event`) so operators +can pick full-chain indexing or per-contract cursors depending on their +use case. + +## Role in Torii + +The production-grade consolidation of `bins/torii-erc20` + equivalent +ERC721/ERC1155 binaries. Most deployments want all three standards in +one place; this is that binary. Every token sink is optional — omit the +`--erc20` / `--erc721` / `--erc1155` / `--include-well-known` flags and +that sink is not installed. See `torii-runtime-common::token_support` for +the boolean trio that drives the install logic. + +## Architecture + +```text ++---------------------------+ +| clap::Parser Config | --mode (block-range | event | global-event) +| (src/config.rs) | --erc20, --erc721, --erc1155 lists +| | --include-well-known (ETH/STRK) +| | --observability (Prometheus) +| | --database-url, --storage-database-url ++--------------+------------+ + | + v ++---------------------------+ +| resolve_token_db_setup | engine.db + erc20.db + erc721.db + erc1155.db +| (torii-runtime-common) | or single Postgres URL for all ++--------------+------------+ + | + v ++------------------------------------------------+ +| main() | +| | +| Extractor selected by --mode: | +| BlockRangeExtractor (one global cursor) | +| EventExtractor (per-contract cursors) | +| GlobalEventExtractor (one cursor via | +| starknet_getEvents) | +| | +| Decoders registered (all three always, but | +| only sinks actually consuming are installed):| +| Erc20Decoder → Erc20Sink + service | +| Erc721Decoder → Erc721Sink + service | +| Erc1155Decoder → Erc1155Sink + service | +| | +| Contract identification: | +| - explicit mappings for each sink's list | +| - well-known ETH + STRK (if requested) | +| - ContractRegistry with Erc20Rule, | +| Erc721Rule, Erc1155Rule | +| - rpc_parallelism = --rpc-parallelism | +| | +| Metadata pipelines: | +| N workers × queue_capacity × max_retries | +| (parallelism=8, queue=2048, retries=5) | +| | +| EtlConcurrencyConfig { | +| max_prefetch_batches: --max-prefetch-batches | +| } | +| | +| grpc_router = builder | +| .add_service(Erc20Server) | +| .add_service(Erc721Server) | +| .add_service(Erc1155Server) | +| + composed reflection (TORII + all 3 descr.) | +| | +| torii::run(config).await | ++------------------------------------------------+ +``` + +## Deep Dive + +### CLI highlights (full list in `src/config.rs`) + +| Flag | Default | Purpose | +|---|---|---| +| `--mode` | `block-range` | Extraction mode: `block-range` / `event` / `global-event` | +| `--rpc-url` (env `STARKNET_RPC_URL`) | Cartridge mainnet | JSON-RPC endpoint | +| `--from-block`, `--to-block` | `0` / none | Range bounds | +| `--db-dir` | `./torii-data` | Root when using SQLite (`engine.db`, `erc20.db`, `erc721.db`, `erc1155.db`) | +| `--database-url` (env `DATABASE_URL`) | — | Postgres URL or SQLite override for the engine DB | +| `--storage-database-url` (env `STORAGE_DATABASE_URL`) | — | Token-storage override (else uses `--database-url`, else `--db-dir`) | +| `--port` | `3000` | gRPC + HTTP port | +| `--observability` | off | Sets `TORII_METRICS_ENABLED=true` via `torii-config-common` | +| `--erc20` / `--erc721` / `--erc1155` | `[]` | Comma-separated contract lists | +| `--include-well-known` | off | Pre-map ETH + STRK | +| `--batch-size` | `50` | Blocks per `BlockRangeExtractor` batch | +| `--event-chunk-size` | `1000` | Events per `starknet_getEvents` page | +| `--event-block-batch-size` | `10000` | Blocks per event-mode iteration | +| `--event-bootstrap-blocks` | `20000` | Scan depth for auto-discovery in event mode | +| `--max-prefetch-batches` | `2` | ETL prefetch queue depth | +| `--cycle-interval` | `3` | Idle retry seconds | +| `--rpc-parallelism` | `0` (auto) | Max concurrent chunked RPC requests | +| `--metadata-parallelism` | `8` | Async metadata fetcher workers | +| `--metadata-queue-capacity` | `2048` | Metadata command queue | +| `--metadata-max-retries` | `5` | Metadata fetch retries | +| `--metadata-mode` | `inline` | `inline` or `deferred` | +| `--metadata-backfill-only` | off | Reserved for future metadata-only workflows | + +### Internal modules + +- `config.rs` — `Config`, `ExtractionMode`, `MetadataMode`, `well_known_erc20_contracts`, `has_tokens`, unit tests for flag parsing. +- `main.rs` — `resolve_token_db_setup` → per-standard storage + sink + decoder + rule → `ContractRegistry.load_from_db` → `ToriiConfig` build → `torii::run`. Composes gRPC reflection from four descriptor sets. + +### Workspace dependencies + +`torii`, `torii-erc20`, `torii-erc721`, `torii-erc1155`, `torii-runtime-common`, `torii-config-common`, `torii-sql` (postgres + sqlite), `starknet`, `tonic`, `tonic-reflection`, `clap`, `tokio`, `tracing`, `anyhow`, `url`. + +### Extension Points + +- Add another token standard → register its decoder + rule + sink + gRPC service + descriptor set; extend `InstalledTokenSupport` in `torii-runtime-common` if the install matrix gets more complex. +- Change the balance model for a single standard → modify the corresponding sink; this binary is pure wiring. + +Paired profiler: `bins/torii-tokens-synth`. diff --git a/bins/torii-tokens/src/main.rs b/bins/torii-tokens/src/main.rs index 3d49c1ff..1e46aba6 100644 --- a/bins/torii-tokens/src/main.rs +++ b/bins/torii-tokens/src/main.rs @@ -61,7 +61,8 @@ use torii_common::{MetadataFetcher, TokenUriService}; use torii_config_common::apply_observability_env; use torii_runtime_common::database::resolve_token_db_setup; #[cfg(feature = "profiling")] -use torii_runtime_common::database::DatabaseBackend; +use torii_sql::DbBackend; +use torii_sql::DbPoolOptions; // Import from ERC20 library crate use torii_erc20::proto::erc20_server::Erc20Server; @@ -97,13 +98,13 @@ async fn contracts_from_registry( for (contract, decoder_ids, _) in mappings { if decoder_ids.contains(&erc20_id) { - erc20.push(contract); + erc20.push(contract.into()); } if decoder_ids.contains(&erc721_id) { - erc721.push(contract); + erc721.push(contract.into()); } if decoder_ids.contains(&erc1155_id) { - erc1155.push(contract); + erc1155.push(contract.into()); } } @@ -188,7 +189,7 @@ async fn bootstrap_registry_for_event_mode( continue; } - let contract_addresses: Vec = batch + let contract_addresses: Vec<_> = batch .events .iter() .map(|event| event.from_address) @@ -445,7 +446,7 @@ async fn run_indexer(config: Config) -> Result<()> { for addr in &all_erc20_addresses { event_configs.push(ContractEventConfig { - address: *addr, + address: (*addr).into(), from_block: config.from_block, to_block, }); @@ -453,7 +454,7 @@ async fn run_indexer(config: Config) -> Result<()> { for addr in &all_erc721_addresses { event_configs.push(ContractEventConfig { - address: *addr, + address: (*addr).into(), from_block: config.from_block, to_block, }); @@ -461,7 +462,7 @@ async fn run_indexer(config: Config) -> Result<()> { for addr in &all_erc1155_addresses { event_configs.push(ContractEventConfig { - address: *addr, + address: (*addr).into(), from_block: config.from_block, to_block, }); @@ -539,7 +540,8 @@ async fn run_indexer(config: Config) -> Result<()> { if create_erc20 { enabled_types.push("ERC20"); - let storage = Arc::new(Erc20Storage::new(&db_setup.erc20_url).await?); + let erc20_pool = Erc20Storage::connect_pool(&db_setup.erc20_url).await?; + let storage = Arc::new(Erc20Storage::from_pool(&db_setup.erc20_url, erc20_pool).await?); tracing::info!("ERC20 database initialized: {}", db_setup.erc20_url); let decoder = Arc::new(Erc20Decoder::new()); @@ -570,7 +572,7 @@ async fn run_indexer(config: Config) -> Result<()> { let erc20_decoder_id = DecoderId::new("erc20"); for address in &all_erc20_addresses { - torii_config = torii_config.map_contract(*address, vec![erc20_decoder_id]); + torii_config = torii_config.map_contract((*address).into(), vec![erc20_decoder_id]); } if all_erc20_addresses.is_empty() { @@ -586,7 +588,8 @@ async fn run_indexer(config: Config) -> Result<()> { if create_erc721 { enabled_types.push("ERC721"); - let storage = Arc::new(Erc721Storage::new(&db_setup.erc721_url).await?); + let erc721_pool = Erc721Storage::connect_pool(&db_setup.erc721_url).await?; + let storage = Arc::new(Erc721Storage::from_pool(&db_setup.erc721_url, erc721_pool).await?); tracing::info!("ERC721 database initialized: {}", db_setup.erc721_url); let decoder = Arc::new(Erc721Decoder::new()); @@ -625,7 +628,7 @@ async fn run_indexer(config: Config) -> Result<()> { let erc721_decoder_id = DecoderId::new("erc721"); for address in &all_erc721_addresses { - torii_config = torii_config.map_contract(*address, vec![erc721_decoder_id]); + torii_config = torii_config.map_contract((*address).into(), vec![erc721_decoder_id]); } if all_erc721_addresses.is_empty() { @@ -641,7 +644,10 @@ async fn run_indexer(config: Config) -> Result<()> { if create_erc1155 { enabled_types.push("ERC1155"); - let storage = Arc::new(Erc1155Storage::new(&db_setup.erc1155_url).await?); + let erc1155_pool = DbPoolOptions::new() + .connect_any(&db_setup.erc1155_url) + .await?; + let storage = Arc::new(Erc1155Storage::new(erc1155_pool, &db_setup.erc1155_url).await?); tracing::info!("ERC1155 database initialized: {}", db_setup.erc1155_url); let decoder = Arc::new(Erc1155Decoder::new()); @@ -679,7 +685,7 @@ async fn run_indexer(config: Config) -> Result<()> { let erc1155_decoder_id = DecoderId::new("erc1155"); for address in &all_erc1155_addresses { - torii_config = torii_config.map_contract(*address, vec![erc1155_decoder_id]); + torii_config = torii_config.map_contract((*address).into(), vec![erc1155_decoder_id]); } if all_erc1155_addresses.is_empty() { @@ -782,7 +788,7 @@ async fn run_indexer(config: Config) -> Result<()> { .duration_since(UNIX_EPOCH) .map(|d| d.as_secs()) .unwrap_or(0); - let db_backend = if db_setup.erc20_backend == DatabaseBackend::Postgres { + let db_backend = if db_setup.erc20_backend == DbBackend::Postgres { "postgres" } else { "sqlite" diff --git a/crates/arcade-sink/Cargo.toml b/crates/arcade-sink/Cargo.toml index 1c182a02..cb3b8447 100644 --- a/crates/arcade-sink/Cargo.toml +++ b/crates/arcade-sink/Cargo.toml @@ -6,8 +6,10 @@ edition = "2021" [dependencies] torii = { path = "../.." } torii-common = { path = "../torii-common" } +torii-dojo = { workspace = true } torii-introspect = { path = "../introspect" } torii-erc721 = { path = "../torii-erc721" } +torii-sql = { path = "../sql", features = ["postgres", "sqlite"] } torii-runtime-common.workspace = true anyhow.workspace = true diff --git a/crates/arcade-sink/README.md b/crates/arcade-sink/README.md new file mode 100644 index 00000000..03e49b4b --- /dev/null +++ b/crates/arcade-sink/README.md @@ -0,0 +1,129 @@ +# arcade-sink (torii-arcade-sink) + +Projects **Cartridge Arcade** entities (Games, Editions, Collections, +Listings, Sales) out of Dojo/Introspect events. The sink watches +`IntrospectMsg` envelopes from the `torii-dojo` decoder for mutations of +Arcade-model tables, plans refresh/delete operations, and calls into an +`ArcadeService` backed by the introspect SQL store + the ERC721 store to +keep a live read-model that the `arcade.v1` gRPC service serves. + +## Role in Torii + +`bins/torii-arcade` is the only consumer. It runs a full Dojo+token +pipeline and this sink sits near the end of it: introspect-sql-sink +materialises the raw Dojo tables, arcade-sink reads from them +(`ArcadeService::refresh_*`) to build Arcade-specific projections, and +exposes them over gRPC to the Arcade UI. ERC721 data is joined in for NFT +edition metadata. + +## Architecture + +```text +DojoDecoder (torii-dojo) + | + v DojoBody { DojoEvent::Introspect(IntrospectMsg) } + | ++--------------------------------+ +| ArcadeSink | +| | +| build_batch_plan(envelopes): | +| - InsertsFields / Delete- | +| Records / CreateTable / | +| RenamePrimary → plan ops | +| - detects requires_full_ | +| rebuild or reload of | +| tracked_tables | +| | +| process(): | +| if full_rebuild: | +| service.bootstrap_from_ | +| source() | +| for op in ops: | +| TrackedTable.refresh() | +| TrackedTable.delete() | +| | +| tracked_tables: | +| HashMap | +| Game | Edition | | +| Collection | Listing | Sale| ++----------+---------------------+ + | + v ++--------------------------------+ +| ArcadeService | +| Arc> | +| (arcade + erc721 DBs) | +| | +| refresh_game/edition/… | +| delete_game/edition/… | +| bootstrap_from_source() | ++--------------------------------+ + | + v + gRPC: arcade.v1 service (proto/arcade.v1.proto) + via tonic + descriptor set (arcade_descriptor) +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `ArcadeSink` | `src/sink.rs` | 16 | Sink — holds `Arc` + `RwLock>` | +| `ArcadeSink::new(db_url, erc721_db_url, max_conns)` | `src/sink.rs` | 111 | Opens the arcade + erc721 pools | +| `ArcadeService` | `src/grpc_service.rs` | — | gRPC service backing the `arcade.v1` RPCs | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs` | 14 | Bytes for gRPC reflection | +| `proto::arcade` | `src/lib.rs` | 4 | Generated `arcade.v1` types | + +Internal: `ProjectionOp` (Refresh / Delete), `ProjectionBatchPlan`, `TrackedTable` (5 known Arcade models + their hard-coded table IDs), `from_table_name()` fallback by model name (`ARCADE-Game`, `ARCADE-Edition`, `ARCADE-Collection`, `ARCADE-Order`/`ARCADE-Listing`, `ARCADE-Sale`). + +### Internal Modules + +- `lib.rs` — re-exports + `tonic::include_proto!("arcade.v1")` + descriptor bytes. +- `sink.rs` — `ArcadeSink`, `ArcadeSink::build_batch_plan`, `TrackedTable` mapping + async `refresh` / `delete` dispatchers. +- `grpc_service.rs` — `ArcadeService`; SQL reader + writer; implements the `arcade.v1` service. + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `"arcade"` | +| `interested_types` | `[DOJO_TYPE_ID]` — consumes all Dojo envelopes and filters for `IntrospectMsg::Inserts/Delete/CreateTable/RenamePrimary` | +| `process(envelopes, _batch)` | Builds a `ProjectionBatchPlan` (maybe with `reload_tracked_tables` or `requires_full_rebuild`), then calls `refresh_*` / `delete_*` on the service per op | +| `topics` | `Vec::new()` — no EventBus topic | +| `build_routes` | `Router::new()` — no HTTP routes (gRPC only) | +| `initialize` | No-op (pools already open from `new`) | + +The gRPC service is **not** registered through `Sink::build_routes` — like +every gRPC-exposing sink, the binary wires it into tonic's router via +`ToriiConfigBuilder::with_grpc_router(Server::builder().add_service(...))` +before calling `torii::run`. + +### Protobuf & gRPC + +- Proto: `proto/arcade.v1.proto`. +- `build.rs` invokes `tonic_build` to generate the module and the descriptor set (`arcade_descriptor`). +- The descriptor is exported so binaries can compose reflection: + `ToriiConfigBuilder::with_custom_reflection(true)` + a user-built `tonic_reflection` registering `TORII_DESCRIPTOR_SET` + `arcade_sink::FILE_DESCRIPTOR_SET` + any token-sink descriptors. + +### Storage + +Two SQL pools: +- **Arcade DB**: Arcade-specific tables populated by `bootstrap_from_source` + `refresh_*`. +- **ERC721 DB**: reads NFT ownership/metadata written by `torii-erc721`. + +Both use sqlx's `Any` backend, so the binary can pick SQLite or PostgreSQL +via `torii-runtime-common::resolve_*_db_setup`. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-arcade` only. +- **Downstream deps**: `torii`, `torii-common`, `torii-dojo`, `torii-introspect`, `torii-erc721`, `torii-sql`, `tonic`, `prost`, `sqlx`, `starknet`, `async-trait`, `tokio`, `tracing`. + +### Extension Points + +- New tracked table → add an `ARCADE-Foo` table ID to `TrackedTable::known_table_ids` + a `TrackedTable::Foo` variant + `refresh_foo` / `delete_foo` on `ArcadeService`. +- New projection operation → extend `ProjectionOp` and add a branch in `build_batch_plan_from_tables` + `process`. +- New RPCs → add to `proto/arcade.v1.proto` and `ArcadeService`. The descriptor set regenerates automatically on build. diff --git a/crates/arcade-sink/src/grpc_service.rs b/crates/arcade-sink/src/grpc_service.rs index 205cf48c..835dc66c 100644 --- a/crates/arcade-sink/src/grpc_service.rs +++ b/crates/arcade-sink/src/grpc_service.rs @@ -4,14 +4,13 @@ use std::sync::Arc; use anyhow::{anyhow, Result}; use serde_json::Value; -use sqlx::{ - any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, -}; +use sqlx::{any::AnyPoolOptions, Any, Pool, QueryBuilder, Row}; use starknet::core::types::{Felt, U256}; use tonic::{Request, Response, Status}; use torii_common::{blob_to_u256, u256_to_blob}; use torii_erc721::{storage::OwnershipCursor, Erc721Storage}; use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; +use torii_sql::DbPool; use crate::proto::arcade::{ arcade_server::Arcade, Collection, Edition, Game, GetPlayerInventoryRequest, @@ -119,11 +118,14 @@ impl ArcadeService { .connect(&database_url) .await?; + let erc721_pool: DbPool = Erc721Storage::connect_pool(erc721_database_url).await?; + let erc721 = Arc::new(Erc721Storage::from_pool(erc721_database_url, erc721_pool).await?); + let service = Self { state: Arc::new(ArcadeState { pool, backend, - erc721: Arc::new(Erc721Storage::new(erc721_database_url).await?), + erc721, }), }; @@ -1754,7 +1756,7 @@ fn row_string(row: &sqlx::any::AnyRow, column: &str) -> Result { fn row_felt_hex(row: &sqlx::any::AnyRow, column: &str) -> Result { if let Ok(bytes) = row.try_get::, _>(column) { - return Ok(felt_hex(Felt::from_bytes_be_slice(&bytes))); + return Ok(felt_hex(felt_from_bytes(&bytes)?)); } let value = row @@ -1887,7 +1889,13 @@ fn felt_from_hex(value: &str) -> Result { } fn felt_from_bytes(value: &[u8]) -> Result { - Ok(Felt::from_bytes_be_slice(value)) + if value.len() > 32 { + return Err(anyhow!("felt bytes exceed 32 bytes")); + } + + let mut padded = [0u8; 32]; + padded[32 - value.len()..].copy_from_slice(value); + Ok(Felt::from_bytes_be(&padded)) } fn decimal_to_bytes(value: &str) -> Result> { @@ -1914,7 +1922,8 @@ fn decimal_to_bytes(value: &str) -> Result> { } } - Ok(u256_to_blob(blob_to_u256(&bytes))) + let value: U256 = blob_to_u256(&bytes); + Ok(u256_to_blob(value)) } fn u256_to_bytes(value: U256) -> Vec { @@ -1928,16 +1937,14 @@ fn sqlite_url(path: &str) -> Result { if path.starts_with("sqlite:") { return Ok(path.to_string()); } - let options = SqliteConnectOptions::from_str(&format!("sqlite://{path}")) - .or_else(|_| Ok::<_, sqlx::Error>(SqliteConnectOptions::new().filename(path)))?; - if let Some(parent) = options - .get_filename() - .parent() - .filter(|path| !path.as_os_str().is_empty()) - { + let path = std::path::Path::new(path); + if let Some(parent) = path.parent().filter(|path| !path.as_os_str().is_empty()) { std::fs::create_dir_all(parent)?; } - Ok(options.to_url_lossy().to_string()) + if !path.exists() { + std::fs::File::create(path)?; + } + Ok(format!("sqlite://{}", path.display())) } fn internal_status(error: impl std::fmt::Display) -> Status { diff --git a/crates/arcade-sink/src/sink.rs b/crates/arcade-sink/src/sink.rs index e95ec912..374b76e3 100644 --- a/crates/arcade-sink/src/sink.rs +++ b/crates/arcade-sink/src/sink.rs @@ -1,17 +1,15 @@ use std::collections::HashMap; -use std::sync::Arc; -use std::sync::RwLock; +use std::sync::{Arc, RwLock}; use anyhow::Result; use async_trait::async_trait; use starknet::core::types::Felt; use torii::axum::Router; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, SinkContext, TopicInfo}, -}; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; +use torii::etl::envelope::{Envelope, TypeId}; +use torii::etl::extractor::ExtractionBatch; +use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; +use torii_dojo::{DojoBody, DojoEvent, DOJO_TYPE_ID}; +use torii_introspect::events::IntrospectMsg; use crate::grpc_service::ArcadeService; @@ -237,19 +235,22 @@ impl ArcadeSink { }; for envelope in envelopes { - if envelope.type_id != TypeId::new("introspect") { + if envelope.type_id != DOJO_TYPE_ID { continue; } - let Some(body) = envelope.downcast_ref::() else { + let Some(body) = envelope.downcast_ref::() else { + continue; + }; + let DojoEvent::Introspect(msg) = &body.msg else { continue; }; - if Self::is_schema_rebuild_msg(&plan.tracked_tables, &body.msg) { + if Self::is_schema_rebuild_msg(&plan.tracked_tables, msg) { plan.requires_full_rebuild = true; } - match &body.msg { + match msg { IntrospectMsg::CreateTable(table) => { Self::update_tracked_table(&mut plan.tracked_tables, table.id, &table.name); plan.reload_tracked_tables = true; @@ -271,7 +272,15 @@ impl ArcadeSink { Self::tracked_table_for_id(&plan.tracked_tables, insert.table) { for record in &insert.records { - let entity_id = Felt::from_bytes_be_slice(&record.id); + let entity_id = Felt::from_bytes_be(&record.id); + if entity_id == Felt::ZERO && record.id.iter().any(|byte| *byte != 0) { + tracing::warn!( + target: "torii_arcade_sink::sink", + tracked_table = ?tracked_table, + "Skipping malformed entity id in introspect insert record" + ); + continue; + } plan.projection_ops .push(ProjectionOp::Refresh(tracked_table, entity_id)); } @@ -314,7 +323,7 @@ impl Sink for ArcadeSink { } fn interested_types(&self) -> Vec { - vec![TypeId::new("introspect")] + vec![DOJO_TYPE_ID] } async fn process(&self, envelopes: &[Envelope], _batch: &ExtractionBatch) -> Result<()> { @@ -372,17 +381,14 @@ mod tests { use std::str::FromStr; use anyhow::Result; - use sqlx::{ - sqlite::{SqliteConnectOptions, SqlitePoolOptions}, - Row, - }; + use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; + use sqlx::Row; use starknet::core::types::Felt; use tempfile::tempdir; use torii::etl::extractor::ExtractionBatch; - use torii::etl::{Envelope, MetaData}; - use torii_introspect::events::{ - InsertsFields, IntrospectBody, IntrospectMsg, Record, RenamePrimary, - }; + use torii::etl::{Envelope, EventContext, EventMsg}; + use torii_dojo::DojoEvent; + use torii_introspect::events::{InsertsFields, IntrospectMsg, Record, RenamePrimary}; use super::*; @@ -391,14 +397,11 @@ mod tests { "0x8c6f6a3efadfd0f4a8408d7ad1b4e37f4f733fe10b8a5ef9a76fd4d8be3b5b"; fn introspect_envelope(msg: IntrospectMsg) -> Envelope { - Envelope::from(IntrospectBody { - metadata: MetaData { - block_number: Some(1), - transaction_hash: Felt::ZERO, - from_address: Felt::ZERO, - }, - msg, - }) + Envelope::from(DojoEvent::Introspect(msg).to_body(EventContext { + block_number: 1, + transaction_hash: Felt::ZERO.into(), + from_address: Felt::ZERO.into(), + })) } fn tracked_tables() -> HashMap { diff --git a/crates/dojo/Cargo.toml b/crates/dojo/Cargo.toml index 0e09b633..e2942159 100644 --- a/crates/dojo/Cargo.toml +++ b/crates/dojo/Cargo.toml @@ -11,27 +11,32 @@ serde.workspace = true starknet-types-core.workspace = true starknet.workspace = true tokio.workspace = true -torii-common.workspace = true -torii.workspace = true anyhow.workspace = true tracing.workspace = true thiserror.workspace = true hex.workspace = true itertools.workspace = true - -dojo-introspect.workspace = true -introspect-types.workspace = true -torii-introspect.workspace = true -torii-postgres.workspace = true -torii-sqlite.workspace = true sqlx = { workspace = true, features = [ - "postgres", - "sqlite", "runtime-tokio-rustls", "macros", "migrate", ] } +starknet-types-raw.workspace = true +dojo-introspect.workspace = true +introspect-types.workspace = true + +torii-introspect.workspace = true +torii-sql.workspace = true +torii-types.workspace = true +torii-common.workspace = true +torii.workspace = true + + +[features] +postgres = ["sqlx/postgres"] +sqlite = ["sqlx/sqlite"] + [build-dependencies] tonic-build.workspace = true diff --git a/crates/dojo/README.md b/crates/dojo/README.md new file mode 100644 index 00000000..96ee1d80 --- /dev/null +++ b/crates/dojo/README.md @@ -0,0 +1,128 @@ +# torii-dojo + +Decoder and type library for **Dojo ECS** worlds. Translates Cairo-level +events emitted by the `dojo::world::World` contract into typed +`IntrospectMsg` payloads that downstream sinks (`introspect-sql-sink`, +`torii-ecs-sink`, `arcade-sink`) materialise. Also owns the +`ExternalContractRegistered` event + a `CommandHandler` that wires runtime +contract registrations back into the `ContractRegistry`. + +## Role in Torii + +`DojoDecoder` is the *one* decoder every Dojo-aware binary registers. It +listens to the world contract for schema mutations (table create/update, +record inserts/deletes, event declarations) and emits `DojoEvent` +envelopes tagged with `DOJO_TYPE_ID`. Any sink that declares +`interested_types() == [DOJO_TYPE_ID]` gets the entire Dojo event stream. + +The secondary responsibility — `ExternalContractRegistered` — +is how a live Dojo world advertises that a deployed contract should be +indexed as an ERC20/721/1155 token: the sink that consumes the event +dispatches a `RegisterExternalContractCommand` through the command bus; +the paired handler updates the `SharedDecoderRegistry` so the +`ContractRegistry` immediately starts routing that contract to the right +decoder. + +## Architecture + +```text ++--------------------------------+ +| StarknetEvent | +| (from Dojo world contract) | ++---------------+----------------+ + | + v ++------------------------------------------+ +| DojoDecoder | +| | +| async fn decode(keys, data, ctx): | +| select on event selector → | +| ModelRegistered, ModelWithSchemaReg- | +| istered, ModelUpgraded, | +| EventRegistered, EventUpgraded, | +| EventEmitted, StoreSetRecord, | +| StoreUpdateRecord, StoreUpdateMember, | +| StoreDelRecord, ExternalContract | +| Registered | +| | +| for table events: look up / insert | +| DojoTableInfo in self.tables | +| (RwLock>)| +| | +| for record events: translate via | +| DojoRecordEvent::event_to_msg() | +| into IntrospectMsg | +| | +| for schema-refresh: fetch schema via | +| DojoSchemaFetcher (F) | ++--------+---------------------------------+ + | | + | | + v v ++----------------+ +------------------------+ +| DojoEvent | | DojoEvent::External | +| ::Introspect | | ContractRegistered | +| (IntrospectMsg| | → EventBody | +| — 13 variants| | → consumed by sinks | +| from torii- | | that dispatch | +| introspect) | | RegisterExternal- | ++-------+--------+ | ContractCommand | + | +-----------+------------+ + | | + | v + | +------------------------+ + | | RegisterExternal | + | | ContractCommand | + | | Handler | + | | updates Shared | + | | {Contract,Decoder} | + | | Registry | + | +------------------------+ + | + v + envelopes (DOJO_TYPE_ID) flow to: + - introspect-sql-sink (persists schema + records) + - torii-ecs-sink (entity/event-message classification) + - arcade-sink (Arcade projections) +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `DojoDecoder` | `src/decoder.rs` | 55 | Decoder generic over a `DojoStoreTrait` impl + a `DojoSchemaFetcher` | +| `DojoEvent`, `DojoBody` | `src/decoder.rs` | 35 / 40 | Envelope union: `Introspect(IntrospectMsg)` \| `ExternalContractRegistered(...)` | +| `DOJO_TYPE_ID` | `src/decoder.rs` | 32 | `TypeId::new("dojo")` — the single type id every Dojo-aware sink subscribes to | +| `DojoTableEvent` | `src/decoder.rs` | 79 | Async trait for model registration events | +| `DojoRecordEvent` | `src/decoder.rs` | 88 | Sync trait for record mutations | +| `DojoTable` | `src/table.rs` | 14 | Dojo model metadata (columns, primary key, key/value field separation) | +| `ExternalContractRegistered`, `ExternalContractRegisteredEvent`, `ExternalContractRegisteredBody` | `src/external_contract.rs` | 73 / 31 | On-chain registration of external token contracts | +| `RegisterExternalContractCommand` | `src/external_contract.rs` | — | Torii command dispatched on registration | +| `RegisterExternalContractCommandHandler` | `src/external_contract.rs` | — | `CommandHandler` that updates the shared registries | +| `SharedContractTypeRegistry`, `SharedDecoderRegistry` | `src/external_contract.rs` | 28 / 29 | `Arc>>` handles shared between sink and handler | +| `RegisteredContractType` | `src/external_contract.rs` | 20 | `World` / `Erc20` / `Erc721` / `Erc1155` / `Other` | +| `resolve_external_contract`, `contract_type_from_decoder_ids` | `src/external_contract.rs` | — | Map a Dojo `contract_name` to `(DecoderIds, RegisteredContractType)` | +| `DojoToriiError`, `DojoToriiResult` | `src/error.rs` | — | Domain errors | + +### Internal Modules + +- `decoder` — `DojoDecoder` + the `Decoder` impl + `DojoEvent` union + helper macros for Cairo deserialisation. +- `event` — lightweight envelope-free wrapper used internally for deserialisation plumbing. +- `table` — `DojoTable` / `DojoTableInfo` + `sort_columns` helper. +- `external_contract` — everything around `ExternalContractRegistered`: types, command, handler, shared registries, `RegisteredContractType` + resolver. +- `error` — `DojoToriiError` (deserialisation failures, schema fetch failures, missing primary key, etc.). +- `store` — `DojoStoreTrait` + no-op SQLite/Postgres impls (retained for callers that need a pass-through store; real persistence happens in `introspect-sql-sink`). + +### Interactions + +- **Upstream (consumers)**: `introspect-sql-sink`, `torii-ecs-sink`, `arcade-sink`, every Dojo-capable binary (`bins/torii-introspect-bin`, `bins/torii-arcade`, `bins/torii-introspect-synth`). +- **Downstream deps**: `torii`, `torii-common`, `torii-introspect`, `torii-sql` (for the optional store impls), `torii-types`, `dojo-introspect`, `introspect-events`, `introspect-rust-macros`, `introspect-types`, `starknet`, `starknet-types-core`, `starknet-types-raw`, `async-trait`, `serde`, `tokio`, `itertools`. +- **Features**: `postgres`, `sqlite` — enable the corresponding sqlx driver in `store/{postgres,sqlite}`. + +### Extension Points + +- New Cairo event → add a variant to `DojoEvent` + a branch in `DojoDecoder::decode`; if it mutates a table, implement `DojoRecordEvent` / `DojoTableEvent` and translate to an `IntrospectMsg` so the existing sinks Just Work. +- New external-contract type → extend `RegisteredContractType` and `resolve_external_contract`. +- Custom store backend → implement `DojoStoreTrait`; wire it into `DojoDecoder::new`. diff --git a/crates/dojo/migrations/001_dojo_store.sql b/crates/dojo/migrations/postgres/001_dojo_store.sql similarity index 100% rename from crates/dojo/migrations/001_dojo_store.sql rename to crates/dojo/migrations/postgres/001_dojo_store.sql diff --git a/crates/dojo/src/decoder.rs b/crates/dojo/src/decoder.rs index 1b95122d..ecf1ed90 100644 --- a/crates/dojo/src/decoder.rs +++ b/crates/dojo/src/decoder.rs @@ -9,6 +9,7 @@ use dojo_introspect::events::{ ModelWithSchemaRegistered, StoreDelRecord, StoreSetRecord, StoreUpdateMember, StoreUpdateRecord, }; +use dojo_introspect::selector::compute_selector_from_dojo_tag; use dojo_introspect::serde::dojo_primary_def; use dojo_introspect::{DojoSchema, DojoSchemaFetcher}; use introspect_types::{ @@ -16,23 +17,46 @@ use introspect_types::{ SliceFeltSource, }; use itertools::Itertools; -use starknet::core::types::EmittedEvent; use starknet_types_core::felt::Felt; -use std::collections::HashMap; +use starknet_types_raw::Felt as RawFelt; +use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::sync::RwLock; -use torii::etl::event::EmittedEventExt; -use torii::etl::{Decoder, Envelope, EventMsg}; +use torii::etl::{Decoder, Envelope, EventBody, EventMsg, TypeId}; use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_introspect::schema::{TableMetadata, TableSchema}; -use torii_introspect::EventId; +use torii_introspect::schema::TableSchema; +use torii_introspect::{CreateTable, EventId}; +use torii_types::event::EventContext; pub const DOJO_ID_FIELD_NAME: &str = "entity_id"; +pub const DOJO_TYPE_ID: TypeId = TypeId::new("dojo"); + +#[derive(Debug, Clone)] +pub enum DojoEvent { + Introspect(IntrospectMsg), + ExternalContractRegistered(ExternalContractRegistered), +} + +pub type DojoBody = EventBody; + +impl EventMsg for DojoEvent { + fn event_id(&self) -> String { + match self { + Self::Introspect(msg) => msg.event_id(), + Self::ExternalContractRegistered(msg) => msg.event_id(), + } + } + + fn envelope_type_id(&self) -> TypeId { + DOJO_TYPE_ID + } +} pub struct DojoDecoder { pub tables: RwLock>, pub store: Store, pub fetcher: F, + pub append_only: HashSet<(Felt, Felt)>, } fn deserialize_data<'a, T>(keys: &[Felt], data: &'a [Felt]) -> DojoToriiResult @@ -47,12 +71,16 @@ where } } +fn raw_to_core(raw: RawFelt) -> Felt { + raw.into() +} + #[async_trait] pub trait DojoTableEvent: Sized + CairoEventInfo + Debug { type Msg: EventId; async fn event_to_msg( self, - raw: &EmittedEvent, + context: EventContext, decoder: &DojoDecoder, ) -> DojoToriiResult; } @@ -65,30 +93,26 @@ pub trait DojoRecordEvent: Sized + CairoEventInfo + Debug { #[async_trait] impl DojoStoreTrait for DojoDecoder where - Store: DojoStoreTrait + Send + Sync, - Store::Error: ToString, - F: Send + Sync + 'static, + Store: DojoStoreTrait + Sync, + F: Sync, { - type Error = DojoToriiError; - + async fn initialize(&self) -> DojoToriiResult<()> { + self.store.initialize().await + } async fn save_table( &self, - owner: &Felt, + owner: Felt, table: &DojoTable, - tx_hash: &Felt, + tx_hash: Felt, block_number: u64, ) -> DojoToriiResult<()> { self.store .save_table(owner, table, tx_hash, block_number) .await - .map_err(DojoToriiError::store_error) } async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { - self.store - .read_tables(owners) - .await - .map_err(DojoToriiError::store_error) + self.store.read_tables(owners).await } } @@ -103,31 +127,36 @@ pub fn primary_field_def() -> PrimaryDef { impl DojoDecoder { pub fn with_table( &self, - id: &Felt, + id: Felt, f: impl FnOnce(&DojoTableInfo) -> DojoToriiResult, ) -> DojoToriiResult { let tables = self.tables.read()?; let table = tables - .get(id) - .ok_or_else(|| DojoToriiError::TableNotFoundById(*id))?; + .get(&id) + .ok_or_else(|| DojoToriiError::TableNotFoundById(id))?; f(table) } } impl DojoDecoder where - Store: DojoStoreTrait + Sync, + Store: DojoStoreTrait + Sync + Send, F: DojoSchemaFetcher + Send + Sync + 'static, { - pub fn new>(store: S, fetcher: F) -> Self { - let store = store.into(); + pub fn new(store: Store, fetcher: F) -> Self { Self { tables: Default::default(), store, fetcher, + append_only: HashSet::new(), + } + } + pub fn append_historical(&mut self, models: HashSet<(Felt, String)>) { + for (address, name) in models { + let table_id = compute_selector_from_dojo_tag(&name).unwrap(); + self.append_only.insert((address, table_id)); } } - pub async fn load_tables(&self, owners: &[Felt]) -> DojoToriiResult<()> { let new = self.read_tables(owners).await?; let mut tables = self.tables.write()?; @@ -155,25 +184,22 @@ where tables: RwLock::new(tables), store, fetcher, + append_only: HashSet::new(), } } pub async fn register_table( &self, - owner: &Felt, namespace: &str, name: &str, schema: DojoSchema, - metadata: &impl TableMetadata, - ) -> DojoToriiResult { + context: EventContext, + ) -> DojoToriiResult { + let owner = raw_to_core(context.from_address); + let tx_hash = raw_to_core(context.transaction_hash); let full_table = DojoTable::from_schema(schema, namespace, name, dojo_primary_def()); - self.save_table( - owner, - &full_table, - metadata.tx_hash(), - metadata.block_number(), - ) - .await?; + self.save_table(owner, &full_table, tx_hash, context.block_number) + .await?; let (id, table) = full_table.clone().into(); { if let Some(existing) = self.tables.read()?.get(&id) { @@ -185,16 +211,20 @@ where } } self.tables.write()?.insert(id, table); - Ok(full_table.into()) + Ok(CreateTable::from_schema( + full_table.into(), + self.append_only.contains(&(owner, id)), + )) } pub async fn update_table( &self, - owner: &Felt, id: Felt, schema: DojoSchema, - meta_data: &impl TableMetadata, + context: EventContext, ) -> DojoToriiResult { + let owner = raw_to_core(context.from_address); + let tx_hash = raw_to_core(context.transaction_hash); let mut info = { let mut tables = self.tables.write()?; match tables.remove(&id) { @@ -207,7 +237,7 @@ where info.key_fields = key_fields; info.value_fields = value_fields; let table = (id, info).into(); - self.save_table(owner, &table, meta_data.tx_hash(), meta_data.block_number()) + self.save_table(owner, &table, tx_hash, context.block_number) .await?; let (_, info) = table.clone().into(); self.tables.write()?.insert(id, info); @@ -216,16 +246,16 @@ where async fn process_table_event<'a, E>( &self, - raw: &EmittedEvent, keys: &'a [Felt], - values: &'a [Felt], + data: &'a [Felt], + context: EventContext, ) -> DojoToriiResult where E: DojoTableEvent + CairoEvent>> + Send, E::Msg: Into, { - deserialize_data::(keys, values)? - .event_to_msg(raw, self) + deserialize_data::(keys, data)? + .event_to_msg(context, self) .await .ok_into() } @@ -233,13 +263,13 @@ where fn process_record_event<'a, E>( &self, keys: &'a [Felt], - values: &'a [Felt], + data: &'a [Felt], ) -> DojoToriiResult where E: DojoRecordEvent + CairoEvent>> + Send, E::Msg: Into, { - deserialize_data::(keys, values)? + deserialize_data::(keys, data)? .event_to_msg(self) .ok_into() } @@ -252,63 +282,82 @@ where deserialize_data::(keys, values).map(Into::into) } - pub async fn decode_event_data( + pub async fn event_with_selector_to_msg( &self, - raw: &EmittedEvent, - selector: &Felt, + selector: Felt, keys: &[Felt], - values: &[Felt], + data: &[Felt], + context: EventContext, ) -> DojoToriiResult { let selector_raw = selector.to_raw(); match selector_raw { ModelRegistered::SELECTOR_RAW => { - self.process_table_event::(raw, keys, values) + self.process_table_event::(keys, data, context) .await } ModelWithSchemaRegistered::SELECTOR_RAW => { - self.process_table_event::(raw, keys, values) + self.process_table_event::(keys, data, context) .await } ModelUpgraded::SELECTOR_RAW => { - self.process_table_event::(raw, keys, values) + self.process_table_event::(keys, data, context) .await } EventRegistered::SELECTOR_RAW => { - self.process_table_event::(raw, keys, values) + self.process_table_event::(keys, data, context) .await } EventUpgraded::SELECTOR_RAW => { - self.process_table_event::(raw, keys, values) + self.process_table_event::(keys, data, context) .await } - StoreSetRecord::SELECTOR_RAW => { - self.process_record_event::(keys, values) - } + StoreSetRecord::SELECTOR_RAW => self.process_record_event::(keys, data), StoreUpdateRecord::SELECTOR_RAW => { - self.process_record_event::(keys, values) + self.process_record_event::(keys, data) } StoreUpdateMember::SELECTOR_RAW => { - self.process_record_event::(keys, values) - } - StoreDelRecord::SELECTOR_RAW => { - self.process_record_event::(keys, values) + self.process_record_event::(keys, data) } - EventEmitted::SELECTOR_RAW => self.process_record_event::(keys, values), - _ => Err(DojoToriiError::UnknownDojoEventSelector(*selector)), + StoreDelRecord::SELECTOR_RAW => self.process_record_event::(keys, data), + EventEmitted::SELECTOR_RAW => self.process_record_event::(keys, data), + _ => Err(DojoToriiError::UnknownDojoEventSelector(selector)), } } - pub async fn decode_raw_event_msg(&self, raw: &EmittedEvent) -> DojoToriiResult { - let (selector, keys) = raw - .split_keys() + pub async fn event_with_selector_to_body( + &self, + selector: Felt, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> DojoToriiResult { + self.event_with_selector_to_msg(selector, keys, data, context) + .await + .map(|msg| msg.to_body(context)) + } + + pub async fn event_to_msg( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> DojoToriiResult { + let (selector, keys) = keys + .split_first() .ok_or(DojoToriiError::MissingEventSelector)?; - self.decode_event_data(raw, selector, keys, &raw.data).await + self.event_with_selector_to_msg(*selector, keys, data, context) + .await } - pub async fn decode_raw_event(&self, raw: &EmittedEvent) -> DojoToriiResult { - self.decode_raw_event_msg(raw) + pub async fn event_to_body( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> DojoToriiResult { + self.event_to_msg(keys, data, context) .await - .map(|msg| msg.to_body(raw)) + .map(|msg| msg.to_body(context)) } } @@ -322,39 +371,46 @@ where "dojo-introspect" } - async fn decode_event(&self, event: &EmittedEvent) -> AnyResult> { - let (selector, keys) = event - .split_keys() + async fn decode( + &self, + keys: &[RawFelt], + data: &[RawFelt], + context: EventContext, + ) -> AnyResult> { + let (&selector, keys) = keys + .split_first() .ok_or(DojoToriiError::MissingEventSelector)?; + let selector = raw_to_core(selector); + let keys: Vec = keys.iter().copied().map(raw_to_core).collect(); + let data: Vec = data.iter().copied().map(raw_to_core).collect(); - if selector.to_raw() == ExternalContractRegisteredEvent::SELECTOR_RAW { + if selector == ExternalContractRegisteredEvent::SELECTOR { return self - .process_external_contract_event(keys, &event.data) - .map(|msg| vec![msg.to_envelope(event)]) + .process_external_contract_event(&keys, &data) + .map(|msg| DojoEvent::ExternalContractRegistered(msg).to_envelopes(context)) .err_into(); } - self.decode_event_data(event, selector, keys, &event.data) + match self + .event_with_selector_to_msg(selector, &keys, &data, context) .await - .map(|msg| vec![msg.to_body(event).into()]) - .err_into() + { + Ok(msg) => DojoEvent::Introspect(msg).to_ok_envelopes(context), + Err(DojoToriiError::UnknownDojoEventSelector(_)) => Ok(Vec::new()), + Err(e) => Err(e.into()), + } } } #[cfg(test)] mod tests { use super::*; - use crate::ExternalContractRegisteredBody; use async_trait::async_trait; use dojo_introspect::DojoIntrospectError; - use introspect_types::{ - utils::string_to_cairo_serialize_byte_array, Attribute, ColumnDef, TypeDef, - }; + use introspect_types::utils::string_to_cairo_serialize_byte_array; + use introspect_types::{Attribute, ColumnDef, TypeDef}; use std::sync::Mutex; - - #[derive(Debug, thiserror::Error)] - #[error("{0}")] - struct FakeStoreError(String); + use torii::etl::StarknetEvent; #[derive(Default)] struct FakeStore { @@ -363,20 +419,21 @@ mod tests { #[async_trait] impl DojoStoreTrait for FakeStore { - type Error = FakeStoreError; - + async fn initialize(&self) -> DojoToriiResult { + Ok(()) + } async fn save_table( &self, - _owner: &Felt, + _owner: Felt, _table: &DojoTable, - _tx_hash: &Felt, + _tx_hash: Felt, block_number: u64, - ) -> Result<(), Self::Error> { + ) -> DojoToriiResult { self.saved_blocks.lock().unwrap().push(block_number); Ok(()) } - async fn read_tables(&self, _owners: &[Felt]) -> Result, Self::Error> { + async fn read_tables(&self, _owners: &[Felt]) -> DojoToriiResult> { Ok(Vec::new()) } } @@ -427,20 +484,23 @@ mod tests { decoder .update_table( - &owner, table_id, schema(&[ (1, "entity_id", true), (2, "health", false), (3, "armor", false), ]), - &(42, Felt::ZERO), + EventContext { + from_address: owner.into(), + transaction_hash: Felt::ZERO.into(), + block_number: 42, + }, ) .await .unwrap(); let parsed = decoder - .with_table(&table_id, |table| Ok(table.columns.len())) + .with_table(table_id, |table| Ok(table.columns.len())) .unwrap(); assert_eq!(parsed, 3); assert_eq!(*decoder.store.saved_blocks.lock().unwrap(), vec![42]); @@ -457,32 +517,36 @@ mod tests { keys.extend(string_to_cairo_serialize_byte_array("eth")); keys.push(Felt::from_hex("0x99").unwrap()); - let event = EmittedEvent { - from_address: Felt::from_hex("0x1").unwrap(), - keys, + let event = StarknetEvent { + from_address: Felt::from_hex("0x1").unwrap().into(), + keys: keys.into_iter().map(Into::into).collect(), data: vec![ - Felt::from_hex("0xabc").unwrap(), - Felt::from_hex("0x1234").unwrap(), - Felt::from(42_u64), + Felt::from_hex("0xabc").unwrap().into(), + Felt::from_hex("0x1234").unwrap().into(), + Felt::from(42_u64).into(), ], - block_hash: None, - block_number: Some(42), - transaction_hash: Felt::from_hex("0xbeef").unwrap(), + block_number: 42, + transaction_hash: Felt::from_hex("0xbeef").unwrap().into(), }; let envelopes = decoder.decode_event(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let body = envelopes[0] - .downcast_ref::() - .unwrap(); + let body = envelopes[0].downcast_ref::().unwrap(); + assert_eq!(envelopes[0].type_id, DOJO_TYPE_ID); + let DojoEvent::ExternalContractRegistered(msg) = &body.msg else { + panic!("expected external contract registered event"); + }; - assert_eq!(body.msg.namespace, "tokens"); - assert_eq!(body.msg.contract_name, "ERC20"); - assert_eq!(body.msg.instance_name, "eth"); - assert_eq!(body.msg.contract_selector, Felt::from_hex("0x99").unwrap()); - assert_eq!(body.msg.class_hash, Felt::from_hex("0xabc").unwrap()); - assert_eq!(body.msg.contract_address, Felt::from_hex("0x1234").unwrap()); - assert_eq!(body.msg.registration_block, 42); - assert_eq!(body.metadata.from_address, Felt::from_hex("0x1").unwrap()); + assert_eq!(msg.namespace, "tokens"); + assert_eq!(msg.contract_name, "ERC20"); + assert_eq!(msg.instance_name, "eth"); + assert_eq!(msg.contract_selector, Felt::from_hex("0x99").unwrap()); + assert_eq!(msg.class_hash, Felt::from_hex("0xabc").unwrap()); + assert_eq!(msg.contract_address, Felt::from_hex("0x1234").unwrap()); + assert_eq!(msg.registration_block, 42); + assert_eq!( + body.context.from_address, + Felt::from_hex("0x1").unwrap().into() + ); } } diff --git a/crates/dojo/src/error.rs b/crates/dojo/src/error.rs index 911f7550..e9d4cc78 100644 --- a/crates/dojo/src/error.rs +++ b/crates/dojo/src/error.rs @@ -1,10 +1,10 @@ -use std::sync::PoisonError; - use dojo_introspect::DojoIntrospectError; use introspect_types::transcode::TranscodeError; use introspect_types::DecodeError; use starknet::core::utils::NonAsciiNameError; use starknet_types_core::felt::Felt; +use std::error::Error as StdError; +use std::sync::PoisonError; #[derive(Debug, thiserror::Error)] pub enum DojoToriiError { @@ -12,8 +12,8 @@ pub enum DojoToriiError { UnknownDojoEventSelector(Felt), #[error("Missing event selector")] MissingEventSelector, - #[error("Column {0:#066x} not found in table {1}")] - ColumnNotFound(Felt, String), + #[error("Column {1:#066x} not found in table {0}")] + ColumnNotFound(String, Felt), #[error("Failed to parse field {0:#066x} in table {1}")] FieldParseError(Felt, String), #[error("Too many values provided for field {0:#066x}")] @@ -26,8 +26,6 @@ pub enum DojoToriiError { TableNotFoundById(Felt), #[error("Failed to acquire lock: {0}")] LockError(String), - #[error("Store error: {0}")] - StoreError(String), #[error("Starknet selector error: {0}")] StarknetSelectorError(#[from] NonAsciiNameError), #[error("Lock poisoned: {0}")] @@ -40,9 +38,13 @@ pub enum DojoToriiError { DojoIntrospectError(#[from] DojoIntrospectError), #[error("Transcode error: {0:?}")] TranscodeError(TranscodeError), + #[error("Store error: {0}")] + StoreError(#[source] Box), + #[error(transparent)] + JsonError(#[from] serde_json::Error), } -pub type DojoToriiResult = std::result::Result; +pub type DojoToriiResult = std::result::Result; impl From> for DojoToriiError { fn from(err: TranscodeError) -> Self { @@ -51,8 +53,8 @@ impl From> for DojoToriiError { } impl DojoToriiError { - pub fn store_error(err: T) -> Self { - Self::StoreError(err.to_string()) + pub fn store_error(err: T) -> Self { + Self::StoreError(Box::new(err)) } } diff --git a/crates/dojo/src/event.rs b/crates/dojo/src/event.rs index 3ae7b61b..cd579b5a 100644 --- a/crates/dojo/src/event.rs +++ b/crates/dojo/src/event.rs @@ -10,9 +10,8 @@ use dojo_introspect::events::{ }; use dojo_introspect::DojoSchemaFetcher; use introspect_types::FeltIds; -use starknet::core::types::EmittedEvent; -use starknet_types_core::felt::Felt; use torii_introspect::events::{CreateTable, DeleteRecords, InsertsFields, UpdateTable}; +use torii_types::event::EventContext; #[async_trait] impl DojoTableEvent for ModelWithSchemaRegistered @@ -23,19 +22,12 @@ where type Msg = CreateTable; async fn event_to_msg( self, - raw: &EmittedEvent, + context: EventContext, decoder: &DojoDecoder, ) -> DojoToriiResult { decoder - .register_table( - &raw.from_address, - &self.namespace, - &self.name, - self.schema, - raw, - ) + .register_table(&self.namespace, &self.name, self.schema, context) .await - .map(Into::into) } } @@ -48,14 +40,13 @@ where type Msg = CreateTable; async fn event_to_msg( self, - raw: &EmittedEvent, + context: EventContext, decoder: &DojoDecoder, ) -> DojoToriiResult { let schema = decoder.fetcher.schema(self.address).await?; decoder - .register_table(&raw.from_address, &self.namespace, &self.name, schema, raw) + .register_table(&self.namespace, &self.name, schema, context) .await - .map(Into::into) } } @@ -68,14 +59,13 @@ where type Msg = CreateTable; async fn event_to_msg( self, - raw: &EmittedEvent, + context: EventContext, decoder: &DojoDecoder, ) -> DojoToriiResult { let schema = decoder.fetcher.schema(self.address).await?; decoder - .register_table(&raw.from_address, &self.namespace, &self.name, schema, raw) + .register_table(&self.namespace, &self.name, schema, context) .await - .map(Into::into) } } @@ -88,12 +78,12 @@ where type Msg = UpdateTable; async fn event_to_msg( self, - raw: &EmittedEvent, + context: EventContext, decoder: &DojoDecoder, ) -> DojoToriiResult { let schema = decoder.fetcher.schema(self.address).await?; decoder - .update_table(&raw.from_address, self.selector, schema, raw) + .update_table(self.selector, schema, context) .await .map(Into::into) } @@ -108,12 +98,12 @@ where type Msg = UpdateTable; async fn event_to_msg( self, - raw: &EmittedEvent, + context: EventContext, decoder: &DojoDecoder, ) -> DojoToriiResult { let schema = decoder.fetcher.schema(self.address).await?; decoder - .update_table(&raw.from_address, self.selector, schema, raw) + .update_table(self.selector, schema, context) .await .map(Into::into) } @@ -122,7 +112,7 @@ where impl DojoRecordEvent for StoreSetRecord { type Msg = InsertsFields; fn event_to_msg(self, decoder: &DojoDecoder) -> DojoToriiResult { - let (columns, data) = decoder.with_table(&self.selector, |table| { + let (columns, data) = decoder.with_table(self.selector, |table| { table.parse_record(self.keys, self.values) })?; Ok(InsertsFields::new_single( @@ -138,7 +128,7 @@ impl DojoRecordEvent for StoreUpdateRecord { type Msg = InsertsFields; fn event_to_msg(self, decoder: &DojoDecoder) -> DojoToriiResult { let (columns, data) = - decoder.with_table(&self.selector, |table| table.parse_values(self.values))?; + decoder.with_table(self.selector, |table| table.parse_values(self.values))?; Ok(InsertsFields::new_single( self.selector, columns, @@ -151,8 +141,8 @@ impl DojoRecordEvent for StoreUpdateRecord { impl DojoRecordEvent for EventEmitted { type Msg = InsertsFields; fn event_to_msg(self, decoder: &DojoDecoder) -> DojoToriiResult { - let primary = Felt::from_bytes_be(&self.keys.hash().into()); - let (columns, data) = decoder.with_table(&self.selector, |table| { + let primary: [u8; 32] = self.keys.hash().into(); + let (columns, data) = decoder.with_table(self.selector, |table| { table.parse_record(self.keys, self.values) })?; Ok(InsertsFields::new_single( @@ -167,7 +157,7 @@ impl DojoRecordEvent for EventEmitted { impl DojoRecordEvent for StoreUpdateMember { type Msg = InsertsFields; fn event_to_msg(self, decoder: &DojoDecoder) -> DojoToriiResult { - let data = decoder.with_table(&self.selector, |table| { + let data = decoder.with_table(self.selector, |table| { table.parse_field(self.member_selector, self.values) })?; Ok(InsertsFields::new_single( diff --git a/crates/dojo/src/external_contract.rs b/crates/dojo/src/external_contract.rs index 5d2d158f..9b1c6b0f 100644 --- a/crates/dojo/src/external_contract.rs +++ b/crates/dojo/src/external_contract.rs @@ -5,7 +5,7 @@ use introspect_types::{ DecodeResult, FeltSource, }; use serde::{Deserialize, Serialize}; -use starknet::core::types::Felt; +use starknet_types_core::felt::Felt; use std::collections::{HashMap, HashSet}; use std::sync::Arc; use tokio::sync::RwLock; @@ -210,7 +210,7 @@ impl CommandHandler for RegisterExternalContractCommandHandler { let command = *command; self.engine_db - .set_contract_decoders(command.contract_address, &command.decoder_ids) + .set_contract_decoders(command.contract_address.into(), &command.decoder_ids) .await .with_context(|| { format!( @@ -379,7 +379,7 @@ mod tests { ); assert_eq!( engine_db - .get_contract_decoders(contract_address) + .get_contract_decoders(contract_address.into()) .await .unwrap() .unwrap(), diff --git a/crates/dojo/src/lib.rs b/crates/dojo/src/lib.rs index 042aab65..71b3bd2c 100644 --- a/crates/dojo/src/lib.rs +++ b/crates/dojo/src/lib.rs @@ -4,6 +4,7 @@ pub mod event; pub mod external_contract; pub mod store; pub mod table; +pub use decoder::{DojoBody, DojoEvent, DOJO_TYPE_ID}; pub use error::{DojoToriiError, DojoToriiResult}; pub use external_contract::{ contract_type_from_decoder_ids, resolve_external_contract, ExternalContractRegistered, diff --git a/crates/dojo/src/store/json.rs b/crates/dojo/src/store/json.rs index a3349671..1ae0d026 100644 --- a/crates/dojo/src/store/json.rs +++ b/crates/dojo/src/store/json.rs @@ -1,6 +1,7 @@ use super::DojoStoreTrait; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiResult}; use async_trait::async_trait; +use introspect_types::ResultInto; use serde_json::Error as JsonError; use starknet_types_core::felt::Felt; use std::fs; @@ -68,19 +69,24 @@ impl JsonStore { #[async_trait] impl DojoStoreTrait for JsonStore { - type Error = JsonError; - + async fn initialize(&self) -> DojoToriiResult { + // No initialization needed for JSON store, but we can check if the path is accessible. + if !self.path.exists() { + std::fs::create_dir_all(&self.path).map_err(JsonError::io)?; + } + Ok(()) + } async fn save_table( &self, - _owner: &Felt, + _owner: Felt, table: &DojoTable, - _tx_hash: &Felt, + _tx_hash: Felt, _block_number: u64, - ) -> Result<(), Self::Error> { - self.dump_table(table) + ) -> DojoToriiResult { + self.dump_table(table).err_into() } - async fn read_tables(&self, _owners: &[Felt]) -> Result, Self::Error> { - self.load_all_tables() + async fn read_tables(&self, _owners: &[Felt]) -> DojoToriiResult> { + self.load_all_tables().err_into() } } diff --git a/crates/dojo/src/store/mod.rs b/crates/dojo/src/store/mod.rs index 148ad129..057ecfa3 100644 --- a/crates/dojo/src/store/mod.rs +++ b/crates/dojo/src/store/mod.rs @@ -1,31 +1,35 @@ pub mod json; + +#[cfg(feature = "postgres")] pub mod postgres; +#[cfg(feature = "sqlite")] pub mod sqlite; +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub mod sql; + use crate::table::DojoTableInfo; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiResult}; use async_trait::async_trait; use starknet_types_core::felt::Felt; use std::collections::HashMap; #[async_trait] -pub trait DojoStoreTrait -where - Self: Send + Sync + 'static + Sized, -{ - type Error: std::error::Error; +pub trait DojoStoreTrait { + async fn initialize(&self) -> DojoToriiResult; async fn save_table( &self, - owner: &Felt, + owner: Felt, table: &DojoTable, - tx_hash: &Felt, + tx_hash: Felt, block_number: u64, - ) -> Result<(), Self::Error>; - async fn read_tables(&self, owners: &[Felt]) -> Result, Self::Error>; + ) -> DojoToriiResult; + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult>; async fn read_table_map( &self, owners: &[Felt], - ) -> Result, Self::Error> { + ) -> DojoToriiResult> { Ok(self .read_tables(owners) .await? diff --git a/crates/dojo/src/store/postgres.rs b/crates/dojo/src/store/postgres.rs index 995aa859..0d5a46d0 100644 --- a/crates/dojo/src/store/postgres.rs +++ b/crates/dojo/src/store/postgres.rs @@ -1,23 +1,20 @@ use super::DojoStoreTrait; use crate::decoder::primary_field_def; use crate::table::DojoTableInfo; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiError, DojoToriiResult}; use async_trait::async_trait; -use introspect_types::{Attribute, ColumnInfo, ResultInto, TypeDef}; +use introspect_types::{Attribute, ColumnInfo, TypeDef}; use itertools::Itertools; use sqlx::migrate::Migrator; -use sqlx::postgres::PgArguments; use sqlx::query::Query; use sqlx::types::Json; -use sqlx::{FromRow, Postgres}; +use sqlx::FromRow; use starknet_types_core::felt::Felt; use std::collections::HashMap; -use std::ops::Deref; -use torii_common::sql::SqlxResult; use torii_introspect::postgres::owned::PgTypeDef; use torii_introspect::postgres::PgFelt; use torii_introspect::schema::ColumnKeyTrait; -use torii_postgres::db::PostgresConnection; +use torii_sql::{PgArguments, PgPool, PoolExt, Postgres, SqlxResult}; pub const FETCH_TABLES_QUERY: &str = r#" SELECT DISTINCT ON (owner, id) @@ -87,7 +84,7 @@ pub const INSERT_COLUMN_QUERY: &str = r#" updated_at = NOW(), updated_tx = EXCLUDED.updated_tx"#; -pub const DOJO_STORE_MIGRATIONS: Migrator = sqlx::migrate!(); +pub const DOJO_STORE_MIGRATIONS: Migrator = sqlx::migrate!("./migrations/postgres"); #[derive(Debug, thiserror::Error)] pub enum DojoPgStoreError { @@ -101,8 +98,6 @@ pub enum DojoPgStoreError { table_id: Felt, column_id: Felt, }, - #[error("Duplicate tables found for owner {owner:?} and table id {table_id}")] - DuplicateTables { owner: Felt, table_id: Felt }, } impl DojoPgStoreError { @@ -191,65 +186,37 @@ where } } -// #[async_trait] -// impl PgTypeDef for DojoTableInfo { -// type Row = DojoTableRow; -// async fn get_rows( -// pool: &PgPool, -// query: &'static str, -// owners: &[Felt], -// ) -> SqlxResult> { -// Self::get_pg_rows(pool, query, owners) -// .await -// .map(|rows| rows.into_iter().map_into().collect_vec()) -// } -// } - -// #[async_trait] -// impl PgTypeDef<()> for DojoTable { -// type Row = DojoTableRow; -// async fn get_rows( -// pool: &PgPool, -// query: &'static str, -// owners: &[Felt], -// ) -> SqlxResult> { -// Self::get_pg_rows(pool, query, owners) -// .await -// .map(|rows| rows.into_iter().map_into().collect_vec()) -// } -// } - pub fn table_insert_query( - owner: &Felt, - id: &Felt, + owner: Felt, + id: Felt, block_number: u64, name: &str, attributes: &[String], keys: &[Felt], values: &[Felt], legacy: bool, - created_tx: &Felt, + created_tx: Felt, ) -> Query<'static, Postgres, PgArguments> { sqlx::query::(INSERT_TABLE_QUERY) - .bind(PgFelt::from(*owner)) - .bind(PgFelt::from(*id)) + .bind(PgFelt::from(owner)) + .bind(PgFelt::from(id)) .bind(block_number.to_string()) .bind(name.to_owned()) .bind(attributes.to_owned()) .bind(keys.iter().copied().map(PgFelt::from).collect_vec()) .bind(values.iter().copied().map(PgFelt::from).collect_vec()) .bind(legacy) - .bind(PgFelt::from(*created_tx)) + .bind(PgFelt::from(created_tx)) } pub fn column_info_insert_query( query: &'static str, - owner: &Felt, - table: &Felt, - id: &Felt, + owner: Felt, + table: Felt, + id: Felt, block_number: u64, info: &ColumnInfo, - created_tx: &Felt, + created_tx: Felt, ) -> Query<'static, Postgres, PgArguments> { column_insert_query( query, @@ -266,36 +233,36 @@ pub fn column_info_insert_query( pub fn column_insert_query( query: &'static str, - owner: &Felt, - table: &Felt, - id: &Felt, + owner: Felt, + table: Felt, + id: Felt, block_number: u64, name: &str, attributes: Vec, type_def: &TypeDef, - created_tx: &Felt, + created_tx: Felt, ) -> Query<'static, Postgres, PgArguments> { sqlx::query::(query) - .bind(PgFelt::from(*owner)) - .bind(PgFelt::from(*table)) - .bind(PgFelt::from(*id)) + .bind(PgFelt::from(owner)) + .bind(PgFelt::from(table)) + .bind(PgFelt::from(id)) .bind(block_number.to_string()) .bind(name.to_owned()) .bind(attributes) .bind(Json(type_def.clone())) - .bind(PgFelt::from(*created_tx)) + .bind(PgFelt::from(created_tx)) } impl DojoTable { pub fn insert_query( &self, - owner: &Felt, - tx_hash: &Felt, + owner: Felt, + tx_hash: Felt, block_number: u64, ) -> Query<'static, Postgres, PgArguments> { table_insert_query( owner, - &self.id, + self.id, block_number, &self.name, &self.attributes, @@ -309,74 +276,82 @@ impl DojoTable { pub struct PgStore(pub T); -impl Deref for PgStore { - type Target = T; - - fn deref(&self) -> &Self::Target { - &self.0 +impl> PoolExt for PgStore { + fn pool(&self) -> &PgPool { + self.0.pool() } } -impl PgStore { +impl + Send + Sync> PgStore { pub async fn initialize(&self) -> SqlxResult<()> { self.migrate(Some("dojo"), DOJO_STORE_MIGRATIONS).await } } -impl From for PgStore { +impl> From for PgStore { fn from(pool: T) -> Self { PgStore(pool) } } #[async_trait] -impl DojoStoreTrait for PgStore { - type Error = DojoPgStoreError; - +impl DojoStoreTrait for PgPool { + async fn initialize(&self) -> DojoToriiResult { + self.migrate(Some("dojo"), DOJO_STORE_MIGRATIONS) + .await + .map_err(DojoToriiError::store_error) + } async fn save_table( &self, - owner: &Felt, + owner: Felt, table: &DojoTable, - tx_hash: &Felt, + tx_hash: Felt, block_number: u64, - ) -> Result<(), Self::Error> { - let mut transaction = self.begin().await?; + ) -> DojoToriiResult { + let mut transaction = self.begin().await.map_err(DojoToriiError::store_error)?; table .insert_query(owner, tx_hash, block_number) .execute(&mut *transaction) - .await?; - for (id, info) in &table.columns { + .await + .map_err(DojoToriiError::store_error)?; + for (&id, info) in &table.columns { column_info_insert_query( INSERT_COLUMN_QUERY, owner, - &table.id, + table.id, id, block_number, info, tx_hash, ) .execute(&mut *transaction) - .await?; + .await + .map_err(DojoToriiError::store_error)?; } - transaction.commit().await.err_into() + transaction + .commit() + .await + .map_err(DojoToriiError::store_error) } - async fn read_tables(&self, owners: &[Felt]) -> Result, Self::Error> { + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { let mut tables = PgTypeDef::get_rows::(self.pool(), FETCH_TABLES_QUERY, owners) - .await? + .await + .map_err(DojoToriiError::store_error)? .into_iter() .map(|row: ((), DojoTable)| row.1) .collect_vec(); let mut columns: HashMap<(Felt, Felt), _> = ColumnInfo::get_hash_map::(self.pool(), FETCH_COLUMNS_QUERY, owners) - .await?; + .await + .map_err(DojoToriiError::store_error)?; for table in &mut tables { for key in table.key_fields.iter().chain(table.value_fields.iter()) { - let column = columns.remove(&(table.id, *key)).ok_or_else(|| { - DojoPgStoreError::column_not_found(table.name.clone(), &(table.id, *key)) - })?; + let column = columns + .remove(&(table.id, *key)) + .ok_or_else(|| DojoToriiError::ColumnNotFound(table.name.clone(), *key))?; table.columns.insert(*key, column); } } diff --git a/crates/dojo/src/store/sql.rs b/crates/dojo/src/store/sql.rs new file mode 100644 index 00000000..e72039e7 --- /dev/null +++ b/crates/dojo/src/store/sql.rs @@ -0,0 +1,35 @@ +use crate::store::DojoStoreTrait; +use crate::{DojoTable, DojoToriiResult}; +use async_trait::async_trait; +use starknet_types_core::felt::Felt; +use torii_sql::DbPool; + +#[async_trait] +impl DojoStoreTrait for DbPool { + async fn initialize(&self) -> DojoToriiResult { + match self { + DbPool::Postgres(pool) => pool.initialize().await, + DbPool::Sqlite(pool) => pool.initialize().await, + } + } + + async fn save_table( + &self, + owner: Felt, + table: &DojoTable, + tx_hash: Felt, + block_number: u64, + ) -> DojoToriiResult { + match self { + DbPool::Postgres(pool) => pool.save_table(owner, table, tx_hash, block_number).await, + DbPool::Sqlite(pool) => pool.save_table(owner, table, tx_hash, block_number).await, + } + } + + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { + match self { + DbPool::Postgres(pool) => pool.read_tables(owners).await, + DbPool::Sqlite(pool) => pool.read_tables(owners).await, + } + } +} diff --git a/crates/dojo/src/store/sqlite.rs b/crates/dojo/src/store/sqlite.rs index fd348d5e..d5f0bfaf 100644 --- a/crates/dojo/src/store/sqlite.rs +++ b/crates/dojo/src/store/sqlite.rs @@ -1,21 +1,20 @@ use super::DojoStoreTrait; use crate::decoder::primary_field_def; -use crate::DojoTable; +use crate::{DojoTable, DojoToriiError, DojoToriiResult}; use async_trait::async_trait; use introspect_types::ColumnInfo; use serde::{Deserialize, Serialize}; use sqlx::migrate::Migrator; use sqlx::sqlite::SqliteArguments; -use sqlx::Arguments; -use sqlx::{FromRow, Sqlite}; +use sqlx::{Arguments, FromRow, Sqlite, SqlitePool}; use starknet_types_core::felt::Felt; +use starknet_types_raw::{error::OverflowError, Felt as RawFelt}; use std::collections::HashMap; use std::io; use std::ops::Deref; -use torii_common::sql::SqlxResult; -use torii_common::{blob_to_felt, felt_to_blob}; use torii_introspect::schema::ColumnKeyTrait; -use torii_sqlite::SqliteConnection; +use torii_sql::sqlite::SqliteDbConnection; +use torii_sql::{PoolExt, SqlxResult}; pub const DOJO_SQLITE_STORE_MIGRATIONS: Migrator = sqlx::migrate!("./migrations/sqlite"); @@ -31,6 +30,8 @@ pub enum DojoSqliteStoreError { table_id: Felt, column_id: Felt, }, + #[error("Felt overflow error: {0}")] + FeltOverflow(#[from] OverflowError), } impl DojoSqliteStoreError { @@ -88,9 +89,13 @@ fn parse_felt_json_array(value: &str) -> Result, DojoSqliteStoreError> .collect() } +fn felt_from_bytes(value: &[u8]) -> Result { + Ok(RawFelt::from_be_bytes_slice(value)?.into()) +} + fn table_row_into_table(value: TableRow) -> Result { Ok(DojoTable { - id: blob_to_felt(&value.id), + id: felt_from_bytes(&value.id)?, name: value.name, attributes: serde_json::from_str(&value.attributes)?, primary: primary_field_def(), @@ -107,7 +112,10 @@ where { let payload: StoredColumnInfo = serde_json::from_str(&value.payload)?; Ok(( - K::from_parts(blob_to_felt(&value.table_id), blob_to_felt(&value.id)), + K::from_parts( + felt_from_bytes(&value.table_id)?, + felt_from_bytes(&value.id)?, + ), ColumnInfo { name: payload.name, attributes: payload.attributes, @@ -116,7 +124,7 @@ where )) } -fn select_table_query<'a>(owners: &[Felt]) -> (String, SqliteArguments<'a>) { +fn select_table_query(owners: &[Felt]) -> (String, SqliteArguments<'_>) { let mut query = String::from( "SELECT id, name, attributes, keys_json, values_json, legacy FROM dojo_tables", ); @@ -128,14 +136,14 @@ fn select_table_query<'a>(owners: &[Felt]) -> (String, SqliteArguments<'a>) { query.push_str(", "); } query.push('?'); - let _ = args.add(felt_to_blob(*owner)); + let _ = args.add(owner.to_bytes_be().to_vec()); } query.push(')'); } (query, args) } -fn select_column_query<'a>(owners: &[Felt]) -> (String, SqliteArguments<'a>) { +fn select_column_query(owners: &[Felt]) -> (String, SqliteArguments<'_>) { let mut query = String::from("SELECT table_id, id, payload FROM dojo_columns"); let mut args = SqliteArguments::default(); if !owners.is_empty() { @@ -145,7 +153,7 @@ fn select_column_query<'a>(owners: &[Felt]) -> (String, SqliteArguments<'a>) { query.push_str(", "); } query.push('?'); - let _ = args.add(felt_to_blob(*owner)); + let _ = args.add(owner.to_bytes_be().to_vec()); } query.push(')'); } @@ -162,31 +170,34 @@ impl Deref for SqliteStore { } } -impl SqliteStore { +impl SqliteStore { pub async fn initialize(&self) -> SqlxResult<()> { self.migrate(Some("dojo"), DOJO_SQLITE_STORE_MIGRATIONS) .await } } -impl From for SqliteStore { +impl From for SqliteStore { fn from(pool: T) -> Self { Self(pool) } } #[async_trait] -impl DojoStoreTrait for SqliteStore { - type Error = DojoSqliteStoreError; - +impl DojoStoreTrait for SqlitePool { + async fn initialize(&self) -> DojoToriiResult { + self.migrate(Some("dojo"), DOJO_SQLITE_STORE_MIGRATIONS) + .await + .map_err(DojoToriiError::store_error) + } async fn save_table( &self, - owner: &Felt, + owner: Felt, table: &DojoTable, - tx_hash: &Felt, + tx_hash: Felt, block_number: u64, - ) -> Result<(), Self::Error> { - let mut transaction = self.begin().await?; + ) -> DojoToriiResult { + let mut transaction = self.begin().await.map_err(DojoToriiError::store_error)?; sqlx::query( r" @@ -215,8 +226,8 @@ impl DojoStoreTrait for SqliteStore updated_tx = excluded.updated_tx ", ) - .bind(felt_to_blob(*owner)) - .bind(felt_to_blob(table.id)) + .bind(owner.to_bytes_be().to_vec()) + .bind(table.id.to_bytes_be().to_vec()) .bind(&table.name) .bind(serde_json::to_string(&table.attributes)?) .bind(serialize_felt_json_array(&table.key_fields)?) @@ -224,10 +235,11 @@ impl DojoStoreTrait for SqliteStore .bind(table.legacy) .bind(block_number as i64) .bind(block_number as i64) - .bind(felt_to_blob(*tx_hash)) - .bind(felt_to_blob(*tx_hash)) + .bind(tx_hash.to_bytes_be().to_vec()) + .bind(tx_hash.to_bytes_be().to_vec()) .execute(&mut *transaction) - .await?; + .await + .map_err(DojoToriiError::store_error)?; for (id, info) in &table.columns { let payload = StoredColumnInfo { @@ -243,42 +255,50 @@ impl DojoStoreTrait for SqliteStore payload = excluded.payload ", ) - .bind(felt_to_blob(*owner)) - .bind(felt_to_blob(table.id)) - .bind(felt_to_blob(*id)) + .bind(owner.to_bytes_be().to_vec()) + .bind(table.id.to_bytes_be().to_vec()) + .bind(id.to_bytes_be().to_vec()) .bind(serde_json::to_string(&payload)?) .execute(&mut *transaction) - .await?; + .await + .map_err(DojoToriiError::store_error)?; } - transaction.commit().await?; + transaction + .commit() + .await + .map_err(DojoToriiError::store_error)?; Ok(()) } - async fn read_tables(&self, owners: &[Felt]) -> Result, Self::Error> { + async fn read_tables(&self, owners: &[Felt]) -> DojoToriiResult> { let (table_query, table_args) = select_table_query(owners); let rows = sqlx::query_as_with::(&table_query, table_args) .fetch_all(self.pool()) - .await?; + .await + .map_err(DojoToriiError::store_error)?; let mut tables = rows .into_iter() .map(table_row_into_table) - .collect::, _>>()?; + .collect::, _>>() + .map_err(DojoToriiError::store_error)?; let (column_query, column_args) = select_column_query(owners); let mut columns: HashMap<(Felt, Felt), ColumnInfo> = sqlx::query_as_with::(&column_query, column_args) .fetch_all(self.pool()) - .await? + .await + .map_err(DojoToriiError::store_error)? .into_iter() .map(column_row_into_entry::<(Felt, Felt)>) - .collect::, _>>()?; + .collect::, _>>() + .map_err(DojoToriiError::store_error)?; for table in &mut tables { for key in table.key_fields.iter().chain(table.value_fields.iter()) { - let column = columns.remove(&(table.id, *key)).ok_or_else(|| { - DojoSqliteStoreError::column_not_found(table.name.clone(), &(table.id, *key)) - })?; + let column = columns + .remove(&(table.id, *key)) + .ok_or_else(|| DojoToriiError::ColumnNotFound(table.name.clone(), *key))?; table.columns.insert(*key, column); } } diff --git a/crates/dojo/src/table.rs b/crates/dojo/src/table.rs index 05048a88..674f4808 100644 --- a/crates/dojo/src/table.rs +++ b/crates/dojo/src/table.rs @@ -178,7 +178,7 @@ impl DojoTable { pub fn get_column(&self, selector: &Felt) -> DojoToriiResult<&ColumnInfo> { self.columns .get(selector) - .ok_or_else(|| DojoToriiError::ColumnNotFound(*selector, self.name.clone())) + .ok_or_else(|| DojoToriiError::ColumnNotFound(self.name.clone(), *selector)) } pub fn selectors(&self) -> impl Iterator + '_ { @@ -270,7 +270,7 @@ impl DojoTableInfo { pub fn get_column(&self, selector: &Felt) -> DojoToriiResult<&ColumnInfo> { self.columns .get(selector) - .ok_or_else(|| DojoToriiError::ColumnNotFound(*selector, self.name.clone())) + .ok_or_else(|| DojoToriiError::ColumnNotFound(self.name.clone(), *selector)) } pub fn selectors(&self) -> impl Iterator + '_ { diff --git a/crates/introspect-postgres-sink/Cargo.toml b/crates/introspect-postgres-sink/Cargo.toml deleted file mode 100644 index 816f88dd..00000000 --- a/crates/introspect-postgres-sink/Cargo.toml +++ /dev/null @@ -1,36 +0,0 @@ -[package] -name = "torii-introspect-postgres-sink" -version = "0.1.0" -edition = "2021" -description = "PostgreSQL sink implementation for Torii runtime" -authors = ["Torii Runtime "] -license = "Apache-2.0" - -[dependencies] -sqlx = { workspace = true, features = [ - "postgres", - "sqlite", - "runtime-tokio-rustls", - "macros", - "migrate", -] } -anyhow.workspace = true -async-trait.workspace = true -metrics.workspace = true -xxhash-rust.workspace = true -hex.workspace = true -serde.workspace = true -serde_json.workspace = true -starknet.workspace = true -tokio.workspace = true -torii.workspace = true -tracing.workspace = true -starknet-types-core.workspace = true -thiserror.workspace = true -introspect-types.workspace = true -itertools.workspace = true - -# Local crates -torii-introspect.workspace = true -torii-postgres.workspace = true -torii-common.workspace = true diff --git a/crates/introspect-postgres-sink/src/error.rs b/crates/introspect-postgres-sink/src/error.rs deleted file mode 100644 index 0dda8017..00000000 --- a/crates/introspect-postgres-sink/src/error.rs +++ /dev/null @@ -1,134 +0,0 @@ -use std::sync::PoisonError; - -use introspect_types::{PrimaryTypeDef, TypeDef}; -use sqlx::Error as SqlxError; -use starknet_types_core::felt::Felt; -use thiserror::Error; - -#[derive(Debug, Error)] -pub enum PgTypeError { - #[error("Unsupported type for {0}")] - UnsupportedType(String), - #[error("Nested arrays are not supported")] - NestedArrays, -} - -pub type PgTypeResult = std::result::Result; - -#[derive(Debug, Error)] -pub enum PgTableError { - #[error("Column with id: {0} not found in table {1}")] - ColumnNotFound(Felt, String), - #[error(transparent)] - TypeError(#[from] PgTypeError), - #[error("Current type mismatch error")] - TypeMismatch, - #[error("Unsupported upgrade for table {table} column {column}: {reason}")] - UnsupportedUpgrade { - table: String, - column: String, - reason: UpgradeError, - }, -} - -pub type TableResult = std::result::Result; - -#[derive(Debug, thiserror::Error)] -pub enum UpgradeError { - #[error("Failed to upgrade type from {old} to {new}")] - TypeUpgradeError { - old: &'static str, - new: &'static str, - }, - #[error("Failed to upgrade primary from {old} to {new}")] - PrimaryUpgradeError { - old: &'static str, - new: &'static str, - }, - #[error(transparent)] - TypeCreationError(#[from] PgTypeError), - #[error("Array length cannot be decreased from {old} to {new}")] - ArrayLengthDecreaseError { old: u32, new: u32 }, - #[error("Cannot reduce element in tuple")] - TupleReductionError, -} - -pub type UpgradeResult = Result; - -impl UpgradeError { - pub fn type_upgrade_err(old: &TypeDef, new: &TypeDef) -> UpgradeResult { - Err(Self::TypeUpgradeError { - old: old.item_name(), - new: new.item_name(), - }) - } - pub fn type_upgrade_to_err(old: &TypeDef, new: &'static str) -> UpgradeResult { - Err(Self::TypeUpgradeError { - old: old.item_name(), - new, - }) - } - pub fn type_cast_err(old: &TypeDef, new: &'static str) -> UpgradeResult { - Err(Self::TypeUpgradeError { - old: old.item_name(), - new, - }) - } - pub fn array_shorten_err(old: u32, new: u32) -> UpgradeResult { - Err(Self::ArrayLengthDecreaseError { old, new }) - } - pub fn primary_upgrade_err(old: &PrimaryTypeDef, new: &PrimaryTypeDef) -> UpgradeResult { - Err(Self::PrimaryUpgradeError { - old: old.item_name(), - new: new.item_name(), - }) - } -} - -pub trait UpgradeResultExt { - fn to_table_result(self, table: &str, column: &str) -> TableResult; -} - -impl UpgradeResultExt for UpgradeResult { - fn to_table_result(self, table: &str, column: &str) -> TableResult { - self.map_err(|err| PgTableError::UnsupportedUpgrade { - table: table.to_string(), - column: column.to_string(), - reason: err, - }) - } -} - -#[derive(Debug, thiserror::Error)] -pub enum PgDbError { - #[error(transparent)] - DatabaseError(#[from] SqlxError), - #[error("Invalid event format: {0}")] - InvalidEventFormat(String), - #[error(transparent)] - JsonError(#[from] serde_json::Error), - #[error(transparent)] - IoError(#[from] std::io::Error), - #[error(transparent)] - TableError(#[from] PgTableError), - #[error(transparent)] - TypeError(#[from] PgTypeError), - #[error("Table with id: {0} already exists, incoming name: {1}, existing name: {2}")] - TableAlreadyExists(Felt, String, String), - #[error("Table not found with id: {0}")] - TableNotFound(Felt), - #[error("Table not alive - id: {0}, name: {1}")] - TableNotAlive(Felt, String), - #[error("Manager does not support updating")] - UpdateNotSupported, - #[error("Table poison error: {0}")] - PoisonError(String), -} - -pub type PgDbResult = std::result::Result; - -impl From> for PgDbError { - fn from(err: PoisonError) -> Self { - Self::PoisonError(err.to_string()) - } -} diff --git a/crates/introspect-postgres-sink/src/lib.rs b/crates/introspect-postgres-sink/src/lib.rs deleted file mode 100644 index 83d42122..00000000 --- a/crates/introspect-postgres-sink/src/lib.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub mod create; -pub mod error; -pub mod json; -pub mod processor; -pub mod query; -pub mod sink; -pub mod table; -pub mod types; -pub mod upgrade; -pub mod utils; - -pub use error::{ - PgDbError, PgDbResult, PgTableError, PgTypeError, PgTypeResult, TableResult, UpgradeError, - UpgradeResult, UpgradeResultExt, -}; -pub use processor::IntrospectPgDb; -pub use types::{ - PgSchema, PostgresArray, PostgresField, PostgresScalar, PostgresType, PrimaryKey, SchemaName, -}; -pub use utils::{truncate, HasherExt}; - -pub const INTROSPECT_PG_SINK_MIGRATIONS: sqlx::migrate::Migrator = sqlx::migrate!("./migrations"); diff --git a/crates/introspect-postgres-sink/src/processor.rs b/crates/introspect-postgres-sink/src/processor.rs deleted file mode 100644 index 196a7ef1..00000000 --- a/crates/introspect-postgres-sink/src/processor.rs +++ /dev/null @@ -1,334 +0,0 @@ -use crate::json::PostgresJsonSerializer; -use crate::query::{fetch_columns, fetch_dead_fields, fetch_tables, CreatePgTable}; -use crate::table::{DeadField, PgTable}; -use crate::{PgDbError, PgDbResult, PgSchema, INTROSPECT_PG_SINK_MIGRATIONS}; -use introspect_types::ColumnInfo; -use serde_json::Serializer as JsonSerializer; -use sqlx::PgPool; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use std::io::Write; -use std::ops::Deref; -use std::rc::Rc; -use std::sync::RwLock; -use torii::etl::envelope::MetaData; -use torii_common::sql::{PgQuery, Queries}; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_introspect::schema::TableSchema; -use torii_introspect::InsertsFields; -use torii_postgres::PostgresConnection; - -pub const COMMIT_CMD: &str = "--COMMIT"; -pub const DEAD_MEMBERS_TABLE: &str = "__introspect_dead_fields"; -pub const TABLES_TABLE: &str = "__introspect_tables"; -pub const COLUMNS_TABLE: &str = "__introspect_columns"; -pub const METADATA_CONFLICTS: &str = "__updated_at = NOW(), __updated_block = EXCLUDED.__updated_block, __updated_tx = EXCLUDED.__updated_tx"; - -#[derive(Debug, Default)] -pub struct PostgresTables(pub RwLock>); - -#[derive(Debug, Default)] -pub struct DeadFields(pub RwLock>>); - -impl Deref for PostgresTables { - type Target = RwLock>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl Deref for DeadFields { - type Target = RwLock>>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl From> for PgSchema { - fn from(value: Option) -> Self { - match value { - Some(s) => Self::Custom(s), - None => PgSchema::Public, - } - } -} - -impl From<()> for PgSchema { - fn from(_: ()) -> Self { - PgSchema::Public - } -} - -impl From for PgSchema { - fn from(value: String) -> Self { - Self::Custom(value) - } -} - -impl From<&str> for PgSchema { - fn from(value: &str) -> Self { - Self::Custom(value.to_string()) - } -} - -impl From> for PgSchema { - fn from(value: Option<&str>) -> Self { - match value { - Some(s) => Self::Custom(s.to_string()), - None => PgSchema::Public, - } - } -} - -impl PostgresTables { - pub fn create_table( - &self, - schema: &Rc, - to_table: impl Into, - metadata: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - let (id, table) = Into::::into(to_table).into(); - self.assert_table_not_exists(&id, &table.name)?; - CreatePgTable::new(schema, &id, &table)?.make_queries(queries); - let table = PgTable::new(schema, table, None); - table.insert_queries( - &id, - None, - metadata.block_number.unwrap_or_default(), - metadata.transaction_hash, - queries, - )?; - let mut tables: std::sync::RwLockWriteGuard<'_, HashMap> = self.write()?; - tables.insert(id, table); - Ok(()) - } - - pub fn update_table( - &self, - to_table: impl Into, - metadata: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - let tx_hash = &metadata.transaction_hash; - let block_number = metadata.block_number.unwrap_or_default(); - let (id, table) = Into::::into(to_table).into(); - let mut tables = self.write()?; - let existing = tables - .get_mut(&id) - .ok_or_else(|| PgDbError::TableNotFound(id))?; - let upgrades = existing.update_from_info(&id, &table)?; - upgrades.to_queries(&id, block_number, tx_hash, queries)?; - existing.insert_queries( - &id, - Some(&upgrades.columns_upgraded), - block_number, - metadata.transaction_hash, - queries, - ) - } - - pub fn assert_table_not_exists(&self, id: &Felt, name: &str) -> PgDbResult<()> { - match self.read()?.get(id) { - Some(existing) => Err(PgDbError::TableAlreadyExists( - *id, - name.to_string(), - existing.name.to_string(), - )), - None => Ok(()), - } - } - - pub fn set_table_dead(&self, id: &Felt) -> PgDbResult<()> { - let mut tables = self.write()?; - match tables.get_mut(id) { - Some(table) => { - table.alive = false; - Ok(()) - } - None => Err(PgDbError::TableNotFound(*id)), - } - } - - pub fn insert_fields( - &self, - event: &InsertsFields, - context: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - let tables = self.read().unwrap(); - let table = match tables.get(&event.table) { - Some(table) => Ok(table), - None => Err(PgDbError::TableNotFound(event.table)), - }?; - if !table.alive { - return Ok(()); - } - let record = table.get_record_schema(&event.columns)?; - let table_name = &table.name; - let mut writer = Vec::new(); - let schema = &table.schema; - write!( - writer, - r#"INSERT INTO "{schema}"."{table_name}" SELECT * FROM jsonb_populate_recordset(NULL::"{schema}"."{table_name}", $$"# - ) - .unwrap(); - record.parse_records_with_metadata( - &event.records, - context, - &mut JsonSerializer::new(&mut writer), - &PostgresJsonSerializer, - )?; - write!( - writer, - r#"$$) ON CONFLICT ("{}") DO UPDATE SET {METADATA_CONFLICTS}"#, - record.primary().name - ) - .unwrap(); - for ColumnInfo { name, .. } in record.columns() { - write!( - writer, - r#", "{name}" = COALESCE(EXCLUDED."{name}", "{table_name}"."{name}")"#, - name = name - ) - .unwrap(); - } - let string = unsafe { String::from_utf8_unchecked(writer) }; - queries.add(string); - Ok(()) - } - - pub fn handle_message( - &self, - schema: &Rc, - msg: &IntrospectMsg, - metadata: &MetaData, - queries: &mut Vec, - ) -> PgDbResult<()> { - match msg { - IntrospectMsg::CreateTable(event) => { - self.create_table(schema, event.clone(), metadata, queries) - } - IntrospectMsg::UpdateTable(event) => { - self.update_table(event.clone(), metadata, queries) - } - IntrospectMsg::AddColumns(event) => self.set_table_dead(&event.table), - IntrospectMsg::DropColumns(event) => self.set_table_dead(&event.table), - IntrospectMsg::RetypeColumns(event) => self.set_table_dead(&event.table), - IntrospectMsg::RetypePrimary(event) => self.set_table_dead(&event.table), - IntrospectMsg::RenameTable(_) - | IntrospectMsg::DropTable(_) - | IntrospectMsg::RenameColumns(_) - | IntrospectMsg::RenamePrimary(_) => Ok(()), - IntrospectMsg::InsertsFields(event) => self.insert_fields(event, metadata, queries), - IntrospectMsg::DeleteRecords(_) | IntrospectMsg::DeletesFields(_) => Ok(()), - } - } -} - -fn make_schema_query(schema: &PgSchema) -> String { - format!(r#"CREATE SCHEMA IF NOT EXISTS "{schema}""#) -} - -pub struct IntrospectPgDb { - tables: PostgresTables, - schema: PgSchema, - pool: T, -} - -impl PostgresConnection for IntrospectPgDb { - fn pool(&self) -> &PgPool { - self.pool.pool() - } -} - -impl IntrospectPgDb { - pub fn new(pool: T, schema: impl Into) -> Self { - Self { - tables: PostgresTables::default(), - schema: schema.into(), - pool, - } - } - - pub async fn load_store_data(&self) -> PgDbResult<()> { - let mut tables = fetch_tables(self.pool(), &self.schema) - .await? - .into_iter() - .map(|t| t.to_table(&self.schema)) - .collect::>(); - for (table_id, id, column_info) in fetch_columns(self.pool(), &self.schema).await? { - if let Some(table) = tables.get_mut(&table_id) { - table.columns.insert(id, column_info); - } - } - for (table_id, id, field) in fetch_dead_fields(self.pool(), &self.schema).await? { - if let Some(table) = tables.get_mut(&table_id) { - table.dead.insert(id, field); - } - } - let mut tables_map = self.tables.write()?; - tables_map.extend(tables); - Ok(()) - } - - pub async fn initialize_introspect_pg_sink(&self) -> PgDbResult<()> { - self.migrate(Some("introspect"), INTROSPECT_PG_SINK_MIGRATIONS) - .await?; - self.execute_queries(make_schema_query(&self.schema)) - .await?; - self.load_store_data().await - } - - pub async fn process_message( - &self, - msg: &IntrospectMsg, - metadata: &MetaData, - ) -> PgDbResult<()> { - let mut queries = Vec::new(); - { - let schema = Rc::new(self.schema.clone()); - self.tables - .handle_message(&schema, msg, metadata, &mut queries)?; - } - self.execute_queries(queries).await?; - Ok(()) - } - - pub async fn process_messages( - &self, - msgs: Vec<&IntrospectBody>, - ) -> PgDbResult>> { - let mut queries = Vec::new(); - let mut results = Vec::with_capacity(msgs.len()); - { - let schema = Rc::new(self.schema.clone()); - for body in msgs { - let (msg, metadata) = body.into(); - results.push( - self.tables - .handle_message(&schema, msg, metadata, &mut queries), - ); - } - } - let mut batch = Vec::new(); - for query in queries { - if query == *COMMIT_CMD { - self.execute_queries(std::mem::take(&mut batch)).await?; - } else { - batch.push(query); - } - } - if !batch.is_empty() { - self.execute_queries(batch).await?; - } - Ok(results) - } -} - -pub struct MessageWithContext<'a, M> { - pub msg: &'a M, - pub context: &'a MetaData, -} diff --git a/crates/introspect-postgres-sink/src/sink.rs b/crates/introspect-postgres-sink/src/sink.rs deleted file mode 100644 index d33b260b..00000000 --- a/crates/introspect-postgres-sink/src/sink.rs +++ /dev/null @@ -1,102 +0,0 @@ -use crate::processor::IntrospectPgDb; -use anyhow::Result; -use async_trait::async_trait; -use std::sync::Arc; -use torii::axum::Router; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, SinkContext, TopicInfo}, -}; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_postgres::PostgresConnection; - -pub const LOGGING_TARGET: &str = "torii::sinks::introspect::postgres"; -const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); - -#[async_trait] -impl Sink for IntrospectPgDb { - fn name(&self) -> &'static str { - "introspect-postgres" - } - - fn interested_types(&self) -> Vec { - vec![INTROSPECT_TYPE] - } - - async fn process(&self, envelopes: &[Envelope], _batch: &ExtractionBatch) -> Result<()> { - let mut processed = 0usize; - let mut create_tables: usize = 0usize; - let mut update_tables = 0usize; - let mut inserts_fields = 0usize; - let mut inserted_records = 0usize; - let mut delete_records = 0usize; - let mut msgs = Vec::with_capacity(envelopes.len()); - for envelope in envelopes { - if envelope.type_id == INTROSPECT_TYPE { - if let Some(body) = envelope.downcast_ref::() { - match &body.msg { - IntrospectMsg::CreateTable(_) => create_tables += 1, - IntrospectMsg::UpdateTable(_) => update_tables += 1, - IntrospectMsg::InsertsFields(event) => { - inserts_fields += 1; - inserted_records += event.records.len(); - } - IntrospectMsg::DeleteRecords(event) => { - delete_records += event.rows.len(); - } - _ => {} - } - processed += 1; - msgs.push(body); - } - } - } - self.process_messages(msgs).await?; - if processed > 0 { - tracing::info!( - target: LOGGING_TARGET, - processed, - create_tables, - update_tables, - inserts_fields, - inserted_records, - delete_records, - "Processed introspect envelopes" - ); - ::metrics::counter!("torii_introspect_sink_messages_total", "message" => "create_table") - .increment(create_tables as u64); - ::metrics::counter!("torii_introspect_sink_messages_total", "message" => "update_table") - .increment(update_tables as u64); - ::metrics::counter!("torii_introspect_sink_messages_total", "message" => "inserts_fields") - .increment(inserts_fields as u64); - ::metrics::counter!("torii_introspect_sink_records_total", "message" => "inserts_fields") - .increment(inserted_records as u64); - ::metrics::counter!("torii_introspect_sink_records_total", "message" => "delete_records") - .increment(delete_records as u64); - } - - Ok(()) - } - - fn topics(&self) -> Vec { - Vec::new() - } - - fn build_routes(&self) -> Router { - Router::new() - } - - async fn initialize( - &mut self, - _event_bus: Arc, - _context: &SinkContext, - ) -> Result<()> { - self.initialize_introspect_pg_sink().await?; - tracing::info!( - target: LOGGING_TARGET, - "Initialized introspect Postgres sink" - ); - Ok(()) - } -} diff --git a/crates/introspect-postgres-sink/src/table.rs b/crates/introspect-postgres-sink/src/table.rs deleted file mode 100644 index eeef1a9c..00000000 --- a/crates/introspect-postgres-sink/src/table.rs +++ /dev/null @@ -1,104 +0,0 @@ -use introspect_types::{ColumnInfo, MemberDef, PrimaryDef, TypeDef}; -use itertools::Itertools; -use sqlx::Error::Encode as EncodeError; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use torii_common::sql::PgQuery; -use torii_introspect::{schema::TableInfo, tables::RecordSchema}; - -use crate::{ - query::{insert_columns_query, insert_table_query}, - PgDbResult, PgSchema, PgTableError, TableResult, -}; - -#[derive(Debug)] -pub struct PgTable { - pub schema: PgSchema, - pub name: String, - pub primary: PrimaryDef, - pub columns: HashMap, - pub alive: bool, - pub dead: HashMap, -} - -#[derive(Debug)] -pub struct DeadField { - pub name: String, - pub type_def: TypeDef, -} - -impl From for DeadField { - fn from(value: MemberDef) -> Self { - DeadField { - name: value.name, - type_def: value.type_def, - } - } -} - -impl From for MemberDef { - fn from(value: DeadField) -> Self { - MemberDef { - name: value.name, - attributes: Vec::new(), - type_def: value.type_def, - } - } -} - -impl PgTable { - pub fn column(&self, id: &Felt) -> TableResult<&ColumnInfo> { - self.columns - .get(id) - .ok_or_else(|| PgTableError::ColumnNotFound(*id, self.name.clone())) - } - - pub fn columns(&self, ids: &[Felt]) -> TableResult> { - ids.iter() - .map(|id| self.column(id)) - .collect::>>() - } - - pub fn new(schema: &PgSchema, info: TableInfo, dead: Option>) -> Self { - PgTable { - schema: schema.clone(), - name: info.name, - primary: info.primary, - columns: info.columns.into_iter().map_into().collect(), - alive: true, - dead: dead.unwrap_or_default().into_iter().collect(), - } - } - pub fn get_record_schema(&self, columns: &[Felt]) -> TableResult> { - Ok(RecordSchema::new(&self.primary, self.columns(columns)?)) - } - pub fn insert_queries( - &self, - id: &Felt, - column_ids: Option<&[Felt]>, - block_number: u64, - transaction_hash: Felt, - queries: &mut Vec, - ) -> PgDbResult<()> { - queries.push( - insert_table_query( - &self.schema, - id, - &self.name, - &self.primary, - block_number, - &transaction_hash, - ) - .map_err(EncodeError)?, - ); - let columns = match column_ids { - Some(ids) => ids.iter().zip(self.columns(ids)?).collect_vec(), - None => self.columns.iter().collect_vec(), - }; - queries.push( - insert_columns_query(&self.schema, id, columns, block_number, &transaction_hash) - .map_err(EncodeError)?, - ); - Ok(()) - } -} diff --git a/crates/introspect-sqlite-sink/Cargo.toml b/crates/introspect-sql-sink/Cargo.toml similarity index 53% rename from crates/introspect-sqlite-sink/Cargo.toml rename to crates/introspect-sql-sink/Cargo.toml index 6113d553..317a535f 100644 --- a/crates/introspect-sqlite-sink/Cargo.toml +++ b/crates/introspect-sql-sink/Cargo.toml @@ -1,36 +1,38 @@ [package] -name = "torii-introspect-sqlite-sink" +name = "torii-introspect-sql-sink" version = "0.1.0" edition = "2021" -description = "SQLite sink implementation for Torii runtime" +description = "Database sink implementation for Torii runtime" authors = ["Torii Runtime "] license = "Apache-2.0" [dependencies] -sqlx = { workspace = true, features = [ - "sqlite", - "runtime-tokio-rustls", - "macros", - "migrate", -] } +sqlx = { workspace = true, features = ["runtime-tokio-rustls", "macros"] } anyhow.workspace = true async-trait.workspace = true metrics.workspace = true hex.workspace = true serde.workspace = true serde_json.workspace = true -starknet.workspace = true +starknet-types-core.workspace = true +starknet-types-raw.workspace = true tokio.workspace = true torii.workspace = true tracing.workspace = true -starknet-types-core.workspace = true thiserror.workspace = true introspect-types.workspace = true itertools.workspace = true primitive-types.workspace = true +# Local crates +torii-dojo.workspace = true torii-introspect.workspace = true -torii-sqlite.workspace = true +torii-sql.workspace = true + +# Postgres features +xxhash-rust = { workspace = true, optional = true } + -[lints] -workspace = true +[features] +postgres = ["sqlx/postgres", "dep:xxhash-rust", "torii-sql/postgres"] +sqlite = ["sqlx/sqlite", "torii-sql/sqlite"] diff --git a/crates/introspect-sql-sink/README.md b/crates/introspect-sql-sink/README.md new file mode 100644 index 00000000..3019a127 --- /dev/null +++ b/crates/introspect-sql-sink/README.md @@ -0,0 +1,130 @@ +# torii-introspect-sql-sink + +Materialises **Dojo / Introspect events** into a live SQL schema. This is +the sink that turns `IntrospectMsg::CreateTable` into a `CREATE TABLE`, +`InsertsFields` into an `INSERT`, `DropTable` into a `DROP TABLE`, and so +on. Generic over a backend (`IntrospectPgDb`, `IntrospectSqliteDb`) via +the `IntrospectProcessor` / `IntrospectQueryMaker` traits. + +## Role in Torii + +This is the first sink in a Dojo pipeline. It owns the SQL schema. Other +Dojo-aware sinks (`torii-ecs-sink`, `arcade-sink`) read from the tables +this sink writes. Every binary that indexes a Dojo world includes it — see +`bins/torii-introspect-bin`, `bins/torii-arcade`. + +## Architecture + +```text +Envelope with DOJO_TYPE_ID (from torii-dojo::DojoDecoder) + | + v ++-----------------------------------------------------+ +| IntrospectDb | +| | +| impl Sink: | +| process(envelopes): | +| filter DojoEvent::Introspect(IntrospectMsg) | +| collect into Vec | +| self.process_messages(...) | +| (IntrospectProcessor trait) | +| | +| state: | +| - tables cache (Vec) | +| - namespace mode (NamespaceMode) | +| - column/field metadata (DbColumn, DbDeadField) | ++------+----------------------------------------------+ + | ^ + | | + v | ++-----------------------------+ | +| IntrospectQueryMaker (SQL) | | +| CREATE/ALTER/DROP TABLE | | +| INSERT/DELETE rows | | +| type mapping | | ++--------+--------------------+ | + | | + v | ++-----------------------------+ +----------------+ +| Postgres backend | | SQLite backend | +| IntrospectPgDb | | IntrospectSqliteDb +| native DDL | | JSON append- | +| per-namespace schemas | | only log per | +| typed columns | | namespace | ++-----------------------------+ +----------------+ + +(feature gated: postgres, sqlite; require at least one) +runtime.rs (both features): DbPool::Postgres | DbPool::Sqlite +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `IntrospectDb` | `src/processor.rs` | — | Generic sink; `Sink` impl lives in `sink.rs` | +| `IntrospectSqlSink` (marker trait) | `src/sink.rs` | 14 | Provides `const NAME: &'static str` returned from `Sink::name` | +| `IntrospectProcessor` | `src/backend.rs` | — | Async backend contract: `process_messages(&[&IntrospectBody])` | +| `IntrospectQueryMaker` | `src/backend.rs` | — | SQL generation contract (DDL + record writes) | +| `IntrospectInitialize` | `src/backend.rs` | — | One-shot schema-setup contract | +| `DbTable`, `DbColumn`, `DbDeadField` | `src/processor.rs` | — | In-memory schema snapshot | +| `NamespaceMode` | `src/namespace.rs` | 11 | `None` / `Single(Arc)` / `Address` / `Named(map)` / `Addresses(set)` | +| `TableKey`, `NamespaceKey` | `src/namespace.rs` | 20 / 32 | Routes a table to its schema bucket | +| `Table`, `DeadField`, `DeadFieldDef` | `src/table.rs` | — | Schema evolution helpers (dropped columns, retyped fields) | +| `IntrospectPgDb` | `src/postgres/mod.rs` (feature `postgres`) | — | Postgres backend (native DDL) | +| `IntrospectSqliteDb` | `src/sqlite/mod.rs` (feature `sqlite`) | — | SQLite backend (JSON append-only log) | +| Errors | `src/error.rs` | — | `DbError`, `TableError`, `TypeError`, `UpgradeError`, etc. | + +### Internal Modules + +- `sink` — `impl Sink for IntrospectDb`; the `process` branch filtering `DojoEvent::Introspect` and forwarding to `process_messages`. +- `processor` — `IntrospectDb` struct + the generic `IntrospectProcessor` impl over `Pool`. Holds the cached `DbTable` list and the `NamespaceMode`. +- `backend` — three traits (`IntrospectQueryMaker`, `IntrospectProcessor`, `IntrospectInitialize`) — the backend interface. +- `namespace` — `NamespaceMode` + `TableKey` / `NamespaceKey`; routes a Dojo table to its physical schema slot. +- `table` / `tables` — `Table`, `DeadField`, cached list of alive tables; used to resolve incremental `UpdateTable` messages against prior state. +- `error` — typed domain errors (most carry a `TableKey` or `ColumnKey` for diagnostics). +- `postgres/` — Postgres backend: DDL via native `CREATE TABLE`, `ALTER TABLE`, JSON serialisation for unknown types, typed columns where possible. +- `sqlite/` — SQLite backend: append-only JSON log per (namespace, table) with type coercion on read. +- `runtime` (feature `postgres + sqlite`) — `IntrospectDb` runtime enum so one binary can open either backend. + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `Backend::NAME` (e.g. `"introspect-pg"`, `"introspect-sqlite"`) | +| `interested_types` | `[DOJO_TYPE_ID]` | +| `process` | Extracts `IntrospectMsg` from each envelope, counts categories (create/update/inserts/deletes) for metrics, calls `self.process_messages(...)`, and logs per-message failures | +| `topics` | `Vec::new()` — no EventBus topic | +| `build_routes` | `Router::new()` — no HTTP | +| `initialize` | Calls `self.initialize_introspect_sql_sink()` — creates catalog tables, idempotent | + +### Namespace modes (NamespaceMode) + +- `None` — single namespace, no routing. +- `Single(Arc)` — all tables in one named namespace. +- `Address` — one namespace per owner Felt address. +- `Named(HashMap>)` — owner → explicit namespace map. +- `Addresses(HashSet)` — allow-list; unknown owners are dropped. + +Pick the mode when the binary constructs the backend; it determines how +tables are grouped in Postgres schemas or SQLite filenames. + +### Metrics + +| Metric | Labels | +|---|---| +| `torii_introspect_sink_messages_total` | `message=create_table|update_table|inserts_fields` | +| `torii_introspect_sink_records_total` | `message=inserts_fields|delete_records` | + +### Interactions + +- **Upstream (consumers)**: `bins/torii-introspect-bin`, `bins/torii-arcade`, `bins/torii-introspect-synth`. +- **Downstream deps**: `torii`, `torii-dojo`, `torii-introspect`, `torii-sql`, `sqlx` (feature-gated), `introspect-types`, `starknet-types-core`, `starknet-types-raw`, `primitive-types`, `xxhash-rust` (feature `postgres`), `tokio`, `async-trait`, `itertools`, `hex`, `metrics`, `tracing`. +- **Features**: `postgres`, `sqlite` (at least one required). The `runtime` module needs both. + +### Extension Points + +- New backend → impl `IntrospectQueryMaker` + `IntrospectProcessor` + `IntrospectInitialize` + provide `IntrospectSqlSink::NAME`. The `Sink` impl is generic; no changes there. +- New namespace strategy → add a variant to `NamespaceMode` and extend every `resolve_*` helper that pattern-matches it. +- Change the SQLite row encoding → swap the JSON writer in `sqlite/` and bump the schema version; catalog tables detect the mismatch on startup. diff --git a/crates/introspect-postgres-sink/migrations/001_domains.sql b/crates/introspect-sql-sink/migrations/postgres/001_domains.sql similarity index 100% rename from crates/introspect-postgres-sink/migrations/001_domains.sql rename to crates/introspect-sql-sink/migrations/postgres/001_domains.sql diff --git a/crates/introspect-postgres-sink/migrations/002_metadata_function.sql b/crates/introspect-sql-sink/migrations/postgres/002_metadata_function.sql similarity index 100% rename from crates/introspect-postgres-sink/migrations/002_metadata_function.sql rename to crates/introspect-sql-sink/migrations/postgres/002_metadata_function.sql diff --git a/crates/introspect-postgres-sink/migrations/003_store.sql b/crates/introspect-sql-sink/migrations/postgres/003_store.sql similarity index 94% rename from crates/introspect-postgres-sink/migrations/003_store.sql rename to crates/introspect-sql-sink/migrations/postgres/003_store.sql index d58e2145..461fd11f 100644 --- a/crates/introspect-postgres-sink/migrations/003_store.sql +++ b/crates/introspect-sql-sink/migrations/postgres/003_store.sql @@ -22,7 +22,10 @@ CREATE TABLE IF NOT EXISTS introspect.db_tables ( "schema" TEXT NOT NULL, id felt252 NOT NULL, name TEXT NOT NULL, + "owner" felt252 NOT NULL, primary_def introspect.primary_def NOT NULL, + append_only BOOLEAN NOT NULL DEFAULT FALSE, + alive BOOLEAN NOT NULL DEFAULT TRUE, created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), created_block uint64 NOT NULL, diff --git a/crates/introspect-sqlite-sink/migrations/001_init.sql b/crates/introspect-sql-sink/migrations/sqlite/001_init.sql similarity index 100% rename from crates/introspect-sqlite-sink/migrations/001_init.sql rename to crates/introspect-sql-sink/migrations/sqlite/001_init.sql diff --git a/crates/introspect-sql-sink/migrations/sqlite/002_schema_state.sql b/crates/introspect-sql-sink/migrations/sqlite/002_schema_state.sql new file mode 100644 index 00000000..1c345fb2 --- /dev/null +++ b/crates/introspect-sql-sink/migrations/sqlite/002_schema_state.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS introspect_db_tables ( + namespace TEXT NOT NULL, + id TEXT NOT NULL, + owner TEXT NOT NULL, + name TEXT NOT NULL, + "primary" TEXT NOT NULL, + columns TEXT NOT NULL, + append_only INTEGER NOT NULL DEFAULT 0, + alive INTEGER NOT NULL DEFAULT 1, + updated_at INTEGER NOT NULL DEFAULT (unixepoch()), + PRIMARY KEY (namespace, id) +); diff --git a/crates/introspect-sql-sink/src/backend.rs b/crates/introspect-sql-sink/src/backend.rs new file mode 100644 index 00000000..563e2e5e --- /dev/null +++ b/crates/introspect-sql-sink/src/backend.rs @@ -0,0 +1,114 @@ +use crate::processor::{messages_to_queries, DbColumn, DbDeadField, DbTable, COMMIT_CMD}; +use crate::table::Table; +use crate::tables::Tables; +use crate::{DbResult, NamespaceMode, RecordResult, TableResult}; +use async_trait::async_trait; +use introspect_types::{ColumnDef, PrimaryDef}; +use sqlx::{Database, Pool}; +use starknet_types_core::felt::Felt; +use std::fmt::Debug; +use torii_introspect::events::IntrospectBody; +use torii_introspect::Record; +use torii_sql::{Executable, FlexQuery, PoolExt}; + +#[async_trait] +pub trait IntrospectProcessor { + async fn process_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>>; +} + +#[async_trait] +pub trait IntrospectInitialize { + async fn initialize(&self) -> DbResult<()>; + async fn load_tables(&self, namespaces: &Option>) -> DbResult>; + async fn load_columns(&self, namespaces: &Option>) -> DbResult>; + async fn load_dead_fields( + &self, + namespaces: &Option>, + ) -> DbResult>; +} + +#[allow(clippy::too_many_arguments)] +pub trait IntrospectQueryMaker: Database { + fn create_table_queries( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> TableResult<()>; + fn update_table_queries( + table: &mut Table, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> TableResult<()>; + fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> RecordResult<()>; +} + +#[async_trait] +pub trait IntrospectPool { + async fn commit_queries(&self, queries: Vec>) -> DbResult<()>; + async fn execute_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + let mut queries = Vec::new(); + let results = messages_to_queries::(namespaces, tables, msgs, &mut queries)?; + self.commit_queries(queries).await?; + Ok(results) + } +} + +#[async_trait] +impl IntrospectPool for Pool +where + Vec>: Executable, + FlexQuery: Debug + Clone, +{ + async fn commit_queries(&self, queries: Vec>) -> DbResult<()> { + let mut batch = Vec::new(); + for query in queries { + if query == *COMMIT_CMD { + self.execute_queries(std::mem::take(&mut batch)).await?; + } else { + batch.push(query); + } + } + if !batch.is_empty() { + match self.execute_queries(batch.clone()).await { + Ok(_) => (), + Err(e) => { + for query in batch { + eprintln!("Failed query: {query:?}"); + } + return Err(e.into()); + } + } + } + Ok(()) + } +} diff --git a/crates/introspect-sql-sink/src/error.rs b/crates/introspect-sql-sink/src/error.rs new file mode 100644 index 00000000..b3b201d7 --- /dev/null +++ b/crates/introspect-sql-sink/src/error.rs @@ -0,0 +1,248 @@ +use crate::TableKey; +use introspect_types::{DecodeError, PrimaryTypeDef, TypeDef}; +use sqlx::error::BoxDynError; +use sqlx::Error as SqlxError; +use starknet_types_core::felt::Felt; +use starknet_types_core::felt::FromStrError; +use std::fmt::Display; +use std::sync::PoisonError; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum TypeError { + #[error("Unsupported type for {0}")] + UnsupportedType(String), + #[error("Nested arrays are not supported")] + NestedArrays, +} + +pub type TypeResult = std::result::Result; + +#[derive(Debug, Error)] +pub enum TableError { + #[error("Table {0} has not got columns: {1:?}")] + ColumnsNotFound(String, ColumnsNotFoundError), + #[error(transparent)] + TypeError(#[from] TypeError), + #[error("Current type mismatch error")] + TypeMismatch, + #[error("Unsupported upgrade for table {table} column {column}: {reason}")] + UnsupportedUpgrade { + table: String, + column: String, + reason: UpgradeError, + }, + #[error(transparent)] + JsonError(#[from] serde_json::Error), + #[error("error occurred while encoding a value: {0}")] + Encode(#[from] BoxDynError), +} + +#[derive(Debug)] +pub struct ColumnNotFoundError(pub Felt); + +#[derive(Debug, Default, Error)] +pub struct ColumnsNotFoundError(pub Vec); + +impl ColumnsNotFoundError { + pub fn to_table_error(self, table: &str) -> TableError { + TableError::ColumnsNotFound(table.to_string(), self) + } +} + +impl Display for ColumnsNotFoundError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some((first, rest)) = self.0.split_first() { + write!(f, "0x{first:#063x}")?; + for col in rest { + write!(f, ", 0x{col:#063x}")?; + } + } + Ok(()) + } +} + +pub trait CollectColumnResults { + fn collect_columns(self) -> Result, ColumnsNotFoundError>; +} + +impl CollectColumnResults for I +where + I: Iterator>, +{ + fn collect_columns(self) -> Result, ColumnsNotFoundError> { + let mut columns = Vec::new(); + let mut not_found = Vec::new(); + for result in self { + match result { + Ok(col) => columns.push(col), + Err(e) => not_found.push(e.0), + } + } + if not_found.is_empty() { + Ok(columns) + } else { + Err(ColumnsNotFoundError(not_found)) + } + } +} + +pub type TableResult = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum UpgradeError { + #[error("Failed to upgrade type from {old} to {new}")] + TypeUpgradeError { + old: &'static str, + new: &'static str, + }, + #[error("Failed to upgrade primary from {old} to {new}")] + PrimaryUpgradeError { + old: &'static str, + new: &'static str, + }, + #[error(transparent)] + TypeCreationError(#[from] TypeError), + #[error("Array length cannot be decreased from {old} to {new}")] + ArrayLengthDecreaseError { old: u32, new: u32 }, + #[error("Cannot reduce element in tuple")] + TupleReductionError, +} + +pub type UpgradeResult = Result; + +impl UpgradeError { + pub fn type_upgrade_err(old: &TypeDef, new: &TypeDef) -> UpgradeResult { + Err(Self::TypeUpgradeError { + old: old.item_name(), + new: new.item_name(), + }) + } + pub fn type_upgrade_to_err(old: &TypeDef, new: &'static str) -> UpgradeResult { + Err(Self::TypeUpgradeError { + old: old.item_name(), + new, + }) + } + pub fn type_cast_err(old: &TypeDef, new: &'static str) -> UpgradeResult { + Err(Self::TypeUpgradeError { + old: old.item_name(), + new, + }) + } + pub fn array_shorten_err(old: u32, new: u32) -> UpgradeResult { + Err(Self::ArrayLengthDecreaseError { old, new }) + } + pub fn primary_upgrade_err(old: &TypeDef, new: &PrimaryTypeDef) -> UpgradeResult { + Err(Self::PrimaryUpgradeError { + old: old.item_name(), + new: new.item_name(), + }) + } +} + +pub trait UpgradeResultExt { + fn to_table_result(self, table: &str, column: &str) -> TableResult; +} + +impl UpgradeResultExt for UpgradeResult { + fn to_table_result(self, table: &str, column: &str) -> TableResult { + self.map_err(|err| TableError::UnsupportedUpgrade { + table: table.to_string(), + column: column.to_string(), + reason: err, + }) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum RecordError { + #[error(transparent)] + JsonError(#[from] serde_json::Error), + #[error(transparent)] + TypeError(#[from] TypeError), + #[error("Record does not match table schema")] + SchemaMismatch, + #[error(transparent)] + DecodeError(#[from] DecodeError), + #[error(transparent)] + SqlxError(#[from] SqlxError), + #[error("Columns with ids: {0} not found")] + ColumnsNotFound(#[from] ColumnsNotFoundError), +} + +pub type RecordResult = std::result::Result; + +pub trait RecordResultExt { + fn to_db_result(self, table: &str) -> DbResult; +} + +impl RecordResultExt for RecordResult { + fn to_db_result(self, table: &str) -> DbResult { + self.map_err(|err| DbError::RecordError(table.to_string(), err)) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum DbError { + #[error(transparent)] + DatabaseError(#[from] SqlxError), + #[error("Invalid event format: {0}")] + InvalidEventFormat(String), + #[error(transparent)] + JsonError(#[from] serde_json::Error), + #[error(transparent)] + IoError(#[from] std::io::Error), + #[error(transparent)] + TableError(#[from] TableError), + #[error(transparent)] + TypeError(#[from] TypeError), + #[error("Table with id: {0} already exists, incoming name: {1}, existing name: {2}")] + TableAlreadyExists(TableKey, String, String), + #[error("Table not found with id: {0}")] + TableNotFound(TableKey), + #[error("Table not alive - id: {0}, name: {1}")] + TableNotAlive(Felt, String), + #[error("Manager does not support updating")] + UpdateNotSupported, + #[error("Table poison error: {0}")] + PoisonError(String), + #[error("Namespace not found for address: {0:#063x}")] + NamespaceNotFound(Felt), + #[error("Could not parse record for table {0}: {1}")] + RecordError(String, RecordError), + #[error("Failed to pass string to felt")] + FeltFromStrError(#[from] FromStrError), +} + +pub type DbResult = std::result::Result; + +impl From> for DbError { + fn from(err: PoisonError) -> Self { + Self::PoisonError(err.to_string()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum NamespaceError { + #[error("Invalid address length for address: {0} should be 63 characters long")] + InvalidAddressLength(String), + #[error(transparent)] + AddressFromStrError(#[from] FromStrError), + #[error("Namespace {0} does not match expected namespace {1}")] + NamespaceMismatch(String, String), + #[error("Namespace {1} not found for address: {0:#063x}")] + AddressNotFound(Felt, String), +} + +pub type NamespaceResult = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum TableLoadError { + #[error(transparent)] + NamespaceError(#[from] NamespaceError), + #[error("Table {0} {1:#063x} not found for column {2} with id: {3:#063x}")] + ColumnTableNotFound(String, Felt, String, Felt), + #[error("Table {0} {1:#063x} not found for dead field {2} with id: {3}")] + TableDeadNotFound(String, Felt, String, u128), +} diff --git a/crates/introspect-sql-sink/src/lib.rs b/crates/introspect-sql-sink/src/lib.rs new file mode 100644 index 00000000..249898a0 --- /dev/null +++ b/crates/introspect-sql-sink/src/lib.rs @@ -0,0 +1,31 @@ +pub mod backend; +pub mod error; +pub mod namespace; +pub mod processor; +pub mod sink; +pub mod table; +pub mod tables; + +pub use backend::{IntrospectInitialize, IntrospectProcessor, IntrospectQueryMaker}; +pub use error::{ + DbError, DbResult, RecordError, RecordResult, TableError, TableResult, TypeError, TypeResult, + UpgradeError, UpgradeResult, UpgradeResultExt, +}; +pub use namespace::{NamespaceKey, NamespaceMode, TableKey}; +pub use processor::{DbColumn, DbDeadField, DbTable, IntrospectDb}; +pub use sink::IntrospectSqlSink; +pub use table::{DeadField, DeadFieldDef, Table}; + +#[cfg(feature = "postgres")] +pub mod postgres; +#[cfg(feature = "postgres")] +pub use postgres::IntrospectPgDb; + +#[cfg(feature = "sqlite")] +pub mod sqlite; +#[cfg(feature = "sqlite")] +pub use sqlite::IntrospectSqliteDb; + +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub mod runtime; diff --git a/crates/introspect-sql-sink/src/namespace.rs b/crates/introspect-sql-sink/src/namespace.rs new file mode 100644 index 00000000..695a3049 --- /dev/null +++ b/crates/introspect-sql-sink/src/namespace.rs @@ -0,0 +1,206 @@ +use crate::error::{NamespaceError, NamespaceResult}; +use crate::{DbError, DbResult}; +use introspect_types::ResultInto; +use itertools::Itertools; +use starknet_types_core::felt::Felt; +use std::collections::{HashMap, HashSet}; +use std::fmt::Display; +use std::hash::{Hash, Hasher}; +use std::sync::Arc; + +pub enum NamespaceMode { + None, + Single(Arc), + Address, + Named(HashMap>), + Addresses(HashSet), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TableKey { + namespace: NamespaceKey, + id: Felt, +} + +impl TableKey { + pub fn new(namespace: NamespaceKey, id: Felt) -> Self { + Self { namespace, id } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NamespaceKey { + None, + Single(Arc), + Address(Felt), + Named(Arc), +} + +pub fn felt_to_namespace(address: &Felt) -> String { + format!("{address:063x}") +} + +impl Hash for TableKey { + fn hash(&self, state: &mut H) { + self.namespace.hash(state); + self.id.hash(state); + } +} + +impl Hash for NamespaceKey { + fn hash(&self, state: &mut H) { + match self { + NamespaceKey::Address(addr) => addr.hash(state), + NamespaceKey::Named(name) => name.hash(state), + NamespaceKey::Single(_) => {} + NamespaceKey::None => {} + } + } +} + +impl Display for NamespaceKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NamespaceKey::Address(addr) => write!(f, "{addr:063x}"), + NamespaceKey::Named(name) | NamespaceKey::Single(name) => name.fmt(f), + NamespaceKey::None => Ok(()), + } + } +} + +impl Display for TableKey { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !matches!(self.namespace, NamespaceKey::Single(_) | NamespaceKey::None) { + write!(f, "{} ", self.namespace)?; + } + write!(f, "{:#063x}", self.id) + } +} + +impl From for NamespaceMode { + fn from(value: String) -> Self { + NamespaceMode::Single(value.into()) + } +} + +impl From<&str> for NamespaceMode { + fn from(value: &str) -> Self { + NamespaceMode::Single(value.into()) + } +} + +impl From> for NamespaceMode { + fn from(value: HashMap) -> Self { + NamespaceMode::Named( + value + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect::>(), + ) + } +} + +impl From<[(Felt, &str); N]> for NamespaceMode { + fn from(value: [(Felt, &str); N]) -> Self { + NamespaceMode::Named( + value + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect::>(), + ) + } +} + +impl From<[(Felt, String); N]> for NamespaceMode { + fn from(value: [(Felt, String); N]) -> Self { + NamespaceMode::Named( + value + .into_iter() + .map(|(k, v)| (k, v.into())) + .collect::>(), + ) + } +} + +impl From<[Felt; N]> for NamespaceMode { + fn from(value: [Felt; N]) -> Self { + NamespaceMode::Addresses(value.into_iter().collect()) + } +} + +impl From> for NamespaceMode { + fn from(value: Vec) -> Self { + NamespaceMode::Addresses(value.into_iter().collect()) + } +} + +fn felt_try_from_namespace(namespace: &str) -> NamespaceResult { + match namespace.len() == 63 { + true => Felt::from_hex(namespace).err_into(), + false => Err(NamespaceError::InvalidAddressLength(namespace.to_string())), + } +} + +impl NamespaceMode { + pub fn namespaces(&self) -> Option> { + match self { + NamespaceMode::None => Some(vec!["".to_string()]), + NamespaceMode::Single(name) => Some(vec![name.to_string()]), + NamespaceMode::Address => None, + NamespaceMode::Named(map) => { + Some(map.values().unique().map(ToString::to_string).collect()) + } + NamespaceMode::Addresses(set) => Some(set.iter().map(felt_to_namespace).collect()), + } + } + + pub fn get_namespace_key( + &self, + namespace: String, + owner: &Felt, + ) -> NamespaceResult { + match self { + NamespaceMode::None => Ok(NamespaceKey::None), + NamespaceMode::Single(s) => match **s == *namespace { + true => Ok(NamespaceKey::Single(s.clone())), + false => Err(NamespaceError::NamespaceMismatch(namespace, s.to_string())), + }, + NamespaceMode::Address => { + felt_try_from_namespace(&namespace).map(NamespaceKey::Address) + } + NamespaceMode::Named(map) => match map.get(owner) { + Some(s) if **s == *namespace => Ok(NamespaceKey::Named(s.clone())), + Some(s) => Err(NamespaceError::NamespaceMismatch(namespace, s.to_string())), + None => Err(NamespaceError::AddressNotFound(*owner, namespace)), + }, + NamespaceMode::Addresses(set) => { + let address = felt_try_from_namespace(&namespace)?; + match set.contains(&address) { + true => Ok(NamespaceKey::Address(address)), + false => Err(NamespaceError::AddressNotFound(address, namespace)), + } + } + } + } + + pub fn get_key(&self, namespace: String, id: Felt, owner: &Felt) -> NamespaceResult { + self.get_namespace_key(namespace, owner) + .map(|k| TableKey::new(k, id)) + } + + pub fn to_namespace(&self, from_address: &Felt) -> DbResult { + match self { + NamespaceMode::None => Ok(NamespaceKey::None), + NamespaceMode::Single(name) => Ok(NamespaceKey::Single(name.clone())), + NamespaceMode::Address => Ok(NamespaceKey::Address(*from_address)), + NamespaceMode::Named(map) => match map.get(from_address) { + Some(namespace) => Ok(NamespaceKey::Named(namespace.clone())), + None => Err(DbError::NamespaceNotFound(*from_address)), + }, + NamespaceMode::Addresses(set) => match set.contains(from_address) { + true => Ok(NamespaceKey::Address(*from_address)), + false => Err(DbError::NamespaceNotFound(*from_address)), + }, + } + } +} diff --git a/crates/introspect-sql-sink/src/postgres/append_only.rs b/crates/introspect-sql-sink/src/postgres/append_only.rs new file mode 100644 index 00000000..9d505c64 --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/append_only.rs @@ -0,0 +1,106 @@ +use super::json::PostgresJsonSerializer; +use crate::postgres::insert::pg_json_felt252; +use crate::{RecordResult, Table}; +use introspect_types::ColumnInfo; +use serde::ser::SerializeMap; +use serde_json::Serializer as JsonSerializer; +use starknet_types_core::felt::Felt; +use std::io::Write; +use torii_introspect::tables::SerializeEntries; +use torii_introspect::Record; +use torii_sql::postgres::PgQuery; +use torii_sql::Queries; + +struct MetaData<'a> { + pub block_number: u64, + pub transaction_hash: &'a Felt, +} + +impl<'a> MetaData<'a> { + pub fn new(block_number: u64, transaction_hash: &'a Felt) -> Self { + Self { + block_number, + transaction_hash, + } + } +} + +impl SerializeEntries for MetaData<'_> { + fn entry_count(&self) -> usize { + 2 + } + fn serialize_entries( + &self, + map: &mut ::SerializeMap, + ) -> Result<(), S::Error> { + let tx_hash = pg_json_felt252(self.transaction_hash); + map.serialize_entry("__created_block", &self.block_number)?; + map.serialize_entry("__created_tx", &tx_hash) + } +} + +#[allow(clippy::too_many_arguments)] +pub fn append_only_record_queries( + table: &Table, + column_ids: &[Felt], + records: &[Record], + _from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let schema = table.get_record_schema(column_ids)?; + let namespace = &table.namespace; + let table_name = &table.name; + let mut writer = Vec::new(); + let metadata = MetaData::new(block_number, transaction_hash); + let primary = schema.primary_name(); + let columns: Vec<(&Felt, &ColumnInfo)> = table.columns.iter().collect(); + + write!( + writer, + r#"WITH latest AS (SELECT DISTINCT ON ("{primary}") * FROM "{namespace}"."{table_name}" ORDER BY "{primary}", "__revision" DESC), + input AS ( SELECT * FROM jsonb_populate_recordset(NULL::"{namespace}"."{table_name}", $$"# + ) + .unwrap(); + + schema.parse_records_with_metadata( + records, + &metadata, + &mut JsonSerializer::new(&mut writer), + &PostgresJsonSerializer, + )?; + + write!( + writer, + r#"$$)) INSERT INTO "{namespace}"."{table_name}" ("{primary}", "__revision", "__created_block", "__created_tx""# + ) + .unwrap(); + + for (_, ColumnInfo { name, .. }) in &columns { + write!(writer, r#", "{name}""#).unwrap(); + } + + write!( + writer, + r#") SELECT i."{primary}", (SELECT COALESCE(MAX("__revision"), 0) + 1 FROM "{namespace}"."{table_name}" WHERE "{primary}" = i."{primary}") AS "__revision", i."__created_block", i."__created_tx""#, + ).unwrap(); + + for (id, ColumnInfo { name, .. }) in &columns { + if column_ids.contains(id) { + write!(writer, r#", i."{name}""#).unwrap(); + } else { + write!(writer, r#", COALESCE(i."{name}", l."{name}")"#).unwrap(); + } + } + + write!( + writer, + r#" FROM input i LEFT JOIN latest l USING ("{primary}")"# + ) + .unwrap(); + + let string = unsafe { String::from_utf8_unchecked(writer) }; + queries.add(string); + Ok(()) +} diff --git a/crates/introspect-sql-sink/src/postgres/backend.rs b/crates/introspect-sql-sink/src/postgres/backend.rs new file mode 100644 index 00000000..31a5c5dd --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/backend.rs @@ -0,0 +1,147 @@ +use super::insert::insert_record_queries; +use super::query::{insert_columns_query, insert_table_query, CreatePgTable}; +use super::upgrade::PgTableUpgrade; +use crate::postgres::append_only::append_only_record_queries; +use crate::processor::IntrospectDb; +use crate::{ + IntrospectQueryMaker, IntrospectSqlSink, RecordResult, Table, TableError, TableResult, +}; +use async_trait::async_trait; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::{ColumnDef, FeltIds, PrimaryDef}; +use starknet_types_core::felt::Felt; +use torii_introspect::Record; +use torii_sql::{PgPool, PgQuery, Postgres}; + +pub type IntrospectPgDb = IntrospectDb; + +#[async_trait] +impl IntrospectQueryMaker for Postgres { + fn create_table_queries( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + let ns = namespace.into(); + CreatePgTable::new(&ns, id, name, primary, columns, append_only)?.make_queries(queries); + store_table_queries( + namespace, + id, + name, + primary, + append_only, + columns, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn update_table_queries( + table: &mut Table, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + let upgrades = table.upgrade_table(name, primary, columns)?; + upgrades.to_queries(block_number, transaction_hash, queries)?; + let columns: Vec<(&Felt, &introspect_types::ColumnInfo)> = table + .columns_with_ids(&upgrades.columns_upgraded) + .map_err(|e| e.to_table_error(&table.name))?; + store_table_queries( + &table.namespace, + &table.id, + name, + primary, + table.append_only, + columns, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> RecordResult<()> { + if table.append_only { + append_only_record_queries( + table, + columns, + records, + from_address, + block_number, + transaction_hash, + queries, + ) + } else { + insert_record_queries( + table, + columns, + records, + from_address, + block_number, + transaction_hash, + queries, + ) + } + } +} + +#[allow(clippy::too_many_arguments)] +fn store_table_queries( + schema: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + append_only: bool, + columns: CS, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, +) -> TableResult<()> +where + CS: Names + FeltIds + TypeDefs, +{ + queries.push( + insert_table_query( + schema, + id, + name, + primary, + from_address, + append_only, + block_number, + transaction_hash, + ) + .map_err(TableError::Encode)?, + ); + + queries.push( + insert_columns_query(schema, id, columns, block_number, transaction_hash) + .map_err(TableError::Encode)?, + ); + Ok(()) +} + +impl IntrospectSqlSink for PgPool { + const NAME: &'static str = "Introspect Postgres"; +} diff --git a/crates/introspect-postgres-sink/src/create.rs b/crates/introspect-sql-sink/src/postgres/create.rs similarity index 84% rename from crates/introspect-postgres-sink/src/create.rs rename to crates/introspect-sql-sink/src/postgres/create.rs index 41bf4d85..34df03c8 100644 --- a/crates/introspect-postgres-sink/src/create.rs +++ b/crates/introspect-sql-sink/src/postgres/create.rs @@ -1,9 +1,7 @@ -use crate::{ - query::{CreatePgTable, CreatesType}, - utils::{AsBytes, HasherExt}, - PgSchema, PgTypeError, PgTypeResult, PostgresField, PostgresScalar, PostgresType, PrimaryKey, - SchemaName, -}; +use super::query::{make_schema_query, CreatePgTable, CreatesType}; +use super::utils::{AsBytes, HasherExt}; +use super::{PostgresField, PostgresScalar, PostgresType, PrimaryKey, SchemaName}; +use crate::{TypeError, TypeResult}; use introspect_types::{ ArrayDef, ColumnDef, EnumDef, FixedArrayDef, MemberDef, OptionDef, PrimaryDef, PrimaryTypeDef, StructDef, TupleDef, TypeDef, VariantDef, @@ -11,17 +9,17 @@ use introspect_types::{ use itertools::Itertools; use starknet_types_core::felt::Felt; use std::rc::Rc; -use torii_common::sql::{PgQuery, Queries}; -use torii_introspect::schema::TableInfo; +use torii_sql::postgres::PgQuery; +use torii_sql::Queries; use xxhash_rust::xxh3::Xxh3; pub trait PostgresTypeExtractor { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult; + ) -> TypeResult; } pub trait PostgresFieldExtractor { @@ -34,10 +32,10 @@ pub trait PostgresFieldExtractor { } fn extract_field( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { Ok(PostgresField::new( self.name().to_string(), self.type_def() @@ -54,7 +52,7 @@ impl PostgresField { } } - pub fn new_composite(name: S, schema: &Rc, type_name: T) -> Self + pub fn new_composite(name: S, schema: &Rc, type_name: T) -> Self where S: Into, T: Into, @@ -111,10 +109,10 @@ impl PostgresFieldExtractor for (&Felt, &VariantDef) { impl PostgresTypeExtractor for TypeDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { match self { TypeDef::None => Ok(PostgresScalar::None.into()), TypeDef::Bool => Ok(PostgresScalar::Boolean.into()), @@ -149,7 +147,7 @@ impl PostgresTypeExtractor for TypeDef { TypeDef::Option(def) => def.type_def.extract_type(schema, branch, creates), TypeDef::Nullable(def) => def.type_def.extract_type(schema, branch, creates), TypeDef::Felt252Dict(_) | TypeDef::Result(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { - Err(PgTypeError::UnsupportedType(format!("{self:?}"))) + Err(TypeError::UnsupportedType(format!("{self:?}"))) } } } @@ -187,10 +185,10 @@ impl From<&PrimaryDef> for PrimaryKey { impl PostgresTypeExtractor for ArrayDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { self.type_def .extract_type(schema, branch, creates)? .to_array(None) @@ -200,10 +198,10 @@ impl PostgresTypeExtractor for ArrayDef { impl PostgresTypeExtractor for FixedArrayDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { self.type_def .extract_type(schema, branch, creates)? .to_array(Some(self.size)) @@ -213,15 +211,15 @@ impl PostgresTypeExtractor for FixedArrayDef { impl PostgresTypeExtractor for StructDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { let members = self .members .iter() .map(|f| f.extract_field(schema, branch, creates)) - .collect::>>()?; + .collect::>>()?; let name = branch.type_name(&self.name); creates.push(CreatesType::new_struct(schema, &name, members).into()); Ok(PostgresType::composite(schema, name)) @@ -231,10 +229,10 @@ impl PostgresTypeExtractor for StructDef { impl PostgresTypeExtractor for EnumDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { let name = branch.type_name(&self.name); let variants_type = branch.type_name(&format!("v_{}", self.name)); let variant_names = self.variants.values().map(|v| v.name.clone()).collect_vec(); @@ -259,10 +257,10 @@ impl PostgresTypeExtractor for EnumDef { impl PostgresTypeExtractor for TupleDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { let mut variants = Vec::with_capacity(self.elements.len()); for (i, element) in self.elements.iter().enumerate() { variants.push( @@ -280,44 +278,51 @@ impl PostgresTypeExtractor for TupleDef { impl PostgresTypeExtractor for OptionDef { fn extract_type( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, creates: &mut Vec>, - ) -> PgTypeResult { + ) -> TypeResult { self.type_def.extract_type(schema, branch, creates) } } impl CreatePgTable { - pub fn new(schema: &Rc, id: &Felt, table: &TableInfo) -> PgTypeResult { - let TableInfo { - name, - attributes: _, - primary, - columns, - } = table; - let mut creates = Vec::new(); + pub fn new( + schema: &Rc, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + ) -> TypeResult { + let mut creates: Vec = Vec::new(); let branch = Xxh3::new_based(id); let primary = primary.into(); let columns = columns .iter() .map(|col| col.extract_field(schema, &branch, &mut creates)) - .collect::>>()?; + .collect::>>()?; Ok(Self { name: SchemaName::new(schema, name), primary, columns, pg_types: creates, + append_only, }) } pub fn make_queries(&self, queries: &mut Vec) { + if !self.name.1.is_empty() { + queries.add(make_schema_query(&self.name.0)); + } for pg_type in &self.pg_types { queries.add(pg_type.to_string()); } queries.add(self.to_string()); - queries.add(format!( - r#"CREATE TRIGGER set_timestamps BEFORE INSERT ON {} FOR EACH ROW EXECUTE FUNCTION introspect.set_default_timestamps();"#, - self.name - )); + if !self.append_only { + queries.add(format!( + r#"CREATE TRIGGER set_timestamps BEFORE INSERT ON {} FOR EACH ROW EXECUTE FUNCTION introspect.set_default_timestamps();"#, + self.name + )); + } } } diff --git a/crates/introspect-sql-sink/src/postgres/handler.rs b/crates/introspect-sql-sink/src/postgres/handler.rs new file mode 100644 index 00000000..c445a50c --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/handler.rs @@ -0,0 +1,28 @@ +use super::query::{fetch_columns, fetch_dead_fields, fetch_tables}; +use crate::backend::IntrospectInitialize; +use crate::{DbColumn, DbDeadField, DbResult, DbTable}; +use async_trait::async_trait; +use introspect_types::ResultInto; +use sqlx::PgPool; +use torii_sql::PoolExt; + +pub const INTROSPECT_PG_SINK_MIGRATIONS: sqlx::migrate::Migrator = + sqlx::migrate!("./migrations/postgres"); + +#[async_trait] +impl IntrospectInitialize for PgPool { + async fn load_tables(&self, schemas: &Option>) -> DbResult> { + fetch_tables(self.pool(), schemas).await.err_into() + } + async fn load_columns(&self, schemas: &Option>) -> DbResult> { + fetch_columns(self.pool(), schemas).await.err_into() + } + async fn load_dead_fields(&self, schemas: &Option>) -> DbResult> { + fetch_dead_fields(self.pool(), schemas).await.err_into() + } + async fn initialize(&self) -> DbResult<()> { + self.migrate(Some("introspect"), INTROSPECT_PG_SINK_MIGRATIONS) + .await + .err_into() + } +} diff --git a/crates/introspect-sql-sink/src/postgres/insert.rs b/crates/introspect-sql-sink/src/postgres/insert.rs new file mode 100644 index 00000000..a49fe6f4 --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/insert.rs @@ -0,0 +1,92 @@ +use super::json::PostgresJsonSerializer; +use crate::{RecordResult, Table}; +use introspect_types::ColumnInfo; +use serde::ser::SerializeMap; +use serde_json::Serializer as JsonSerializer; +use starknet_types_core::felt::Felt; +use std::io::Write; +use torii_introspect::tables::SerializeEntries; +use torii_introspect::Record; +use torii_sql::postgres::PgQuery; +use torii_sql::Queries; + +pub const METADATA_CONFLICTS: &str = "__updated_at = NOW(), __updated_block = EXCLUDED.__updated_block, __updated_tx = EXCLUDED.__updated_tx"; + +struct MetaData<'a> { + pub block_number: u64, + pub transaction_hash: &'a Felt, +} + +impl<'a> MetaData<'a> { + pub fn new(block_number: u64, transaction_hash: &'a Felt) -> Self { + Self { + block_number, + transaction_hash, + } + } +} + +pub fn pg_json_felt252(value: &Felt) -> String { + format!("\\x{}", hex::encode(value.to_bytes_be())) +} + +impl SerializeEntries for MetaData<'_> { + fn entry_count(&self) -> usize { + 4 + } + fn serialize_entries( + &self, + map: &mut ::SerializeMap, + ) -> Result<(), S::Error> { + let tx_hash = pg_json_felt252(self.transaction_hash); + map.serialize_entry("__created_block", &self.block_number)?; + map.serialize_entry("__updated_block", &self.block_number)?; + map.serialize_entry("__created_tx", &tx_hash)?; + map.serialize_entry("__updated_tx", &tx_hash) + } +} + +#[allow(clippy::too_many_arguments)] +pub fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + _from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let schema = table.get_record_schema(columns)?; + let namespace = &table.namespace; + let table_name = &table.name; + let mut writer = Vec::new(); + let metadata = MetaData::new(block_number, transaction_hash); + write!( + writer, + r#"INSERT INTO "{namespace}"."{table_name}" SELECT * FROM jsonb_populate_recordset(NULL::"{namespace}"."{table_name}", $$"# + ) + .unwrap(); + schema.parse_records_with_metadata( + records, + &metadata, + &mut JsonSerializer::new(&mut writer), + &PostgresJsonSerializer, + )?; + write!( + writer, + r#"$$) ON CONFLICT ("{}") DO UPDATE SET {METADATA_CONFLICTS}"#, + schema.primary().name + ) + .unwrap(); + for ColumnInfo { name, .. } in schema.columns() { + write!( + writer, + r#", "{name}" = COALESCE(EXCLUDED."{name}", "{table_name}"."{name}")"#, + name = name + ) + .unwrap(); + } + let string = unsafe { String::from_utf8_unchecked(writer) }; + queries.add(string); + Ok(()) +} diff --git a/crates/introspect-postgres-sink/src/json.rs b/crates/introspect-sql-sink/src/postgres/json.rs similarity index 64% rename from crates/introspect-postgres-sink/src/json.rs rename to crates/introspect-sql-sink/src/postgres/json.rs index 7e8cc911..02d78188 100644 --- a/crates/introspect-postgres-sink/src/json.rs +++ b/crates/introspect-sql-sink/src/postgres/json.rs @@ -1,8 +1,9 @@ use introspect_types::serialize::ToCairoDeSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; -use introspect_types::{CairoDeserializer, ResultDef, TupleDef, TypeDef}; -use serde::ser::SerializeMap; +use introspect_types::{CairoDeserializer, EnumDef, ResultDef, TupleDef, TypeDef, VariantDef}; +use serde::ser::{Error as SerError, SerializeMap}; use serde::Serializer; +use starknet_types_core::felt::Felt; pub struct PostgresJsonSerializer; @@ -48,27 +49,40 @@ impl CairoTypeSerialization for PostgresJsonSerializer { seq.end() } - fn serialize_variant<'a, S: Serializer>( + fn serialize_enum<'a, S: Serializer>( &'a self, data: &mut impl CairoDeserializer, serializer: S, - name: &str, - type_def: &'a TypeDef, + enum_def: &'a EnumDef, + variant: Felt, ) -> Result { - match type_def { - TypeDef::None => { - let mut map = serializer.serialize_map(Some(1))?; - map.serialize_entry("_variant", name)?; - map - } - _ => { - let mut map = serializer.serialize_map(Some(2))?; - map.serialize_entry("_variant", name)?; - map.serialize_entry(name, &type_def.to_de_se(data, self))?; - map + let VariantDef { name, type_def, .. } = + enum_def.get_variant(&variant).map_err(S::Error::custom)?; + let mut map = serializer.serialize_map(None)?; + map.serialize_entry("_variant", name)?; + if type_def != &TypeDef::None { + map.serialize_entry(name, &type_def.to_de_se(data, self))?; + } + for v in &enum_def.order { + if v != &variant { + let VariantDef { name, type_def, .. } = + enum_def.get_variant(v).map_err(S::Error::custom)?; + if type_def != &TypeDef::None { + map.serialize_entry(name, &())?; + } } } - .end() + map.end() + } + + fn serialize_variant<'a, S: Serializer>( + &'a self, + _data: &mut impl CairoDeserializer, + _serializer: S, + _name: &str, + _type_def: &'a TypeDef, + ) -> Result { + unimplemented!("variant serialization is only supported within enums, and should not be called directly") } fn serialize_result<'a, S: Serializer>( diff --git a/crates/introspect-sql-sink/src/postgres/mod.rs b/crates/introspect-sql-sink/src/postgres/mod.rs new file mode 100644 index 00000000..5bdfab78 --- /dev/null +++ b/crates/introspect-sql-sink/src/postgres/mod.rs @@ -0,0 +1,16 @@ +pub mod append_only; +pub mod backend; +pub mod create; +pub mod handler; +pub mod insert; +pub mod json; +pub mod query; +pub mod types; +pub mod upgrade; +pub mod utils; + +pub use backend::IntrospectPgDb; +pub use types::{ + PostgresArray, PostgresField, PostgresScalar, PostgresType, PrimaryKey, SchemaName, +}; +pub use utils::{truncate, HasherExt}; diff --git a/crates/introspect-postgres-sink/src/query.rs b/crates/introspect-sql-sink/src/postgres/query.rs similarity index 74% rename from crates/introspect-postgres-sink/src/query.rs rename to crates/introspect-sql-sink/src/postgres/query.rs index 63b96983..9a1adc1b 100644 --- a/crates/introspect-postgres-sink/src/query.rs +++ b/crates/introspect-sql-sink/src/postgres/query.rs @@ -1,59 +1,62 @@ -use introspect_types::{ColumnDef, ColumnInfo, MemberDef, PrimaryDef, TypeDef}; +use super::{PostgresField, PostgresType, PrimaryKey, SchemaName}; +use crate::{DbColumn, DbDeadField, DbTable, DeadFieldDef, TableError, TableResult}; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::{ColumnDef, FeltIds, MemberDef, PrimaryDef, TypeDef}; use itertools::Itertools; use sqlx::error::BoxDynError; +use sqlx::postgres::{PgArguments, PgRow}; use sqlx::prelude::FromRow; -use sqlx::Error::Encode as EncodeError; -use sqlx::{postgres::PgArguments, types::Json}; +use sqlx::query::QueryAs; +use sqlx::types::Json; use sqlx::{Arguments, Executor, Postgres}; use starknet_types_core::felt::Felt; -use torii_common::sql::{PgQuery, Queries, SqlxResult}; +use std::collections::HashMap; +use std::fmt::{Display, Formatter, Result as FmtResult, Write}; +use std::rc::Rc; use torii_introspect::postgres::types::{PgPrimary, Uint128}; use torii_introspect::postgres::PgFelt; +use torii_sql::postgres::PgQuery; +use torii_sql::{Queries, SqlxResult}; -use crate::table::PgTable; -use crate::{ - processor::COMMIT_CMD, table::DeadField, PgSchema, PostgresField, PostgresType, PrimaryKey, - SchemaName, -}; -use std::collections::HashMap; -use std::{ - fmt::{Display, Formatter, Result as FmtResult, Write}, - rc::Rc, -}; +pub const COMMIT_CMD: &str = "--COMMIT"; -const CREATE_METADATA_COLUMNS: &str = "__created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __created_block public.uint64 NOT NULL, __updated_block public.uint64 NOT NULL, __created_tx public.felt252 NOT NULL, __updated_tx public.felt252 NOT NULL);"; +const CREATE_METADATA_COLUMNS: &str = "__created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __created_block public.uint64 NOT NULL, __updated_block public.uint64 NOT NULL, __created_tx public.felt252 NOT NULL, __updated_tx public.felt252 NOT NULL"; +const CREATE_APPEND_ONLY_METADATA_COLUMNS: &str = "__created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), __created_block public.uint64 NOT NULL, __created_tx public.felt252 NOT NULL"; +const APPEND_ONLY_REVISION_COLUMN: &str = r#""__revision" bigint NOT NULL, "#; +const INSERT_TABLE_QUERY: &str = r#"INSERT INTO introspect.db_tables + ("schema", id, owner, name, primary_def, append_only, updated_at, created_block, updated_block, created_tx, updated_tx) + VALUES ($1, $2, $3, $4, $5, $6, NOW(), $7::uint64, $7::uint64, $8, $8) + ON CONFLICT ("schema", id) DO UPDATE SET + name = EXCLUDED.name, append_only = EXCLUDED.append_only, primary_def = EXCLUDED.primary_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; const INSERT_DEAD_MEMBER_QUERY: &str = r#"INSERT INTO introspect.db_dead_fields ("schema", "table", id, name, type_def, updated_at, created_block, updated_block, created_tx, updated_tx) SELECT $1, $2, unnest($3::bigint[]), unnest($4::text[]), unnest($5::jsonb[]), NOW(), $6::uint64, $6::uint64, $7, $7 ON CONFLICT ("schema", "table", id) DO UPDATE SET name = EXCLUDED.name, type_def = EXCLUDED.type_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; -const INSERT_TABLE_QUERY: &str = r#"INSERT INTO introspect.db_tables - ("schema", id, name, primary_def, updated_at, created_block, updated_block, created_tx, updated_tx) - VALUES ($1, $2, $3, $4, NOW(), $5::uint64, $5::uint64, $6, $6) - ON CONFLICT ("schema", id) DO UPDATE SET - name = EXCLUDED.name, primary_def = EXCLUDED.primary_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; + const INSERT_COLUMN_QUERY: &str = r#"INSERT INTO introspect.db_columns ("schema", "table", id, name, type_def, updated_at, created_block, updated_block, created_tx, updated_tx) SELECT $1, $2, unnest($3::felt252[]), unnest($4::text[]), unnest($5::jsonb[]), NOW(), $6::uint64, $6::uint64, $7, $7 ON CONFLICT ("schema", "table", id) DO UPDATE SET name = EXCLUDED.name, type_def = EXCLUDED.type_def, updated_at = NOW(), updated_block = EXCLUDED.updated_block, updated_tx = EXCLUDED.updated_tx"#; -const FETCH_TABLES_QUERY: &str = - r#"SELECT id, name, primary_def FROM introspect.db_tables WHERE "schema" = $1"#; -const FETCH_COLUMNS_QUERY: &str = - r#"SELECT "table", id, name, type_def FROM introspect.db_columns WHERE "schema" = $1"#; -const FETCH_DEAD_FIELDS_QUERY: &str = - r#"SELECT "table", id, name, type_def FROM introspect.db_dead_fields WHERE "schema" = $1"#; +const FETCH_TABLES_QUERY: &str = r#"SELECT "schema", id, name, primary_def, owner, append_only FROM introspect.db_tables WHERE $1::text[] = '{}'::text[] OR "schema" = ANY($1::text[])"#; +const FETCH_COLUMNS_QUERY: &str = r#"SELECT "schema", "table", id, name, type_def FROM introspect.db_columns WHERE $1::text[] = '{}'::text[] OR "schema" = ANY($1::text[])"#; +const FETCH_DEAD_FIELDS_QUERY: &str = r#"SELECT "schema", "table", id, name, type_def FROM introspect.db_dead_fields WHERE $1::text[] = '{}'::text[] OR "schema" = ANY($1::text[])"#; #[derive(FromRow)] pub struct TableRow { + pub schema: String, pub id: PgFelt, pub name: String, pub primary_def: PgPrimary, + pub owner: PgFelt, + pub append_only: bool, } #[derive(FromRow)] pub struct ColumnRow { + pub schema: String, pub table: PgFelt, pub id: PgFelt, pub name: String, @@ -62,36 +65,33 @@ pub struct ColumnRow { #[derive(FromRow)] pub struct DeadFieldRow { + pub schema: String, pub table: PgFelt, pub id: Uint128, pub name: String, pub type_def: Json, } -pub struct PgTableHead { - id: Felt, - name: String, - primary: PrimaryDef, -} - #[derive(Debug)] pub struct CreatePgTable { pub name: SchemaName, pub primary: PrimaryKey, pub columns: Vec, pub pg_types: Vec, + pub append_only: bool, } #[derive(Debug)] pub struct TableUpgrade { - pub schema: Rc, + pub schema: Rc, + pub id: Felt, pub name: String, pub old_name: Option, pub atomic: Vec, pub alters: Vec, pub columns: Vec, pub columns_upgraded: Vec, - pub dead: Vec, + pub dead: Vec, pub col_alters: Vec, } @@ -99,7 +99,7 @@ pub struct TableUpgrade { pub struct ColumnUpgrade { pub atomic: Vec, pub alters: Vec, - pub dead: Vec, + pub dead: Vec, pub altered: bool, pub upgraded: bool, } @@ -156,35 +156,6 @@ pub struct EnumUpgrade { add: Vec, } -#[derive(Debug)] -pub struct DeadFieldWithId { - pub id: u128, - pub name: String, - pub type_def: TypeDef, -} - -impl From for (u128, DeadField) { - fn from(value: DeadFieldWithId) -> Self { - ( - value.id, - DeadField { - name: value.name, - type_def: value.type_def, - }, - ) - } -} - -impl From<(u128, DeadField)> for DeadFieldWithId { - fn from(value: (u128, DeadField)) -> Self { - DeadFieldWithId { - id: value.0, - name: value.1.name, - type_def: value.1.type_def, - } - } -} - #[derive(Debug)] pub enum CreatesType { Struct(CreateStruct), @@ -198,10 +169,25 @@ impl Display for CreatePgTable { r#"CREATE TABLE IF NOT EXISTS {} ({}, "#, self.name, self.primary )?; + if self.append_only { + APPEND_ONLY_REVISION_COLUMN.fmt(f)?; + } for column in &self.columns { write!(f, "{column}, ")?; } - CREATE_METADATA_COLUMNS.fmt(f) + if self.append_only { + write!( + f, + r#"{CREATE_APPEND_ONLY_METADATA_COLUMNS}, PRIMARY KEY ("{}", "__revision"));"#, + self.primary.name + ) + } else { + write!( + f, + r#"{CREATE_METADATA_COLUMNS}, PRIMARY KEY ("{}"));"#, + self.primary.name + ) + } } } @@ -242,7 +228,7 @@ impl Display for CreatesType { impl CreatesType { pub fn new_struct>( - schema: &Rc, + schema: &Rc, name: S, fields: Vec, ) -> Self { @@ -252,11 +238,7 @@ impl CreatesType { }) } - pub fn new_enum>( - schema: &Rc, - name: S, - variants: Vec, - ) -> Self { + pub fn new_enum>(schema: &Rc, name: S, variants: Vec) -> Self { Self::Enum(CreateEnum { name: SchemaName::new(schema, name), variants, @@ -265,9 +247,10 @@ impl CreatesType { } impl TableUpgrade { - pub fn new>(schema: &Rc, name: S) -> Self { + pub fn new>(schema: &Rc, id: Felt, name: S) -> Self { Self { schema: schema.clone(), + id, name: name.into(), old_name: None, columns: Vec::new(), @@ -338,31 +321,31 @@ impl TableUpgrade { pub fn to_queries( &self, - table_id: &Felt, block_number: u64, transaction_hash: &Felt, queries: &mut Vec, - ) -> SqlxResult<()> { - let schema = &self.schema; - let name = &self.name; + ) -> TableResult<()> { queries.add( insert_dead_member_query( &self.schema, - table_id, + &self.id, &self.dead, block_number, transaction_hash, ) - .map_err(EncodeError)?, + .map_err(TableError::Encode)?, ); if let Some(old_name) = &self.old_name { queries.add(format!( - r#"ALTER TABLE "{schema}"."{old_name}" RENAME TO "{name}";"# + r#"ALTER TABLE {} RENAME TO "{}";"#, + SchemaName::new(&self.schema, old_name), + self.name )); } self.atomic.iter().for_each(|m| m.to_queries(queries)); + let name = SchemaName::new(&self.schema, &self.name); if let Some((last, columns)) = self.columns.split_last() { - let mut alterations = format!(r#"ALTER TABLE "{schema}"."{name}" "#); + let mut alterations = format!(r#"ALTER TABLE {name} "#); columns .iter() .for_each(|m| write!(alterations, "{m}, ").unwrap()); @@ -375,8 +358,8 @@ impl TableUpgrade { fn alter_queries(&self, queries: &mut Vec) { if let Some((last, others)) = self.col_alters.split_last() { - let (schema, name) = (&self.schema, &self.name); - let mut forward = format!(r#"ALTER TABLE "{schema}"."{name}" "#); + let table_name = SchemaName::new(&self.schema, &self.name); + let mut forward = format!(r#"ALTER TABLE {table_name} "#); let mut reverse = forward.clone(); for PostgresField { name: col, pg_type } in others { write!( @@ -496,7 +479,7 @@ impl TypeMods for Vec { impl ColumnUpgrade { pub fn maybe_alter( &mut self, - schema: &Rc, + schema: &Rc, name: &str, field: &str, pg_type: Option, @@ -513,7 +496,7 @@ impl ColumnUpgrade { pub fn add_struct_mod>( &mut self, - schema: &Rc, + schema: &Rc, name: S, mods: Vec, ) { @@ -527,7 +510,7 @@ impl ColumnUpgrade { } pub fn add_enum_mod>( &mut self, - schema: &Rc, + schema: &Rc, name: S, rename: Vec<(String, String)>, add: Vec, @@ -544,7 +527,7 @@ impl ColumnUpgrade { } pub fn add_dead_member(&mut self, id: u128, member: &MemberDef) { self.upgraded = true; - self.dead.push(DeadFieldWithId { + self.dead.push(DeadFieldDef { id, name: member.name.clone(), type_def: member.type_def.clone(), @@ -577,64 +560,50 @@ impl From for TypeMod { } } -impl From for (Felt, Felt, ColumnInfo) { +impl From for DbColumn { fn from(value: ColumnRow) -> Self { - ( - value.table.into(), - value.id.into(), - ColumnInfo { - name: value.name, - attributes: Vec::new(), - type_def: value.type_def.0, - }, - ) + DbColumn { + namespace: value.schema, + table: value.table.into(), + id: value.id.into(), + name: value.name, + type_def: value.type_def.0, + } } } -impl From for (Felt, u128, DeadField) { +impl From for DbDeadField { fn from(value: DeadFieldRow) -> Self { - ( - value.table.into(), - value.id.into(), - DeadField { - name: value.name, - type_def: value.type_def.0, - }, - ) + DbDeadField { + namespace: value.schema, + table: value.table.into(), + id: value.id.into(), + name: value.name, + type_def: value.type_def.0, + } } } -impl From for PgTableHead { +impl From for DbTable { fn from(value: TableRow) -> Self { - let row = value; - PgTableHead { - id: row.id.into(), - name: row.name, - primary: row.primary_def.into(), + DbTable { + namespace: value.schema, + id: value.id.into(), + owner: value.owner.into(), + name: value.name, + primary: value.primary_def.into(), + columns: HashMap::new(), + dead: HashMap::new(), + append_only: value.append_only, + alive: true, } } } -impl PgTableHead { - pub fn to_table(self, schema: &PgSchema) -> (Felt, PgTable) { - ( - self.id, - PgTable { - schema: schema.clone(), - name: self.name, - primary: self.primary, - columns: HashMap::new(), - alive: true, - dead: HashMap::new(), - }, - ) - } -} - fn insert_dead_member_query( - schema: &PgSchema, + schema: &str, table: &Felt, - fields: &[DeadFieldWithId], + fields: &[DeadFieldDef], block_number: u64, transaction_hash: &Felt, ) -> Result { @@ -650,55 +619,71 @@ fn insert_dead_member_query( Ok(PgQuery::new(INSERT_DEAD_MEMBER_QUERY, args)) } -pub fn insert_columns_query( - schema: &PgSchema, +pub fn insert_columns_query( + schema: &str, table: &Felt, - columns: Vec<(&Felt, &ColumnInfo)>, + columns: CS, block_number: u64, transaction_hash: &Felt, -) -> Result { +) -> Result +where + CS: Names + FeltIds + TypeDefs, +{ let mut args = PgArguments::default(); args.add(schema.to_string())?; args.add(PgFelt::from(*table))?; - args.add( - columns - .iter() - .map(|(id, _)| PgFelt::from(*id)) - .collect_vec(), - )?; - args.add(columns.iter().map(|(_, c)| c.name.clone()).collect_vec())?; - args.add(columns.iter().map(|(_, c)| Json(&c.type_def)).collect_vec())?; + args.add(columns.ids().iter().map_into::().collect_vec())?; + args.add(columns.names())?; + args.add(columns.type_defs().into_iter().map(Json).collect_vec())?; args.add(block_number.to_string())?; args.add(PgFelt::from(*transaction_hash))?; Ok(PgQuery::new(INSERT_COLUMN_QUERY, args)) } +#[allow(clippy::too_many_arguments)] pub fn insert_table_query( - schema: &PgSchema, + schema: &str, id: &Felt, name: &str, primary_def: &PrimaryDef, + from_address: &Felt, + append_only: bool, block_number: u64, transaction_hash: &Felt, ) -> Result { let mut args = PgArguments::default(); args.add(schema.to_string())?; args.add(PgFelt::from(*id))?; + args.add(PgFelt::from(*from_address))?; args.add(name.to_owned())?; args.add(PgPrimary::from(primary_def))?; + args.add(append_only)?; args.add(block_number.to_string())?; args.add(PgFelt::from(*transaction_hash))?; Ok(PgQuery::new(INSERT_TABLE_QUERY, args)) } +pub fn schema_query<'a, R>( + query: &'a str, + schemas: &'a Option>, +) -> QueryAs<'a, Postgres, R, PgArguments> +where + R: for<'r> FromRow<'r, PgRow>, +{ + let query = sqlx::query_as::<_, R>(query); + match schemas { + Some(schemas) => query.bind(schemas), + None => query.bind("{}".to_string()), + } +} + pub async fn fetch_tables<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( conn: E, - schema: &PgSchema, -) -> SqlxResult> { - sqlx::query_as::<_, TableRow>(FETCH_TABLES_QUERY) - .bind(schema.to_string()) + schemas: &Option>, +) -> SqlxResult> { + schema_query::(FETCH_TABLES_QUERY, schemas) .fetch_all(conn) .await .map(|rows| rows.into_iter().map_into().collect()) @@ -706,10 +691,9 @@ pub async fn fetch_tables<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( pub async fn fetch_columns<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( conn: E, - schema: &PgSchema, -) -> SqlxResult> { - sqlx::query_as::<_, ColumnRow>(FETCH_COLUMNS_QUERY) - .bind(schema.to_string()) + schemas: &Option>, +) -> SqlxResult> { + schema_query::(FETCH_COLUMNS_QUERY, schemas) .fetch_all(conn) .await .map(|rows| rows.into_iter().map_into().collect()) @@ -717,11 +701,14 @@ pub async fn fetch_columns<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( pub async fn fetch_dead_fields<'e, 'c, E: 'e + Executor<'c, Database = Postgres>>( conn: E, - schema: &PgSchema, -) -> SqlxResult> { - sqlx::query_as::<_, DeadFieldRow>(FETCH_DEAD_FIELDS_QUERY) - .bind(schema.to_string()) + schemas: &Option>, +) -> SqlxResult> { + schema_query::(FETCH_DEAD_FIELDS_QUERY, schemas) .fetch_all(conn) .await .map(|rows| rows.into_iter().map_into().collect()) } + +pub fn make_schema_query(schema: &str) -> String { + format!(r#"CREATE SCHEMA IF NOT EXISTS "{schema}""#) +} diff --git a/crates/introspect-postgres-sink/src/types.rs b/crates/introspect-sql-sink/src/postgres/types.rs similarity index 83% rename from crates/introspect-postgres-sink/src/types.rs rename to crates/introspect-sql-sink/src/postgres/types.rs index 7354b8b4..57f97de6 100644 --- a/crates/introspect-postgres-sink/src/types.rs +++ b/crates/introspect-sql-sink/src/postgres/types.rs @@ -1,20 +1,11 @@ +use crate::{TypeError, TypeResult}; use serde::{Deserialize, Serialize}; -use std::{ - collections::VecDeque, - fmt::{Display, Formatter, Result as FmtResult}, - rc::Rc, -}; - -use crate::{PgTypeError, PgTypeResult}; - -#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] -pub enum PgSchema { - Public, - Custom(String), -} +use std::collections::VecDeque; +use std::fmt::{Display, Formatter, Result as FmtResult}; +use std::rc::Rc; #[derive(Clone, Deserialize, Serialize, PartialEq, Debug)] -pub struct SchemaName(pub Rc, pub String); +pub struct SchemaName(pub Rc, pub String); #[derive(Clone, Deserialize, Serialize, PartialEq, Debug)] pub enum PostgresScalar { @@ -75,18 +66,12 @@ impl From for PostgresType { } } -impl Display for PgSchema { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - PgSchema::Custom(namespace) => write!(f, "{namespace}",), - PgSchema::Public => write!(f, "public"), - } - } -} - impl Display for SchemaName { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - write!(f, r#""{}"."{}""#, self.0, self.1) + match self.0.is_empty() { + true => write!(f, r#""{}""#, self.1), + false => write!(f, r#""{}"."{}""#, self.0, self.1), + } } } @@ -136,7 +121,7 @@ impl Display for PostgresType { impl Display for PrimaryKey { fn fmt(&self, f: &mut Formatter<'_>) -> FmtResult { - write!(f, r#""{}" {} PRIMARY KEY"#, self.name, self.pg_type) + write!(f, r#""{}" {}"#, self.name, self.pg_type) } } @@ -156,7 +141,7 @@ impl From for PostgresType { } impl SchemaName { - pub fn new>(schema: &Rc, name: T) -> Self { + pub fn new>(schema: &Rc, name: T) -> Self { Self(schema.clone(), name.into()) } pub fn replace>(&mut self, name: S) -> String { @@ -168,7 +153,7 @@ impl PostgresType { pub fn is_composite(&self) -> bool { matches!(self.scalar, PostgresScalar::Composite(_)) } - pub fn to_array(self, size: Option) -> PgTypeResult { + pub fn to_array(self, size: Option) -> TypeResult { let arr = match (self.array, size) { (PostgresArray::None, None) => PostgresArray::Dynamic, (PostgresArray::None, Some(size)) => PostgresArray::Fixed(VecDeque::from([size])), @@ -176,7 +161,7 @@ impl PostgresType { sizes.push_back(size); PostgresArray::Fixed(sizes) } - _ => return Err(PgTypeError::NestedArrays), + _ => return Err(TypeError::NestedArrays), }; Ok(Self { scalar: self.scalar, @@ -184,7 +169,7 @@ impl PostgresType { }) } - pub fn composite>(schema: &Rc, name: S) -> Self { + pub fn composite>(schema: &Rc, name: S) -> Self { Self { scalar: PostgresScalar::Composite(SchemaName::new(schema, name)), array: PostgresArray::None, diff --git a/crates/introspect-postgres-sink/src/upgrade.rs b/crates/introspect-sql-sink/src/postgres/upgrade.rs similarity index 83% rename from crates/introspect-postgres-sink/src/upgrade.rs rename to crates/introspect-sql-sink/src/postgres/upgrade.rs index f565d01d..ac6681ca 100644 --- a/crates/introspect-postgres-sink/src/upgrade.rs +++ b/crates/introspect-sql-sink/src/postgres/upgrade.rs @@ -1,33 +1,38 @@ +use super::create::PostgresTypeExtractor; +use super::query::{ColumnUpgrade, StructMod, StructMods, TableUpgrade}; +use super::{HasherExt, PostgresScalar, PostgresType}; use crate::{ - create::PostgresTypeExtractor, - query::{ColumnUpgrade, StructMod, StructMods, TableUpgrade}, - table::{DeadField, PgTable}, - HasherExt, PgSchema, PgTypeError, PgTypeResult, PostgresScalar, PostgresType, TableResult, - UpgradeError, UpgradeResult, UpgradeResultExt, + DeadField, Table, TableResult, TypeError, TypeResult, UpgradeError, UpgradeResult, + UpgradeResultExt, }; use introspect_types::{ ArrayDef, ColumnDef, EnumDef, FixedArrayDef, MemberDef, OptionDef, PrimaryDef, PrimaryTypeDef, ResultInto, StructDef, TupleDef, TypeDef, VariantDef, }; -use starknet_types_core::felt::Felt; -use std::{collections::HashMap, rc::Rc}; -use torii_introspect::schema::TableInfo; +use std::collections::HashMap; +use std::rc::Rc; use xxhash_rust::xxh3::Xxh3; -impl PgTable { - pub fn update_from_info(&mut self, id: &Felt, info: &TableInfo) -> TableResult { - self.update(id, &info.name, &info.primary, &info.columns) - } - pub fn update( +pub trait PgTableUpgrade { + fn upgrade_table( + &mut self, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + ) -> TableResult; + fn retype_primary(&mut self, new: &PrimaryTypeDef) -> UpgradeResult>; +} + +impl PgTableUpgrade for Table { + fn upgrade_table( &mut self, - id: &Felt, name: &str, primary: &PrimaryDef, columns: &[ColumnDef], ) -> TableResult { - let branch = Xxh3::new_based(id); - let schema = Rc::new(self.schema.clone()); - let mut table_mod = TableUpgrade::new(&schema, self.name.clone()); + let branch = Xxh3::new_based(&self.id); + let schema: Rc = self.namespace(); + let mut table_mod = TableUpgrade::new(&schema, self.id, self.name.clone()); table_mod.rename_table(name); table_mod.rename_column(&mut self.primary.name, &primary.name); let pg_type = self @@ -64,58 +69,66 @@ impl PgTable { Ok(table_mod) } fn retype_primary(&mut self, new: &PrimaryTypeDef) -> UpgradeResult> { - use crate::PostgresScalar::{ + use super::PostgresScalar::{ BigInt, Felt252 as PgFelt252, Int, Int128, SmallInt, Uint128, Uint16, Uint32, Uint64, Uint8, }; use introspect_types::PrimaryTypeDef::{ + Bool as PBool, Bytes31 as PBytes31, Bytes31Encoded as PBytes31Encoded, + ClassHash as PClassHash, ContractAddress as PContractAddress, + EthAddress as PEthAddress, Felt252 as PFelt252, ShortUtf8 as PShortUtf8, + StorageAddress as PStorageAddress, StorageBaseAddress as PStorageBaseAddress, + I128 as PI128, I16 as PI16, I32 as PI32, I64 as PI64, I8 as PI8, U128 as PU128, + U16 as PU16, U32 as PU32, U64 as PU64, U8 as PU8, + }; + use introspect_types::TypeDef::{ Bool, Bytes31, Bytes31Encoded, ClassHash, ContractAddress, EthAddress, Felt252, ShortUtf8, StorageAddress, StorageBaseAddress, I128, I16, I32, I64, I8, U128, U16, U32, U64, U8, }; match (&self.primary.type_def, new) { - (Bool, Bool) - | (U8, U8) - | (U16, U16) - | (U32, U32) - | (U64, U64) - | (U128, U128) - | (I8, I8) - | (I16, I16) - | (I32, I32) - | (I64, I64) - | (I128, I128) - | (ShortUtf8, ShortUtf8) - | (EthAddress, EthAddress) - | (ClassHash, ClassHash) - | (ContractAddress, ContractAddress) - | (StorageAddress, StorageAddress) - | (StorageBaseAddress, StorageBaseAddress) - | (Bytes31, Bytes31) - | (Bytes31Encoded(_), Bytes31Encoded(_)) - | (Felt252, Felt252) => Ok(None), - (Bool, U8) => self.primary.type_def.update_as(U8, Uint8), - (Bool | U8, U16) => self.primary.type_def.update_as(U16, Uint16), - (Bool | U8 | U16, U32) => self.primary.type_def.update_as(U32, Uint32), - (Bool | U8 | U16 | U32, U64) => self.primary.type_def.update_as(U64, Uint64), - (Bool | U8 | U16 | U32 | U64, U128) => self.primary.type_def.update_as(U128, Uint128), - (Bool | U8, I8) => self.primary.type_def.update_as(I8, SmallInt), - (Bool | U8 | I8, I16) => self.primary.type_def.update_as(I16, SmallInt), - (Bool | U8 | U16 | I8 | I16, I32) => self.primary.type_def.update_as(I32, Int), - (Bool | U8 | U16 | U32 | I8 | I16 | I32, I64) => { + (Bool, PBool) + | (U8, PU8) + | (U16, PU16) + | (U32, PU32) + | (U64, PU64) + | (U128, PU128) + | (I8, PI8) + | (I16, PI16) + | (I32, PI32) + | (I64, PI64) + | (I128, PI128) + | (ShortUtf8, PShortUtf8) + | (EthAddress, PEthAddress) + | (ClassHash, PClassHash) + | (ContractAddress, PContractAddress) + | (StorageAddress, PStorageAddress) + | (StorageBaseAddress, PStorageBaseAddress) + | (Bytes31, PBytes31) + | (Bytes31Encoded(_), PBytes31Encoded(_)) + | (Felt252, PFelt252) => Ok(None), + (Bool, PU8) => self.primary.type_def.update_as(U8, Uint8), + (Bool | U8, PU16) => self.primary.type_def.update_as(U16, Uint16), + (Bool | U8 | U16, PU32) => self.primary.type_def.update_as(U32, Uint32), + (Bool | U8 | U16 | U32, PU64) => self.primary.type_def.update_as(U64, Uint64), + (Bool | U8 | U16 | U32 | U64, PU128) => self.primary.type_def.update_as(U128, Uint128), + (Bool | U8, PI8) => self.primary.type_def.update_as(I8, SmallInt), + (Bool | U8 | I8, PI16) => self.primary.type_def.update_as(I16, SmallInt), + (Bool | U8 | U16 | I8 | I16, PI32) => self.primary.type_def.update_as(I32, Int), + (Bool | U8 | U16 | U32 | I8 | I16 | I32, PI64) => { self.primary.type_def.update_as(I64, BigInt) } - (Bool | U8 | U16 | U32 | U64 | I8 | I16 | I32 | I64, I128) => { + (Bool | U8 | U16 | U32 | U64 | I8 | I16 | I32 | I64, PI128) => { self.primary.type_def.update_as(I128, Int128) } ( EthAddress, - ClassHash | ContractAddress | StorageAddress | StorageBaseAddress | Felt252, - ) => self.primary.type_def.update_to(new, PgFelt252), + PClassHash | PContractAddress | PStorageAddress | PStorageBaseAddress | PFelt252, + ) => self.primary.type_def.update_to(&new.into(), PgFelt252), ( ClassHash | ContractAddress | StorageAddress | StorageBaseAddress | Felt252, - ClassHash | ContractAddress | StorageAddress | StorageBaseAddress | Felt252, - ) => self.primary.type_def.update_no_change(new), + PClassHash | PContractAddress | PStorageAddress | PStorageBaseAddress | PFelt252, + ) => self.primary.type_def.update_no_change(&new.into()), _ => UpgradeError::primary_upgrade_err(&self.primary.type_def, new), } } @@ -129,7 +142,7 @@ pub trait ExtractItem { fn as_tuple(&mut self) -> UpgradeResult<&mut TupleDef>; fn update_as_array( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &ArrayDef, dead: &mut HashMap, @@ -137,13 +150,13 @@ pub trait ExtractItem { ) -> UpgradeResult>; fn update_as_option( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &TypeDef, dead: &mut HashMap, queries: &mut ColumnUpgrade, ) -> UpgradeResult>; - fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> PgTypeResult; + fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> TypeResult; } impl ExtractItem for TypeDef { @@ -181,7 +194,7 @@ impl ExtractItem for TypeDef { fn update_as_array( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &ArrayDef, dead: &mut HashMap, @@ -195,7 +208,7 @@ impl ExtractItem for TypeDef { } fn update_as_option( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &TypeDef, dead: &mut HashMap, @@ -210,7 +223,7 @@ impl ExtractItem for TypeDef { } } } - fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> PgTypeResult { + fn get_pg_type(&self, schema: &Rc, branch: &Xxh3) -> TypeResult { match self { TypeDef::None => Ok(PostgresScalar::None.into()), TypeDef::Bool => Ok(PostgresScalar::Boolean.into()), @@ -248,7 +261,7 @@ impl ExtractItem for TypeDef { TypeDef::Option(def) => def.get_pg_type(schema, branch), TypeDef::Nullable(def) => def.get_pg_type(schema, branch), TypeDef::Felt252Dict(_) | TypeDef::Result(_) | TypeDef::Ref(_) | TypeDef::Custom(_) => { - Err(PgTypeError::UnsupportedType(format!("{self:?}"))) + Err(TypeError::UnsupportedType(format!("{self:?}"))) } } } @@ -257,7 +270,7 @@ impl ExtractItem for TypeDef { pub trait CompareType { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &Self, dead: &mut HashMap, @@ -297,7 +310,7 @@ pub trait UpgradeField { fn name(&self) -> &str; fn upgrade_field( &mut self, - schema: &Rc, + schema: &Rc, name: &str, branch: &Xxh3, new: &Self, @@ -312,7 +325,7 @@ pub trait UpgradeField { } fn add_field( &self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, queries: &mut ColumnUpgrade, ) -> UpgradeResult { @@ -350,7 +363,7 @@ impl UpgradeField for VariantDef { impl CompareType for TypeDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &TypeDef, dead: &mut HashMap, @@ -439,7 +452,7 @@ impl CompareType for TypeDef { impl CompareType for StructDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &StructDef, dead: &mut HashMap, @@ -477,7 +490,7 @@ impl CompareType for StructDef { impl CompareType for EnumDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &EnumDef, dead: &mut HashMap, @@ -517,7 +530,7 @@ impl CompareType for EnumDef { impl CompareType for FixedArrayDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &Self, dead: &mut HashMap, @@ -540,7 +553,7 @@ impl CompareType for FixedArrayDef { impl CompareType for TupleDef { fn compare_type( &mut self, - schema: &Rc, + schema: &Rc, branch: &Xxh3, new: &Self, dead: &mut HashMap, diff --git a/crates/introspect-postgres-sink/src/utils.rs b/crates/introspect-sql-sink/src/postgres/utils.rs similarity index 100% rename from crates/introspect-postgres-sink/src/utils.rs rename to crates/introspect-sql-sink/src/postgres/utils.rs diff --git a/crates/introspect-sql-sink/src/processor.rs b/crates/introspect-sql-sink/src/processor.rs new file mode 100644 index 00000000..77dabbca --- /dev/null +++ b/crates/introspect-sql-sink/src/processor.rs @@ -0,0 +1,277 @@ +use crate::backend::{IntrospectInitialize, IntrospectPool, IntrospectProcessor}; +use crate::error::TableLoadError; +use crate::table::{DeadField, Table}; +use crate::tables::Tables; +use crate::{DbResult, IntrospectQueryMaker, NamespaceKey, NamespaceMode}; +use async_trait::async_trait; +use introspect_types::{ColumnInfo, PrimaryDef, TypeDef}; +use itertools::Itertools; +use sqlx::{Database, Pool}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::fmt::Debug; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; +use torii_sql::{Executable, FlexQuery, PoolExt}; + +pub const COMMIT_CMD: &str = "--COMMIT"; + +pub struct IntrospectDb { + tables: Tables, + namespaces: NamespaceMode, + db: Backend, +} + +pub struct DbTable { + pub namespace: String, + pub id: Felt, + pub owner: Felt, + pub name: String, + pub primary: PrimaryDef, + pub columns: HashMap, + pub dead: HashMap, + pub append_only: bool, + pub alive: bool, +} + +pub struct DbColumn { + pub namespace: String, + pub table: Felt, + pub id: Felt, + pub name: String, + pub type_def: TypeDef, +} + +pub struct DbDeadField { + pub namespace: String, + pub table: Felt, + pub id: u128, + pub name: String, + pub type_def: TypeDef, +} + +impl PoolExt for IntrospectDb> { + fn pool(&self) -> &Pool { + &self.db + } +} + +pub trait IntoHashMap { + fn into_hash_map(self) -> HashMap; +} + +impl IntoHashMap for Vec +where + T: Into<(K, V)>, + K: std::hash::Hash + Eq, +{ + fn into_hash_map(self) -> HashMap { + self.into_iter().map_into().collect() + } +} + +#[async_trait] +impl IntrospectProcessor for Pool +where + Vec>: Executable, + FlexQuery: Debug + Clone, +{ + async fn process_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + self.execute_msgs(namespaces, tables, msgs).await + } +} + +impl IntrospectDb { + pub fn new(pool: Backend, namespaces: impl Into) -> Self { + Self { + tables: Tables::default(), + namespaces: namespaces.into(), + db: pool, + } + } +} +impl IntrospectDb { + pub async fn initialize_introspect_sql_sink(&self) -> DbResult> { + self.db.initialize().await?; + self.load_store_data().await + } + + pub async fn process_messages( + &self, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + self.db + .process_msgs(&self.namespaces, &self.tables, msgs) + .await + } + + pub async fn load_store_data(&self) -> DbResult> { + let mut errors = Vec::new(); + let namespaces = self.namespaces.namespaces(); + let mut tables: HashMap<(String, Felt), Table> = + self.db.load_tables(&namespaces).await?.into_hash_map(); + for column in self.db.load_columns(&namespaces).await? { + let (namespace, table_id, id, column_info) = column.into(); + if let Some(table) = tables.get_mut(&(namespace.clone(), table_id)) { + table.columns.insert(id, column_info); + } else { + errors.push(TableLoadError::ColumnTableNotFound( + namespace, + table_id, + column_info.name, + id, + )); + } + } + for dead_field in self.db.load_dead_fields(&namespaces).await? { + let (namespace, table_id, id, field) = dead_field.into(); + if let Some(table) = tables.get_mut(&(namespace.clone(), table_id)) { + table.dead.insert(id, field); + } else { + errors.push(TableLoadError::TableDeadNotFound( + namespace, table_id, field.name, id, + )); + } + } + let mut map = self.tables.write()?; + for ((namespace, id), table) in tables { + match self.namespaces.get_key(namespace, id, &table.owner) { + Ok(key) => { + map.insert(key, table); + } + Err(err) => errors.push(TableLoadError::NamespaceError(err)), + } + } + Ok(errors) + } +} + +impl From for ((String, Felt), Table) { + fn from(value: DbTable) -> Self { + ( + (value.namespace.clone(), value.id), + Table { + id: value.id, + namespace: value.namespace, + name: value.name, + owner: value.owner, + primary: value.primary.into(), + columns: value.columns, + dead: value.dead, + append_only: value.append_only, + alive: value.alive, + }, + ) + } +} + +impl From for (String, Felt, Felt, ColumnInfo) { + fn from(value: DbColumn) -> Self { + ( + value.namespace, + value.table, + value.id, + ColumnInfo { + name: value.name, + attributes: Vec::new(), + type_def: value.type_def, + }, + ) + } +} + +impl From for (String, Felt, u128, DeadField) { + fn from(value: DbDeadField) -> Self { + ( + value.namespace, + value.table, + value.id, + DeadField { + name: value.name, + type_def: value.type_def, + }, + ) + } +} + +pub fn messages_to_queries( + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + queries: &mut Vec>, +) -> DbResult>> { + let mut results = Vec::with_capacity(msgs.len()); + for body in msgs { + let (msg, metadata) = body.into(); + let from_address: Felt = metadata.from_address.into(); + let transaction_hash: Felt = metadata.transaction_hash.into(); + let namespace = namespaces.to_namespace(&from_address)?; + results.push(handle_message::( + namespace, + tables, + msg, + &from_address, + metadata.block_number, + &transaction_hash, + queries, + )); + } + Ok(results) +} + +pub fn handle_message( + namespace: NamespaceKey, + tables: &Tables, + msg: &IntrospectMsg, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, +) -> DbResult<()> { + match msg { + IntrospectMsg::CreateTable(event) => tables.create_table::( + namespace, + &event.id, + &event.name, + &event.primary, + &event.columns, + event.append_only, + from_address, + block_number, + transaction_hash, + queries, + ), + IntrospectMsg::UpdateTable(event) => tables.update_table::( + namespace, + &event.id, + &event.name, + &event.primary, + &event.columns, + from_address, + block_number, + transaction_hash, + queries, + ), + IntrospectMsg::AddColumns(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::DropColumns(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::RetypeColumns(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::RetypePrimary(event) => tables.set_table_dead(namespace, event.table), + IntrospectMsg::RenameTable(_) + | IntrospectMsg::DropTable(_) + | IntrospectMsg::RenameColumns(_) + | IntrospectMsg::RenamePrimary(_) => Ok(()), + IntrospectMsg::InsertsFields(event) => tables.insert_fields::( + namespace, + event, + from_address, + block_number, + transaction_hash, + queries, + ), + IntrospectMsg::DeleteRecords(_) | IntrospectMsg::DeletesFields(_) => Ok(()), + } +} diff --git a/crates/introspect-sql-sink/src/runtime.rs b/crates/introspect-sql-sink/src/runtime.rs new file mode 100644 index 00000000..1c3667eb --- /dev/null +++ b/crates/introspect-sql-sink/src/runtime.rs @@ -0,0 +1,58 @@ +use crate::tables::Tables; +use crate::{ + DbColumn, DbDeadField, DbResult, DbTable, IntrospectInitialize, IntrospectProcessor, + IntrospectSqlSink, NamespaceMode, +}; +use async_trait::async_trait; +use torii_introspect::events::IntrospectBody; +use torii_sql::DbPool; + +impl IntrospectSqlSink for DbPool { + const NAME: &'static str = "introspect-sql"; +} + +#[async_trait] +impl IntrospectInitialize for DbPool { + async fn initialize(&self) -> DbResult { + match self { + DbPool::Postgres(pg) => pg.initialize().await, + DbPool::Sqlite(site) => site.initialize().await, + } + } + async fn load_tables(&self, namespaces: &Option>) -> DbResult> { + match self { + DbPool::Postgres(pg) => pg.load_tables(namespaces).await, + DbPool::Sqlite(site) => site.load_tables(namespaces).await, + } + } + async fn load_columns(&self, namespaces: &Option>) -> DbResult> { + match self { + DbPool::Postgres(pg) => pg.load_columns(namespaces).await, + DbPool::Sqlite(site) => site.load_columns(namespaces).await, + } + } + async fn load_dead_fields( + &self, + namespaces: &Option>, + ) -> DbResult> { + match self { + DbPool::Postgres(pg) => pg.load_dead_fields(namespaces).await, + DbPool::Sqlite(site) => site.load_dead_fields(namespaces).await, + } + } +} + +#[async_trait] +impl IntrospectProcessor for DbPool { + async fn process_msgs( + &self, + namespaces: &NamespaceMode, + tables: &Tables, + msgs: Vec<&IntrospectBody>, + ) -> DbResult>> { + match self { + DbPool::Postgres(pg) => pg.process_msgs(namespaces, tables, msgs).await, + DbPool::Sqlite(site) => site.process_msgs(namespaces, tables, msgs).await, + } + } +} diff --git a/crates/introspect-sqlite-sink/src/sink.rs b/crates/introspect-sql-sink/src/sink.rs similarity index 57% rename from crates/introspect-sqlite-sink/src/sink.rs rename to crates/introspect-sql-sink/src/sink.rs index 86706aed..d3ffec80 100644 --- a/crates/introspect-sqlite-sink/src/sink.rs +++ b/crates/introspect-sql-sink/src/sink.rs @@ -1,58 +1,67 @@ -use crate::processor::IntrospectSqliteDb; use anyhow::Result; use async_trait::async_trait; use std::sync::Arc; use torii::axum::Router; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, SinkContext, TopicInfo}, -}; +use torii::etl::envelope::{Envelope, TypeId}; +use torii::etl::extractor::ExtractionBatch; +use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; +use torii_dojo::{DojoBody, DojoEvent, DOJO_TYPE_ID}; use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_sqlite::SqliteConnection; -pub const LOGGING_TARGET: &str = "torii::sinks::introspect::sqlite"; -const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); +use crate::{IntrospectDb, IntrospectInitialize, IntrospectProcessor}; + +pub const LOGGING_TARGET: &str = "torii::sinks::introspect-sql"; +pub trait IntrospectSqlSink { + const NAME: &'static str; +} #[async_trait] -impl Sink for IntrospectSqliteDb { +impl Sink + for IntrospectDb +{ fn name(&self) -> &'static str { - "introspect-sqlite" + Backend::NAME } fn interested_types(&self) -> Vec { - vec![TypeId::new("introspect")] + vec![DOJO_TYPE_ID] } async fn process(&self, envelopes: &[Envelope], _batch: &ExtractionBatch) -> Result<()> { let mut processed = 0usize; - let mut create_tables = 0usize; + let mut create_tables: usize = 0usize; let mut update_tables = 0usize; let mut inserts_fields = 0usize; let mut inserted_records = 0usize; let mut delete_records = 0usize; let mut msgs = Vec::with_capacity(envelopes.len()); for envelope in envelopes { - if envelope.type_id == INTROSPECT_TYPE { - if let Some(body) = envelope.downcast_ref::() { - match &body.msg { - IntrospectMsg::CreateTable(_) => create_tables += 1, - IntrospectMsg::UpdateTable(_) => update_tables += 1, - IntrospectMsg::InsertsFields(event) => { - inserts_fields += 1; - inserted_records += event.records.len(); - } - IntrospectMsg::DeleteRecords(event) => { - delete_records += event.rows.len(); + if envelope.type_id == DOJO_TYPE_ID { + if let Some(body) = envelope.downcast_ref::() { + if let DojoEvent::Introspect(msg) = &body.msg { + match msg { + IntrospectMsg::CreateTable(_) => create_tables += 1, + IntrospectMsg::UpdateTable(_) => update_tables += 1, + IntrospectMsg::InsertsFields(event) => { + inserts_fields += 1; + inserted_records += event.records.len(); + } + IntrospectMsg::DeleteRecords(event) => { + delete_records += event.rows.len(); + } + _ => {} } - _ => {} + processed += 1; + msgs.push(IntrospectBody { + context: body.context, + msg: msg.clone(), + }); } - processed += 1; - msgs.push(body); } } } - let results = self.process_messages(msgs).await?; + let msg_refs = msgs.iter().collect::>(); + let results = self.process_messages(msg_refs).await?; let failed = results.iter().filter(|r| r.is_err()).count(); if failed > 0 { tracing::error!( @@ -102,10 +111,11 @@ impl Sink for IntrospectSqliteDb { _event_bus: Arc, _context: &SinkContext, ) -> Result<()> { - self.initialize_introspect_sqlite_sink().await?; + self.initialize_introspect_sql_sink().await?; tracing::info!( target: LOGGING_TARGET, - "Initialized introspect SQLite sink" + "Connected to introspect SQL sink with database: {}", + Backend::NAME ); Ok(()) } diff --git a/crates/introspect-sql-sink/src/sqlite/append_only.rs b/crates/introspect-sql-sink/src/sqlite/append_only.rs new file mode 100644 index 00000000..5607c0a2 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/append_only.rs @@ -0,0 +1,71 @@ +use crate::sqlite::record::SqliteDeserializer; +use crate::sqlite::table::qualified_table_name; +use crate::sqlite::types::{SqliteType, TypeDefSqliteExt}; +use crate::{RecordResult, Table}; +use introspect_types::bytes::IntoByteSource; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::ColumnInfo; +use itertools::Itertools; +use sqlx::Arguments; +use sqlx::Error::Encode as EncodeError; +use starknet_types_core::felt::Felt; +use std::fmt::Write as FmtWrite; +use std::sync::Arc; +use torii_introspect::Record; +use torii_sql::{Queries, SqliteArguments, SqliteQuery}; + +pub fn append_only_record_queries( + table: &Table, + column_ids: &[Felt], + records: &[Record], + _from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let table_name = qualified_table_name(&table.namespace, &table.name); + let primary = &table.primary.name; + let columns = table.columns.iter().collect_vec(); + let mut sql = format!(r#"INSERT INTO "{table_name}" ("{primary}", "__revision""#,); + for name in columns.names() { + write!(sql, r#", "{name}""#).unwrap(); + } + write!(sql, r#") VALUES (?, (SELECT COALESCE(MAX("__revision"), 0) + 1 FROM "{table_name}" WHERE "{primary}" = ?1)"#).unwrap(); + for (id, ColumnInfo { name, type_def, .. }) in &columns { + match column_ids.iter().position(|c| &c == id) { + Some(index) => write!(sql, r#", {}"#, type_def.index_placeholder(index + 2)?).unwrap(), + None => match type_def.try_into()? { + SqliteType::Json => write!( + sql, + r#", (SELECT jsonb("{name}") FROM "{table_name}" WHERE "{primary}" = ?1 ORDER BY "__revision" DESC LIMIT 1)"# + ).unwrap(), + _ => write!( + sql, + r#", (SELECT "{name}" FROM "{table_name}" WHERE "{primary}" = ?1 ORDER BY "__revision" DESC LIMIT 1)"# + ).unwrap(), + } + } + } + let schema = table.get_record_schema(column_ids)?; + sql.push(')'); + let sql: Arc = sql.into(); + for record in records { + let mut arguments: SqliteArguments<'static> = SqliteArguments::default(); + let mut primary_data = record.id.as_slice().into_source(); + let mut data = record.values.as_slice().into_source(); + arguments + .add( + schema + .primary_type_def() + .deserialize_column(&mut primary_data)?, + ) + .map_err(EncodeError)?; + for type_def in schema.columns().type_defs() { + arguments + .add(type_def.deserialize_column(&mut data)?) + .map_err(EncodeError)?; + } + queries.add((sql.clone(), arguments)); + } + Ok(()) +} diff --git a/crates/introspect-sql-sink/src/sqlite/backend.rs b/crates/introspect-sql-sink/src/sqlite/backend.rs new file mode 100644 index 00000000..6f0d4999 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/backend.rs @@ -0,0 +1,190 @@ +use crate::sqlite::append_only::append_only_record_queries; +use crate::sqlite::record::insert_record_queries; +use crate::sqlite::table::{ + create_table_query, persist_table_state_query, qualified_table_name, update_column, + update_columns, FETCH_TABLES_QUERY, +}; +use crate::{ + DbColumn, DbDeadField, DbResult, DbTable, IntrospectDb, IntrospectInitialize, + IntrospectQueryMaker, IntrospectSqlSink, RecordResult, Table, TableResult, UpgradeResultExt, +}; +use async_trait::async_trait; +use introspect_types::{ColumnDef, ColumnInfo, PrimaryDef, ResultInto}; +use itertools::Itertools; +use sqlx::prelude::FromRow; +use sqlx::types::Json; +use starknet_types_core::felt::{Felt, FromStrError}; +use std::collections::HashMap; +use torii_introspect::Record; +use torii_sql::{PoolExt, Queries, Sqlite, SqlitePool, SqliteQuery}; + +pub const INTROSPECT_SQLITE_SINK_MIGRATIONS: sqlx::migrate::Migrator = + sqlx::migrate!("./migrations/sqlite"); + +pub type IntrospectSqliteDb = IntrospectDb; + +#[derive(FromRow)] +pub struct SqliteTableRow { + namespace: String, + id: String, + owner: String, + name: String, + primary: Json, + columns: Json>, + append_only: bool, + alive: bool, +} + +impl TryFrom for DbTable { + type Error = FromStrError; + fn try_from(value: SqliteTableRow) -> Result { + Ok(DbTable { + namespace: value.namespace, + id: Felt::from_hex(&value.id)?, + owner: Felt::from_hex(&value.owner)?, + name: value.name, + primary: value.primary.0, + columns: value.columns.0.into_iter().map_into().collect(), + dead: HashMap::new(), + append_only: value.append_only, + alive: value.alive, + }) + } +} + +#[async_trait] +impl IntrospectQueryMaker for Sqlite { + fn create_table_queries( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + queries.add(create_table_query( + namespace, + name, + primary, + columns, + append_only, + )?); + persist_table_state_query( + namespace, + id, + name, + primary, + columns, + append_only, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn update_table_queries( + table: &mut Table, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec, + ) -> TableResult<()> { + let table_name = qualified_table_name(&table.namespace, name); + if table.name != name { + queries.add(format!( + r#"ALTER TABLE "{}" RENAME TO "{table_name}""#, + qualified_table_name(&table.namespace, &table.name), + )); + table.name = name.to_string(); + } + update_column( + &table_name, + &mut table.primary, + &primary.name, + &((&primary.type_def).into()), + queries, + ) + .to_table_result(&table_name, "primary")?; + update_columns(&mut table.columns, &table_name, columns, queries)?; + persist_table_state_query( + &table.namespace, + &table.id, + &table.name, + primary, + &table.columns.iter().collect_vec(), + table.append_only, + from_address, + block_number, + transaction_hash, + queries, + ) + } + fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + _from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, + ) -> RecordResult<()> { + if table.append_only { + append_only_record_queries( + table, + columns, + records, + _from_address, + _block_number, + _transaction_hash, + queries, + ) + } else { + insert_record_queries( + table, + columns, + records, + _from_address, + _block_number, + _transaction_hash, + queries, + ) + } + } +} + +impl IntrospectSqlSink for SqlitePool { + const NAME: &'static str = "Introspect Sqlite"; +} + +#[async_trait] +impl IntrospectInitialize for SqlitePool { + async fn load_tables(&self, _schemas: &Option>) -> DbResult> { + let rows: Vec = sqlx::query_as(FETCH_TABLES_QUERY) + .fetch_all(self.pool()) + .await?; + + let tables: Vec = rows + .into_iter() + .map(|row| row.try_into()) + .collect::>()?; + Ok(tables) + } + async fn load_columns(&self, _schemas: &Option>) -> DbResult> { + Ok(Vec::new()) + } + async fn load_dead_fields(&self, _schemas: &Option>) -> DbResult> { + Ok(Vec::new()) + } + async fn initialize(&self) -> DbResult<()> { + self.migrate(Some("introspect"), INTROSPECT_SQLITE_SINK_MIGRATIONS) + .await + .err_into() + } +} diff --git a/crates/introspect-sqlite-sink/src/json.rs b/crates/introspect-sql-sink/src/sqlite/json.rs similarity index 92% rename from crates/introspect-sqlite-sink/src/json.rs rename to crates/introspect-sql-sink/src/sqlite/json.rs index 3b7c28d8..7cf2f75b 100644 --- a/crates/introspect-sqlite-sink/src/json.rs +++ b/crates/introspect-sql-sink/src/sqlite/json.rs @@ -2,7 +2,7 @@ use introspect_types::serialize::ToCairoDeSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; use introspect_types::{CairoDeserializer, ResultDef, TupleDef, TypeDef}; use primitive_types::{U256, U512}; -use serde::ser::SerializeMap; +use serde::ser::{SerializeMap, SerializeTuple}; use serde::Serializer; pub struct SqliteJsonSerializer; @@ -54,9 +54,9 @@ impl CairoTypeSerialization for SqliteJsonSerializer { serializer: S, tuple: &'a TupleDef, ) -> Result { - let mut seq = serializer.serialize_map(Some(tuple.elements.len()))?; - for (index, element) in tuple.elements.iter().enumerate() { - seq.serialize_entry(&format!("_{index}"), &element.to_de_se(data, self))?; + let mut seq = serializer.serialize_tuple(tuple.elements.len())?; + for element in tuple.elements.iter() { + seq.serialize_element(&element.to_de_se(data, self))?; } seq.end() } diff --git a/crates/introspect-sql-sink/src/sqlite/mod.rs b/crates/introspect-sql-sink/src/sqlite/mod.rs new file mode 100644 index 00000000..3f8c4ef1 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/mod.rs @@ -0,0 +1,12 @@ +pub mod append_only; +pub mod backend; +pub mod json; +pub mod record; +pub mod table; +pub mod types; + +use sqlx::migrate::Migrator; + +pub use backend::IntrospectSqliteDb; + +pub const INTROSPECT_SQLITE_SINK_MIGRATIONS: Migrator = sqlx::migrate!("./migrations/sqlite"); diff --git a/crates/introspect-sql-sink/src/sqlite/record.rs b/crates/introspect-sql-sink/src/sqlite/record.rs new file mode 100644 index 00000000..ab7d5d48 --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/record.rs @@ -0,0 +1,230 @@ +use crate::sqlite::json::SqliteJsonSerializer; +use crate::sqlite::table::qualified_table_name; +use crate::sqlite::types::{SqliteColumn, SqliteType}; +use crate::{RecordResult, Table, TypeResult}; +use introspect_types::bytes::IntoByteSource; +use introspect_types::schema::{Names, TypeDefs}; +use introspect_types::serialize::CairoSeFrom; +use introspect_types::{CairoDeserializer, DecodeError, EthAddress, ResultInto, TypeDef}; +use itertools::Itertools; +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::sqlite::SqliteArgumentValue; +use sqlx::Error::Encode as EncodeError; +use sqlx::{Arguments, Encode, Sqlite, Type}; +use starknet_types_core::felt::Felt; +use std::sync::Arc; +use torii_introspect::Record; +use torii_sql::{Queries, SqliteArguments, SqliteQuery}; + +pub fn coalesce_sql<'a>(table_name: &str, column: &SqliteColumn<'a>) -> String { + let column_name = column.name; + match column.sql_type { + SqliteType::Json => { + format!( + r#""{column_name}" = COALESCE(jsonb(excluded."{column_name}"), "{table_name}"."{column_name}")"# + ) + } + _ => format!( + r#""{column_name}" = COALESCE(excluded."{column_name}", "{table_name}"."{column_name}")"# + ), + } +} + +pub enum SqliteValue { + Null, + Integer(i64), + Text(String), + Blob(Vec), +} + +impl SqliteValue { + fn integer(n: impl Into) -> Self { + SqliteValue::Integer(n.into()) + } + fn text(s: impl ToString) -> Self { + SqliteValue::Text(s.to_string()) + } +} + +impl From for SqliteValue { + fn from(value: String) -> Self { + SqliteValue::Text(value) + } +} + +impl From> for SqliteValue { + fn from(value: Vec) -> Self { + SqliteValue::Blob(value) + } +} + +impl From<[u8; N]> for SqliteValue { + fn from(value: [u8; N]) -> Self { + SqliteValue::Blob(value.to_vec()) + } +} + +impl From for SqliteValue { + fn from(value: Felt) -> Self { + SqliteValue::Text(format!("{value:#064x}")) + } +} + +impl From for SqliteValue { + fn from(value: EthAddress) -> Self { + SqliteValue::Text(format!("0x{}", hex::encode(value.0))) + } +} + +impl From for SqliteValue { + fn from(value: bool) -> Self { + SqliteValue::Integer(if value { 1 } else { 0 }) + } +} + +pub trait SqliteDeserializer { + fn deserialize_column(&self, data: &mut impl CairoDeserializer) -> RecordResult; + fn deserialize_json(&self, data: &mut impl CairoDeserializer) -> RecordResult; +} + +impl SqliteDeserializer for TypeDef { + fn deserialize_column(&self, data: &mut impl CairoDeserializer) -> RecordResult { + match self { + TypeDef::None => Ok(SqliteValue::Null), + TypeDef::Felt252 + | TypeDef::ClassHash + | TypeDef::ContractAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress => data.next_felt().result_into(), + TypeDef::ShortUtf8 => data.next_short_string().result_into(), + TypeDef::Bytes31 | TypeDef::Bytes31Encoded(_) => data.next_bytes::<31>().result_into(), + TypeDef::Bool => data.next_bool().result_into(), + TypeDef::U8 => data.next_u8().map(SqliteValue::integer).err_into(), + TypeDef::U16 => data.next_u16().map(SqliteValue::integer).err_into(), + TypeDef::U32 => data.next_u32().map(SqliteValue::integer).err_into(), + TypeDef::U64 => data.next_u64().map(SqliteValue::text).err_into(), + TypeDef::U128 => data.next_u128().map(SqliteValue::text).err_into(), + TypeDef::U256 => data.next_u256().map(SqliteValue::text).err_into(), + TypeDef::U512 => data.next_u512().map(SqliteValue::text).err_into(), + TypeDef::I8 => data.next_i8().map(SqliteValue::integer).err_into(), + TypeDef::I16 => data.next_i16().map(SqliteValue::integer).err_into(), + TypeDef::I32 => data.next_i32().map(SqliteValue::integer).err_into(), + TypeDef::I64 => data.next_i64().map(SqliteValue::integer).err_into(), + TypeDef::I128 => data.next_i128().map(SqliteValue::text).err_into(), + TypeDef::EthAddress => data.next_eth_address().result_into(), + TypeDef::Utf8String => data.next_string().result_into(), + TypeDef::ByteArray | TypeDef::ByteArrayEncoded(_) | TypeDef::Custom(_) => { + data.next_byte_array_bytes().result_into() + } + TypeDef::Tuple(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Felt252Dict(_) + | TypeDef::Struct(_) + | TypeDef::Enum(_) + | TypeDef::Option(_) + | TypeDef::Result(_) + | TypeDef::Nullable(_) => self.deserialize_json(data), + TypeDef::Ref(_) => Err(DecodeError::message( + "TypeDef Ref needs to be expanded before transoding", + )) + .err_into(), + } + } + fn deserialize_json(&self, data: &mut impl CairoDeserializer) -> RecordResult { + let se = CairoSeFrom::new(self, data, &SqliteJsonSerializer); + serde_json::to_string(&se).result_into() + } +} + +impl From for SqliteArgumentValue<'_> { + fn from(value: SqliteValue) -> Self { + match value { + SqliteValue::Null => SqliteArgumentValue::Null, + SqliteValue::Integer(n) => SqliteArgumentValue::Int64(n), + SqliteValue::Text(s) => SqliteArgumentValue::Text(s.into()), + SqliteValue::Blob(b) => SqliteArgumentValue::Blob(b.into()), + } + } +} + +impl Type for SqliteValue { + fn type_info() -> ::TypeInfo { + // SqliteValue is dynamically typed; report as Text since SQLite is flexible with types. + >::type_info() + } + + fn compatible(ty: &::TypeInfo) -> bool { + >::compatible(ty) + || >::compatible(ty) + || as Type>::compatible(ty) + } +} + +impl<'q> Encode<'q, Sqlite> for SqliteValue { + fn encode_by_ref(&self, buf: &mut Vec>) -> Result { + match self { + SqliteValue::Null => Ok(IsNull::Yes), + SqliteValue::Integer(n) => >::encode_by_ref(n, buf), + SqliteValue::Text(s) => >::encode_by_ref(s, buf), + SqliteValue::Blob(b) => as Encode>::encode_by_ref(b, buf), + } + } +} + +pub fn insert_record_queries( + table: &Table, + columns: &[Felt], + records: &[Record], + _from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, +) -> RecordResult<()> { + let schema = table.get_record_schema(columns)?; + let table_name = qualified_table_name(&table.namespace, &table.name); + let all_columns = schema.all_columns(); + let sql_columns = all_columns + .iter() + .map(|c| (*c).try_into()) + .collect::>>()?; + let column_names = all_columns.names(); + let placeholders = sql_columns.iter().map(SqliteColumn::placeholder).join(", "); + let coalesce = sql_columns[1..] + .iter() + .map(|col| coalesce_sql(&table_name, col)) + .join(", "); + let sql: Arc = format!( + r#"INSERT INTO "{table_name}" ({}) VALUES ({}) ON CONFLICT("{}") DO UPDATE SET {}"#, + column_names + .iter() + .map(|name| format!(r#""{name}""#)) + .collect::>() + .join(", "), + placeholders, + schema.primary_name(), + coalesce + ) + .into(); + + for record in records { + let mut arguments: SqliteArguments<'static> = SqliteArguments::default(); + let mut primary_data = record.id.as_slice().into_source(); + let mut data = record.values.as_slice().into_source(); + arguments + .add( + schema + .primary_type_def() + .deserialize_column(&mut primary_data)?, + ) + .map_err(EncodeError)?; + for type_def in schema.columns().type_defs() { + arguments + .add(type_def.deserialize_column(&mut data)?) + .map_err(EncodeError)?; + } + queries.add((sql.clone(), arguments)); + } + Ok(()) +} diff --git a/crates/introspect-sql-sink/src/sqlite/table.rs b/crates/introspect-sql-sink/src/sqlite/table.rs new file mode 100644 index 00000000..317f837f --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/table.rs @@ -0,0 +1,217 @@ +use crate::sqlite::types::SqliteType; +use crate::{TableResult, UpgradeError, UpgradeResult, UpgradeResultExt}; +use introspect_types::schema::AsColumnRef; +use introspect_types::{ColumnDef, ColumnInfo, PrimaryDef, TypeDef}; +use serde::ser::SerializeMap; +use serde::Serializer; +use serde_json::{Result as JsonResult, Serializer as JsonSerializer}; +use sqlx::Arguments; +use starknet_types_core::felt::Felt; +use starknet_types_raw::Felt as RawFelt; +use std::collections::HashMap; +use std::fmt::{Display, Write}; +use torii_sql::types::SqlFelt; +use torii_sql::{Queries, SqliteArguments, SqliteQuery}; + +pub const FETCH_TABLES_QUERY: &str = r#" + SELECT namespace, id, owner, name, "primary", columns, append_only, alive + FROM introspect_db_tables + ORDER BY updated_at ASC +"#; + +const INSERT_TABLE_QUERY: &str = r#" INSERT INTO introspect_db_tables + (namespace, id, owner, name, "primary", columns, append_only, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, unixepoch()) + ON CONFLICT (namespace, id) DO UPDATE SET + owner = excluded.owner, name = excluded.name, "primary" = excluded."primary", columns = excluded.columns, append_only = excluded.append_only, updated_at = unixepoch() +"#; + +struct TableName<'a>(&'a str, &'a str); + +impl Display for TableName<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if !self.0.is_empty() { + write!(f, "{}__", self.0)?; + } + self.1.fmt(f) + } +} + +pub fn qualified_table_name(namespace: &str, table_name: &str) -> String { + if namespace.is_empty() { + table_name.to_string() + } else { + format!("{}__{}", namespace, table_name) + } +} + +pub fn serialize_columns<'a>(columns: &'a [impl AsColumnRef<'a>]) -> JsonResult { + let mut data = Vec::new(); + let mut serializer = JsonSerializer::new(&mut data); + let mut array = serializer.serialize_map(Some(columns.len()))?; + for column in columns { + let (id, info) = column.as_entry(); + array.serialize_entry(id, &info)?; + } + array.end()?; + Ok(unsafe { String::from_utf8_unchecked(data) }) +} + +#[allow(clippy::too_many_arguments)] +pub fn persist_table_state_query<'a>( + namespace: &str, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &'a [impl AsColumnRef<'a>], + append_only: bool, + from_address: &Felt, + _block_number: u64, + _transaction_hash: &Felt, + queries: &mut Vec, +) -> TableResult<()> { + let mut args = SqliteArguments::default(); + args.add(namespace.to_string())?; + args.add(SqlFelt::from(RawFelt::from(*id)))?; + args.add(SqlFelt::from(RawFelt::from(*from_address)))?; + args.add(name.to_string())?; + args.add(serde_json::to_string(primary)?)?; + args.add(serialize_columns(columns)?)?; + args.add(append_only)?; + queries.add((INSERT_TABLE_QUERY, args)); + Ok(()) +} + +pub fn create_table_query( + namespace: &str, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, +) -> TableResult { + let table_name = TableName(namespace, name); + let mut query = format!( + r#"CREATE TABLE IF NOT EXISTS "{table_name}" ("{}" {}"#, + primary.name, + TryInto::::try_into(primary)? + ); + if append_only { + query.push_str(r#", "__revision" INTEGER NOT NULL"#); + } + for column in columns { + let sql_type: SqliteType = column.try_into()?; + write!(query, r#", "{}" {sql_type}"#, column.name).unwrap(); + } + if append_only { + write!( + query, + r#", PRIMARY KEY ("{}", "__revision"));"#, + primary.name + ) + .unwrap(); + } else { + write!(query, r#", PRIMARY KEY ("{}"));"#, primary.name).unwrap(); + } + Ok(query) +} + +pub fn update_columns( + columns: &mut HashMap, + table_name: &str, + new: &[ColumnDef], + queries: &mut Vec, +) -> TableResult<()> { + for column in new { + let result = match columns.get_mut(&column.id) { + Some(existing) => update_column( + table_name, + existing, + &column.name, + &column.type_def, + queries, + ), + None => { + columns.insert( + column.id, + ColumnInfo { + name: column.name.clone(), + type_def: column.type_def.clone(), + attributes: column.attributes.clone(), + }, + ); + create_column_query(table_name, column).map(|query| queries.add(query)) + } + }; + result.to_table_result(table_name, &column.name)?; + } + Ok(()) +} + +pub fn update_column( + table_name: &str, + column: &mut ColumnInfo, + new_name: &str, + new_type: &TypeDef, + queries: &mut Vec, +) -> UpgradeResult { + use introspect_types::TypeDef::{ + Array, Bool, ByteArray, ByteArrayEncoded, Bytes31, Bytes31Encoded, ClassHash, + ContractAddress, Custom, Enum, EthAddress, Felt252, FixedArray, Nullable, + Option as TDOption, Result as TDResult, ShortUtf8, StorageAddress, StorageBaseAddress, + Struct, Tuple, Utf8String, I128, I16, I32, I64, I8, U128, U16, U256, U32, U512, U64, U8, + }; + if column.name != new_name { + queries.add(format!( + r#"ALTER TABLE "{table_name}" RENAME COLUMN "{}" TO "{new_name}";"#, + column.name + )); + column.name = new_name.to_string(); + } + let cast = match (&column.type_def, &new_type) { + (Bool | U8 | U16 | U32, Bool | U8 | U16 | U32 | I8 | I16 | I32 | I64) + | (I8 | I16 | I32 | I64, I8 | I16 | I32 | I64) + | (U64 | U128 | U256 | U512, U64 | U128 | U256 | U512 | I128) + | (I128, I128) + | ( + Felt252 | ClassHash | ContractAddress | EthAddress | StorageAddress + | StorageBaseAddress, + Felt252 | ClassHash | ContractAddress | EthAddress | StorageAddress + | StorageBaseAddress, + ) + | (ShortUtf8 | Utf8String, ShortUtf8 | Utf8String) + | ( + Bytes31 | Bytes31Encoded(_) | ByteArray | ByteArrayEncoded(_) | Custom(_), + Bytes31 | Bytes31Encoded(_) | ByteArray | ByteArrayEncoded(_) | Custom(_), + ) + | ( + Tuple(_) | Array(_) | FixedArray(_) | Struct(_) | Enum(_) | TDOption(_) | TDResult(_) + | Nullable(_), + Tuple(_) | Array(_) | FixedArray(_) | Struct(_) | Enum(_) | TDOption(_) | TDResult(_) + | Nullable(_), + ) => None, + (Bool | U8 | U16 | U32, U64 | U128 | U256 | U512 | I128) | (I8 | I16 | I32 | I64, I128) => { + Some(format!(r#"CAST("{new_name}" AS TEXT)"#)) + } + ( + Bool | U8 | U16 | U32, + Felt252 | ClassHash | ContractAddress | EthAddress | StorageAddress + | StorageBaseAddress, + ) => Some(format!(r#"printf('0x%064x', "{new_name}")"#)), + _ => return UpgradeError::type_upgrade_err(&column.type_def, new_type), + }; + column.type_def = new_type.clone(); + if let Some(cast) = cast { + queries.add(format!( + r#"UPDATE "{table_name}" SET "{new_name}" = {cast};"# + )); + } + Ok(()) +} + +pub fn create_column_query(table_name: &str, column: &ColumnDef) -> UpgradeResult { + let sql_type: SqliteType = column.try_into()?; + Ok(format!( + r#"ALTER TABLE "{table_name}" ADD COLUMN "{}" {sql_type};"#, + column.name + )) +} diff --git a/crates/introspect-sql-sink/src/sqlite/types.rs b/crates/introspect-sql-sink/src/sqlite/types.rs new file mode 100644 index 00000000..749e69eb --- /dev/null +++ b/crates/introspect-sql-sink/src/sqlite/types.rs @@ -0,0 +1,161 @@ +use std::fmt::Display; + +use introspect_types::{ColumnDef, ColumnInfo, PrimaryDef, PrimaryTypeDef, TypeDef}; + +use crate::{TypeError, TypeResult}; + +pub enum SqliteType { + Null, + Text, + Integer, + Real, + Blob, + Json, +} + +impl Display for SqliteType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SqliteType::Null => write!(f, "NULL"), + SqliteType::Text => write!(f, "TEXT"), + SqliteType::Integer => write!(f, "INTEGER"), + SqliteType::Real => write!(f, "REAL"), + SqliteType::Blob => write!(f, "BLOB"), + SqliteType::Json => write!(f, "TEXT"), + } + } +} + +pub struct SqliteColumn<'a> { + pub name: &'a str, + pub sql_type: SqliteType, +} + +impl TryFrom<&TypeDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &TypeDef) -> Result { + match value { + TypeDef::None => Ok(SqliteType::Null), + TypeDef::Bool + | TypeDef::U8 + | TypeDef::U16 + | TypeDef::U32 + | TypeDef::I8 + | TypeDef::I16 + | TypeDef::I32 + | TypeDef::I64 => Ok(SqliteType::Integer), + TypeDef::U64 | TypeDef::U128 | TypeDef::U256 | TypeDef::U512 | TypeDef::I128 => { + Ok(SqliteType::Text) + } + TypeDef::Felt252 + | TypeDef::ClassHash + | TypeDef::ContractAddress + | TypeDef::EthAddress + | TypeDef::StorageAddress + | TypeDef::StorageBaseAddress => Ok(SqliteType::Text), + TypeDef::ShortUtf8 | TypeDef::Utf8String => Ok(SqliteType::Text), + TypeDef::Bytes31 + | TypeDef::Bytes31Encoded(_) + | TypeDef::ByteArray + | TypeDef::ByteArrayEncoded(_) => Ok(SqliteType::Blob), + TypeDef::Struct(_) + | TypeDef::Enum(_) + | TypeDef::Tuple(_) + | TypeDef::Array(_) + | TypeDef::FixedArray(_) + | TypeDef::Option(_) + | TypeDef::Nullable(_) + | TypeDef::Result(_) => Ok(SqliteType::Json), + TypeDef::Custom(_) => Ok(SqliteType::Blob), + _ => Err(TypeError::UnsupportedType(value.item_name().to_string())), + } + } +} + +impl TryFrom<&ColumnDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &ColumnDef) -> Result { + (&value.type_def).try_into() + } +} + +impl TryFrom<&ColumnInfo> for SqliteType { + type Error = TypeError; + + fn try_from(value: &ColumnInfo) -> Result { + (&value.type_def).try_into() + } +} + +impl TryFrom<&PrimaryTypeDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &PrimaryTypeDef) -> Result { + (&Into::::into(value)).try_into() + } +} + +impl TryFrom<&PrimaryDef> for SqliteType { + type Error = TypeError; + + fn try_from(value: &PrimaryDef) -> Result { + (&value.type_def).try_into() + } +} + +impl<'a> TryFrom<&'a ColumnInfo> for SqliteColumn<'a> { + type Error = TypeError; + + fn try_from(value: &'a ColumnInfo) -> Result { + let sql_type = (&value.type_def).try_into()?; + Ok(SqliteColumn { + name: &value.name, + sql_type, + }) + } +} + +impl SqliteType { + pub fn placeholder(&self) -> &'static str { + match self { + SqliteType::Null => "NULL", + SqliteType::Text | SqliteType::Integer | SqliteType::Real | SqliteType::Blob => "?", + SqliteType::Json => "jsonb(?)", + } + } + pub fn index_placeholder(&self, index: usize) -> String { + match self { + SqliteType::Null => "NULL".to_string(), + SqliteType::Text | SqliteType::Integer | SqliteType::Real | SqliteType::Blob => { + format!("?{index}") + } + SqliteType::Json => format!("jsonb(?{index})"), + } + } +} + +impl<'a> SqliteColumn<'a> { + pub fn placeholder(&self) -> &'static str { + self.sql_type.placeholder() + } + pub fn index_placeholder(&self, index: usize) -> String { + self.sql_type.index_placeholder(index) + } +} + +pub trait TypeDefSqliteExt { + fn placeholder(&self) -> TypeResult<&'static str>; + fn index_placeholder(&self, index: usize) -> TypeResult; +} + +impl TypeDefSqliteExt for TypeDef { + fn placeholder(&self) -> TypeResult<&'static str> { + self.try_into().map(|t: SqliteType| t.placeholder()) + } + fn index_placeholder(&self, index: usize) -> TypeResult { + self.try_into() + .map(|t: SqliteType| t.index_placeholder(index)) + } +} diff --git a/crates/introspect-sql-sink/src/table.rs b/crates/introspect-sql-sink/src/table.rs new file mode 100644 index 00000000..7d9242ee --- /dev/null +++ b/crates/introspect-sql-sink/src/table.rs @@ -0,0 +1,137 @@ +use crate::error::{CollectColumnResults, ColumnNotFoundError, ColumnsNotFoundError}; +use introspect_types::{ColumnDef, ColumnInfo, MemberDef, PrimaryDef, TypeDef}; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::rc::Rc; +use torii_introspect::tables::RecordSchema; + +#[derive(Debug)] +pub struct Table { + pub id: Felt, + pub namespace: String, + pub name: String, + pub owner: Felt, + pub primary: ColumnInfo, + pub columns: HashMap, + pub dead: HashMap, + pub append_only: bool, + pub alive: bool, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeadField { + pub name: String, + pub type_def: TypeDef, +} + +#[derive(Debug)] +pub struct DeadFieldDef { + pub id: u128, + pub name: String, + pub type_def: TypeDef, +} + +impl From for DeadField { + fn from(value: MemberDef) -> Self { + DeadField { + name: value.name, + type_def: value.type_def, + } + } +} + +impl From for MemberDef { + fn from(value: DeadField) -> Self { + MemberDef { + name: value.name, + attributes: Vec::new(), + type_def: value.type_def, + } + } +} + +impl From for (u128, DeadField) { + fn from(value: DeadFieldDef) -> Self { + ( + value.id, + DeadField { + name: value.name, + type_def: value.type_def, + }, + ) + } +} + +impl From<(u128, DeadField)> for DeadFieldDef { + fn from(value: (u128, DeadField)) -> Self { + DeadFieldDef { + id: value.0, + name: value.1.name, + type_def: value.1.type_def, + } + } +} + +impl Table { + pub fn column(&self, id: &Felt) -> Result<&ColumnInfo, ColumnNotFoundError> { + self.columns.get(id).ok_or(ColumnNotFoundError(*id)) + } + + pub fn namespace(&self) -> Rc { + self.namespace.as_str().into() + } + + pub fn columns(&self, ids: &[Felt]) -> Result, ColumnsNotFoundError> { + ids.iter().map(|id| self.column(id)).collect_columns() + } + + pub fn all_columns(&self) -> Vec<&ColumnInfo> { + self.columns.values().collect() + } + + pub fn columns_with_ids<'a>( + &'a self, + ids: &'a [Felt], + ) -> Result, ColumnsNotFoundError> { + ids.iter() + .map(|id| self.column(id).map(|col| (id, col))) + .collect_columns() + } + + pub fn all_columns_with_ids(&self) -> Vec<(&Felt, &ColumnInfo)> { + self.columns.iter().collect() + } + + #[allow(clippy::too_many_arguments)] + pub fn new( + namespace: String, + id: Felt, + owner: Felt, + name: String, + primary: PrimaryDef, + columns: &[ColumnDef], + dead: Option>, + append_only: bool, + ) -> Self { + Table { + id, + namespace, + owner, + name, + primary: primary.into(), + columns: columns.iter().cloned().map_into().collect(), + dead: dead.unwrap_or_default().into_iter().collect(), + append_only, + alive: true, + } + } + + pub fn get_record_schema( + &self, + columns: &[Felt], + ) -> Result, ColumnsNotFoundError> { + Ok(RecordSchema::new(&self.primary, self.columns(columns)?)) + } +} diff --git a/crates/introspect-sql-sink/src/tables.rs b/crates/introspect-sql-sink/src/tables.rs new file mode 100644 index 00000000..95693cdb --- /dev/null +++ b/crates/introspect-sql-sink/src/tables.rs @@ -0,0 +1,155 @@ +use crate::backend::IntrospectQueryMaker; +use crate::error::RecordResultExt; +use crate::namespace::{NamespaceKey, TableKey}; +use crate::table::Table; +use crate::{DbError, DbResult}; +use introspect_types::{ColumnDef, PrimaryDef, ResultInto}; +use starknet_types_core::felt::Felt; +use std::collections::HashMap; +use std::ops::Deref; +use std::sync::RwLock; +use torii_introspect::InsertsFields; +use torii_sql::FlexQuery; + +#[derive(Debug, Default)] +pub struct Tables(pub RwLock>); + +impl Deref for Tables { + type Target = RwLock>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +#[allow(clippy::too_many_arguments)] +impl Tables { + pub fn create_table( + &self, + namespace_key: NamespaceKey, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + append_only: bool, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> DbResult<()> { + let namespace = namespace_key.to_string(); + + let key = TableKey::new(namespace_key, *id); + self.assert_table_not_exists(&key, name)?; + DB::create_table_queries( + &namespace, + id, + name, + primary, + columns, + append_only, + from_address, + block_number, + transaction_hash, + queries, + )?; + let mut tables = self.write()?; + tables.insert( + key, + Table::new( + namespace, + *id, + *from_address, + name.to_string(), + primary.clone(), + columns, + None, + append_only, + ), + ); + Ok(()) + } + + pub fn update_table( + &self, + namespace_key: NamespaceKey, + id: &Felt, + name: &str, + primary: &PrimaryDef, + columns: &[ColumnDef], + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> DbResult<()> { + let mut tables = self.write()?; + let key = TableKey::new(namespace_key, *id); + let table = tables + .get_mut(&key) + .ok_or_else(|| DbError::TableNotFound(key.clone()))?; + DB::update_table_queries( + table, + name, + primary, + columns, + from_address, + block_number, + transaction_hash, + queries, + ) + .err_into() + } + + pub fn assert_table_not_exists(&self, id: &TableKey, name: &str) -> DbResult<()> { + match self.read()?.get(id) { + Some(existing) => Err(DbError::TableAlreadyExists( + id.clone(), + name.to_string(), + existing.name.to_string(), + )), + None => Ok(()), + } + } + + pub fn set_table_dead(&self, namespace: NamespaceKey, id: Felt) -> DbResult<()> { + let mut tables = self.write()?; + let key = TableKey::new(namespace, id); + match tables.get_mut(&key) { + Some(table) => { + table.alive = false; + Ok(()) + } + None => Err(DbError::TableNotFound(key)), + } + } + + pub fn insert_fields( + &self, + namespace: NamespaceKey, + event: &InsertsFields, + from_address: &Felt, + block_number: u64, + transaction_hash: &Felt, + queries: &mut Vec>, + ) -> DbResult<()> { + let tables = self.read().unwrap(); + let key = TableKey::new(namespace, event.table); + let table = match tables.get(&key) { + Some(table) => Ok(table), + None => Err(DbError::TableNotFound(key)), + }?; + if !table.alive { + return Ok(()); + } + DB::insert_record_queries( + table, + &event.columns, + &event.records, + from_address, + block_number, + transaction_hash, + queries, + ) + .to_db_result(&table.name) + } +} diff --git a/crates/introspect-sqlite-sink/migrations/002_schema_state.sql b/crates/introspect-sqlite-sink/migrations/002_schema_state.sql deleted file mode 100644 index caa564d4..00000000 --- a/crates/introspect-sqlite-sink/migrations/002_schema_state.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE IF NOT EXISTS introspect_sink_schema_state ( - table_id TEXT PRIMARY KEY, - table_schema_json TEXT NOT NULL, - alive INTEGER NOT NULL DEFAULT 1, - updated_at INTEGER NOT NULL DEFAULT (unixepoch()) -); diff --git a/crates/introspect-sqlite-sink/src/lib.rs b/crates/introspect-sqlite-sink/src/lib.rs deleted file mode 100644 index ac41c665..00000000 --- a/crates/introspect-sqlite-sink/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod json; -pub mod processor; -pub mod sink; -pub mod table; - -use sqlx::migrate::Migrator; - -pub const INTROSPECT_SQLITE_SINK_MIGRATIONS: Migrator = sqlx::migrate!("./migrations"); diff --git a/crates/introspect-sqlite-sink/src/processor.rs b/crates/introspect-sqlite-sink/src/processor.rs deleted file mode 100644 index d3a00a93..00000000 --- a/crates/introspect-sqlite-sink/src/processor.rs +++ /dev/null @@ -1,650 +0,0 @@ -use crate::json::SqliteJsonSerializer; -use crate::table::{SqliteTable, SqliteTableError}; -use crate::INTROSPECT_SQLITE_SINK_MIGRATIONS; -use introspect_types::{PrimaryTypeDef, TypeDef}; -use serde_json::{Serializer as JsonSerializer, Value}; -use sqlx::Error as SqlxError; -use sqlx::Row; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use std::fmt::Display; -use std::ops::Deref; -use std::sync::{PoisonError, RwLock}; -use torii::etl::envelope::MetaData; -use torii::etl::EventMsg; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; -use torii_introspect::schema::TableSchema; -use torii_introspect::InsertsFields; -use torii_sqlite::SqliteConnection; - -#[derive(Debug, thiserror::Error)] -pub enum SqliteDbError { - #[error(transparent)] - DatabaseError(#[from] SqlxError), - #[error(transparent)] - JsonError(#[from] serde_json::Error), - #[error(transparent)] - TableError(#[from] SqliteTableError), - #[error("record frame must serialize to an object")] - InvalidRecordFrame, - #[error("Table with id: {0} already exists, incoming name: {1}, existing name: {2}")] - TableAlreadyExists(Felt, String, String), - #[error("Table not found with id: {0}")] - TableNotFound(Felt), - #[error("Table poison error: {0}")] - PoisonError(String), -} - -type SqliteDbResult = std::result::Result; - -impl From> for SqliteDbError { - fn from(err: PoisonError) -> Self { - Self::PoisonError(err.to_string()) - } -} - -#[derive(Debug, Default)] -pub struct SqliteTables(pub RwLock>); - -impl Deref for SqliteTables { - type Target = RwLock>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[derive(Debug, Clone, Default)] -pub enum SqliteNamespace { - #[default] - None, - Custom(String), -} - -impl SqliteNamespace { - pub fn prefix(&self) -> &str { - match self { - Self::None => "", - Self::Custom(prefix) => prefix, - } - } -} - -impl From<()> for SqliteNamespace { - fn from((): ()) -> Self { - Self::None - } -} - -impl From for SqliteNamespace { - fn from(value: String) -> Self { - if value.is_empty() { - Self::None - } else { - Self::Custom(value) - } - } -} - -impl From<&str> for SqliteNamespace { - fn from(value: &str) -> Self { - if value.is_empty() { - Self::None - } else { - Self::Custom(value.to_string()) - } - } -} - -impl Display for SqliteNamespace { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::None => f.write_str("main"), - Self::Custom(prefix) => f.write_str(prefix), - } - } -} - -impl SqliteTables { - pub fn assert_table_not_exists(&self, id: &Felt, name: &str) -> SqliteDbResult<()> { - match self.read()?.get(id) { - Some(existing) => Err(SqliteDbError::TableAlreadyExists( - *id, - name.to_string(), - existing.name.clone(), - )), - None => Ok(()), - } - } - - pub fn create_table( - &self, - namespace: &SqliteNamespace, - to_table: impl Into, - ) -> SqliteDbResult<(Felt, String)> { - let table = to_table.into(); - self.assert_table_not_exists(&table.id, &table.name)?; - let (id, sqlite_table) = SqliteTable::new_from_table(namespace.prefix(), table); - let create_query = create_table_query(&sqlite_table); - self.write()?.insert(id, sqlite_table); - Ok((id, create_query)) - } - - pub fn set_table_dead(&self, id: &Felt) -> SqliteDbResult<()> { - if let Some(table) = self.write()?.get_mut(id) { - table.alive = false; - return Ok(()); - } - Err(SqliteDbError::TableNotFound(*id)) - } -} - -fn sqlite_column_type(type_def: &TypeDef) -> &'static str { - if is_json_type(type_def) { - "JSONB" - } else if matches!( - type_def, - TypeDef::Bool - | TypeDef::I8 - | TypeDef::I16 - | TypeDef::I32 - | TypeDef::U8 - | TypeDef::U16 - | TypeDef::U32 - ) { - "INTEGER" - } else { - "TEXT" - } -} - -fn sqlite_primary_type(type_def: &PrimaryTypeDef) -> &'static str { - if matches!( - type_def, - PrimaryTypeDef::Bool - | PrimaryTypeDef::I8 - | PrimaryTypeDef::I16 - | PrimaryTypeDef::I32 - | PrimaryTypeDef::U8 - | PrimaryTypeDef::U16 - | PrimaryTypeDef::U32 - ) { - "INTEGER" - } else { - "TEXT" - } -} - -fn is_json_type(type_def: &TypeDef) -> bool { - matches!( - type_def, - TypeDef::Struct(_) - | TypeDef::Enum(_) - | TypeDef::Tuple(_) - | TypeDef::Array(_) - | TypeDef::FixedArray(_) - | TypeDef::Option(_) - | TypeDef::Nullable(_) - | TypeDef::Result(_) - ) -} - -fn create_table_query(table: &SqliteTable) -> String { - let primary_type = sqlite_primary_type(&table.primary.type_def); - let mut columns = Vec::with_capacity(table.columns.len() + 1); - columns.push(format!( - r#""{}" {primary_type} PRIMARY KEY"#, - table.primary.name - )); - for column_id in &table.order { - let column = &table.columns[column_id]; - let col_type = sqlite_column_type(&column.type_def); - columns.push(format!(r#""{}" {col_type}"#, column.name)); - } - format!( - r#"CREATE TABLE IF NOT EXISTS "{}" ({});"#, - table.storage_name, - columns.join(", ") - ) -} - -enum SqliteBindValue { - Null, - Integer(i64), - Text(String), -} - -fn to_bind_value(value: &Value, type_def: &TypeDef) -> SqliteBindValue { - if value.is_null() { - return SqliteBindValue::Null; - } - - match type_def { - TypeDef::Bool => match value.as_bool() { - Some(b) => SqliteBindValue::Integer(i64::from(b)), - None => SqliteBindValue::Null, - }, - TypeDef::I8 | TypeDef::I16 | TypeDef::I32 | TypeDef::U8 | TypeDef::U16 | TypeDef::U32 => { - match value.as_i64() { - Some(n) => SqliteBindValue::Integer(n), - None => SqliteBindValue::Null, - } - } - TypeDef::U64 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - TypeDef::I64 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - TypeDef::Felt252 - | TypeDef::ClassHash - | TypeDef::ContractAddress - | TypeDef::StorageAddress - | TypeDef::StorageBaseAddress - | TypeDef::EthAddress - | TypeDef::U256 - | TypeDef::U512 => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - - TypeDef::U128 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{:032x}", n as u128)), - None => match value.as_str() { - Some(s) => match s.parse::() { - Ok(n) => SqliteBindValue::Text(format!("0x{n:032x}")), - Err(_) => SqliteBindValue::Text(s.to_string()), - }, - None => SqliteBindValue::Null, - }, - }, - TypeDef::I128 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("{}", n as i128)), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - TypeDef::Struct(_) - | TypeDef::Enum(_) - | TypeDef::Tuple(_) - | TypeDef::Array(_) - | TypeDef::FixedArray(_) - | TypeDef::Option(_) - | TypeDef::Nullable(_) - | TypeDef::Result(_) => SqliteBindValue::Text(value.to_string()), - - _ => match value { - Value::String(s) => SqliteBindValue::Text(s.clone()), - _ => SqliteBindValue::Text(value.to_string()), - }, - } -} - -fn primary_to_bind_value(value: &Value, type_def: &PrimaryTypeDef) -> SqliteBindValue { - if value.is_null() { - return SqliteBindValue::Null; - } - - match type_def { - PrimaryTypeDef::Bool => match value.as_bool() { - Some(b) => SqliteBindValue::Integer(i64::from(b)), - None => SqliteBindValue::Null, - }, - PrimaryTypeDef::I8 - | PrimaryTypeDef::I16 - | PrimaryTypeDef::I32 - | PrimaryTypeDef::U8 - | PrimaryTypeDef::U16 - | PrimaryTypeDef::U32 => match value.as_i64() { - Some(n) => SqliteBindValue::Integer(n), - None => SqliteBindValue::Null, - }, - PrimaryTypeDef::U64 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - PrimaryTypeDef::I64 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("0x{n:x}")), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - PrimaryTypeDef::Felt252 - | PrimaryTypeDef::ClassHash - | PrimaryTypeDef::ContractAddress - | PrimaryTypeDef::StorageAddress - | PrimaryTypeDef::StorageBaseAddress - | PrimaryTypeDef::EthAddress => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - - PrimaryTypeDef::U128 => match value.as_u64() { - Some(n) => SqliteBindValue::Text(format!("0x{:032x}", n as u128)), - None => match value.as_str() { - Some(s) => match s.parse::() { - Ok(n) => SqliteBindValue::Text(format!("0x{n:032x}")), - Err(_) => SqliteBindValue::Text(s.to_string()), - }, - None => SqliteBindValue::Null, - }, - }, - PrimaryTypeDef::I128 => match value.as_i64() { - Some(n) => SqliteBindValue::Text(format!("{}", n as i128)), - None => match value.as_str() { - Some(s) => SqliteBindValue::Text(s.to_string()), - None => SqliteBindValue::Null, - }, - }, - - _ => match value { - Value::String(s) => SqliteBindValue::Text(s.clone()), - _ => SqliteBindValue::Text(value.to_string()), - }, - } -} - -pub struct IntrospectSqliteDb { - tables: SqliteTables, - namespace: SqliteNamespace, - pool: T, -} - -impl SqliteConnection for IntrospectSqliteDb { - fn pool(&self) -> &sqlx::SqlitePool { - self.pool.pool() - } -} - -impl IntrospectSqliteDb { - pub fn new(pool: T, namespace: impl Into) -> Self { - Self { - tables: SqliteTables::default(), - namespace: namespace.into(), - pool, - } - } - - pub async fn initialize_introspect_sqlite_sink(&self) -> SqliteDbResult<()> { - self.migrate(Some("introspect"), INTROSPECT_SQLITE_SINK_MIGRATIONS) - .await?; - self.load_persisted_state().await?; - Ok(()) - } - - async fn load_persisted_state(&self) -> SqliteDbResult<()> { - let rows = sqlx::query( - r" - SELECT table_schema_json, alive - FROM introspect_sink_schema_state - ORDER BY updated_at ASC - ", - ) - .fetch_all(self.pool()) - .await?; - - let mut tables = self.tables.write()?; - for row in rows { - let schema_json: String = row.try_get("table_schema_json")?; - let alive: i64 = row.try_get("alive")?; - let table_schema: TableSchema = serde_json::from_str(&schema_json)?; - let (id, mut table) = - SqliteTable::new_from_table(self.namespace.prefix(), table_schema); - table.alive = alive != 0; - tables.insert(id, table); - } - - Ok(()) - } - - async fn persist_table_state(&self, table: &TableSchema, alive: bool) -> SqliteDbResult<()> { - let schema_json = serde_json::to_string(table)?; - let alive = i64::from(alive); - sqlx::query( - r" - INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) - VALUES (?1, ?2, ?3, unixepoch()) - ON CONFLICT (table_id) - DO UPDATE SET - table_schema_json = excluded.table_schema_json, - alive = excluded.alive, - updated_at = unixepoch() - ", - ) - .bind(format!("{:#x}", table.id)) - .bind(schema_json) - .bind(alive) - .execute(self.pool()) - .await?; - Ok(()) - } - - async fn update_table(&self, event: impl Into) -> SqliteDbResult<()> { - let table_schema: TableSchema = event.into(); - let id = table_schema.id; - let exists_in_memory = self.tables.read()?.contains_key(&id); - - if !exists_in_memory { - let (_, query) = self - .tables - .create_table(&self.namespace, table_schema.clone())?; - self.execute_queries(&[query]).await?; - self.persist_table_state(&table_schema, true).await?; - return Ok(()); - } - - let (old_columns, storage_name) = { - let tables = self.tables.read()?; - let old = tables.get(&id).unwrap(); - (old.columns.clone(), old.storage_name.clone()) - }; - - let (_, new_table) = - SqliteTable::new_from_table(self.namespace.prefix(), table_schema.clone()); - - let mut alter_queries = Vec::new(); - for (col_id, col_info) in &new_table.columns { - if !old_columns.contains_key(col_id) { - let col_type = sqlite_column_type(&col_info.type_def); - alter_queries.push(format!( - r#"ALTER TABLE "{storage_name}" ADD COLUMN "{}" {col_type}"#, - col_info.name - )); - } - } - - if !alter_queries.is_empty() { - self.execute_queries(&alter_queries).await?; - } - - self.tables.write()?.insert(id, new_table); - self.persist_table_state(&table_schema, true).await?; - Ok(()) - } - - pub fn load_tables_no_commit(&self, table_schemas: Vec) -> SqliteDbResult<()> { - let mut tables = self.tables.write()?; - for table in table_schemas { - let (id, sqlite_table) = SqliteTable::new_from_table(self.namespace.prefix(), table); - tables.insert(id, sqlite_table); - } - Ok(()) - } - - pub async fn process_message( - &self, - msg: &IntrospectMsg, - metadata: &MetaData, - ) -> SqliteDbResult<()> { - match msg { - IntrospectMsg::CreateTable(event) => { - let (_, query) = self.tables.create_table(&self.namespace, event.clone())?; - self.execute_queries(&[query]).await?; - self.persist_table_state(&event.clone().into(), true) - .await?; - Ok(()) - } - IntrospectMsg::UpdateTable(event) => self.update_table(event.clone()).await, - IntrospectMsg::AddColumns(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "AddColumns received — table kept alive, new columns ignored until next UpdateTable" - ); - Ok(()) - } - IntrospectMsg::DropColumns(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "DropColumns received — table kept alive, columns left in place" - ); - Ok(()) - } - IntrospectMsg::RetypeColumns(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "RetypeColumns received — table kept alive, types unchanged" - ); - Ok(()) - } - IntrospectMsg::RetypePrimary(event) => { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - table = %event.table, - "RetypePrimary received — table kept alive, primary type unchanged" - ); - Ok(()) - } - IntrospectMsg::RenameTable(_) - | IntrospectMsg::DropTable(_) - | IntrospectMsg::RenameColumns(_) - | IntrospectMsg::RenamePrimary(_) - | IntrospectMsg::DeleteRecords(_) - | IntrospectMsg::DeletesFields(_) => Ok(()), - IntrospectMsg::InsertsFields(event) => self.insert_fields(event, metadata).await, - } - } - - pub async fn process_messages( - &self, - msgs: Vec<&IntrospectBody>, - ) -> SqliteDbResult>> { - let mut results = Vec::with_capacity(msgs.len()); - for body in msgs { - let (msg, metadata) = body.into(); - let result = self.process_message(msg, metadata).await; - if let Err(ref err) = result { - tracing::warn!( - target: "torii::introspect_sqlite_sink", - event_id = msg.event_id(), - error = %err, - "Failed to process introspect message" - ); - } - results.push(result); - } - Ok(results) - } - - async fn insert_fields( - &self, - event: &InsertsFields, - _metadata: &MetaData, - ) -> SqliteDbResult<()> { - let table = self - .tables - .read()? - .get(&event.table) - .ok_or(SqliteDbError::TableNotFound(event.table))? - .clone(); - if !table.alive { - return Ok(()); - } - - let record_schema = table.get_schema(&event.columns)?; - let column_names = std::iter::once(table.primary.name.as_str()) - .chain( - event - .columns - .iter() - .map(|id| table.columns[id].name.as_str()), - ) - .collect::>(); - - let column_type_defs: Vec<&TypeDef> = event - .columns - .iter() - .map(|id| &table.columns[id].type_def) - .collect(); - - let mut bytes = Vec::new(); - let mut serializer = JsonSerializer::new(&mut bytes); - record_schema.parse_records_with_metadata( - &event.records, - &(), - &mut serializer, - &SqliteJsonSerializer, - )?; - let rows = serde_json::from_slice::>(&bytes)?; - - let mut tx = self.begin().await?; - for value in rows { - let object = value.as_object().ok_or(SqliteDbError::InvalidRecordFrame)?; - - let mut query = sqlx::query(&table.upsert_sql); - - let primary_value = object - .get(table.primary.name.as_str()) - .cloned() - .unwrap_or(Value::Null); - match primary_to_bind_value(&primary_value, &table.primary.type_def) { - SqliteBindValue::Null => { - query = query.bind(None::>); - } - SqliteBindValue::Integer(n) => { - query = query.bind(n); - } - SqliteBindValue::Text(s) => { - query = query.bind(s); - } - } - - for (column_name, type_def) in column_names.iter().skip(1).zip(column_type_defs.iter()) - { - let val = object.get(*column_name).cloned().unwrap_or(Value::Null); - match to_bind_value(&val, type_def) { - SqliteBindValue::Null => { - query = query.bind(None::); - } - SqliteBindValue::Integer(n) => { - query = query.bind(n); - } - SqliteBindValue::Text(s) => { - query = query.bind(s); - } - } - } - query.execute(&mut *tx).await?; - } - tx.commit().await?; - Ok(()) - } -} diff --git a/crates/introspect-sqlite-sink/src/table.rs b/crates/introspect-sqlite-sink/src/table.rs deleted file mode 100644 index 8bbb653e..00000000 --- a/crates/introspect-sqlite-sink/src/table.rs +++ /dev/null @@ -1,151 +0,0 @@ -use introspect_types::{ColumnDef, ColumnInfo, FeltIds, PrimaryDef}; -use itertools::Itertools; -use starknet_types_core::felt::Felt; -use std::collections::HashMap; -use thiserror::Error; -use torii_introspect::schema::TableSchema; -use torii_introspect::tables::RecordSchema; - -#[derive(Debug, Error)] -pub enum SqliteTableError { - #[error("Column with id: {0} not found in table {1}")] - ColumnNotFound(Felt, String), -} - -pub type TableResult = std::result::Result; - -#[derive(Debug, Clone)] -pub struct SqliteTable { - pub name: String, - pub storage_name: String, - pub primary: PrimaryDef, - pub columns: HashMap, - pub order: Vec, - pub upsert_sql: String, - pub alive: bool, -} - -impl SqliteTable { - pub fn new( - storage_name: String, - name: String, - primary: PrimaryDef, - columns: Vec, - ) -> Self { - Self { - name, - storage_name, - primary, - order: columns.ids(), - columns: columns.into_iter().map_into().collect(), - upsert_sql: String::new(), - alive: true, - } - .with_upsert_sql() - } - - pub fn new_from_table(namespace: &str, table: impl Into) -> (Felt, Self) { - let table = table.into(); - let storage_name = if namespace.is_empty() { - table.name.clone() - } else { - format!("{namespace}__{}", table.name) - }; - ( - table.id, - Self::new(storage_name, table.name, table.primary, table.columns), - ) - } - - pub fn get_column(&self, selector: &Felt) -> TableResult<&ColumnInfo> { - self.columns - .get(selector) - .ok_or_else(|| SqliteTableError::ColumnNotFound(*selector, self.name.clone())) - } - - pub fn get_schema(&self, column_ids: &[Felt]) -> TableResult> { - let columns = column_ids - .iter() - .map(|selector| self.get_column(selector)) - .collect::, _>>()?; - Ok(RecordSchema::new(&self.primary, columns)) - } - - fn with_upsert_sql(mut self) -> Self { - self.upsert_sql = build_upsert_sql(&self); - self - } -} - -fn sqlite_column_type(type_def: &introspect_types::TypeDef) -> &'static str { - if matches!( - type_def, - introspect_types::TypeDef::Struct(_) - | introspect_types::TypeDef::Enum(_) - | introspect_types::TypeDef::Tuple(_) - | introspect_types::TypeDef::Array(_) - | introspect_types::TypeDef::FixedArray(_) - | introspect_types::TypeDef::Option(_) - | introspect_types::TypeDef::Nullable(_) - | introspect_types::TypeDef::Result(_) - ) { - "JSONB" - } else { - "" - } -} - -fn build_upsert_sql(table: &SqliteTable) -> String { - let column_names = std::iter::once(table.primary.name.as_str()) - .chain(table.order.iter().map(|id| table.columns[id].name.as_str())) - .collect::>(); - let column_type_defs = table - .order - .iter() - .map(|id| &table.columns[id].type_def) - .collect::>(); - - let placeholders = std::iter::once("?".to_string()) - .chain(column_type_defs.iter().map(|td| { - if sqlite_column_type(td) == "JSONB" { - "jsonb(?)".to_string() - } else { - "?".to_string() - } - })) - .collect::>() - .join(", "); - - let update_columns = column_names - .iter() - .skip(1) - .zip(column_type_defs.iter()) - .map(|(name, td)| { - if sqlite_column_type(td) == "JSONB" { - format!( - r#""{name}" = COALESCE(jsonb(excluded."{name}"), "{table_name}"."{name}")"#, - table_name = table.storage_name - ) - } else { - format!( - r#""{name}" = COALESCE(excluded."{name}", "{table_name}"."{name}")"#, - table_name = table.storage_name - ) - } - }) - .collect::>() - .join(", "); - - format!( - r#"INSERT INTO "{}" ({}) VALUES ({}) ON CONFLICT("{}") DO UPDATE SET {}"#, - table.storage_name, - column_names - .iter() - .map(|name| format!(r#""{name}""#)) - .collect::>() - .join(", "), - placeholders, - table.primary.name, - update_columns - ) -} diff --git a/crates/introspect/Cargo.toml b/crates/introspect/Cargo.toml index 4e00c48e..e9fbcd5a 100644 --- a/crates/introspect/Cargo.toml +++ b/crates/introspect/Cargo.toml @@ -10,8 +10,8 @@ introspect-types.workspace = true introspect-rust-macros.workspace = true serde.workspace = true serde_json.workspace = true -starknet.workspace = true starknet-types-core.workspace = true +starknet-types-raw.workspace = true torii.workspace = true blake3.workspace = true thiserror.workspace = true @@ -28,6 +28,7 @@ async-trait.workspace = true bigdecimal.workspace = true torii-common.workspace = true +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } [build-dependencies] tonic-build = "0.12" diff --git a/crates/introspect/README.md b/crates/introspect/README.md new file mode 100644 index 00000000..74a2c592 --- /dev/null +++ b/crates/introspect/README.md @@ -0,0 +1,81 @@ +# torii-introspect + +The **schema event vocabulary** shared between `torii-dojo` and every +introspect-aware sink. No decoding, no persistence — just the data types +that describe a table being created, renamed, retyped, repopulated, or +deleted at runtime. + +## Role in Torii + +`DojoDecoder` (`crates/dojo/src/decoder.rs`) takes Cairo-level Dojo events +and translates them into this crate's `IntrospectMsg` enum. The resulting +envelopes are then consumed by `introspect-sql-sink` (materialises the +tables to SQL), `torii-ecs-sink` (classifies as entity vs event-message), +and `arcade-sink` (projects into Arcade models). Because everybody agrees +on this vocabulary, the same event stream fans out to all three without +sharing any code. + +## Architecture + +```text +Cairo events from the Dojo world contract + | + v + torii-dojo::DojoDecoder + | + v IntrospectMsg (13 variants, see below) + | + +---> introspect-sql-sink (DDL + records) + +---> torii-ecs-sink (entity/event-message classification) + +---> arcade-sink (Arcade projections) + ++---------------------------------------------------------+ +| IntrospectMsg enum | +| | +| Schema lifecycle Column lifecycle Record lifecyc.| +| - CreateTable - AddColumns - InsertsFields| +| - UpdateTable - RenameColumns - DeleteRecords| +| - RenameTable - RetypeColumns - DeletesFields| +| - RenamePrimary - DropColumns | +| - RetypePrimary - DropTable | ++---------------------------------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `IntrospectMsg` (13 variants) | `src/events.rs` | 16 | The union type every Dojo-aware sink pattern-matches | +| `IntrospectBody = EventBody` | `src/events.rs` | 32 | Convenience alias for the envelope body | +| `EventId` (trait) | `src/events.rs` | 34 | Every variant has a stable `event_id()` for dedup | +| `CreateTable` | `src/events.rs` | 69 | `id`, `name`, `attributes`, `primary`, `columns`, `append_only` | +| `UpdateTable` | `src/events.rs` | 79 | Same as `CreateTable` minus `append_only` | +| `RenameTable`, `RenamePrimary`, `RetypePrimary`, `RenameColumns`, `RetypeColumns`, `AddColumns`, `DropColumns`, `DropTable` | `src/events.rs` | 88–128 | Schema mutations | +| `InsertsFields`, `Record` | `src/events.rs` | 144 / 138 | Bulk insert of rows (row id + raw serialized values) | +| `DeleteRecords`, `DeletesFields` | `src/events.rs` | 151 / 157 | Row and field deletions | +| `ColumnKey` | `src/schema.rs` | — | Newtype wrapping column identity | +| `TableSchema` (`From`, `From`) | `src/schema.rs` | — | Non-event-shaped view of a table (id + name + attrs + primary + columns) used internally | + +### Internal Modules + +- `events` — `IntrospectMsg` + 13 variant structs + their `EventId` impls + `EventMsg` impl (`TypeId::new("introspect")`). +- `schema` — `TableSchema` + `ColumnKey` newtype + conversions to/from `CreateTable`/`UpdateTable`. +- `tables` — multi-table cache helpers (used by the SQL sink). +- `types` — `TypeLibrary` trait for cairo type reference expansion. +- `store` — trait stub (no persistence lives here). +- `manager` — reserved for a future schema manager facade; currently inert. +- `postgres/` — PostgreSQL-specific type mappers; consumed by `introspect-sql-sink` when the `postgres` feature is on. + +### Interactions + +- **Upstream (consumers)**: `torii-dojo` (emits `IntrospectMsg` inside `DojoEvent::Introspect`), `introspect-sql-sink` (materialises), `torii-ecs-sink` (classifies), `arcade-sink` (projects). +- **Downstream deps**: `torii`, `torii-common`, `torii-sql`, `introspect-events`, `introspect-rust-macros`, `introspect-types`, `starknet-types-raw`, `starknet-types-core`, `serde`, `sqlx` (only when feature `postgres` is on). +- **Features**: `postgres` — pulls in `sqlx` and compiles the `postgres/` mappers. + +### Extension Points + +- New schema event → add a variant to `IntrospectMsg` + `EventId` impl + bump every matching sink. Because `IntrospectMsg` is an exhaustive enum, the compiler enforces the fan-out. +- New column attribute → extend `introspect-types::Attribute` (external crate); no change needed here. +- The type itself is ABI-stable on the wire — changing variant names or orderings breaks every sink. diff --git a/crates/introspect/src/events.rs b/crates/introspect/src/events.rs index c4797559..591e4de1 100644 --- a/crates/introspect/src/events.rs +++ b/crates/introspect/src/events.rs @@ -12,7 +12,7 @@ use torii::etl::{EventBody, TypeId}; use crate::schema::TableSchema; -#[derive(EnumFrom, Debug)] +#[derive(EnumFrom, Debug, Clone)] pub enum IntrospectMsg { CreateTable(CreateTable), UpdateTable(UpdateTable), @@ -72,6 +72,7 @@ pub struct CreateTable { pub attributes: Vec, pub primary: PrimaryDef, pub columns: Vec, + pub append_only: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -288,6 +289,7 @@ impl From for CreateTable { attributes: schema.attributes, primary: schema.primary, columns: schema.columns, + append_only: false, } } } @@ -351,6 +353,19 @@ impl InsertsFields { } } +impl CreateTable { + pub fn from_schema(schema: TableSchema, append_only: bool) -> Self { + Self { + id: schema.id, + name: schema.name, + attributes: schema.attributes, + primary: schema.primary, + columns: schema.columns, + append_only, + } + } +} + impl DeleteRecords { pub fn new(table: Felt, rows: Vec) -> Self { Self { table, rows } @@ -382,3 +397,9 @@ impl ToKeyBytes for Felt { self.to_bytes_be() } } + +impl ToKeyBytes for [u8; 32] { + fn to_key_bytes(&self) -> [u8; 32] { + *self + } +} diff --git a/crates/introspect/src/postgres/global.rs b/crates/introspect/src/postgres/global.rs index 41f23449..d651ce22 100644 --- a/crates/introspect/src/postgres/global.rs +++ b/crates/introspect/src/postgres/global.rs @@ -6,7 +6,7 @@ use itertools::Itertools; use sqlx::types::Json; use sqlx::{FromRow, PgPool}; use starknet_types_core::felt::Felt; -use torii_common::sql::SqlxResult; +use torii_sql::SqlxResult; use crate::postgres::{attribute_type, felt252_type, string_type, PgAttribute, PgFelt}; diff --git a/crates/introspect/src/postgres/owned.rs b/crates/introspect/src/postgres/owned.rs index c781c520..bc0c2521 100644 --- a/crates/introspect/src/postgres/owned.rs +++ b/crates/introspect/src/postgres/owned.rs @@ -10,7 +10,7 @@ use sqlx::types::Json; use sqlx::{FromRow, PgPool, Postgres}; use starknet_types_core::felt::Felt; use std::collections::HashMap; -use torii_common::sql::SqlxResult; +use torii_sql::SqlxResult; pub const TABLE_INSERT_QUERY: &str = " INSERT INTO introspect.tables (owner, id, name, attributes, primary_def, column_ids, updated_at, created_block, updated_block, created_tx, updated_tx) diff --git a/crates/introspect/src/postgres/types.rs b/crates/introspect/src/postgres/types.rs index b06dced8..70d75b8b 100644 --- a/crates/introspect/src/postgres/types.rs +++ b/crates/introspect/src/postgres/types.rs @@ -2,13 +2,11 @@ use std::fmt::Display; use introspect_types::{Attribute, PrimaryDef, PrimaryTypeDef}; use itertools::Itertools; -use sqlx::{ - encode::IsNull, - error::BoxDynError, - postgres::{PgArgumentBuffer, PgHasArrayType, PgTypeInfo, PgValueRef}, - types::{BigDecimal, Json}, - Decode, Encode, Postgres, Type, -}; +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::postgres::{PgArgumentBuffer, PgHasArrayType, PgTypeInfo, PgValueRef}; +use sqlx::types::{BigDecimal, Json}; +use sqlx::{Decode, Encode, Postgres, Type}; use starknet_types_core::felt::Felt; #[derive(sqlx::Type, Debug)] diff --git a/crates/introspect/src/schema.rs b/crates/introspect/src/schema.rs index 3586e6d7..a521a6ef 100644 --- a/crates/introspect/src/schema.rs +++ b/crates/introspect/src/schema.rs @@ -1,9 +1,6 @@ -use std::collections::HashMap; - use introspect_types::{Attribute, ColumnDef, PrimaryDef}; -use starknet::core::types::EmittedEvent; use starknet_types_core::felt::Felt; -use torii::etl::EventContext; +use std::collections::HashMap; #[derive(PartialEq, Eq, Hash, Clone, Copy)] pub struct ColumnKey { @@ -76,48 +73,6 @@ pub struct Table { pub order: Vec, pub alive: bool, } - -pub trait TableMetadata { - fn block_number(&self) -> u64; - fn tx_hash(&self) -> &Felt; -} - -impl TableMetadata for EmittedEvent { - fn block_number(&self) -> u64 { - self.block_number.unwrap_or_default() - } - fn tx_hash(&self) -> &Felt { - &self.transaction_hash - } -} - -impl TableMetadata for EventContext { - fn block_number(&self) -> u64 { - self.block.number - } - fn tx_hash(&self) -> &Felt { - &self.transaction.hash - } -} - -impl TableMetadata for (u64, Felt) { - fn block_number(&self) -> u64 { - self.0 - } - fn tx_hash(&self) -> &Felt { - &self.1 - } -} - -impl TableMetadata for (u64, &Felt) { - fn block_number(&self) -> u64 { - self.0 - } - fn tx_hash(&self) -> &Felt { - self.1 - } -} - // impl From<(Felt, TableInfo)> for TableSchema { // fn from(value: (Felt, TableInfo)) -> Self { // let (id, info) = value; diff --git a/crates/introspect/src/tables.rs b/crates/introspect/src/tables.rs index 43775d93..be081a9c 100644 --- a/crates/introspect/src/tables.rs +++ b/crates/introspect/src/tables.rs @@ -1,34 +1,40 @@ use crate::Record; use introspect_types::bytes::IntoByteSource; +use introspect_types::schema::Names; use introspect_types::serialize::CairoSeFrom; use introspect_types::serialize_def::CairoTypeSerialization; use introspect_types::{CairoDeserializer, ColumnInfo, PrimaryDef, PrimaryTypeDef, TypeDef}; use serde::ser::{SerializeMap, SerializeSeq}; use serde::{Serialize, Serializer}; -use starknet_types_core::felt::Felt; use std::ops::Deref; -use torii::etl::envelope::MetaData; pub struct RecordSchema<'a> { - primary: &'a PrimaryDef, + primary: &'a ColumnInfo, columns: Vec<&'a ColumnInfo>, } impl<'a> RecordSchema<'a> { - pub fn new(primary: &'a PrimaryDef, columns: Vec<&'a ColumnInfo>) -> Self { + pub fn new(primary: &'a ColumnInfo, columns: Vec<&'a ColumnInfo>) -> Self { Self { primary, columns } } pub fn columns(&self) -> &[&'a ColumnInfo] { &self.columns } - + pub fn all_columns(&self) -> Vec<&'a ColumnInfo> { + std::iter::once(self.primary) + .chain(self.columns.iter().copied()) + .collect() + } pub fn column_names(&self) -> Vec<&str> { - self.columns.iter().map(|col| col.name.as_str()).collect() + self.columns.names() } - pub fn primary(&self) -> &PrimaryDef { + pub fn primary(&self) -> &ColumnInfo { self.primary } + pub fn primary_type_def(&self) -> &TypeDef { + &self.primary.type_def + } pub fn primary_name(&self) -> &str { &self.primary.name @@ -132,7 +138,7 @@ impl<'a, M: SerializeEntries> RecordWithMetadata<'a, M> { } pub struct RecordFrame<'a, C: CairoTypeSerialization, M: SerializeEntries> { - primary: &'a PrimaryDef, + primary: &'a ColumnInfo, columns: &'a [&'a ColumnInfo], id: &'a [u8; 32], values: &'a [u8], @@ -161,7 +167,7 @@ impl<'a, C: CairoTypeSerialization, M: SerializeEntries> RecordFrame<'a, C, M> { &self, map: &mut ::SerializeMap, ) -> Result<(), S::Error> { - let mut id: introspect_types::bytes::DerefBytesSource<&[u8]> = self.id.into_source(); + let mut id = self.id.into_source(); map.serialize_entry( &self.primary.name, &CairoSeFrom::new(&self.primary.type_def, &mut id, self.cairo_se), @@ -215,23 +221,3 @@ impl SerializeEntries for () { Ok(()) } } - -pub fn pg_json_felt252(value: &Felt) -> String { - format!("\\x{}", hex::encode(value.to_bytes_be())) -} - -impl SerializeEntries for MetaData { - fn entry_count(&self) -> usize { - 4 - } - fn serialize_entries( - &self, - map: &mut ::SerializeMap, - ) -> Result<(), S::Error> { - let tx_hash = pg_json_felt252(&self.transaction_hash); - map.serialize_entry("__created_block", &self.block_number)?; - map.serialize_entry("__updated_block", &self.block_number)?; - map.serialize_entry("__created_tx", &tx_hash)?; - map.serialize_entry("__updated_tx", &tx_hash) - } -} diff --git a/crates/pathfinder/Cargo.toml b/crates/pathfinder/Cargo.toml index 35a1306b..ed3778e2 100644 --- a/crates/pathfinder/Cargo.toml +++ b/crates/pathfinder/Cargo.toml @@ -5,22 +5,22 @@ edition = "2021" description = "Pathfinding utilities for Torii" [dependencies] -starknet.workspace = true -rusqlite = { version = "0.32.1", features = ["bundled"] } +rusqlite = { version = "0.32.1" } zstd = { version = "0.13.2", features = ["experimental"] } serde.workspace = true bincode = { workspace = true, features = ["serde"] } thiserror.workspace = true rand = "0.8.5" -torii-starknet.workspace = true +starknet-types-raw = { workspace = true, features = ["serde"] } +torii-types.workspace = true # Etl dependencies -torii = { workspace = true, optional = true } -async-trait = { workspace = true, optional = true } anyhow = { workspace = true, optional = true } +async-trait = { workspace = true, optional = true } +torii = { workspace = true, optional = true } [lints] workspace = true [features] -etl = ["torii", "async-trait", "anyhow", "torii-starknet/starknet"] +etl = ["dep:async-trait", "dep:anyhow", "dep:torii"] diff --git a/crates/pathfinder/README.md b/crates/pathfinder/README.md new file mode 100644 index 00000000..33cb2e01 --- /dev/null +++ b/crates/pathfinder/README.md @@ -0,0 +1,90 @@ +# torii-pathfinder + +Reads Starknet events directly from a **Pathfinder SQLite database**. Used +when indexing from an archival Pathfinder node is faster than calling +`starknet_getEvents` over RPC — typically for backfills. Under the `etl` +feature it also exposes an `Extractor` implementation so Torii can plug it +into the normal ETL loop. + +## Role in Torii + +Ordinary binaries read from an RPC endpoint via `BlockRangeExtractor` or +`EventExtractor` (`src/etl/extractor/`). For bulk historical ingests, where +going through RPC is prohibitively slow, this crate lets a binary point at +a local Pathfinder SQLite DB and stream events out directly — decoding the +binary event-column format Pathfinder uses (zstd-compressed bincode over +felt arrays). + +## Architecture + +```text ++------------------------------------------------------+ +| torii-pathfinder | +| | +| utils.rs sqlite.rs | +| +----------+ +-------------------------+ | +| | connect()|----->| SqliteExt trait | | +| | (rusqlite) | - get_block_context_rows| | +| +----------+ | - get_block_events_rows | | +| | - get_block_tx_hash_rows| | +| +-------------------------+ | +| | | +| v | +| decoding.rs | +| +----------------+ zstd-decompress + | +| | BlockEvents | bincode-decode + | +| | BlockTxHashes | felt-array parsing | +| +----------------+ | +| | | +| v | +| fetcher.rs | +| +---------------------------------+ | +| | EventFetcher (trait) | | +| | get_events(from, to) | | +| | get_events_with_context(...) | | +| | impl EventFetcher for Connection| | +| +---------------------------------+ | +| | | +| v (feature = "etl") | +| extractor.rs | +| +---------------------------------+ | +| | PathfinderExtractor | | +| | PathfinderCombinedExtractor |--- impl Extractor +| +---------------------------------+ | ++------------------------------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `connect(path)` | `src/utils.rs` | — | Open a `rusqlite::Connection` with the read-only pragmas Pathfinder expects | +| `EventFetcher` trait | `src/fetcher.rs` | 19 | `get_events(from, to)` / `get_events_with_context(from, to)` — blanket impl for `rusqlite::Connection` | +| `BlockEvents` / `BlockTxHashes` | `src/decoding.rs` | — | Zstd-bincode decoders for Pathfinder's blob columns | +| `BlockContextRow` | `src/sqlite.rs` | — | Row shape for `block_headers` | +| `SqliteExt` trait | `src/sqlite.rs` | — | Raw SQL readers used by `EventFetcher` | +| `PathfinderExtractor` | `src/extractor.rs` (feature `etl`) | 8 | Stateful block-range extractor keeping a `Mutex` | +| `PathfinderCombinedExtractor` | `src/extractor.rs` (feature `etl`) | 17 | Switches from the Pathfinder DB to a live-head extractor `T` once caught up | +| `PFError` / `PFResult` | `src/error.rs` | — | Domain error covering IO, SQLite, bincode, zstd, and invariant failures | + +### Internal Modules + +- `decoding` — zstd + bincode felt-array parsing. +- `error` — `PFError` (with typed constructors) and `PFResult`. +- `fetcher` — `EventFetcher` trait and its blanket impl over `rusqlite::Connection`. +- `sqlite` — raw SQL extensions and row structs. +- `utils` — `connect` (read-only open). +- `extractor` (feature `etl`) — `PathfinderExtractor` + combined variant that swaps to a live extractor once the range is exhausted. + +### Interactions + +- **Upstream (consumers)**: `examples/pathfinder/main.rs` demonstrates usage; binaries that want Pathfinder backfill opt in by enabling the `etl` feature. +- **Downstream deps**: `rusqlite` (bundled), `zstd`, `bincode`, `serde`, `starknet-types-raw`, `torii-types`. With `etl`: `torii`, `async-trait`, `anyhow`. +- **Features**: `etl` — adds the `Extractor` implementation and the `torii` dep. Binaries that use only `EventFetcher` do not need it. + +### Extension Points + +- New snapshot source: add a module beside `sqlite` (e.g. `postgres.rs`) and implement `EventFetcher` for its connection. `PathfinderExtractor` becomes reusable because it consumes `EventFetcher` traits, not concrete types. +- New blob format: add a module under `decoding` and widen `BlockEvents::try_from(row)` to dispatch by column version. diff --git a/crates/pathfinder/src/decoding.rs b/crates/pathfinder/src/decoding.rs index eb7cd0d1..1e8694c3 100644 --- a/crates/pathfinder/src/decoding.rs +++ b/crates/pathfinder/src/decoding.rs @@ -1,12 +1,12 @@ use crate::sqlite::BlockEventsRow; use crate::{PFError, PFResult}; use serde::{Deserialize, Serialize}; +use starknet_types_raw::event::Event; +use starknet_types_raw::Felt; use std::cell::RefCell; use std::fmt::{Formatter, Result as FmtResult}; use std::io::Result as IoResult; use std::sync::LazyLock; -use torii_starknet::event::Event; -use torii_starknet::Felt; use zstd::bulk::Decompressor; // Taken from pathfinder-common but with some optimizations diff --git a/crates/pathfinder/src/extractor.rs b/crates/pathfinder/src/extractor.rs index 46b86456..3dcf5201 100644 --- a/crates/pathfinder/src/extractor.rs +++ b/crates/pathfinder/src/extractor.rs @@ -1,13 +1,9 @@ use crate::{connect, EventFetcher, PFResult}; -use anyhow::Result as AnyResult; -use async_trait::async_trait; use rusqlite::Connection; -use starknet::core::types::EmittedEvent; -use std::collections::HashMap; use std::path::Path; -use std::sync::{Arc, Mutex}; -use torii::etl::{BlockContext, EngineDb, ExtractionBatch, Extractor}; - +use std::sync::Mutex; +use torii_types::block::BlockContext; +use torii_types::event::StarknetEvent; #[derive(Debug)] pub struct PathfinderExtractor { pub conn: Mutex, @@ -18,7 +14,7 @@ pub struct PathfinderExtractor { } #[derive(Debug)] -pub struct PathfinderCombinedExtractor { +pub struct PathfinderCombinedExtractor { pub pathfinder: PathfinderExtractor, pub head: T, pub on_head: bool, @@ -35,7 +31,7 @@ impl PathfinderExtractor { }) } - pub fn next_batch(&mut self) -> PFResult<(Vec, Vec)> { + pub fn next_batch(&mut self) -> PFResult<(Vec, Vec)> { let mut last = self.current + self.batch; if last >= self.end { last = self.end; @@ -44,17 +40,17 @@ impl PathfinderExtractor { let (blocks, events) = self .conn .lock()? - .get_emitted_events_with_context(self.current, last)?; + .get_events_with_context(self.current, last)?; let last_block = blocks.last().map_or(self.current, |b| b.number); if last_block < last { self.finished = true; } self.current = last_block + 1; - Ok((blocks, events.into_iter().map(Into::into).collect())) + Ok((blocks, events)) } } -impl PathfinderCombinedExtractor { +impl PathfinderCombinedExtractor { pub fn new>( path: P, batch: u64, @@ -70,64 +66,74 @@ impl PathfinderCombinedExtractor { } } -#[async_trait] -impl Extractor for PathfinderExtractor { - fn set_start_block(&mut self, start_block: u64) { - self.current = start_block.max(self.current); - } - async fn extract( - &mut self, - _cursor: Option, - _engine_db: &EngineDb, - ) -> AnyResult { - let (blocks, events) = self.next_batch()?; - let blocks = blocks - .into_iter() - .map(|b| (b.number, Arc::new(b))) - .collect(); - Ok(ExtractionBatch { - events, - blocks, - transactions: HashMap::new(), - declared_classes: Vec::new(), - deployed_contracts: Vec::new(), - cursor: None, - chain_head: None, - }) - } - fn is_finished(&self) -> bool { - self.current >= self.end - } - fn as_any(&self) -> &dyn std::any::Any { - self - } -} +#[cfg(feature = "etl")] +mod etl { + use super::{PathfinderCombinedExtractor, PathfinderExtractor}; + use anyhow::Result as AnyResult; + use async_trait::async_trait; + use std::collections::HashMap; + use std::sync::Arc; + use torii::etl::{EngineDb, ExtractionBatch, Extractor}; -#[async_trait] -impl Extractor for PathfinderCombinedExtractor { - fn set_start_block(&mut self, start_block: u64) { - self.pathfinder.set_start_block(start_block); - self.head.set_start_block(start_block); - } - async fn extract( - &mut self, - cursor: Option, - engine_db: &EngineDb, - ) -> AnyResult { - if self.on_head { - self.head.extract(cursor, engine_db).await - } else if self.pathfinder.is_finished() { - self.on_head = true; - self.head.set_start_block(self.pathfinder.current); - self.head.extract(cursor, engine_db).await - } else { - self.pathfinder.extract(cursor, engine_db).await + #[async_trait] + impl Extractor for PathfinderExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.current = start_block.max(self.current); + } + async fn extract( + &mut self, + _cursor: Option, + _engine_db: &EngineDb, + ) -> AnyResult { + let (blocks, events) = self.next_batch()?; + let blocks = blocks + .into_iter() + .map(|b| (b.number, Arc::new(b))) + .collect(); + Ok(ExtractionBatch { + events, + blocks, + transactions: HashMap::new(), + declared_classes: Vec::new(), + deployed_contracts: Vec::new(), + cursor: None, + chain_head: None, + }) + } + fn is_finished(&self) -> bool { + self.current >= self.end + } + fn as_any(&self) -> &dyn std::any::Any { + self } } - fn is_finished(&self) -> bool { - self.pathfinder.is_finished() && self.head.is_finished() - } - fn as_any(&self) -> &dyn std::any::Any { - self + + #[async_trait] + impl Extractor for PathfinderCombinedExtractor { + fn set_start_block(&mut self, start_block: u64) { + self.pathfinder.set_start_block(start_block); + self.head.set_start_block(start_block); + } + async fn extract( + &mut self, + cursor: Option, + engine_db: &EngineDb, + ) -> AnyResult { + if self.on_head { + self.head.extract(cursor, engine_db).await + } else if self.pathfinder.is_finished() { + self.on_head = true; + self.head.set_start_block(self.pathfinder.current); + self.head.extract(cursor, engine_db).await + } else { + self.pathfinder.extract(cursor, engine_db).await + } + } + fn is_finished(&self) -> bool { + self.pathfinder.is_finished() && self.head.is_finished() + } + fn as_any(&self) -> &dyn std::any::Any { + self + } } } diff --git a/crates/pathfinder/src/fetcher.rs b/crates/pathfinder/src/fetcher.rs index e3aebabf..f5182cc6 100644 --- a/crates/pathfinder/src/fetcher.rs +++ b/crates/pathfinder/src/fetcher.rs @@ -1,75 +1,37 @@ -use std::collections::HashMap; - +use crate::decoding::BlockEvents; use crate::sqlite::{BlockContextRow, SqliteExt}; -use crate::PFError; -use crate::{decoding::BlockEvents, PFResult}; +use crate::{PFError, PFResult}; use rusqlite::Connection; - -#[cfg(not(feature = "etl"))] -/// Block context information Copied from core to avoid importing the entire crate -#[derive(Debug, Clone, Default)] -pub struct BlockContext { - pub number: u64, - pub hash: Felt, - pub parent_hash: Felt, - pub timestamp: u64, -} - -#[cfg(feature = "etl")] -pub use torii::etl::extractor::BlockContext; -use torii_starknet::event::EmittedEvent; -use torii_starknet::Felt; +use torii_types::block::BlockContext; +use torii_types::event::StarknetEvent; impl From for BlockContext { fn from(value: BlockContextRow) -> Self { BlockContext { number: value.number, - hash: value.hash.into(), - parent_hash: value.parent_hash.into(), + hash: value.hash, + parent_hash: value.parent_hash, timestamp: value.timestamp, } } } -pub trait EmittedEventExt { - fn with_block_hash(self, block_hashes: &HashMap) -> PFResult; -} - -impl EmittedEventExt for EmittedEvent { - fn with_block_hash(mut self, block_hashes: &HashMap) -> PFResult { - let block_number = self.block_number.unwrap(); - match block_hashes.get(&block_number) { - Some(block_hash) => self.block_hash = Some(*block_hash), - None => return Err(PFError::block_hash_missing(block_number)), - } - Ok(self) - } -} - pub trait EventFetcher { - fn get_emitted_events_wo_block_hashes( - &self, - from_block: u64, - to_block: u64, - ) -> PFResult>; - fn get_emitted_events(&self, from_block: u64, to_block: u64) -> PFResult>; - fn get_emitted_events_with_context( + fn get_events(&self, from_block: u64, to_block: u64) -> PFResult>; + fn get_events_with_context( &self, from_block: u64, to_block: u64, - ) -> PFResult<(Vec, Vec)>; + ) -> PFResult<(Vec, Vec)>; } + impl EventFetcher for Connection { - fn get_emitted_events_wo_block_hashes( - &self, - from_block: u64, - to_block: u64, - ) -> PFResult> { + fn get_events(&self, from_block: u64, to_block: u64) -> PFResult> { let total_events = self.get_number_of_events_for_blocks(from_block, to_block)?; let mut tx_hashes = self .get_block_tx_hash_rows(from_block, to_block)? .into_iter(); - let mut emitted_events = Vec::with_capacity(total_events as usize); + let mut events = Vec::with_capacity(total_events as usize); for row in self.get_block_events_rows(from_block, to_block)? { let block: BlockEvents = row.try_into()?; @@ -79,9 +41,8 @@ impl EventFetcher for Connection { .map(|r| r.1) .ok_or_else(|| PFError::tx_hash_mismatch(block.block_number))?; for event in transaction { - emitted_events.push(EmittedEvent { - block_hash: None, - block_number: Some(block.block_number), + events.push(StarknetEvent { + block_number: block.block_number, data: event.data, from_address: event.from_address, keys: event.keys, @@ -90,57 +51,15 @@ impl EventFetcher for Connection { } } } - Ok(emitted_events) - } - fn get_emitted_events(&self, from_block: u64, to_block: u64) -> PFResult> { - let contexts = self.get_block_hash_rows_with_count(from_block, to_block)?; - let total_events: u64 = contexts.iter().map(|r| r.event_count).sum(); - let mut tx_hashes = self - .get_block_tx_hash_rows(from_block, to_block)? - .into_iter(); - let mut hash_rows = contexts.into_iter(); - let mut emitted_events = Vec::with_capacity(total_events as usize); - - for row in self.get_block_events_rows(from_block, to_block)? { - let block: BlockEvents = row.try_into()?; - let block_hash = loop { - match hash_rows.next() { - Some(hash_row) => { - if hash_row.number == block.block_number { - break hash_row.hash; - } else if hash_row.number > block.block_number { - return Err(PFError::block_hash_missing(block.block_number)); - } - } - None => return Err(PFError::block_hash_missing(block.block_number)), - } - }; - for transaction in block.transactions { - let transaction_hash = tx_hashes - .next() - .map(|r| r.1) - .ok_or_else(|| PFError::tx_hash_mismatch(block.block_number))?; - for event in transaction { - emitted_events.push(EmittedEvent { - block_hash: Some(block_hash), - block_number: Some(block.block_number), - data: event.data, - from_address: event.from_address, - keys: event.keys, - transaction_hash, - }); - } - } - } - Ok(emitted_events) + Ok(events) } - fn get_emitted_events_with_context( + fn get_events_with_context( &self, from_block: u64, to_block: u64, - ) -> PFResult<(Vec, Vec)> { + ) -> PFResult<(Vec, Vec)> { let context_rows = self.get_block_context_rows(from_block, to_block)?; let total_events: u64 = context_rows.iter().map(|r| r.event_count).sum(); let mut tx_hashes = self @@ -170,9 +89,8 @@ impl EventFetcher for Connection { .map(|r| r.1) .ok_or_else(|| PFError::tx_hash_mismatch(block.block_number))?; for event in transaction { - emitted_events.push(EmittedEvent { - block_hash: Some(ctx.hash), - block_number: Some(block.block_number), + emitted_events.push(StarknetEvent { + block_number: block.block_number, data: event.data, from_address: event.from_address, keys: event.keys, diff --git a/crates/pathfinder/src/lib.rs b/crates/pathfinder/src/lib.rs index 2286741f..487298d4 100644 --- a/crates/pathfinder/src/lib.rs +++ b/crates/pathfinder/src/lib.rs @@ -11,5 +11,5 @@ pub mod extractor; mod test; pub use error::{PFError, PFResult}; -pub use fetcher::{BlockContext, EventFetcher}; +pub use fetcher::EventFetcher; pub use utils::connect; diff --git a/crates/pathfinder/src/sqlite.rs b/crates/pathfinder/src/sqlite.rs index fb71976c..33757937 100644 --- a/crates/pathfinder/src/sqlite.rs +++ b/crates/pathfinder/src/sqlite.rs @@ -1,9 +1,6 @@ -use rusqlite::params; use rusqlite::types::ValueRef; -use rusqlite::Connection; -use rusqlite::Error as SqliteError; -use rusqlite::Row; -use torii_starknet::Felt; +use rusqlite::{params, Connection, Error as SqliteError, Row}; +use starknet_types_raw::Felt; pub type SqliteResult = Result; @@ -91,7 +88,9 @@ pub trait RowExt { impl RowExt for Row<'_> { fn get_felt(&self, idx: usize) -> SqliteResult { match self.get_ref(idx)? { - ValueRef::Blob(b) => Ok(Felt::from_bytes_be_slice(b)), + ValueRef::Blob(b) => Felt::from_be_bytes_slice(b).map_err(|e| { + SqliteError::FromSqlConversionFailure(idx, rusqlite::types::Type::Blob, Box::new(e)) + }), _ => Err(SqliteError::InvalidColumnType( idx, "felt".into(), @@ -102,7 +101,9 @@ impl RowExt for Row<'_> { fn get_felt_opt(&self, idx: usize) -> SqliteResult> { match self.get_ref(idx)? { - ValueRef::Blob(b) => Ok(Some(Felt::from_bytes_be_slice(b))), + ValueRef::Blob(b) => Ok(Some(Felt::from_be_bytes_slice(b).map_err(|e| { + SqliteError::FromSqlConversionFailure(idx, rusqlite::types::Type::Blob, Box::new(e)) + })?)), ValueRef::Null => Ok(None), _ => Err(SqliteError::InvalidColumnType( idx, diff --git a/crates/pathfinder/src/test.rs b/crates/pathfinder/src/test.rs index 33324343..8451ce6f 100644 --- a/crates/pathfinder/src/test.rs +++ b/crates/pathfinder/src/test.rs @@ -1,8 +1,12 @@ -use crate::{connect, extractor::PathfinderExtractor, fetcher::EventFetcher}; +use crate::connect; +#[cfg(feature = "etl")] +use crate::extractor::PathfinderExtractor; +use crate::fetcher::EventFetcher; const DB_PATH: &str = "/mnt/store/mainnet.sqlite"; #[test] +#[cfg(feature = "etl")] #[ignore = "requires /mnt/store/mainnet.sqlite snapshot"] fn test_emitted_events() { let mut extractor = @@ -25,7 +29,7 @@ fn test_emitted_events() { fn test_get_emitted_events_with_context() { let conn = connect(DB_PATH).unwrap(); let (blocks, events) = conn - .get_emitted_events_with_context(3000000, 3000010) + .get_events_with_context(3000000, 3000010) .expect("failed to fetch events with context"); println!( "Fetched {} blocks and {} events", diff --git a/crates/postgres/Cargo.toml b/crates/postgres/Cargo.toml deleted file mode 100644 index 9624b2f6..00000000 --- a/crates/postgres/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "torii-postgres" -version = "0.1.0" -edition = "2021" -description = "PostgreSQL utils for Torii runtime" -authors = ["Torii Runtime "] -license = "Apache-2.0" - -[dependencies] -sqlx = { workspace = true, features = [ - "postgres", - "runtime-tokio-rustls", - "macros", - "migrate", -] } -anyhow.workspace = true -async-trait.workspace = true -xxhash-rust.workspace = true -hex.workspace = true -serde.workspace = true -serde_json.workspace = true -tokio.workspace = true -torii.workspace = true -tracing.workspace = true -starknet-types-core.workspace = true -thiserror.workspace = true -futures.workspace = true -crc.workspace = true - - -torii-common.workspace = true diff --git a/crates/postgres/src/db.rs b/crates/postgres/src/db.rs deleted file mode 100644 index ef5d608d..00000000 --- a/crates/postgres/src/db.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::migration::SchemaMigrator; -use async_trait::async_trait; -use sqlx::{migrate::Migrator, Postgres}; -pub use sqlx::{PgPool, Transaction}; -use std::ops::Deref; -use torii_common::sql::{Executable, SqlxResult}; - -#[async_trait] -pub trait PostgresConnection { - fn pool(&self) -> &PgPool; - - async fn begin(&self) -> SqlxResult> { - Ok(self.pool().begin().await?) - } - async fn migrate(&self, schema: Option<&'static str>, migrator: Migrator) -> SqlxResult<()> { - let result = match schema { - Some(schema) => SchemaMigrator::new(schema, migrator).run(self.pool()).await, - None => migrator.run(self.pool()).await, - }; - Ok(result?) - } - async fn execute_queries(&self, queries: impl Executable + Send) -> SqlxResult<()> { - let mut transaction = self.begin().await?; - queries.execute(&mut transaction).await?; - transaction.commit().await - } -} - -#[allow(clippy::explicit_auto_deref)] -#[async_trait] -impl + Send + Sync + 'static> PostgresConnection for T { - fn pool(&self) -> &PgPool { - &**self - } -} diff --git a/crates/postgres/src/lib.rs b/crates/postgres/src/lib.rs deleted file mode 100644 index a2da4928..00000000 --- a/crates/postgres/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod db; -pub mod metadata; -pub mod migration; -pub use db::PostgresConnection; diff --git a/crates/postgres/src/metadata.rs b/crates/postgres/src/metadata.rs deleted file mode 100644 index 3368b76c..00000000 --- a/crates/postgres/src/metadata.rs +++ /dev/null @@ -1,72 +0,0 @@ -use sqlx::{postgres::PgArguments, query::Query, Postgres}; -use std::fmt::Write; -use torii::etl::EventContext; - -pub const INSERTS: &str = "__created_block, __updated_block, __created_tx, __updated_tx"; -pub const CONFLICTS: &str = "__updated_at = NOW(), __updated_block = EXCLUDED.__updated_block, __updated_tx = EXCLUDED.__updated_tx"; - -pub trait PgMetadata { - fn insert_string( - &self, - schema: &str, - table: &str, - primary_name: &str, - primary_value: String, - ) -> String; - fn static_insert_query(schema: &str, table: &str, primary_name: &str) -> String { - format!( - r#" - INSERT INTO "{schema}"."{table}" ("{primary_name}", {INSERTS}) - VALUES ($1, $2, $2, $3, $3) - ON CONFLICT ({primary_name}) DO UPDATE SET {CONFLICTS}"# - ) - } - fn insert_values(&self, string: &mut String); - fn bind_query<'a>( - &self, - query: &'a str, - primary_value: String, - ) -> Query<'a, Postgres, PgArguments>; -} - -impl PgMetadata for EventContext { - fn insert_string( - &self, - schema: &str, - table: &str, - primary_name: &str, - primary_value: String, - ) -> String { - let mut string = format!( - r#"INSERT INTO "{schema}"."{table}" ("{primary_name}", {INSERTS}) VALUES ({primary_value}, "# - ); - self.insert_values(&mut string); - write!( - string, - ") ON CONFLICT ({primary_name}) DO UPDATE SET {CONFLICTS}" - ) - .unwrap(); - string - } - fn bind_query<'a>( - &self, - query: &'a str, - primary_value: String, - ) -> Query<'a, Postgres, PgArguments> { - let block_number = self.block.number.to_string(); - let tx_hash = self.transaction.hash.to_bytes_be(); - sqlx::query::(query) - .bind(primary_value.clone()) - .bind(block_number) - .bind(tx_hash) - } - fn insert_values(&self, string: &mut String) { - let block_number = self.block.number; - let tx_hash = hex::encode(self.transaction.hash.to_bytes_be()); - write!( - string, - "{block_number}, {block_number}, '\\x{tx_hash}', '\\x{tx_hash}'" - ) - .unwrap(); - } -} diff --git a/crates/postgres/src/migration.rs b/crates/postgres/src/migration.rs deleted file mode 100644 index 9be9fce7..00000000 --- a/crates/postgres/src/migration.rs +++ /dev/null @@ -1,443 +0,0 @@ -use futures::future::BoxFuture; -use sqlx::{ - migrate::{AppliedMigration, Migrate, MigrateError, Migration, MigrationSource, Migrator}, - query, query_scalar, Acquire, PgConnection, Postgres, -}; -use sqlx::{query_as, Executor}; -use std::{ - collections::{HashMap, HashSet}, - ops::{Deref, DerefMut}, - slice, - time::{Duration, Instant}, -}; - -pub struct SchemaMigrator { - pub migrator: Migrator, - pub schema: &'static str, -} - -pub struct PgAcquiredSchema<'a, A> -where - A: Acquire<'a>, -{ - pub connection: >::Connection, - pub schema: &'static str, -} - -impl<'a, A> Deref for PgAcquiredSchema<'a, A> -where - A: Acquire<'a, Database = Postgres>, -{ - type Target = PgConnection; - fn deref(&self) -> &Self::Target { - &self.connection - } -} - -impl<'a, A> DerefMut for PgAcquiredSchema<'a, A> -where - A: Acquire<'a, Database = Postgres>, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.connection - } -} - -impl<'a, A> Migrate for PgAcquiredSchema<'a, A> -where - A: Acquire<'a, Database = Postgres>, - A: Executor<'a, Database = Postgres>, -{ - fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - // language=SQL - self.connection - .execute( - format!( - r#" -CREATE SCHEMA IF NOT EXISTS {schema}; -CREATE TABLE IF NOT EXISTS {schema}._sqlx_migrations ( - version BIGINT PRIMARY KEY, - description TEXT NOT NULL, - installed_on TIMESTAMPTZ NOT NULL DEFAULT now(), - success BOOLEAN NOT NULL, - checksum BYTEA NOT NULL, - execution_time BIGINT NOT NULL -); - "#, - schema = self.schema - ) - .as_str(), - ) - .await?; - - Ok(()) - }) - } - - fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - // language=SQL - - let row: Option<(i64,)> = query_as( - format!("SELECT version FROM {schema}._sqlx_migrations WHERE success = false ORDER BY version LIMIT 1", schema = self.schema).as_str() - ) - .fetch_optional(&mut *self.connection) - .await?; - - Ok(row.map(|r: (i64,)| r.0)) - }) - } - - fn list_applied_migrations( - &mut self, - ) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - // language=SQL - let rows: Vec<(i64, Vec)> = query_as( - format!( - "SELECT version, checksum FROM {schema}._sqlx_migrations ORDER BY version", - schema = self.schema - ) - .as_str(), - ) - .fetch_all(&mut *self.connection) - .await?; - - let migrations = rows - .into_iter() - .map(|(version, checksum)| AppliedMigration { - version, - checksum: checksum.into(), - }) - .collect(); - - Ok(migrations) - }) - } - - fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - let database_name = current_database(&mut self.connection).await?; - let lock_id = generate_lock_id(&database_name); - - // create an application lock over the database - // this function will not return until the lock is acquired - - // https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS - // https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS-TABLE - - // language=SQL - let _ = query("SELECT pg_advisory_lock($1)") - .bind(lock_id) - .execute(&mut *self.connection) - .await?; - - Ok(()) - }) - } - - fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - let database_name = current_database(self).await?; - let lock_id = generate_lock_id(&database_name); - - // language=SQL - let _ = query("SELECT pg_advisory_unlock($1)") - .bind(lock_id) - .execute(&mut *self.connection) - .await?; - - Ok(()) - }) - } - - fn apply<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let start = Instant::now(); - let schema = self.schema; - // execute migration queries - if migration.no_tx { - execute_migration(self, schema, migration).await?; - } else { - // Use a single transaction for the actual migration script and the essential bookeeping so we never - // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. - // The `execution_time` however can only be measured for the whole transaction. This value _only_ exists for - // data lineage and debugging reasons, so it is not super important if it is lost. So we initialize it to -1 - // and update it once the actual transaction completed. - let mut tx = self.begin().await?; - execute_migration(&mut tx, schema, migration).await?; - tx.commit().await?; - } - - // Update `elapsed_time`. - // NOTE: The process may disconnect/die at this point, so the elapsed time value might be lost. We accept - // this small risk since this value is not super important. - let elapsed = start.elapsed(); - - // language=SQL - #[allow(clippy::cast_possible_truncation)] - let _ = query(&format!( - r#" - UPDATE {schema}._sqlx_migrations - SET execution_time = $1 - WHERE version = $2 - "#, - schema = self.schema - )) - .bind(elapsed.as_nanos() as i64) - .bind(migration.version) - .execute(&mut *self.connection) - .await?; - - Ok(elapsed) - }) - } - - fn revert<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let start = Instant::now(); - let schema = self.schema; - // execute migration queries - if migration.no_tx { - revert_migration(&mut self.connection, schema, migration).await?; - } else { - // Use a single transaction for the actual migration script and the essential bookeeping so we never - // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. - let mut tx = self.begin().await?; - revert_migration(&mut tx, schema, migration).await?; - tx.commit().await?; - } - - let elapsed = start.elapsed(); - - Ok(elapsed) - }) - } -} - -async fn current_database(conn: &mut PgConnection) -> Result { - // language=SQL - Ok(query_scalar("SELECT current_database()") - .fetch_one(conn) - .await?) -} - -fn generate_lock_id(database_name: &str) -> i64 { - const CRC_IEEE: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); - // 0x3d32ad9e chosen by fair dice roll - 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) -} - -async fn execute_migration( - conn: &mut PgConnection, - schema: &'static str, - migration: &Migration, -) -> Result<(), MigrateError> { - let _ = conn - .execute(&*migration.sql) - .await - .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; - - // language=SQL - let _ = query( - format!(r#" - INSERT INTO {schema}._sqlx_migrations ( version, description, success, checksum, execution_time ) - VALUES ( $1, $2, TRUE, $3, -1 ) - "#).as_str() - ) - .bind(migration.version) - .bind(&*migration.description) - .bind(&*migration.checksum) - .execute(conn) - .await?; - - Ok(()) -} - -async fn revert_migration( - conn: &mut PgConnection, - schema: &'static str, - migration: &Migration, -) -> Result<(), MigrateError> { - let _ = conn - .execute(&*migration.sql) - .await - .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; - - // language=SQL - let _ = query(format!(r#"DELETE FROM {schema}._sqlx_migrations WHERE version = $1"#).as_str()) - .bind(migration.version) - .execute(conn) - .await?; - - Ok(()) -} - -impl SchemaMigrator { - pub const fn new(schema: &'static str, migrator: Migrator) -> Self { - Self { migrator, schema } - } - pub async fn new_from_source<'s, S>( - schema: &'static str, - source: S, - ) -> Result - where - S: MigrationSource<'s>, - { - Migrator::new(source) - .await - .map(|migrator| Self { migrator, schema }) - } - - /// Specify whether applied migrations that are missing from the resolved migrations should be ignored. - pub fn set_ignore_missing(&mut self, ignore_missing: bool) -> &Self { - self.migrator.ignore_missing = ignore_missing; - self - } - - /// Specify whether or not to lock the database during migration. Defaults to `true`. - /// - /// ### Warning - /// Disabling locking can lead to errors or data loss if multiple clients attempt to apply migrations simultaneously - /// without some sort of mutual exclusion. - /// - /// This should only be used if the database does not support locking, e.g. CockroachDB which talks the Postgres - /// protocol but does not support advisory locks used by SQLx's migrations support for Postgres. - pub fn set_locking(&mut self, locking: bool) -> &Self { - self.migrator.locking = locking; - self - } - - /// Get an iterator over all known migrations. - pub fn iter(&self) -> slice::Iter<'_, Migration> { - self.migrator.migrations.iter() - } - - /// Check if a migration version exists. - pub fn version_exists(&self, version: i64) -> bool { - self.iter().any(|m| m.version == version) - } - - /// Run any pending migrations against the database; and, validate previously applied migrations - /// against the current migration source to detect accidental changes in previously-applied migrations. - /// - /// # Examples - /// - /// ```rust,no_run - /// # use sqlx::migrate::MigrateError; - /// # fn main() -> Result<(), MigrateError> { - /// # sqlx::__rt::test_block_on(async move { - /// use sqlx::migrate::Migrator; - /// use sqlx::sqlite::SqlitePoolOptions; - /// - /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; - /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; - /// m.run(&pool).await - /// # }) - /// # } - /// ``` - pub async fn run<'a, A>(&self, migrator: A) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Postgres>, - PgAcquiredSchema<'a, A>: Migrate, - { - let mut conn = PgAcquiredSchema { - connection: migrator.acquire().await?, - schema: self.schema, - }; - self.migrator.run_direct(&mut conn).await - } - - /// Run down migrations against the database until a specific version. - /// - /// # Examples - /// - /// ```rust,no_run - /// # use sqlx::migrate::MigrateError; - /// # fn main() -> Result<(), MigrateError> { - /// # sqlx::__rt::test_block_on(async move { - /// use sqlx::migrate::Migrator; - /// use sqlx::sqlite::SqlitePoolOptions; - /// - /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; - /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; - /// m.undo(&pool, 4).await - /// # }) - /// # } - /// ``` - pub async fn undo<'a, A>(&self, migrator: A, target: i64) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Postgres>, - PgAcquiredSchema<'a, A>: Migrate, - { - let mut conn = PgAcquiredSchema { - connection: migrator.acquire().await?, - schema: self.schema, - }; - // lock the database for exclusive access by the migrator - if self.migrator.locking { - conn.lock().await?; - } - - // creates [_migrations] table only if needed - // eventually this will likely migrate previous versions of the table - conn.ensure_migrations_table().await?; - - let version = conn.dirty_version().await?; - if let Some(version) = version { - return Err(MigrateError::Dirty(version)); - } - - let applied_migrations = conn.list_applied_migrations().await?; - validate_applied_migrations(&applied_migrations, &self.migrator)?; - - let applied_migrations: HashMap<_, _> = applied_migrations - .into_iter() - .map(|m| (m.version, m)) - .collect(); - - for migration in self - .iter() - .rev() - .filter(|m| m.migration_type.is_down_migration()) - .filter(|m| applied_migrations.contains_key(&m.version)) - .filter(|m| m.version > target) - { - conn.revert(migration).await?; - } - - // unlock the migrator to allow other migrators to run - // but do nothing as we already migrated - if self.migrator.locking { - conn.unlock().await?; - } - - Ok(()) - } -} - -fn validate_applied_migrations( - applied_migrations: &[AppliedMigration], - migrator: &Migrator, -) -> Result<(), MigrateError> { - if migrator.ignore_missing { - return Ok(()); - } - - let migrations: HashSet<_> = migrator.iter().map(|m| m.version).collect(); - - for applied_migration in applied_migrations { - if !migrations.contains(&applied_migration.version) { - return Err(MigrateError::VersionMissing(applied_migration.version)); - } - } - - Ok(()) -} diff --git a/crates/sql/Cargo.toml b/crates/sql/Cargo.toml new file mode 100644 index 00000000..9743060e --- /dev/null +++ b/crates/sql/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "torii-sql" +version = "0.1.0" +edition = "2021" +description = "Common utilities for Torii token indexers" + +[dependencies] +async-trait = "0.1" +sqlx = { workspace = true, features = ["runtime-tokio-rustls"] } +itertools.workspace = true +futures.workspace = true +crc.workspace = true +starknet-types-raw.workspace = true +log.workspace = true + +[features] +postgres = ["sqlx/postgres"] +sqlite = ["sqlx/sqlite"] +mysql = ["sqlx/mysql"] + +[lints] +workspace = true diff --git a/crates/sql/README.md b/crates/sql/README.md new file mode 100644 index 00000000..7427ebed --- /dev/null +++ b/crates/sql/README.md @@ -0,0 +1,98 @@ +# torii-sql + +The database abstraction layer. Wraps `sqlx` to give every Torii sink a +unified API over **SQLite** and **PostgreSQL** (MySQL has hooks but is +unused), plus a set of helpers for building pools, running migrations, and +composing parametric queries across backends. + +## Role in Torii + +Every sink that persists state uses this crate: `torii-sql-sink`, +`torii-controllers-sink`, `torii-ecs-sink`, `introspect-sql-sink`, and the +three token sinks. `EngineDb` in the root crate does *not* use this layer +(it is pinned to raw `sqlx::Any`) — everything else does. The +`DbBackend` enum defined here is the same one `torii-runtime-common` returns +from `backend_from_url_or_path`, so a single value flows from CLI → runtime +resolver → pool → migrations → queries. + +## Architecture + +```text ++-------------------------------------------------------------------+ +| torii-sql | +| | +| connection.rs | +| +-------------------+ DbBackend::{Sqlite,Postgres} | +| | DbBackend | DbOption (backend-tagged pair) | +| | POSTGRES_URL_... | | +| | SQLITE_URL_... | | +| +-------------------+ | +| | +| pool.rs | +| +---------------------+ PoolExt / PoolConfig / DbPoolOptions | +| | Connection pools | (pool tuning helpers) | +| +---------------------+ | +| | +| migrate.rs | +| +---------------------+ SchemaMigrator (versioned, backend- | +| | Schema migrations | aware), AcquiredSchema (lock guard) | +| +---------------------+ | +| | +| query.rs | +| +---------------------+ Executable trait + FlexQuery + Queries | +| | Query composition | placeholder-normalisation for ?/$n | +| +---------------------+ | +| | +| postgres/ (feature: postgres) sqlite/ (feature: sqlite) | +| +------------------+ +------------------+ | +| | Postgres + Pg* | | Sqlite + Sqlite* | | +| +------------------+ +------------------+ | +| | +| runtime.rs (feature: postgres + sqlite) | +| +----------------------+ | +| | DbPool / DbConnection| runtime-picked backend | +| | Options | | +| +----------------------+ | ++-------------------------------------------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `DbBackend` | `src/connection.rs` | 4 | `Postgres` / `Sqlite`; `FromStr` accepts `postgres://`, `postgresql://`, `sqlite*`, `:memory:` | +| `DbOption` | `src/connection.rs` | 10 | Backend-indexed pair (stores two values, picks one at runtime) | +| `POSTGRES_URL_SCHEMES` / `SQLITE_URL_SCHEMES` | `src/connection.rs` | 15 / 16 | URL prefix constants | +| `PoolExt` / `PoolConfig` / `DbPoolOptions` | `src/pool.rs` | — | Pool tuning + trait-object friendly pool abstraction | +| `SchemaMigrator` / `AcquiredSchema` | `src/migrate.rs` | — | Versioned schema migrations with cross-process locks | +| `Executable` | `src/query.rs` | — | Execute + fetch over any backend | +| `FlexQuery` / `Queries` | `src/query.rs` | — | Normalises `?` / `$n` placeholders and bindings | +| `Postgres`, `PgPool`, `PgQuery`, `PgArguments`, `PgDbConnection` | `src/postgres/mod.rs` | — | feature `postgres` | +| `Sqlite`, `SqlitePool`, `SqliteQuery`, `SqliteArguments`, `SqliteDbConnection` | `src/sqlite/mod.rs` | — | feature `sqlite` | +| `DbPool` / `DbConnectionOptions` | `src/runtime.rs` | — | feature `postgres + sqlite`; runtime enum wrapping either side | +| `SqlxResult` alias | `src/lib.rs` | 13 | `Result` | + +### Internal Modules + +- `connection` — `DbBackend` + URL classification + `DbOption` pair. +- `pool` — pool configuration helpers (size, timeouts) and the `PoolExt` blanket-impl. +- `migrate` — versioned schema migrator backed by a per-backend lock. +- `query` — `FlexQuery` and friends; bridges SQLite `?` vs. Postgres `$n` placeholder conventions. +- `postgres/` — sqlx-postgres wrappers + Postgres-specific migrate helpers + type adapters. +- `sqlite/` — sqlx-sqlite wrappers + SQLite migrate helpers. +- `runtime` — `DbPool::Postgres(pool) | DbPool::Sqlite(pool)` runtime enum (only compiled when both features are on). +- `types` — shared numeric/date conversions. + +### Interactions + +- **Upstream (consumers)**: `torii-runtime-common` (reads `DbBackend`), every SQL-backed sink, `introspect-sql-sink` (both PG + SQLite via its own features). +- **Downstream deps**: `sqlx` (patched fork from `bengineer42/sqlx`, see workspace `Cargo.toml` line 81), `async-trait`, `futures`. +- **Features**: `postgres`, `sqlite`, `mysql` — each pulls the corresponding sqlx driver. The `runtime` module and `DbPool` only exist when `postgres + sqlite` are both enabled. + +### Extension Points + +- New backend: add a feature flag + a sibling module mirroring `postgres/` and `sqlite/`, expose a `Pool`, a `Query`, and a `DbConnection`, and extend `DbBackend`. +- New cross-backend query helper: put it in `query.rs`. Keep backend-specific quirks (JSON operators, upsert syntax) behind `DbOption<&'static str>` lookups. +- New migration surface: add a `SchemaMigrator` subscriber; do not add a second migrator, the backend-aware one is authoritative. diff --git a/crates/sql/src/connection.rs b/crates/sql/src/connection.rs new file mode 100644 index 00000000..9e544687 --- /dev/null +++ b/crates/sql/src/connection.rs @@ -0,0 +1,76 @@ +use crate::SqlxError; +use std::fmt::Display; +use std::str::FromStr; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DbBackend { + Postgres, + Sqlite, +} + +pub struct DbOption { + postgres: T, + sqlite: T, +} + +pub const POSTGRES_URL_SCHEMES: [&str; 2] = ["postgres", "postgresql"]; +pub const SQLITE_URL_SCHEMES: [&str; 1] = ["sqlite"]; + +impl FromStr for DbBackend { + type Err = SqlxError; + + fn from_str(s: &str) -> Result { + if s.starts_with("postgres") || s.starts_with("postgresql") { + Ok(DbBackend::Postgres) + } else if s.starts_with("sqlite") || s == ":memory:" || s == "memory" { + Ok(DbBackend::Sqlite) + } else { + Err(SqlxError::Configuration( + format!("Unsupported database url: {s}").into(), + )) + } + } +} + +impl Display for DbBackend { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_str().fmt(f) + } +} + +impl DbBackend { + pub fn as_str(&self) -> &'static str { + match self { + DbBackend::Postgres => "postgres", + DbBackend::Sqlite => "sqlite", + } + } +} + +impl TryFrom<&str> for DbBackend { + type Error = SqlxError; + + fn try_from(value: &str) -> Result { + DbBackend::from_str(value) + } +} + +impl TryFrom for DbBackend { + type Error = SqlxError; + + fn try_from(value: String) -> Result { + DbBackend::try_from(value.as_str()) + } +} + +impl DbOption { + pub fn new(postgres: T, sqlite: T) -> Self { + Self { postgres, sqlite } + } + + pub fn value(self, db_type: &DbBackend) -> T { + match db_type { + DbBackend::Postgres => self.postgres, + DbBackend::Sqlite => self.sqlite, + } + } +} diff --git a/crates/sql/src/lib.rs b/crates/sql/src/lib.rs new file mode 100644 index 00000000..13e98d24 --- /dev/null +++ b/crates/sql/src/lib.rs @@ -0,0 +1,31 @@ +pub mod connection; +pub mod migrate; +pub mod pool; +pub mod query; +pub mod types; + +pub use connection::DbBackend; +pub use migrate::{AcquiredSchema, SchemaMigrator}; +pub use pool::{DbPoolOptions, PoolConfig, PoolExt}; +pub use query::{Executable, FlexQuery, Queries}; + +pub use sqlx::Error as SqlxError; +pub type SqlxResult = std::result::Result; + +#[cfg(feature = "postgres")] +pub mod postgres; +#[cfg(feature = "postgres")] +pub use postgres::{PgArguments, PgDbConnection, PgPool, PgQuery, Postgres}; + +#[cfg(feature = "sqlite")] +pub mod sqlite; +#[cfg(feature = "sqlite")] +pub use sqlite::{Sqlite, SqliteArguments, SqliteDbConnection, SqlitePool, SqliteQuery}; + +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub mod runtime; + +#[cfg(feature = "postgres")] +#[cfg(feature = "sqlite")] +pub use runtime::{DbConnectionOptions, DbPool}; diff --git a/crates/sql/src/migrate.rs b/crates/sql/src/migrate.rs new file mode 100644 index 00000000..42093cdc --- /dev/null +++ b/crates/sql/src/migrate.rs @@ -0,0 +1,218 @@ +use sqlx::{ + migrate::{AppliedMigration, Migrate, MigrateError, Migration, MigrationSource, Migrator}, + Acquire, Connection, Database, +}; +use std::{ + collections::HashMap, + ops::{Deref, DerefMut}, +}; +use std::{collections::HashSet, slice}; + +pub struct SchemaMigrator { + pub migrator: Migrator, + pub schema: &'static str, +} + +pub struct AcquiredSchema +where + DB: Database, + C: Connection, +{ + pub connection: C, + pub schema: &'static str, +} + +impl Deref for AcquiredSchema +where + DB: Database, + C: Connection, +{ + type Target = C; + fn deref(&self) -> &Self::Target { + &self.connection + } +} + +impl DerefMut for AcquiredSchema +where + DB: Database, + C: Connection, +{ + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.connection + } +} + +impl AcquiredSchema +where + DB: Database, + C: Connection, +{ + pub fn new(connection: C, schema: &'static str) -> Self { + Self { connection, schema } + } +} + +impl SchemaMigrator { + pub const fn new(schema: &'static str, migrator: Migrator) -> Self { + Self { migrator, schema } + } + pub async fn new_from_source<'s, S>( + schema: &'static str, + source: S, + ) -> Result + where + S: MigrationSource<'s>, + { + Migrator::new(source) + .await + .map(|migrator| Self { migrator, schema }) + } + + /// Specify whether applied migrations that are missing from the resolved migrations should be ignored. + pub fn set_ignore_missing(&mut self, ignore_missing: bool) -> &Self { + self.migrator.ignore_missing = ignore_missing; + self + } + + /// Specify whether or not to lock the database during migration. Defaults to `true`. + /// + /// ### Warning + /// Disabling locking can lead to errors or data loss if multiple clients attempt to apply migrations simultaneously + /// without some sort of mutual exclusion. + /// + /// This should only be used if the database does not support locking, e.g. CockroachDB which talks the Postgres + /// protocol but does not support advisory locks used by SQLx's migrations support for Postgres. + pub fn set_locking(&mut self, locking: bool) -> &Self { + self.migrator.locking = locking; + self + } + + /// Get an iterator over all known migrations. + pub fn iter(&self) -> slice::Iter<'_, Migration> { + self.migrator.migrations.iter() + } + + /// Check if a migration version exists. + pub fn version_exists(&self, version: i64) -> bool { + self.iter().any(|m| m.version == version) + } + + /// Run any pending migrations against the database; and, validate previously applied migrations + /// against the current migration source to detect accidental changes in previously-applied migrations. + /// + /// # Examples + /// + /// ```rust,no_run + /// # use sqlx::migrate::MigrateError; + /// # fn main() -> Result<(), MigrateError> { + /// # sqlx::__rt::test_block_on(async move { + /// use sqlx::migrate::Migrator; + /// use sqlx::sqlite::SqlitePoolOptions; + /// + /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; + /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; + /// m.run(&pool).await + /// # }) + /// # } + /// ``` + pub async fn run<'a, A>(&self, migrator: A) -> Result<(), MigrateError> + where + A: Acquire<'a>, + >::Connection: Connection, + AcquiredSchema: Migrate, + { + let mut conn = AcquiredSchema { + connection: migrator.acquire().await?, + schema: self.schema, + }; + self.migrator.run_direct(&mut conn).await + } + + /// Run down migrations against the database until a specific version. + /// + /// # Examples + /// + /// ```rust,no_run + /// # use sqlx::migrate::MigrateError; + /// # fn main() -> Result<(), MigrateError> { + /// # sqlx::__rt::test_block_on(async move { + /// use sqlx::migrate::Migrator; + /// use sqlx::sqlite::SqlitePoolOptions; + /// + /// let m = Migrator::new(std::path::Path::new("./migrations")).await?; + /// let pool = SqlitePoolOptions::new().connect("sqlite::memory:").await?; + /// m.undo(&pool, 4).await + /// # }) + /// # } + /// ``` + pub async fn undo<'a, A>(&self, migrator: A, target: i64) -> Result<(), MigrateError> + where + A: Acquire<'a>, + >::Connection: Connection, + AcquiredSchema: Migrate, + { + let mut conn = AcquiredSchema:: { + connection: migrator.acquire().await?, + schema: self.schema, + }; + // lock the database for exclusive access by the migrator + if self.migrator.locking { + conn.lock().await?; + } + + // creates [_migrations] table only if needed + // eventually this will likely migrate previous versions of the table + conn.ensure_migrations_table().await?; + + let version = conn.dirty_version().await?; + if let Some(version) = version { + return Err(MigrateError::Dirty(version)); + } + + let applied_migrations = conn.list_applied_migrations().await?; + validate_applied_migrations(&applied_migrations, &self.migrator)?; + + let applied_migrations: HashMap<_, _> = applied_migrations + .into_iter() + .map(|m| (m.version, m)) + .collect(); + + for migration in self + .iter() + .rev() + .filter(|m| m.migration_type.is_down_migration()) + .filter(|m| applied_migrations.contains_key(&m.version)) + .filter(|m| m.version > target) + { + conn.revert(migration).await?; + } + + // unlock the migrator to allow other migrators to run + // but do nothing as we already migrated + if self.migrator.locking { + conn.unlock().await?; + } + + Ok(()) + } +} + +fn validate_applied_migrations( + applied_migrations: &[AppliedMigration], + migrator: &Migrator, +) -> Result<(), MigrateError> { + if migrator.ignore_missing { + return Ok(()); + } + + let migrations: HashSet<_> = migrator.iter().map(|m| m.version).collect(); + + for applied_migration in applied_migrations { + if !migrations.contains(&applied_migration.version) { + return Err(MigrateError::VersionMissing(applied_migration.version)); + } + } + + Ok(()) +} diff --git a/crates/sql/src/pool.rs b/crates/sql/src/pool.rs new file mode 100644 index 00000000..73387198 --- /dev/null +++ b/crates/sql/src/pool.rs @@ -0,0 +1,302 @@ +use std::time::Duration; + +use crate::query::Executable; +use crate::{AcquiredSchema, SqlxResult}; +use async_trait::async_trait; +use log::LevelFilter; +use sqlx::migrate::{Migrate, Migrator}; +use sqlx::pool::PoolOptions; +use sqlx::{Connection, Database, Pool, Transaction}; + +#[async_trait] +pub trait PoolExt { + fn pool(&self) -> &Pool; + async fn begin(&self) -> SqlxResult> { + Ok(self.pool().begin().await?) + } + async fn migrate(&self, schema: Option<&'static str>, migrator: Migrator) -> SqlxResult<()> + where + ::Connection: Migrate, + AcquiredSchema::Connection>: Migrate, + { + let result = match schema { + Some(schema) => { + let mut conn: AcquiredSchema::Connection> = AcquiredSchema { + connection: self.pool().acquire().await?.detach(), + schema, + }; + migrator.run_direct(&mut conn).await + } + None => migrator.run(self.pool()).await, + }; + Ok(result?) + } + async fn execute_queries + Send>(&self, queries: E) -> SqlxResult<()> { + let mut transaction: Transaction<'_, DB> = self.begin().await?; + queries.execute(&mut transaction).await?; + transaction.commit().await + } +} + +impl PoolExt for Pool { + fn pool(&self) -> &Pool { + self + } +} + +const DEFAULT_TEST_BEFORE_ACQUIRE: bool = true; +const DEFAULT_MAX_CONNECTIONS: u32 = 10; +const DEFAULT_MIN_CONNECTIONS: u32 = 0; +const DEFAULT_ACQUIRE_TIME_LEVEL: LevelFilter = LevelFilter::Off; +const DEFAULT_ACQUIRE_SLOW_LEVEL: LevelFilter = LevelFilter::Warn; +const DEFAULT_ACQUIRE_SLOW_THRESHOLD: Duration = Duration::from_secs(2); +const DEFAULT_ACQUIRE_TIMEOUT: Duration = Duration::from_secs(30); +const DEFAULT_IDLE_TIMEOUT: Option = Some(Duration::from_secs(10 * 60)); +const DEFAULT_MAX_LIFETIME: Option = Some(Duration::from_secs(30 * 60)); +const DEFAULT_FAIR: bool = true; + +#[derive(Debug, Clone, Copy)] +pub struct DbPoolOptions { + pub test_before_acquire: bool, + pub max_connections: u32, + pub acquire_time_level: LevelFilter, + pub acquire_slow_level: LevelFilter, + pub acquire_slow_threshold: Duration, + pub acquire_timeout: Duration, + pub min_connections: u32, + pub max_lifetime: Option, + pub idle_timeout: Option, + pub fair: bool, +} + +#[derive(Debug, Clone)] +pub struct PoolConfig { + pub url: String, + pub options: DbPoolOptions, +} + +impl PoolConfig { + pub fn new(url: String) -> Self { + Self { + url, + options: DbPoolOptions::new(), + } + } + pub async fn connect(&self) -> SqlxResult> { + self.options.connect(&self.url).await + } + pub fn options(&self) -> PoolOptions { + self.options.options() + } + pub fn max_connections(mut self, max: u32) -> Self { + self.options.max_connections = max; + self + } + + pub fn get_max_connections(&self) -> u32 { + self.options.max_connections + } + + pub fn min_connections(mut self, min: u32) -> Self { + self.options.min_connections = min; + self + } + + pub fn get_min_connections(&self) -> u32 { + self.options.min_connections + } + + pub fn acquire_time_level(mut self, level: LevelFilter) -> Self { + self.options.acquire_time_level = level; + self + } + + pub fn acquire_slow_level(mut self, level: LevelFilter) -> Self { + self.options.acquire_slow_level = level; + self + } + + pub fn acquire_slow_threshold(mut self, threshold: Duration) -> Self { + self.options.acquire_slow_threshold = threshold; + self + } + + pub fn get_acquire_slow_threshold(&self) -> Duration { + self.options.acquire_slow_threshold + } + + pub fn acquire_timeout(mut self, timeout: Duration) -> Self { + self.options.acquire_timeout = timeout; + self + } + + pub fn get_acquire_timeout(&self) -> Duration { + self.options.acquire_timeout + } + + pub fn max_lifetime(mut self, lifetime: impl Into>) -> Self { + self.options.max_lifetime = lifetime.into(); + self + } + + pub fn get_max_lifetime(&self) -> Option { + self.options.max_lifetime + } + + pub fn idle_timeout(mut self, timeout: impl Into>) -> Self { + self.options.idle_timeout = timeout.into(); + self + } + + pub fn get_idle_timeout(&self) -> Option { + self.options.idle_timeout + } + + pub fn test_before_acquire(mut self, test: bool) -> Self { + self.options.test_before_acquire = test; + self + } + + pub fn get_test_before_acquire(&self) -> bool { + self.options.test_before_acquire + } + + pub fn fair(mut self, fair: bool) -> Self { + self.options.fair = fair; + self + } + + pub fn get_fair(&self) -> bool { + self.options.fair + } +} + +impl Default for DbPoolOptions { + fn default() -> Self { + DbPoolOptions::new() + } +} + +impl DbPoolOptions { + pub fn new() -> Self { + Self { + test_before_acquire: DEFAULT_TEST_BEFORE_ACQUIRE, + max_connections: DEFAULT_MAX_CONNECTIONS, + acquire_time_level: DEFAULT_ACQUIRE_TIME_LEVEL, + acquire_slow_level: DEFAULT_ACQUIRE_SLOW_LEVEL, + acquire_slow_threshold: DEFAULT_ACQUIRE_SLOW_THRESHOLD, + acquire_timeout: DEFAULT_ACQUIRE_TIMEOUT, + min_connections: DEFAULT_MIN_CONNECTIONS, + max_lifetime: DEFAULT_MAX_LIFETIME, + idle_timeout: DEFAULT_IDLE_TIMEOUT, + fair: DEFAULT_FAIR, + } + } + + pub async fn connect(&self, url: &str) -> SqlxResult> { + self.options::().connect(url).await + } + + pub async fn connect_with( + &self, + options: ::Options, + ) -> SqlxResult> { + self.options::().connect_with(options).await + } + + pub fn options(&self) -> PoolOptions { + PoolOptions::::new() + .test_before_acquire(self.test_before_acquire) + .max_connections(self.max_connections) + .acquire_time_level(self.acquire_time_level) + .acquire_slow_level(self.acquire_slow_level) + .acquire_slow_threshold(self.acquire_slow_threshold) + .acquire_timeout(self.acquire_timeout) + .min_connections(self.min_connections) + .max_lifetime(self.max_lifetime) + .idle_timeout(self.idle_timeout) + .__fair(self.fair) + } + + pub fn max_connections(mut self, max: u32) -> Self { + self.max_connections = max; + self + } + + pub fn get_max_connections(&self) -> u32 { + self.max_connections + } + + pub fn min_connections(mut self, min: u32) -> Self { + self.min_connections = min; + self + } + + pub fn get_min_connections(&self) -> u32 { + self.min_connections + } + + pub fn acquire_time_level(mut self, level: LevelFilter) -> Self { + self.acquire_time_level = level; + self + } + + pub fn acquire_slow_level(mut self, level: LevelFilter) -> Self { + self.acquire_slow_level = level; + self + } + + pub fn acquire_slow_threshold(mut self, threshold: Duration) -> Self { + self.acquire_slow_threshold = threshold; + self + } + + pub fn get_acquire_slow_threshold(&self) -> Duration { + self.acquire_slow_threshold + } + + pub fn acquire_timeout(mut self, timeout: Duration) -> Self { + self.acquire_timeout = timeout; + self + } + + pub fn get_acquire_timeout(&self) -> Duration { + self.acquire_timeout + } + + pub fn max_lifetime(mut self, lifetime: impl Into>) -> Self { + self.max_lifetime = lifetime.into(); + self + } + + pub fn get_max_lifetime(&self) -> Option { + self.max_lifetime + } + + pub fn idle_timeout(mut self, timeout: impl Into>) -> Self { + self.idle_timeout = timeout.into(); + self + } + + pub fn get_idle_timeout(&self) -> Option { + self.idle_timeout + } + + pub fn test_before_acquire(mut self, test: bool) -> Self { + self.test_before_acquire = test; + self + } + + pub fn get_test_before_acquire(&self) -> bool { + self.test_before_acquire + } + + pub fn fair(mut self, fair: bool) -> Self { + self.fair = fair; + self + } + + pub fn get_fair(&self) -> bool { + self.fair + } +} diff --git a/crates/sql/src/postgres/migrate.rs b/crates/sql/src/postgres/migrate.rs new file mode 100644 index 00000000..02982cc6 --- /dev/null +++ b/crates/sql/src/postgres/migrate.rs @@ -0,0 +1,227 @@ +use crate::AcquiredSchema; +use futures::future::BoxFuture; +use sqlx::migrate::{AppliedMigration, Migrate, MigrateError, Migration}; +use sqlx::{query, query_as, query_scalar, Connection, Executor, PgConnection, Postgres}; +use std::time::{Duration, Instant}; + +impl Migrate for AcquiredSchema { + fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + // language=SQL + self.connection + .execute( + format!( + "CREATE SCHEMA IF NOT EXISTS {schema}; + CREATE TABLE IF NOT EXISTS {schema}._sqlx_migrations ( + version BIGINT PRIMARY KEY, + description TEXT NOT NULL, + installed_on TIMESTAMPTZ NOT NULL DEFAULT now(), + success BOOLEAN NOT NULL, + checksum BYTEA NOT NULL, + execution_time BIGINT NOT NULL + );", + schema = self.schema + ) + .as_str(), + ) + .await?; + + Ok(()) + }) + } + + fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + // language=SQL + + let row: Option<(i64,)> = query_as( + format!("SELECT version FROM {schema}._sqlx_migrations WHERE success = false ORDER BY version LIMIT 1", schema = self.schema).as_str() + ) + .fetch_optional(&mut self.connection) + .await?; + + Ok(row.map(|r: (i64,)| r.0)) + }) + } + + fn list_applied_migrations( + &mut self, + ) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + // language=SQL + let rows: Vec<(i64, Vec)> = query_as( + format!( + "SELECT version, checksum FROM {schema}._sqlx_migrations ORDER BY version", + schema = self.schema + ) + .as_str(), + ) + .fetch_all(&mut self.connection) + .await?; + + let migrations = rows + .into_iter() + .map(|(version, checksum)| AppliedMigration { + version, + checksum: checksum.into(), + }) + .collect(); + + Ok(migrations) + }) + } + + fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + let database_name = current_database(&mut self.connection).await?; + let lock_id = generate_lock_id(&database_name); + + // create an application lock over the database + // this function will not return until the lock is acquired + + // https://www.postgresql.org/docs/current/explicit-locking.html#ADVISORY-LOCKS + // https://www.postgresql.org/docs/current/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS-TABLE + + // language=SQL + let _ = query("SELECT pg_advisory_lock($1)") + .bind(lock_id) + .execute(&mut self.connection) + .await?; + + Ok(()) + }) + } + + fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + let database_name = current_database(self).await?; + let lock_id = generate_lock_id(&database_name); + + // language=SQL + let _ = query("SELECT pg_advisory_unlock($1)") + .bind(lock_id) + .execute(&mut self.connection) + .await?; + + Ok(()) + }) + } + + fn apply<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let start = Instant::now(); + let schema = self.schema; + // execute migration queries + if migration.no_tx { + execute_migration(self, schema, migration).await?; + } else { + // Use a single transaction for the actual migration script and the essential bookeeping so we never + // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. + // The `execution_time` however can only be measured for the whole transaction. This value _only_ exists for + // data lineage and debugging reasons, so it is not super important if it is lost. So we initialize it to -1 + // and update it once the actual transaction completed. + let mut tx = Connection::begin(&mut self.connection).await?; + execute_migration(&mut tx, schema, migration).await?; + tx.commit().await?; + } + + // Update `elapsed_time`. + // NOTE: The process may disconnect/die at this point, so the elapsed time value might be lost. We accept + // this small risk since this value is not super important. + let elapsed = start.elapsed(); + + // language=SQL + #[allow(clippy::cast_possible_truncation)] + let _ = query(&format!( + "UPDATE {schema}._sqlx_migrations SET execution_time = $1 WHERE version = $2", + schema = self.schema + )) + .bind(elapsed.as_nanos() as i64) + .bind(migration.version) + .execute(&mut self.connection) + .await?; + + Ok(elapsed) + }) + } + + fn revert<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let start = Instant::now(); + let schema = self.schema; + // execute migration queries + if migration.no_tx { + revert_migration(&mut self.connection, schema, migration).await?; + } else { + // Use a single transaction for the actual migration script and the essential bookeeping so we never + // execute migrations twice. See https://github.com/launchbadge/sqlx/issues/1966. + let mut tx = Connection::begin(&mut self.connection).await?; + revert_migration(&mut tx, schema, migration).await?; + tx.commit().await?; + } + + let elapsed = start.elapsed(); + + Ok(elapsed) + }) + } +} + +async fn current_database(conn: &mut PgConnection) -> Result { + // language=SQL + Ok(query_scalar("SELECT current_database()") + .fetch_one(conn) + .await?) +} + +fn generate_lock_id(database_name: &str) -> i64 { + const CRC_IEEE: crc::Crc = crc::Crc::::new(&crc::CRC_32_ISO_HDLC); + // 0x3d32ad9e chosen by fair dice roll + 0x3d32ad9e * (CRC_IEEE.checksum(database_name.as_bytes()) as i64) +} + +async fn execute_migration( + conn: &mut PgConnection, + schema: &'static str, + migration: &Migration, +) -> Result<(), MigrateError> { + let _ = conn + .execute(&*migration.sql) + .await + .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; + + // language=SQL + query( + format!(r#"INSERT INTO "{schema}"._sqlx_migrations ( version, description, success, checksum, execution_time ) VALUES ( $1, $2, TRUE, $3, -1 )"#).as_str() + ) + .bind(migration.version) + .bind(&*migration.description) + .bind(&*migration.checksum) + .execute(conn) + .await?; + Ok(()) +} + +async fn revert_migration( + conn: &mut PgConnection, + schema: &'static str, + migration: &Migration, +) -> Result<(), MigrateError> { + let _ = conn + .execute(&*migration.sql) + .await + .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; + + query(format!(r#"DELETE FROM "{schema}"._sqlx_migrations WHERE version = $1"#).as_str()) + .bind(migration.version) + .execute(conn) + .await?; + + Ok(()) +} diff --git a/crates/sql/src/postgres/mod.rs b/crates/sql/src/postgres/mod.rs new file mode 100644 index 00000000..379307b3 --- /dev/null +++ b/crates/sql/src/postgres/mod.rs @@ -0,0 +1,35 @@ +pub mod migrate; +pub mod types; + +pub use sqlx::postgres::PgArguments; +pub use sqlx::{PgPool, Postgres}; + +use crate::{Executable, FlexQuery, SqlxResult}; +use futures::future::BoxFuture; +use sqlx::{Executor, PgTransaction}; + +pub type PgQuery = crate::FlexQuery; + +pub trait PgDbConnection: crate::PoolExt {} +impl crate::PgDbConnection for T {} + +impl Executable for FlexQuery { + fn execute<'t>(self, transaction: &'t mut PgTransaction) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + match self.args { + Some(args) => { + transaction + .execute(sqlx::query_with(self.sql.as_ref(), args)) + .await?; + } + None => { + transaction.execute(self.sql.as_ref()).await?; + } + } + Ok(()) + }) + } +} diff --git a/crates/sql/src/postgres/types.rs b/crates/sql/src/postgres/types.rs new file mode 100644 index 00000000..3ed6d57e --- /dev/null +++ b/crates/sql/src/postgres/types.rs @@ -0,0 +1,38 @@ +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::postgres::{PgArgumentBuffer, PgHasArrayType, PgTypeInfo, PgValueRef}; +use sqlx::{Decode, Encode, Postgres, Type}; + +use crate::types::SqlFelt; + +impl Type for SqlFelt { + fn type_info() -> PgTypeInfo { + PgTypeInfo::with_name("felt252") + } + + fn compatible(ty: &PgTypeInfo) -> bool { + *ty == PgTypeInfo::with_name("felt252") || <[u8] as Type>::compatible(ty) + } +} + +impl PgHasArrayType for SqlFelt { + fn array_type_info() -> PgTypeInfo { + PgTypeInfo::with_name("_felt252") + } +} + +impl Encode<'_, Postgres> for SqlFelt { + fn encode_by_ref(&self, buf: &mut PgArgumentBuffer) -> Result { + <&[u8] as Encode>::encode(self.0.as_slice(), buf) + } +} + +impl Decode<'_, Postgres> for SqlFelt { + fn decode(value: PgValueRef<'_>) -> Result { + let bytes = <&[u8] as Decode>::decode(value)?; + let arr: [u8; 32] = bytes + .try_into() + .map_err(|_| format!("expected 32 bytes for felt252, got {}", bytes.len()))?; + Ok(SqlFelt(arr)) + } +} diff --git a/crates/sql/src/query.rs b/crates/sql/src/query.rs new file mode 100644 index 00000000..fe916a37 --- /dev/null +++ b/crates/sql/src/query.rs @@ -0,0 +1,315 @@ +use crate::SqlxResult; +use futures::future::BoxFuture; +use itertools::Itertools; +use sqlx::{Database, Executor, Transaction}; +use std::fmt::Display; +use std::sync::Arc; + +pub trait Executable { + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't; +} + +impl Executable for &str +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self).await?; + Ok(()) + }) + } +} + +impl Executable for &String +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self.as_str()).await?; + Ok(()) + }) + } +} + +impl Executable for String +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self.as_str()).await?; + Ok(()) + }) + } +} + +impl Executable for FlexStr +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + transaction.execute(self.as_ref()).await?; + Ok(()) + }) + } +} + +#[derive(Debug, Clone)] +pub enum FlexStr { + Owned(String), + Static(&'static str), + Shared(Arc), +} + +impl Display for FlexStr { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.as_ref().fmt(f) + } +} + +impl FlexStr { + pub fn as_str(&self) -> &str { + match self { + FlexStr::Owned(s) => s.as_str(), + FlexStr::Shared(s) => s, + FlexStr::Static(s) => s, + } + } +} + +impl AsRef for FlexStr { + fn as_ref(&self) -> &str { + self.as_str() + } +} + +impl PartialEq for FlexStr { + fn eq(&self, other: &str) -> bool { + self.as_ref() == other + } +} + +/// A query with optional bind arguments. +/// +/// SQL can be any `FlexStr` (String, Arc, &'static str). +/// The `Bound` variant carries SQL + pre-built arguments. +/// The per-database `Executable` impls handle the lifetime requirements: +/// Postgres needs no special treatment; SQLite uses an unsafe lifetime extension +/// that is sound because the `FlexStr` outlives the `.await` point. +pub struct FlexQuery { + pub(crate) sql: FlexStr, + pub(crate) args: Option<::Arguments<'static>>, +} + +impl Clone for FlexQuery +where + DB::Arguments<'static>: Clone, +{ + fn clone(&self) -> Self { + Self { + sql: self.sql.clone(), + args: self.args.clone(), + } + } +} + +impl std::fmt::Debug for FlexQuery +where + DB::Arguments<'static>: std::fmt::Debug, +{ + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FlexQuery") + .field("sql", &self.sql) + .field("args", &self.args) + .finish() + } +} + +impl FlexQuery { + pub fn new(sql: impl Into, args: ::Arguments<'static>) -> Self { + FlexQuery { + sql: sql.into(), + args: Some(args), + } + } + + pub fn from_sql(sql: impl Into) -> Self { + FlexQuery { + sql: sql.into(), + args: None, + } + } +} + +impl PartialEq for FlexQuery { + fn eq(&self, other: &str) -> bool { + self.sql.as_ref() == other + } +} + +impl From for FlexStr { + fn from(s: String) -> Self { + FlexStr::Owned(s) + } +} + +impl From<&'static str> for FlexStr { + fn from(s: &'static str) -> Self { + FlexStr::Static(s) + } +} + +impl From> for FlexStr { + fn from(s: Arc) -> Self { + FlexStr::Shared(s) + } +} + +impl From for FlexQuery { + fn from(sql: FlexStr) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From<&'static str> for FlexQuery { + fn from(sql: &'static str) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From for FlexQuery { + fn from(sql: String) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From> for FlexQuery { + fn from(sql: Arc) -> Self { + FlexQuery::from_sql(sql) + } +} + +impl From<(S, A)> for FlexQuery +where + S: Into, + A: Into<::Arguments<'static>>, +{ + fn from((sql, args): (S, A)) -> Self { + FlexQuery::new(sql, args.into()) + } +} + +pub trait Queries { + fn add(&mut self, query: impl Into>); + fn adds(&mut self, queries: impl IntoIterator>>); +} + +impl Queries for Vec> { + fn add(&mut self, query: impl Into>) { + self.push(query.into()); + } + fn adds(&mut self, queries: impl IntoIterator>>) { + self.extend(queries.into_iter().map_into()); + } +} + +impl Executable for &[String; N] +where + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for query in self { + transaction.execute(query.as_str()).await?; + } + Ok(()) + }) + } +} + +impl + Send> Executable for Vec { + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} + +impl<'a, DB: Database, T> Executable for &'a Vec +where + &'a T: Executable + Send, + T: Send + Sync, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} + +impl<'a, DB: Database, T> Executable for &'a [T] +where + &'a T: Executable + Send, + T: Send + Sync, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} + +impl Executable for [T; N] +where + T: Executable + Send, +{ + fn execute<'t>(self, transaction: &'t mut Transaction<'_, DB>) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + for item in self { + item.execute(transaction).await?; + } + Ok(()) + }) + } +} diff --git a/crates/sql/src/runtime.rs b/crates/sql/src/runtime.rs new file mode 100644 index 00000000..e9c16bec --- /dev/null +++ b/crates/sql/src/runtime.rs @@ -0,0 +1,58 @@ +use std::str::FromStr; + +use crate::connection::DbBackend; +use crate::{DbPoolOptions, PoolConfig, SqlxError, SqlxResult}; +use sqlx::postgres::PgConnectOptions; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{Pool, Postgres, Sqlite}; + +#[derive(Clone)] +pub enum DbPool { + Postgres(Pool), + Sqlite(Pool), +} + +#[derive(Debug, Clone)] +pub enum DbConnectionOptions { + Postgres(PgConnectOptions), + Sqlite(SqliteConnectOptions), +} + +impl FromStr for DbConnectionOptions { + type Err = SqlxError; + fn from_str(s: &str) -> Result { + match DbBackend::from_str(s)? { + DbBackend::Postgres => PgConnectOptions::from_str(s).map(DbConnectionOptions::Postgres), + DbBackend::Sqlite => SqliteConnectOptions::from_str(s).map(DbConnectionOptions::Sqlite), + } + } +} + +impl PoolConfig { + pub async fn connect_any(&self) -> SqlxResult { + match DbBackend::try_from(self.url.as_str()) { + Ok(DbBackend::Postgres) => self.connect::().await.map(DbPool::Postgres), + Ok(DbBackend::Sqlite) => self.connect::().await.map(DbPool::Sqlite), + Err(err) => Err(SqlxError::Configuration(err.into())), + } + } +} + +impl DbPoolOptions { + pub async fn connect_any(&self, url: &str) -> SqlxResult { + match DbBackend::try_from(url) { + Ok(DbBackend::Postgres) => self.connect::(url).await.map(DbPool::Postgres), + Ok(DbBackend::Sqlite) => self.connect::(url).await.map(DbPool::Sqlite), + Err(err) => Err(SqlxError::Configuration(err.into())), + } + } + + pub async fn connect_any_with(&self, options: DbConnectionOptions) -> SqlxResult { + match options { + DbConnectionOptions::Postgres(opts) => { + self.connect_with(opts).await.map(DbPool::Postgres) + } + DbConnectionOptions::Sqlite(opts) => self.connect_with(opts).await.map(DbPool::Sqlite), + } + } +} diff --git a/crates/sql/src/sqlite/migrate.rs b/crates/sql/src/sqlite/migrate.rs new file mode 100644 index 00000000..11c1ca22 --- /dev/null +++ b/crates/sql/src/sqlite/migrate.rs @@ -0,0 +1,160 @@ +use crate::AcquiredSchema; +use futures::future::BoxFuture; +use sqlx::migrate::{AppliedMigration, Migrate, MigrateError, Migration}; +use sqlx::{query, query_as, Acquire, Executor, Sqlite, SqliteConnection}; +use std::borrow::Cow; +use std::time::{Duration, Instant}; + +impl AcquiredSchema { + fn table_name(&self) -> Cow<'static, str> { + Cow::Owned(format!("_sqlx_migrations_{}", self.schema)) + } +} + +impl Migrate for AcquiredSchema { + fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { + let table_name = self.table_name(); + self.connection + .execute( + format!( + r#" +CREATE TABLE IF NOT EXISTS "{table_name}" ( + version BIGINT PRIMARY KEY, + description TEXT NOT NULL, + installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + success BOOLEAN NOT NULL, + checksum BLOB NOT NULL, + execution_time BIGINT NOT NULL +); + "# + ) + .as_str(), + ) + .await?; + + Ok(()) + }) + } + + fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + let table_name = self.table_name(); + let row: Option<(i64,)> = query_as( + format!( + r#"SELECT version FROM "{table_name}" WHERE success = false ORDER BY version LIMIT 1"# + ) + .as_str(), + ) + .fetch_optional(&mut self.connection) + .await?; + + Ok(row.map(|row| row.0)) + }) + } + + fn list_applied_migrations( + &mut self, + ) -> BoxFuture<'_, Result, MigrateError>> { + Box::pin(async move { + let table_name = self.table_name(); + let rows: Vec<(i64, Vec)> = query_as( + format!(r#"SELECT version, checksum FROM "{table_name}" ORDER BY version"#) + .as_str(), + ) + .fetch_all(&mut self.connection) + .await?; + + Ok(rows + .into_iter() + .map(|(version, checksum)| AppliedMigration { + version, + checksum: checksum.into(), + }) + .collect()) + }) + } + + fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { Ok(()) }) + } + + fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { + Box::pin(async move { Ok(()) }) + } + + fn apply<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let table_name = self.table_name(); + let mut tx = self.begin().await?; + let start = Instant::now(); + + let _ = tx + .execute(&*migration.sql) + .await + .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; + + let _ = query( + format!( + r#" +INSERT INTO "{table_name}" (version, description, success, checksum, execution_time) +VALUES (?1, ?2, TRUE, ?3, -1) + "# + ) + .as_str(), + ) + .bind(migration.version) + .bind(&*migration.description) + .bind(&*migration.checksum) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + let elapsed = start.elapsed(); + + #[allow(clippy::cast_possible_truncation)] + let _ = query( + format!( + r#" +UPDATE "{table_name}" +SET execution_time = ?1 +WHERE version = ?2 + "# + ) + .as_str(), + ) + .bind(elapsed.as_nanos() as i64) + .bind(migration.version) + .execute(&mut self.connection) + .await?; + + Ok(elapsed) + }) + } + + fn revert<'e: 'm, 'm>( + &'e mut self, + migration: &'m Migration, + ) -> BoxFuture<'m, Result> { + Box::pin(async move { + let table_name = self.table_name(); + let mut tx = self.begin().await?; + let start = Instant::now(); + + let _ = tx.execute(&*migration.sql).await?; + + let _ = query(format!(r#"DELETE FROM "{table_name}" WHERE version = ?1"#).as_str()) + .bind(migration.version) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + + Ok(start.elapsed()) + }) + } +} diff --git a/crates/sql/src/sqlite/mod.rs b/crates/sql/src/sqlite/mod.rs new file mode 100644 index 00000000..d797fb48 --- /dev/null +++ b/crates/sql/src/sqlite/mod.rs @@ -0,0 +1,68 @@ +pub mod migrate; +pub mod types; + +use futures::future::BoxFuture; +pub use sqlx::sqlite::SqliteArguments; +use sqlx::{Executor, SqliteTransaction}; +pub use sqlx::{Sqlite, SqlitePool}; + +use sqlx::sqlite::SqliteConnectOptions; +use std::str::FromStr; + +use crate::{Executable, SqlxResult}; + +pub type SqliteQuery = super::FlexQuery; + +pub trait SqliteDbConnection: super::PoolExt {} +impl> SqliteDbConnection for T {} + +pub fn is_sqlite_memory_path(path: &str) -> bool { + path == ":memory:" + || path == "sqlite::memory:" + || path == "sqlite://:memory:" + || path.contains("mode=memory") +} + +pub fn sqlite_connect_options(path: &str) -> Result { + if path == ":memory:" || path == "sqlite::memory:" { + return SqliteConnectOptions::from_str("sqlite::memory:"); + } + + let options = if path.starts_with("sqlite:") { + SqliteConnectOptions::from_str(path)? + } else { + SqliteConnectOptions::new().filename(path) + }; + + if path.starts_with("sqlite:") && path.contains("mode=") { + Ok(options) + } else { + Ok(options.create_if_missing(true)) + } +} + +#[allow(unsafe_code)] +impl Executable for SqliteQuery { + fn execute<'t>(self, transaction: &'t mut SqliteTransaction) -> BoxFuture<'t, SqlxResult<()>> + where + Self: 't, + { + Box::pin(async move { + match self.args { + Some(args) => { + // SAFETY: `self.sql` is moved into this async block and lives for + // its entire duration. The extended reference is only used by + // `query_with` which is immediately awaited, so it cannot outlive + // the backing data in `FlexStr`. + let sql_ref: &'static str = + unsafe { std::mem::transmute::<&str, &'static str>(self.sql.as_ref()) }; + transaction.execute(sqlx::query_with(sql_ref, args)).await?; + } + None => { + transaction.execute(self.sql.as_ref()).await?; + } + } + Ok(()) + }) + } +} diff --git a/crates/sql/src/sqlite/types.rs b/crates/sql/src/sqlite/types.rs new file mode 100644 index 00000000..8d3ad9ab --- /dev/null +++ b/crates/sql/src/sqlite/types.rs @@ -0,0 +1,51 @@ +use crate::types::SqlFelt; +use sqlx::encode::IsNull; +use sqlx::error::BoxDynError; +use sqlx::{Decode, Encode, Sqlite, Type}; + +impl Type for SqlFelt { + fn type_info() -> ::TypeInfo { + >::type_info() + } +} + +impl<'q> Encode<'q, Sqlite> for SqlFelt { + fn encode_by_ref( + &self, + buf: &mut ::ArgumentBuffer<'q>, + ) -> Result { + let mut hex = String::with_capacity(66); + hex.push_str("0x"); + for byte in &self.0 { + use std::fmt::Write; + write!(hex, "{byte:02x}").unwrap(); + } + Encode::::encode(hex, buf) + } +} + +impl Decode<'_, Sqlite> for SqlFelt { + fn decode(value: ::ValueRef<'_>) -> Result { + let s = >::decode(value)?; + let s = s.strip_prefix("0x").unwrap_or(&s); + if s.len() != 64 { + return Err(format!("expected 64 hex chars for felt252, got {}", s.len()).into()); + } + let mut arr = [0u8; 32]; + for (i, chunk) in s.as_bytes().chunks(2).enumerate() { + let hi = hex_nibble(chunk[0])?; + let lo = hex_nibble(chunk[1])?; + arr[i] = (hi << 4) | lo; + } + Ok(SqlFelt(arr)) + } +} + +fn hex_nibble(c: u8) -> Result { + match c { + b'0'..=b'9' => Ok(c - b'0'), + b'a'..=b'f' => Ok(c - b'a' + 10), + b'A'..=b'F' => Ok(c - b'A' + 10), + _ => Err(format!("invalid hex char: {}", c as char).into()), + } +} diff --git a/crates/sql/src/types.rs b/crates/sql/src/types.rs new file mode 100644 index 00000000..347e1aef --- /dev/null +++ b/crates/sql/src/types.rs @@ -0,0 +1,22 @@ +use starknet_types_raw::Felt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct SqlFelt(pub [u8; 32]); + +impl From for Felt { + fn from(value: SqlFelt) -> Self { + value.0.into() + } +} + +impl From for SqlFelt { + fn from(value: Felt) -> Self { + SqlFelt(value.to_be_bytes()) + } +} + +impl From<&Felt> for SqlFelt { + fn from(value: &Felt) -> Self { + SqlFelt(value.to_be_bytes()) + } +} diff --git a/crates/sqlite/Cargo.toml b/crates/sqlite/Cargo.toml deleted file mode 100644 index 08f62825..00000000 --- a/crates/sqlite/Cargo.toml +++ /dev/null @@ -1,19 +0,0 @@ -[package] -name = "torii-sqlite" -version = "0.1.0" -edition = "2021" -description = "SQLite connection helpers for Torii storage crates" - -[dependencies] -async-trait.workspace = true -futures.workspace = true -sqlx = { workspace = true, features = [ - "sqlite", - "runtime-tokio-rustls", - "migrate", -] } - -torii-common.workspace = true - -[lints] -workspace = true diff --git a/crates/sqlite/src/db.rs b/crates/sqlite/src/db.rs deleted file mode 100644 index c9f4a200..00000000 --- a/crates/sqlite/src/db.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::ops::Deref; -use std::str::FromStr; - -pub use async_trait::async_trait; -use sqlx::migrate::Migrator; -use sqlx::sqlite::SqliteConnectOptions; -use sqlx::Sqlite; -pub use sqlx::{SqlitePool, Transaction}; -use torii_common::sql::SqlxResult; - -use crate::migration::NamespaceMigrator; - -#[async_trait] -pub trait SqliteConnection { - fn pool(&self) -> &SqlitePool; - - async fn begin(&self) -> SqlxResult> { - Ok(self.pool().begin().await?) - } - - async fn migrate(&self, namespace: Option<&'static str>, migrator: Migrator) -> SqlxResult<()> { - let result = match namespace { - Some(namespace) => { - NamespaceMigrator::new(namespace, migrator) - .run(self.pool()) - .await - } - None => migrator.run(self.pool()).await, - }; - Ok(result?) - } - - async fn execute_queries(&self, queries: &[String]) -> SqlxResult<()> { - let mut transaction = self.begin().await?; - for query in queries { - sqlx::query(query).execute(&mut *transaction).await?; - } - transaction.commit().await - } -} - -#[allow(clippy::explicit_auto_deref)] -#[async_trait] -impl + Send + Sync + 'static> SqliteConnection for T { - fn pool(&self) -> &SqlitePool { - &**self - } -} - -pub fn is_sqlite_memory_path(path: &str) -> bool { - path == ":memory:" - || path == "sqlite::memory:" - || path == "sqlite://:memory:" - || path.contains("mode=memory") -} - -pub fn sqlite_connect_options(path: &str) -> Result { - if path == ":memory:" || path == "sqlite::memory:" { - return SqliteConnectOptions::from_str("sqlite::memory:"); - } - - let options = if path.starts_with("sqlite:") { - SqliteConnectOptions::from_str(path)? - } else { - SqliteConnectOptions::new().filename(path) - }; - - if path.starts_with("sqlite:") && path.contains("mode=") { - Ok(options) - } else { - Ok(options.create_if_missing(true)) - } -} diff --git a/crates/sqlite/src/lib.rs b/crates/sqlite/src/lib.rs deleted file mode 100644 index e0e1dc65..00000000 --- a/crates/sqlite/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub mod db; -pub mod migration; - -pub use db::{is_sqlite_memory_path, sqlite_connect_options, SqliteConnection}; diff --git a/crates/sqlite/src/migration.rs b/crates/sqlite/src/migration.rs deleted file mode 100644 index 8b9e3eef..00000000 --- a/crates/sqlite/src/migration.rs +++ /dev/null @@ -1,320 +0,0 @@ -use futures::future::BoxFuture; -use sqlx::migrate::{ - AppliedMigration, Migrate, MigrateError, Migration, MigrationSource, Migrator, -}; -use sqlx::sqlite::SqliteConnection; -use sqlx::{query, query_as, Acquire, Executor, Sqlite}; -use std::borrow::Cow; -use std::collections::{HashMap, HashSet}; -use std::ops::{Deref, DerefMut}; -use std::slice; -use std::time::{Duration, Instant}; - -pub struct NamespaceMigrator { - pub migrator: Migrator, - pub namespace: &'static str, -} - -pub struct SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a>, -{ - pub connection: >::Connection, - pub namespace: &'static str, -} - -impl<'a, A> Deref for SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite>, -{ - type Target = SqliteConnection; - - fn deref(&self) -> &Self::Target { - &self.connection - } -} - -impl<'a, A> DerefMut for SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite>, -{ - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.connection - } -} - -impl<'a, A> SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite>, -{ - fn table_name(&self) -> Cow<'static, str> { - Cow::Owned(format!("_sqlx_migrations_{}", self.namespace)) - } -} - -impl<'a, A> Migrate for SqliteAcquiredNamespace<'a, A> -where - A: Acquire<'a, Database = Sqlite> + Executor<'a, Database = Sqlite>, -{ - fn ensure_migrations_table(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { - let table_name = self.table_name(); - self.connection - .execute( - format!( - r#" -CREATE TABLE IF NOT EXISTS "{table_name}" ( - version BIGINT PRIMARY KEY, - description TEXT NOT NULL, - installed_on TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - success BOOLEAN NOT NULL, - checksum BLOB NOT NULL, - execution_time BIGINT NOT NULL -); - "# - ) - .as_str(), - ) - .await?; - - Ok(()) - }) - } - - fn dirty_version(&mut self) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - let table_name = self.table_name(); - let row: Option<(i64,)> = query_as( - format!( - r#"SELECT version FROM "{table_name}" WHERE success = false ORDER BY version LIMIT 1"# - ) - .as_str(), - ) - .fetch_optional(&mut *self.connection) - .await?; - - Ok(row.map(|row| row.0)) - }) - } - - fn list_applied_migrations( - &mut self, - ) -> BoxFuture<'_, Result, MigrateError>> { - Box::pin(async move { - let table_name = self.table_name(); - let rows: Vec<(i64, Vec)> = query_as( - format!(r#"SELECT version, checksum FROM "{table_name}" ORDER BY version"#) - .as_str(), - ) - .fetch_all(&mut *self.connection) - .await?; - - Ok(rows - .into_iter() - .map(|(version, checksum)| AppliedMigration { - version, - checksum: checksum.into(), - }) - .collect()) - }) - } - - fn lock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { Ok(()) }) - } - - fn unlock(&mut self) -> BoxFuture<'_, Result<(), MigrateError>> { - Box::pin(async move { Ok(()) }) - } - - fn apply<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let table_name = self.table_name(); - let mut tx = self.begin().await?; - let start = Instant::now(); - - let _ = tx - .execute(&*migration.sql) - .await - .map_err(|e| MigrateError::ExecuteMigration(e, migration.version))?; - - let _ = query( - format!( - r#" -INSERT INTO "{table_name}" (version, description, success, checksum, execution_time) -VALUES (?1, ?2, TRUE, ?3, -1) - "# - ) - .as_str(), - ) - .bind(migration.version) - .bind(&*migration.description) - .bind(&*migration.checksum) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - let elapsed = start.elapsed(); - - #[allow(clippy::cast_possible_truncation)] - let _ = query( - format!( - r#" -UPDATE "{table_name}" -SET execution_time = ?1 -WHERE version = ?2 - "# - ) - .as_str(), - ) - .bind(elapsed.as_nanos() as i64) - .bind(migration.version) - .execute(&mut *self.connection) - .await?; - - Ok(elapsed) - }) - } - - fn revert<'e: 'm, 'm>( - &'e mut self, - migration: &'m Migration, - ) -> BoxFuture<'m, Result> { - Box::pin(async move { - let table_name = self.table_name(); - let mut tx = self.begin().await?; - let start = Instant::now(); - - let _ = tx.execute(&*migration.sql).await?; - - let _ = query(format!(r#"DELETE FROM "{table_name}" WHERE version = ?1"#).as_str()) - .bind(migration.version) - .execute(&mut *tx) - .await?; - - tx.commit().await?; - - Ok(start.elapsed()) - }) - } -} - -impl NamespaceMigrator { - pub const fn new(namespace: &'static str, migrator: Migrator) -> Self { - Self { - migrator, - namespace, - } - } - - pub async fn new_from_source<'s, S>( - namespace: &'static str, - source: S, - ) -> Result - where - S: MigrationSource<'s>, - { - Migrator::new(source).await.map(|migrator| Self { - migrator, - namespace, - }) - } - - pub fn set_ignore_missing(&mut self, ignore_missing: bool) -> &Self { - self.migrator.ignore_missing = ignore_missing; - self - } - - pub fn set_locking(&mut self, locking: bool) -> &Self { - self.migrator.locking = locking; - self - } - - pub fn iter(&self) -> slice::Iter<'_, Migration> { - self.migrator.migrations.iter() - } - - pub fn version_exists(&self, version: i64) -> bool { - self.iter().any(|migration| migration.version == version) - } - - pub async fn run<'a, A>(&self, migrator: A) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Sqlite>, - SqliteAcquiredNamespace<'a, A>: Migrate, - { - let mut conn = SqliteAcquiredNamespace { - connection: migrator.acquire().await?, - namespace: self.namespace, - }; - self.migrator.run_direct(&mut conn).await - } - - pub async fn undo<'a, A>(&self, migrator: A, target: i64) -> Result<(), MigrateError> - where - A: Acquire<'a, Database = Sqlite>, - SqliteAcquiredNamespace<'a, A>: Migrate, - { - let mut conn = SqliteAcquiredNamespace { - connection: migrator.acquire().await?, - namespace: self.namespace, - }; - - if self.migrator.locking { - conn.lock().await?; - } - - conn.ensure_migrations_table().await?; - - if let Some(version) = conn.dirty_version().await? { - return Err(MigrateError::Dirty(version)); - } - - let applied_migrations = conn.list_applied_migrations().await?; - validate_applied_migrations(&applied_migrations, &self.migrator)?; - - let applied_migrations: HashMap<_, _> = applied_migrations - .into_iter() - .map(|migration| (migration.version, migration)) - .collect(); - - for migration in self - .iter() - .rev() - .filter(|migration| migration.migration_type.is_down_migration()) - .filter(|migration| applied_migrations.contains_key(&migration.version)) - .filter(|migration| migration.version > target) - { - conn.revert(migration).await?; - } - - if self.migrator.locking { - conn.unlock().await?; - } - - Ok(()) - } -} - -fn validate_applied_migrations( - applied_migrations: &[AppliedMigration], - migrator: &Migrator, -) -> Result<(), MigrateError> { - if migrator.ignore_missing { - return Ok(()); - } - - let migrations: HashSet<_> = migrator.iter().map(|migration| migration.version).collect(); - - for applied_migration in applied_migrations { - if !migrations.contains(&applied_migration.version) { - return Err(MigrateError::VersionMissing(applied_migration.version)); - } - } - - Ok(()) -} diff --git a/crates/starknet/Cargo.toml b/crates/starknet/Cargo.toml deleted file mode 100644 index 9517b699..00000000 --- a/crates/starknet/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "torii-starknet" -version = "0.1.0" -edition = "2021" -description = "StarkNet utilities for Torii" - -[dependencies] -starknet = { workspace = true, optional = true } -serde = { workspace = true, optional = true } - -[dev-dependencies] -pretty_assertions_sorted = "1.2.3" -assert_matches = "1.5.0" -serde_json.workspace = true -bincode = { workspace = true, features = ["serde"] } - -[features] -default = ["serde"] -starknet = ["dep:starknet"] -serde = ["dep:serde"] - -[lints] -workspace = true diff --git a/crates/starknet/src/event.rs b/crates/starknet/src/event.rs deleted file mode 100644 index d04e86d6..00000000 --- a/crates/starknet/src/event.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::Felt; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct EmittedEvent { - pub from_address: Felt, - pub keys: Vec, - pub data: Vec, - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub block_hash: Option, - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - pub block_number: Option, - pub transaction_hash: Felt, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -pub struct Event { - pub from_address: Felt, - pub keys: Vec, - pub data: Vec, -} diff --git a/crates/starknet/src/felt.rs b/crates/starknet/src/felt.rs deleted file mode 100644 index 90dbb411..00000000 --- a/crates/starknet/src/felt.rs +++ /dev/null @@ -1,714 +0,0 @@ -use std::borrow::Cow; -use std::error::Error; - -/// Starknet Field Element. -/// -/// A field element is a number 0..p-1 with p=2^{251}+17*2^{192}+1, and it forms -/// the basic building block of most Starknet interactions. -#[repr(transparent)] -#[derive(Clone, Copy, PartialEq, Hash, Eq, PartialOrd, Ord)] -pub struct Felt(pub(crate) [u8; 32]); - -const MODULUS_U64: [u64; 4] = [576460752303423505u64, 0, 0, 1]; - -impl std::fmt::Debug for Felt { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "0x{self:x}") - } -} - -impl std::fmt::Display for Felt { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:x}") - } -} - -impl std::fmt::LowerHex for Felt { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let bytes = &self.0; - let first = bytes.iter().position(|&b| b != 0).unwrap_or(31); - write!(f, "{:x}", bytes[first])?; - bytes[first + 1..] - .iter() - .try_for_each(|&b| write!(f, "{b:02x}")) - } -} - -impl std::fmt::UpperHex for Felt { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let bytes = &self.0; - let first = bytes.iter().position(|&b| b != 0).unwrap_or(31); - write!(f, "{:X}", bytes[first])?; - bytes[first + 1..] - .iter() - .try_for_each(|&b| write!(f, "{b:02X}")) - } -} -impl Default for Felt { - fn default() -> Self { - Felt::ZERO - } -} - -/// Error returned by [Felt::from_be_bytes] indicating the maximum field value -/// was exceeded. -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct OverflowError; - -impl Error for OverflowError {} - -const OVERFLOW_MSG: &str = "The maximum field value was exceeded."; - -impl std::fmt::Display for OverflowError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(OVERFLOW_MSG) - } -} - -impl Felt { - pub const ZERO: Felt = Felt([0u8; 32]); - pub const ONE: Felt = Self::from_u64(1); - - /// Return true if the element is zero. - pub fn is_zero(&self) -> bool { - self == &Felt::ZERO - } - - /// Returns the big-endian representation of this [Felt]. - pub const fn to_be_bytes(self) -> [u8; 32] { - self.0 - } - - /// Returns the little-endian representation of this [Felt]. - pub fn to_le_bytes(self) -> [u8; 32] { - let mut tmp = self.0; - tmp.reverse(); - tmp - } - - /// Big-endian representation of this [Felt]. - pub const fn as_be_bytes(&self) -> &[u8; 32] { - &self.0 - } - - /// Big-endian mutable representation of this [Felt]. - pub fn as_mut_be_bytes(&mut self) -> &mut [u8; 32] { - &mut self.0 - } - - /// Convenience function which extends [Felt::from_be_bytes] to work with - /// slices. - pub const fn from_be_slice(bytes: &[u8]) -> Result { - if bytes.len() > 32 { - return Err(OverflowError); - } - - let mut buf = [0u8; 32]; - let mut index = 0; - loop { - if index == bytes.len() { - break; - } - buf[32 - bytes.len() + index] = bytes[index]; - index += 1; - } - - Felt::from_be_bytes(buf) - } - - /// Creates a [Felt] from big-endian bytes. - /// - /// Returns [OverflowError] if not less than the field modulus. - pub const fn from_be_bytes(bytes: [u8; 32]) -> Result { - #[rustfmt::skip] - let limbs = [ - u64::from_be_bytes([ - bytes[0], bytes[1], bytes[2], bytes[3], - bytes[4], bytes[5], bytes[6], bytes[7], - ]), - u64::from_be_bytes([ - bytes[8], bytes[9], bytes[10], bytes[11], - bytes[12], bytes[13], bytes[14], bytes[15], - ]), - u64::from_be_bytes([ - bytes[16], bytes[17], bytes[18], bytes[19], - bytes[20], bytes[21], bytes[22], bytes[23], - ]), - u64::from_be_bytes([ - bytes[24], bytes[25], bytes[26], bytes[27], - bytes[28], bytes[29], bytes[30], bytes[31], - ]), - ]; - - // Loop over each word, if all previous are equal and current is less, we are - // good. - let mut maybe_overflow = true; - let mut i = 0; - while i < 4 && maybe_overflow { - if limbs[i] < MODULUS_U64[i] { - maybe_overflow = false; - } else if limbs[i] > MODULUS_U64[i] { - return Err(OverflowError); - } - i += 1; - } - if maybe_overflow { - Err(OverflowError) - } else { - Ok(Felt(bytes)) - } - } - - /// Creates a Felt from a big-endian byte slice of up to 32 bytes. - /// Panics if slice is longer than 32 bytes. - pub fn from_bytes_be_slice(bytes: &[u8]) -> Self { - assert!(bytes.len() <= 32, "slice too long"); - let mut buf = [0u8; 32]; - buf[32 - bytes.len()..].copy_from_slice(bytes); - Self(buf) - } - - /// Creates a Felt from a little-endian byte slice of up to 32 bytes. - pub fn from_bytes_le_slice(bytes: &[u8]) -> Self { - assert!(bytes.len() <= 32, "slice too long"); - let mut buf = [0u8; 32]; - for (i, &b) in bytes.iter().enumerate() { - buf[31 - i] = b; - } - Self(buf) - } - - /// Returns `true` if the value of [`Felt`] is larger than `2^251 - 1`. - /// - /// Every [`Felt`] that is used to traverse a Merkle-Patricia Tree - /// must not exceed 251 bits, since 251 is the height of the tree. - pub const fn has_more_than_251_bits(&self) -> bool { - self.0[0] & 0b1111_1000 > 0 - } - - pub const fn from_u64(u: u64) -> Self { - const_expect!( - Self::from_be_slice(&u.to_be_bytes()), - "64 bits is less than 251 bits" - ) - } - - pub const fn from_u128(u: u128) -> Self { - const_expect!( - Self::from_be_slice(&u.to_be_bytes()), - "128 bits is less than 251 bits" - ) - } -} - -macro_rules! const_expect { - ($e:expr, $why:expr) => {{ - match $e { - Ok(x) => x, - Err(_) => panic!(concat!("Expectation failed: ", $why)), - } - }}; -} - -use const_expect; - -impl From for Felt { - fn from(value: u64) -> Self { - Self::from_u64(value) - } -} - -impl From for Felt { - fn from(value: usize) -> Self { - Self::from_u64(value.try_into().expect("ptr size is 64 bits")) - } -} - -impl From for Felt { - fn from(value: u128) -> Self { - Self::from_u128(value) - } -} - -impl From<[u8; 32]> for Felt { - fn from(bytes: [u8; 32]) -> Self { - Self(bytes) - } -} - -impl From<[u8; 31]> for Felt { - fn from(bytes: [u8; 31]) -> Self { - let mut buf = [0u8; 32]; - buf[1..].copy_from_slice(&bytes); - Self(buf) - } -} - -impl TryInto for Felt { - type Error = OverflowError; - - fn try_into(self) -> Result { - let initial_zeroes = self.0.iter().take_while(|b| **b == 0).count(); - const EXPECTED_ZEROES: usize = (32 - u128::BITS / u8::BITS) as usize; - - if initial_zeroes < EXPECTED_ZEROES { - return Err(OverflowError); - } - - let bytes = self.0[EXPECTED_ZEROES..] - .try_into() - .expect("Should match u128 size"); - Ok(u128::from_be_bytes(bytes)) - } -} - -impl TryInto for Felt { - type Error = OverflowError; - - fn try_into(self) -> Result { - let initial_zeroes = self.0.iter().take_while(|b| **b == 0).count(); - const EXPECTED_ZEROES: usize = (32 - u64::BITS / u8::BITS) as usize; - - if initial_zeroes < EXPECTED_ZEROES { - return Err(OverflowError); - } - let bytes = self.0[EXPECTED_ZEROES..] - .try_into() - .expect("Should match u64 size"); - Ok(u64::from_be_bytes(bytes)) - } -} - -impl Felt { - /// A convenience function which parses a hex string into a [Felt]. - /// - /// Supports both upper and lower case hex strings, as well as an optional - /// "0x" prefix. - pub const fn from_hex_str(hex_str: &str) -> Result { - const fn parse_hex_digit(digit: u8) -> Result { - match digit { - b'0'..=b'9' => Ok(digit - b'0'), - b'A'..=b'F' => Ok(digit - b'A' + 10), - b'a'..=b'f' => Ok(digit - b'a' + 10), - other => Err(HexParseError::InvalidNibble(other)), - } - } - - let bytes = hex_str.as_bytes(); - let start = if bytes.len() >= 2 && bytes[0] == b'0' && bytes[1] == b'x' { - 2 - } else { - 0 - }; - let len = bytes.len() - start; - - if len > 64 { - return Err(HexParseError::InvalidLength { - max: 64, - actual: bytes.len(), - }); - } - - let mut buf = [0u8; 32]; - - // We want the result in big-endian so reverse iterate over each pair of - // nibbles. let chunks = hex_str.as_bytes().rchunks_exact(2); - - // Handle a possible odd nibble remaining nibble. - if len % 2 == 1 { - let idx = len / 2; - buf[31 - idx] = match parse_hex_digit(bytes[start]) { - Ok(b) => b, - Err(e) => return Err(e), - }; - } - - let chunks = len / 2; - let mut chunk = 0; - - while chunk < chunks { - let lower = match parse_hex_digit(bytes[bytes.len() - chunk * 2 - 1]) { - Ok(b) => b, - Err(e) => return Err(e), - }; - let upper = match parse_hex_digit(bytes[bytes.len() - chunk * 2 - 2]) { - Ok(b) => b, - Err(e) => return Err(e), - }; - buf[31 - chunk] = (upper << 4) | lower; - chunk += 1; - } - - let felt = match Felt::from_be_bytes(buf) { - Ok(felt) => felt, - Err(OverflowError) => return Err(HexParseError::Overflow), - }; - Ok(felt) - } - - /// The first stage of conversion - skip leading zeros - fn skip_zeros(&self) -> (impl Iterator, usize, usize) { - // Skip all leading zero bytes - let it = self.0.iter().skip_while(|&&b| b == 0); - let num_bytes = it.clone().count(); - let skipped = self.0.len() - num_bytes; - // The first high nibble can be 0 - let start = if self.0[skipped] < 0x10 { 1 } else { 2 }; - // Number of characters to display - let len = start + num_bytes * 2; - (it, start, len) - } - - /// The second stage of conversion - map bytes to hex str - fn it_to_hex_str<'a>( - it: impl Iterator, - start: usize, - len: usize, - buf: &'a mut [u8], - ) -> &'a [u8] { - const LUT: [u8; 16] = *b"0123456789abcdef"; - buf[0] = b'0'; - // Same small lookup table is ~25% faster than hex::encode_from_slice 🤷 - it.enumerate().for_each(|(i, &b)| { - let idx = b as usize; - let pos = start + i * 2; - let x = [LUT[(idx & 0xf0) >> 4], LUT[idx & 0x0f]]; - buf[pos..pos + 2].copy_from_slice(&x); - }); - buf[1] = b'x'; - &buf[..len] - } - - /// A convenience function which produces a "0x" prefixed hex str slice in a - /// given buffer `buf` from a [Felt]. - /// Panics if `self.0.len() * 2 + 2 > buf.len()` - pub fn as_hex_str<'a>(&'a self, buf: &'a mut [u8]) -> &'a str { - let expected_buf_len = self.0.len() * 2 + 2; - assert!( - buf.len() >= expected_buf_len, - "buffer size is {}, expected at least {}", - buf.len(), - expected_buf_len - ); - - if !self.0.iter().any(|b| *b != 0) { - return "0x0"; - } - - let (it, start, len) = self.skip_zeros(); - let res = Self::it_to_hex_str(it, start, len, buf); - // Unwrap is safe because `buf` holds valid UTF8 characters. - std::str::from_utf8(res).unwrap() - } - - /// A convenience function which produces a "0x" prefixed hex string from a - /// [Felt]. - pub fn to_hex_str(&self) -> Cow<'static, str> { - if !self.0.iter().any(|b| *b != 0) { - return Cow::from("0x0"); - } - let (it, start, len) = self.skip_zeros(); - let mut buf = vec![0u8; len]; - Self::it_to_hex_str(it, start, len, &mut buf); - // Unwrap is safe as the buffer contains valid utf8 - String::from_utf8(buf).unwrap().into() - } -} - -/// Error returned by [Felt::from_hex_str] indicating an invalid hex string. -#[derive(Debug, PartialEq, Eq)] -pub enum HexParseError { - InvalidNibble(u8), - InvalidLength { max: usize, actual: usize }, - Overflow, -} - -impl Error for HexParseError {} - -impl From for HexParseError { - fn from(_: OverflowError) -> Self { - Self::Overflow - } -} - -impl std::fmt::Display for HexParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::InvalidNibble(n) => f.write_fmt(format_args!("Invalid nibble found: 0x{:x}", *n)), - Self::InvalidLength { max, actual } => { - f.write_fmt(format_args!("More than {} digits found: {}", *max, *actual)) - } - Self::Overflow => f.write_str(OVERFLOW_MSG), - } - } -} - -#[cfg(test)] -mod tests { - use pretty_assertions_sorted::assert_eq; - - use super::*; - - const MODULUS_U8: [u8; 32] = [ - 8, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, - ]; - - #[test] - fn bytes_round_trip() { - let original = [ - 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, - 0x0E, 0x0F, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1A, 0x1B, - 0x1C, 0x1D, 0x1E, 0x1F, - ]; - let hash = Felt::from_be_bytes(original).unwrap(); - let bytes = hash.to_be_bytes(); - assert_eq!(bytes, original); - } - - #[test] - fn from_bytes_overflow() { - // Field modulus - assert_eq!(Felt::from_be_bytes(MODULUS_U8), Err(OverflowError)); - // Field modulus - 1 - let mut max_val = MODULUS_U8; - max_val[31] -= 1; - Felt::from_be_bytes(max_val).unwrap(); - } - - mod from_be_slice { - use pretty_assertions_sorted::assert_eq; - - use super::*; - - #[test] - fn round_trip() { - let original = Felt::from_hex_str("abcdef0123456789").unwrap(); - let bytes = original.to_be_bytes(); - let result = Felt::from_be_slice(&bytes[..]).unwrap(); - - assert_eq!(result, original); - } - - #[test] - fn too_long() { - let original = Felt::from_hex_str("abcdef0123456789").unwrap(); - let mut bytes = original.to_be_bytes().to_vec(); - bytes.push(0); - Felt::from_be_slice(&bytes[..]).unwrap_err(); - } - - #[test] - fn short_slice() { - let original = Felt::from_hex_str("abcdef0123456789").unwrap(); - let bytes = original.to_be_bytes(); - let result = Felt::from_be_slice(&bytes[24..]); - - assert_eq!(result, Ok(original)); - } - - #[test] - fn max() { - let mut max_val = MODULUS_U8; - max_val[31] -= 1; - Felt::from_be_slice(&max_val[..]).unwrap(); - } - - #[test] - fn overflow() { - assert_eq!(Felt::from_be_slice(&MODULUS_U8[..]), Err(OverflowError)); - } - } - - mod fmt { - use pretty_assertions_sorted::assert_eq; - - use super::Felt; - - #[test] - fn debug() { - let hex_str = "1234567890abcdef000edcba0987654321"; - let felt = Felt::from_hex_str(hex_str).unwrap(); - let result = format!("{felt:?}"); - - let expected = format!("0x{felt}"); - - assert_eq!(result, expected); - } - - #[test] - fn fmt() { - let hex_str = "1234567890abcdef000edcba0987654321"; - let starkhash = Felt::from_hex_str(hex_str).unwrap(); - let result = format!("{starkhash:x}"); - - // We don't really care which casing is used by fmt. - assert_eq!(result.to_lowercase(), hex_str.to_lowercase()); - } - - #[test] - fn lower_hex() { - let hex_str = "1234567890abcdef000edcba0987654321"; - let starkhash = Felt::from_hex_str(hex_str).unwrap(); - let result = format!("{starkhash:x}"); - - assert_eq!(result, hex_str.to_lowercase()); - } - - #[test] - fn upper_hex() { - let hex_str = "1234567890abcdef000edcba0987654321"; - let starkhash = Felt::from_hex_str(hex_str).unwrap(); - let result = format!("{starkhash:X}"); - - assert_eq!(result, hex_str.to_uppercase()); - } - } - - mod from_hex_str { - use assert_matches::assert_matches; - use pretty_assertions_sorted::assert_eq; - - use super::*; - - /// Test hex string with its expected [Felt]. - fn test_data() -> (&'static str, Felt) { - let mut expected = [0; 32]; - expected[31] = 0xEF; - expected[30] = 0xCD; - expected[29] = 0xAB; - expected[28] = 0xef; - expected[27] = 0xcd; - expected[26] = 0xab; - expected[25] = 0x89; - expected[24] = 0x67; - expected[23] = 0x45; - expected[22] = 0x23; - expected[21] = 0x01; - let expected = Felt::from_be_bytes(expected).unwrap(); - - ("0123456789abcdefABCDEF", expected) - } - - #[test] - fn simple() { - let (test_str, expected) = test_data(); - let uut = Felt::from_hex_str(test_str).unwrap(); - assert_eq!(uut, expected); - } - - #[test] - fn prefix() { - let (test_str, expected) = test_data(); - let uut = Felt::from_hex_str(&format!("0x{test_str}")).unwrap(); - assert_eq!(uut, expected); - } - - #[test] - fn leading_zeros() { - let (test_str, expected) = test_data(); - let uut = Felt::from_hex_str(&format!("000000000{test_str}")).unwrap(); - assert_eq!(uut, expected); - } - - #[test] - fn prefix_and_leading_zeros() { - let (test_str, expected) = test_data(); - let uut = Felt::from_hex_str(&format!("0x000000000{test_str}")).unwrap(); - assert_eq!(uut, expected); - } - - #[test] - fn invalid_nibble() { - assert_matches!(Felt::from_hex_str("0x123z").unwrap_err(), HexParseError::InvalidNibble(n) => assert_eq!(n, b'z')); - } - - #[test] - fn invalid_len() { - assert_matches!(Felt::from_hex_str(&"1".repeat(65)).unwrap_err(), HexParseError::InvalidLength{max: 64, actual: n} => assert_eq!(n, 65)); - } - - #[test] - fn overflow() { - // Field modulus - let mut modulus = - "0x800000000000011000000000000000000000000000000000000000000000001".to_string(); - assert_eq!( - Felt::from_hex_str(&modulus).unwrap_err(), - HexParseError::Overflow - ); - // Field modulus - 1 - modulus.pop(); - modulus.push('0'); - Felt::from_hex_str(&modulus).unwrap(); - } - } - - mod to_hex_str { - use pretty_assertions_sorted::assert_eq; - - use super::*; - - const ODD: &str = "0x1234567890abcde"; - const EVEN: &str = "0x1234567890abcdef"; - const MAX: &str = "0x800000000000011000000000000000000000000000000000000000000000000"; - - #[test] - fn zero() { - assert_eq!(Felt::ZERO.to_hex_str(), "0x0"); - let mut buf = [0u8; 66]; - assert_eq!(Felt::ZERO.as_hex_str(&mut buf), "0x0"); - } - - #[test] - fn odd() { - let hash = Felt::from_hex_str(ODD).unwrap(); - assert_eq!(hash.to_hex_str(), ODD); - let mut buf = [0u8; 66]; - assert_eq!(hash.as_hex_str(&mut buf), ODD); - } - - #[test] - fn even() { - let hash = Felt::from_hex_str(EVEN).unwrap(); - assert_eq!(hash.to_hex_str(), EVEN); - let mut buf = [0u8; 66]; - assert_eq!(hash.as_hex_str(&mut buf), EVEN); - } - - #[test] - fn max() { - let hash = Felt::from_hex_str(MAX).unwrap(); - assert_eq!(hash.to_hex_str(), MAX); - let mut buf = [0u8; 66]; - assert_eq!(hash.as_hex_str(&mut buf), MAX); - } - - #[test] - #[should_panic(expected = "buffer size is 65, expected at least 66")] - fn buffer_too_small() { - let mut buf = [0u8; 65]; - Felt::ZERO.as_hex_str(&mut buf); - } - } - - mod has_more_than_251_bits { - use super::*; - - #[test] - fn has_251_bits() { - let mut bytes = [0xFFu8; 32]; - bytes[0] = 0x07; - let h = Felt::from_be_bytes(bytes).unwrap(); - assert!(!h.has_more_than_251_bits()); - } - - #[test] - fn has_252_bits() { - let mut bytes = [0u8; 32]; - bytes[0] = 0x08; - let h = Felt::from_be_bytes(bytes).unwrap(); - assert!(h.has_more_than_251_bits()); - } - } -} diff --git a/crates/starknet/src/lib.rs b/crates/starknet/src/lib.rs deleted file mode 100644 index 54ae3603..00000000 --- a/crates/starknet/src/lib.rs +++ /dev/null @@ -1,10 +0,0 @@ -pub mod event; -pub mod felt; - -#[cfg(feature = "serde")] -pub mod serde; - -#[cfg(feature = "starknet")] -pub mod starknet; - -pub use felt::Felt; diff --git a/crates/starknet/src/serde.rs b/crates/starknet/src/serde.rs deleted file mode 100644 index 1aece7f2..00000000 --- a/crates/starknet/src/serde.rs +++ /dev/null @@ -1,134 +0,0 @@ -use serde::{Deserialize, Serialize}; - -use crate::Felt; - -impl Serialize for Felt { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - if serializer.is_human_readable() { - serializer.serialize_str(&self.to_hex_str()) - } else { - let bytes = self.as_be_bytes(); - let first = bytes.iter().position(|&b| b != 0).unwrap_or(31); - serializer.serialize_bytes(&bytes[first..]) - } - } -} - -impl<'de> Deserialize<'de> for Felt { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - if deserializer.is_human_readable() { - deserializer.deserialize_str(FeltVisitor) - } else { - deserializer.deserialize_bytes(FeltVisitor) - } - } -} - -struct FeltVisitor; - -impl serde::de::Visitor<'_> for FeltVisitor { - type Value = Felt; - - fn expecting(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "a hex string prefixed with 0x or up to 32 bytes") - } - - fn visit_str(self, value: &str) -> Result { - Felt::from_hex_str(value).map_err(|e| E::custom(e.to_string())) - } - - fn visit_bytes(self, value: &[u8]) -> Result { - if value.len() > 32 { - return Err(E::invalid_length(value.len(), &self)); - } - let mut buf = [0u8; 32]; - buf[32 - value.len()..].copy_from_slice(value); - Ok(Felt(buf)) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json; - - #[test] - fn serialize_zero_human_readable() { - let felt = Felt::ZERO; - assert_eq!(serde_json::to_string(&felt).unwrap(), "\"0x0\""); - } - - #[test] - fn serialize_one_human_readable() { - let felt = Felt::ONE; - assert_eq!(serde_json::to_string(&felt).unwrap(), "\"0x1\""); - } - - #[test] - fn serialize_human_readable_round_trip() { - let original = Felt::from_hex_str("0xdeadbeef").unwrap(); - let serialized = serde_json::to_string(&original).unwrap(); - let deserialized: Felt = serde_json::from_str(&serialized).unwrap(); - assert_eq!(original, deserialized); - } - - #[test] - fn serialize_max_human_readable() { - let max = - Felt::from_hex_str("0x800000000000011000000000000000000000000000000000000000000000000") - .unwrap(); - let serialized = serde_json::to_string(&max).unwrap(); - let deserialized: Felt = serde_json::from_str(&serialized).unwrap(); - assert_eq!(max, deserialized); - } - - #[test] - fn deserialize_overflow_fails() { - // Field modulus - should fail - assert!(serde_json::from_str::( - "\"0x800000000000011000000000000000000000000000000000000000000000001\"" - ) - .is_err()); - } - - #[test] - fn serialize_binary_round_trip() { - let original = Felt::from_hex_str("0xdeadbeef").unwrap(); - let serialized = - bincode::serde::encode_to_vec(original, bincode::config::standard()).unwrap(); - let deserialized: Felt = - bincode::serde::decode_from_slice(&serialized, bincode::config::standard()) - .unwrap() - .0; - assert_eq!(original, deserialized); - } - - #[test] - fn serialize_binary_zero() { - let felt = Felt::ZERO; - let serialized = bincode::serde::encode_to_vec(felt, bincode::config::standard()).unwrap(); - let deserialized: Felt = - bincode::serde::decode_from_slice(&serialized, bincode::config::standard()) - .unwrap() - .0; - assert_eq!(felt, deserialized); - } - - #[test] - fn serialize_binary_compact() { - let felt = Felt::from_hex_str("0xbabe").unwrap(); - let serialized = bincode::serde::encode_to_vec(felt, bincode::config::standard()).unwrap(); - assert!(serialized.len() < 32); - let deserialized: Felt = - bincode::serde::decode_from_slice(&serialized, bincode::config::standard()) - .unwrap() - .0; - assert_eq!(felt, deserialized); - } -} diff --git a/crates/starknet/src/starknet.rs b/crates/starknet/src/starknet.rs deleted file mode 100644 index d3ac96c7..00000000 --- a/crates/starknet/src/starknet.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::event::{EmittedEvent, Event}; -use crate::Felt; -use starknet::core::types::Felt as SnFelt; -use starknet::core::types::{EmittedEvent as SnEmittedEvent, Event as SnEvent}; - -impl From for Felt { - fn from(value: SnFelt) -> Self { - Self(value.to_bytes_be()) - } -} - -impl From<&SnFelt> for Felt { - fn from(value: &SnFelt) -> Self { - Self(value.to_bytes_be()) - } -} - -impl From for SnFelt { - fn from(value: Felt) -> Self { - Self::from_bytes_be(&value.0) - } -} -impl From<&Felt> for SnFelt { - fn from(value: &Felt) -> Self { - Self::from_bytes_be(&value.0) - } -} - -impl From for Event { - fn from(value: SnEvent) -> Self { - Self { - from_address: value.from_address.into(), - keys: value.keys.into_iter().map(Into::into).collect(), - data: value.data.into_iter().map(Into::into).collect(), - } - } -} - -impl From for EmittedEvent { - fn from(value: SnEmittedEvent) -> Self { - Self { - from_address: value.from_address.into(), - keys: value.keys.into_iter().map(Into::into).collect(), - data: value.data.into_iter().map(Into::into).collect(), - block_hash: value.block_hash.map(Into::into), - block_number: value.block_number, - transaction_hash: value.transaction_hash.into(), - } - } -} - -impl From for SnEvent { - fn from(value: Event) -> Self { - Self { - from_address: value.from_address.into(), - keys: value.keys.into_iter().map(Into::into).collect(), - data: value.data.into_iter().map(Into::into).collect(), - } - } -} - -impl From for SnEmittedEvent { - fn from(value: EmittedEvent) -> Self { - Self { - from_address: value.from_address.into(), - keys: value.keys.into_iter().map(Into::into).collect(), - data: value.data.into_iter().map(Into::into).collect(), - block_hash: value.block_hash.map(Into::into), - block_number: value.block_number, - transaction_hash: value.transaction_hash.into(), - } - } -} diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 37f3c534..fd74cc53 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -12,7 +12,7 @@ async-trait.workspace = true resolve-path.workspace = true serde.workspace = true serde_json.workspace = true -starknet.workspace = true dojo-introspect.workspace = true introspect-types.workspace = true starknet-types-core.workspace = true +starknet-types-raw.workspace = true diff --git a/crates/testing/README.md b/crates/testing/README.md new file mode 100644 index 00000000..97be69af --- /dev/null +++ b/crates/testing/README.md @@ -0,0 +1,73 @@ +# torii-test-utils + +Test-only helpers for working with captured Starknet event fixtures and +Dojo schemas. Kept out of `torii-common` because it pulls in `resolve-path`, +`alphanumeric-sort`, and `dojo-introspect` — weight no production binary +should carry. + +## Role in Torii + +Unit and integration tests across the workspace replay on-chain events from +JSON fixtures instead of hitting a real RPC. `EventIterator` turns a +directory of `events_*.json` batch files into an `Iterator`; `MultiContractEventIterator` round-robins between several +such directories to mimic multi-contract workloads. `FakeProvider` gives +`torii-dojo` tests a `DojoSchemaFetcher` that reads schemas from disk instead +of performing a `starknet_getClass` RPC call. + +## Architecture + +```text ++---------------------------------------------------------+ +| torii-test-utils | +| | +| event_reader.rs | +| +---------------------+ Event (serde-compatible JSON)| +| | EventBatch |---+ | +| | EventIterator | | | +| | MultiContractEvent- | v | +| | Iterator | starknet_types_raw:: | +| +---------------------+ EmittedEvent | +| | +| dojo.rs | +| +---------------------+ reads {contract:#066x}.json | +| | FakeProvider |---> impls DojoSchemaFetcher | +| +---------------------+ | +| | +| utils.rs | +| +---------------------+ | +| | read_json_file | `~/` expansion via resolve_ | +| | resolve_path_like | path | +| +---------------------+ | ++---------------------------------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `EventIterator` | `src/event_reader.rs` | 39 | Streams `EmittedEvent`s from a directory of batch JSON files (sorted alphanumerically) | +| `MultiContractEventIterator` | `src/event_reader.rs` | 48 | Round-robin merge of several `EventIterator`s | +| `EventBatch` / `Event` | `src/event_reader.rs` | 9 / 32 | Serde-compatible wire shapes used by fixture files | +| `FakeProvider` | `src/dojo.rs` | 11 | `DojoSchemaFetcher` impl that loads `{contract:#066x}.json` from disk | +| `read_json_file(path)` | `src/utils.rs` | — | Typed JSON loader used by both modules | +| `resolve_path_like` | `src/utils.rs` | — | Expands `~`-relative paths via `resolve-path` | + +### Internal Modules + +- `event_reader` — two iterators and their batch JSON schemas. +- `dojo` — a single `DojoSchemaFetcher` impl for test-time introspection. +- `utils` — JSON loader + `~/` path expander (the module is `pub` but the functions are re-exported at the crate root). + +### Interactions + +- **Upstream (consumers)**: tests and benches across the workspace; the root crate lists this in its `[dev-dependencies]` (`Cargo.toml:244`). +- **Downstream deps**: `dojo-introspect`, `introspect-types`, `starknet-types-core`, `starknet-types-raw`, `serde`, `serde_json`, `resolve-path`, `alphanumeric-sort`, `async-trait`. +- **Workspace deps**: none at runtime — it is only pulled in as a dev-dep. + +### Extension Points + +- New fixture shape: add a module, re-export from `lib.rs`, keep fixtures beside their tests rather than under this crate. +- Do **not** put production code here — anything in this crate's deps (e.g. `resolve-path`) will leak into binaries if they accidentally take a runtime dep. diff --git a/crates/testing/src/event_reader.rs b/crates/testing/src/event_reader.rs index 9b79a376..be9c4509 100644 --- a/crates/testing/src/event_reader.rs +++ b/crates/testing/src/event_reader.rs @@ -1,7 +1,7 @@ use crate::{read_json_file, resolve_path_like}; use serde::Deserialize; -use starknet::core::types::EmittedEvent; -use starknet_types_core::felt::Felt; +use starknet_types_raw::event::EmittedEvent; +use starknet_types_raw::Felt; use std::collections::VecDeque; use std::fs::read_dir; use std::path::PathBuf; @@ -45,6 +45,10 @@ pub struct EventIterator { pub event: usize, } +pub struct MultiContractEventIterator { + iterators: VecDeque, +} + impl EventIterator { pub fn new>(path: P) -> Self { let path = resolve_path_like(path); @@ -63,6 +67,16 @@ impl EventIterator { } } +impl MultiContractEventIterator { + pub fn new>(paths: Vec

) -> Self { + let iterators = paths + .into_iter() + .map(|p| EventIterator::new(p)) + .collect::>(); + Self { iterators } + } +} + impl Iterator for EventIterator { type Item = EmittedEvent; @@ -81,3 +95,18 @@ impl Iterator for EventIterator { } } } + +impl Iterator for MultiContractEventIterator { + type Item = EmittedEvent; + + fn next(&mut self) -> Option { + while let Some(iterator) = self.iterators.front_mut() { + if let Some(event) = iterator.next() { + self.iterators.rotate_left(1); + return Some(event); + } + self.iterators.pop_front(); + } + None + } +} diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 1635a8e8..70afaa2d 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -2,5 +2,5 @@ mod dojo; mod event_reader; pub mod utils; pub use dojo::FakeProvider; -pub use event_reader::EventIterator; +pub use event_reader::{EventIterator, MultiContractEventIterator}; pub use utils::{read_json_file, resolve_path_like}; diff --git a/crates/torii-common/Cargo.toml b/crates/torii-common/Cargo.toml index 9112e39f..572afe1d 100644 --- a/crates/torii-common/Cargo.toml +++ b/crates/torii-common/Cargo.toml @@ -5,7 +5,6 @@ edition = "2021" description = "Common utilities for Torii token indexers" [dependencies] -starknet = "0.17" anyhow = "1.0" tracing = "0.1" tokio = { version = "1", features = ["full"] } @@ -15,13 +14,9 @@ urlencoding = "2" async-trait = "0.1" serde.workspace = true serde_json.workspace = true -sqlx = { workspace = true, features = [ - "postgres", - "runtime-tokio-rustls", - "mysql", - "sqlite", -] } -itertools.workspace = true +starknet.workspace = true +starknet-types-raw.workspace = true +primitive-types.workspace = true [lints] workspace = true diff --git a/crates/torii-common/README.md b/crates/torii-common/README.md new file mode 100644 index 00000000..b93036d4 --- /dev/null +++ b/crates/torii-common/README.md @@ -0,0 +1,70 @@ +# torii-common + +Shared low-level utilities used by every Torii sink and decoder: compact U256 +and Felt BLOB encodings, JSON helpers, token metadata fetching, and a +pluggable token-URI resolver. This crate has no knowledge of the ETL loop — +it is a pure toolbox. + +## Role in Torii + +`torii-common` is consumed everywhere amounts, addresses, or token metadata +need to move between the chain and the database. The token sinks +(`torii-erc20`, `torii-erc721`, `torii-erc1155`) store amounts and token IDs +as variable-length BLOBs via `U256Blob`, and fetch name/symbol/decimals from +on-chain views or off-chain metadata URIs via `MetadataFetcher` + +`TokenUriService`. + +## Architecture + +```text ++-----------------------------------------------------------+ +| torii-common | +| | +| src/lib.rs src/metadata.rs src/token_uri.rs| +| +-----------+ +----------------+ +--------------+| +| | U256Blob | | MetadataFetcher| |TokenUriService| +| | FeltBlob | | TokenMetadata | |TokenStandard | +| +-----------+ +----------------+ |TokenUriRequest| +| ^ ^ |TokenUriStore | +| | | +--------------+| +| | | ^ | +| | | | | +| Sinks / Decoders <---------+--------------------+ | +| (torii-erc20, torii-erc721, torii-erc1155) | ++-----------------------------------------------------------+ +``` + +## Deep Dive + +### Public API + +| Type / Fn | File | Line | Purpose | +|---|---|---|---| +| `U256Blob` | `src/lib.rs` | 22 | Compact big-endian BLOB codec for U256 (leading zeros stripped, zero → `[0]`) | +| `FeltBlob` | `src/lib.rs` | 103 | BLOB codec for `RawFelt` / `CoreFelt` | +| `u256_to_blob` / `blob_to_u256` | `src/lib.rs` | 72 / 80 | Generic helpers over `U256Blob` | +| `felt_to_blob` / `blob_to_felt` | `src/lib.rs` | 134 / 148 | Generic helpers over `FeltBlob` | +| `MetadataFetcher` | `src/metadata.rs` | — | Async HTTP + on-chain metadata enrichment | +| `TokenMetadata` | `src/metadata.rs` | — | Name / symbol / decimals / URI | +| `TokenUriService` | `src/token_uri.rs` | — | Resolves on-chain `token_uri` / `uri` to structured `TokenUriResult`s | +| `TokenStandard` | `src/token_uri.rs` | — | ERC20 / ERC721 / ERC1155 tag | +| `process_token_uri_request` | `src/token_uri.rs` | — | One-shot resolver used by metadata command handlers | + +### Internal Modules + +- `json` — small serde helpers (hex parsing, optional fields). +- `metadata` — async fetcher, retry policy, error types, unit-tested around HTTP mocking. +- `token_uri` — token-standard-aware URI resolution (`ipfs://`, `data:` URIs, on-chain strings). +- `utils` — miscellaneous helpers (hex printing, string normalization). + +### Interactions + +- **Upstream (consumers)**: `torii-erc20`, `torii-erc721`, `torii-erc1155`, `torii-dojo`, `arcade-sink`, `introspect-sql-sink`. The root crate re-exports `StarknetEvent` from `torii-types`, not from here. +- **Downstream deps**: `reqwest` (HTTP), `serde` / `serde_json`, `primitive-types`, `starknet` / `starknet-types-raw`, `tokio`. +- **Workspace deps**: none (pure library). + +### Extension Points + +- Adding a new token standard: extend `TokenStandard` in `src/token_uri.rs` and the corresponding branches of `process_token_uri_request`. +- New wire encoding: impl `U256Blob` or `FeltBlob` for a custom numeric type; the existing helpers become generic over it automatically. +- New metadata source: wrap `MetadataFetcher` or register a new `CommandHandler` in the binary (see `torii-erc20::FetchErc20MetadataCommand`). diff --git a/crates/torii-common/src/lib.rs b/crates/torii-common/src/lib.rs index 5a96b8dc..36dcfb3d 100644 --- a/crates/torii-common/src/lib.rs +++ b/crates/torii-common/src/lib.rs @@ -5,104 +5,149 @@ pub mod json; pub mod metadata; -pub mod sql; pub mod token_uri; pub mod utils; -use starknet::core::types::{Felt, U256}; - pub use metadata::{MetadataFetcher, TokenMetadata}; +use primitive_types::U256 as PrimitiveU256; +use starknet::core::types::{Felt as CoreFelt, U256 as CoreU256}; +use starknet_types_raw::Felt as RawFelt; pub use token_uri::{ process_token_uri_request, TokenStandard, TokenUriRequest, TokenUriResult, TokenUriSender, TokenUriService, TokenUriStore, }; -// ===== Felt conversions ===== +// ===== U256 conversions ===== -/// Convert Felt to 32-byte BLOB for storage (big-endian) -pub fn felt_to_blob(felt: Felt) -> Vec { - felt.to_bytes_be().to_vec() +pub trait U256Blob: Sized { + fn to_blob_bytes(self) -> Vec; + fn from_blob_bytes(bytes: &[u8]) -> Self; } -/// Convert BLOB back to Felt (big-endian) -pub fn blob_to_felt(bytes: &[u8]) -> Felt { - let mut arr = [0u8; 32]; - let len = bytes.len().min(32); - // Right-align for big-endian (pad zeros on the left) - arr[32 - len..].copy_from_slice(&bytes[..len]); - Felt::from_bytes_be(&arr) -} +impl U256Blob for PrimitiveU256 { + fn to_blob_bytes(self) -> Vec { + if self.is_zero() { + return vec![0]; + } -/// Parse bytes to Felt (returns None if > 32 bytes) -pub fn bytes_to_felt(bytes: &[u8]) -> Option { - if bytes.len() > 32 { - return None; + let bytes = self.to_big_endian(); + let start = bytes.iter().position(|&b| b != 0).unwrap_or(31); + bytes[start..].to_vec() } - Some(blob_to_felt(bytes)) -} -// ===== U256 conversions ===== - -/// Convert U256 to variable-length BLOB for storage (big-endian, compact) -/// -/// Compression strategy: -/// - Zero value: 1 byte (0x00) -/// - Values < 2^128: 1-16 bytes (minimal encoding of low word) -/// - Values >= 2^128: Full encoding (17-32 bytes) -pub fn u256_to_blob(value: U256) -> Vec { - let high = value.high(); - let low = value.low(); - - if high == 0 { - if low == 0 { - return vec![0u8]; - } - let bytes = low.to_be_bytes(); - let start = bytes.iter().position(|&b| b != 0).unwrap_or(15); - return bytes[start..].to_vec(); + fn from_blob_bytes(bytes: &[u8]) -> Self { + Self::from_big_endian(bytes) } - - let mut result = Vec::with_capacity(32); - let high_bytes = high.to_be_bytes(); - let high_start = high_bytes.iter().position(|&b| b != 0).unwrap_or(15); - result.extend_from_slice(&high_bytes[high_start..]); - result.extend_from_slice(&low.to_be_bytes()); - result } -/// Convert BLOB back to U256 (big-endian) -pub fn blob_to_u256(bytes: &[u8]) -> U256 { - let len = bytes.len(); +impl U256Blob for CoreU256 { + fn to_blob_bytes(self) -> Vec { + if self == Self::from(0u8) { + return vec![0]; + } - if len == 0 { - return U256::from(0u64); + let mut bytes = [0u8; 32]; + bytes[..16].copy_from_slice(&self.high().to_be_bytes()); + bytes[16..].copy_from_slice(&self.low().to_be_bytes()); + let start = bytes.iter().position(|&b| b != 0).unwrap_or(31); + bytes[start..].to_vec() } - if len <= 16 { - let mut low_bytes = [0u8; 16]; - low_bytes[16 - len..].copy_from_slice(bytes); - let low = u128::from_be_bytes(low_bytes); - U256::from_words(low, 0) - } else { - let high_len = len - 16; - let mut high_bytes = [0u8; 16]; - high_bytes[16 - high_len..].copy_from_slice(&bytes[..high_len]); - let high = u128::from_be_bytes(high_bytes); - - let mut low_bytes = [0u8; 16]; - low_bytes.copy_from_slice(&bytes[high_len..]); - let low = u128::from_be_bytes(low_bytes); - - U256::from_words(low, high) + fn from_blob_bytes(bytes: &[u8]) -> Self { + let mut padded = [0u8; 32]; + let source = if bytes.len() > 32 { + &bytes[bytes.len() - 32..] + } else { + bytes + }; + let offset = 32 - source.len(); + padded[offset..].copy_from_slice(source); + let high = u128::from_be_bytes(padded[..16].try_into().unwrap()); + let low = u128::from_be_bytes(padded[16..].try_into().unwrap()); + Self::from_words(low, high) } } +/// Convert U256 to variable-length BLOB for storage (big-endian, compact) +pub fn u256_to_blob(value: U) -> Vec +where + U: U256Blob, +{ + value.to_blob_bytes() +} + +/// Convert BLOB back to U256 (big-endian) +pub fn blob_to_u256(bytes: &[u8]) -> U +where + U: U256Blob, +{ + U::from_blob_bytes(bytes) +} + /// Alias for u256_to_blob (same format, different context name) -pub fn u256_to_bytes(value: U256) -> Vec { +pub fn u256_to_bytes(value: U) -> Vec +where + U: U256Blob, +{ u256_to_blob(value) } /// Alias for blob_to_u256 (same format, different context name) -pub fn bytes_to_u256(bytes: &[u8]) -> U256 { +pub fn bytes_to_u256(bytes: &[u8]) -> U +where + U: U256Blob, +{ blob_to_u256(bytes) } + +pub trait FeltBlob: Sized { + fn to_blob_bytes(self) -> Vec; + fn from_blob_bytes(bytes: &[u8]) -> Option; +} + +impl FeltBlob for RawFelt { + fn to_blob_bytes(self) -> Vec { + self.to_be_bytes_vec() + } + + fn from_blob_bytes(bytes: &[u8]) -> Option { + Self::from_be_bytes_slice(bytes).ok() + } +} + +impl FeltBlob for CoreFelt { + fn to_blob_bytes(self) -> Vec { + self.to_bytes_be().to_vec() + } + + fn from_blob_bytes(bytes: &[u8]) -> Option { + if bytes.len() > 32 { + return None; + } + + let mut padded = [0u8; 32]; + padded[32 - bytes.len()..].copy_from_slice(bytes); + Some(Self::from_bytes_be(&padded)) + } +} + +pub fn felt_to_blob(value: F) -> Vec +where + F: FeltBlob, +{ + value.to_blob_bytes() +} + +pub fn bytes_to_felt(bytes: &[u8]) -> Option +where + F: FeltBlob, +{ + F::from_blob_bytes(bytes) +} + +pub fn blob_to_felt(bytes: &[u8]) -> F +where + F: FeltBlob, +{ + bytes_to_felt(bytes).expect("database stores canonical felt blobs") +} diff --git a/crates/torii-common/src/metadata.rs b/crates/torii-common/src/metadata.rs index 8c8bd24c..8259a340 100644 --- a/crates/torii-common/src/metadata.rs +++ b/crates/torii-common/src/metadata.rs @@ -4,16 +4,19 @@ //! making `starknet_call` requests. Handles both snake_case and camelCase //! selectors, felt-encoded strings and ByteArray returns. +use primitive_types::U256; use starknet::core::codec::Decode; -use starknet::core::types::{ - requests::CallRequest, BlockId, BlockTag, ByteArray, Felt, FunctionCall, U256, -}; +use starknet::core::types::requests::CallRequest; +use starknet::core::types::{BlockId, BlockTag, ByteArray, Felt as SnFelt, FunctionCall}; use starknet::core::utils::parse_cairo_short_string; use starknet::macros::selector; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::sync::Arc; +use crate::utils::parse_u256_result; + /// Token metadata (common fields for all ERC standards) #[derive(Debug, Clone, Default)] pub struct TokenMetadata { @@ -85,11 +88,13 @@ impl MetadataFetcher { /// Returns None if the call fails or returns empty data. pub async fn fetch_token_uri(&self, contract: Felt, token_id: Felt) -> Option { // Try snake_case first, then camelCase + let contract = contract.into(); + let token_id = token_id.into(); for sel in [selector!("token_uri"), selector!("tokenURI")] { let call = FunctionCall { contract_address: contract, entry_point_selector: sel, - calldata: vec![token_id, Felt::ZERO], // u256: (low, high) + calldata: vec![token_id, SnFelt::ZERO], // u256: (low, high) }; if let Ok(result) = self @@ -132,10 +137,13 @@ impl MetadataFetcher { /// Fetch `uri(token_id)` for ERC1155 tokens. pub async fn fetch_uri(&self, contract: Felt, token_id: Felt) -> Option { // ERC1155 uses `uri(token_id)` — u256 arg + let contract = contract.into(); + let token_id = token_id.into(); + let call = FunctionCall { contract_address: contract, entry_point_selector: selector!("uri"), - calldata: vec![token_id, Felt::ZERO], + calldata: vec![token_id, SnFelt::ZERO], }; if let Ok(result) = self @@ -206,7 +214,7 @@ impl MetadataFetcher { }; let call = FunctionCall { - contract_address: contract, + contract_address: contract.into(), entry_point_selector: sel, calldata: vec![], }; @@ -233,7 +241,7 @@ impl MetadataFetcher { /// Fetch `decimals()` from an ERC20 contract. async fn fetch_decimals(&self, contract: Felt) -> Option { let call = FunctionCall { - contract_address: contract, + contract_address: contract.into(), entry_point_selector: selector!("decimals"), calldata: vec![], }; @@ -276,7 +284,7 @@ impl MetadataFetcher { async fn fetch_total_supply(&self, contract: Felt) -> Option { for sel in [selector!("total_supply"), selector!("totalSupply")] { let call = FunctionCall { - contract_address: contract, + contract_address: contract.into(), entry_point_selector: sel, calldata: vec![], }; @@ -290,13 +298,9 @@ impl MetadataFetcher { continue; } // U256 return: [low, high] or single felt - let low: u128 = result[0].try_into().unwrap_or(0); - return Some(if result.len() == 1 { - U256::from(low) - } else { - let high: u128 = result[1].try_into().unwrap_or(0); - U256::from_words(low, high) - }); + return Some(parse_u256_result( + result.into_iter().map(Into::into).collect(), + )); } } @@ -309,7 +313,7 @@ impl MetadataFetcher { /// 1. **Single short string** (felt): via `parse_cairo_short_string` /// 2. **Cairo ByteArray**: via `ByteArray::decode` (the standard Cairo string type) /// 3. **Legacy array**: `[len, felt1, felt2, ...]` where each felt is a short string segment - fn decode_string_result(result: &[Felt]) -> Option { + fn decode_string_result(result: &[SnFelt]) -> Option { if result.is_empty() { return None; } @@ -355,7 +359,7 @@ impl MetadataFetcher { async fn fetch_string_calls_batch( &self, requests: &[(Felt, Felt)], - attempts: &[(Felt, bool)], + attempts: &[(SnFelt, bool)], ) -> Vec> { if requests.is_empty() { return Vec::new(); @@ -374,13 +378,13 @@ impl MetadataFetcher { .map(|&idx| { let (contract, token_id) = requests[idx]; let calldata = if use_u256 { - vec![token_id, Felt::ZERO] + vec![token_id.into(), SnFelt::ZERO] } else { - vec![token_id] + vec![token_id.into()] }; ProviderRequestData::Call(CallRequest { request: FunctionCall { - contract_address: contract, + contract_address: contract.into(), entry_point_selector: selector, calldata, }, @@ -435,13 +439,13 @@ mod tests { #[test] fn test_parse_short_string() { // "ETH" = 0x455448 - let felt = Felt::from(0x455448u64); + let felt = SnFelt::from(0x455448u64); assert_eq!(parse_cairo_short_string(&felt).unwrap(), "ETH".to_string()); } #[test] fn test_decode_single_felt_string() { - let result = vec![Felt::from(0x455448u64)]; // "ETH" + let result = vec![SnFelt::from(0x455448u64)]; // "ETH" assert_eq!( MetadataFetcher::decode_string_result(&result), Some("ETH".to_string()) @@ -452,9 +456,9 @@ mod tests { fn test_decode_byte_array() { // ByteArray: [data_len=0, pending_word="ETH", pending_word_len=3] let result = vec![ - Felt::from(0u64), // data_len = 0 chunks - Felt::from(0x455448u64), // pending_word = "ETH" - Felt::from(3u64), // pending_word_len = 3 + SnFelt::from(0u64), // data_len = 0 chunks + SnFelt::from(0x455448u64), // pending_word = "ETH" + SnFelt::from(3u64), // pending_word_len = 3 ]; assert_eq!( MetadataFetcher::decode_string_result(&result), diff --git a/crates/torii-common/src/sql.rs b/crates/torii-common/src/sql.rs deleted file mode 100644 index 531adaf0..00000000 --- a/crates/torii-common/src/sql.rs +++ /dev/null @@ -1,241 +0,0 @@ -use async_trait::async_trait; -use itertools::Itertools; -use sqlx::{Database, Executor, Postgres}; -pub use sqlx::{PgPool, Transaction}; -use std::borrow::Cow; -use std::sync::Arc; - -pub use sqlx::Error as SqlxError; - -pub type SqlxResult = std::result::Result; - -#[async_trait] -pub trait Executable { - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()>; -} - -#[async_trait] -impl Executable for &str -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self).await?; - Ok(()) - } -} - -#[async_trait] -impl Executable for &String -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_str()).await?; - Ok(()) - } -} - -#[async_trait] -impl Executable for String -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_str()).await?; - Ok(()) - } -} - -#[async_trait] -impl Executable for Cow<'_, str> -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_ref()).await?; - Ok(()) - } -} - -pub struct QueryLike, DB: Database> { - sql: S, - args: Option<::Arguments<'static>>, -} - -impl, DB: Database> QueryLike { - pub fn new(sql: impl Into, args: ::Arguments<'static>) -> Self { - Self { - sql: sql.into(), - args: Some(args), - } - } - - pub fn from_sql(sql: S) -> Self { - Self { sql, args: None } - } -} - -#[async_trait] -impl Executable for FlexStr -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - transaction.execute(self.as_ref()).await?; - Ok(()) - } -} - -pub enum FlexStr { - Owned(String), - Static(&'static str), - Shared(Arc), -} - -impl AsRef for FlexStr { - fn as_ref(&self) -> &str { - match self { - FlexStr::Owned(s) => s.as_str(), - FlexStr::Static(s) => s, - FlexStr::Shared(s) => s, - } - } -} - -impl PartialEq for FlexStr { - fn eq(&self, other: &str) -> bool { - self.as_ref() == other - } -} - -impl, DB: Database> PartialEq for QueryLike { - fn eq(&self, other: &str) -> bool { - self.sql.as_ref() == other - } -} - -impl From for FlexStr { - fn from(s: String) -> Self { - FlexStr::Owned(s) - } -} - -impl From<&'static str> for FlexStr { - fn from(s: &'static str) -> Self { - FlexStr::Static(s) - } -} - -impl From> for FlexStr { - fn from(s: Arc) -> Self { - FlexStr::Shared(s) - } -} - -impl From for QueryLike { - fn from(sql: FlexStr) -> Self { - QueryLike::from_sql(sql) - } -} - -impl From<&'static str> for QueryLike { - fn from(sql: &'static str) -> Self { - QueryLike::from_sql(sql.into()) - } -} - -impl From for QueryLike { - fn from(sql: String) -> Self { - QueryLike::from_sql(sql.into()) - } -} - -impl From> for QueryLike { - fn from(sql: Arc) -> Self { - QueryLike::from_sql(sql.into()) - } -} - -pub trait Queries { - fn add(&mut self, query: impl Into>); - fn adds(&mut self, queries: impl IntoIterator>>); -} - -impl Queries for Vec> { - fn add(&mut self, query: impl Into>) { - self.push(query.into()); - } - fn adds(&mut self, queries: impl IntoIterator>>) { - self.extend(queries.into_iter().map_into()); - } -} - -#[async_trait] -impl Executable for &[String; N] -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for query in self { - transaction.execute(query.as_str()).await?; - } - Ok(()) - } -} - -#[async_trait] -impl + Send> Executable for Vec { - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for item in self { - item.execute(transaction).await?; - } - Ok(()) - } -} - -#[async_trait] -impl<'a, DB: sqlx::Database, T> Executable for &'a Vec -where - &'a T: Executable + Send, - T: Send + Sync, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for item in self { - item.execute(transaction).await?; - } - Ok(()) - } -} - -#[async_trait] -impl<'a, DB: sqlx::Database, T> Executable for &'a [T] -where - &'a T: Executable + Send, - T: Send + Sync, -{ - async fn execute(self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - for item in self { - item.execute(transaction).await?; - } - Ok(()) - } -} - -#[async_trait] -impl + Send, DB: sqlx::Database> Executable for QueryLike -where - for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, - for<'q> ::Arguments<'static>: sqlx::IntoArguments<'q, DB>, -{ - async fn execute(mut self, transaction: &mut Transaction<'_, DB>) -> SqlxResult<()> { - let sql = self.sql.as_ref(); - let args = self.args.take().unwrap_or_default(); - transaction.execute(sqlx::query_with(sql, args)).await?; - Ok(()) - } -} - -pub type PgQuery = QueryLike; -pub type MySqlQuery = QueryLike; -pub type SqliteQuery = QueryLike; diff --git a/crates/torii-common/src/token_uri.rs b/crates/torii-common/src/token_uri.rs index 71cdd249..43d831e0 100644 --- a/crates/torii-common/src/token_uri.rs +++ b/crates/torii-common/src/token_uri.rs @@ -17,7 +17,8 @@ //! - JSON sanitization for broken metadata (control chars, unescaped quotes) //! - Raw JSON fallback for inline metadata -use starknet::core::types::{Felt, U256}; +use primitive_types::U256; +use starknet_types_raw::Felt; use std::collections::{HashMap, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -470,7 +471,8 @@ impl TokenUriService { let mut erc1155_requests = Vec::new(); for (idx, request) in requests.iter().enumerate() { - let token_id = Felt::from(request.token_id.low()); + // let token_id = Felt::from(request.token_id.low()); + let token_id = Felt::from_le_words(request.token_id.0); // CHECK: same logic match request.standard { TokenStandard::Erc721 => { erc721_positions.push(idx); @@ -804,7 +806,7 @@ async fn fetch_token_uri_with_retry( standard: TokenStandard, ) -> Option { // Use the MetadataFetcher which already tries multiple selectors - let token_id_felt = Felt::from(token_id.low()); + let token_id_felt = Felt::from_le_words(token_id.0); // CHECK: same logic as before match standard { TokenStandard::Erc721 => fetcher.fetch_token_uri(contract, token_id_felt).await, diff --git a/crates/torii-common/src/utils.rs b/crates/torii-common/src/utils.rs index f442b9ca..d13adcf8 100644 --- a/crates/torii-common/src/utils.rs +++ b/crates/torii-common/src/utils.rs @@ -1,3 +1,6 @@ +use primitive_types::U256; +use starknet_types_raw::Felt; + pub trait ElementsInto { fn elements_into(self) -> Vec; } @@ -23,3 +26,70 @@ where vec.into_iter().map(U::from).collect() } } + +/// Parse a U256 result from balance_of return value +/// +/// ERC20 balance_of typically returns: +/// - Cairo 0: A single felt (fits in 252 bits, usually enough for balances) +/// - Cairo 1 with u256: Two felts [low, high] representing a 256-bit value +pub fn parse_u256_result(result: Vec) -> U256 { + match result.len() { + 0 => U256::from(0u64), + 1 => { + // Single felt - convert to U256 + // Felt is 252 bits max, so it fits in the low part + // Take the lower 16 bytes for u128 (fits any felt value) + felt_to_u256(result.into_iter().next().unwrap()) + } + _ => { + // Two felts: [low, high] for u256 + let mut iter = result.into_iter(); + felt_pair_to_u256(iter.next().unwrap(), iter.next().unwrap()) + } + } +} + +pub fn felt_pair_to_u256(low: Felt, high: Felt) -> U256 { + let [l0, l1, _, _] = low.to_le_words(); + let [h0, h1, _, _] = high.to_le_words(); + U256([l0, l1, h0, h1]) +} + +pub fn felt_to_u256(value: Felt) -> U256 { + U256(value.to_le_words()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_u256_empty() { + let result = parse_u256_result(vec![]); + assert_eq!(result, U256::from(0u64)); + } + + #[test] + fn test_parse_u256_single_felt() { + let felt = Felt::from(1000u64); + let result = parse_u256_result(vec![felt]); + assert_eq!(result, U256::from(1000u64)); + } + + #[test] + fn test_parse_u256_two_felts() { + // low = 100, high = 0 + let low = Felt::from(100u64); + let high = Felt::from(0u64); + let result = parse_u256_result(vec![low, high]); + assert_eq!(result, U256::from(100u64)); + + // Test with high value + let low = Felt::from(0u64); + let high = Felt::from(1u64); + let result = parse_u256_result(vec![low, high]); + // high = 1 means value = 1 * 2^128 + let expected = U256::from(1u64) << 128; + assert_eq!(result, expected); + } +} diff --git a/crates/torii-config-common/README.md b/crates/torii-config-common/README.md index b27cea7f..7bc86268 100644 --- a/crates/torii-config-common/README.md +++ b/crates/torii-config-common/README.md @@ -1,19 +1,60 @@ # torii-config-common -Shared CLI configuration helpers for Torii binaries. +Shared **CLI configuration** helpers for Torii binaries. A tiny crate whose +only purpose is to keep clap parsers honest: same flags → same behaviour +across every binary under `bins/`. -## What it provides +## Role in Torii -- `apply_observability_env(enabled: bool)` - - Sets `TORII_METRICS_ENABLED` from CLI flags so runtime behavior is explicit. -- `require_postgres_url(url, arg_name)` - - Validates that a URL is PostgreSQL and returns a user-facing error if not. +Every binary uses `clap` to build its own `Config` struct, then calls into +this crate to (a) turn observability flags into environment variables the +core library reads (`TORII_METRICS_ENABLED`) and (b) validate that strings +purporting to be PostgreSQL URLs actually are. Everything that needs a real +database URL lives in `torii-runtime-common`; this crate is pure validation +and env plumbing. -## Example +## Architecture -```rust -use torii_config_common::{apply_observability_env, require_postgres_url}; - -apply_observability_env(config.observability); -let storage_url = require_postgres_url(&config.storage_database_url, "--storage-database-url")?; +```text + clap parser (in each bin) + | + v + +-----------------------------+ + | torii-config-common | + | | + | apply_observability_env() |---> sets TORII_METRICS_ENABLED + | is_postgres_url() | + | require_postgres_url() |---> anyhow::Error on bad URL + +-----------------------------+ + | + v + torii-runtime-common::resolve_*_db_setup + | + v + torii::ToriiConfigBuilder ``` + +## Deep Dive + +### Public API + +| Fn | File | Purpose | +|---|---|---| +| `apply_observability_env(enabled: bool)` | `src/lib.rs` | Sets `TORII_METRICS_ENABLED=true/false` so the metrics subsystem in `src/metrics.rs::init_from_env` picks it up | +| `is_postgres_url(url: &str) -> bool` | `src/lib.rs` | Prefix check for `postgres://` / `postgresql://` | +| `require_postgres_url(url: &str, arg_name: &str)` | `src/lib.rs` | Returns `anyhow::Error` with a user-facing message if the URL is not PostgreSQL; used by binaries that only support PG (synthetic profilers) | + +### Internal Modules + +- `lib.rs` — single file, < 100 LoC, no sub-modules. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-arcade`, `bins/torii-introspect-bin`, `bins/torii-tokens`, the three `-synth` profilers. +- **Downstream deps**: `anyhow` only. +- **Workspace deps**: none. + +### Extension Points + +- New cross-binary CLI rules (e.g., conflicting-flag validation) belong here so every binary inherits them. +- Resist adding DB-setup logic — that belongs in `torii-runtime-common`. diff --git a/crates/torii-controllers-sink/Cargo.toml b/crates/torii-controllers-sink/Cargo.toml index 4f5c07a6..6396be7c 100644 --- a/crates/torii-controllers-sink/Cargo.toml +++ b/crates/torii-controllers-sink/Cargo.toml @@ -4,18 +4,25 @@ version = "0.1.0" edition = "2021" [dependencies] -torii = { path = "../.." } +torii.workspace = true torii-runtime-common.workspace = true +torii-sql.workspace = true anyhow.workspace = true async-trait.workspace = true chrono.workspace = true +hex.workspace = true metrics.workspace = true reqwest = { version = "0.12", features = ["json", "rustls-tls"] } serde.workspace = true serde_json.workspace = true -sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "postgres", "any"] } -starknet.workspace = true +sqlx = { workspace = true, features = [ + "runtime-tokio", + "sqlite", + "postgres", + "any", +] } +starknet-types-raw.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/crates/torii-controllers-sink/README.md b/crates/torii-controllers-sink/README.md new file mode 100644 index 00000000..3b7b5703 --- /dev/null +++ b/crates/torii-controllers-sink/README.md @@ -0,0 +1,111 @@ +# torii-controllers-sink + +Syncs **Cartridge controller accounts** (username + address + deployment +timestamp) from the Cartridge GraphQL API into a local SQL table. Unlike +most sinks, this one is **API-driven** — it does not consume `Envelope`s; +it watches the block timestamps moving in the `ExtractionBatch` and uses +that as a tick to fetch fresh controllers from +`https://api.cartridge.gg/query`. + +## Role in Torii + +Binaries that serve the Cartridge ecosystem (`bins/torii-arcade`, +`bins/torii-introspect-bin`) plug this sink in alongside the token / dojo +sinks. It provides the `controllers` table other sinks JOIN against to +enrich queries with a human-readable `username` for a given on-chain +address. + +## Architecture + +```text + +------------------------------+ + | ControllersSink | + | | + | process(batch): | + | time_window := min/max | + | block.timestamp | + | if window > synced_until: | + | fetch_controllers(GQL) |---> https://api.cartridge.gg/query + | upsert batch(store) | + +------------------------------+ + | + v + +------------------------------+ + | ControllersStore | + | Arc> | + | backend = Sqlite | Postgres | + | | + | controllers (id PK, addr, | + | username, deployed_at, | + | updated_at) | + | torii_controller_sync_state | + | (key='synced_until', | + | value=) | + +------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `ControllersSink` | `src/lib.rs` | 260 | Sink — wraps `ControllersStore`, reqwest client, GraphQL URL | +| `ControllersSink::new(db_url, max_conns, api_url)` | `src/lib.rs` | 267 | Opens the store, initialises the HTTP client | +| `CONTROLLERS_TABLE`, `CONTROLLERS_STATE_TABLE` | `src/lib.rs` | 21 / 22 | Table-name constants | +| `DEFAULT_API_QUERY_URL` | `src/lib.rs` | 20 | `https://api.cartridge.gg/query` | + +Internal (not re-exported): `ControllersStore`, `StoredController`, `ControllerNode`, `CONTROLLERS_TYPE = TypeId::new("controllers.sync")`. + +### Internal Modules + +- `lib.rs` — the entire crate; one file. Sections: GraphQL response types, `ControllersStore` (schema + upsert + cursor), `ControllersSink` (GraphQL query + polling loop + `Sink` impl). + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `"controllers"` | +| `interested_types` | `[TypeId::new("controllers.sync")]` — ignored; `process` uses `batch`, not envelopes | +| `process(_envelopes, batch)` | Computes `batch_time_window`; if greater than stored `synced_until`, issues GraphQL query filtered by `createdAtGT` / `createdAtLTE`, upserts results in SQLite-sized chunks (199) or Postgres-sized chunks (10 000) | +| `topics` | `Vec::new()` — no EventBus topic | +| `build_routes` | `Router::new()` — no HTTP endpoints | +| `initialize` | Creates schema; if table is empty, performs a **full sync** from epoch to `now` to seed the data | + +### Storage + +- **Tables** (both created with backend-aware DDL): + - `controllers`: `id PK` (hex address), `address UNIQUE`, `username`, `deployed_at` (RFC 3339), `updated_at` (unix seconds). + - `torii_controller_sync_state`: key/value pair; tracks `synced_until` cursor. +- **Upsert batching**: `SQLITE_CONTROLLER_UPSERT_BATCH_SIZE = 199`, `POSTGRES_CONTROLLER_UPSERT_BATCH_SIZE = 10_000` (both avoid the 32 767-parameter SQLite limit and fit well within PG). +- **Address normalisation**: `0x` + `hex::encode(felt.to_be_bytes_vec())` so lookups match whatever shape the on-chain address landed in. + +### GraphQL contract + +Query shape (`src/lib.rs::ControllersSink::build_query`): + +```graphql +query { + controllers( + where: { createdAtGT: "", createdAtLTE: "" }, + orderBy: { field: CREATED_AT, direction: ASC } + ) { + edges { node { address createdAt account { username } } } + } +} +``` + +Retries: `MAX_RETRIES = 3`, exponential backoff starting at +`INITIAL_BACKOFF = 2s`. Partial errors from the API are logged and +dropped; total failures bubble up from `process`. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-arcade`, `bins/torii-introspect-bin`. +- **Downstream deps**: `torii`, `torii-runtime-common`, `torii-sql`, `sqlx` (sqlite/postgres/any), `reqwest`, `chrono`, `hex`, `metrics`, `tracing`, `serde`. + +### Extension Points + +- Point at a different backend API → pass `api_url` override to `ControllersSink::new`. +- Add more fields → extend `ControllerNode`, `StoredController`, schema migration, and the upsert SQL in `ControllersStore::upsert_controllers_and_store_synced_until`. +- Expose the data over gRPC → build a second sink that reads the `controllers` table; keep this sink single-responsibility (sync only). diff --git a/crates/torii-controllers-sink/src/lib.rs b/crates/torii-controllers-sink/src/lib.rs index ce02c551..a4f5d05c 100644 --- a/crates/torii-controllers-sink/src/lib.rs +++ b/crates/torii-controllers-sink/src/lib.rs @@ -1,22 +1,21 @@ -use std::str::FromStr; -use std::sync::Arc; -use std::time::Duration; - use anyhow::{anyhow, Context, Result}; use async_trait::async_trait; use chrono::{DateTime, TimeZone, Utc}; use reqwest::Client; use serde::Deserialize; use serde_json::json; -use sqlx::{ - any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, QueryBuilder, Row, -}; -use starknet::core::types::Felt; +use sqlx::any::AnyPoolOptions; +use sqlx::{Any, Pool, QueryBuilder, Row}; +use starknet_types_raw::Felt; +use std::str::FromStr; +use std::sync::Arc; +use std::time::Duration; use torii::axum::Router; use torii::etl::extractor::ExtractionBatch; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; use torii::etl::TypeId; -use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; +use torii_runtime_common::database::{backend_from_url_or_path, DEFAULT_SQLITE_MAX_CONNECTIONS}; +use torii_sql::DbBackend; pub const DEFAULT_API_QUERY_URL: &str = "https://api.cartridge.gg/query"; pub const CONTROLLERS_TABLE: &str = "controllers"; @@ -28,23 +27,6 @@ const CONTROLLERS_TYPE: TypeId = TypeId::new("controllers.sync"); const CONTROLLER_PROCESSING_BATCH_SIZE: usize = 10_000; const SQLITE_CONTROLLER_UPSERT_BATCH_SIZE: usize = 199; const POSTGRES_CONTROLLER_UPSERT_BATCH_SIZE: usize = 10_000; - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Sqlite, - Postgres, -} - -impl DbBackend { - fn detect(database_url: &str) -> Self { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - Self::Postgres - } else { - Self::Sqlite - } - } -} - #[derive(Debug, Clone, Deserialize)] struct ControllerAccount { username: String, @@ -111,7 +93,7 @@ impl TryFrom for StoredController { value.address ) })?; - let normalized = format!("{felt_addr:#066x}"); + let normalized = normalize_controller_address(&felt_addr); Ok(Self { id: normalized.clone(), address: normalized, @@ -122,6 +104,10 @@ impl TryFrom for StoredController { } } +fn normalize_controller_address(address: &Felt) -> String { + format!("0x{}", hex::encode(address.to_be_bytes_vec())) +} + struct ControllersStore { pool: Pool, backend: DbBackend, @@ -131,7 +117,7 @@ impl ControllersStore { async fn new(database_url: &str, max_connections: Option) -> Result { sqlx::any::install_default_drivers(); - let backend = DbBackend::detect(database_url); + let backend = backend_from_url_or_path(database_url); let database_url = match backend { DbBackend::Postgres => database_url.to_string(), DbBackend::Sqlite => sqlite_url(database_url)?, @@ -554,16 +540,17 @@ fn sqlite_url(path: &str) -> Result { if path.starts_with("sqlite:") { return Ok(path.to_string()); } - let options = SqliteConnectOptions::from_str(&format!("sqlite://{path}")) - .or_else(|_| Ok::<_, sqlx::Error>(SqliteConnectOptions::new().filename(path)))?; - if let Some(parent) = options - .get_filename() + let path = std::path::Path::new(path); + if let Some(parent) = path .parent() .filter(|parent| !parent.as_os_str().is_empty()) { std::fs::create_dir_all(parent)?; } - Ok(options.to_url_lossy().to_string()) + if !path.exists() { + std::fs::File::create(path)?; + } + Ok(format!("sqlite://{}", path.display())) } #[cfg(test)] @@ -572,7 +559,9 @@ mod tests { use serde_json::Value; use sqlx::query_scalar; use tokio::net::TcpListener; - use torii::axum::{extract::State, routing::post, Json, Router}; + use torii::axum::extract::State; + use torii::axum::routing::post; + use torii::axum::{Json, Router}; use torii::command::CommandBus; use torii::grpc::SubscriptionManager; @@ -620,6 +609,26 @@ mod tests { batch } + fn temp_sqlite_path(name: &str) -> String { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock before epoch") + .as_nanos(); + std::env::temp_dir() + .join(format!("torii-controllers-sink-{name}-{nonce}.db")) + .to_string_lossy() + .to_string() + } + + #[tokio::test] + async fn store_accepts_plain_sqlite_path() { + let path = temp_sqlite_path("plain-path"); + let store = ControllersStore::new(&path, Some(1)).await.unwrap(); + + store.initialize().await.unwrap(); + assert!(std::path::Path::new(&path).exists()); + } + #[tokio::test] async fn controllers_sink_persists_rows_and_progress() { let api_url = spawn_graphql_server(json!({ @@ -649,13 +658,18 @@ mod tests { .await .unwrap(); - let username: String = query_scalar(&format!( - "SELECT username FROM {CONTROLLERS_TABLE} WHERE address = ?1" - )) - .bind("0x0000000000000000000000000000000000000000000000000000000000000123") - .fetch_one(&sink.store.pool) - .await - .unwrap(); + let controller_count: i64 = + query_scalar(&format!("SELECT COUNT(*) FROM {CONTROLLERS_TABLE}")) + .fetch_one(&sink.store.pool) + .await + .unwrap(); + assert_eq!(controller_count, 1); + + let username: String = + query_scalar(&format!("SELECT username FROM {CONTROLLERS_TABLE} LIMIT 1")) + .fetch_one(&sink.store.pool) + .await + .unwrap(); assert_eq!(username, "test_user"); let synced_until: i64 = query_scalar(&format!( @@ -709,13 +723,18 @@ mod tests { .await .unwrap(); - let username: String = query_scalar(&format!( - "SELECT username FROM {CONTROLLERS_TABLE} WHERE id = ?1" - )) - .bind("0x0000000000000000000000000000000000000000000000000000000000000123") - .fetch_one(&sink.store.pool) - .await - .unwrap(); + let controller_count: i64 = + query_scalar(&format!("SELECT COUNT(*) FROM {CONTROLLERS_TABLE}")) + .fetch_one(&sink.store.pool) + .await + .unwrap(); + assert_eq!(controller_count, 1); + + let username: String = + query_scalar(&format!("SELECT username FROM {CONTROLLERS_TABLE} LIMIT 1")) + .fetch_one(&sink.store.pool) + .await + .unwrap(); assert_eq!(username, "user_one"); } @@ -802,6 +821,24 @@ mod tests { assert_eq!(username, "test_user"); } + #[test] + fn stored_controller_normalizes_addresses_to_fixed_hex() { + let controller = StoredController::try_from(ControllerNode { + address: "0x123".to_string(), + created_at: "2024-03-20T12:00:00Z".to_string(), + account: ControllerAccount { + username: "test_user".to_string(), + }, + }) + .unwrap(); + + assert_eq!( + controller.address, + "0x0000000000000000000000000000000000000000000000000000000000000123" + ); + assert_eq!(controller.id, controller.address); + } + #[tokio::test] async fn initialize_runs_full_sync_when_table_is_empty() { let api_url = spawn_graphql_server(json!({ @@ -827,13 +864,18 @@ mod tests { .unwrap(); initialize_sink(&mut sink).await; - let username: String = query_scalar(&format!( - "SELECT username FROM {CONTROLLERS_TABLE} WHERE address = ?1" - )) - .bind("0x0000000000000000000000000000000000000000000000000000000000000456") - .fetch_one(&sink.store.pool) - .await - .unwrap(); + let controller_count: i64 = + query_scalar(&format!("SELECT COUNT(*) FROM {CONTROLLERS_TABLE}")) + .fetch_one(&sink.store.pool) + .await + .unwrap(); + assert_eq!(controller_count, 1); + + let username: String = + query_scalar(&format!("SELECT username FROM {CONTROLLERS_TABLE} LIMIT 1")) + .fetch_one(&sink.store.pool) + .await + .unwrap(); assert_eq!(username, "boot_user"); let synced_until: i64 = query_scalar(&format!( diff --git a/crates/torii-ecs-sink/Cargo.toml b/crates/torii-ecs-sink/Cargo.toml index 3598c8c9..f582e783 100644 --- a/crates/torii-ecs-sink/Cargo.toml +++ b/crates/torii-ecs-sink/Cargo.toml @@ -4,12 +4,15 @@ version = "0.1.0" edition = "2021" [dependencies] -torii = { path = "../.." } -torii-dojo = { path = "../dojo" } -torii-introspect = { path = "../introspect" } +torii.workspace = true +torii-dojo = { workspace = true, features = ["postgres", "sqlite"] } +torii-introspect.workspace = true torii-runtime-common.workspace = true +torii-sql.workspace = true + dojo-introspect.workspace = true introspect-types.workspace = true +starknet-types-raw.workspace = true anyhow.workspace = true async-trait.workspace = true @@ -20,8 +23,12 @@ prost.workspace = true prost-types.workspace = true serde.workspace = true serde_json.workspace = true -sqlx = { workspace = true, features = ["runtime-tokio", "sqlite", "postgres", "any"] } -starknet.workspace = true +sqlx = { workspace = true, features = [ + "runtime-tokio", + "sqlite", + "postgres", + "any", +] } tokio.workspace = true tokio-stream.workspace = true tonic.workspace = true diff --git a/crates/torii-ecs-sink/README.md b/crates/torii-ecs-sink/README.md index 67cf3597..a6440acc 100644 --- a/crates/torii-ecs-sink/README.md +++ b/crates/torii-ecs-sink/README.md @@ -1,88 +1,129 @@ -# Torii ECS Sink - -`torii-ecs-sink` exposes the legacy `world.World` gRPC API on top of the data already indexed by `torii-introspect`. - -## What It Serves - -The sink mounts these legacy RPCs into `torii-server` on the same gRPC port: - -- `Worlds` -- `RetrieveEntities` -- `RetrieveEventMessages` -- `RetrieveEvents` -- `SubscribeContracts` -- `SubscribeEntities` -- `SubscribeEventMessages` -- `SubscribeEvents` -- `UpdateEntitiesSubscription` -- `UpdateEventMessagesSubscription` - -The read path is backed by: - -- `torii_dojo_manager_state` for Dojo table metadata -- `torii_ecs_entity_meta` and `torii_ecs_entity_models` for entity and event-message snapshots -- `torii_ecs_events` for raw events -- `torii_ecs_table_kinds` for entity vs event-message classification - -## Manual Validation - -With `torii-server` running on the default port: - -```bash -grpcurl -plaintext localhost:3000 list -grpcurl -plaintext localhost:3000 describe world.World -grpcurl -plaintext -d '{}' localhost:3000 world.World/Worlds -grpcurl -max-time 10 -plaintext -d '{"query":{"pagination":{"limit":1,"direction":"FORWARD"}}}' localhost:3000 world.World/RetrieveEvents +# torii-ecs-sink + +Serves the **legacy `world.World` gRPC API** on top of the data already +persisted by `introspect-sql-sink`. It is the compatibility shim that lets +existing Dojo clients (Cartridge UI, dojo.js SDK) talk to the refactored +Torii pipeline. + +## Role in Torii + +Dojo worlds emit introspect events that `introspect-sql-sink` turns into +relational rows. `torii-ecs-sink` consumes the same `DojoEvent` stream but +only cares about two things: classifying each table as an *entity* or +*event-message*, and handling runtime registration of external contracts +(ERC20/ERC721/ERC1155) so the `EcsService` can enrich its responses. It +does not write business data; it reads from the introspect tables populated +earlier in the sink pipeline to answer `RetrieveEntities`, +`RetrieveEventMessages`, `RetrieveEvents`, and the matching subscription +streams. + +## Architecture + +```text +DojoDecoder (torii-dojo) + | + v DojoBody { DojoEvent::Introspect(...) | DojoEvent::External(...) } + | ++----------------------------------------------+ +| EcsSink | +| | +| process(envelopes, batch): | +| - IntrospectMsg → update table_kind cache | +| (entity | event_message) | +| - ExternalContractRegistered → dispatch | +| RegisterExternalContractCommand on the | +| command bus (if indexing enabled) | +| | +| installed_external_decoders: HashSet | +| contract_types: SharedContractTypeRegistry | ++----------------------+-----------------------+ + | + v ++----------------------------------------------+ +| EcsService | +| | +| sqlx pools: world + optional erc20/721/1155 | +| tables: torii_dojo_manager_state, | +| torii_ecs_entity_meta, | +| torii_ecs_entity_models, | +| torii_ecs_events, | +| torii_ecs_table_kinds | +| | +| gRPC server: world.World | +| - Worlds | +| - RetrieveEntities | +| - RetrieveEventMessages | +| - RetrieveEvents | +| - SubscribeContracts | +| - SubscribeEntities | +| - SubscribeEventMessages | +| - SubscribeEvents | +| - UpdateEntitiesSubscription | +| - UpdateEventMessagesSubscription | ++----------------------------------------------+ ``` -Important: +## Deep Dive -- `RetrieveEntities` only returns models classified as `entity` -- `RetrieveEventMessages` only returns models classified as `event_message` -- a model such as `ARCADE-Sale` can be valid for `RetrieveEventMessages` and intentionally return nothing from `RetrieveEntities` +### Public API -## End-to-End Test Script +| Item | File | Line | Purpose | +|---|---|---|---| +| `EcsSink` | `src/sink.rs` | 23 | Sink wrapping `EcsService` + the shared contract-type registry | +| `EcsSink::new(db_url, max_conns, erc20?, erc721?, erc1155?, types_registry, from_block, enable_indexing, installed_decoders)` | `src/sink.rs` | 33 | Multi-URL constructor | +| `EcsSink::get_grpc_service_impl` | `src/sink.rs` | 63 | Returns `Arc` for `with_grpc_router` | +| `EcsService` | `src/grpc_service.rs` | — | Tonic impl of `world.World` | +| `TableKind` | `src/grpc_service.rs` | — | `Entity` / `EventMessage` classifier | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs` | — | `world.proto` + `types.proto` descriptor bytes | -The repository ships a runnable script at `./scripts/test-ecs-grpc-e2e.sh`. +### Internal Modules -From the repository root: +- `lib.rs` — re-exports + generated `world`, `types`, reflection bytes. +- `sink.rs` — `Sink` impl; consumes `DojoEvent::Introspect` for `TableKind` classification, `DojoEvent::External` for external-contract registration (dispatched via `CommandBusSender` → `RegisterExternalContractCommand`). +- `grpc_service.rs` — `EcsService`; pagination, subscription streams, cross-DB joins between introspect + token tables. -```bash -./scripts/test-ecs-grpc-e2e.sh -``` +### Sink trait wiring -By default the script: +| Method | Behavior | +|---|---| +| `name` | `"ecs"` | +| `interested_types` | `[DOJO_TYPE_ID]` | +| `process(envelopes, batch)` | Downcasts each envelope to `DojoBody`; for `IntrospectMsg::CreateTable/RenameTable/DropTable` updates `torii_ecs_table_kinds`; for `ExternalContractRegistered` dispatches `RegisterExternalContractCommand` through the `CommandBusSender` captured at `initialize` | +| `topics` | Empty — uses gRPC subscription streams directly, not the central EventBus | +| `build_routes` | `Router::new()` — gRPC only | +| `initialize(event_bus, ctx)` | Captures `ctx.command_bus` into its `RwLock>` so `process` can dispatch commands | -- targets `localhost:3000` -- reads sqlite state from `./torii-data/introspect.db` -- autodiscovers one world that has both `entity` and `event_message` models -- validates reflection, `Worlds`, `RetrieveEntities`, `RetrieveEventMessages`, and `RetrieveEvents` +### Storage (reads only — writes via introspect-sql-sink) -For Postgres-backed deployments, set `DB_URL` or `DATABASE_URL` to the same storage database used by `torii-server`. The script will use `psql` for metadata discovery instead of `sqlite3`. +- `torii_dojo_manager_state` — Dojo table metadata +- `torii_ecs_entity_meta` — entity snapshots +- `torii_ecs_entity_models` — per-entity model rows +- `torii_ecs_events` — raw events +- `torii_ecs_table_kinds` — `entity` vs `event_message` classification -### Environment Overrides +**Gotcha**: `RetrieveEntities` only returns tables classified as `entity`; `RetrieveEventMessages` only returns `event_message`. A model like `ARCADE-Sale` can be a valid `event_message` and intentionally return nothing from `RetrieveEntities`. -You can override discovery when needed: +### Interactions -```bash -GRPC_ADDR=localhost:3000 \ -DB_PATH=./torii-data/introspect.db \ -WORLD_ADDRESS=0x7a079295990e43441a7389fdc3b9ba063c6cd6aee16fb846f598c42a9f04ff7 \ -ENTITY_MODEL=ARCADE-Order \ -EVENT_MESSAGE_MODEL=ARCADE-Sale \ -./scripts/test-ecs-grpc-e2e.sh -``` +- **Upstream (consumers)**: `bins/torii-introspect-bin`, `bins/torii-arcade`. +- **Downstream deps**: `torii`, `torii-dojo`, `torii-introspect`, `torii-runtime-common`, `torii-sql`, `sqlx`, `tonic`, `prost`, `starknet-types-raw`, `tokio`, `async-trait`, `tracing`, `metrics`, `hex`. -If `WORLD_ADDRESS`, `ENTITY_MODEL`, and `EVENT_MESSAGE_MODEL` are provided, the script does not need sqlite introspection metadata to choose test cases. +### End-to-end validation -Postgres example: +Run the bundled script against a live `torii-server`: ```bash -GRPC_ADDR=localhost:3000 \ +./scripts/test-ecs-grpc-e2e.sh # auto-discovers via SQLite DB_URL=postgres://torii:torii@localhost:5432/torii \ -./scripts/test-ecs-grpc-e2e.sh + ./scripts/test-ecs-grpc-e2e.sh # Postgres-backed ``` -## Kulala +Overrides: `GRPC_ADDR`, `DB_PATH`, `WORLD_ADDRESS`, `ENTITY_MODEL`, `EVENT_MESSAGE_MODEL`. + +Editor-driven checks: `crates/torii-ecs-sink/ecs-grpc.http` (Kulala-compatible). + +### Extension Points -For editor-driven checks, the crate also includes `./crates/torii-ecs-sink/ecs-grpc.http`. +- New RPCs → `proto/world.proto` + `EcsService`. Keep the stable `world.World` service name — dojo.js depends on it. +- Cross-DB joins → widen `EcsService::new` with a new URL parameter and a new pool. The sink already uses one pool per token standard. +- Avoid adding writes here. Writes belong in `introspect-sql-sink`; this sink is the *read* facade. diff --git a/crates/torii-ecs-sink/src/grpc_service.rs b/crates/torii-ecs-sink/src/grpc_service.rs index 565828cf..060840d6 100644 --- a/crates/torii-ecs-sink/src/grpc_service.rs +++ b/crates/torii-ecs-sink/src/grpc_service.rs @@ -1,10 +1,24 @@ -use std::collections::{HashMap, HashSet, VecDeque}; -use std::pin::Pin; -use std::str::FromStr; -use std::sync::Arc; -use std::task::{Context, Poll}; -use std::time::Instant; - +use crate::proto::types::clause::ClauseType; +use crate::proto::types::member_value::ValueType; +use crate::proto::types::{ + self, ComparisonOperator, ContractType, LogicalOperator, PaginationDirection, PatternMatching, +}; +use crate::proto::world::world_server::World; +use crate::proto::world::{ + RetrieveContractsRequest, RetrieveContractsResponse, RetrieveControllersRequest, + RetrieveControllersResponse, RetrieveEntitiesRequest, RetrieveEntitiesResponse, + RetrieveEventsRequest, RetrieveEventsResponse, RetrieveTokenBalancesRequest, + RetrieveTokenBalancesResponse, RetrieveTokenContractsRequest, RetrieveTokenContractsResponse, + RetrieveTokenTransfersRequest, RetrieveTokenTransfersResponse, RetrieveTokensRequest, + RetrieveTokensResponse, RetrieveTransactionsRequest, RetrieveTransactionsResponse, + SubscribeContractsRequest, SubscribeContractsResponse, SubscribeEntitiesRequest, + SubscribeEntityResponse, SubscribeEventsRequest, SubscribeEventsResponse, + SubscribeTokenBalancesRequest, SubscribeTokenBalancesResponse, SubscribeTokenTransfersRequest, + SubscribeTokenTransfersResponse, SubscribeTokensRequest, SubscribeTokensResponse, + SubscribeTransactionsRequest, SubscribeTransactionsResponse, UpdateEntitiesSubscriptionRequest, + UpdateTokenBalancesSubscriptionRequest, UpdateTokenSubscriptionRequest, + UpdateTokenTransfersSubscriptionRequest, WorldsRequest, WorldsResponse, +}; use anyhow::{anyhow, Result}; use chrono::Utc; use introspect_types::serialize::ToCairoDeSeFrom; @@ -19,46 +33,32 @@ use prost::Message; use serde::ser::SerializeMap; use serde::Serializer; use serde_json::{Map, Serializer as JsonSerializer, Value}; -use sqlx::AnyConnection; -use sqlx::{ - any::AnyPoolOptions, pool::PoolConnection, postgres::PgPoolOptions, - sqlite::SqliteConnectOptions, sqlite::SqlitePoolOptions, Any, Column, ConnectOptions, Pool, - QueryBuilder, Row, -}; -use starknet::core::types::Felt; +use sqlx::any::AnyPoolOptions; +use sqlx::pool::PoolConnection; +use sqlx::postgres::PgPoolOptions; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use sqlx::{Any, AnyConnection, Column, Pool, QueryBuilder, Row}; +use starknet_types_raw::Felt; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::fs; +use std::pin::Pin; +use std::str::FromStr; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Instant; use tokio::sync::mpsc::error::TrySendError; use tokio::sync::{mpsc, Mutex, RwLock}; use tokio::time::{sleep, Duration}; -use tokio_stream::{wrappers::ReceiverStream, Stream}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::Stream; use tonic::{Request, Response, Status}; -use torii_dojo::store::postgres::PgStore; use torii_dojo::store::sqlite::SqliteStore; use torii_dojo::store::DojoStoreTrait; use torii_dojo::DojoTable; use torii_introspect::events::{CreateTable, Record, UpdateTable}; use torii_introspect::schema::TableSchema; use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; - -use crate::proto::types::{ - self, clause::ClauseType, member_value::ValueType, ComparisonOperator, ContractType, - LogicalOperator, PaginationDirection, PatternMatching, -}; -use crate::proto::world::{ - world_server::World, RetrieveContractsRequest, RetrieveContractsResponse, - RetrieveControllersRequest, RetrieveControllersResponse, RetrieveEntitiesRequest, - RetrieveEntitiesResponse, RetrieveEventsRequest, RetrieveEventsResponse, - RetrieveTokenBalancesRequest, RetrieveTokenBalancesResponse, RetrieveTokenContractsRequest, - RetrieveTokenContractsResponse, RetrieveTokenTransfersRequest, RetrieveTokenTransfersResponse, - RetrieveTokensRequest, RetrieveTokensResponse, RetrieveTransactionsRequest, - RetrieveTransactionsResponse, SubscribeContractsRequest, SubscribeContractsResponse, - SubscribeEntitiesRequest, SubscribeEntityResponse, SubscribeEventsRequest, - SubscribeEventsResponse, SubscribeTokenBalancesRequest, SubscribeTokenBalancesResponse, - SubscribeTokenTransfersRequest, SubscribeTokenTransfersResponse, SubscribeTokensRequest, - SubscribeTokensResponse, SubscribeTransactionsRequest, SubscribeTransactionsResponse, - UpdateEntitiesSubscriptionRequest, UpdateTokenBalancesSubscriptionRequest, - UpdateTokenSubscriptionRequest, UpdateTokenTransfersSubscriptionRequest, WorldsRequest, - WorldsResponse, -}; +use torii_sql::DbBackend; const SUBSCRIPTION_SEEN_CACHE_CAPACITY: usize = 4096; @@ -85,29 +85,6 @@ impl TableKind { } } -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Sqlite, - Postgres, -} - -impl DbBackend { - fn detect(database_url: &str) -> Self { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - Self::Postgres - } else { - Self::Sqlite - } - } - - fn as_str(self) -> &'static str { - match self { - Self::Sqlite => "sqlite", - Self::Postgres => "postgres", - } - } -} - #[derive(Clone)] pub struct EcsService { state: Arc, @@ -646,18 +623,17 @@ impl EcsService { ) -> Result { sqlx::any::install_default_drivers(); - let backend = DbBackend::detect(database_url); + let backend = get_db_backend(database_url); let database_url = match backend { DbBackend::Postgres => database_url.to_string(), DbBackend::Sqlite => sqlite_url(database_url)?, }; - let has_erc20 = erc20_url.is_some(); let has_erc721 = erc721_url.is_some(); let has_erc1155 = erc1155_url.is_some(); - let erc20_url = erc20_url.map(std::string::ToString::to_string); - let erc721_url = erc721_url.map(std::string::ToString::to_string); - let erc1155_url = erc1155_url.map(std::string::ToString::to_string); + let erc20_url = normalize_attached_database_url(backend, erc20_url)?; + let erc721_url = normalize_attached_database_url(backend, erc721_url)?; + let erc1155_url = normalize_attached_database_url(backend, erc1155_url)?; let pool_options = AnyPoolOptions::new().max_connections(max_connections.unwrap_or( if backend == DbBackend::Sqlite { @@ -912,15 +888,23 @@ impl EcsService { ("erc1155", &self.state.erc1155_url), ] { if let Some(url) = url { - let path = sqlite_db_path(url); - let file_exists = std::path::Path::new(&path).exists(); - tracing::info!( - schema, - path = %path, - file_exists, - "Attaching ERC database" - ); - attach_sqlite_database(&mut conn, schema, url).await?; + let options = SqliteConnectOptions::from_str(url)?; + #[allow(clippy::match_bool)] + #[allow(clippy::single_match_else)] + match options.is_in_memory() { + true => tracing::info!(schema, "Attaching in-memory ERC database"), + false => { + let path = options.get_filename(); + let file_exists = path.exists(); + tracing::info!( + schema, + path = %path.display(), + file_exists, + "Attaching ERC database" + ); + } + } + attach_sqlite_database(&mut conn, schema, &options).await?; match sqlx::query(sqlite_master_preview_sql(schema)) .fetch_all(&mut *conn) .await @@ -1013,7 +997,7 @@ impl EcsService { let contract = self .load_contracts(&types::ContractQuery { - contract_addresses: vec![contract_address.to_bytes_be().to_vec()], + contract_addresses: vec![contract_address.into()], contract_types: vec![], }) .await? @@ -1166,7 +1150,7 @@ impl EcsService { .bind(kind.as_str()) .bind(felt_hex(world_address)) .bind(felt_hex(table_id)) - .bind(felt_hex(Felt::from_bytes_be(&record.id))) + .bind(felt_hex(record.id)) .bind(row_json) .bind(executed_at as i64) .execute(&self.state.pool) @@ -1263,15 +1247,9 @@ impl EcsService { .await?; self.publish_event_update(types::Event { - keys: keys - .iter() - .map(|felt| felt.to_bytes_be().to_vec()) - .collect(), - data: data - .iter() - .map(|felt| felt.to_bytes_be().to_vec()) - .collect(), - transaction_hash: transaction_hash.to_bytes_be().to_vec(), + keys: keys.iter().map(Into::into).collect(), + data: data.iter().map(Into::into).collect(), + transaction_hash: transaction_hash.into(), }) .await; @@ -1466,7 +1444,7 @@ impl EcsService { { let mut separated = builder.separated(", "); for address in &query.contract_addresses { - separated.push_bind(felt_hex(felt_from_bytes(address)?)); + separated.push_bind(felt_hex(Felt::from_be_bytes_slice(address)?)); } } builder.push(")"); @@ -1492,7 +1470,7 @@ impl EcsService { let contract_address = row.try_get::("contract_address")?; let contract_type = row.try_get::("contract_type")?; let contract = types::Contract { - contract_address: felt_from_hex(&contract_address)?.to_bytes_be().to_vec(), + contract_address: felt_from_hex(&contract_address)?.into(), contract_type, head: row .try_get::, _>("head")? @@ -1507,7 +1485,7 @@ impl EcsService { .try_get::, _>("last_pending_block_tx")? .map(|value| felt_from_hex(&value)) .transpose()? - .map(|felt| felt.to_bytes_be().to_vec()), + .map(Into::into), updated_at: row.try_get::("updated_at")? as u64, created_at: row.try_get::("created_at")? as u64, }; @@ -1544,7 +1522,7 @@ impl EcsService { { let mut separated = builder.separated(", "); for address in &query.contract_addresses { - separated.push_bind(format!("{:#066x}", felt_from_bytes(address)?)); + separated.push_bind(felt_hex(Felt::from_be_bytes_slice(address)?)); } } builder.push(")"); @@ -1601,7 +1579,7 @@ impl EcsService { .map(|value| value.timestamp() as u64) .unwrap_or_default(); Ok(types::Controller { - address: felt_from_hex(&address)?.to_bytes_be().to_vec(), + address: felt_from_hex(&address)?.into(), username: row .try_get::, _>("username")? .unwrap_or_default(), @@ -1992,7 +1970,8 @@ impl EcsService { Ok(rows) => { for row in rows { let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + Felt::from_be_bytes_slice(&row.try_get::, _>("token")?)? + .to_be_bytes_vec(); // Why not just get vec? items.push(types::TokenContract { contract_address, contract_type: ContractType::Erc20 as i32, @@ -2036,8 +2015,7 @@ impl EcsService { )); push_blob_in_filter(&mut builder, "tm.token", &query.contract_addresses); for row in builder.build().fetch_all(&mut *conn).await? { - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? let total_supply = canonical_optional_u256_bytes_from_db( row.try_get::>, _>("total_supply")?, )? @@ -2078,8 +2056,7 @@ impl EcsService { )); push_blob_in_filter(&mut builder, "tm.token", &query.contract_addresses); for row in builder.build().fetch_all(&mut *conn).await? { - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + let contract_address = row.try_get::, _>("token")?; //TODO: Can it just be a vec? items.push(types::TokenContract { contract_address, contract_type: ContractType::Erc1155 as i32, @@ -2120,8 +2097,7 @@ impl EcsService { )); push_blob_in_filter(&mut builder, "token", &query.contract_addresses); for row in builder.build().fetch_all(&mut *conn).await? { - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + let contract_address = row.try_get::, _>("token")?; //TODO: Can it just be a vec? items.push(types::Token { token_id: None, contract_address, @@ -2218,8 +2194,7 @@ impl EcsService { .map(|row| { let token_id = canonical_u256_bytes_from_db(&row.try_get::, _>("token_id")?)?; - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? Ok(types::Token { token_id: Some(token_id), contract_address, @@ -2308,10 +2283,8 @@ impl EcsService { for row in rows { let balance = canonical_u256_bytes_from_db(&row.try_get::, _>("balance")?)?; - let account_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("wallet")?)?; - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + let account_address = row.try_get::, _>("wallet")?; // TODO: can it just be a vec? + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? items.push(types::TokenBalance { balance, account_address, @@ -2376,10 +2349,8 @@ impl EcsService { "RetrieveTokenBalances queried ERC721 ownership" ); for row in rows { - let account_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("owner")?)?; - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; + let account_address = row.try_get::, _>("owner")?; // TODO: can it just be a vec? + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? let token_id = canonical_u256_bytes_from_db(&row.try_get::, _>("token_id")?)?; items.push(types::TokenBalance { @@ -2449,10 +2420,8 @@ impl EcsService { for row in rows { let balance = canonical_u256_bytes_from_db(&row.try_get::, _>("balance")?)?; - let account_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("wallet")?)?; - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("contract")?)?; + let account_address = row.try_get::, _>("wallet")?; // TODO: can it just be a vec? + let contract_address = row.try_get::, _>("contract")?; // TODO: can it just be a vec? let token_id = canonical_u256_bytes_from_db(&row.try_get::, _>("token_id")?)?; items.push(types::TokenBalance { @@ -2621,7 +2590,8 @@ impl EcsService { { let mut separated = builder.separated(", "); for contract in &filter.contract_addresses { - separated.push_bind(felt_hex(felt_from_bytes(contract)?)); + separated.push_bind(hex::encode(contract)); // TODO: Is okay? + // separated.push_bind(felt_hex(felt_from_bytes(contract)?)); } } builder.push(")"); @@ -2630,9 +2600,7 @@ impl EcsService { rows.into_iter() .map(|row| { Ok(( - felt_from_hex(&row.try_get::("transaction_hash")?)? - .to_bytes_be() - .to_vec(), + felt_from_hex(&row.try_get::("transaction_hash")?)?.into(), row.try_get::("block_number")? as u64, row.try_get::("executed_at")? as u64, )) @@ -2738,12 +2706,9 @@ impl EcsService { let rows = builder.build().fetch_all(&mut **conn).await?; rows.into_iter() .map(|row| { - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; - let from_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("from_addr")?)?; - let to_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("to_addr")?)?; + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? + let from_address = row.try_get::, _>("from_addr")?; // TODO: can it just be a vec? + let to_address = row.try_get::, _>("to_addr")?; // TODO: can it just be a vec? let amount = canonical_u256_bytes_from_db(&row.try_get::, _>("amount")?)?; Ok(types::TokenTransfer { id: format!("erc20:{}", row.try_get::("id")?), @@ -2782,12 +2747,9 @@ impl EcsService { let rows = builder.build().fetch_all(&mut **conn).await?; rows.into_iter() .map(|row| { - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; - let from_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("from_addr")?)?; - let to_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("to_addr")?)?; + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? + let from_address = row.try_get::, _>("from_addr")?; // TODO: can it just be a vec? + let to_address = row.try_get::, _>("to_addr")?; // TODO: can it just be a vec? let token_id = canonical_u256_bytes_from_db(&row.try_get::, _>("token_id")?)?; Ok(types::TokenTransfer { @@ -2827,12 +2789,9 @@ impl EcsService { let rows = builder.build().fetch_all(&mut **conn).await?; rows.into_iter() .map(|row| { - let contract_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("token")?)?; - let from_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("from_addr")?)?; - let to_address = - canonical_felt_bytes_from_db(&row.try_get::, _>("to_addr")?)?; + let contract_address = row.try_get::, _>("token")?; // TODO: can it just be a vec? + let from_address = row.try_get::, _>("from_addr")?; // TODO: can it just be a vec? + let to_address = row.try_get::, _>("to_addr")?; // TODO: can it just be a vec? let token_id = canonical_u256_bytes_from_db(&row.try_get::, _>("token_id")?)?; let amount = canonical_u256_bytes_from_db(&row.try_get::, _>("amount")?)?; @@ -3244,7 +3203,7 @@ impl EcsService { continue; } by_world - .entry(table.world_address.to_bytes_be().to_vec()) + .entry(table.world_address.into()) .or_default() .push(model_from_table(&table)); } @@ -3369,7 +3328,7 @@ impl EcsService { .max_connections(1) .connect_with(SqliteConnectOptions::from_str(&self.state.database_url)?) .await?; - let store = SqliteStore(Arc::new(pool)); + let store = SqliteStore(pool); Ok(store.read_tables(&[]).await?) } DbBackend::Postgres => { @@ -3377,8 +3336,7 @@ impl EcsService { .max_connections(1) .connect(&self.state.database_url) .await?; - let store = PgStore(Arc::new(pool)); - Ok(store.read_tables(&[]).await?) + Ok(pool.read_tables(&[]).await?) } } } @@ -3613,7 +3571,7 @@ impl EcsService { for table in models { let rows = self.fetch_table_rows(&table).await?; let meta = self - .load_entity_meta(table.kind, table.world_address, table.table.id) + .load_entity_meta(table.kind, table.world_address, table.table.id.into()) .await?; for row in rows { let entity_id = row @@ -3622,7 +3580,7 @@ impl EcsService { .unwrap_or(Value::Null); let entity_felt = value_to_primary_felt(&entity_id, &table.table.primary.type_def) .unwrap_or(Felt::ZERO); - let entity_key = entity_felt.to_bytes_be().to_vec(); + let entity_key: Vec = entity_felt.into(); if meta .get(&felt_hex(entity_felt)) .is_some_and(|meta| meta.deleted) @@ -3632,12 +3590,9 @@ impl EcsService { let model = row_to_model_struct(&table.table, &row)?; let aggregate = entities - .entry(( - table.world_address.to_bytes_be().to_vec(), - entity_key.clone(), - )) + .entry((table.world_address.into(), entity_key.clone())) .or_insert_with(|| EntityAggregate { - world_address: table.world_address.to_bytes_be().to_vec(), + world_address: table.world_address.into(), hashed_keys: entity_key.clone(), ..EntityAggregate::default() }); @@ -3724,14 +3679,14 @@ impl EcsService { let query = types::Query { clause: Some(types::Clause { clause_type: Some(ClauseType::HashedKeys(types::HashedKeysClause { - hashed_keys: vec![entity_id.to_bytes_be().to_vec()], + hashed_keys: vec![entity_id.into()], })), }), no_hashed_keys: false, models: vec![], pagination: None, historical: false, - world_addresses: vec![world_address.to_bytes_be().to_vec()], + world_addresses: vec![world_address.into()], }; Ok(self .load_entity_page(kind, &query) @@ -3949,8 +3904,8 @@ impl EcsService { .await?; let managed_tables = self.load_managed_table_map().await?; let mut aggregate = EntityAggregate { - world_address: world_address.to_bytes_be().to_vec(), - hashed_keys: entity_id.to_bytes_be().to_vec(), + world_address: world_address.into(), + hashed_keys: entity_id.into(), ..EntityAggregate::default() }; @@ -4147,15 +4102,14 @@ impl EcsService { let event = types::Event { keys: keys_hex .into_iter() - .map(|value| felt_from_hex(&value).map(|felt| felt.to_bytes_be().to_vec())) - .collect::>>()?, + .map(|value| Felt::from_hex(&value).map(|felt| felt.to_be_bytes_vec())) + .collect::, _>>()?, data: data_hex .into_iter() - .map(|value| felt_from_hex(&value).map(|felt| felt.to_bytes_be().to_vec())) - .collect::>>()?, - transaction_hash: felt_from_hex(&row.try_get::("transaction_hash")?)? - .to_bytes_be() - .to_vec(), + .map(|value| Felt::from_hex(&value).map(|felt| felt.to_be_bytes_vec())) + .collect::, _>>()?, + transaction_hash: Felt::from_hex(&row.try_get::("transaction_hash")?)? + .to_be_bytes_vec(), }; if query .keys @@ -4922,16 +4876,28 @@ async fn attach_sqlite_databases( ("erc1155", erc1155_url), ] { if let Some(url) = url { - attach_sqlite_database(conn, schema, url).await?; + let options = SqliteConnectOptions::from_str(url)?; + attach_sqlite_database(conn, schema, &options).await?; } } Ok(()) } +fn normalize_attached_database_url( + backend: DbBackend, + url: Option<&str>, +) -> Result> { + url.map(|url| match backend { + DbBackend::Postgres => Ok(url.to_string()), + DbBackend::Sqlite => sqlite_url(url), + }) + .transpose() +} + async fn attach_sqlite_database( conn: &mut AnyConnection, schema: &str, - url: &str, + options: &SqliteConnectOptions, ) -> sqlx::Result<()> { let attached = sqlx::query_scalar::("SELECT 1 FROM pragma_database_list WHERE name = ? LIMIT 1") @@ -4942,23 +4908,15 @@ async fn attach_sqlite_database( if attached { return Ok(()); } + let path = options.get_filename(); - let db_path = sqlite_db_path(url); - if !is_sqlite_memory_url(url) { - let path = std::path::Path::new(&db_path); - if let Some(parent) = path.parent().filter(|path| !path.as_os_str().is_empty()) { - std::fs::create_dir_all(parent)?; - } - if !path.exists() { - std::fs::OpenOptions::new() - .create(true) - .write(true) - .truncate(false) - .open(path)?; - } + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; } - - let path = db_path.replace('\'', "''"); + if !path.exists() { + fs::File::create(path)?; + } + let path = path.to_string_lossy().replace('\'', "''"); // sqlite-dynamic-ok: ATTACH requires the database path and schema identifier in SQL text. sqlx::query(&format!("ATTACH DATABASE '{path}' AS {schema}")) .execute(&mut *conn) @@ -4966,11 +4924,6 @@ async fn attach_sqlite_database( Ok(()) } -fn is_sqlite_memory_url(url: &str) -> bool { - matches!(url, ":memory:" | "sqlite::memory:") - || (url.starts_with("sqlite:file:") && url.contains("mode=memory")) -} - fn sqlite_master_preview_sql(schema: &str) -> &'static str { match schema { "erc20" => "SELECT name FROM erc20.sqlite_master WHERE type='table' LIMIT 5", @@ -4980,18 +4933,12 @@ fn sqlite_master_preview_sql(schema: &str) -> &'static str { } } -fn sqlite_db_path(url: &str) -> String { - let path = url - .strip_prefix("sqlite://") - .or_else(|| url.strip_prefix("sqlite:")) - .unwrap_or(url); - let p = std::path::Path::new(path); - if let (Some(parent), Some(file_name)) = (p.parent(), p.file_name()) { - if let Ok(abs_parent) = parent.canonicalize() { - return abs_parent.join(file_name).to_string_lossy().into_owned(); - } +fn get_db_backend(url: &str) -> DbBackend { + if url.starts_with("postgres") { + DbBackend::Postgres + } else { + DbBackend::Sqlite } - path.to_string() } fn sqlite_url(path: &str) -> Result { @@ -5001,16 +4948,14 @@ fn sqlite_url(path: &str) -> Result { if path.starts_with("sqlite:") { return Ok(path.to_string()); } - let options = SqliteConnectOptions::from_str(&format!("sqlite://{path}")) - .or_else(|_| Ok::<_, sqlx::Error>(SqliteConnectOptions::new().filename(path)))?; - if let Some(parent) = options - .get_filename() - .parent() - .filter(|path| !path.as_os_str().is_empty()) - { + let path = std::path::Path::new(path); + if let Some(parent) = path.parent().filter(|path| !path.as_os_str().is_empty()) { std::fs::create_dir_all(parent)?; } - Ok(options.to_url_lossy().to_string()) + if !path.exists() { + std::fs::File::create(path)?; + } + Ok(format!("sqlite://{}", path.display())) } impl CairoTypeSerialization for SnapshotJsonSerializer { @@ -5118,7 +5063,7 @@ fn model_from_table(table: &ManagedTable) -> types::Model { schema: serde_json::to_vec(&table.table).unwrap_or_default(), contract_address: Vec::new(), use_legacy_store: table.table.legacy, - world_address: table.world_address.to_bytes_be().to_vec(), + world_address: Vec::::from(table.world_address), } } @@ -5161,7 +5106,7 @@ fn record_to_json_map( .columns .iter() .cloned() - .map(|column| (column.id, column)) + .map(|column| (Felt::from(column.id), column)) .collect::>(); let schema_columns = columns .iter() @@ -5173,15 +5118,15 @@ fn record_to_json_map( .collect::>>()? .into_iter() .cloned() - .map(|column| { - let (_, info): (Felt, ColumnInfo) = column.into(); - info + .map(|column| ColumnInfo { + name: column.name, + attributes: column.attributes, + type_def: column.type_def, }) .collect::>(); - let schema = torii_introspect::tables::RecordSchema::new( - &schema_table.primary, - schema_columns.iter().collect(), - ); + let primary = schema_table.primary.into(); + let schema = + torii_introspect::tables::RecordSchema::new(&primary, schema_columns.iter().collect()); let mut bytes = Vec::new(); let mut serializer = JsonSerializer::new(&mut bytes); @@ -5541,13 +5486,13 @@ fn value_as_string(value: &Value) -> String { fn value_as_felt_bytes(value: &Value) -> Result> { if value.is_null() { - return Ok(Felt::ZERO.to_bytes_be().to_vec()); + return Ok(Felt::ZERO.into()); } let string = value_as_string(value); let trimmed = string.trim(); if trimmed.is_empty() { - return Ok(Felt::ZERO.to_bytes_be().to_vec()); + return Ok(Felt::ZERO.into()); } felt_like_string_to_bytes(trimmed) @@ -5694,7 +5639,7 @@ fn felt_like_string_to_bytes(raw: &str) -> Result> { return Ok(bytes); } - Ok(felt_from_hex(trimmed)?.to_bytes_be().to_vec()) + Ok(felt_from_hex(trimmed)?.into()) } fn decode_prefixed_hex_to_padded_bytes(raw: &str, width: usize) -> Result>> { @@ -5888,10 +5833,6 @@ fn token_balance_subscription_state(balances: &[types::TokenBalance]) -> HashMap .collect() } -fn canonical_felt_bytes_from_db(bytes: &[u8]) -> Result> { - Ok(felt_from_bytes(bytes)?.to_bytes_be().to_vec()) -} - fn canonical_optional_u256_bytes_from_db(bytes: Option>) -> Result>> { bytes .as_deref() @@ -6336,16 +6277,17 @@ fn contract_matches_query(contract: &types::Contract, query: &types::ContractQue || query.contract_types.contains(&contract.contract_type)) } -fn felt_hex(value: Felt) -> String { - format!("{value:#x}") +fn felt_hex>(value: T) -> String { + let value: Felt = value.into(); + format!("0x{}", hex::encode(<[u8; 32]>::from(value))) } -fn felt_from_hex(value: &str) -> Result { - Felt::from_str(value).map_err(|err| anyhow!("invalid felt {value}: {err}")) +fn felt_from_bytes(value: &[u8]) -> Result { + Felt::from_be_bytes_slice(value).map_err(|err| anyhow!("invalid felt bytes: {err}")) } -fn felt_from_bytes(value: &[u8]) -> Result { - Ok(Felt::from_bytes_be_slice(value)) +fn felt_from_hex(value: &str) -> Result { + Felt::from_str(value).map_err(|err| anyhow!("invalid felt {value}: {err}")) } #[cfg(test)] @@ -6374,6 +6316,17 @@ mod tests { format!("sqlite://{}", path.display()) } + fn temp_sqlite_path(name: &str) -> String { + let nonce = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("clock before epoch") + .as_nanos(); + std::env::temp_dir() + .join(format!("torii-ecs-sink-{name}-{nonce}.db")) + .to_string_lossy() + .to_string() + } + fn sql_rows_to_maps(rows: &[types::SqlRow]) -> Vec> { rows.iter() .map(|row| { @@ -6630,7 +6583,7 @@ mod tests { .expect("create dojo_columns"); let table = CreateTable { - id: table_id, + id: table_id.into(), name: table_name.to_string(), attributes: vec![], primary: PrimaryDef { @@ -6639,11 +6592,12 @@ mod tests { type_def: PrimaryTypeDef::Felt252, }, columns: vec![ColumnDef { - id: Felt::from(1_u64), + id: Felt::from(1_u64).into(), name: "open".to_string(), attributes: vec![], type_def: TypeDef::Bool, }], + append_only: false, }; service.cache_created_table(world_address, &table).await; @@ -6725,7 +6679,7 @@ mod tests { .expect("create dojo_columns"); let table = CreateTable { - id: table_id, + id: table_id.into(), name: table_name.to_string(), attributes: vec![], primary: PrimaryDef { @@ -6734,11 +6688,12 @@ mod tests { type_def: PrimaryTypeDef::Felt252, }, columns: vec![ColumnDef { - id: Felt::from(1_u64), + id: Felt::from(1_u64).into(), name: field_name.to_string(), attributes: vec![], type_def: TypeDef::Bool, }], + append_only: false, }; service.cache_created_table(world_address, &table).await; @@ -6781,9 +6736,7 @@ mod tests { key: true, ty: Some(types::Ty { ty_type: Some(types::ty::TyType::Primitive(types::Primitive { - primitive_type: Some(types::primitive::PrimitiveType::Felt252( - value.to_bytes_be().to_vec(), - )), + primitive_type: Some(types::primitive::PrimitiveType::Felt252(value.into())), })), }), } @@ -6834,7 +6787,7 @@ mod tests { .expect("create dojo_columns"); let table = CreateTable { - id: table_id, + id: table_id.into(), name: table_name.to_string(), attributes: vec![], primary: PrimaryDef { @@ -6843,11 +6796,12 @@ mod tests { type_def: PrimaryTypeDef::Felt252, }, columns: vec![ColumnDef { - id: Felt::from(1_u64), + id: Felt::from(1_u64).into(), name: key_field_name.to_string(), attributes: vec![Attribute::new_empty("key".to_string())], type_def: TypeDef::Felt252, }], + append_only: false, }; service.cache_created_table(world_address, &table).await; @@ -6884,6 +6838,43 @@ mod tests { .expect("insert entity model"); } + #[tokio::test] + async fn service_accepts_plain_sqlite_paths_for_attached_token_databases() { + let db_path = temp_sqlite_path("ecs-main-plain"); + let erc20_url = temp_sqlite_path("erc20-plain"); + let erc721_url = temp_sqlite_path("erc721-plain"); + let erc1155_url = temp_sqlite_path("erc1155-plain"); + + let service = EcsService::new( + &db_path, + Some(1), + Some(&erc20_url), + Some(&erc721_url), + Some(&erc1155_url), + ) + .await + .expect("service init"); + + assert!(service.state.has_erc20); + assert!(service.state.has_erc721); + assert!(service.state.has_erc1155); + assert!(service + .state + .erc20_url + .as_deref() + .is_some_and(|url| url.starts_with("sqlite:"))); + assert!(service + .state + .erc721_url + .as_deref() + .is_some_and(|url| url.starts_with("sqlite:"))); + assert!(service + .state + .erc1155_url + .as_deref() + .is_some_and(|url| url.starts_with("sqlite:"))); + } + #[tokio::test] async fn subscribe_entities_update_flow() { let db_path = test_db_path("entities-sub"); @@ -6907,7 +6898,7 @@ mod tests { let response = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: Some(member_bool_clause("test-Lobby", "open", true)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities"); @@ -6932,13 +6923,13 @@ mod tests { .expect("entity ok"); assert_eq!(matched.subscription_id, subscription_id); let matched_entity = matched.entity.expect("entity payload"); - assert_eq!(matched_entity.world_address, world.to_bytes_be().to_vec()); + assert_eq!(matched_entity.world_address, Vec::::from(world)); service .update_entities_subscription(Request::new(UpdateEntitiesSubscriptionRequest { subscription_id, clause: Some(member_bool_clause("test-Lobby", "open", false)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("update entities subscription"); @@ -6962,7 +6953,7 @@ mod tests { let response = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: None, - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities"); @@ -6992,7 +6983,7 @@ mod tests { let response = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: None, - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities"); @@ -7023,14 +7014,14 @@ mod tests { let response_a = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: Some(clause.clone()), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities a"); let response_b = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: Some(clause), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities b"); @@ -7072,14 +7063,14 @@ mod tests { let response_a = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: Some(member_bool_clause("test-Lobby", "open", true)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities a"); let response_b = service .subscribe_entities(Request::new(SubscribeEntitiesRequest { clause: Some(member_bool_clause("test-Lobby", "open", true)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe entities b"); @@ -7103,7 +7094,7 @@ mod tests { .update_entities_subscription(Request::new(UpdateEntitiesSubscriptionRequest { subscription_id: setup_a.subscription_id, clause: Some(member_bool_clause("test-Lobby", "open", false)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("update entities subscription"); @@ -7140,7 +7131,7 @@ mod tests { let response = service .subscribe_event_messages(Request::new(SubscribeEntitiesRequest { clause: Some(member_bool_clause("test-EventMessage", "open", true)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe event messages"); @@ -7170,7 +7161,7 @@ mod tests { .update_event_messages_subscription(Request::new(UpdateEntitiesSubscriptionRequest { subscription_id, clause: Some(member_bool_clause("test-EventMessage", "open", false)), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("update event messages subscription"); @@ -7194,7 +7185,7 @@ mod tests { let response = service .subscribe_event_messages(Request::new(SubscribeEntitiesRequest { clause: None, - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], })) .await .expect("subscribe event messages"); @@ -7226,7 +7217,7 @@ mod tests { let response = service .subscribe_events(Request::new(SubscribeEventsRequest { keys: vec![types::KeysClause { - keys: vec![key_match.to_bytes_be().to_vec()], + keys: vec![key_match.into()], pattern_matching: PatternMatching::VariableLen as i32, models: vec![], }], @@ -7270,10 +7261,10 @@ mod tests { .expect("matched frame") .expect("matched ok"); let event = matched.event.expect("event payload"); - assert_eq!(event.transaction_hash, tx2.to_bytes_be().to_vec()); + assert_eq!(event.transaction_hash, Vec::::from(tx2)); assert_eq!( event.keys.first().cloned(), - Some(key_match.to_bytes_be().to_vec()) + Some(Vec::::from(key_match)) ); } @@ -7314,7 +7305,7 @@ mod tests { let response = service .subscribe_contracts(Request::new(SubscribeContractsRequest { query: Some(types::ContractQuery { - contract_addresses: vec![world_contract.to_bytes_be().to_vec()], + contract_addresses: vec![world_contract.into()], contract_types: vec![ContractType::World as i32], }), })) @@ -7342,10 +7333,7 @@ mod tests { .expect("matched frame") .expect("matched ok"); let contract = matched.contract.expect("contract payload"); - assert_eq!( - contract.contract_address, - world_contract.to_bytes_be().to_vec() - ); + assert_eq!(contract.contract_address, Vec::::from(world_contract)); assert_eq!(contract.contract_type, ContractType::World as i32); } @@ -7471,8 +7459,8 @@ mod tests { .await .expect("create erc20 balances"); - let token = Felt::from(0x99_u64).to_bytes_be().to_vec(); - let wallet = Felt::from(0x55_u64).to_bytes_be().to_vec(); + let token: Vec = Felt::from(0x99_u64).into(); + let wallet: Vec = Felt::from(0x55_u64).into(); let original_balance = u256_bytes_from_u64(1); let updated_balance = u256_bytes_from_u64(5); @@ -7680,7 +7668,7 @@ mod tests { clause_type: Some(ClauseType::Keys(types::KeysClause { models: vec!["NUMS-Config".to_string()], pattern_matching: PatternMatching::VariableLen as i32, - keys: vec![Felt::from(0x222_u64).to_bytes_be().to_vec()], + keys: vec![Felt::from(0x222_u64).into()], })), }; @@ -7781,7 +7769,7 @@ mod tests { direction: PaginationDirection::Forward as i32, order_by: vec![], }), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], models: vec![], clause: Some(member_bool_clause("NUMS-QuestDefinition", "enabled", true)), no_hashed_keys: false, @@ -7844,7 +7832,7 @@ mod tests { direction: PaginationDirection::Forward as i32, order_by: vec![], }), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], models: vec![], clause: Some(member_bool_clause("NUMS-QuestDefinition", "enabled", true)), no_hashed_keys: false, @@ -7912,13 +7900,13 @@ mod tests { direction: PaginationDirection::Forward as i32, order_by: vec![], }), - world_addresses: vec![world.to_bytes_be().to_vec()], + world_addresses: vec![world.into()], models: vec![], clause: Some(types::Clause { clause_type: Some(ClauseType::Keys(types::KeysClause { models: vec!["NUMS-Config".to_string()], pattern_matching: PatternMatching::VariableLen as i32, - keys: vec![matching_key.to_bytes_be().to_vec()], + keys: vec![matching_key.into()], })), }), no_hashed_keys: false, @@ -7932,7 +7920,7 @@ mod tests { assert_eq!(response.entities.len(), 1); assert_eq!( response.entities[0].hashed_keys, - matching_entity.to_bytes_be().to_vec() + Vec::::from(matching_entity) ); assert!(response.next_cursor.is_empty()); } @@ -8085,8 +8073,8 @@ mod tests { "INSERT INTO erc20.balances (token, wallet, balance, last_block, last_tx_hash) VALUES (?1, ?2, ?3, ?4, ?5)", ) - .bind(Felt::from(0x99_u64).to_bytes_be().to_vec()) - .bind(Felt::from(0x55_u64).to_bytes_be().to_vec()) + .bind(Vec::::from(Felt::from(0x99_u64))) + .bind(Vec::::from(Felt::from(0x55_u64))) .bind(vec![1_u8]) .bind("1") .bind(vec![0_u8; 32]) @@ -8112,9 +8100,9 @@ mod tests { "INSERT INTO erc721.nft_ownership (token, token_id, owner, block_number, tx_hash, timestamp) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", ) - .bind(Felt::from(0x1137_u64).to_bytes_be().to_vec()) + .bind(Vec::::from(Felt::from(0x1137_u64))) .bind(vec![0x04_u8]) - .bind(Felt::from(0x25145_u64).to_bytes_be().to_vec()) + .bind(Vec::::from(Felt::from(0x25145_u64))) .bind("1") .bind(vec![1_u8; 32]) .bind("1774350232") @@ -8228,8 +8216,8 @@ mod tests { "INSERT INTO erc20.balances (token, wallet, balance, last_block, last_tx_hash) VALUES (?1, ?2, ?3, ?4, ?5)", ) - .bind(Felt::from(0x99_u64).to_bytes_be().to_vec()) - .bind(Felt::from(0x55_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x99_u64).as_be_bytes_slice()) + .bind(Felt::from(0x55_u64).as_be_bytes_slice()) .bind(vec![1_u8]) .bind("1") .bind(vec![0_u8; 32]) @@ -8264,11 +8252,11 @@ mod tests { assert_eq!(response.balances.len(), 1); assert_eq!( response.balances[0].contract_address, - Felt::from(0x99_u64).to_bytes_be().to_vec() + Felt::from(0x99_u64).to_be_bytes_vec() ); assert_eq!( response.balances[0].account_address, - Felt::from(0x55_u64).to_bytes_be().to_vec() + Felt::from(0x55_u64).to_be_bytes_vec() ); assert_eq!(response.balances[0].balance, u256_bytes_from_u64(1)); assert_eq!(response.balances[0].token_id, None); @@ -8428,7 +8416,7 @@ mod tests { "INSERT INTO erc20.token_metadata (token, name, symbol, decimals, total_supply) VALUES (?1, ?2, ?3, ?4, ?5)", ) - .bind(Felt::from(0x99_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x99_u64).to_be_bytes_vec()) .bind("Nums") .bind("NUMS") .bind("18") @@ -8461,7 +8449,7 @@ mod tests { canonical_u256_bytes_from_db(&[0x12, 0x34, 0x56]).expect("canonical"); assert_eq!( contract.contract_address, - Felt::from(0x99_u64).to_bytes_be().to_vec() + Felt::from(0x99_u64).to_be_bytes_vec() ); assert_eq!(contract.total_supply.as_ref(), Some(&expected_total_supply)); assert_eq!(contract.metadata, Vec::::new()); @@ -8499,9 +8487,9 @@ mod tests { "INSERT INTO erc721.nft_ownership (token, token_id, owner) VALUES (?1, ?2, ?3)", ) - .bind(Felt::from(0x1137_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x1137_u64).to_be_bytes_vec()) .bind(vec![0x04_u8]) - .bind(Felt::from(0x25145_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x25145_u64).to_be_bytes_vec()) .execute(&mut *conn) .await .expect("insert ownership"); @@ -8530,11 +8518,11 @@ mod tests { let expected_token_id = u256_bytes_from_u64(4); assert_eq!( balance.contract_address, - Felt::from(0x1137_u64).to_bytes_be().to_vec() + Felt::from(0x1137_u64).to_be_bytes_vec() ); assert_eq!( balance.account_address, - Felt::from(0x25145_u64).to_bytes_be().to_vec() + Felt::from(0x25145_u64).to_be_bytes_vec() ); assert_eq!(balance.balance, u256_bytes_from_u64(1)); assert_eq!(balance.token_id.as_ref(), Some(&expected_token_id)); @@ -8606,8 +8594,8 @@ mod tests { "INSERT INTO erc20.balances (token, wallet, balance, last_block, last_tx_hash) VALUES (?1, ?2, ?3, ?4, ?5)", ) - .bind(Felt::from(0x30_u64).to_bytes_be().to_vec()) - .bind(Felt::from(0x10_u64).to_bytes_be().to_vec()) + .bind(Vec::::from(Felt::from(0x30_u64))) + .bind(Vec::::from(Felt::from(0x10_u64))) .bind(vec![9_u8]) .bind("1") .bind(vec![0_u8; 32]) @@ -8618,9 +8606,9 @@ mod tests { "INSERT INTO erc721.nft_ownership (token, token_id, owner) VALUES (?1, ?2, ?3)", ) - .bind(Felt::from(0x10_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x10_u64).to_be_bytes_vec()) .bind(vec![0x02_u8]) - .bind(Felt::from(0x20_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x20_u64).to_be_bytes_vec()) .execute(&mut *conn) .await .expect("insert erc721 ownership"); @@ -8628,8 +8616,8 @@ mod tests { "INSERT INTO erc1155.erc1155_balances (contract, wallet, token_id, balance, last_block) VALUES (?1, ?2, ?3, ?4, ?5)", ) - .bind(Felt::from(0x20_u64).to_bytes_be().to_vec()) - .bind(Felt::from(0x05_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x20_u64).to_be_bytes_vec()) + .bind(Felt::from(0x05_u64).to_be_bytes_vec()) .bind(vec![0x01_u8]) .bind(vec![7_u8]) .bind("1") @@ -8659,11 +8647,11 @@ mod tests { assert_eq!(response.balances.len(), 2); assert_eq!( response.balances[0].contract_address, - Felt::from(0x10_u64).to_bytes_be().to_vec() + Felt::from(0x10_u64).to_be_bytes_vec() ); assert_eq!( response.balances[0].account_address, - Felt::from(0x20_u64).to_bytes_be().to_vec() + Felt::from(0x20_u64).to_be_bytes_vec() ); assert_eq!( response.balances[0].token_id.as_ref(), @@ -8672,11 +8660,11 @@ mod tests { assert_eq!( response.balances[1].contract_address, - Felt::from(0x20_u64).to_bytes_be().to_vec() + Felt::from(0x20_u64).to_be_bytes_vec() ); assert_eq!( response.balances[1].account_address, - Felt::from(0x05_u64).to_bytes_be().to_vec() + Felt::from(0x05_u64).to_be_bytes_vec() ); assert_eq!( response.balances[1].token_id.as_ref(), @@ -8715,9 +8703,9 @@ mod tests { "INSERT INTO erc721.nft_ownership (token, token_id, owner) VALUES (?1, ?2, ?3)", ) - .bind(Felt::from(0x99_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x99_u64).to_be_bytes_vec()) .bind(vec![0x04_u8]) - .bind(Felt::from(0x77_u64).to_bytes_be().to_vec()) + .bind(Felt::from(0x77_u64).to_be_bytes_vec()) .execute(&mut *conn) .await .expect("insert ownership"); @@ -8744,11 +8732,11 @@ mod tests { assert_eq!(response.balances.len(), 1); assert_eq!( response.balances[0].contract_address, - Felt::from(0x99_u64).to_bytes_be().to_vec() + Felt::from(0x99_u64).to_be_bytes_vec() ); assert_eq!( response.balances[0].account_address, - Felt::from(0x77_u64).to_bytes_be().to_vec() + Felt::from(0x77_u64).to_be_bytes_vec() ); assert_eq!( response.balances[0].token_id.as_ref(), @@ -8790,7 +8778,7 @@ mod tests { let service = EcsService::new(&db_path, Some(1), None, None, None) .await .expect("service init"); - let controller = format!("{:#066x}", Felt::from(0x123_u64)); + let controller = felt_hex(Felt::from(0x123_u64)); sqlx::query( "CREATE TABLE controllers ( @@ -8820,7 +8808,7 @@ mod tests { let response = service .retrieve_controllers(Request::new(RetrieveControllersRequest { query: Some(types::ControllerQuery { - contract_addresses: vec![Felt::from(0x123_u64).to_bytes_be().to_vec()], + contract_addresses: vec![Felt::from(0x123_u64).into()], usernames: Vec::new(), pagination: Some(types::Pagination { cursor: String::new(), @@ -8838,7 +8826,7 @@ mod tests { assert!(response.next_cursor.is_empty()); assert_eq!( response.controllers[0].address, - Felt::from(0x123_u64).to_bytes_be().to_vec() + Vec::::from(Felt::from(0x123_u64)) ); assert_eq!(response.controllers[0].username, "alice"); assert_eq!(response.controllers[0].deployed_at_timestamp, 1_710_936_000); @@ -8871,7 +8859,7 @@ mod tests { .await .expect("begin controller insert tx"); for i in 0..1_500_u64 { - let controller = format!("{:#066x}", Felt::from(i + 1)); + let controller = felt_hex(Felt::from(i + 1)); sqlx::query( "INSERT INTO controllers (id, address, username, deployed_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5)", @@ -8911,14 +8899,14 @@ mod tests { .controllers .first() .map(|controller| controller.address.clone()), - Some(Felt::from(1_u64).to_bytes_be().to_vec()) + Some(Vec::::from(Felt::from(1_u64))) ); assert_eq!( response .controllers .last() .map(|controller| controller.address.clone()), - Some(Felt::from(1_200_u64).to_bytes_be().to_vec()) + Some(Vec::::from(Felt::from(1_200_u64))) ); } } diff --git a/crates/torii-ecs-sink/src/sink.rs b/crates/torii-ecs-sink/src/sink.rs index 2e0680d2..30013862 100644 --- a/crates/torii-ecs-sink/src/sink.rs +++ b/crates/torii-ecs-sink/src/sink.rs @@ -3,24 +3,20 @@ use std::sync::{Arc, RwLock}; use anyhow::Result; use async_trait::async_trait; -use dojo_introspect::events::{ - EventEmitted, StoreDelRecord, StoreSetRecord, StoreUpdateMember, StoreUpdateRecord, -}; -use introspect_types::event::CairoEventInfo; -use starknet::core::types::Felt; +use starknet_types_raw::Felt; use torii::axum::Router; use torii::command::CommandBusSender; use torii::etl::decoder::DecoderId; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, SinkContext, TopicInfo}, -}; +use torii::etl::envelope::{Envelope, TypeId}; +use torii::etl::extractor::ExtractionBatch; +use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; +use torii::etl::EventContext; use torii_dojo::external_contract::{ - resolve_external_contract, ExternalContractRegisteredBody, RegisterExternalContractCommand, + resolve_external_contract, ExternalContractRegistered, RegisterExternalContractCommand, RegisteredContractType, SharedContractTypeRegistry, }; -use torii_introspect::events::{IntrospectBody, IntrospectMsg}; +use torii_dojo::{DojoBody, DojoEvent, DOJO_TYPE_ID}; +use torii_introspect::events::IntrospectMsg; use crate::grpc_service::{EcsService, TableKind}; @@ -73,7 +69,7 @@ impl EcsSink { .contract_types .read() .await - .get(&contract_address) + .get(&contract_address.into()) .copied() .unwrap_or(RegisteredContractType::World); @@ -86,22 +82,26 @@ impl EcsSink { } } - async fn handle_external_contract_registered(&self, body: &ExternalContractRegisteredBody) { + async fn handle_external_contract_registered( + &self, + context: &EventContext, + msg: &ExternalContractRegistered, + ) { if !self.external_contract_indexing { tracing::debug!( target: "torii::ecs_sink", - contract = format!("{:#x}", body.msg.contract_address), + contract = format!("{:#x}", msg.contract_address), "External contract registration ignored because runtime indexing is disabled" ); return; } - let Some(resolved) = resolve_external_contract(&body.msg.contract_name) else { + let Some(resolved) = resolve_external_contract(&msg.contract_name) else { tracing::warn!( target: "torii::ecs_sink", - world = format!("{:#x}", body.metadata.from_address), - contract = format!("{:#x}", body.msg.contract_address), - contract_name = %body.msg.contract_name, + world = format!("{:#x}", context.from_address), + contract = format!("{:#x}", msg.contract_address), + contract_name = %msg.contract_name, "Unsupported external contract registration; skipping runtime indexing" ); return; @@ -116,9 +116,9 @@ impl EcsSink { if !missing_decoders.is_empty() { tracing::warn!( target: "torii::ecs_sink", - world = format!("{:#x}", body.metadata.from_address), - contract = format!("{:#x}", body.msg.contract_address), - contract_name = %body.msg.contract_name, + world = format!("{:#x}", context.from_address), + contract = format!("{:#x}", msg.contract_address), + contract_name = %msg.contract_name, missing_decoders = ?missing_decoders, "External contract registration skipped because matching decoders are not installed" ); @@ -133,27 +133,27 @@ impl EcsSink { let Some(command_bus) = command_bus else { tracing::warn!( target: "torii::ecs_sink", - contract = format!("{:#x}", body.msg.contract_address), + contract = format!("{:#x}", msg.contract_address), "External contract registration dropped because the command bus is unavailable" ); return; }; if let Err(error) = command_bus.dispatch(RegisterExternalContractCommand { - world_address: body.metadata.from_address, - contract_address: body.msg.contract_address, - contract_name: body.msg.contract_name.clone(), - namespace: body.msg.namespace.clone(), - instance_name: body.msg.instance_name.clone(), + world_address: context.from_address.into(), + contract_address: msg.contract_address, + contract_name: msg.contract_name.clone(), + namespace: msg.namespace.clone(), + instance_name: msg.instance_name.clone(), from_block: self.external_contract_from_block, - registration_block: body.msg.registration_block, + registration_block: msg.registration_block, contract_type: resolved.contract_type, decoder_ids: resolved.decoder_ids, }) { tracing::warn!( target: "torii::ecs_sink", - world = format!("{:#x}", body.metadata.from_address), - contract = format!("{:#x}", body.msg.contract_address), + world = format!("{:#x}", context.from_address), + contract = format!("{:#x}", msg.contract_address), error = %error, "Failed to enqueue external contract registration command" ); @@ -168,23 +168,25 @@ impl Sink for EcsSink { } fn interested_types(&self) -> Vec { - vec![ - TypeId::new("introspect"), - TypeId::new("dojo.external_contract_registered"), - ] + vec![DOJO_TYPE_ID] } async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> Result<()> { for (ordinal, event) in batch.events.iter().enumerate() { - let context = batch - .get_event_context(&event.transaction_hash, event.from_address) - .unwrap_or_default(); + let block_number = batch + .transactions + .get(&event.transaction_hash) + .map_or(event.block_number, |tx| tx.block_number); + let timestamp = batch + .blocks + .get(&block_number) + .map_or(0, |block| block.timestamp); self.service .store_event( event.from_address, event.transaction_hash, - context.transaction.block_number, - context.block.timestamp, + block_number, + timestamp, &event.keys, &event.data, ordinal, @@ -195,8 +197,8 @@ impl Sink for EcsSink { .record_contract_progress( event.from_address, self.contract_type_for(event.from_address).await, - context.transaction.block_number, - context.block.timestamp, + block_number, + timestamp, None, ) .await?; @@ -204,13 +206,13 @@ impl Sink for EcsSink { let Some(selector) = event.keys.first() else { continue; }; - let selector_raw = selector.to_raw(); + let selector_raw = *selector; if matches!( selector_raw, - StoreSetRecord::SELECTOR_RAW - | StoreUpdateRecord::SELECTOR_RAW - | StoreUpdateMember::SELECTOR_RAW - | StoreDelRecord::SELECTOR_RAW + s if s == Felt::selector("StoreSetRecord") + || s == Felt::selector("StoreUpdateRecord") + || s == Felt::selector("StoreUpdateMember") + || s == Felt::selector("StoreDelRecord") ) { if event.keys.len() >= 3 { self.service @@ -222,12 +224,12 @@ impl Sink for EcsSink { event.from_address, event.keys[1], event.keys[2], - context.block.timestamp, - selector_raw == StoreDelRecord::SELECTOR_RAW, + timestamp, + selector_raw == Felt::selector("StoreDelRecord"), ) .await?; } - } else if selector_raw == EventEmitted::SELECTOR_RAW && event.keys.len() >= 2 { + } else if selector_raw == Felt::selector("EventEmitted") && event.keys.len() >= 2 { self.service .record_table_kind(event.from_address, event.keys[1], TableKind::EventMessage) .await?; @@ -235,48 +237,59 @@ impl Sink for EcsSink { } for envelope in envelopes { - if envelope.type_id == TypeId::new("dojo.external_contract_registered") { - if let Some(body) = envelope.downcast_ref::() { - self.handle_external_contract_registered(body).await; - } + if envelope.type_id != DOJO_TYPE_ID { continue; } - if envelope.type_id != TypeId::new("introspect") { + let Some(body) = envelope.downcast_ref::() else { continue; - } - - let Some(body) = envelope.downcast_ref::() else { + }; + let DojoEvent::Introspect(msg) = &body.msg else { + if let DojoEvent::ExternalContractRegistered(msg) = &body.msg { + self.handle_external_contract_registered(&body.context, msg) + .await; + } continue; }; - let context = batch - .get_event_context(&body.metadata.transaction_hash, body.metadata.from_address) - .unwrap_or_default(); - match &body.msg { + let timestamp = batch + .transactions + .get(&body.context.transaction_hash) + .and_then(|tx| batch.blocks.get(&tx.block_number)) + .map_or(0, |block| block.timestamp); + + match msg { IntrospectMsg::CreateTable(table) => { self.service - .cache_created_table(body.metadata.from_address, table) + .cache_created_table(body.context.from_address, table) .await; self.service - .record_table_kind(body.metadata.from_address, table.id, TableKind::Entity) + .record_table_kind( + body.context.from_address, + table.id.into(), + TableKind::Entity, + ) .await .ok(); } IntrospectMsg::UpdateTable(table) => { self.service - .cache_updated_table(body.metadata.from_address, table) + .cache_updated_table(body.context.from_address, table) .await; self.service - .record_table_kind(body.metadata.from_address, table.id, TableKind::Entity) + .record_table_kind( + body.context.from_address, + table.id.into(), + TableKind::Entity, + ) .await .ok(); } IntrospectMsg::InsertsFields(insert) => { let kind = if batch.events.iter().any(|event| { event.keys.first().is_some_and(|selector| { - selector.to_raw() == EventEmitted::SELECTOR_RAW - && event.keys.get(1).copied() == Some(insert.table) + *selector == Felt::selector("EventEmitted") + && event.keys.get(1).copied() == Some(insert.table.into()) }) }) { TableKind::EventMessage @@ -284,59 +297,65 @@ impl Sink for EcsSink { TableKind::Entity }; self.service - .record_table_kind(body.metadata.from_address, insert.table, kind) + .record_table_kind(body.context.from_address, insert.table.into(), kind) .await?; + let raw_columns: Vec = + insert.columns.iter().copied().map(Into::into).collect(); for record in &insert.records { - let entity_id = Felt::from_bytes_be(&record.id); + let entity_id = Felt::from_be_bytes_slice(&record.id)?; self.service .upsert_entity_meta( kind, - body.metadata.from_address, - insert.table, + body.context.from_address, + insert.table.into(), entity_id, - context.block.timestamp, + timestamp, false, ) .await?; self.service .upsert_entity_model( kind, - body.metadata.from_address, - insert.table, - &insert.columns, + body.context.from_address, + insert.table.into(), + &raw_columns, record, - context.block.timestamp, + timestamp, ) .await?; self.service - .publish_entity_update(kind, body.metadata.from_address, entity_id) + .publish_entity_update(kind, body.context.from_address, entity_id) .await?; } } IntrospectMsg::DeleteRecords(delete) => { - let kind = self.service.table_kind(delete.table).await?; + let kind = self.service.table_kind(delete.table.into()).await?; for row in &delete.rows { let entity_id = row.to_felt(); self.service .upsert_entity_meta( kind, - body.metadata.from_address, - delete.table, - entity_id, - context.block.timestamp, + body.context.from_address, + delete.table.into(), + entity_id.into(), + timestamp, true, ) .await?; self.service .delete_entity_model( kind, - body.metadata.from_address, - delete.table, - entity_id, + body.context.from_address, + delete.table.into(), + entity_id.into(), ) .await?; self.service - .publish_entity_update(kind, body.metadata.from_address, entity_id) + .publish_entity_update( + kind, + body.context.from_address, + entity_id.into(), + ) .await?; } } diff --git a/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs b/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs index 58487346..c6a1c590 100644 --- a/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs +++ b/crates/torii-ecs-sink/tests/sqlite_prepared_statements.rs @@ -23,8 +23,7 @@ fn runtime_sqlite_queries_avoid_uncached_prepare_and_inline_format_sql() { let root = workspace_root(); let files = [ "crates/torii-ecs-sink/src/grpc_service.rs", - "crates/torii-entities-historical-sink/src/lib.rs", - "crates/introspect-sqlite-sink/src/processor.rs", + "crates/introspect-sql-sink/src/sqlite/table.rs", "crates/torii-erc20/src/storage.rs", "crates/torii-erc721/src/storage.rs", "crates/torii-erc1155/src/storage.rs", diff --git a/crates/torii-entities-historical-sink/Cargo.toml b/crates/torii-entities-historical-sink/Cargo.toml deleted file mode 100644 index 710230ef..00000000 --- a/crates/torii-entities-historical-sink/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "torii-entities-historical-sink" -version = "0.1.0" -edition = "2021" -description = "Append-only historical entity sink for Torii introspect models" -authors = ["Torii Runtime "] -license = "Apache-2.0" - -[dependencies] -anyhow.workspace = true -async-trait.workspace = true -serde.workspace = true -serde_json.workspace = true -sqlx = { workspace = true, features = ["any", "postgres", "sqlite", "runtime-tokio-rustls"] } -starknet.workspace = true -tokio.workspace = true -torii.workspace = true -torii-runtime-common.workspace = true -tracing.workspace = true -thiserror.workspace = true -introspect-types.workspace = true - -torii-introspect.workspace = true - -[lints] -workspace = true diff --git a/crates/torii-entities-historical-sink/src/lib.rs b/crates/torii-entities-historical-sink/src/lib.rs deleted file mode 100644 index 025d2182..00000000 --- a/crates/torii-entities-historical-sink/src/lib.rs +++ /dev/null @@ -1,1199 +0,0 @@ -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; - -use anyhow::{anyhow, Context, Result}; -use async_trait::async_trait; -use sqlx::any::AnyPoolOptions; -use sqlx::{Any, Pool, Row}; -use starknet::core::types::Felt; -use tokio::sync::RwLock; -use torii::axum::Router; -use torii::etl::envelope::{Envelope, TypeId}; -use torii::etl::extractor::ExtractionBatch; -use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; -use torii_introspect::events::{CreateTable, IntrospectBody, IntrospectMsg, UpdateTable}; -use torii_introspect::schema::TableSchema; -use torii_runtime_common::database::DEFAULT_SQLITE_MAX_CONNECTIONS; - -const INTROSPECT_TYPE: TypeId = TypeId::new("introspect"); -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum DbBackend { - Sqlite, - Postgres, -} - -impl DbBackend { - fn detect(database_url: &str) -> Self { - if database_url.starts_with("postgres://") || database_url.starts_with("postgresql://") { - Self::Postgres - } else { - Self::Sqlite - } - } -} - -#[derive(Clone, Debug, Default)] -pub enum HistoricalNamespace { - #[default] - Default, - Custom(String), -} - -impl HistoricalNamespace { - fn sqlite_prefix(&self) -> &str { - match self { - Self::Default => "", - Self::Custom(prefix) => prefix, - } - } - - fn postgres_schema(&self) -> &str { - match self { - Self::Default => "public", - Self::Custom(schema) => schema, - } - } -} - -impl From<()> for HistoricalNamespace { - fn from((): ()) -> Self { - Self::Default - } -} - -impl From for HistoricalNamespace { - fn from(value: String) -> Self { - if value.is_empty() { - Self::Default - } else { - Self::Custom(value) - } - } -} - -impl From<&str> for HistoricalNamespace { - fn from(value: &str) -> Self { - if value.is_empty() { - Self::Default - } else { - Self::Custom(value.to_string()) - } - } -} - -#[derive(Clone, Debug)] -struct HistoryColumn { - name: String, - type_sql: String, -} - -#[derive(Clone, Debug)] -struct TrackedTable { - logical_name: String, - base_name: String, - history_name: String, - columns: Vec, - sqlite_queries: Option, -} - -#[derive(Clone, Debug)] -struct TrackedTableSqliteQueries { - next_revision: String, - copy_current_row: String, - insert_tombstone: String, -} - -#[derive(Clone)] -pub struct EntitiesHistoricalSink { - pool: Pool, - backend: DbBackend, - namespace: HistoricalNamespace, - tracked_names: HashSet, - tracked_tables: Arc>>, -} - -impl EntitiesHistoricalSink { - pub async fn new( - database_url: &str, - max_connections: Option, - namespace: impl Into, - tracked_models: Vec, - ) -> Result { - sqlx::any::install_default_drivers(); - - let backend = DbBackend::detect(database_url); - let database_url = match backend { - DbBackend::Postgres => database_url.to_string(), - DbBackend::Sqlite => sqlite_url(database_url)?, - }; - let pool = AnyPoolOptions::new() - .max_connections(max_connections.unwrap_or(if backend == DbBackend::Sqlite { - DEFAULT_SQLITE_MAX_CONNECTIONS - } else { - 5 - })) - .connect(&database_url) - .await?; - - Ok(Self { - pool, - backend, - namespace: namespace.into(), - tracked_names: tracked_models - .into_iter() - .filter(|name| !name.is_empty()) - .collect(), - tracked_tables: Arc::new(RwLock::new(HashMap::new())), - }) - } - - async fn bootstrap(&self) -> Result<()> { - if self.tracked_names.is_empty() { - return Ok(()); - } - - if self.backend == DbBackend::Sqlite { - sqlx::query("PRAGMA journal_mode=WAL") - .execute(&self.pool) - .await - .ok(); - } - - match self.backend { - DbBackend::Sqlite => { - let rows = sqlx::query( - "SELECT table_schema_json - FROM introspect_sink_schema_state - WHERE alive != 0", - ) - .fetch_all(&self.pool) - .await?; - for row in rows { - let schema_json: String = row.try_get("table_schema_json")?; - let schema: TableSchema = serde_json::from_str(&schema_json)?; - if self.tracked_names.contains(&schema.name) { - self.sync_tracked_table(schema.id, &schema.name).await?; - } - } - } - DbBackend::Postgres => { - let rows = sqlx::query( - "SELECT id, name - FROM introspect.db_tables - WHERE \"schema\" = $1", - ) - .bind(self.namespace.postgres_schema()) - .fetch_all(&self.pool) - .await?; - for row in rows { - let table_name: String = row.try_get("name")?; - if !self.tracked_names.contains(&table_name) { - continue; - } - let table_id = felt_from_row_hex(&row, "id")?; - self.sync_tracked_table(table_id, &table_name).await?; - } - } - } - - Ok(()) - } - - async fn resolve_tracked_table(&self, table_id: Felt) -> Result> { - let tracked = { - let tracked_tables = self.tracked_tables.read().await; - tracked_tables.get(&table_id).cloned() - }; - - if let Some(table) = tracked { - return Ok(Some(table)); - } - - let resolved = self.lookup_table_name(table_id).await?; - let Some(table_name) = resolved else { - return Ok(None); - }; - if !self.tracked_names.contains(&table_name) { - return Ok(None); - } - - let tracked = self.sync_tracked_table(table_id, &table_name).await?; - Ok(Some(tracked)) - } - - async fn lookup_table_name(&self, table_id: Felt) -> Result> { - let (canonical_table_id, compact_table_id) = felt_hex_variants(table_id); - match self.backend { - DbBackend::Sqlite => { - let row = sqlx::query( - "SELECT table_schema_json - FROM introspect_sink_schema_state - WHERE table_id = ?1 OR table_id = ?2 - LIMIT 1", - ) - .bind(canonical_table_id) - .bind(compact_table_id) - .fetch_optional(&self.pool) - .await?; - row.map(|row| { - let schema_json: String = row.try_get("table_schema_json")?; - let schema: TableSchema = serde_json::from_str(&schema_json)?; - Ok::<_, anyhow::Error>(schema.name) - }) - .transpose() - } - DbBackend::Postgres => { - let row = sqlx::query( - "SELECT name - FROM introspect.db_tables - WHERE \"schema\" = $1 AND (id::text = $2 OR id::text = $3) - LIMIT 1", - ) - .bind(self.namespace.postgres_schema()) - .bind(canonical_table_id) - .bind(compact_table_id) - .fetch_optional(&self.pool) - .await?; - row.map(|row| row.try_get("name").map_err(Into::into)) - .transpose() - } - } - } - - async fn sync_tracked_table(&self, table_id: Felt, logical_name: &str) -> Result { - let base_name = match self.backend { - DbBackend::Sqlite => sqlite_storage_name(self.namespace.sqlite_prefix(), logical_name), - DbBackend::Postgres => logical_name.to_string(), - }; - let history_name = match self.backend { - DbBackend::Sqlite => sqlite_storage_name( - self.namespace.sqlite_prefix(), - &format!("{logical_name}_historical"), - ), - DbBackend::Postgres => format!("{logical_name}_historical"), - }; - let columns = self.load_source_columns(&base_name).await?; - if !columns.iter().any(|column| column.name == "entity_id") { - return Err(anyhow!( - "tracked table '{logical_name}' does not expose entity_id column" - )); - } - let sqlite_queries = (self.backend == DbBackend::Sqlite) - .then(|| build_tracked_table_sqlite_queries(&base_name, &history_name, &columns)); - let tracked = TrackedTable { - logical_name: logical_name.to_string(), - base_name, - history_name, - sqlite_queries, - columns, - }; - self.ensure_history_table(&tracked).await?; - self.tracked_tables - .write() - .await - .insert(table_id, tracked.clone()); - Ok(tracked) - } - - async fn load_source_columns(&self, base_name: &str) -> Result> { - match self.backend { - DbBackend::Sqlite => { - // sqlite-dynamic-ok: PRAGMA table_info requires the table identifier in SQL text. - let rows = sqlx::query(&format!( - "PRAGMA table_info({})", - quote_sqlite_identifier(base_name) - )) - .fetch_all(&self.pool) - .await?; - let mut columns = Vec::with_capacity(rows.len()); - for row in rows { - let name: String = row.try_get("name")?; - let type_sql: String = row.try_get("type")?; - columns.push(HistoryColumn { name, type_sql }); - } - Ok(columns) - } - DbBackend::Postgres => { - let rows = sqlx::query( - "SELECT a.attname AS column_name, - pg_catalog.format_type(a.atttypid, a.atttypmod) AS column_type - FROM pg_catalog.pg_attribute a - JOIN pg_catalog.pg_class c ON c.oid = a.attrelid - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 - AND c.relname = $2 - AND a.attnum > 0 - AND NOT a.attisdropped - ORDER BY a.attnum", - ) - .bind(self.namespace.postgres_schema()) - .bind(base_name) - .fetch_all(&self.pool) - .await?; - - let mut columns = Vec::new(); - for row in rows { - let name: String = row.try_get("column_name")?; - if name.starts_with("__") { - continue; - } - let type_sql: String = row.try_get("column_type")?; - columns.push(HistoryColumn { name, type_sql }); - } - Ok(columns) - } - } - } - - async fn ensure_history_table(&self, tracked: &TrackedTable) -> Result<()> { - let create_sql = match self.backend { - DbBackend::Sqlite => self.create_history_table_sqlite(tracked), - DbBackend::Postgres => self.create_history_table_postgres(tracked), - }; - sqlx::query(&create_sql).execute(&self.pool).await?; - - let existing_columns = self.load_existing_history_columns(tracked).await?; - for column in &tracked.columns { - if existing_columns.contains(&column.name) { - continue; - } - let add_sql = match self.backend { - DbBackend::Sqlite => format!( - "ALTER TABLE {} ADD COLUMN {} {}", - quote_sqlite_identifier(&tracked.history_name), - quote_ident(&column.name), - column.type_sql - ), - DbBackend::Postgres => format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS {} {}", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name), - quote_ident(&column.name), - column.type_sql - ), - }; - sqlx::query(&add_sql).execute(&self.pool).await?; - } - - for meta_sql in self.ensure_history_meta_columns_sql(tracked, &existing_columns) { - sqlx::query(&meta_sql).execute(&self.pool).await?; - } - - for index_sql in self.ensure_history_indexes_sql(tracked) { - sqlx::query(&index_sql).execute(&self.pool).await?; - } - - Ok(()) - } - - async fn load_existing_history_columns( - &self, - tracked: &TrackedTable, - ) -> Result> { - let rows = match self.backend { - DbBackend::Sqlite => { - // sqlite-dynamic-ok: PRAGMA table_info requires the table identifier in SQL text. - sqlx::query(&format!( - "PRAGMA table_info({})", - quote_sqlite_identifier(&tracked.history_name) - )) - .fetch_all(&self.pool) - .await? - } - DbBackend::Postgres => { - sqlx::query( - "SELECT a.attname AS column_name - FROM pg_catalog.pg_attribute a - JOIN pg_catalog.pg_class c ON c.oid = a.attrelid - JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = $1 - AND c.relname = $2 - AND a.attnum > 0 - AND NOT a.attisdropped", - ) - .bind(self.namespace.postgres_schema()) - .bind(&tracked.history_name) - .fetch_all(&self.pool) - .await? - } - }; - let mut columns = HashSet::with_capacity(rows.len()); - for row in rows { - let name = if self.backend == DbBackend::Sqlite { - row.try_get("name")? - } else { - row.try_get("column_name")? - }; - columns.insert(name); - } - Ok(columns) - } - - fn create_history_table_sqlite(&self, tracked: &TrackedTable) -> String { - let model_columns = tracked - .columns - .iter() - .map(|column| { - if column.name == "entity_id" { - format!("{} {} NOT NULL", quote_ident(&column.name), column.type_sql) - } else { - format!("{} {}", quote_ident(&column.name), column.type_sql) - } - }) - .collect::>() - .join(", "); - format!( - "CREATE TABLE IF NOT EXISTS {} ({model_columns}, \ - \"revision\" INTEGER NOT NULL DEFAULT 0, \ - \"historical_deleted\" INTEGER NOT NULL DEFAULT 0, \ - \"historical_block_number\" INTEGER, \ - \"historical_tx_hash\" TEXT NOT NULL DEFAULT '', \ - \"historical_executed_at\" INTEGER NOT NULL DEFAULT 0)", - quote_sqlite_identifier(&tracked.history_name) - ) - } - - fn create_history_table_postgres(&self, tracked: &TrackedTable) -> String { - let model_columns = tracked - .columns - .iter() - .map(|column| { - if column.name == "entity_id" { - format!("{} {} NOT NULL", quote_ident(&column.name), column.type_sql) - } else { - format!("{} {}", quote_ident(&column.name), column.type_sql) - } - }) - .collect::>() - .join(", "); - format!( - "CREATE TABLE IF NOT EXISTS {} ({model_columns}, \ - \"revision\" BIGINT NOT NULL DEFAULT 0, \ - \"historical_deleted\" BOOLEAN NOT NULL DEFAULT FALSE, \ - \"historical_block_number\" BIGINT, \ - \"historical_tx_hash\" TEXT NOT NULL DEFAULT '', \ - \"historical_executed_at\" BIGINT NOT NULL DEFAULT 0)", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ) - } - - fn ensure_history_meta_columns_sql( - &self, - tracked: &TrackedTable, - existing_columns: &HashSet, - ) -> Vec { - match self.backend { - DbBackend::Sqlite => { - let mut sql = Vec::new(); - let target = quote_sqlite_identifier(&tracked.history_name); - if !existing_columns.contains("revision") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"revision\" INTEGER NOT NULL DEFAULT 0" - )); - } - if !existing_columns.contains("historical_deleted") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_deleted\" INTEGER NOT NULL DEFAULT 0" - )); - } - if !existing_columns.contains("historical_block_number") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_block_number\" INTEGER" - )); - } - if !existing_columns.contains("historical_tx_hash") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_tx_hash\" TEXT NOT NULL DEFAULT ''" - )); - } - if !existing_columns.contains("historical_executed_at") { - sql.push(format!( - "ALTER TABLE {target} ADD COLUMN \"historical_executed_at\" INTEGER NOT NULL DEFAULT 0" - )); - } - sql - } - DbBackend::Postgres => vec![ - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"revision\" BIGINT NOT NULL DEFAULT 0", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_deleted\" BOOLEAN NOT NULL DEFAULT FALSE", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_block_number\" BIGINT", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_tx_hash\" TEXT NOT NULL DEFAULT ''", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "ALTER TABLE {} ADD COLUMN IF NOT EXISTS \"historical_executed_at\" BIGINT NOT NULL DEFAULT 0", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - ], - } - } - - fn ensure_history_indexes_sql(&self, tracked: &TrackedTable) -> Vec { - let unique_index = format!("{}_entity_revision_idx", tracked.history_name); - let entity_index = format!("{}_entity_idx", tracked.history_name); - match self.backend { - DbBackend::Sqlite => vec![ - format!( - "CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} (\"entity_id\", \"revision\")", - quote_ident(&unique_index), - quote_sqlite_identifier(&tracked.history_name) - ), - format!( - "CREATE INDEX IF NOT EXISTS {} ON {} (\"entity_id\")", - quote_ident(&entity_index), - quote_sqlite_identifier(&tracked.history_name) - ), - ], - DbBackend::Postgres => vec![ - format!( - "CREATE UNIQUE INDEX IF NOT EXISTS {} ON {} (\"entity_id\", \"revision\")", - quote_ident(&unique_index), - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - format!( - "CREATE INDEX IF NOT EXISTS {} ON {} (\"entity_id\")", - quote_ident(&entity_index), - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - ], - } - } - - async fn next_revision( - &self, - tracked: &TrackedTable, - canonical_entity_id_hex: &str, - compact_entity_id_hex: &str, - ) -> Result { - let sql = match self.backend { - DbBackend::Sqlite => tracked - .sqlite_queries - .as_ref() - .expect("sqlite queries available for sqlite backend") - .next_revision - .clone(), - DbBackend::Postgres => format!( - "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ - FROM {} WHERE \"entity_id\"::text = $1 OR \"entity_id\"::text = $2", - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name) - ), - }; - let row = sqlx::query(&sql) - .bind(canonical_entity_id_hex) - .bind(compact_entity_id_hex) - .fetch_one(&self.pool) - .await?; - Ok(row.try_get::("next_revision")?) - } - - async fn persist_snapshot( - &self, - tracked: &TrackedTable, - entity_id: Felt, - block_number: Option, - tx_hash: Felt, - executed_at: u64, - deleted: bool, - ) -> Result<()> { - let (canonical_entity_id_hex, compact_entity_id_hex) = felt_hex_variants(entity_id); - let revision = self - .next_revision(tracked, &canonical_entity_id_hex, &compact_entity_id_hex) - .await?; - let copied = self - .copy_current_row_into_history( - tracked, - &canonical_entity_id_hex, - &compact_entity_id_hex, - revision, - block_number, - tx_hash, - executed_at, - deleted, - ) - .await?; - if copied == 0 { - if deleted { - self.insert_tombstone_only( - tracked, - &canonical_entity_id_hex, - revision, - block_number, - tx_hash, - executed_at, - ) - .await?; - return Ok(()); - } - return Err(anyhow!( - "unable to load latest row for tracked model '{}' entity {}", - tracked.logical_name, - canonical_entity_id_hex - )); - } - Ok(()) - } - - async fn copy_current_row_into_history( - &self, - tracked: &TrackedTable, - canonical_entity_id_hex: &str, - compact_entity_id_hex: &str, - revision: i64, - block_number: Option, - tx_hash: Felt, - executed_at: u64, - deleted: bool, - ) -> Result { - let sql = match self.backend { - DbBackend::Sqlite => tracked - .sqlite_queries - .as_ref() - .expect("sqlite queries available for sqlite backend") - .copy_current_row - .clone(), - DbBackend::Postgres => { - let source_columns = tracked - .columns - .iter() - .map(|column| quote_ident(&column.name)) - .collect::>() - .join(", "); - let insert_columns = format!( - "{source_columns}, \"revision\", \"historical_deleted\", \ - \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" - ); - let history_target = - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name); - let source_target = - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.base_name); - format!( - "INSERT INTO {history_target} ({insert_columns}) \ - SELECT {source_columns}, $1, $2, $3, $4, $5 \ - FROM {source_target} WHERE (\"entity_id\"::text = $6 OR \"entity_id\"::text = $7) LIMIT 1" - ) - } - }; - let mut query = sqlx::query(&sql).bind(revision); - query = match self.backend { - DbBackend::Sqlite => query.bind(i64::from(deleted)), - DbBackend::Postgres => query.bind(deleted), - }; - let result = query - .bind(block_number.map(|value| value as i64)) - .bind(felt_hex(tx_hash)) - .bind(executed_at as i64) - .bind(canonical_entity_id_hex) - .bind(compact_entity_id_hex) - .execute(&self.pool) - .await?; - Ok(result.rows_affected()) - } - - async fn insert_tombstone_only( - &self, - tracked: &TrackedTable, - entity_id_hex: &str, - revision: i64, - block_number: Option, - tx_hash: Felt, - executed_at: u64, - ) -> Result<()> { - let sql = match self.backend { - DbBackend::Sqlite => tracked - .sqlite_queries - .as_ref() - .expect("sqlite queries available for sqlite backend") - .insert_tombstone - .clone(), - DbBackend::Postgres => { - let mut columns = Vec::with_capacity(tracked.columns.len() + 5); - let mut values_sql = Vec::with_capacity(tracked.columns.len() + 5); - - let mut bind_index = 1_usize; - for column in &tracked.columns { - columns.push(quote_ident(&column.name)); - if column.name == "entity_id" { - values_sql.push(self.entity_cast_placeholder(bind_index, &column.type_sql)); - bind_index += 1; - } else { - values_sql.push("NULL".to_string()); - } - } - - columns.extend([ - "\"revision\"".to_string(), - "\"historical_deleted\"".to_string(), - "\"historical_block_number\"".to_string(), - "\"historical_tx_hash\"".to_string(), - "\"historical_executed_at\"".to_string(), - ]); - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - bind_index += 1; - values_sql.push(self.value_placeholder(bind_index)); - - let target = - quote_pg_qualified(self.namespace.postgres_schema(), &tracked.history_name); - format!( - "INSERT INTO {target} ({}) VALUES ({})", - columns.join(", "), - values_sql.join(", ") - ) - } - }; - - let mut query = sqlx::query(&sql).bind(entity_id_hex).bind(revision); - query = match self.backend { - DbBackend::Sqlite => query.bind(1_i64), - DbBackend::Postgres => query.bind(true), - }; - query - .bind(block_number.map(|value| value as i64)) - .bind(felt_hex(tx_hash)) - .bind(executed_at as i64) - .execute(&self.pool) - .await?; - - Ok(()) - } - - fn value_placeholder(&self, index: usize) -> String { - match self.backend { - DbBackend::Sqlite => "?".to_string(), - DbBackend::Postgres => format!("${index}"), - } - } - - fn entity_cast_placeholder(&self, index: usize, type_sql: &str) -> String { - match self.backend { - DbBackend::Sqlite => "?".to_string(), - DbBackend::Postgres => format!("CAST(${index} AS {type_sql})"), - } - } - - async fn process_table_schema(&self, table: &CreateTable) -> Result<()> { - if self.tracked_names.contains(&table.name) { - self.sync_tracked_table(table.id, &table.name).await?; - } - Ok(()) - } - - async fn process_table_update(&self, table: &UpdateTable) -> Result<()> { - if self.tracked_names.contains(&table.name) { - self.sync_tracked_table(table.id, &table.name).await?; - } - Ok(()) - } -} - -#[async_trait] -impl Sink for EntitiesHistoricalSink { - fn name(&self) -> &'static str { - "entities-historical" - } - - fn interested_types(&self) -> Vec { - vec![INTROSPECT_TYPE] - } - - async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> Result<()> { - for envelope in envelopes { - if envelope.type_id != INTROSPECT_TYPE { - continue; - } - let Some(body) = envelope.downcast_ref::() else { - continue; - }; - let context = batch - .get_event_context(&body.metadata.transaction_hash, body.metadata.from_address) - .unwrap_or_default(); - let block_number = body - .metadata - .block_number - .or(Some(context.transaction.block_number)); - let executed_at = context.block.timestamp; - - match &body.msg { - IntrospectMsg::CreateTable(table) => { - self.process_table_schema(table).await?; - } - IntrospectMsg::UpdateTable(table) => { - self.process_table_update(table).await?; - } - IntrospectMsg::InsertsFields(insert) => { - let Some(tracked) = self.resolve_tracked_table(insert.table).await? else { - continue; - }; - for record in &insert.records { - self.persist_snapshot( - &tracked, - Felt::from_bytes_be(&record.id), - block_number, - body.metadata.transaction_hash, - executed_at, - false, - ) - .await - .with_context(|| { - format!( - "failed to persist historical snapshot for '{}' insert", - tracked.logical_name - ) - })?; - } - } - IntrospectMsg::DeleteRecords(delete) => { - let Some(tracked) = self.resolve_tracked_table(delete.table).await? else { - continue; - }; - for row in &delete.rows { - self.persist_snapshot( - &tracked, - row.to_felt(), - block_number, - body.metadata.transaction_hash, - executed_at, - true, - ) - .await - .with_context(|| { - format!( - "failed to persist historical tombstone for '{}'", - tracked.logical_name - ) - })?; - } - } - _ => {} - } - } - - Ok(()) - } - - fn topics(&self) -> Vec { - Vec::new() - } - - fn build_routes(&self) -> Router { - Router::new() - } - - async fn initialize( - &mut self, - _event_bus: Arc, - _context: &SinkContext, - ) -> Result<()> { - self.bootstrap().await?; - Ok(()) - } -} - -fn sqlite_storage_name(prefix: &str, logical_name: &str) -> String { - if prefix.is_empty() { - logical_name.to_string() - } else { - format!("{prefix}__{logical_name}") - } -} - -fn sqlite_url(database_url: &str) -> Result { - if database_url == ":memory:" { - return Ok("sqlite::memory:".to_string()); - } - if database_url.starts_with("sqlite:") { - return Ok(database_url.to_string()); - } - Ok(format!("sqlite://{database_url}")) -} - -fn quote_ident(identifier: &str) -> String { - format!("\"{}\"", identifier.replace('"', "\"\"")) -} - -fn quote_sqlite_identifier(table: &str) -> String { - quote_ident(table) -} - -fn quote_pg_qualified(schema: &str, table: &str) -> String { - format!("{}.{}", quote_ident(schema), quote_ident(table)) -} - -fn build_tracked_table_sqlite_queries( - base_name: &str, - history_name: &str, - columns: &[HistoryColumn], -) -> TrackedTableSqliteQueries { - let history_target = quote_sqlite_identifier(history_name); - let source_target = quote_sqlite_identifier(base_name); - let source_columns = columns - .iter() - .map(|column| quote_ident(&column.name)) - .collect::>() - .join(", "); - let insert_columns = format!( - "{source_columns}, \"revision\", \"historical_deleted\", \ - \"historical_block_number\", \"historical_tx_hash\", \"historical_executed_at\"" - ); - - let mut tombstone_columns = Vec::with_capacity(columns.len() + 5); - let mut tombstone_values = Vec::with_capacity(columns.len() + 5); - for column in columns { - tombstone_columns.push(quote_ident(&column.name)); - if column.name == "entity_id" { - tombstone_values.push("?".to_string()); - } else { - tombstone_values.push("NULL".to_string()); - } - } - tombstone_columns.extend([ - "\"revision\"".to_string(), - "\"historical_deleted\"".to_string(), - "\"historical_block_number\"".to_string(), - "\"historical_tx_hash\"".to_string(), - "\"historical_executed_at\"".to_string(), - ]); - tombstone_values.extend([ - "?".to_string(), - "?".to_string(), - "?".to_string(), - "?".to_string(), - "?".to_string(), - ]); - - TrackedTableSqliteQueries { - next_revision: format!( - "SELECT COALESCE(MAX(\"revision\"), 0) + 1 AS next_revision \ - FROM {history_target} WHERE \"entity_id\" = ?1 OR \"entity_id\" = ?2" - ), - copy_current_row: format!( - "INSERT INTO {history_target} ({insert_columns}) \ - SELECT {source_columns}, ?1, ?2, ?3, ?4, ?5 \ - FROM {source_target} \ - WHERE (\"entity_id\" = ?6 OR \"entity_id\" = ?7) \ - LIMIT 1" - ), - insert_tombstone: format!( - "INSERT INTO {history_target} ({}) VALUES ({})", - tombstone_columns.join(", "), - tombstone_values.join(", ") - ), - } -} - -fn felt_hex(value: Felt) -> String { - format!("{value:#x}") -} - -fn canonical_felt_hex(value: Felt) -> String { - format!("{value:#066x}") -} - -fn compact_hex_str(value: &str) -> String { - let value = value.trim(); - let Some(hex) = value.strip_prefix("0x") else { - return value.to_string(); - }; - let trimmed = hex.trim_start_matches('0'); - if trimmed.is_empty() { - "0x0".to_string() - } else { - format!("0x{trimmed}") - } -} - -fn felt_hex_variants(value: Felt) -> (String, String) { - let canonical = canonical_felt_hex(value); - let compact = compact_hex_str(&canonical); - (canonical, compact) -} - -fn felt_from_row_hex(row: &sqlx::any::AnyRow, column: &str) -> Result { - let value: String = row.try_get(column)?; - Felt::from_hex(&value).map_err(Into::into) -} - -#[cfg(test)] -mod tests { - use super::*; - use introspect_types::{ColumnDef, PrimaryDef, PrimaryTypeDef, TypeDef}; - use torii::etl::envelope::{EventBody, MetaData}; - use torii::etl::extractor::ExtractionBatch; - use torii_introspect::events::{DeleteRecords, InsertsFields, IntrospectMsg, Record}; - - fn sqlite_table_schema(name: &str, id: Felt) -> TableSchema { - TableSchema { - id, - name: name.to_string(), - attributes: Vec::new(), - primary: PrimaryDef { - name: "entity_id".to_string(), - attributes: Vec::new(), - type_def: PrimaryTypeDef::Felt252, - }, - columns: vec![ - ColumnDef { - id: Felt::from(1_u8), - name: "name".to_string(), - attributes: Vec::new(), - type_def: TypeDef::ByteArray, - }, - ColumnDef { - id: Felt::from(2_u8), - name: "score".to_string(), - attributes: Vec::new(), - type_def: TypeDef::U32, - }, - ], - } - } - - async fn sqlite_sink() -> Result { - let sink = EntitiesHistoricalSink::new( - "sqlite::memory:", - Some(1), - (), - vec!["NUMS-Game".to_string()], - ) - .await?; - sqlx::query( - "CREATE TABLE introspect_sink_schema_state ( - table_id TEXT PRIMARY KEY, - table_schema_json TEXT NOT NULL, - alive INTEGER NOT NULL DEFAULT 1, - updated_at INTEGER NOT NULL DEFAULT (unixepoch()) - )", - ) - .execute(&sink.pool) - .await?; - sqlx::query( - "CREATE TABLE \"NUMS-Game\" ( - \"entity_id\" TEXT PRIMARY KEY, - \"name\" TEXT, - \"score\" INTEGER - )", - ) - .execute(&sink.pool) - .await?; - let schema = sqlite_table_schema("NUMS-Game", Felt::from(9_u8)); - sqlx::query( - "INSERT INTO introspect_sink_schema_state (table_id, table_schema_json, alive, updated_at) - VALUES (?1, ?2, 1, unixepoch())", - ) - .bind(canonical_felt_hex(schema.id)) - .bind(serde_json::to_string(&schema)?) - .execute(&sink.pool) - .await?; - Ok(sink) - } - - #[tokio::test] - async fn initializes_history_table_from_sqlite_schema_state() -> Result<()> { - let sink = sqlite_sink().await?; - sink.bootstrap().await?; - - let row = sqlx::query( - "SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'NUMS-Game_historical'", - ) - .fetch_one(&sink.pool) - .await?; - let name: String = row.try_get("name")?; - assert_eq!(name, "NUMS-Game_historical"); - Ok(()) - } - - #[tokio::test] - async fn appends_revisions_and_delete_tombstones_in_sqlite() -> Result<()> { - let sink = sqlite_sink().await?; - sink.bootstrap().await?; - - let entity_id = Felt::from(77_u8); - sqlx::query( - "INSERT INTO \"NUMS-Game\" (\"entity_id\", \"name\", \"score\") VALUES (?1, ?2, ?3)", - ) - .bind(canonical_felt_hex(entity_id)) - .bind("first") - .bind(10_i64) - .execute(&sink.pool) - .await?; - - let insert = EventBody { - metadata: MetaData { - block_number: Some(1), - transaction_hash: Felt::from(100_u16), - from_address: Felt::ZERO, - }, - msg: IntrospectMsg::InsertsFields(InsertsFields::new( - Felt::from(9_u8), - Vec::new(), - vec![Record::new(entity_id, Vec::new())], - )), - }; - sink.process(&[Envelope::from(insert)], &ExtractionBatch::empty()) - .await?; - - sqlx::query("UPDATE \"NUMS-Game\" SET \"score\" = 25 WHERE \"entity_id\" = ?1") - .bind(canonical_felt_hex(entity_id)) - .execute(&sink.pool) - .await?; - let update = EventBody { - metadata: MetaData { - block_number: Some(2), - transaction_hash: Felt::from(101_u16), - from_address: Felt::ZERO, - }, - msg: IntrospectMsg::InsertsFields(InsertsFields::new( - Felt::from(9_u8), - Vec::new(), - vec![Record::new(entity_id, Vec::new())], - )), - }; - sink.process(&[Envelope::from(update)], &ExtractionBatch::empty()) - .await?; - - let delete = EventBody { - metadata: MetaData { - block_number: Some(3), - transaction_hash: Felt::from(102_u16), - from_address: Felt::ZERO, - }, - msg: IntrospectMsg::DeleteRecords(DeleteRecords::new( - Felt::from(9_u8), - vec![entity_id.into()], - )), - }; - sink.process(&[Envelope::from(delete)], &ExtractionBatch::empty()) - .await?; - - let rows = sqlx::query( - "SELECT revision, historical_deleted, score FROM \"NUMS-Game_historical\" ORDER BY revision", - ) - .fetch_all(&sink.pool) - .await?; - - assert_eq!(rows.len(), 3); - assert_eq!(rows[0].try_get::("revision")?, 1); - assert_eq!(rows[0].try_get::("historical_deleted")?, 0); - assert_eq!(rows[0].try_get::("score")?, 10); - assert_eq!(rows[1].try_get::("revision")?, 2); - assert_eq!(rows[1].try_get::("score")?, 25); - assert_eq!(rows[2].try_get::("revision")?, 3); - assert_eq!(rows[2].try_get::("historical_deleted")?, 1); - assert_eq!(rows[2].try_get::("score")?, 25); - Ok(()) - } -} diff --git a/crates/torii-erc1155/Cargo.toml b/crates/torii-erc1155/Cargo.toml index 961ca495..0d660758 100644 --- a/crates/torii-erc1155/Cargo.toml +++ b/crates/torii-erc1155/Cargo.toml @@ -8,6 +8,8 @@ description = "ERC1155 semi-fungible token indexer library for Torii" # Torii core torii = { path = "../.." } torii-common = { path = "../torii-common" } +torii-types = { path = "../types" } +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } # Async tokio = { version = "1", features = ["full"] } @@ -15,6 +17,7 @@ async-trait = "0.1" # Starknet starknet = "0.17" +starknet-types-raw.workspace = true # Storage rusqlite = { version = "0.32", features = ["bundled"] } @@ -26,6 +29,7 @@ serde_json = "1.0" # Encoding hex = "0.4" +primitive-types.workspace = true # gRPC/Protobuf prost = "0.13" @@ -43,6 +47,7 @@ axum = "0.7" # Utilities anyhow = "1.0" tracing = "0.1" +thiserror.workspace = true [build-dependencies] tonic-build = "0.12" diff --git a/crates/torii-erc1155/README.md b/crates/torii-erc1155/README.md new file mode 100644 index 00000000..4ba7d080 --- /dev/null +++ b/crates/torii-erc1155/README.md @@ -0,0 +1,130 @@ +# torii-erc1155 + +End-to-end **ERC1155 semi-fungible token indexer**. Decodes TransferSingle, +TransferBatch, ApprovalForAll, and URI events; stores transfer history, +operator approvals, and URI updates; serves a gRPC query/subscribe API. +**Balance tracking is intentionally not authoritative** — the sink writes +balance deltas but clients should query the chain for authoritative state +(see note below). + +## Role in Torii + +Same shape as `torii-erc20` / `torii-erc721` but for ERC1155. The +`Erc1155Rule` auto-identifier keys off `safeTransferFrom` + `balanceOf` + +`TransferSingle`. The sink emits three EventBus topics: `erc1155.transfer`, +`erc1155.metadata`, `erc1155.uri`. + +## Architecture + +```text +StarknetEvent (ERC1155 selector) + | + v ++-------------------------------+ +| Erc1155Decoder | identification::Erc1155Rule +| TransferSingle | Batch | | (safeTransferFrom + balanceOf + +| ApprovalForAll | URI | TransferSingle) +| → Envelope(ERC1155_TYPE_ID) | ++---------------+---------------+ + | + v ++-------------------------------+ +| Erc1155Sink | +| | +| process(): | +| - persist transfers | +| (batch expanded to N rows| +| one per (id,amount)) | +| - update operator approval | +| - URI update → enqueue | +| Erc1155TokenUriCommand | +| - balance delta tracked | +| in erc1155_balance | +| (reconciled on negative | +| via Erc1155BalanceFetcher +| when enabled) | +| - broadcast only when | +| batch.is_live(100) | ++---------------+---------------+ + | + v ++-------------------------------+ +| Erc1155Storage | +| erc1155_transfer (cursor) | +| erc1155_operator_approval | +| erc1155_uri | +| erc1155_balance | +| erc1155_balance_adjustment | +| erc1155_metadata | +| erc1155_token_uri | ++---------------+---------------+ + | + v ++-------------------------------+ +| Erc1155Service | +| gRPC: torii.sinks.erc1155 | ++-------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Purpose | +|---|---|---| +| `Erc1155Decoder`, `Erc1155Message`, `Erc1155Body`, `ERC1155_TYPE_ID` | `src/decoder.rs` | Decoder + event enum + wrapper body | +| `TransferData`, `OperatorApproval`, `UriUpdate` | `src/decoder.rs` | Typed event payloads | +| `Erc1155Rule` | `src/identification.rs` | ABI-based identification | +| `Erc1155Sink` | `src/sink.rs` | `Sink` impl | +| `Erc1155Storage`, `TokenTransferData`, `TokenUriData`, `Erc1155BalanceData`, `Erc1155BalanceAdjustment`, `TransferCursor` | `src/storage.rs` | Persistence layer | +| `Erc1155Service` | `src/grpc_service.rs` | gRPC service | +| `Erc1155BalanceFetcher`, `Erc1155BalanceFetchRequest` | `src/balance_fetcher.rs` | On-chain balance reconciliation | +| `Erc1155MetadataCommandHandler`, `Erc1155TokenUriCommandHandler` | `src/handlers.rs` | Async command handlers | +| `SyntheticErc1155Config`, `SyntheticErc1155Extractor` | `src/synthetic.rs` | Deterministic generator | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs:55` | Descriptor bytes | + +### Internal Modules + +- `decoder` — matches ERC1155 selectors, **expands `TransferBatch` into N transfers** (one per `(id, amount)` pair) so downstream code treats single and batch identically. +- `identification` — requires `safeTransferFrom`, `balanceOf`, `TransferSingle`. +- `sink` — processes envelopes; dispatches metadata + token-URI commands. +- `storage` — transfer history + balance + URI tables. +- `balance_fetcher` — optional reconciliation path using `JsonRpcClient`. +- `handlers` — two `CommandHandler`s parallel to ERC721. +- `grpc_service` — pagination + broadcast channel. +- `synthetic` — `SyntheticErc1155Extractor`. + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `"erc1155"` | +| `interested_types` | `[ERC1155_TYPE_ID]` | +| `process` | Persists expanded transfers, updates operator-approval bitmap, stores URI updates, updates balance delta, broadcasts when `batch.is_live(100)` | +| `topics` | 3 topics: `erc1155.transfer` (filters: `token`, `from`, `to`, `wallet`), `erc1155.metadata` (filter: `token`), `erc1155.uri` (filters: `token`, `token_id`) | +| `build_routes` | `Router::new()` — gRPC only | +| `initialize` | Captures `event_bus` + `ctx.command_bus` | + +### Storage note + +ERC1155 maintains `erc1155_balance (token, account, token_id, amount)` and +an `erc1155_balance_adjustment` audit table, but the lib docs (`src/lib.rs:3`) state: + +> Like ERC20, we only track transfer history - NOT balances. Clients should +> query the chain for actual balances to avoid inaccuracies from genesis +> allocations. + +Treat the balance tables as a **best-effort cache** that the optional +`Erc1155BalanceFetcher` reconciles on divergence; authoritative balance +queries belong on-chain. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-tokens`, `bins/torii-tokens-synth`, `bins/torii-introspect-bin`, `bins/torii-arcade`. +- **Downstream deps**: `torii`, `torii-common`, `torii-types`, `sqlx`, `tonic`, `prost`, `starknet`, `starknet-types-raw`, `primitive-types`, `tokio`, `async-trait`, `chrono`, `hex`, `metrics`, `tracing`. + +### Extension Points + +- TransferBatch-level aggregation → add a `TokenBatchTransferData` alongside expanded rows. +- Additional filters → extend the `TopicInfo.available_filters` lists and the matching `matches_*_filters` helper in `sink.rs`. +- Flip balance tracking from best-effort to authoritative → wire a continuous reconciler through `Erc1155BalanceFetcher` + CommandBus; keep the note in `lib.rs` in sync. diff --git a/crates/torii-erc1155/src/balance_fetcher.rs b/crates/torii-erc1155/src/balance_fetcher.rs index 5fe7dbf8..c5b0d49e 100644 --- a/crates/torii-erc1155/src/balance_fetcher.rs +++ b/crates/torii-erc1155/src/balance_fetcher.rs @@ -6,11 +6,14 @@ //! ERC1155 has a different signature: balance_of(account, token_id) use anyhow::{Context, Result}; -use starknet::core::types::{requests::CallRequest, BlockId, Felt, FunctionCall, U256}; +use primitive_types::U256; +use starknet::core::types::{requests::CallRequest, BlockId, FunctionCall}; use starknet::macros::selector; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::sync::Arc; +use torii_common::utils::{felt_pair_to_u256, felt_to_u256}; /// Request for fetching an ERC1155 balance at a specific block #[derive(Debug, Clone)] @@ -54,15 +57,20 @@ impl Erc1155BalanceFetcher { let calldata = build_balance_of_calldata(wallet, token_id); let call = FunctionCall { - contract_address: contract, + contract_address: to_starknet_felt(contract), entry_point_selector: selector!("balance_of"), - calldata, + calldata: calldata.into_iter().map(to_starknet_felt).collect(), }; let block_id = BlockId::Number(block_number); match self.provider.call(call, block_id).await { - Ok(result) => Ok(parse_u256_result(&result)), + Ok(result) => Ok(parse_u256_result( + &result + .into_iter() + .map(from_starknet_felt) + .collect::>(), + )), Err(e) => { tracing::warn!( target: "torii_erc1155::balance_fetcher", @@ -98,9 +106,9 @@ impl Erc1155BalanceFetcher { let calldata = build_balance_of_calldata(req.wallet, req.token_id); ProviderRequestData::Call(CallRequest { request: FunctionCall { - contract_address: req.contract, + contract_address: to_starknet_felt(req.contract), entry_point_selector: selector!("balance_of"), - calldata, + calldata: calldata.into_iter().map(to_starknet_felt).collect(), }, block_id: BlockId::Number(req.block_number), }) @@ -119,7 +127,12 @@ impl Erc1155BalanceFetcher { for (idx, response) in responses.into_iter().enumerate() { let req = &requests[idx]; let balance = if let ProviderResponseData::Call(felts) = response { - parse_u256_result(&felts) + parse_u256_result( + &felts + .into_iter() + .map(from_starknet_felt) + .collect::>(), + ) } else { tracing::warn!( target: "torii_erc1155::balance_fetcher", @@ -147,9 +160,7 @@ impl Erc1155BalanceFetcher { /// Build calldata for balance_of(account, token_id) /// ERC1155 expects: [account, token_id_low, token_id_high] fn build_balance_of_calldata(wallet: Felt, token_id: U256) -> Vec { - // Convert U256 to two Felts (low, high) - let low = Felt::from(token_id.low()); - let high = Felt::from(token_id.high()); + let (low, high) = u256_low_high(token_id); vec![wallet, low, high] } @@ -161,23 +172,25 @@ fn build_balance_of_calldata(wallet: Felt, token_id: U256) -> Vec { fn parse_u256_result(result: &[Felt]) -> U256 { match result.len() { 0 => U256::from(0u64), - 1 => { - // Single felt - convert to U256 - let bytes = result[0].to_bytes_be(); - let low = u128::from_be_bytes(bytes[16..32].try_into().unwrap()); - U256::from_words(low, 0) - } - _ => { - // Two felts: [low, high] for u256 - let low_bytes = result[0].to_bytes_be(); - let high_bytes = result[1].to_bytes_be(); + 1 => felt_to_u256(result[0]), + _ => felt_pair_to_u256(result[0], result[1]), + } +} + +fn to_starknet_felt(value: Felt) -> starknet::core::types::Felt { + starknet::core::types::Felt::from_bytes_be(&value.to_be_bytes()) +} - let low = u128::from_be_bytes(low_bytes[16..32].try_into().unwrap()); - let high = u128::from_be_bytes(high_bytes[16..32].try_into().unwrap()); +fn from_starknet_felt(value: starknet::core::types::Felt) -> Felt { + Felt::from_be_bytes_slice(&value.to_bytes_be()).unwrap_or(Felt::ZERO) +} - U256::from_words(low, high) - } - } +fn u256_low_high(value: U256) -> (Felt, Felt) { + let [l0, l1, h0, h1] = value.0; + ( + Felt::from_le_words([l0, l1, 0, 0]), + Felt::from_le_words([h0, h1, 0, 0]), + ) } #[cfg(test)] diff --git a/crates/torii-erc1155/src/decoder.rs b/crates/torii-erc1155/src/decoder.rs index 22b7b5ea..0950ed38 100644 --- a/crates/torii-erc1155/src/decoder.rs +++ b/crates/torii-erc1155/src/decoder.rs @@ -2,77 +2,34 @@ use anyhow::Result; use async_trait::async_trait; +use primitive_types::U256; use starknet::core::codec::Decode; -use starknet::core::types::{ByteArray, EmittedEvent, Felt, U256}; +use starknet::core::types::ByteArray; use starknet::core::utils::parse_cairo_short_string; -use starknet::macros::selector; -use std::any::Any; -use std::collections::HashMap; -use torii::etl::{Decoder, Envelope, TypedBody}; -use torii_common::bytes_to_u256; - -/// TransferSingle event from ERC1155 token -#[derive(Debug, Clone)] -pub struct TransferSingle { - pub operator: Felt, - pub from: Felt, - pub to: Felt, - /// Token ID as U256 (256-bit) - pub id: U256, - /// Amount as U256 (256-bit) - pub value: U256, - pub token: Felt, - pub block_number: u64, - pub transaction_hash: Felt, -} - -impl TypedBody for TransferSingle { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc1155.transfer_single") - } +use starknet_types_raw::event::EmittedEvent; +use starknet_types_raw::Felt; +use torii::etl::envelope::EventMsg; +use torii::etl::{Decoder, Envelope, EventBody, TypeId}; +use torii_common::utils::{felt_pair_to_u256, felt_to_u256}; +use torii_types::event::EventContext; - fn as_any(&self) -> &dyn Any { - self - } +pub const ERC1155_TYPE_ID: TypeId = TypeId::new("erc1155"); - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - -/// TransferBatch event from ERC1155 token (denormalized into individual transfers) -#[derive(Debug, Clone)] -pub struct TransferBatch { +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TransferData { pub operator: Felt, pub from: Felt, pub to: Felt, - /// Token ID for this specific transfer in the batch - pub id: U256, - /// Amount for this specific transfer in the batch - pub value: U256, - /// Index in the original batch + pub token_id: U256, + pub amount: U256, + pub is_batch: bool, pub batch_index: u32, pub token: Felt, pub block_number: u64, pub transaction_hash: Felt, } -impl TypedBody for TransferBatch { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc1155.transfer_batch") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - -/// ApprovalForAll event from ERC1155 token -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct OperatorApproval { pub owner: Felt, pub operator: Felt, @@ -82,22 +39,7 @@ pub struct OperatorApproval { pub transaction_hash: Felt, } -impl TypedBody for OperatorApproval { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc1155.approval_for_all") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - -/// URI event from ERC1155 token -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct UriUpdate { pub token: Felt, pub token_id: U256, @@ -106,28 +48,66 @@ pub struct UriUpdate { pub transaction_hash: Felt, } -impl TypedBody for UriUpdate { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc1155.uri") - } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Erc1155Message { + Transfer(TransferData), + ApprovalForAll(OperatorApproval), + Uri(UriUpdate), +} - fn as_any(&self) -> &dyn Any { - self +pub type Erc1155Body = EventBody; + +impl EventMsg for Erc1155Message { + fn event_id(&self) -> String { + match self { + Self::Transfer(transfer) => { + if transfer.is_batch { + format!( + "erc1155_transfer_batch_{}_{}_{}", + transfer.block_number, + format!("{:#x}", transfer.transaction_hash), + transfer.batch_index + ) + } else { + format!( + "erc1155_transfer_single_{}_{}", + transfer.block_number, + format!("{:#x}", transfer.transaction_hash) + ) + } + } + Self::ApprovalForAll(approval) => { + format!( + "erc1155_approval_for_all_{}_{}", + approval.block_number, + format!("{:#x}", approval.transaction_hash) + ) + } + Self::Uri(uri) => format!( + "erc1155_uri_{}_{}", + uri.block_number, + format!("{:#x}", uri.transaction_hash) + ), + } } - fn as_any_mut(&mut self) -> &mut dyn Any { - self + fn envelope_type_id(&self) -> TypeId { + ERC1155_TYPE_ID } } -/// ERC1155 event decoder -/// -/// Decodes multiple ERC1155 events: -/// - TransferSingle(operator, from, to, id, value) -/// - TransferBatch(operator, from, to, ids, values) -/// - ApprovalForAll(owner, operator, approved) -/// -/// Supports both modern (keys) and legacy (data-only) formats. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct TransferPart { + operator: Felt, + from: Felt, + to: Felt, + token_id: U256, + amount: U256, + batch_index: u32, + is_batch: bool, +} + +/// ERC1155 event decoder. pub struct Erc1155Decoder; impl Erc1155Decoder { @@ -135,28 +115,20 @@ impl Erc1155Decoder { Self } - fn felt_to_u256(felt: Felt) -> U256 { - bytes_to_u256(&felt.to_bytes_be()) - } - - /// TransferSingle event selector: sn_keccak("TransferSingle") - fn transfer_single_selector() -> Felt { - selector!("TransferSingle") + pub fn transfer_single_selector() -> Felt { + Felt::selector("TransferSingle") } - /// TransferBatch event selector: sn_keccak("TransferBatch") - fn transfer_batch_selector() -> Felt { - selector!("TransferBatch") + pub fn transfer_batch_selector() -> Felt { + Felt::selector("TransferBatch") } - /// ApprovalForAll event selector: sn_keccak("ApprovalForAll") - fn approval_for_all_selector() -> Felt { - selector!("ApprovalForAll") + pub fn approval_for_all_selector() -> Felt { + Felt::selector("ApprovalForAll") } - /// URI event selector: sn_keccak("URI") - fn uri_selector() -> Felt { - selector!("URI") + pub fn uri_selector() -> Felt { + Felt::selector("URI") } fn decode_string_result(result: &[Felt]) -> Option { @@ -165,12 +137,17 @@ impl Erc1155Decoder { } if result.len() == 1 { - return parse_cairo_short_string(&result[0]) + return parse_cairo_short_string(&to_starknet_felt(result[0])) .ok() .filter(|s| !s.is_empty()); } - if let Ok(byte_array) = ByteArray::decode(result) { + let starknet_felts = result + .iter() + .copied() + .map(to_starknet_felt) + .collect::>(); + if let Ok(byte_array) = ByteArray::decode(&starknet_felts) { if let Ok(s) = String::try_from(byte_array) { if !s.is_empty() { return Some(s); @@ -178,11 +155,11 @@ impl Erc1155Decoder { } } - let len: usize = result[0].try_into().unwrap_or(0usize); + let len = felt_to_usize(result[0]); if len > 0 && len < 1000 && result.len() > len { let mut out = String::new(); for felt in &result[1..=len] { - if let Ok(chunk) = parse_cairo_short_string(felt) { + if let Ok(chunk) = parse_cairo_short_string(&to_starknet_felt(*felt)) { out.push_str(&chunk); } } @@ -194,190 +171,77 @@ impl Erc1155Decoder { None } - /// Decode TransferSingle event into envelope - /// - /// TransferSingle event signatures: - /// - /// Modern ERC1155: - /// - keys[0]: TransferSingle selector - /// - keys[1]: operator address - /// - keys[2]: from address - /// - keys[3]: to address - /// - data[0]: id_low (u128) - /// - data[1]: id_high (u128) - /// - data[2]: value_low (u128) - /// - data[3]: value_high (u128) - /// - /// Legacy ERC1155: - /// - keys[0]: TransferSingle selector - /// - data[0]: operator address - /// - data[1]: from address - /// - data[2]: to address - /// - data[3]: id_low (u128) - /// - data[4]: id_high (u128) - /// - data[5]: value_low (u128) - /// - data[6]: value_high (u128) - async fn decode_transfer_single(&self, event: &EmittedEvent) -> Result> { - let operator; - let from; - let to; - let id: U256; - let value: U256; - + fn parse_transfer_single(&self, event: &EmittedEvent) -> Option { if event.keys.len() == 4 && event.data.len() == 4 { - // Modern format: operator, from, to in keys; id, value in data - operator = event.keys[1]; - from = event.keys[2]; - to = event.keys[3]; - let id_low: u128 = event.data[0].try_into().unwrap_or(0); - let id_high: u128 = event.data[1].try_into().unwrap_or(0); - id = U256::from_words(id_low, id_high); - let value_low: u128 = event.data[2].try_into().unwrap_or(0); - let value_high: u128 = event.data[3].try_into().unwrap_or(0); - value = U256::from_words(value_low, value_high); - } else if event.keys.len() == 1 && event.data.len() == 7 { - // Legacy format: all in data - operator = event.data[0]; - from = event.data[1]; - to = event.data[2]; - let id_low: u128 = event.data[3].try_into().unwrap_or(0); - let id_high: u128 = event.data[4].try_into().unwrap_or(0); - id = U256::from_words(id_low, id_high); - let value_low: u128 = event.data[5].try_into().unwrap_or(0); - let value_high: u128 = event.data[6].try_into().unwrap_or(0); - value = U256::from_words(value_low, value_high); - } else if event.keys.len() == 4 && event.data.len() == 2 { - // Alternative modern format with single felt id and value - operator = event.keys[1]; - from = event.keys[2]; - to = event.keys[3]; - id = Self::felt_to_u256(event.data[0]); - value = Self::felt_to_u256(event.data[1]); - } else { - tracing::warn!( - target: "torii_erc1155::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - keys_len = event.keys.len(), - data_len = event.data.len(), - "Malformed ERC1155 TransferSingle event" - ); - return Ok(None); + return Some(TransferPart { + operator: event.keys[1], + from: event.keys[2], + to: event.keys[3], + token_id: felt_pair_to_u256(event.data[0], event.data[1]), + amount: felt_pair_to_u256(event.data[2], event.data[3]), + batch_index: 0, + is_batch: false, + }); } - let transfer = TransferSingle { - operator, - from, - to, - id, - value, - token: event.from_address, - block_number: event.block_number.unwrap_or(0), - transaction_hash: event.transaction_hash, - }; + if event.keys.len() == 1 && event.data.len() == 7 { + return Some(TransferPart { + operator: event.data[0], + from: event.data[1], + to: event.data[2], + token_id: felt_pair_to_u256(event.data[3], event.data[4]), + amount: felt_pair_to_u256(event.data[5], event.data[6]), + batch_index: 0, + is_batch: false, + }); + } - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); + if event.keys.len() == 4 && event.data.len() == 2 { + return Some(TransferPart { + operator: event.keys[1], + from: event.keys[2], + to: event.keys[3], + token_id: felt_to_u256(event.data[0]), + amount: felt_to_u256(event.data[1]), + batch_index: 0, + is_batch: false, + }); + } - let envelope_id = format!( - "erc1155_transfer_single_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); + None + } + + fn parse_u256_slice(items: &[Felt], as_pairs: bool) -> Vec { + if as_pairs { + return items + .chunks_exact(2) + .map(|chunk| felt_pair_to_u256(chunk[0], chunk[1])) + .collect(); + } - Ok(Some(Envelope::new( - envelope_id, - Box::new(transfer), - metadata, - ))) + items.iter().copied().map(felt_to_u256).collect() } - /// Decode TransferBatch event into multiple envelopes (one per id/value pair) - /// - /// TransferBatch event signatures: - /// - /// Modern ERC1155: - /// - keys[0]: TransferBatch selector - /// - keys[1]: operator address - /// - keys[2]: from address - /// - keys[3]: to address - /// - data[0]: ids_len - /// - data[1..1+ids_len*2]: ids (low, high pairs) - /// - data[1+ids_len*2]: values_len - /// - data[2+ids_len*2..]: values (low, high pairs) - /// - /// Legacy: all in data - async fn decode_transfer_batch(&self, event: &EmittedEvent) -> Result> { - let operator; - let from; - let to; - let mut data_offset = 0; - - if event.keys.len() == 4 { - // Modern format: operator, from, to in keys - operator = event.keys[1]; - from = event.keys[2]; - to = event.keys[3]; + fn parse_transfer_batch(&self, event: &EmittedEvent) -> Option> { + let (operator, from, to, mut data_offset) = if event.keys.len() == 4 { + (event.keys[1], event.keys[2], event.keys[3], 0usize) } else if event.keys.len() == 1 && event.data.len() >= 3 { - // Legacy format: operator, from, to at start of data - operator = event.data[0]; - from = event.data[1]; - to = event.data[2]; - data_offset = 3; + (event.data[0], event.data[1], event.data[2], 3usize) } else { - tracing::warn!( - target: "torii_erc1155::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - keys_len = event.keys.len(), - data_len = event.data.len(), - "Malformed ERC1155 TransferBatch event" - ); - return Ok(vec![]); - } + return None; + }; - // Parse ids array if event.data.len() <= data_offset { - return Ok(vec![]); + return Some(Vec::new()); } - let ids_len: usize = event.data[data_offset].try_into().unwrap_or(0); + let ids_len = felt_to_usize(event.data[data_offset]); data_offset += 1; - fn parse_u256_slice(items: &[Felt], as_pairs: bool) -> Vec { - if as_pairs { - let mut out = Vec::with_capacity(items.len() / 2); - for chunk in items.chunks_exact(2) { - let low: u128 = chunk[0].try_into().unwrap_or(0); - let high: u128 = chunk[1].try_into().unwrap_or(0); - out.push(U256::from_words(low, high)); - } - out - } else { - items - .iter() - .copied() - .map(Erc1155Decoder::felt_to_u256) - .collect() - } - } - - let mut ids: Vec = Vec::new(); - let mut values: Vec = Vec::new(); + let mut ids = Vec::new(); + let mut values = Vec::new(); let mut parsed = false; - // Try both pair-based and single-felt array layouts for ids and values. - // Standard Starknet ERC1155 uses U256 pairs, but some contracts emit felt arrays. for ids_as_pairs in [true, false] { let id_words = if ids_as_pairs { 2 } else { 1 }; let ids_end = data_offset.saturating_add(ids_len.saturating_mul(id_words)); @@ -385,8 +249,9 @@ impl Erc1155Decoder { continue; } - let candidate_ids = parse_u256_slice(&event.data[data_offset..ids_end], ids_as_pairs); - let values_len: usize = event.data[ids_end].try_into().unwrap_or(0); + let candidate_ids = + Self::parse_u256_slice(&event.data[data_offset..ids_end], ids_as_pairs); + let values_len = felt_to_usize(event.data[ids_end]); let values_start = ids_end + 1; for values_as_pairs in [true, false] { @@ -398,7 +263,8 @@ impl Erc1155Decoder { } ids.clone_from(&candidate_ids); - values = parse_u256_slice(&event.data[values_start..values_end], values_as_pairs); + values = + Self::parse_u256_slice(&event.data[values_start..values_end], values_as_pairs); parsed = true; break; } @@ -408,166 +274,99 @@ impl Erc1155Decoder { } } - if !parsed { - tracing::warn!( - target: "torii_erc1155::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - ids_len = ids_len, - "Failed to parse ERC1155 TransferBatch ids/values arrays" - ); - return Ok(vec![]); + if !parsed || ids.len() != values.len() { + return None; } - // Create envelope for each id/value pair - let mut envelopes = Vec::new(); - for (i, (id, value)) in ids.iter().zip(values.iter()).enumerate() { - let transfer = TransferBatch { - operator, - from, - to, - id: *id, - value: *value, - batch_index: i as u32, + Some( + ids.into_iter() + .zip(values) + .enumerate() + .map(|(batch_index, (token_id, amount))| TransferPart { + operator, + from, + to, + token_id, + amount, + batch_index: batch_index as u32, + is_batch: true, + }) + .collect(), + ) + } + + fn parse_approval_for_all(&self, event: &EmittedEvent) -> Option { + if event.keys.len() == 3 && event.data.len() == 1 { + return Some(OperatorApproval { + owner: event.keys[1], + operator: event.keys[2], + approved: event.data[0] != Felt::ZERO, token: event.from_address, block_number: event.block_number.unwrap_or(0), transaction_hash: event.transaction_hash, - }; - - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); - metadata.insert("batch_index".to_string(), i.to_string()); - - let envelope_id = format!( - "erc1155_transfer_batch_{}_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash), - i - ); + }); + } - envelopes.push(Envelope::new(envelope_id, Box::new(transfer), metadata)); + if event.keys.len() == 1 && event.data.len() == 3 { + return Some(OperatorApproval { + owner: event.data[0], + operator: event.data[1], + approved: event.data[2] != Felt::ZERO, + token: event.from_address, + block_number: event.block_number.unwrap_or(0), + transaction_hash: event.transaction_hash, + }); } - Ok(envelopes) + None } - /// Decode ApprovalForAll event into envelope - async fn decode_approval_for_all(&self, event: &EmittedEvent) -> Result> { - let owner; - let operator; - let approved: bool; - - if event.keys.len() == 3 && event.data.len() == 1 { - // Modern format: owner, operator in keys; approved in data - owner = event.keys[1]; - operator = event.keys[2]; - approved = event.data[0] != Felt::ZERO; - } else if event.keys.len() == 1 && event.data.len() == 3 { - // Legacy format: all in data - owner = event.data[0]; - operator = event.data[1]; - approved = event.data[2] != Felt::ZERO; - } else { - tracing::warn!( - target: "torii_erc1155::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - keys_len = event.keys.len(), - data_len = event.data.len(), - "Malformed ERC1155 ApprovalForAll event" - ); - return Ok(None); + fn parse_uri(&self, event: &EmittedEvent) -> Option { + if event.keys.len() < 2 || event.data.is_empty() { + return None; } - let approval = OperatorApproval { - owner, - operator, - approved, + let uri = Self::decode_string_result(&event.data)?; + Some(UriUpdate { token: event.from_address, + token_id: felt_to_u256(event.keys[1]), + uri, block_number: event.block_number.unwrap_or(0), transaction_hash: event.transaction_hash, - }; - - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); + }) + } - let envelope_id = format!( - "erc1155_approval_for_all_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) + fn log_malformed(&self, event: &EmittedEvent, event_name: &str) { + tracing::warn!( + target: "torii_erc1155::decoder", + token = %format!("{:#x}", event.from_address), + tx_hash = %format!("{:#x}", event.transaction_hash), + block_number = event.block_number.unwrap_or(0), + keys_len = event.keys.len(), + data_len = event.data.len(), + "Malformed ERC1155 {event_name} event" ); - - Ok(Some(Envelope::new( - envelope_id, - Box::new(approval), - metadata, - ))) } - /// Decode URI event into envelope - /// - /// Common ERC1155 Starknet layout: - /// - keys[0]: URI selector - /// - keys[1]: token id (felt-encoded) - /// - data: URI payload (short string or ByteArray) - async fn decode_uri(&self, event: &EmittedEvent) -> Result> { - if event.keys.len() < 2 || event.data.is_empty() { - return Ok(None); - } - - let token_id = Self::felt_to_u256(event.keys[1]); - let Some(uri) = Self::decode_string_result(&event.data) else { - return Ok(None); - }; - - let uri_update = UriUpdate { + fn transfer_envelope( + &self, + event: &EmittedEvent, + context: EventContext, + part: TransferPart, + ) -> Envelope { + Erc1155Message::Transfer(TransferData { + operator: part.operator, + from: part.from, + to: part.to, + token_id: part.token_id, + amount: part.amount, + is_batch: part.is_batch, + batch_index: part.batch_index, token: event.from_address, - token_id, - uri, block_number: event.block_number.unwrap_or(0), transaction_hash: event.transaction_hash, - }; - - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); - - let envelope_id = format!( - "erc1155_uri_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); - - Ok(Some(Envelope::new( - envelope_id, - Box::new(uri_update), - metadata, - ))) + }) + .to_envelope(context) } } @@ -583,128 +382,185 @@ impl Decoder for Erc1155Decoder { "erc1155" } - async fn decode_event(&self, event: &EmittedEvent) -> Result> { - if event.keys.is_empty() { + async fn decode( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> Result> { + let event = EmittedEvent { + from_address: context.from_address, + keys: keys.to_vec(), + data: data.to_vec(), + block_hash: None, + block_number: Some(context.block_number), + transaction_hash: context.transaction_hash, + }; + + let Some(selector) = event.keys.first().copied() else { return Ok(Vec::new()); + }; + + if selector == Self::transfer_single_selector() { + return Ok(if let Some(part) = self.parse_transfer_single(&event) { + vec![self.transfer_envelope(&event, context, part)] + } else { + self.log_malformed(&event, "TransferSingle"); + Vec::new() + }); } - let selector = event.keys[0]; + if selector == Self::transfer_batch_selector() { + return Ok(if let Some(parts) = self.parse_transfer_batch(&event) { + parts + .into_iter() + .map(|part| self.transfer_envelope(&event, context, part)) + .collect() + } else { + self.log_malformed(&event, "TransferBatch"); + Vec::new() + }); + } - if selector == Self::transfer_single_selector() { - if let Some(envelope) = self.decode_transfer_single(event).await? { - return Ok(vec![envelope]); - } - } else if selector == Self::transfer_batch_selector() { - return self.decode_transfer_batch(event).await; - } else if selector == Self::approval_for_all_selector() { - if let Some(envelope) = self.decode_approval_for_all(event).await? { - return Ok(vec![envelope]); - } - } else if selector == Self::uri_selector() { - if let Some(envelope) = self.decode_uri(event).await? { - return Ok(vec![envelope]); - } - } else { - tracing::trace!( - target: "torii_erc1155::decoder", - token = %format!("{:#x}", event.from_address), - selector = %format!("{:#x}", selector), - keys_len = event.keys.len(), - data_len = event.data.len(), - block_number = event.block_number.unwrap_or(0), - tx_hash = %format!("{:#x}", event.transaction_hash), - "Unhandled event selector" + if selector == Self::approval_for_all_selector() { + return Ok( + if let Some(approval) = self.parse_approval_for_all(&event) { + vec![Erc1155Message::ApprovalForAll(approval).to_envelope(context)] + } else { + self.log_malformed(&event, "ApprovalForAll"); + Vec::new() + }, ); } + if selector == Self::uri_selector() { + return Ok(match self.parse_uri(&event) { + Some(uri) => vec![Erc1155Message::Uri(uri).to_envelope(context)], + None => Vec::new(), + }); + } + + tracing::trace!( + target: "torii_erc1155::decoder", + token = %format!("{:#x}", event.from_address), + selector = %format!("{:#x}", selector), + keys_len = event.keys.len(), + data_len = event.data.len(), + block_number = event.block_number.unwrap_or(0), + tx_hash = %format!("{:#x}", event.transaction_hash), + "Unhandled event selector" + ); + Ok(Vec::new()) } } +fn to_starknet_felt(value: Felt) -> starknet::core::types::Felt { + starknet::core::types::Felt::from_bytes_be(&value.to_be_bytes()) +} + +fn felt_to_usize(value: Felt) -> usize { + usize::try_from(value.to_le_words()[0]).unwrap_or(0) +} + #[cfg(test)] mod tests { use super::*; + fn decode_raw(decoder: &Erc1155Decoder, event: &EmittedEvent) -> Vec { + futures::executor::block_on(decoder.decode( + &event.keys, + &event.data, + EventContext { + from_address: event.from_address, + block_number: event.block_number.unwrap_or(0), + transaction_hash: event.transaction_hash, + }, + )) + .unwrap() + } + + fn body(envelope: &Envelope) -> &Erc1155Body { + envelope + .body + .as_any() + .downcast_ref::() + .unwrap() + } + #[tokio::test] async fn test_decode_transfer_single_modern() { let decoder = Erc1155Decoder::new(); - - // Modern format: operator, from, to in keys; id, value in data let event = EmittedEvent { from_address: Felt::from(0x123u64), keys: vec![ Erc1155Decoder::transfer_single_selector(), - Felt::from(0x1u64), // operator - Felt::from(0x2u64), // from - Felt::from(0x3u64), // to + Felt::from(0x1u64), + Felt::from(0x2u64), + Felt::from(0x3u64), ], data: vec![ - Felt::from(42u64), // id_low - Felt::ZERO, // id_high - Felt::from(100u64), // value_low - Felt::ZERO, // value_high + Felt::from(42u64), + Felt::ZERO, + Felt::from(100u64), + Felt::ZERO, ], block_hash: None, block_number: Some(100), transaction_hash: Felt::from(0xabcdu64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decode_raw(&decoder, &event); assert_eq!(envelopes.len(), 1); - - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(transfer.operator, Felt::from(0x1u64)); - assert_eq!(transfer.from, Felt::from(0x2u64)); - assert_eq!(transfer.to, Felt::from(0x3u64)); - assert_eq!(transfer.id, U256::from(42u64)); - assert_eq!(transfer.value, U256::from(100u64)); + assert_eq!(envelopes[0].type_id, ERC1155_TYPE_ID); + + match &body(&envelopes[0]).msg { + Erc1155Message::Transfer(transfer) => { + assert_eq!(transfer.operator, Felt::from(0x1u64)); + assert_eq!(transfer.from, Felt::from(0x2u64)); + assert_eq!(transfer.to, Felt::from(0x3u64)); + assert_eq!(transfer.token_id, U256::from(42u64)); + assert_eq!(transfer.amount, U256::from(100u64)); + assert!(!transfer.is_batch); + } + other => panic!("unexpected message: {other:?}"), + } } #[tokio::test] async fn test_decode_approval_for_all() { let decoder = Erc1155Decoder::new(); - let event = EmittedEvent { from_address: Felt::from(0x456u64), keys: vec![ Erc1155Decoder::approval_for_all_selector(), - Felt::from(0xau64), // owner - Felt::from(0xbu64), // operator + Felt::from(0xau64), + Felt::from(0xbu64), ], - data: vec![Felt::from(1u64)], // approved = true + data: vec![Felt::from(1u64)], block_hash: None, block_number: Some(200), transaction_hash: Felt::from(0xef01u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decode_raw(&decoder, &event); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(approval.owner, Felt::from(0xau64)); - assert_eq!(approval.operator, Felt::from(0xbu64)); - assert!(approval.approved); + match &body(&envelopes[0]).msg { + Erc1155Message::ApprovalForAll(approval) => { + assert_eq!(approval.owner, Felt::from(0xau64)); + assert_eq!(approval.operator, Felt::from(0xbu64)); + assert!(approval.approved); + } + other => panic!("unexpected message: {other:?}"), + } } #[tokio::test] async fn test_decode_transfer_single_single_felt_preserves_full_felt() { let decoder = Erc1155Decoder::new(); - - let id_felt = - Felt::from_hex("0x100000000000000000000000000000001").expect("invalid id felt"); - let value_felt = - Felt::from_hex("0x200000000000000000000000000000003").expect("invalid value felt"); - + let id_felt = Felt::from_hex("0x100000000000000000000000000000001").unwrap(); + let value_felt = Felt::from_hex("0x200000000000000000000000000000003").unwrap(); let event = EmittedEvent { from_address: Felt::from(0x123u64), keys: vec![ @@ -719,92 +575,79 @@ mod tests { transaction_hash: Felt::from(0xabcdu64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); - assert_eq!(envelopes.len(), 1); - - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(transfer.id, Erc1155Decoder::felt_to_u256(id_felt)); - assert_eq!(transfer.value, Erc1155Decoder::felt_to_u256(value_felt)); + let envelopes = decode_raw(&decoder, &event); + match &body(&envelopes[0]).msg { + Erc1155Message::Transfer(transfer) => { + assert_eq!(transfer.token_id, felt_to_u256(id_felt)); + assert_eq!(transfer.amount, felt_to_u256(value_felt)); + } + other => panic!("unexpected message: {other:?}"), + } } #[tokio::test] async fn test_decode_transfer_batch_single_felt_arrays() { let decoder = Erc1155Decoder::new(); - let event = EmittedEvent { from_address: Felt::from(0x123u64), keys: vec![ Erc1155Decoder::transfer_batch_selector(), - Felt::from(0x1u64), // operator - Felt::from(0x2u64), // from - Felt::from(0x3u64), // to + Felt::from(0x1u64), + Felt::from(0x2u64), + Felt::from(0x3u64), ], data: vec![ - Felt::from(2u64), // ids_len - Felt::from(11u64), // id[0] - Felt::from(12u64), // id[1] - Felt::from(2u64), // values_len - Felt::from(101u64), // value[0] - Felt::from(102u64), // value[1] + Felt::from(2u64), + Felt::from(11u64), + Felt::from(12u64), + Felt::from(2u64), + Felt::from(101u64), + Felt::from(102u64), ], block_hash: None, block_number: Some(102), transaction_hash: Felt::from(0xabcfu64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decode_raw(&decoder, &event); assert_eq!(envelopes.len(), 2); - let first = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - let second = envelopes[1] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(first.id, U256::from(11u64)); - assert_eq!(first.value, U256::from(101u64)); - assert_eq!(second.id, U256::from(12u64)); - assert_eq!(second.value, U256::from(102u64)); + match (&body(&envelopes[0]).msg, &body(&envelopes[1]).msg) { + (Erc1155Message::Transfer(first), Erc1155Message::Transfer(second)) => { + assert_eq!(first.token_id, U256::from(11u64)); + assert_eq!(first.amount, U256::from(101u64)); + assert!(first.is_batch); + assert_eq!(first.batch_index, 0); + assert_eq!(second.token_id, U256::from(12u64)); + assert_eq!(second.amount, U256::from(102u64)); + assert_eq!(second.batch_index, 1); + } + other => panic!("unexpected messages: {other:?}"), + } } #[tokio::test] async fn test_decode_uri_event() { let decoder = Erc1155Decoder::new(); - - // "abc" as short string felt - let uri_felt = Felt::from(0x616263u64); let event = EmittedEvent { from_address: Felt::from(0x123u64), - keys: vec![ - Erc1155Decoder::uri_selector(), - Felt::from(7u64), // token id - ], - data: vec![uri_felt], + keys: vec![Erc1155Decoder::uri_selector(), Felt::from(7u64)], + data: vec![Felt::from(0x616263u64)], block_hash: None, block_number: Some(103), transaction_hash: Felt::from(0xabd0u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decode_raw(&decoder, &event); assert_eq!(envelopes.len(), 1); - let uri = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - assert_eq!(uri.token, Felt::from(0x123u64)); - assert_eq!(uri.token_id, U256::from(7u64)); - assert_eq!(uri.uri, "abc".to_string()); + match &body(&envelopes[0]).msg { + Erc1155Message::Uri(uri) => { + assert_eq!(uri.token, Felt::from(0x123u64)); + assert_eq!(uri.token_id, U256::from(7u64)); + assert_eq!(uri.uri, "abc"); + } + other => panic!("unexpected message: {other:?}"), + } } } diff --git a/crates/torii-erc1155/src/grpc_service.rs b/crates/torii-erc1155/src/grpc_service.rs index d9b99c74..41ac9fbc 100644 --- a/crates/torii-erc1155/src/grpc_service.rs +++ b/crates/torii-erc1155/src/grpc_service.rs @@ -13,17 +13,21 @@ use crate::proto::{ use crate::storage::{Erc1155Storage, TokenTransferData, TransferCursor}; use async_trait::async_trait; use futures::stream::Stream; -use starknet::core::types::Felt; -use starknet::core::types::U256; +use primitive_types::U256; +use starknet_types_raw::Felt; use std::collections::{HashMap, HashSet}; use std::pin::Pin; use std::sync::Arc; use tokio::sync::broadcast; use tonic::{Request, Response, Status}; -use torii_common::{bytes_to_felt, bytes_to_u256, u256_to_bytes}; +use torii_common::{bytes_to_u256, u256_to_bytes}; const DEFAULT_PROJECT_ID: &str = "arcade-main"; +fn bytes_to_felt(bytes: &[u8]) -> Option { + Felt::from_be_bytes_slice(bytes).ok() +} + /// gRPC service implementation for ERC1155 #[derive(Clone)] pub struct Erc1155Service { @@ -55,14 +59,14 @@ impl Erc1155Service { /// Convert storage TokenTransferData to proto TokenTransfer fn transfer_data_to_proto(data: &TokenTransferData) -> TokenTransfer { TokenTransfer { - token: data.token.to_bytes_be().to_vec(), - operator: data.operator.to_bytes_be().to_vec(), - from: data.from.to_bytes_be().to_vec(), - to: data.to.to_bytes_be().to_vec(), + token: data.token.to_be_bytes_vec(), + operator: data.operator.to_be_bytes_vec(), + from: data.from.to_be_bytes_vec(), + to: data.to.to_be_bytes_vec(), token_id: u256_to_bytes(data.token_id), amount: u256_to_bytes(data.amount), block_number: data.block_number, - tx_hash: data.tx_hash.to_bytes_be().to_vec(), + tx_hash: data.tx_hash.to_be_bytes_vec(), timestamp: data.timestamp.unwrap_or(0), is_batch: data.is_batch, batch_index: data.batch_index, @@ -223,7 +227,7 @@ impl Erc1155Service { }; CollectionToken { - contract_address: contract.to_bytes_be().to_vec(), + contract_address: contract.to_be_bytes_vec(), token_id: token_id_bytes, uri, metadata_json, @@ -361,7 +365,7 @@ impl Erc1155Trait for Erc1155Service { let entries = match self.storage.get_token_metadata(token).await { Ok(Some((name, symbol, total_supply))) => vec![TokenMetadataEntry { - token: token.to_bytes_be().to_vec(), + token: token.to_be_bytes_vec(), name, symbol, total_supply: total_supply.map(u256_to_bytes), @@ -392,7 +396,7 @@ impl Erc1155Trait for Erc1155Service { let entries = all .into_iter() .map(|(token, name, symbol, total_supply)| TokenMetadataEntry { - token: token.to_bytes_be().to_vec(), + token: token.to_be_bytes_vec(), name, symbol, total_supply: total_supply.map(u256_to_bytes), @@ -401,7 +405,7 @@ impl Erc1155Trait for Erc1155Service { Ok(Response::new(GetTokenMetadataResponse { tokens: entries, - next_cursor: next_cursor.map(|c| c.to_bytes_be().to_vec()), + next_cursor: next_cursor.map(|c| c.to_be_bytes_vec()), })) } @@ -586,7 +590,7 @@ impl Erc1155Trait for Erc1155Service { let traits = Self::build_trait_summaries(&facets); overviews.push(ContractCollectionOverview { - contract_address: contract.to_bytes_be().to_vec(), + contract_address: contract.to_be_bytes_vec(), tokens, next_cursor_token_id, total_hits, diff --git a/crates/torii-erc1155/src/handlers.rs b/crates/torii-erc1155/src/handlers.rs index f82ae36a..ac7ce153 100644 --- a/crates/torii-erc1155/src/handlers.rs +++ b/crates/torii-erc1155/src/handlers.rs @@ -1,16 +1,19 @@ use anyhow::Result; use async_trait::async_trait; +use primitive_types::U256; use prost::Message; use prost_types::Any; -use starknet::core::types::Felt; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; +use starknet_types_raw::Felt; use std::collections::HashSet; use std::path::PathBuf; use std::sync::{Arc, Mutex}; use torii::command::CommandHandler; use torii::etl::sink::EventBus; use torii::UpdateType; -use torii_common::{process_token_uri_request, u256_to_bytes, MetadataFetcher, TokenUriRequest}; +use torii_common::{ + bytes_to_u256, process_token_uri_request, u256_to_bytes, MetadataFetcher, TokenUriRequest, +}; use crate::proto; use crate::storage::Erc1155Storage; @@ -23,7 +26,7 @@ pub struct FetchErc1155MetadataCommand { #[derive(Debug, Clone)] pub struct RefreshErc1155TokenUriCommand { pub contract: Felt, - pub token_id: starknet::core::types::U256, + pub token_id: U256, } pub struct Erc1155MetadataCommandHandler { @@ -98,14 +101,15 @@ impl CommandHandler for Erc1155MetadataCommandHandler { command.token, meta.name.as_deref(), meta.symbol.as_deref(), - meta.total_supply, + meta.total_supply + .map(|value| bytes_to_u256(&u256_to_bytes(value))), ) .await?; let event_bus = self.event_bus.lock().unwrap().clone(); if let Some(event_bus) = event_bus { let meta_entry = proto::TokenMetadataEntry { - token: command.token.to_bytes_be().to_vec(), + token: command.token.to_be_bytes_vec(), name: meta.name, symbol: meta.symbol, total_supply: meta.total_supply.map(u256_to_bytes), @@ -142,7 +146,7 @@ pub struct Erc1155TokenUriCommandHandler { fetcher: Arc, storage: Arc, image_cache_dir: Option, - in_flight: Mutex>, + in_flight: Mutex>, } impl Erc1155TokenUriCommandHandler { @@ -193,7 +197,7 @@ impl CommandHandler for Erc1155TokenUriCommandHandler { self.storage.as_ref(), &TokenUriRequest { contract: command.contract, - token_id: command.token_id, + token_id: bytes_to_u256(&u256_to_bytes(command.token_id)), standard: torii_common::TokenStandard::Erc1155, }, self.image_cache_dir.as_deref(), diff --git a/crates/torii-erc1155/src/identification.rs b/crates/torii-erc1155/src/identification.rs index 80a8b44d..a400d958 100644 --- a/crates/torii-erc1155/src/identification.rs +++ b/crates/torii-erc1155/src/identification.rs @@ -6,7 +6,7 @@ //! - `TransferBatch` event use anyhow::Result; -use starknet::core::types::Felt; +use starknet_types_raw::Felt; use torii::etl::decoder::DecoderId; use torii::etl::extractor::ContractAbi; use torii::etl::identification::IdentificationRule; diff --git a/crates/torii-erc1155/src/lib.rs b/crates/torii-erc1155/src/lib.rs index 23b94621..6aed0f6d 100644 --- a/crates/torii-erc1155/src/lib.rs +++ b/crates/torii-erc1155/src/lib.rs @@ -20,7 +20,8 @@ //! use torii_erc1155::proto::erc1155_server::Erc1155Server; //! //! // Create storage -//! let storage = Arc::new(Erc1155Storage::new("./erc1155.db")?); +//! let pool = torii_sql::DbPoolOptions::new().connect_any("./erc1155.db").await?; +//! let storage = Arc::new(Erc1155Storage::new(pool, "./erc1155.db").await?); //! //! // Create gRPC service //! let grpc_service = Erc1155Service::new(storage.clone()); @@ -55,7 +56,10 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!("generated/erc1155_descrip // Re-export main types for convenience pub use balance_fetcher::{Erc1155BalanceFetchRequest, Erc1155BalanceFetcher}; -pub use decoder::{Erc1155Decoder, OperatorApproval, TransferBatch, TransferSingle, UriUpdate}; +pub use decoder::{ + Erc1155Body, Erc1155Decoder, Erc1155Message, OperatorApproval, TransferData, UriUpdate, + ERC1155_TYPE_ID, +}; pub use grpc_service::Erc1155Service; pub use handlers::{Erc1155MetadataCommandHandler, Erc1155TokenUriCommandHandler}; pub use identification::Erc1155Rule; diff --git a/crates/torii-erc1155/src/sink.rs b/crates/torii-erc1155/src/sink.rs index a3b1b1e8..d3b195ad 100644 --- a/crates/torii-erc1155/src/sink.rs +++ b/crates/torii-erc1155/src/sink.rs @@ -12,21 +12,18 @@ //! fetches the actual balance from the chain and adjusts use crate::balance_fetcher::Erc1155BalanceFetcher; -use crate::decoder::{ - OperatorApproval as DecodedOperatorApproval, TransferBatch as DecodedTransferBatch, - TransferSingle as DecodedTransferSingle, UriUpdate as DecodedUriUpdate, -}; +use crate::decoder::{Erc1155Body as DecodedErc1155Body, Erc1155Message, ERC1155_TYPE_ID}; use crate::grpc_service::Erc1155Service; use crate::handlers::{FetchErc1155MetadataCommand, RefreshErc1155TokenUriCommand}; use crate::proto; use crate::storage::{Erc1155Storage, OperatorApprovalData, TokenTransferData, TokenUriData}; -use anyhow::Result; use async_trait::async_trait; use axum::Router; +use primitive_types::U256; use prost::Message; use prost_types::Any; -use starknet::core::types::{Felt, U256}; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; +use starknet_types_raw::Felt; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -34,12 +31,26 @@ use torii::command::CommandBusSender; use torii::etl::sink::{EventBus, TopicInfo}; use torii::etl::{Envelope, ExtractionBatch, Sink, TypeId}; use torii::grpc::UpdateType; -use torii_common::{u256_to_bytes, TokenStandard, TokenUriRequest, TokenUriSender}; +use torii_common::{bytes_to_u256, u256_to_bytes, TokenStandard, TokenUriRequest, TokenUriSender}; /// Default threshold for "live" detection: 100 blocks from chain head. /// Events from blocks older than this won't be broadcast to real-time subscribers. const LIVE_THRESHOLD_BLOCKS: u64 = 100; +type SinkResult = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Erc1155SinkError { + #[error("failed to batch insert transfers")] + InsertTransfers(#[source] anyhow::Error), + #[error("failed to batch insert operator approvals")] + InsertOperatorApprovals(#[source] anyhow::Error), + #[error("failed to batch upsert token URI updates")] + UpsertTokenUriUpdates(#[source] anyhow::Error), + #[error("failed to encode ERC1155 protobuf payload")] + Encode(#[from] prost::EncodeError), +} + /// ERC1155 token sink /// /// Processes ERC1155 TransferSingle, TransferBatch, and ApprovalForAll events: @@ -193,7 +204,7 @@ impl Erc1155Sink { if let Some(sender) = &self.token_uri_sender { return sender.request_update(TokenUriRequest { contract, - token_id, + token_id: bytes_to_u256(&u256_to_bytes(token_id)), standard: TokenStandard::Erc1155, }); } @@ -214,35 +225,23 @@ impl Erc1155Sink { false } -} - -#[async_trait] -impl Sink for Erc1155Sink { - fn name(&self) -> &'static str { - "erc1155" - } - - fn interested_types(&self) -> Vec { - vec![ - TypeId::new("erc1155.transfer_single"), - TypeId::new("erc1155.transfer_batch"), - TypeId::new("erc1155.approval_for_all"), - TypeId::new("erc1155.uri"), - ] - } - async fn initialize( + async fn initialize_sink( &mut self, event_bus: Arc, context: &torii::etl::sink::SinkContext, - ) -> Result<()> { + ) -> SinkResult<()> { self.event_bus = Some(event_bus); self.command_bus = Some(context.command_bus.clone()); tracing::info!(target: "torii_erc1155::sink", "ERC1155 sink initialized"); Ok(()) } - async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> Result<()> { + async fn process_sink( + &self, + envelopes: &[Envelope], + batch: &ExtractionBatch, + ) -> SinkResult<()> { let mut transfers: Vec = Vec::with_capacity(envelopes.len()); let mut operator_approvals: Vec = Vec::with_capacity(envelopes.len()); let mut uri_updates: Vec = Vec::with_capacity(envelopes.len()); @@ -258,86 +257,53 @@ impl Sink for Erc1155Sink { .collect(); for envelope in envelopes { - // Handle single transfers - if envelope.type_id == TypeId::new("erc1155.transfer_single") { - if let Some(transfer) = envelope - .body - .as_any() - .downcast_ref::() - { - let timestamp = block_timestamps.get(&transfer.block_number).copied(); - transfers.push(TokenTransferData { - id: None, - token: transfer.token, - operator: transfer.operator, - from: transfer.from, - to: transfer.to, - token_id: transfer.id, - amount: transfer.value, - is_batch: false, - batch_index: 0, - block_number: transfer.block_number, - tx_hash: transfer.transaction_hash, - timestamp, - }); - } + if envelope.type_id != ERC1155_TYPE_ID { + continue; } - // Handle batch transfers - else if envelope.type_id == TypeId::new("erc1155.transfer_batch") { - if let Some(transfer) = envelope - .body - .as_any() - .downcast_ref::() - { - let timestamp = block_timestamps.get(&transfer.block_number).copied(); - transfers.push(TokenTransferData { - id: None, - token: transfer.token, - operator: transfer.operator, - from: transfer.from, - to: transfer.to, - token_id: transfer.id, - amount: transfer.value, - is_batch: true, - batch_index: transfer.batch_index, - block_number: transfer.block_number, - tx_hash: transfer.transaction_hash, - timestamp, - }); - } - } - // Handle approval for all - else if envelope.type_id == TypeId::new("erc1155.approval_for_all") { - if let Some(approval) = envelope - .body - .as_any() - .downcast_ref::() - { - let timestamp = block_timestamps.get(&approval.block_number).copied(); - operator_approvals.push(OperatorApprovalData { - id: None, - token: approval.token, - owner: approval.owner, - operator: approval.operator, - approved: approval.approved, - block_number: approval.block_number, - tx_hash: approval.transaction_hash, - timestamp, - }); - } - } - // Handle URI updates - else if envelope.type_id == TypeId::new("erc1155.uri") { - if let Some(uri) = envelope.body.as_any().downcast_ref::() { - let timestamp = block_timestamps.get(&uri.block_number).copied(); - uri_updates.push(TokenUriData { - token: uri.token, - token_id: uri.token_id, - uri: uri.uri.clone(), - block_number: uri.block_number, - tx_hash: uri.transaction_hash, - timestamp, - }); + + if let Some(body) = envelope.body.as_any().downcast_ref::() { + match &body.msg { + Erc1155Message::Transfer(transfer) => { + let timestamp = block_timestamps.get(&transfer.block_number).copied(); + transfers.push(TokenTransferData { + id: None, + token: transfer.token, + operator: transfer.operator, + from: transfer.from, + to: transfer.to, + token_id: transfer.token_id, + amount: transfer.amount, + is_batch: transfer.is_batch, + batch_index: transfer.batch_index, + block_number: transfer.block_number, + tx_hash: transfer.transaction_hash, + timestamp, + }); + } + Erc1155Message::ApprovalForAll(approval) => { + let timestamp = block_timestamps.get(&approval.block_number).copied(); + operator_approvals.push(OperatorApprovalData { + id: None, + token: approval.token, + owner: approval.owner, + operator: approval.operator, + approved: approval.approved, + block_number: approval.block_number, + tx_hash: approval.transaction_hash, + timestamp, + }); + } + Erc1155Message::Uri(uri) => { + let timestamp = block_timestamps.get(&uri.block_number).copied(); + uri_updates.push(TokenUriData { + token: uri.token, + token_id: uri.token_id, + uri: uri.uri.clone(), + block_number: uri.block_number, + tx_hash: uri.transaction_hash, + timestamp, + }); + } } } } @@ -449,14 +415,14 @@ impl Sink for Erc1155Sink { if !transfers.is_empty() { let transfer_count = match self.storage.insert_transfers_batch(&transfers).await { Ok(count) => count, - Err(e) => { + Err(error) => { tracing::error!( target: "torii_erc1155::sink", count = transfers.len(), - error = %e, + error = %error, "Failed to batch insert transfers" ); - return Err(e); + return Err(Erc1155SinkError::InsertTransfers(error)); } }; @@ -471,9 +437,7 @@ impl Sink for Erc1155Sink { "Batch inserted token transfers" ); - // Update balances if balance tracking is enabled if let Some(ref fetcher) = self.balance_fetcher { - // Step 1: Check which balances need adjustment (would go negative) let adjustment_requests = match self .storage .check_balances_batch(&transfers) @@ -490,7 +454,6 @@ impl Sink for Erc1155Sink { } }; - // Step 2: Batch fetch actual balances from RPC for inconsistent wallets let mut adjustments: HashMap<(Felt, Felt, U256), U256> = HashMap::new(); if !adjustment_requests.is_empty() { tracing::info!( @@ -511,7 +474,6 @@ impl Sink for Erc1155Sink { error = %e, "Failed to fetch balances from RPC, using 0 for adjustments" ); - // On failure, use 0 for all requested adjustments for req in &adjustment_requests { adjustments.insert( (req.contract, req.wallet, req.token_id), @@ -522,7 +484,6 @@ impl Sink for Erc1155Sink { } } - // Step 3: Apply transfers with adjustments to update balances if let Err(e) = self .storage .apply_transfers_with_adjustments(&transfers, &adjustments) @@ -533,30 +494,26 @@ impl Sink for Erc1155Sink { error = %e, "Failed to apply balance updates" ); - // Don't fail the whole batch - transfers are already inserted } } - // Only broadcast to real-time subscribers when near chain head let is_live = batch.is_live(LIVE_THRESHOLD_BLOCKS); if is_live { - // Publish transfer events for transfer in &transfers { let proto_transfer = proto::TokenTransfer { - token: transfer.token.to_bytes_be().to_vec(), - operator: transfer.operator.to_bytes_be().to_vec(), - from: transfer.from.to_bytes_be().to_vec(), - to: transfer.to.to_bytes_be().to_vec(), + token: transfer.token.to_be_bytes_vec(), + operator: transfer.operator.to_be_bytes_vec(), + from: transfer.from.to_be_bytes_vec(), + to: transfer.to.to_be_bytes_vec(), token_id: u256_to_bytes(transfer.token_id), amount: u256_to_bytes(transfer.amount), block_number: transfer.block_number, - tx_hash: transfer.tx_hash.to_bytes_be().to_vec(), + tx_hash: transfer.tx_hash.to_be_bytes_vec(), timestamp: transfer.timestamp.unwrap_or(0), is_batch: transfer.is_batch, batch_index: transfer.batch_index, }; - // Publish to EventBus if let Some(event_bus) = &self.event_bus { let mut buf = Vec::new(); proto_transfer.encode(&mut buf)?; @@ -576,7 +533,6 @@ impl Sink for Erc1155Sink { ); } - // Broadcast to gRPC service if let Some(grpc_service) = &self.grpc_service { grpc_service.broadcast_transfer(proto_transfer); } @@ -585,7 +541,6 @@ impl Sink for Erc1155Sink { } } - // Batch insert operator approvals if !operator_approvals.is_empty() { match self .storage @@ -603,19 +558,18 @@ impl Sink for Erc1155Sink { "Batch inserted operator approvals" ); } - Err(e) => { + Err(error) => { tracing::error!( target: "torii_erc1155::sink", - "Failed to batch insert {} operator approvals: {}", - operator_approvals.len(), - e + count = operator_approvals.len(), + error = %error, + "Failed to batch insert operator approvals" ); - return Err(e); + return Err(Erc1155SinkError::InsertOperatorApprovals(error)); } } } - // Batch upsert token URI updates if !uri_updates.is_empty() { match self.storage.upsert_token_uris_batch(&uri_updates).await { Ok(count) => { @@ -629,11 +583,10 @@ impl Sink for Erc1155Sink { "Batch upserted token URI updates" ); - // Publish URI updates to topic subscribers if let Some(event_bus) = &self.event_bus { for uri in &uri_updates { let proto_uri = proto::TokenUri { - token: uri.token.to_bytes_be().to_vec(), + token: uri.token.to_be_bytes_vec(), token_id: u256_to_bytes(uri.token_id), uri: uri.uri.clone(), block_number: uri.block_number, @@ -658,19 +611,18 @@ impl Sink for Erc1155Sink { } } } - Err(e) => { + Err(error) => { tracing::error!( target: "torii_erc1155::sink", count = uri_updates.len(), - error = %e, + error = %error, "Failed to batch upsert token URI updates" ); - return Err(e); + return Err(Erc1155SinkError::UpsertTokenUriUpdates(error)); } } } - // Log combined statistics without full-table scans. if inserted_transfers > 0 || inserted_operator_approvals > 0 || inserted_uri_updates > 0 { tracing::info!( target: "torii_erc1155::sink", @@ -687,6 +639,33 @@ impl Sink for Erc1155Sink { Ok(()) } +} + +#[async_trait] +impl Sink for Erc1155Sink { + fn name(&self) -> &'static str { + "erc1155" + } + + fn interested_types(&self) -> Vec { + vec![ERC1155_TYPE_ID] + } + + async fn initialize( + &mut self, + event_bus: Arc, + context: &torii::etl::sink::SinkContext, + ) -> anyhow::Result<()> { + self.initialize_sink(event_bus, context) + .await + .map_err(Into::into) + } + + async fn process(&self, envelopes: &[Envelope], batch: &ExtractionBatch) -> anyhow::Result<()> { + self.process_sink(envelopes, batch) + .await + .map_err(Into::into) + } fn topics(&self) -> Vec { vec![ diff --git a/crates/torii-erc1155/src/storage.rs b/crates/torii-erc1155/src/storage.rs index 1886977b..d672c9ac 100644 --- a/crates/torii-erc1155/src/storage.rs +++ b/crates/torii-erc1155/src/storage.rs @@ -7,14 +7,14 @@ //! - Records all adjustments in an audit table for debugging use anyhow::Result; +use primitive_types::U256; use rusqlite::{params, params_from_iter, Connection, ToSql}; -use starknet::core::types::{Felt, U256}; +use starknet_types_raw::Felt; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, Mutex}; use tokio_postgres::{types::ToSql as PgToSql, Client, NoTls}; -use torii_common::{ - blob_to_felt, blob_to_u256, felt_to_blob, u256_to_blob, TokenUriResult, TokenUriStore, -}; +use torii_common::{blob_to_u256, u256_to_blob, TokenUriResult, TokenUriStore}; +use torii_sql::DbPool; use crate::balance_fetcher::Erc1155BalanceFetchRequest; @@ -23,7 +23,15 @@ const SQLITE_TOKEN_BATCH_SIZE: usize = SQLITE_MAX_BIND_VARS; const SQLITE_TOKEN_PAIR_BATCH_SIZE: usize = SQLITE_MAX_BIND_VARS / 2; /// Maximum value for U256 (2^256 - 1) -const U256_MAX: U256 = U256::from_words(u128::MAX, u128::MAX); +const U256_MAX: U256 = U256([u64::MAX, u64::MAX, u64::MAX, u64::MAX]); + +fn felt_to_blob(value: Felt) -> Vec { + value.to_be_bytes_vec() +} + +fn blob_to_felt(bytes: &[u8]) -> Felt { + Felt::from_be_bytes_slice(bytes).unwrap_or(Felt::ZERO) +} /// Safely adds two U256 values, capping at U256::MAX on overflow. /// @@ -54,6 +62,7 @@ fn safe_u256_add(a: U256, b: U256) -> U256 { /// Storage for ERC1155 token data pub struct Erc1155Storage { + pool: DbPool, backend: StorageBackend, conn: Arc>, pg_conn: Option>>, @@ -157,10 +166,14 @@ pub struct Erc1155BalanceAdjustment { } impl Erc1155Storage { + pub fn pool(&self) -> &DbPool { + &self.pool + } + /// Create or open the database - pub async fn new(db_path: &str) -> Result { - if db_path.starts_with("postgres://") || db_path.starts_with("postgresql://") { - let (client, connection) = tokio_postgres::connect(db_path, NoTls).await?; + pub async fn new(pool: DbPool, database_url: &str) -> Result { + if matches!(&pool, DbPool::Postgres(_)) { + let (client, connection) = tokio_postgres::connect(database_url, NoTls).await?; tokio::spawn(async move { if let Err(e) = connection.await { tracing::error!(target: "torii_erc1155::storage", error = %e, "PostgreSQL connection task failed"); @@ -305,13 +318,14 @@ impl Erc1155Storage { tracing::info!(target: "torii_erc1155::storage", "PostgreSQL storage initialized"); return Ok(Self { + pool, backend: StorageBackend::Postgres, conn: Arc::new(Mutex::new(Connection::open_in_memory()?)), pg_conn: Some(Arc::new(tokio::sync::Mutex::new(client))), }); } - let conn = Connection::open(db_path)?; + let conn = Connection::open(database_url)?; // Enable WAL mode + Performance PRAGMAs conn.execute_batch( @@ -554,9 +568,10 @@ impl Erc1155Storage { [], )?; - tracing::info!(target: "torii_erc1155::storage", db_path = %db_path, "ERC1155 database initialized"); + tracing::info!(target: "torii_erc1155::storage", db_path = %database_url, "ERC1155 database initialized"); Ok(Self { + pool, backend: StorageBackend::Sqlite, conn: Arc::new(Mutex::new(conn)), pg_conn: None, @@ -3090,7 +3105,13 @@ impl TokenUriStore for Erc1155Storage { .await?; } - pg_sync_facets_for_token(&tx, result.contract, result.token_id, attrs).await?; + pg_sync_facets_for_token( + &tx, + result.contract, + blob_to_u256(&u256_to_blob(result.token_id)), + attrs, + ) + .await?; } tx.commit().await?; @@ -3145,7 +3166,12 @@ impl TokenUriStore for Erc1155Storage { insert_attr_stmt.execute(params![&token_blob, &token_id_blob, key, value])?; } - sqlite_sync_facets_for_token(&tx, result.contract, result.token_id, attrs)?; + sqlite_sync_facets_for_token( + &tx, + result.contract, + blob_to_u256(&u256_to_blob(result.token_id)), + attrs, + )?; } } diff --git a/crates/torii-erc1155/src/synthetic.rs b/crates/torii-erc1155/src/synthetic.rs index 6b11ec97..a9609e01 100644 --- a/crates/torii-erc1155/src/synthetic.rs +++ b/crates/torii-erc1155/src/synthetic.rs @@ -5,9 +5,11 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt, U256}; -use starknet::macros::selector; +use primitive_types::U256; +use starknet_types_raw::event::EmittedEvent; +use starknet_types_raw::Felt; use torii::etl::extractor::{ExtractionBatch, SyntheticExtractor}; +use torii_types::event::StarknetEvent; const EXTRACTOR_NAME: &str = "synthetic_erc1155"; @@ -239,16 +241,13 @@ impl SyntheticErc1155Extractor { Erc1155EventType::TransferSingle => { let id = self.token_id_for(block_number, tx_index, 0); let value = self.value_for(block_number, tx_index, 0); + let (id_low, id_high) = u256_low_high(id); + let (value_low, value_high) = u256_low_high(value); EmittedEvent { from_address: token, - keys: vec![selector!("TransferSingle"), operator, from, to], - data: vec![ - Felt::from(id.low()), - Felt::from(id.high()), - Felt::from(value.low()), - Felt::from(value.high()), - ], + keys: vec![Felt::selector("TransferSingle"), operator, from, to], + data: vec![id_low, id_high, value_low, value_high], block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), block_number: Some(block_number), transaction_hash: tx_hash, @@ -261,22 +260,24 @@ impl SyntheticErc1155Extractor { for i in 0..batch_size { let id = self.token_id_for(block_number, tx_index, i); - ids_data.push(Felt::from(id.low())); - ids_data.push(Felt::from(id.high())); + let (low, high) = u256_low_high(id); + ids_data.push(low); + ids_data.push(high); } values_data.push(Felt::from(batch_size as u64)); for i in 0..batch_size { let value = self.value_for(block_number, tx_index, i); - values_data.push(Felt::from(value.low())); - values_data.push(Felt::from(value.high())); + let (low, high) = u256_low_high(value); + values_data.push(low); + values_data.push(high); } ids_data.extend(values_data); EmittedEvent { from_address: token, - keys: vec![selector!("TransferBatch"), operator, from, to], + keys: vec![Felt::selector("TransferBatch"), operator, from, to], data: ids_data, block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), block_number: Some(block_number), @@ -285,7 +286,7 @@ impl SyntheticErc1155Extractor { } Erc1155EventType::ApprovalForAll => EmittedEvent { from_address: token, - keys: vec![selector!("ApprovalForAll"), from, to], + keys: vec![Felt::selector("ApprovalForAll"), from, to], data: vec![Felt::from(1u64)], block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), block_number: Some(block_number), @@ -294,9 +295,10 @@ impl SyntheticErc1155Extractor { Erc1155EventType::Uri => { let id = self.token_id_for(block_number, tx_index, 0); let uri_felt = Felt::from(0x69706673u64); + let (id_low, _) = u256_low_high(id); EmittedEvent { from_address: token, - keys: vec![selector!("URI"), Felt::from(id.low())], + keys: vec![Felt::selector("URI"), id_low], data: vec![uri_felt], block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), block_number: Some(block_number), @@ -304,7 +306,17 @@ impl SyntheticErc1155Extractor { } } }; - batch.add_event_with_tx_context(event, Some(operator), vec![token, from, to]); + batch.add_event_with_tx_context( + StarknetEvent::new( + event.from_address, + event.keys, + event.data, + event.block_number.unwrap_or(0), + event.transaction_hash, + ), + Some(operator), + vec![token, from, to], + ); } } @@ -314,6 +326,14 @@ impl SyntheticErc1155Extractor { } } +fn u256_low_high(value: U256) -> (Felt, Felt) { + let [l0, l1, h0, h1] = value.0; + ( + Felt::from_le_words([l0, l1, 0, 0]), + Felt::from_le_words([h0, h1, 0, 0]), + ) +} + #[derive(Debug, Clone, Copy, PartialEq)] enum Erc1155EventType { TransferSingle, @@ -407,10 +427,10 @@ mod tests { let mut extractor = SyntheticErc1155Extractor::new(cfg).unwrap(); let batch = extractor.extract(None).await.unwrap(); - let transfer_single_selector = selector!("TransferSingle"); - let transfer_batch_selector = selector!("TransferBatch"); - let approval_for_all_selector = selector!("ApprovalForAll"); - let uri_selector = selector!("URI"); + let transfer_single_selector = Felt::selector("TransferSingle"); + let transfer_batch_selector = Felt::selector("TransferBatch"); + let approval_for_all_selector = Felt::selector("ApprovalForAll"); + let uri_selector = Felt::selector("URI"); let mut has_transfer_single = false; let mut has_transfer_batch = false; @@ -451,12 +471,12 @@ mod tests { let mut extractor = SyntheticErc1155Extractor::new(cfg).unwrap(); let batch = extractor.extract(None).await.unwrap(); - let transfer_batch_selector = selector!("TransferBatch"); + let transfer_batch_selector = Felt::selector("TransferBatch"); let mut batch_sizes = Vec::new(); for event in &batch.events { if event.keys[0] == transfer_batch_selector { - let batch_size: usize = event.data[0].try_into().unwrap_or(0); + let batch_size = usize::try_from(event.data[0].to_le_words()[0]).unwrap_or(0); batch_sizes.push(batch_size); } } diff --git a/crates/torii-erc20/Cargo.toml b/crates/torii-erc20/Cargo.toml index aac69e0e..43d63788 100644 --- a/crates/torii-erc20/Cargo.toml +++ b/crates/torii-erc20/Cargo.toml @@ -6,47 +6,50 @@ description = "ERC20 token indexer library for Torii" [dependencies] # Torii core -torii = { path = "../.." } -torii-common = { path = "../torii-common" } +torii.workspace = true +torii-common.workspace = true +torii-types.workspace = true # Async -tokio = { version = "1", features = ["full"] } -async-trait = "0.1" +tokio = { workspace = true, features = ["full"] } +async-trait.workspace = true -# Starknet -starknet = "0.17" +starknet-types-raw.workspace = true +starknet.workspace = true # Storage -rusqlite = { version = "0.32", features = ["bundled"] } +rusqlite = { workspace = true } tokio-postgres = "0.7" +torii-sql = { workspace = true, features = ["postgres", "sqlite"] } # Serialization -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true # Encoding -hex = "0.4" +hex.workspace = true +primitive-types.workspace = true # gRPC/Protobuf -prost = "0.13" -prost-types = "0.13" -tonic = "0.12" -chrono = "0.4" +prost.workspace = true +prost-types.workspace = true +tonic.workspace = true +chrono.workspace = true # Streaming -async-stream = "0.3" -futures = "0.3" +async-stream.workspace = true +futures.workspace = true # Web framework -axum = "0.7" +axum.workspace = true # Utilities -anyhow = "1.0" -tracing = "0.1" -metrics = "0.24" +anyhow.workspace = true +tracing.workspace = true +metrics.workspace = true [build-dependencies] -tonic-build = "0.12" +tonic-build.workspace = true [lints] workspace = true diff --git a/crates/torii-erc20/README.md b/crates/torii-erc20/README.md new file mode 100644 index 00000000..75fb6491 --- /dev/null +++ b/crates/torii-erc20/README.md @@ -0,0 +1,145 @@ +# torii-erc20 + +End-to-end **ERC20 token indexer**. Ships a `Decoder` for Transfer / Approval +events, an `IdentificationRule` for ABI-based auto-discovery, a `Sink` that +persists the history + maintains balances with on-chain reconciliation, +and a gRPC service that answers queries and pushes live updates. A paired +synthetic extractor makes reproducible profiling workloads possible +without a live RPC endpoint. + +## Role in Torii + +Slotted into any binary as (a) an auto-identification rule registered with +the `ContractRegistry` so Torii learns which contracts are ERC20s, (b) the +`Erc20Decoder` that turns Transfer/Approval selectors into typed envelopes, +and (c) the `Erc20Sink` that writes the rows, keeps balances sane, and +serves the `torii.sinks.erc20.Erc20` gRPC service. Wires cleanly into +`torii::run` via `ToriiConfigBuilder::with_grpc_router` + +`with_contract_identifier`. + +## Architecture + +```text +StarknetEvent (Transfer/Approval selector) + | + v ++------------------------------+ +| Erc20Decoder | identification::Erc20Rule +| Transfer|Approval → Envelope| (ABI-based auto-discovery) +| ERC20_TYPE_ID | ++--------------+---------------+ + | + v ++------------------------------+ +| Erc20Sink | +| | +| process(): | +| - store transfer/approval | +| - compute balance delta | +| - detect negative | +| balance → enqueue | +| BalanceFetcher | +| - if batch.is_live(100): | +| EventBus publish | +| grpc broadcast | +| - dispatch | +| FetchErc20MetadataCmd | +| for unseen tokens | +| AtomicU64 counters | ++--------------+---------------+ + | + v ++------------------------------+ +| Erc20Storage | +| | +| erc20_transfer (cursor PK) | +| erc20_approval (cursor PK) | +| erc20_balance (token, acc,| +| amount U256)| +| erc20_balance_adjustment | +| erc20_metadata | +| | +| BLOB encoded via torii- | +| common::U256Blob / FeltBlob | ++--------------+---------------+ + | + v ++------------------------------+ +| Erc20Service | +| gRPC: torii.sinks.erc20 | +| cursor-paginated queries | +| broadcast subscriptions | ++------------------------------+ + ^ + | ++------------------------------+ +| Erc20MetadataCommandHandler| <-- CommandBus +| (fetches name/symbol/ | FetchErc20MetadataCommand +| decimals via JsonRpcClient| +| + torii-common::Metadata- | +| Fetcher) | ++------------------------------+ + +(optional) synthetic::SyntheticErc20Extractor feeds deterministic events +for benchmarking — impl SyntheticExtractor, fed via +ToriiConfigBuilder::with_synthetic_extractor +``` + +## Deep Dive + +### Public API + +| Item | File | Purpose | +|---|---|---| +| `Erc20Decoder`, `Transfer`, `Approval`, `Erc20Event`, `ERC20_TYPE_ID` | `src/decoder.rs` | Decoder impl + event variants + the `TypeId` sinks subscribe to | +| `Erc20Rule` | `src/identification.rs` | `IdentificationRule` — ABI must expose `transfer`, `balance_of`, and a `Transfer` event | +| `Erc20Sink` | `src/sink.rs` | `Sink` impl with fluent setters: `with_grpc_service`, `with_balance_tracking(provider)`, `with_metadata_pipeline(...)` | +| `Erc20Storage`, `TransferData`, `ApprovalData`, `BalanceData`, `BalanceAdjustment`, cursors | `src/storage.rs` | Persistence layer | +| `Erc20Service` | `src/grpc_service.rs` | gRPC queries + subscriptions | +| `BalanceFetcher`, `BalanceFetchRequest` | `src/balance_fetcher.rs` | Queued on-chain balance lookups (via JsonRpcClient) | +| `Erc20MetadataCommandHandler`, `FetchErc20MetadataCommand` | `src/handlers.rs` | `CommandHandler` that fetches name/symbol/decimals out of band | +| `SyntheticErc20Config`, `SyntheticErc20Extractor` | `src/synthetic.rs` | Deterministic generator for profiling | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs:53` | Descriptor bytes for reflection | + +### Internal Modules + +- `decoder` — matches the Transfer/Approval selectors, parses the felt payload, attaches block+tx metadata to the envelope. +- `identification` — `Erc20Rule`: returns `[DecoderId::new("erc20")]` when the ABI has the three required items. +- `sink` — the core `Sink` impl (see "Sink trait wiring" below). +- `storage` — sqlx-based CRUD over `erc20_transfer` / `erc20_approval` / `erc20_balance` / `erc20_balance_adjustment` / `erc20_metadata`. +- `balance_fetcher` — fetches on-chain `balance_of` results to reconcile negative-balance anomalies (genesis, airdrop). +- `handlers` — async metadata pipeline; the sink dispatches `FetchErc20MetadataCommand`s that the handler processes off the hot path. +- `grpc_service` — tonic service; cursor-paginated queries + `tokio::sync::broadcast` live updates. +- `synthetic` — `SyntheticErc20Extractor` implementing `SyntheticExtractor`. + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `"erc20"` | +| `interested_types` | `[ERC20_TYPE_ID]` | +| `process(envelopes, batch)` | Persists rows; updates the `AtomicU64` counters; updates balance; on negative balance queues a `BalanceFetcher` request. **Broadcasts to EventBus + gRPC only when `batch.is_live(LIVE_THRESHOLD_BLOCKS=100)`** — historical indexing stays silent to avoid flooding clients. Dispatches a metadata command once per new token. | +| `topics` | 3 topics: `erc20.transfer` (filters: `token`, `from`, `to`, `wallet` — OR semantics for `wallet`), `erc20.approval` (filters: `token`, `owner`, `spender`, `account`), `erc20.metadata` (filter: `token`) | +| `build_routes` | `Router::new()` — gRPC only | +| `initialize` | Captures `event_bus` and `ctx.command_bus` | + +### Storage + +- Amounts stored via `torii_common::u256_to_bytes` → **variable-length big-endian BLOB** (zero is `[0]`, no leading-zero padding). +- Addresses via `FeltBlob`. +- Cursors: `(block_number, event_idx)` monotonic pagination keys. +- Balances: optimistic — computed delta, fetched and adjusted on inconsistency. +- Audit: `erc20_balance_adjustment` records every reconciled value for traceability. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-erc20`, `bins/torii-erc20-synth`, `bins/torii-tokens`, `bins/torii-tokens-synth`, `bins/torii-introspect-bin`, `bins/torii-arcade` (any binary with ERC20 support). +- **Downstream deps**: `torii`, `torii-common`, `torii-types`, `sqlx`, `tonic`, `prost`, `starknet`, `starknet-types-raw`, `primitive-types`, `tokio`, `async-trait`, `chrono`, `hex`, `metrics`, `tracing`. + +### Extension Points + +- New event type → add a variant to `Erc20Event` + decoder branch + sink branch + proto message. +- New filter key → add to the relevant `TopicInfo.available_filters` + the matching `matches_*_filters` helper. +- New RPC → proto + `Erc20Service` + regenerate via `build.rs`. +- Different balance strategy → swap `BalanceFetcher` for a batched variant; `Erc20Sink::with_balance_tracking` is the only integration point. +- Benchmarking → compose `SyntheticErc20Extractor` with `ToriiConfigBuilder::with_synthetic_extractor` (see `bins/torii-erc20-synth`). diff --git a/crates/torii-erc20/src/balance_fetcher.rs b/crates/torii-erc20/src/balance_fetcher.rs index 83339202..530f3a18 100644 --- a/crates/torii-erc20/src/balance_fetcher.rs +++ b/crates/torii-erc20/src/balance_fetcher.rs @@ -7,11 +7,15 @@ use anyhow::{Context, Result}; /// Maximum number of requests per batch to avoid RPC limits const MAX_BATCH_SIZE: usize = 500; -use starknet::core::types::{requests::CallRequest, BlockId, Felt, FunctionCall, U256}; +use primitive_types::U256; +use starknet::core::types::requests::CallRequest; +use starknet::core::types::{BlockId, FunctionCall}; use starknet::macros::selector; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::sync::Arc; +use torii_common::utils::parse_u256_result; /// Request for fetching a balance at a specific block #[derive(Debug, Clone)] @@ -49,17 +53,19 @@ impl BalanceFetcher { block_number: u64, ) -> Result { let call = FunctionCall { - contract_address: token, + contract_address: token.into(), // TODO: Some contracts use balance_of (snake_case), others use balanceOf (camelCase). // We need a strategy to handle both - possibly try one, fallback to other, or use ABI. entry_point_selector: selector!("balanceOf"), - calldata: vec![wallet], + calldata: vec![wallet.into()], }; let block_id = BlockId::Number(block_number); match self.provider.call(call, block_id).await { - Ok(result) => Ok(parse_u256_result(&result)), + Ok(result) => Ok(parse_u256_result( + result.into_iter().map(Into::into).collect(), + )), Err(e) => { tracing::warn!( target: "torii_erc20::balance_fetcher", @@ -101,9 +107,9 @@ impl BalanceFetcher { .map(|req| { ProviderRequestData::Call(CallRequest { request: FunctionCall { - contract_address: req.token, + contract_address: req.token.into(), entry_point_selector: selector!("balanceOf"), - calldata: vec![req.wallet], + calldata: vec![req.wallet.into()], }, block_id: BlockId::Number(req.block_number), }) @@ -121,7 +127,7 @@ impl BalanceFetcher { for (idx, response) in responses.into_iter().enumerate() { let req = &chunk[idx]; let balance = if let ProviderResponseData::Call(felts) = response { - parse_u256_result(&felts) + parse_u256_result(felts.into_iter().map(Into::into).collect()) } else { tracing::warn!( target: "torii_erc20::balance_fetcher", @@ -145,67 +151,3 @@ impl BalanceFetcher { Ok(all_results) } } - -/// Parse a U256 result from balance_of return value -/// -/// ERC20 balance_of typically returns: -/// - Cairo 0: A single felt (fits in 252 bits, usually enough for balances) -/// - Cairo 1 with u256: Two felts [low, high] representing a 256-bit value -fn parse_u256_result(result: &[Felt]) -> U256 { - match result.len() { - 0 => U256::from(0u64), - 1 => { - // Single felt - convert to U256 - // Felt is 252 bits max, so it fits in the low part - let bytes = result[0].to_bytes_be(); - // Take the lower 16 bytes for u128 (fits any felt value) - let low = u128::from_be_bytes(bytes[16..32].try_into().unwrap()); - U256::from_words(low, 0) - } - _ => { - // Two felts: [low, high] for u256 - let low_bytes = result[0].to_bytes_be(); - let high_bytes = result[1].to_bytes_be(); - - let low = u128::from_be_bytes(low_bytes[16..32].try_into().unwrap()); - let high = u128::from_be_bytes(high_bytes[16..32].try_into().unwrap()); - - U256::from_words(low, high) - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_u256_empty() { - let result = parse_u256_result(&[]); - assert_eq!(result, U256::from(0u64)); - } - - #[test] - fn test_parse_u256_single_felt() { - let felt = Felt::from(1000u64); - let result = parse_u256_result(&[felt]); - assert_eq!(result, U256::from(1000u64)); - } - - #[test] - fn test_parse_u256_two_felts() { - // low = 100, high = 0 - let low = Felt::from(100u64); - let high = Felt::from(0u64); - let result = parse_u256_result(&[low, high]); - assert_eq!(result, U256::from(100u64)); - - // Test with high value - let low = Felt::from(0u64); - let high = Felt::from(1u64); - let result = parse_u256_result(&[low, high]); - // high = 1 means value = 1 * 2^128 - let expected = U256::from_words(0, 1); - assert_eq!(result, expected); - } -} diff --git a/crates/torii-erc20/src/decoder.rs b/crates/torii-erc20/src/decoder.rs index b54dcb79..aa26d235 100644 --- a/crates/torii-erc20/src/decoder.rs +++ b/crates/torii-erc20/src/decoder.rs @@ -2,12 +2,18 @@ use anyhow::Result; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt, U256}; -use starknet::macros::selector; +use primitive_types::U256; +use starknet_types_raw::event::EmittedEvent; +use starknet_types_raw::Felt; use std::any::Any; use std::collections::HashMap; -use torii::etl::{Decoder, Envelope, TypedBody}; +use torii::etl::{Decoder, Envelope, TypeId, TypedBody}; +use torii_common::utils::{felt_pair_to_u256, felt_to_u256}; +use torii_types::event::EventContext; +pub const ERC20_TYPE_ID: TypeId = torii::etl::envelope::TypeId::new("erc20"); +pub const TRANSFER_SELECTOR: Felt = Felt::selector("Transfer"); +pub const APPROVAL_SELECTOR: Felt = Felt::selector("Approval"); /// Transfer event from ERC20 token #[derive(Debug, Clone)] pub struct Transfer { @@ -20,20 +26,6 @@ pub struct Transfer { pub transaction_hash: Felt, } -impl TypedBody for Transfer { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc20.transfer") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - /// Approval event from ERC20 token #[derive(Debug, Clone)] pub struct Approval { @@ -46,9 +38,15 @@ pub struct Approval { pub transaction_hash: Felt, } -impl TypedBody for Approval { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc20.approval") +#[derive(Debug, Clone)] +pub enum Erc20Event { + Transfer(Transfer), + Approval(Approval), +} + +impl TypedBody for Erc20Event { + fn envelope_type_id(&self) -> TypeId { + ERC20_TYPE_ID } fn as_any(&self) -> &dyn Any { @@ -60,6 +58,37 @@ impl TypedBody for Approval { } } +impl Erc20Event { + fn event_id(&self) -> String { + match self { + Self::Transfer(transfer) => format!( + "erc20_transfer_{}_{:#x}", + transfer.block_number, transfer.transaction_hash + ), + Self::Approval(approval) => format!( + "erc20_approval_{}_{:#x}", + approval.block_number, approval.transaction_hash + ), + } + } + + fn into_envelope(self, event: &EmittedEvent) -> Envelope { + let envelope_id = self.event_id(); + let mut metadata = HashMap::new(); + metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); + metadata.insert( + "block_number".to_string(), + event.block_number.unwrap_or(0).to_string(), + ); + metadata.insert( + "tx_hash".to_string(), + format!("{:#x}", event.transaction_hash), + ); + + Envelope::new(envelope_id, Box::new(self), metadata) + } +} + /// ERC20 event decoder /// /// Decodes multiple ERC20 events: @@ -71,7 +100,7 @@ impl TypedBody for Approval { /// This decoder showcases the recommended pattern for handling multiple event types: /// 1. Check selector first (fast O(1) comparison) /// 2. Dispatch to specific decode_X() method -/// 3. Each method returns Result> (None if not interested/malformed) +/// 3. Each method returns Result> (None if malformed) /// 4. Main decode_event() collects results /// /// This pattern scales cleanly to many event types without complex branching. @@ -84,12 +113,43 @@ impl Erc20Decoder { /// Transfer event selector: sn_keccak("Transfer") fn transfer_selector() -> Felt { - selector!("Transfer") + TRANSFER_SELECTOR } /// Approval event selector: sn_keccak("Approval") fn approval_selector() -> Felt { - selector!("Approval") + APPROVAL_SELECTOR + } + + pub async fn decode(&self, event: &EmittedEvent) -> Result> { + if event.keys.is_empty() { + return Ok(Vec::new()); + } + + let selector = event.keys[0]; + + if selector == Self::transfer_selector() { + if let Some(event_body) = self.decode_transfer(event).await? { + return Ok(vec![event_body.into_envelope(event)]); + } + } else if selector == Self::approval_selector() { + if let Some(event_body) = self.decode_approval(event).await? { + return Ok(vec![event_body.into_envelope(event)]); + } + } else { + tracing::trace!( + target: "torii_erc20::decoder", + token = %format!("{:#x}", event.from_address), + selector = %format!("{:#x}", selector), + keys_len = event.keys.len(), + data_len = event.data.len(), + block_number = event.block_number.unwrap_or(0), + tx_hash = %format!("{:#x}", event.transaction_hash), + "Unhandled event selector" + ); + } + + Ok(Vec::new()) } /// Decode Transfer event into envelope @@ -119,7 +179,7 @@ impl Erc20Decoder { /// - data[3]: amount_high (u128) /// /// Felt-based variants use single felt for amount instead of U256. - async fn decode_transfer(&self, event: &EmittedEvent) -> Result> { + async fn decode_transfer(&self, event: &EmittedEvent) -> Result> { let from; let to; let amount: U256; @@ -128,9 +188,9 @@ impl Erc20Decoder { // All-in-keys format: selector, from, to, amount_low, amount_high all in keys from = event.keys[1]; to = event.keys[2]; - let low: u128 = event.keys[3].try_into().unwrap_or(0); - let high: u128 = event.keys[4].try_into().unwrap_or(0); - amount = U256::from_words(low, high); + let low: Felt = event.keys[3]; + let high: Felt = event.keys[4]; + amount = felt_pair_to_u256(low, high); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -143,8 +203,7 @@ impl Erc20Decoder { // All-in-keys format with felt amount: selector, from, to, amount in keys from = event.keys[1]; to = event.keys[2]; - let amount_felt: u128 = event.keys[3].try_into().unwrap_or(0); - amount = U256::from(amount_felt); + amount = felt_to_u256(event.keys[3]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -157,22 +216,17 @@ impl Erc20Decoder { // Legacy ERC20: from, to, amount_low, amount_high in data from = event.data[0]; to = event.data[1]; - let low: u128 = event.data[2].try_into().unwrap_or(0); - let high: u128 = event.data[3].try_into().unwrap_or(0); - amount = U256::from_words(low, high); + amount = felt_pair_to_u256(event.data[2], event.data[3]); } else if event.keys.len() == 3 && event.data.len() == 2 { // Modern ERC20: from, to in keys; amount_low, amount_high in data from = event.keys[1]; to = event.keys[2]; - let low: u128 = event.data[0].try_into().unwrap_or(0); - let high: u128 = event.data[1].try_into().unwrap_or(0); - amount = U256::from_words(low, high); + amount = felt_pair_to_u256(event.data[0], event.data[1]); } else if event.keys.len() == 3 && event.data.len() == 1 { // Felt-based modern format: from, to in keys; amount as single felt (fits in u128) from = event.keys[1]; to = event.keys[2]; - let amount_felt: u128 = event.data[0].try_into().unwrap_or(0); - amount = U256::from(amount_felt); + amount = felt_to_u256(event.data[0]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -185,8 +239,7 @@ impl Erc20Decoder { // Felt-based legacy format: from, to, amount in data from = event.data[0]; to = event.data[1]; - let amount_felt: u128 = event.data[2].try_into().unwrap_or(0); - amount = U256::from(amount_felt); + amount = felt_to_u256(event.data[2]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -199,7 +252,7 @@ impl Erc20Decoder { // All-in-keys format with zero amount (or amount omitted): from, to in keys, no data from = event.keys[1]; to = event.keys[2]; - amount = U256::from(0u64); + amount = U256::zero(); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -229,28 +282,7 @@ impl Erc20Decoder { transaction_hash: event.transaction_hash, }; - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); - - let envelope_id = format!( - "erc20_transfer_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); - - Ok(Some(Envelope::new( - envelope_id, - Box::new(transfer), - metadata, - ))) + Ok(Some(Erc20Event::Transfer(transfer))) } /// Decode Approval event into envelope @@ -280,7 +312,7 @@ impl Erc20Decoder { /// - data[3]: amount_high (u128) /// /// Felt-based variants use single felt for amount instead of U256. - async fn decode_approval(&self, event: &EmittedEvent) -> Result> { + async fn decode_approval(&self, event: &EmittedEvent) -> Result> { let owner; let spender; let amount: U256; @@ -289,9 +321,7 @@ impl Erc20Decoder { // All-in-keys format: selector, owner, spender, amount_low, amount_high all in keys owner = event.keys[1]; spender = event.keys[2]; - let low: u128 = event.keys[3].try_into().unwrap_or(0); - let high: u128 = event.keys[4].try_into().unwrap_or(0); - amount = U256::from_words(low, high); + amount = felt_pair_to_u256(event.keys[3], event.keys[4]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -304,8 +334,7 @@ impl Erc20Decoder { // All-in-keys format with felt amount: selector, owner, spender, amount in keys owner = event.keys[1]; spender = event.keys[2]; - let amount_felt: u128 = event.keys[3].try_into().unwrap_or(0); - amount = U256::from(amount_felt); + amount = felt_to_u256(event.keys[3]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -318,22 +347,17 @@ impl Erc20Decoder { // Legacy ERC20: owner, spender, amount_low, amount_high in data owner = event.data[0]; spender = event.data[1]; - let low: u128 = event.data[2].try_into().unwrap_or(0); - let high: u128 = event.data[3].try_into().unwrap_or(0); - amount = U256::from_words(low, high); + amount = felt_pair_to_u256(event.data[2], event.data[3]); } else if event.keys.len() == 3 && event.data.len() == 2 { // Modern ERC20: owner, spender in keys; amount_low, amount_high in data owner = event.keys[1]; spender = event.keys[2]; - let low: u128 = event.data[0].try_into().unwrap_or(0); - let high: u128 = event.data[1].try_into().unwrap_or(0); - amount = U256::from_words(low, high); + amount = felt_pair_to_u256(event.data[0], event.data[1]); } else if event.keys.len() == 3 && event.data.len() == 1 { // Felt-based modern format: owner, spender in keys; amount as single felt owner = event.keys[1]; spender = event.keys[2]; - let amount_felt: u128 = event.data[0].try_into().unwrap_or(0); - amount = U256::from(amount_felt); + amount = felt_to_u256(event.data[0]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -346,8 +370,7 @@ impl Erc20Decoder { // Felt-based legacy format: owner, spender, amount in data owner = event.data[0]; spender = event.data[1]; - let amount_felt: u128 = event.data[2].try_into().unwrap_or(0); - amount = U256::from(amount_felt); + amount = felt_to_u256(event.data[2]); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -360,7 +383,7 @@ impl Erc20Decoder { // All-in-keys format with zero amount (or amount omitted) owner = event.keys[1]; spender = event.keys[2]; - amount = U256::from(0u64); + amount = U256::zero(); tracing::debug!( target: "torii_erc20::decoder", token = %format!("{:#x}", event.from_address), @@ -390,28 +413,7 @@ impl Erc20Decoder { transaction_hash: event.transaction_hash, }; - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); - - let envelope_id = format!( - "erc20_approval_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); - - Ok(Some(Envelope::new( - envelope_id, - Box::new(approval), - metadata, - ))) + Ok(Some(Erc20Event::Approval(approval))) } } @@ -427,37 +429,22 @@ impl Decoder for Erc20Decoder { "erc20" } - async fn decode_event(&self, event: &EmittedEvent) -> Result> { - if event.keys.is_empty() { - return Ok(Vec::new()); - } - - let selector = event.keys[0]; - - if selector == Self::transfer_selector() { - if let Some(envelope) = self.decode_transfer(event).await? { - return Ok(vec![envelope]); - } - } else if selector == Self::approval_selector() { - if let Some(envelope) = self.decode_approval(event).await? { - return Ok(vec![envelope]); - } - } else { - // Log unhandled selectors to help identify missing event types. - // This is expected for contracts that emit other events besides ERC20 Transfer/Approval. - tracing::trace!( - target: "torii_erc20::decoder", - token = %format!("{:#x}", event.from_address), - selector = %format!("{:#x}", selector), - keys_len = event.keys.len(), - data_len = event.data.len(), - block_number = event.block_number.unwrap_or(0), - tx_hash = %format!("{:#x}", event.transaction_hash), - "Unhandled event selector" - ); - } + async fn decode( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> Result> { + let event = EmittedEvent { + from_address: context.from_address, + keys: keys.to_vec(), + data: data.to_vec(), + block_hash: None, + block_number: Some(context.block_number), + transaction_hash: context.transaction_hash, + }; - Ok(Vec::new()) + Erc20Decoder::decode(self, &event).await } } @@ -465,6 +452,22 @@ impl Decoder for Erc20Decoder { mod tests { use super::*; + fn transfer(envelope: &Envelope) -> &Transfer { + assert_eq!(envelope.type_id, ERC20_TYPE_ID); + match envelope.body.as_any().downcast_ref::().unwrap() { + Erc20Event::Transfer(transfer) => transfer, + Erc20Event::Approval(_) => panic!("expected transfer event"), + } + } + + fn approval(envelope: &Envelope) -> &Approval { + assert_eq!(envelope.type_id, ERC20_TYPE_ID); + match envelope.body.as_any().downcast_ref::().unwrap() { + Erc20Event::Approval(approval) => approval, + Erc20Event::Transfer(_) => panic!("expected approval event"), + } + } + #[tokio::test] async fn test_decode_transfer() { let decoder = Erc20Decoder::new(); @@ -485,14 +488,10 @@ mod tests { transaction_hash: Felt::from(0xabcdu64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let transfer = transfer(&envelopes[0]); assert_eq!(transfer.from, Felt::from(0x1u64)); assert_eq!(transfer.to, Felt::from(0x2u64)); @@ -520,14 +519,10 @@ mod tests { transaction_hash: Felt::from(0xdef0u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let approval = approval(&envelopes[0]); assert_eq!(approval.owner, Felt::from(0xau64)); assert_eq!(approval.spender, Felt::from(0xbu64)); @@ -551,7 +546,7 @@ mod tests { transaction_hash: Felt::from(0x1234u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 0); // Should return empty vec for unknown events } @@ -575,14 +570,10 @@ mod tests { transaction_hash: Felt::from(0xabcdu64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let transfer = transfer(&envelopes[0]); assert_eq!(transfer.from, Felt::from(0x1u64)); assert_eq!(transfer.to, Felt::from(0x2u64)); @@ -608,14 +599,10 @@ mod tests { transaction_hash: Felt::from(0xef01u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let transfer = transfer(&envelopes[0]); assert_eq!(transfer.from, Felt::from(0xau64)); assert_eq!(transfer.to, Felt::from(0xbu64)); @@ -642,14 +629,10 @@ mod tests { transaction_hash: Felt::from(0xabc1u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let approval = approval(&envelopes[0]); assert_eq!(approval.owner, Felt::from(0xcu64)); assert_eq!(approval.spender, Felt::from(0xdu64)); @@ -677,14 +660,10 @@ mod tests { transaction_hash: Felt::from(0xdef2u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let approval = approval(&envelopes[0]); assert_eq!(approval.owner, Felt::from(0xeu64)); assert_eq!(approval.spender, Felt::from(0xfu64)); @@ -710,14 +689,10 @@ mod tests { transaction_hash: Felt::from(0xef03u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let approval = approval(&envelopes[0]); assert_eq!(approval.owner, Felt::from(0x10u64)); assert_eq!(approval.spender, Felt::from(0x11u64)); @@ -745,14 +720,10 @@ mod tests { transaction_hash: Felt::from(0xfff1u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let transfer = transfer(&envelopes[0]); assert_eq!(transfer.from, Felt::from(0x20u64)); assert_eq!(transfer.to, Felt::from(0x21u64)); @@ -779,14 +750,10 @@ mod tests { transaction_hash: Felt::from(0xfff2u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let transfer = transfer(&envelopes[0]); assert_eq!(transfer.from, Felt::from(0x30u64)); assert_eq!(transfer.to, Felt::from(0x31u64)); @@ -814,14 +781,10 @@ mod tests { transaction_hash: Felt::from(0xfff3u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let approval = approval(&envelopes[0]); assert_eq!(approval.owner, Felt::from(0x40u64)); assert_eq!(approval.spender, Felt::from(0x41u64)); @@ -848,14 +811,10 @@ mod tests { transaction_hash: Felt::from(0xfff4u64), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); + let envelopes = decoder.decode(&event).await.unwrap(); assert_eq!(envelopes.len(), 1); - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); + let approval = approval(&envelopes[0]); assert_eq!(approval.owner, Felt::from(0x50u64)); assert_eq!(approval.spender, Felt::from(0x51u64)); diff --git a/crates/torii-erc20/src/grpc_service.rs b/crates/torii-erc20/src/grpc_service.rs index 8d2bc4e6..fbdcae21 100644 --- a/crates/torii-erc20/src/grpc_service.rs +++ b/crates/torii-erc20/src/grpc_service.rs @@ -5,25 +5,26 @@ //! - Real-time subscriptions with filtering (SubscribeTransfers, SubscribeApprovals) //! - Indexer statistics (GetStats) +use crate::proto::erc20_server::Erc20 as Erc20Trait; use crate::proto::{ - erc20_server::Erc20 as Erc20Trait, Approval, ApprovalFilter, ApprovalUpdate, BalanceEntry, - Cursor, GetApprovalsRequest, GetApprovalsResponse, GetBalanceRequest, GetBalanceResponse, - GetBalancesRequest, GetBalancesResponse, GetStatsRequest, GetStatsResponse, - GetTokenMetadataRequest, GetTokenMetadataResponse, GetTransfersRequest, GetTransfersResponse, - SubscribeApprovalsRequest, SubscribeTransfersRequest, TokenMetadataEntry, Transfer, - TransferFilter, TransferUpdate, + Approval, ApprovalFilter, ApprovalUpdate, BalanceEntry, Cursor, GetApprovalsRequest, + GetApprovalsResponse, GetBalanceRequest, GetBalanceResponse, GetBalancesRequest, + GetBalancesResponse, GetStatsRequest, GetStatsResponse, GetTokenMetadataRequest, + GetTokenMetadataResponse, GetTransfersRequest, GetTransfersResponse, SubscribeApprovalsRequest, + SubscribeTransfersRequest, TokenMetadataEntry, Transfer, TransferFilter, TransferUpdate, }; use crate::storage::{ ApprovalCursor, ApprovalData, Erc20Storage, TransferCursor, TransferData, TransferDirection, }; use async_trait::async_trait; use futures::stream::Stream; -use starknet::core::types::Felt; +use primitive_types::U256; +use starknet_types_raw::Felt; use std::pin::Pin; use std::sync::Arc; use tokio::sync::broadcast; use tonic::{Request, Response, Status}; -use torii_common::{bytes_to_felt, u256_to_bytes}; +use torii_common::u256_to_bytes; /// gRPC service implementation for ERC20 #[derive(Clone)] @@ -72,12 +73,12 @@ impl Erc20Service { /// Convert storage TransferData to proto Transfer fn transfer_data_to_proto(data: &TransferData) -> Transfer { Transfer { - token: data.token.to_bytes_be().to_vec(), - from: data.from.to_bytes_be().to_vec(), - to: data.to.to_bytes_be().to_vec(), + token: data.token.to_be_bytes_vec(), + from: data.from.to_be_bytes_vec(), + to: data.to.to_be_bytes_vec(), amount: u256_to_bytes(data.amount), block_number: data.block_number, - tx_hash: data.tx_hash.to_bytes_be().to_vec(), + tx_hash: data.tx_hash.to_be_bytes_vec(), timestamp: data.timestamp.unwrap_or(0), } } @@ -85,12 +86,12 @@ impl Erc20Service { /// Convert storage ApprovalData to proto Approval fn approval_data_to_proto(data: &ApprovalData) -> Approval { Approval { - token: data.token.to_bytes_be().to_vec(), - owner: data.owner.to_bytes_be().to_vec(), - spender: data.spender.to_bytes_be().to_vec(), + token: data.token.to_be_bytes_vec(), + owner: data.owner.to_be_bytes_vec(), + spender: data.spender.to_be_bytes_vec(), amount: u256_to_bytes(data.amount), block_number: data.block_number, - tx_hash: data.tx_hash.to_bytes_be().to_vec(), + tx_hash: data.tx_hash.to_be_bytes_vec(), timestamp: data.timestamp.unwrap_or(0), } } @@ -213,13 +214,22 @@ impl Erc20Trait for Erc20Service { let filter = req.filter.unwrap_or_default(); // Parse filter fields - let wallet = filter.wallet.as_ref().and_then(|b| bytes_to_felt(b)); - let from = filter.from.as_ref().and_then(|b| bytes_to_felt(b)); - let to = filter.to.as_ref().and_then(|b| bytes_to_felt(b)); + let wallet = filter + .wallet + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); + let from = filter + .from + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); + let to = filter + .to + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); let tokens: Vec = filter .tokens .iter() - .filter_map(|b| bytes_to_felt(b)) + .filter_map(|b| Felt::from_be_bytes_slice(b).ok()) .collect(); let direction = match crate::proto::TransferDirection::try_from(filter.direction) { @@ -298,13 +308,22 @@ impl Erc20Trait for Erc20Service { let filter = req.filter.unwrap_or_default(); // Parse filter fields - let account = filter.account.as_ref().and_then(|b| bytes_to_felt(b)); - let owner = filter.owner.as_ref().and_then(|b| bytes_to_felt(b)); - let spender = filter.spender.as_ref().and_then(|b| bytes_to_felt(b)); + let account = filter + .account + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); + let owner = filter + .owner + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); + let spender = filter + .spender + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); let tokens: Vec = filter .tokens .iter() - .filter_map(|b| bytes_to_felt(b)) + .filter_map(|b| Felt::from_be_bytes_slice(b).ok()) .collect(); // Parse cursor @@ -374,10 +393,10 @@ impl Erc20Trait for Erc20Service { ) -> Result, Status> { let req = request.into_inner(); - let token = bytes_to_felt(&req.token) - .ok_or_else(|| Status::invalid_argument("Invalid token address"))?; - let wallet = bytes_to_felt(&req.wallet) - .ok_or_else(|| Status::invalid_argument("Invalid wallet address"))?; + let token = Felt::from_be_bytes_slice(&req.token) + .map_err(|_| Status::invalid_argument("Invalid token address"))?; + let wallet = Felt::from_be_bytes_slice(&req.wallet) + .map_err(|_| Status::invalid_argument("Invalid wallet address"))?; tracing::debug!( target: "torii_erc20::grpc", @@ -391,7 +410,7 @@ impl Erc20Trait for Erc20Service { .get_balance_with_block(token, wallet) .await .map_err(|e| Status::internal(format!("Query failed: {e}")))? - .unwrap_or((starknet::core::types::U256::from(0u64), 0)); + .unwrap_or((U256::zero(), 0)); Ok(Response::new(GetBalanceResponse { balance: u256_to_bytes(balance), @@ -406,8 +425,14 @@ impl Erc20Trait for Erc20Service { ) -> Result, Status> { let req = request.into_inner(); - let token = req.token.as_ref().and_then(|b| bytes_to_felt(b)); - let wallet = req.wallet.as_ref().and_then(|b| bytes_to_felt(b)); + let token = req + .token + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); + let wallet = req + .wallet + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); let cursor = req.cursor; let limit = if req.limit == 0 { 1000 @@ -433,8 +458,8 @@ impl Erc20Trait for Erc20Service { let rows = balances .into_iter() .map(|b| BalanceEntry { - token: b.token.to_bytes_be().to_vec(), - wallet: b.wallet.to_bytes_be().to_vec(), + token: b.token.to_be_bytes_vec(), + wallet: b.wallet.to_be_bytes_vec(), balance: u256_to_bytes(b.balance), last_block: b.last_block, }) @@ -454,12 +479,12 @@ impl Erc20Trait for Erc20Service { let req = request.into_inner(); if let Some(token_bytes) = req.token { - let token = bytes_to_felt(&token_bytes) - .ok_or_else(|| Status::invalid_argument("Invalid token address"))?; + let token = Felt::from_be_bytes_slice(&token_bytes) + .map_err(|_| Status::invalid_argument("Invalid token address"))?; let entries = match self.storage.get_token_metadata(token).await { Ok(Some((name, symbol, decimals, total_supply))) => vec![TokenMetadataEntry { - token: token.to_bytes_be().to_vec(), + token: token.to_be_bytes_vec(), name, symbol, decimals: decimals.map(|d| d as u32), @@ -475,7 +500,10 @@ impl Erc20Trait for Erc20Service { })); } - let cursor = req.cursor.as_ref().and_then(|b| bytes_to_felt(b)); + let cursor = req + .cursor + .as_ref() + .and_then(|b| Felt::from_be_bytes_slice(b).ok()); let limit = if req.limit == 0 { 100 } else { @@ -492,7 +520,7 @@ impl Erc20Trait for Erc20Service { .into_iter() .map( |(token, name, symbol, decimals, total_supply)| TokenMetadataEntry { - token: token.to_bytes_be().to_vec(), + token: token.to_be_bytes_vec(), name, symbol, decimals: decimals.map(|d| d as u32), @@ -503,7 +531,7 @@ impl Erc20Trait for Erc20Service { Ok(Response::new(GetTokenMetadataResponse { tokens: entries, - next_cursor: next_cursor.map(|c| c.to_bytes_be().to_vec()), + next_cursor: next_cursor.map(|c| c.to_be_bytes_vec()), })) } diff --git a/crates/torii-erc20/src/handlers.rs b/crates/torii-erc20/src/handlers.rs index cc4b4f8a..3fbd03dc 100644 --- a/crates/torii-erc20/src/handlers.rs +++ b/crates/torii-erc20/src/handlers.rs @@ -2,8 +2,8 @@ use anyhow::Result; use async_trait::async_trait; use prost::Message; use prost_types::Any; -use starknet::core::types::Felt; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; +use starknet_types_raw::Felt; use std::collections::HashSet; use std::sync::{Arc, Mutex}; use torii::command::CommandHandler; @@ -112,7 +112,7 @@ impl CommandHandler for Erc20MetadataCommandHandler { let event_bus = self.event_bus.lock().unwrap().clone(); if let Some(event_bus) = event_bus { let meta_entry = proto::TokenMetadataEntry { - token: command.token.to_bytes_be().to_vec(), + token: command.token.to_be_bytes_vec(), name: meta.name, symbol: meta.symbol, decimals: meta.decimals.map(|d| d as u32), diff --git a/crates/torii-erc20/src/identification.rs b/crates/torii-erc20/src/identification.rs index 2868ba3c..98ea750f 100644 --- a/crates/torii-erc20/src/identification.rs +++ b/crates/torii-erc20/src/identification.rs @@ -6,7 +6,7 @@ //! - `Transfer` event use anyhow::Result; -use starknet::core::types::Felt; +use starknet_types_raw::Felt; use torii::etl::decoder::DecoderId; use torii::etl::extractor::ContractAbi; use torii::etl::identification::IdentificationRule; diff --git a/crates/torii-erc20/src/lib.rs b/crates/torii-erc20/src/lib.rs index 70e2b54c..eb596175 100644 --- a/crates/torii-erc20/src/lib.rs +++ b/crates/torii-erc20/src/lib.rs @@ -19,7 +19,7 @@ //! use torii_erc20::proto::erc20_server::Erc20Server; //! //! // Create storage -//! let storage = Arc::new(Erc20Storage::new("./erc20.db")?); +//! let storage = Arc::new(Erc20Storage::new("./erc20.db").await?); //! //! // Create gRPC service //! let grpc_service = Erc20Service::new(storage.clone()); @@ -54,7 +54,7 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!("generated/erc20_descripto // Re-export main types for convenience pub use balance_fetcher::{BalanceFetchRequest, BalanceFetcher}; -pub use decoder::{Approval, Erc20Decoder, Transfer}; +pub use decoder::{Approval, Erc20Decoder, Erc20Event, Transfer, ERC20_TYPE_ID}; pub use grpc_service::Erc20Service; pub use handlers::Erc20MetadataCommandHandler; pub use identification::Erc20Rule; diff --git a/crates/torii-erc20/src/sink.rs b/crates/torii-erc20/src/sink.rs index 4a09e295..65534c0e 100644 --- a/crates/torii-erc20/src/sink.rs +++ b/crates/torii-erc20/src/sink.rs @@ -12,7 +12,7 @@ //! fetches the actual balance from the chain and adjusts use crate::balance_fetcher::BalanceFetcher; -use crate::decoder::{Approval as DecodedApproval, Transfer as DecodedTransfer}; +use crate::decoder::{Erc20Event, ERC20_TYPE_ID}; use crate::grpc_service::Erc20Service; use crate::handlers::FetchErc20MetadataCommand; use crate::proto; @@ -20,10 +20,11 @@ use crate::storage::{ApprovalData, Erc20Storage, TransferData}; use anyhow::Result; use async_trait::async_trait; use axum::Router; +use primitive_types::U256; use prost::Message; use prost_types::Any; -use starknet::core::types::{Felt, U256}; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; +use starknet_types_raw::Felt; use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -217,7 +218,7 @@ impl Sink for Erc20Sink { } fn interested_types(&self) -> Vec { - vec![TypeId::new("erc20.transfer"), TypeId::new("erc20.approval")] + vec![ERC20_TYPE_ID] } async fn initialize( @@ -245,36 +246,38 @@ impl Sink for Erc20Sink { .collect(); for envelope in envelopes { - // Handle transfers - if envelope.type_id == TypeId::new("erc20.transfer") { - if let Some(transfer) = envelope.body.as_any().downcast_ref::() { - let timestamp = block_timestamps.get(&transfer.block_number).copied(); - transfers.push(TransferData { - id: None, - token: transfer.token, - from: transfer.from, - to: transfer.to, - amount: transfer.amount, - block_number: transfer.block_number, - tx_hash: transfer.transaction_hash, - timestamp, - }); - } + if envelope.type_id != ERC20_TYPE_ID { + continue; } - // Handle approvals - else if envelope.type_id == TypeId::new("erc20.approval") { - if let Some(approval) = envelope.body.as_any().downcast_ref::() { - let timestamp = block_timestamps.get(&approval.block_number).copied(); - approvals.push(ApprovalData { - id: None, - token: approval.token, - owner: approval.owner, - spender: approval.spender, - amount: approval.amount, - block_number: approval.block_number, - tx_hash: approval.transaction_hash, - timestamp, - }); + + if let Some(event) = envelope.body.as_any().downcast_ref::() { + match event { + Erc20Event::Transfer(transfer) => { + let timestamp = block_timestamps.get(&transfer.block_number).copied(); + transfers.push(TransferData { + id: None, + token: transfer.token, + from: transfer.from, + to: transfer.to, + amount: transfer.amount, + block_number: transfer.block_number, + tx_hash: transfer.transaction_hash, + timestamp, + }); + } + Erc20Event::Approval(approval) => { + let timestamp = block_timestamps.get(&approval.block_number).copied(); + approvals.push(ApprovalData { + id: None, + token: approval.token, + owner: approval.owner, + spender: approval.spender, + amount: approval.amount, + block_number: approval.block_number, + tx_hash: approval.transaction_hash, + timestamp, + }); + } } } } @@ -449,12 +452,12 @@ impl Sink for Erc20Sink { // Publish transfer events for transfer in &transfers { let proto_transfer = proto::Transfer { - token: transfer.token.to_bytes_be().to_vec(), - from: transfer.from.to_bytes_be().to_vec(), - to: transfer.to.to_bytes_be().to_vec(), + token: transfer.token.to_be_bytes_vec(), + from: transfer.from.to_be_bytes_vec(), + to: transfer.to.to_be_bytes_vec(), amount: u256_to_bytes(transfer.amount), block_number: transfer.block_number, - tx_hash: transfer.tx_hash.to_bytes_be().to_vec(), + tx_hash: transfer.tx_hash.to_be_bytes_vec(), timestamp: transfer.timestamp.unwrap_or(0), }; @@ -522,12 +525,12 @@ impl Sink for Erc20Sink { // Publish approval events for approval in &approvals { let proto_approval = proto::Approval { - token: approval.token.to_bytes_be().to_vec(), - owner: approval.owner.to_bytes_be().to_vec(), - spender: approval.spender.to_bytes_be().to_vec(), + token: approval.token.to_be_bytes_vec(), + owner: approval.owner.to_be_bytes_vec(), + spender: approval.spender.to_be_bytes_vec(), amount: u256_to_bytes(approval.amount), block_number: approval.block_number, - tx_hash: approval.tx_hash.to_bytes_be().to_vec(), + tx_hash: approval.tx_hash.to_be_bytes_vec(), timestamp: approval.timestamp.unwrap_or(0), }; diff --git a/crates/torii-erc20/src/storage.rs b/crates/torii-erc20/src/storage.rs index 82b0296e..aa50ecd9 100644 --- a/crates/torii-erc20/src/storage.rs +++ b/crates/torii-erc20/src/storage.rs @@ -10,19 +10,19 @@ //! - Records all adjustments in an audit table for debugging use anyhow::Result; +use primitive_types::U256; use rusqlite::{params, params_from_iter, Connection, ToSql}; -use starknet::core::types::{Felt, U256}; +use starknet_types_raw::Felt; use std::collections::{HashMap, HashSet, VecDeque}; -use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use tokio_postgres::types::ToSql as PgToSql; use tokio_postgres::{Client, NoTls}; use torii_common::{blob_to_felt, blob_to_u256, felt_to_blob, u256_to_blob}; +use torii_sql::{DbPool, DbPoolOptions}; use crate::balance_fetcher::BalanceFetchRequest; /// Maximum value for U256 (2^256 - 1) -const U256_MAX: U256 = U256::from_words(u128::MAX, u128::MAX); const SQLITE_MAX_BIND_VARS: usize = 900; const SQLITE_TOKEN_WALLET_QUERY_CHUNK: usize = SQLITE_MAX_BIND_VARS - 1; const SQLITE_ACTIVITY_INSERT_CHUNK: usize = SQLITE_MAX_BIND_VARS / 5; @@ -30,6 +30,21 @@ const SQLITE_BALANCE_UPSERT_CHUNK: usize = SQLITE_MAX_BIND_VARS / 5; const SQLITE_ADJUSTMENT_INSERT_CHUNK: usize = SQLITE_MAX_BIND_VARS / 6; const DEFAULT_BALANCE_CACHE_CAPACITY: usize = 300_000; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum StorageBackend { + Sqlite, + Postgres, +} + +/// Storage for ERC20 token data. +pub struct Erc20Storage { + pool: DbPool, + backend: StorageBackend, + conn: Arc>, + balance_cache: Arc>, + pg_conn: Option>>, +} + #[derive(Debug)] struct BalanceCacheState { enabled: bool, @@ -108,6 +123,21 @@ impl BalanceCacheState { } } +fn db_pool_url(db_path: &str) -> String { + if db_path == ":memory:" || db_path == "sqlite::memory:" { + return "sqlite::memory:".to_owned(); + } + + if db_path.starts_with("postgres://") + || db_path.starts_with("postgresql://") + || db_path.starts_with("sqlite:") + { + return db_path.to_owned(); + } + + format!("sqlite://{db_path}?mode=rwc") +} + struct ActivityInsertRow { account: Vec, token: Vec, @@ -146,15 +176,15 @@ struct AdjustmentInsertRow { /// - Accumulation of many transfers to the same address fn safe_u256_add(a: U256, b: U256) -> U256 { // Check if addition would overflow - // If a > U256_MAX - b, then a + b would overflow - let max_minus_b = U256_MAX - b; + // If a > U256::MAX - b, then a + b would overflow + let max_minus_b = U256::MAX - b; if a > max_minus_b { tracing::warn!( "U256 addition overflow detected: {} + {} would exceed U256::MAX, capping at maximum", a, b ); - U256_MAX + U256::MAX } else { a + b } @@ -172,21 +202,6 @@ pub enum TransferDirection { Received, } -/// Storage for ERC20 transfers and approvals -pub struct Erc20Storage { - backend: StorageBackend, - conn: Arc>, - balance_cache: Arc>, - pg_conns: Option>>>, - pg_rr: AtomicUsize, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum StorageBackend { - Sqlite, - Postgres, -} - /// Transfer data for batch insertion pub struct TransferData { pub id: Option, @@ -319,33 +334,32 @@ impl Erc20Storage { ::metrics::gauge!("torii_erc20_balance_cache_size").set(cache.size() as f64); } + pub fn pool(&self) -> &DbPool { + &self.pool + } + + pub async fn connect_pool(db_path: &str) -> Result { + Ok(DbPoolOptions::new() + .connect_any(&db_pool_url(db_path)) + .await?) + } + /// Create or open the database pub async fn new(db_path: &str) -> Result { + let pool = Self::connect_pool(db_path).await?; + Self::from_pool(db_path, pool).await + } + + pub async fn from_pool(db_path: &str, pool: DbPool) -> Result { let balance_cache = Self::build_balance_cache(); - if db_path.starts_with("postgres://") || db_path.starts_with("postgresql://") { - let pool_size = std::env::var("TORII_ERC20_PG_POOL_SIZE") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(8) - .max(1); - - let mut pg_conns = Vec::with_capacity(pool_size); - - for _ in 0..pool_size { - let (client, connection) = tokio_postgres::connect(db_path, NoTls).await?; - tokio::spawn(async move { - if let Err(e) = connection.await { - tracing::error!(target: "torii_erc20::storage", error = %e, "PostgreSQL connection task failed"); - } - }); - pg_conns.push(Arc::new(tokio::sync::Mutex::new(client))); - } + if matches!(&pool, DbPool::Postgres(_)) { + let (client, connection) = tokio_postgres::connect(db_path, NoTls).await?; + tokio::spawn(async move { + if let Err(e) = connection.await { + tracing::error!(target: "torii_erc20::storage", error = %e, "PostgreSQL connection task failed"); + } + }); - let schema_client = pg_conns - .first() - .expect("PostgreSQL connection pool must contain at least one client") - .clone(); - let client = schema_client.lock().await; client .batch_execute( r" @@ -446,13 +460,13 @@ impl Erc20Storage { ) .await?; - tracing::info!(target: "torii_erc20::storage", pool_size, "PostgreSQL storage initialized"); + tracing::info!(target: "torii_erc20::storage", "PostgreSQL storage initialized"); return Ok(Self { + pool, backend: StorageBackend::Postgres, conn: Arc::new(Mutex::new(Connection::open_in_memory()?)), balance_cache, - pg_conns: Some(pg_conns), - pg_rr: AtomicUsize::new(0), + pg_conn: Some(Arc::new(tokio::sync::Mutex::new(client))), }); } @@ -737,11 +751,11 @@ impl Erc20Storage { tracing::info!(target: "torii_erc20::storage", db_path = %db_path, "Database initialized"); Ok(Self { + pool, backend: StorageBackend::Sqlite, conn: Arc::new(Mutex::new(conn)), balance_cache, - pg_conns: None, - pg_rr: AtomicUsize::new(0), + pg_conn: None, }) } @@ -2181,12 +2195,11 @@ impl Erc20Storage { } async fn pg_client(&self) -> Result> { - let conns = self - .pg_conns + let conn = self + .pg_conn .as_ref() - .ok_or_else(|| anyhow::anyhow!("PostgreSQL connections not initialized"))?; - let idx = self.pg_rr.fetch_add(1, Ordering::Relaxed) % conns.len(); - Ok(conns[idx].lock().await) + .ok_or_else(|| anyhow::anyhow!("PostgreSQL connection not initialized"))?; + Ok(conn.lock().await) } fn pg_next_param( diff --git a/crates/torii-erc20/src/synthetic.rs b/crates/torii-erc20/src/synthetic.rs index e0032323..7d9798eb 100644 --- a/crates/torii-erc20/src/synthetic.rs +++ b/crates/torii-erc20/src/synthetic.rs @@ -5,9 +5,11 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; -use starknet::macros::selector; +use starknet_types_raw::Felt; use torii::etl::extractor::{ExtractionBatch, SyntheticExtractor}; +use torii::etl::StarknetEvent; + +use crate::decoder::{APPROVAL_SELECTOR, TRANSFER_SELECTOR}; const EXTRACTOR_NAME: &str = "synthetic_erc20"; @@ -160,23 +162,21 @@ impl SyntheticErc20Extractor { let amount_low = self.amount_low_for(block_number, tx_index); let event = if self.is_approval(tx_index) { - EmittedEvent { - from_address: token, - keys: vec![selector!("Approval"), from, to], - data: vec![amount_low, Felt::ZERO], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - } + StarknetEvent::new( + token, + vec![APPROVAL_SELECTOR, from, to], + vec![amount_low, Felt::ZERO], + block_number, + tx_hash, + ) } else { - EmittedEvent { - from_address: token, - keys: vec![selector!("Transfer"), from, to], - data: vec![amount_low, Felt::ZERO], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - } + StarknetEvent::new( + token, + vec![TRANSFER_SELECTOR, from, to], + vec![amount_low, Felt::ZERO], + block_number, + tx_hash, + ) }; batch.add_event_with_tx_context( event, diff --git a/crates/torii-erc721/Cargo.toml b/crates/torii-erc721/Cargo.toml index 078e257f..87298e96 100644 --- a/crates/torii-erc721/Cargo.toml +++ b/crates/torii-erc721/Cargo.toml @@ -8,6 +8,7 @@ description = "ERC721 NFT token indexer library for Torii" # Torii core torii = { path = "../.." } torii-common = { path = "../torii-common" } +torii-sql = { path = "../sql", features = ["postgres", "sqlite"] } # Async tokio = { version = "1", features = ["full"] } @@ -15,6 +16,8 @@ async-trait = "0.1" # Starknet starknet = "0.17" +starknet-types-raw.workspace = true +primitive-types.workspace = true # Storage rusqlite = { version = "0.32", features = ["bundled"] } diff --git a/crates/torii-erc721/README.md b/crates/torii-erc721/README.md new file mode 100644 index 00000000..bc07a246 --- /dev/null +++ b/crates/torii-erc721/README.md @@ -0,0 +1,133 @@ +# torii-erc721 + +End-to-end **ERC721 NFT indexer**. Decodes Transfer / Approval / +ApprovalForAll / MetadataUpdate / BatchMetadataUpdate (EIP-4906) events, +tracks per-token ownership in a dedicated ownership table, maintains +token-URI metadata out of band, and exposes a gRPC service for queries and +live updates. + +## Role in Torii + +Mirror of `torii-erc20` but for non-fungible tokens. The big shape +differences: balance tracking is replaced with an explicit `ownership` +row per `(token, token_id)`, metadata has a richer lifecycle because +individual tokens can change URI, and the `Erc721Rule` uses the presence +of `owner_of` (not `balance_of`) to distinguish ERC721 from ERC20. + +## Architecture + +```text +StarknetEvent (ERC721 selector) + | + v ++--------------------------------+ +| Erc721Decoder | identification::Erc721Rule +| Transfer | Approval | | (ABI needs owner_of + Transfer evt) +| ApprovalForAll | | +| MetadataUpdate | | +| BatchMetadataUpdate | +| → Envelope(ERC721_TYPE) | ++---------------+----------------+ + | + v ++--------------------------------+ +| Erc721Sink | +| | +| process(): | +| - store transfer / approval | +| - upsert ownership | +| - MetadataUpdate → enqueue | +| Erc721TokenUriCommand | +| via command bus | +| - if batch.is_live(100): | +| EventBus publish | +| gRPC broadcast | ++---------------+----------------+ + | + v ++--------------------------------+ +| Erc721Storage | +| erc721_transfer (cursor) | +| erc721_ownership | +| erc721_approval | +| erc721_operator_approval | +| erc721_metadata | +| erc721_token_uri | ++---------------+----------------+ + | + v ++--------------------------------+ +| Erc721Service | +| gRPC: torii.sinks.erc721 | ++--------------------------------+ + ^ + | ++--------------------------------+ +| Erc721MetadataCommandHandler | <- CommandBus +| Erc721TokenUriCommandHandler | (name/symbol, +| (uses torii-common:: | token URIs, +| MetadataFetcher + | per-token meta) +| TokenUriService) | ++--------------------------------+ +``` + +## Deep Dive + +### Public API + +| Item | File | Purpose | +|---|---|---| +| `Erc721Decoder`, `Erc721Msg`, `Erc721Body`, `ERC721_TYPE` | `src/decoder.rs` | Decoder + event enum + wrapper body | +| `NftTransfer`, `NftApproval`, `OperatorApproval` | `src/decoder.rs` | Transfer + per-token Approval + ApprovalForAll | +| `MetadataUpdate`, `BatchMetadataUpdate` | `src/decoder.rs` | EIP-4906 metadata invalidation events | +| `Erc721Rule` | `src/identification.rs` | ABI-based identification (requires `owner_of`) | +| `Erc721Sink` | `src/sink.rs` | `Sink` impl with `with_grpc_service`, `with_token_uri_pipeline`, etc. | +| `Erc721Storage`, `NftOwnershipData`, `NftTransferData`, `TransferCursor` | `src/storage.rs` | Persistence + ownership | +| `Erc721Service` | `src/grpc_service.rs` | tonic service | +| `Erc721MetadataCommandHandler`, `Erc721TokenUriCommandHandler` | `src/handlers.rs` | `CommandHandler` implementations | +| `SyntheticErc721Config`, `SyntheticErc721Extractor` | `src/synthetic.rs` | Deterministic generator | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs:53` | Descriptor bytes | + +### Internal Modules + +- `decoder` — matches Transfer / Approval / ApprovalForAll / MetadataUpdate / BatchMetadataUpdate. Token IDs are `U256`, stored via `torii-common::U256Blob`. +- `identification` — `Erc721Rule`: requires `owner_of` + `balance_of` + `Transfer` event. Keeps ERC721 and ERC20 distinct under auto-identification (ERC20 has `balance_of` but no `owner_of`). +- `sink` — writes the row, updates ownership, dispatches metadata + token-URI commands, broadcasts when live. +- `storage` — `erc721_transfer`, `erc721_ownership`, `erc721_approval`, `erc721_operator_approval`, `erc721_metadata`, `erc721_token_uri`. +- `grpc_service` — paginated queries + broadcast channel. +- `handlers` — two `CommandHandler`s: contract-level metadata + per-token URI. +- `synthetic` — `SyntheticErc721Extractor` implementing `SyntheticExtractor`. +- `conversions` — proto ↔ storage conversions. + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `"erc721"` | +| `interested_types` | `[ERC721_TYPE]` (one `TypeId::new("erc721")`) | +| `process` | Persist transfer + upsert ownership + optional approval row; dispatch metadata/token-URI commands; broadcast live updates if `batch.is_live(100)` | +| `topics` | 2 topics: `erc721.transfer` (filters: `token`, `from`, `to`, `wallet`) and `erc721.metadata` (filter: `token`) | +| `build_routes` | `Router::new()` — gRPC only | +| `initialize` | Captures `event_bus` + `ctx.command_bus` | + +### Storage (key tables) + +- `erc721_transfer (token, token_id, from, to, block_number, tx_hash, ..)` — event history +- `erc721_ownership (token, token_id, owner)` — one row per NFT; upserted on every Transfer +- `erc721_approval (token, token_id, owner, approved)` — per-token approvals (overwritten) +- `erc721_operator_approval (token, owner, operator, approved)` — ApprovalForAll bitmap +- `erc721_metadata (token, name, symbol, …)` — collection-level metadata +- `erc721_token_uri (token, token_id, uri, fetched_json)` — per-token URI + cached off-chain JSON (populated asynchronously via command handler) + +All `Felt`/`U256` fields are BLOB encoded via `torii-common::{FeltBlob,U256Blob}`. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-tokens`, `bins/torii-introspect-bin`, `bins/torii-arcade` (via `ArcadeService`), `bins/torii-tokens-synth`. +- **Downstream deps**: `torii`, `torii-common`, `torii-types`, `sqlx`, `tonic`, `prost`, `starknet`, `starknet-types-raw`, `primitive-types`, `tokio`, `async-trait`, `chrono`, `hex`, `metrics`, `tracing`. + +### Extension Points + +- Additional per-token metadata → add columns to `erc721_token_uri` + proto + `Erc721TokenUriCommandHandler`. +- New event (e.g. Pausable, EnumerableExt) → extend `Erc721Msg` + decoder branch + sink branch. +- Ownership history → add an `erc721_ownership_history` table; the sink already has the transfer insert site to hook into. diff --git a/crates/torii-erc721/src/conversions.rs b/crates/torii-erc721/src/conversions.rs new file mode 100644 index 00000000..ad012384 --- /dev/null +++ b/crates/torii-erc721/src/conversions.rs @@ -0,0 +1,55 @@ +use primitive_types::U256 as PrimitiveU256; +use starknet::core::types::{Felt, U256}; +use starknet_types_raw::Felt as RawFelt; + +pub(crate) fn felt_to_blob(value: Felt) -> Vec { + value.to_bytes_be().to_vec() +} + +pub(crate) fn blob_to_felt(bytes: &[u8]) -> Felt { + assert!(bytes.len() <= 32, "database stores canonical felt blobs"); + let mut padded = [0u8; 32]; + padded[32 - bytes.len()..].copy_from_slice(bytes); + Felt::from_bytes_be(&padded) +} + +pub(crate) fn u256_to_blob(value: U256) -> Vec { + if value == U256::from(0u8) { + return vec![0]; + } + + let mut bytes = [0u8; 32]; + bytes[..16].copy_from_slice(&value.high().to_be_bytes()); + bytes[16..].copy_from_slice(&value.low().to_be_bytes()); + let start = bytes.iter().position(|&b| b != 0).unwrap_or(31); + bytes[start..].to_vec() +} + +pub(crate) fn blob_to_u256(bytes: &[u8]) -> U256 { + let mut padded = [0u8; 32]; + let offset = 32usize.saturating_sub(bytes.len()); + padded[offset..].copy_from_slice(&bytes[bytes.len().saturating_sub(32)..]); + let high = u128::from_be_bytes(padded[..16].try_into().unwrap()); + let low = u128::from_be_bytes(padded[16..].try_into().unwrap()); + U256::from_words(low, high) +} + +pub(crate) fn u256_to_bytes(value: U256) -> Vec { + u256_to_blob(value) +} + +pub(crate) fn bytes_to_u256(bytes: &[u8]) -> U256 { + blob_to_u256(bytes) +} + +pub(crate) fn core_to_raw_felt(value: Felt) -> RawFelt { + RawFelt::from_be_bytes(value.to_bytes_be()).unwrap() +} + +pub(crate) fn core_to_primitive_u256(value: U256) -> PrimitiveU256 { + PrimitiveU256::from_big_endian(&u256_to_bytes(value)) +} + +pub(crate) fn primitive_to_core_u256(value: PrimitiveU256) -> U256 { + blob_to_u256(&value.to_big_endian()) +} diff --git a/crates/torii-erc721/src/decoder.rs b/crates/torii-erc721/src/decoder.rs index 15ed89a9..6a99faf4 100644 --- a/crates/torii-erc721/src/decoder.rs +++ b/crates/torii-erc721/src/decoder.rs @@ -2,11 +2,14 @@ use anyhow::Result; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt, U256}; +use starknet::core::types::{Felt, U256}; use starknet::macros::selector; +use starknet_types_raw::event::EmittedEvent; +use starknet_types_raw::Felt as RawFelt; use std::any::Any; -use std::collections::HashMap; -use torii::etl::{Decoder, Envelope, TypedBody}; +use torii::etl::{Decoder, Envelope, EventContext, TypeId, TypedBody}; + +pub const ERC721_TYPE: TypeId = TypeId::new("erc721"); /// Transfer event from ERC721 token #[derive(Debug, Clone)] @@ -20,20 +23,6 @@ pub struct NftTransfer { pub transaction_hash: Felt, } -impl TypedBody for NftTransfer { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc721.transfer") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - /// Approval event from ERC721 token (single token approval) #[derive(Debug, Clone)] pub struct NftApproval { @@ -46,20 +35,6 @@ pub struct NftApproval { pub transaction_hash: Felt, } -impl TypedBody for NftApproval { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc721.approval") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - /// ApprovalForAll event from ERC721 token (operator approval) #[derive(Debug, Clone)] pub struct OperatorApproval { @@ -71,20 +46,6 @@ pub struct OperatorApproval { pub transaction_hash: Felt, } -impl TypedBody for OperatorApproval { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc721.approval_for_all") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - /// MetadataUpdate event (EIP-4906) — single token #[derive(Debug, Clone)] pub struct MetadataUpdate { @@ -94,20 +55,6 @@ pub struct MetadataUpdate { pub transaction_hash: Felt, } -impl TypedBody for MetadataUpdate { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc721.metadata_update") - } - - fn as_any(&self) -> &dyn Any { - self - } - - fn as_any_mut(&mut self) -> &mut dyn Any { - self - } -} - /// BatchMetadataUpdate event (EIP-4906) — range of tokens #[derive(Debug, Clone)] pub struct BatchMetadataUpdate { @@ -118,9 +65,31 @@ pub struct BatchMetadataUpdate { pub transaction_hash: Felt, } -impl TypedBody for BatchMetadataUpdate { - fn envelope_type_id(&self) -> torii::etl::envelope::TypeId { - torii::etl::envelope::TypeId::new("erc721.batch_metadata_update") +#[derive(Debug, Clone)] +pub enum Erc721Msg { + Transfer(NftTransfer), + Approval(NftApproval), + ApprovalForAll(OperatorApproval), + MetadataUpdate(MetadataUpdate), + BatchMetadataUpdate(BatchMetadataUpdate), +} + +#[derive(Debug, Clone, Copy)] +pub struct Erc721Metadata { + pub block_number: u64, + pub transaction_hash: Felt, + pub from_address: Felt, +} + +#[derive(Debug, Clone)] +pub struct Erc721Body { + pub metadata: Erc721Metadata, + pub msg: Erc721Msg, +} + +impl TypedBody for Erc721Body { + fn envelope_type_id(&self) -> TypeId { + ERC721_TYPE } fn as_any(&self) -> &dyn Any { @@ -132,6 +101,56 @@ impl TypedBody for BatchMetadataUpdate { } } +impl Erc721Msg { + fn event_id(&self) -> String { + match self { + Self::Transfer(msg) => { + format!( + "erc721.transfer.{}.{}.{:#x}.", + msg.block_number, msg.transaction_hash, msg.token + ) + &msg.token_id.to_string() + } + Self::Approval(msg) => { + format!( + "erc721.approval.{}.{}.{:#x}.", + msg.block_number, msg.transaction_hash, msg.token + ) + &msg.token_id.to_string() + } + Self::ApprovalForAll(msg) => format!( + "erc721.approval_for_all.{}.{:#x}.{:#x}.{:#x}", + msg.block_number, msg.transaction_hash, msg.token, msg.owner + ), + Self::MetadataUpdate(msg) => { + format!( + "erc721.metadata_update.{}.{}.{:#x}.", + msg.block_number, msg.transaction_hash, msg.token + ) + &msg.token_id.to_string() + } + Self::BatchMetadataUpdate(msg) => { + format!( + "erc721.batch_metadata_update.{}.{}.{:#x}.{}.", + msg.block_number, msg.transaction_hash, msg.token, msg.from_token_id + ) + &msg.to_token_id.to_string() + } + } + } + + fn into_envelope(self, event: &EmittedEvent) -> Envelope { + Envelope::new( + self.event_id(), + Box::new(Erc721Body { + metadata: Erc721Metadata { + block_number: event.block_number.unwrap_or(0), + transaction_hash: Erc721Decoder::felt(event.transaction_hash), + from_address: Erc721Decoder::felt(event.from_address), + }, + msg: self, + }), + Default::default(), + ) + } +} + /// ERC721 event decoder /// /// Decodes multiple ERC721 events: @@ -174,417 +193,234 @@ impl Erc721Decoder { selector!("BatchMetadataUpdate") } - /// Decode Transfer event into envelope - /// - /// Transfer event signatures (supports both modern and legacy): - /// - /// Modern ERC721: - /// - keys[0]: Transfer selector - /// - keys[1]: from address - /// - keys[2]: to address - /// - keys[3]: token_id_low (u128) - /// - keys[4]: token_id_high (u128) - /// - /// Legacy ERC721 (all in data): - /// - keys[0]: Transfer selector - /// - data[0]: from address - /// - data[1]: to address - /// - data[2]: token_id_low (u128) - /// - data[3]: token_id_high (u128) - async fn decode_transfer(&self, event: &EmittedEvent) -> Result> { - let from; - let to; - let token_id: U256; + fn felt(value: RawFelt) -> Felt { + Felt::from_bytes_be(&value.to_be_bytes()) + } + + fn felt_to_u128(value: RawFelt) -> u128 { + value.try_into().unwrap_or(0) + } + fn parse_split_or_felt_u256(first: RawFelt, second: Option) -> U256 { + match second { + Some(high) => U256::from_words(Self::felt_to_u128(first), Self::felt_to_u128(high)), + None => U256::from(Self::felt_to_u128(first)), + } + } + + fn parse_address_pair_and_token_id(event: &EmittedEvent) -> Option<(Felt, Felt, U256)> { if event.keys.len() == 5 && event.data.is_empty() { - // Modern ERC721: from, to, token_id_low, token_id_high all in keys - from = event.keys[1]; - to = event.keys[2]; - let low: u128 = event.keys[3].try_into().unwrap_or(0); - let high: u128 = event.keys[4].try_into().unwrap_or(0); - token_id = U256::from_words(low, high); - } else if event.keys.len() == 1 && event.data.len() == 4 { - // Legacy ERC721: from, to, token_id_low, token_id_high in data - from = event.data[0]; - to = event.data[1]; - let low: u128 = event.data[2].try_into().unwrap_or(0); - let high: u128 = event.data[3].try_into().unwrap_or(0); - token_id = U256::from_words(low, high); - } else if event.keys.len() == 4 && event.data.is_empty() { - // Alternative modern format with single felt token_id - from = event.keys[1]; - to = event.keys[2]; - let id_felt: u128 = event.keys[3].try_into().unwrap_or(0); - token_id = U256::from(id_felt); - } else if event.keys.len() == 1 && event.data.len() == 3 { - // Alternative legacy format with single felt token_id - from = event.data[0]; - to = event.data[1]; - let id_felt: u128 = event.data[2].try_into().unwrap_or(0); - token_id = U256::from(id_felt); - } else { - tracing::warn!( - target: "torii_erc721::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - keys_len = event.keys.len(), - data_len = event.data.len(), - "Malformed ERC721 Transfer event" - ); - return Ok(None); + return Some(( + Self::felt(event.keys[1]), + Self::felt(event.keys[2]), + Self::parse_split_or_felt_u256(event.keys[3], Some(event.keys[4])), + )); } - let transfer = NftTransfer { - from, - to, - token_id, - token: event.from_address, - block_number: event.block_number.unwrap_or(0), - transaction_hash: event.transaction_hash, - }; + if event.keys.len() == 1 && event.data.len() == 4 { + return Some(( + Self::felt(event.data[0]), + Self::felt(event.data[1]), + Self::parse_split_or_felt_u256(event.data[2], Some(event.data[3])), + )); + } - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); + if event.keys.len() == 4 && event.data.is_empty() { + return Some(( + Self::felt(event.keys[1]), + Self::felt(event.keys[2]), + Self::parse_split_or_felt_u256(event.keys[3], None), + )); + } - let envelope_id = format!( - "erc721_transfer_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); + if event.keys.len() == 1 && event.data.len() == 3 { + return Some(( + Self::felt(event.data[0]), + Self::felt(event.data[1]), + Self::parse_split_or_felt_u256(event.data[2], None), + )); + } - Ok(Some(Envelope::new( - envelope_id, - Box::new(transfer), - metadata, - ))) - } - - /// Decode Approval event into envelope - /// - /// Approval event signatures (supports both modern and legacy): - /// - /// Modern ERC721: - /// - keys[0]: Approval selector - /// - keys[1]: owner address - /// - keys[2]: approved address - /// - keys[3]: token_id_low (u128) - /// - keys[4]: token_id_high (u128) - /// - /// Legacy ERC721: - /// - keys[0]: Approval selector - /// - data[0]: owner address - /// - data[1]: approved address - /// - data[2]: token_id_low (u128) - /// - data[3]: token_id_high (u128) - async fn decode_approval(&self, event: &EmittedEvent) -> Result> { - let owner; - let approved; - let token_id: U256; + None + } - if event.keys.len() == 5 && event.data.is_empty() { - // Modern ERC721: owner, approved, token_id_low, token_id_high all in keys - owner = event.keys[1]; - approved = event.keys[2]; - let low: u128 = event.keys[3].try_into().unwrap_or(0); - let high: u128 = event.keys[4].try_into().unwrap_or(0); - token_id = U256::from_words(low, high); - } else if event.keys.len() == 1 && event.data.len() == 4 { - // Legacy ERC721: owner, approved, token_id_low, token_id_high in data - owner = event.data[0]; - approved = event.data[1]; - let low: u128 = event.data[2].try_into().unwrap_or(0); - let high: u128 = event.data[3].try_into().unwrap_or(0); - token_id = U256::from_words(low, high); - } else if event.keys.len() == 4 && event.data.is_empty() { - // Alternative modern format with single felt token_id - owner = event.keys[1]; - approved = event.keys[2]; - let id_felt: u128 = event.keys[3].try_into().unwrap_or(0); - token_id = U256::from(id_felt); - } else if event.keys.len() == 1 && event.data.len() == 3 { - // Alternative legacy format with single felt token_id - owner = event.data[0]; - approved = event.data[1]; - let id_felt: u128 = event.data[2].try_into().unwrap_or(0); - token_id = U256::from(id_felt); - } else { - tracing::warn!( - target: "torii_erc721::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - keys_len = event.keys.len(), - data_len = event.data.len(), - "Malformed ERC721 Approval event" - ); - return Ok(None); + fn parse_operator_approval(event: &EmittedEvent) -> Option<(Felt, Felt, bool)> { + if event.keys.len() == 3 && event.data.len() == 1 { + return Some(( + Self::felt(event.keys[1]), + Self::felt(event.keys[2]), + event.data[0] != RawFelt::ZERO, + )); } - let approval = NftApproval { - owner, - approved, - token_id, - token: event.from_address, - block_number: event.block_number.unwrap_or(0), - transaction_hash: event.transaction_hash, - }; + if event.keys.len() == 1 && event.data.len() == 3 { + return Some(( + Self::felt(event.data[0]), + Self::felt(event.data[1]), + event.data[2] != RawFelt::ZERO, + )); + } - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); + None + } - let envelope_id = format!( - "erc721_approval_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); + fn parse_single_token_id(event: &EmittedEvent) -> Option { + if event.data.len() >= 2 { + return Some(Self::parse_split_or_felt_u256( + event.data[0], + Some(event.data[1]), + )); + } - Ok(Some(Envelope::new( - envelope_id, - Box::new(approval), - metadata, - ))) - } - - /// Decode ApprovalForAll event into envelope - /// - /// ApprovalForAll event signatures (supports both modern and legacy): - /// - /// Modern ERC721: - /// - keys[0]: ApprovalForAll selector - /// - keys[1]: owner address - /// - keys[2]: operator address - /// - data[0]: approved (bool as felt) - /// - /// Legacy ERC721: - /// - keys[0]: ApprovalForAll selector - /// - data[0]: owner address - /// - data[1]: operator address - /// - data[2]: approved (bool as felt) - async fn decode_approval_for_all(&self, event: &EmittedEvent) -> Result> { - let owner; - let operator; - let approved: bool; + if event.keys.len() >= 3 { + return Some(Self::parse_split_or_felt_u256( + event.keys[1], + Some(event.keys[2]), + )); + } - if event.keys.len() == 3 && event.data.len() == 1 { - // Modern ERC721: owner, operator in keys; approved in data - owner = event.keys[1]; - operator = event.keys[2]; - approved = event.data[0] != Felt::ZERO; - } else if event.keys.len() == 1 && event.data.len() == 3 { - // Legacy ERC721: owner, operator, approved all in data - owner = event.data[0]; - operator = event.data[1]; - approved = event.data[2] != Felt::ZERO; - } else { - tracing::warn!( - target: "torii_erc721::decoder", - token = %format!("{:#x}", event.from_address), - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number.unwrap_or(0), - keys_len = event.keys.len(), - data_len = event.data.len(), - "Malformed ERC721 ApprovalForAll event" - ); - return Ok(None); + if event.data.len() == 1 { + return Some(Self::parse_split_or_felt_u256(event.data[0], None)); } - let approval = OperatorApproval { - owner, - operator, - approved, - token: event.from_address, - block_number: event.block_number.unwrap_or(0), - transaction_hash: event.transaction_hash, - }; + if event.keys.len() == 2 { + return Some(Self::parse_split_or_felt_u256(event.keys[1], None)); + } - let mut metadata = HashMap::new(); - metadata.insert("token".to_string(), format!("{:#x}", event.from_address)); - metadata.insert( - "block_number".to_string(), - event.block_number.unwrap_or(0).to_string(), - ); - metadata.insert( - "tx_hash".to_string(), - format!("{:#x}", event.transaction_hash), - ); + None + } - let envelope_id = format!( - "erc721_approval_for_all_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) - ); + fn parse_token_range(event: &EmittedEvent) -> Option<(U256, U256)> { + if event.data.len() >= 4 { + return Some(( + Self::parse_split_or_felt_u256(event.data[0], Some(event.data[1])), + Self::parse_split_or_felt_u256(event.data[2], Some(event.data[3])), + )); + } - Ok(Some(Envelope::new( - envelope_id, - Box::new(approval), - metadata, - ))) - } - - /// Decode MetadataUpdate event (EIP-4906) - /// - /// MetadataUpdate(uint256 tokenId): - /// - keys[0]: selector - /// - data[0]: token_id_low - /// - data[1]: token_id_high - /// OR: - /// - keys[0]: selector - /// - keys[1]: token_id_low - /// - keys[2]: token_id_high - async fn decode_metadata_update(&self, event: &EmittedEvent) -> Result> { - let token_id: U256; + if event.keys.len() >= 5 { + return Some(( + Self::parse_split_or_felt_u256(event.keys[1], Some(event.keys[2])), + Self::parse_split_or_felt_u256(event.keys[3], Some(event.keys[4])), + )); + } - if event.data.len() >= 2 { - let low: u128 = event.data[0].try_into().unwrap_or(0); - let high: u128 = event.data[1].try_into().unwrap_or(0); - token_id = U256::from_words(low, high); - } else if event.keys.len() >= 3 { - let low: u128 = event.keys[1].try_into().unwrap_or(0); - let high: u128 = event.keys[2].try_into().unwrap_or(0); - token_id = U256::from_words(low, high); - } else if event.data.len() == 1 { - let id: u128 = event.data[0].try_into().unwrap_or(0); - token_id = U256::from(id); - } else if event.keys.len() == 2 { - let id: u128 = event.keys[1].try_into().unwrap_or(0); - token_id = U256::from(id); - } else { - tracing::warn!( - target: "torii_erc721::decoder", - token = %format!("{:#x}", event.from_address), - "Malformed MetadataUpdate event" - ); + None + } + + fn log_malformed_event(&self, name: &str, event: &EmittedEvent) { + tracing::warn!( + target: "torii_erc721::decoder", + token = %format!("{:#x}", event.from_address), + tx_hash = %format!("{:#x}", event.transaction_hash), + block_number = event.block_number.unwrap_or(0), + keys_len = event.keys.len(), + data_len = event.data.len(), + "Malformed ERC721 {name} event" + ); + } + + /// Decode Transfer event into a message. + async fn decode_transfer(&self, event: &EmittedEvent) -> Result> { + let Some((from, to, token_id)) = Self::parse_address_pair_and_token_id(event) else { + self.log_malformed_event("Transfer", event); return Ok(None); - } + }; - let update = MetadataUpdate { - token: event.from_address, + Ok(Some(Erc721Msg::Transfer(NftTransfer { + from, + to, token_id, + token: Self::felt(event.from_address), block_number: event.block_number.unwrap_or(0), - transaction_hash: event.transaction_hash, - }; + transaction_hash: Self::felt(event.transaction_hash), + }))) + } - let envelope_id = format!( - "erc721_metadata_update_{}_{:#x}", - event.block_number.unwrap_or(0), - event.transaction_hash - ); + /// Decode Approval event into a message. + async fn decode_approval(&self, event: &EmittedEvent) -> Result> { + let Some((owner, approved, token_id)) = Self::parse_address_pair_and_token_id(event) else { + self.log_malformed_event("Approval", event); + return Ok(None); + }; - Ok(Some(Envelope::new( - envelope_id, - Box::new(update), - HashMap::new(), - ))) + Ok(Some(Erc721Msg::Approval(NftApproval { + owner, + approved, + token_id, + token: Self::felt(event.from_address), + block_number: event.block_number.unwrap_or(0), + transaction_hash: Self::felt(event.transaction_hash), + }))) } - /// Decode BatchMetadataUpdate event (EIP-4906) - /// - /// BatchMetadataUpdate(uint256 fromTokenId, uint256 toTokenId) - async fn decode_batch_metadata_update(&self, event: &EmittedEvent) -> Result> { - let from_token_id: U256; - let to_token_id: U256; - - if event.data.len() >= 4 { - let low: u128 = event.data[0].try_into().unwrap_or(0); - let high: u128 = event.data[1].try_into().unwrap_or(0); - from_token_id = U256::from_words(low, high); - let low: u128 = event.data[2].try_into().unwrap_or(0); - let high: u128 = event.data[3].try_into().unwrap_or(0); - to_token_id = U256::from_words(low, high); - } else if event.keys.len() >= 5 { - let low: u128 = event.keys[1].try_into().unwrap_or(0); - let high: u128 = event.keys[2].try_into().unwrap_or(0); - from_token_id = U256::from_words(low, high); - let low: u128 = event.keys[3].try_into().unwrap_or(0); - let high: u128 = event.keys[4].try_into().unwrap_or(0); - to_token_id = U256::from_words(low, high); - } else { - tracing::warn!( - target: "torii_erc721::decoder", - token = %format!("{:#x}", event.from_address), - "Malformed BatchMetadataUpdate event" - ); + /// Decode ApprovalForAll event into a message. + async fn decode_approval_for_all(&self, event: &EmittedEvent) -> Result> { + let Some((owner, operator, approved)) = Self::parse_operator_approval(event) else { + self.log_malformed_event("ApprovalForAll", event); return Ok(None); - } + }; - let update = BatchMetadataUpdate { - token: event.from_address, - from_token_id, - to_token_id, + Ok(Some(Erc721Msg::ApprovalForAll(OperatorApproval { + owner, + operator, + approved, + token: Self::felt(event.from_address), block_number: event.block_number.unwrap_or(0), - transaction_hash: event.transaction_hash, - }; + transaction_hash: Self::felt(event.transaction_hash), + }))) + } - let envelope_id = format!( - "erc721_batch_metadata_update_{}_{:#x}", - event.block_number.unwrap_or(0), - event.transaction_hash - ); + /// Decode MetadataUpdate event (EIP-4906). + async fn decode_metadata_update(&self, event: &EmittedEvent) -> Result> { + let Some(token_id) = Self::parse_single_token_id(event) else { + self.log_malformed_event("MetadataUpdate", event); + return Ok(None); + }; - Ok(Some(Envelope::new( - envelope_id, - Box::new(update), - HashMap::new(), - ))) + Ok(Some(Erc721Msg::MetadataUpdate(MetadataUpdate { + token: Self::felt(event.from_address), + token_id, + block_number: event.block_number.unwrap_or(0), + transaction_hash: Self::felt(event.transaction_hash), + }))) } -} -impl Default for Erc721Decoder { - fn default() -> Self { - Self::new() - } -} + /// Decode BatchMetadataUpdate event (EIP-4906). + async fn decode_batch_metadata_update( + &self, + event: &EmittedEvent, + ) -> Result> { + let Some((from_token_id, to_token_id)) = Self::parse_token_range(event) else { + self.log_malformed_event("BatchMetadataUpdate", event); + return Ok(None); + }; -#[async_trait] -impl Decoder for Erc721Decoder { - fn decoder_name(&self) -> &'static str { - "erc721" + Ok(Some(Erc721Msg::BatchMetadataUpdate(BatchMetadataUpdate { + token: Self::felt(event.from_address), + from_token_id, + to_token_id, + block_number: event.block_number.unwrap_or(0), + transaction_hash: Self::felt(event.transaction_hash), + }))) } - async fn decode_event(&self, event: &EmittedEvent) -> Result> { + async fn decode_raw_event(&self, event: &EmittedEvent) -> Result> { if event.keys.is_empty() { return Ok(Vec::new()); } - let selector = event.keys[0]; - - if selector == Self::transfer_selector() { - if let Some(envelope) = self.decode_transfer(event).await? { - return Ok(vec![envelope]); - } + let selector = Self::felt(event.keys[0]); + let msg = if selector == Self::transfer_selector() { + self.decode_transfer(event).await? } else if selector == Self::approval_selector() { - if let Some(envelope) = self.decode_approval(event).await? { - return Ok(vec![envelope]); - } + self.decode_approval(event).await? } else if selector == Self::approval_for_all_selector() { - if let Some(envelope) = self.decode_approval_for_all(event).await? { - return Ok(vec![envelope]); - } + self.decode_approval_for_all(event).await? } else if selector == Self::metadata_update_selector() { - if let Some(envelope) = self.decode_metadata_update(event).await? { - return Ok(vec![envelope]); - } + self.decode_metadata_update(event).await? } else if selector == Self::batch_metadata_update_selector() { - if let Some(envelope) = self.decode_batch_metadata_update(event).await? { - return Ok(vec![envelope]); - } + self.decode_batch_metadata_update(event).await? } else { tracing::trace!( target: "torii_erc721::decoder", @@ -596,113 +432,157 @@ impl Decoder for Erc721Decoder { tx_hash = %format!("{:#x}", event.transaction_hash), "Unhandled event selector" ); - } + None + }; + + Ok(msg + .into_iter() + .map(|msg| msg.into_envelope(event)) + .collect()) + } +} + +impl Default for Erc721Decoder { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Decoder for Erc721Decoder { + fn decoder_name(&self) -> &'static str { + "erc721" + } + + async fn decode( + &self, + keys: &[RawFelt], + data: &[RawFelt], + context: EventContext, + ) -> Result> { + let event = EmittedEvent { + from_address: context.from_address, + keys: keys.to_vec(), + data: data.to_vec(), + block_hash: None, + block_number: Some(context.block_number), + transaction_hash: context.transaction_hash, + }; - Ok(Vec::new()) + self.decode_raw_event(&event).await } } #[cfg(test)] mod tests { use super::*; + use torii::etl::StarknetEvent; - #[tokio::test] - async fn test_decode_modern_transfer() { + fn raw(value: u64) -> RawFelt { + RawFelt::from(value) + } + + fn raw_selector(selector: Felt) -> RawFelt { + RawFelt::from_be_bytes(selector.to_bytes_be()).unwrap() + } + + fn starknet_event(event: EmittedEvent) -> StarknetEvent { + StarknetEvent::new( + event.from_address, + event.keys, + event.data, + event.block_number.unwrap_or(0), + event.transaction_hash, + ) + } + + fn decode_body(event: EmittedEvent) -> Erc721Body { let decoder = Erc721Decoder::new(); + let envelopes = + futures::executor::block_on(decoder.decode_event(&starknet_event(event))).unwrap(); + assert_eq!(envelopes.len(), 1); + envelopes[0] + .body + .as_any() + .downcast_ref::() + .unwrap() + .clone() + } - // Modern format: all in keys + #[tokio::test] + async fn test_decode_modern_transfer() { let event = EmittedEvent { - from_address: Felt::from(0x123u64), + from_address: raw(0x123), keys: vec![ - Erc721Decoder::transfer_selector(), - Felt::from(0x1u64), // from - Felt::from(0x2u64), // to - Felt::from(42u64), // token_id_low - Felt::ZERO, // token_id_high + raw_selector(Erc721Decoder::transfer_selector()), + raw(0x1), + raw(0x2), + raw(42), + RawFelt::ZERO, ], data: vec![], block_hash: None, block_number: Some(100), - transaction_hash: Felt::from(0xabcdu64), + transaction_hash: raw(0xabcd), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); - assert_eq!(envelopes.len(), 1); - - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(transfer.from, Felt::from(0x1u64)); - assert_eq!(transfer.to, Felt::from(0x2u64)); - assert_eq!(transfer.token_id, U256::from(42u64)); - assert_eq!(transfer.token, Felt::from(0x123u64)); + let body = decode_body(event); + match body.msg { + Erc721Msg::Transfer(transfer) => { + assert_eq!(transfer.from, Felt::from(0x1u64)); + assert_eq!(transfer.to, Felt::from(0x2u64)); + assert_eq!(transfer.token_id, U256::from(42u64)); + assert_eq!(transfer.token, Felt::from(0x123u64)); + } + msg => panic!("unexpected message: {msg:?}"), + } } #[tokio::test] async fn test_decode_legacy_transfer() { - let decoder = Erc721Decoder::new(); - - // Legacy format: all in data let event = EmittedEvent { - from_address: Felt::from(0x456u64), - keys: vec![Erc721Decoder::transfer_selector()], - data: vec![ - Felt::from(0xau64), // from - Felt::from(0xbu64), // to - Felt::from(100u64), // token_id_low - Felt::ZERO, // token_id_high - ], + from_address: raw(0x456), + keys: vec![raw_selector(Erc721Decoder::transfer_selector())], + data: vec![raw(0xa), raw(0xb), raw(100), RawFelt::ZERO], block_hash: None, block_number: Some(200), - transaction_hash: Felt::from(0xef01u64), + transaction_hash: raw(0xef01), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); - assert_eq!(envelopes.len(), 1); - - let transfer = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(transfer.from, Felt::from(0xau64)); - assert_eq!(transfer.to, Felt::from(0xbu64)); - assert_eq!(transfer.token_id, U256::from(100u64)); + let body = decode_body(event); + match body.msg { + Erc721Msg::Transfer(transfer) => { + assert_eq!(transfer.from, Felt::from(0xau64)); + assert_eq!(transfer.to, Felt::from(0xbu64)); + assert_eq!(transfer.token_id, U256::from(100u64)); + } + msg => panic!("unexpected message: {msg:?}"), + } } #[tokio::test] async fn test_decode_approval_for_all() { - let decoder = Erc721Decoder::new(); - - // Modern format let event = EmittedEvent { - from_address: Felt::from(0x789u64), + from_address: raw(0x789), keys: vec![ - Erc721Decoder::approval_for_all_selector(), - Felt::from(0xcu64), // owner - Felt::from(0xdu64), // operator + raw_selector(Erc721Decoder::approval_for_all_selector()), + raw(0xc), + raw(0xd), ], - data: vec![Felt::from(1u64)], // approved = true + data: vec![raw(1)], block_hash: None, block_number: Some(300), - transaction_hash: Felt::from(0x2345u64), + transaction_hash: raw(0x2345), }; - let envelopes = decoder.decode_event(&event).await.unwrap(); - assert_eq!(envelopes.len(), 1); - - let approval = envelopes[0] - .body - .as_any() - .downcast_ref::() - .unwrap(); - - assert_eq!(approval.owner, Felt::from(0xcu64)); - assert_eq!(approval.operator, Felt::from(0xdu64)); - assert!(approval.approved); + let body = decode_body(event); + match body.msg { + Erc721Msg::ApprovalForAll(approval) => { + assert_eq!(approval.owner, Felt::from(0xcu64)); + assert_eq!(approval.operator, Felt::from(0xdu64)); + assert!(approval.approved); + } + msg => panic!("unexpected message: {msg:?}"), + } } } diff --git a/crates/torii-erc721/src/grpc_service.rs b/crates/torii-erc721/src/grpc_service.rs index ae25479a..52daf44b 100644 --- a/crates/torii-erc721/src/grpc_service.rs +++ b/crates/torii-erc721/src/grpc_service.rs @@ -1,5 +1,6 @@ //! gRPC service implementation for ERC721 queries and subscriptions +use crate::conversions::{bytes_to_u256, u256_to_bytes}; use crate::proto::{ erc721_server::Erc721 as Erc721Trait, AttributeFacetCount, CollectionToken, ContractCollectionOverview, Cursor, GetCollectionOverviewRequest, @@ -20,7 +21,16 @@ use std::pin::Pin; use std::sync::Arc; use tokio::sync::broadcast; use tonic::{Request, Response, Status}; -use torii_common::{bytes_to_felt, bytes_to_u256, u256_to_bytes}; + +fn bytes_to_felt(bytes: &[u8]) -> Option { + if bytes.len() > 32 { + return None; + } + + let mut padded = [0u8; 32]; + padded[32 - bytes.len()..].copy_from_slice(bytes); + Some(Felt::from_bytes_be(&padded)) +} const DEFAULT_PROJECT_ID: &str = "arcade-main"; diff --git a/crates/torii-erc721/src/handlers.rs b/crates/torii-erc721/src/handlers.rs index bfe65d47..066847e0 100644 --- a/crates/torii-erc721/src/handlers.rs +++ b/crates/torii-erc721/src/handlers.rs @@ -1,3 +1,6 @@ +use crate::conversions::{ + core_to_primitive_u256, core_to_raw_felt, primitive_to_core_u256, u256_to_bytes, +}; use anyhow::Result; use async_trait::async_trait; use prost::Message; @@ -10,7 +13,7 @@ use std::sync::{Arc, Mutex}; use torii::command::CommandHandler; use torii::etl::sink::EventBus; use torii::UpdateType; -use torii_common::{process_token_uri_request, u256_to_bytes, MetadataFetcher, TokenUriRequest}; +use torii_common::{process_token_uri_request, MetadataFetcher, TokenUriRequest}; use crate::proto; use crate::storage::Erc721Storage; @@ -114,7 +117,10 @@ impl CommandHandler for Erc721MetadataCommandHandler { let mut attempt = 1; let result = loop { - let meta = self.fetcher.fetch_erc721_metadata(command.token).await; + let meta = self + .fetcher + .fetch_erc721_metadata(core_to_raw_felt(command.token)) + .await; if !Self::metadata_complete(&meta) && attempt < self.max_retries { let next_attempt = attempt + 1; let delay = Self::metadata_retry_delay(next_attempt); @@ -140,7 +146,7 @@ impl CommandHandler for Erc721MetadataCommandHandler { command.token, meta.name.as_deref(), meta.symbol.as_deref(), - meta.total_supply, + meta.total_supply.map(primitive_to_core_u256), ) .await?; @@ -150,7 +156,10 @@ impl CommandHandler for Erc721MetadataCommandHandler { token: command.token.to_bytes_be().to_vec(), name: meta.name, symbol: meta.symbol, - total_supply: meta.total_supply.map(u256_to_bytes), + total_supply: meta + .total_supply + .map(primitive_to_core_u256) + .map(u256_to_bytes), }; let mut buf = Vec::new(); @@ -255,8 +264,8 @@ impl CommandHandler for Erc721TokenUriCommandHandler { self.fetcher.as_ref(), self.storage.as_ref(), &TokenUriRequest { - contract: command.contract, - token_id: command.token_id, + contract: core_to_raw_felt(command.contract), + token_id: core_to_primitive_u256(command.token_id), standard: torii_common::TokenStandard::Erc721, }, self.image_cache_dir.as_deref(), diff --git a/crates/torii-erc721/src/identification.rs b/crates/torii-erc721/src/identification.rs index 13a7b22a..f158c433 100644 --- a/crates/torii-erc721/src/identification.rs +++ b/crates/torii-erc721/src/identification.rs @@ -6,7 +6,7 @@ //! - `Transfer` event (with token_id) use anyhow::Result; -use starknet::core::types::Felt; +use starknet_types_raw::Felt; use torii::etl::decoder::DecoderId; use torii::etl::extractor::ContractAbi; use torii::etl::identification::IdentificationRule; diff --git a/crates/torii-erc721/src/lib.rs b/crates/torii-erc721/src/lib.rs index ec73d2d1..5acc1841 100644 --- a/crates/torii-erc721/src/lib.rs +++ b/crates/torii-erc721/src/lib.rs @@ -7,8 +7,8 @@ //! # Components //! //! - [`Erc721Decoder`]: Decodes ERC721 Transfer, Approval, and ApprovalForAll events -//! - [`Erc721Sink`]: Processes decoded events, stores in SQLite, and publishes updates -//! - [`Erc721Storage`]: SQLite storage with ownership tracking and efficient pagination +//! - [`Erc721Sink`]: Processes decoded events, stores them, and publishes updates +//! - [`Erc721Storage`]: DbPool-backed storage with ownership tracking and efficient pagination //! - [`Erc721Service`]: gRPC service for queries and real-time subscriptions //! //! # Example @@ -35,6 +35,7 @@ //! .add_service(Erc721Server::new(grpc_service)); //! ``` +mod conversions; pub mod decoder; pub mod grpc_service; pub mod handlers; @@ -53,7 +54,8 @@ pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!("generated/erc721_descript // Re-export main types for convenience pub use decoder::{ - BatchMetadataUpdate, Erc721Decoder, MetadataUpdate, NftApproval, NftTransfer, OperatorApproval, + BatchMetadataUpdate, Erc721Body, Erc721Decoder, Erc721Msg, MetadataUpdate, NftApproval, + NftTransfer, OperatorApproval, ERC721_TYPE, }; pub use grpc_service::Erc721Service; pub use handlers::{Erc721MetadataCommandHandler, Erc721TokenUriCommandHandler}; diff --git a/crates/torii-erc721/src/sink.rs b/crates/torii-erc721/src/sink.rs index cd8c8438..de12be89 100644 --- a/crates/torii-erc721/src/sink.rs +++ b/crates/torii-erc721/src/sink.rs @@ -1,9 +1,7 @@ //! ERC721 sink for processing NFT transfers, approvals, and ownership -use crate::decoder::{ - BatchMetadataUpdate as DecodedBatchMetadataUpdate, MetadataUpdate as DecodedMetadataUpdate, - NftTransfer as DecodedNftTransfer, OperatorApproval as DecodedOperatorApproval, -}; +use crate::conversions::{core_to_primitive_u256, core_to_raw_felt, u256_to_bytes}; +use crate::decoder::{Erc721Body, Erc721Msg, ERC721_TYPE}; use crate::grpc_service::Erc721Service; use crate::handlers::{FetchErc721MetadataCommand, RefreshErc721TokenUriCommand}; use crate::proto; @@ -22,7 +20,7 @@ use torii::command::CommandBusSender; use torii::etl::sink::{EventBus, TopicInfo}; use torii::etl::{Envelope, ExtractionBatch, Sink, TypeId}; use torii::grpc::UpdateType; -use torii_common::{u256_to_bytes, TokenStandard, TokenUriRequest, TokenUriSender}; +use torii_common::{TokenStandard, TokenUriRequest, TokenUriSender}; /// Default threshold for "live" detection: 100 blocks from chain head. /// Events from blocks older than this won't be broadcast to real-time subscribers. @@ -135,8 +133,8 @@ impl Erc721Sink { fn enqueue_token_uri_request(&self, contract: Felt, token_id: U256) -> bool { if let Some(sender) = &self.token_uri_sender { return sender.request_update(TokenUriRequest { - contract, - token_id, + contract: core_to_raw_felt(contract), + token_id: core_to_primitive_u256(token_id), standard: TokenStandard::Erc721, }); } @@ -174,7 +172,16 @@ impl Erc721Sink { .collect::>(); if let Some(sender) = &self.token_uri_sender { - let accepted = sender.request_batch(contract, &token_ids, TokenStandard::Erc721); + let primitive_token_ids = token_ids + .iter() + .copied() + .map(core_to_primitive_u256) + .collect::>(); + let accepted = sender.request_batch( + core_to_raw_felt(contract), + &primitive_token_ids, + TokenStandard::Erc721, + ); if accepted != token_ids.len() { tracing::warn!( target: "torii_erc721::sink", @@ -200,13 +207,7 @@ impl Sink for Erc721Sink { } fn interested_types(&self) -> Vec { - vec![ - TypeId::new("erc721.transfer"), - TypeId::new("erc721.approval"), - TypeId::new("erc721.approval_for_all"), - TypeId::new("erc721.metadata_update"), - TypeId::new("erc721.batch_metadata_update"), - ] + vec![ERC721_TYPE] } async fn initialize( @@ -234,10 +235,16 @@ impl Sink for Erc721Sink { .collect(); for envelope in envelopes { - // Handle transfers - if envelope.type_id == TypeId::new("erc721.transfer") { - if let Some(transfer) = envelope.body.as_any().downcast_ref::() - { + if envelope.type_id != ERC721_TYPE { + continue; + } + + let Some(body) = envelope.body.as_any().downcast_ref::() else { + continue; + }; + + match &body.msg { + Erc721Msg::Transfer(transfer) => { let timestamp = block_timestamps.get(&transfer.block_number).copied(); transfers.push(NftTransferData { id: None, @@ -250,14 +257,7 @@ impl Sink for Erc721Sink { timestamp, }); } - } - // Handle approval for all - else if envelope.type_id == TypeId::new("erc721.approval_for_all") { - if let Some(approval) = envelope - .body - .as_any() - .downcast_ref::() - { + Erc721Msg::ApprovalForAll(approval) => { let timestamp = block_timestamps.get(&approval.block_number).copied(); operator_approvals.push(OperatorApprovalData { id: None, @@ -270,26 +270,12 @@ impl Sink for Erc721Sink { timestamp, }); } - } - // Handle MetadataUpdate (EIP-4906) — single token - else if envelope.type_id == TypeId::new("erc721.metadata_update") { - if let Some(update) = envelope - .body - .as_any() - .downcast_ref::() - { + Erc721Msg::MetadataUpdate(update) => { if self.token_uri_commands_enabled { self.enqueue_token_uri_request(update.token, update.token_id); } } - } - // Handle BatchMetadataUpdate (EIP-4906) — range of tokens - else if envelope.type_id == TypeId::new("erc721.batch_metadata_update") { - if let Some(update) = envelope - .body - .as_any() - .downcast_ref::() - { + Erc721Msg::BatchMetadataUpdate(update) => { if self.token_uri_commands_enabled { self.enqueue_token_uri_range_refresh( update.token, @@ -299,9 +285,10 @@ impl Sink for Erc721Sink { .await; } } + Erc721Msg::Approval(_) => { + // Single-token approvals are decoded for completeness but not stored yet. + } } - // Note: erc721.approval (single token approval) could be handled similarly - // but is less commonly needed for indexing purposes } // Fetch metadata for any new token contracts. diff --git a/crates/torii-erc721/src/storage.rs b/crates/torii-erc721/src/storage.rs index 658043fa..00541b68 100644 --- a/crates/torii-erc721/src/storage.rs +++ b/crates/torii-erc721/src/storage.rs @@ -3,15 +3,15 @@ //! Uses binary (BLOB) storage for efficiency. //! Tracks current NFT ownership state (who owns each token). +use crate::conversions::{blob_to_felt, blob_to_u256, felt_to_blob, u256_to_blob}; use anyhow::Result; use rusqlite::{params, params_from_iter, Connection, ToSql}; use starknet::core::types::{Felt, U256}; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::{Arc, Mutex}; use tokio_postgres::{types::ToSql as PgToSql, Client, NoTls}; -use torii_common::{ - blob_to_felt, blob_to_u256, felt_to_blob, u256_to_blob, TokenUriResult, TokenUriStore, -}; +use torii_common::{TokenUriResult, TokenUriStore}; +use torii_sql::{DbPool, DbPoolOptions}; const SQLITE_MAX_BIND_VARS: usize = 900; const SQLITE_TOKEN_BATCH_SIZE: usize = SQLITE_MAX_BIND_VARS; @@ -19,9 +19,40 @@ const SQLITE_TOKEN_PAIR_BATCH_SIZE: usize = SQLITE_MAX_BIND_VARS / 2; /// Storage for ERC721 NFT data pub struct Erc721Storage { - backend: StorageBackend, + db: Erc721Db, +} + +struct Erc721Db { + backend: Backend, + runtime: StorageRuntime, +} + +enum StorageRuntime { + Sqlite(SqliteStorageRuntime), + Postgres(PostgresStorageRuntime), +} + +struct SqliteStorageRuntime { conn: Arc>, - pg_conn: Option>>, +} + +struct PostgresStorageRuntime { + conn: Arc>, +} + +impl Erc721Db { + fn new(backend: Backend, runtime: StorageRuntime) -> Self { + Self { backend, runtime } + } +} + +impl StorageRuntime { + fn backend(&self) -> StorageBackend { + match self { + Self::Sqlite(_) => StorageBackend::Sqlite, + Self::Postgres(_) => StorageBackend::Postgres, + } + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -113,7 +144,18 @@ struct ResolvedFacetFilter { impl Erc721Storage { /// Create or open the database pub async fn new(db_path: &str) -> Result { - if db_path.starts_with("postgres://") || db_path.starts_with("postgresql://") { + let pool = Self::connect_pool(db_path).await?; + Self::from_pool(db_path, pool).await + } + + pub async fn connect_pool(db_path: &str) -> Result { + Ok(DbPoolOptions::new() + .connect_any(&db_pool_url(db_path)) + .await?) + } + + pub async fn from_pool(db_path: &str, pool: DbPool) -> Result { + let runtime = if matches!(pool, DbPool::Postgres(_)) { let (client, connection) = tokio_postgres::connect(db_path, NoTls).await?; tokio::spawn(async move { if let Err(e) = connection.await { @@ -251,33 +293,30 @@ impl Erc721Storage { ).await?; tracing::info!(target: "torii_erc721::storage", "PostgreSQL storage initialized"); - return Ok(Self { - backend: StorageBackend::Postgres, - conn: Arc::new(Mutex::new(Connection::open_in_memory()?)), - pg_conn: Some(Arc::new(tokio::sync::Mutex::new(client))), - }); - } - - let conn = Connection::open(db_path)?; - - // Enable WAL mode + Performance PRAGMAs - conn.execute_batch( - "PRAGMA journal_mode=WAL; - PRAGMA synchronous=NORMAL; - PRAGMA foreign_keys=ON; - PRAGMA cache_size=-64000; - PRAGMA temp_store=MEMORY; - PRAGMA mmap_size=268435456; - PRAGMA wal_autocheckpoint=10000; - PRAGMA page_size=4096; - PRAGMA busy_timeout=5000;", - )?; + StorageRuntime::Postgres(PostgresStorageRuntime { + conn: Arc::new(tokio::sync::Mutex::new(client)), + }) + } else { + let conn = Connection::open(db_path)?; + + // Enable WAL mode + Performance PRAGMAs + conn.execute_batch( + "PRAGMA journal_mode=WAL; + PRAGMA synchronous=NORMAL; + PRAGMA foreign_keys=ON; + PRAGMA cache_size=-64000; + PRAGMA temp_store=MEMORY; + PRAGMA mmap_size=268435456; + PRAGMA wal_autocheckpoint=10000; + PRAGMA page_size=4096; + PRAGMA busy_timeout=5000;", + )?; - tracing::info!(target: "torii_erc721::storage", "SQLite configured: WAL mode, 64MB cache, 256MB mmap, NORMAL sync"); + tracing::info!(target: "torii_erc721::storage", "SQLite configured: WAL mode, 64MB cache, 256MB mmap, NORMAL sync"); - // NFT ownership (current state) - one owner per NFT - conn.execute( - "CREATE TABLE IF NOT EXISTS nft_ownership ( + // NFT ownership (current state) - one owner per NFT + conn.execute( + "CREATE TABLE IF NOT EXISTS nft_ownership ( id INTEGER PRIMARY KEY AUTOINCREMENT, token BLOB NOT NULL, token_id BLOB NOT NULL, @@ -287,34 +326,34 @@ impl Erc721Storage { timestamp TEXT, UNIQUE(token, token_id) )", - [], - )?; + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_ownership_owner ON nft_ownership(owner)", - [], - )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_ownership_owner ON nft_ownership(owner)", + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_ownership_token ON nft_ownership(token)", - [], - )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_ownership_token ON nft_ownership(token)", + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_ownership_token_owner_token_id_ord + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_ownership_token_owner_token_id_ord ON nft_ownership(token, owner, length(token_id), token_id)", - [], - )?; + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_ownership_owner_token_token_id_ord + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_ownership_owner_token_token_id_ord ON nft_ownership(owner, token, length(token_id), token_id)", - [], - )?; + [], + )?; - // Transfer history - conn.execute( - "CREATE TABLE IF NOT EXISTS nft_transfers ( + // Transfer history + conn.execute( + "CREATE TABLE IF NOT EXISTS nft_transfers ( id INTEGER PRIMARY KEY AUTOINCREMENT, token BLOB NOT NULL, token_id BLOB NOT NULL, @@ -325,37 +364,37 @@ impl Erc721Storage { timestamp TEXT, UNIQUE(token, tx_hash, token_id, from_addr, to_addr) )", - [], - )?; + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_transfers_token ON nft_transfers(token)", - [], - )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_transfers_token ON nft_transfers(token)", + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_transfers_from ON nft_transfers(from_addr)", - [], - )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_transfers_from ON nft_transfers(from_addr)", + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_transfers_to ON nft_transfers(to_addr)", - [], - )?; + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_transfers_to ON nft_transfers(to_addr)", + [], + )?; - conn.execute( + conn.execute( "CREATE INDEX IF NOT EXISTS idx_nft_transfers_block ON nft_transfers(block_number DESC)", [], )?; - conn.execute( + conn.execute( "CREATE INDEX IF NOT EXISTS idx_nft_transfers_token_id ON nft_transfers(token, token_id)", [], )?; - // Wallet activity table for efficient OR queries - conn.execute( - "CREATE TABLE IF NOT EXISTS nft_wallet_activity ( + // Wallet activity table for efficient OR queries + conn.execute( + "CREATE TABLE IF NOT EXISTS nft_wallet_activity ( id INTEGER PRIMARY KEY AUTOINCREMENT, wallet_address BLOB NOT NULL, token BLOB NOT NULL, @@ -364,24 +403,24 @@ impl Erc721Storage { block_number TEXT NOT NULL, FOREIGN KEY (transfer_id) REFERENCES nft_transfers(id) )", - [], - )?; + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_wallet_activity_wallet_block + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_wallet_activity_wallet_block ON nft_wallet_activity(wallet_address, block_number DESC)", - [], - )?; + [], + )?; - conn.execute( - "CREATE INDEX IF NOT EXISTS idx_nft_wallet_activity_wallet_token + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_nft_wallet_activity_wallet_token ON nft_wallet_activity(wallet_address, token, block_number DESC)", - [], - )?; + [], + )?; - // Approvals (single token) - conn.execute( - "CREATE TABLE IF NOT EXISTS nft_approvals ( + // Approvals (single token) + conn.execute( + "CREATE TABLE IF NOT EXISTS nft_approvals ( id INTEGER PRIMARY KEY AUTOINCREMENT, token BLOB NOT NULL, token_id BLOB NOT NULL, @@ -391,12 +430,12 @@ impl Erc721Storage { tx_hash BLOB NOT NULL, timestamp TEXT )", - [], - )?; + [], + )?; - // Operator approvals (all tokens) - conn.execute( - "CREATE TABLE IF NOT EXISTS nft_operators ( + // Operator approvals (all tokens) + conn.execute( + "CREATE TABLE IF NOT EXISTS nft_operators ( id INTEGER PRIMARY KEY AUTOINCREMENT, token BLOB NOT NULL, owner BLOB NOT NULL, @@ -407,21 +446,21 @@ impl Erc721Storage { timestamp TEXT, UNIQUE(token, owner, operator) )", - [], - )?; + [], + )?; - // Token metadata table - conn.execute( - "CREATE TABLE IF NOT EXISTS token_metadata ( + // Token metadata table + conn.execute( + "CREATE TABLE IF NOT EXISTS token_metadata ( token BLOB PRIMARY KEY, name TEXT, symbol TEXT, total_supply BLOB )", - [], - )?; + [], + )?; - conn.execute_batch( + conn.execute_batch( "CREATE TABLE IF NOT EXISTS token_uris ( token BLOB NOT NULL, token_id BLOB NOT NULL, @@ -480,25 +519,54 @@ impl Erc721Storage { CREATE INDEX IF NOT EXISTS idx_facet_token_map_token_token_id_key_value ON facet_token_map(token, token_id, facet_key_id, facet_value_id);", )?; - tracing::info!(target: "torii_erc721::storage", db_path = %db_path, "ERC721 database initialized"); + tracing::info!(target: "torii_erc721::storage", db_path = %db_path, "ERC721 database initialized"); + + StorageRuntime::Sqlite(SqliteStorageRuntime { + conn: Arc::new(Mutex::new(conn)), + }) + }; Ok(Self { - backend: StorageBackend::Sqlite, - conn: Arc::new(Mutex::new(conn)), - pg_conn: None, + db: Erc721Db::new(pool, runtime), }) } + pub fn pool(&self) -> &DbPool { + &self.db.backend + } + + fn is_postgres(&self) -> bool { + self.db.runtime.backend() == StorageBackend::Postgres + } + + fn sqlite_conn(&self) -> Result> { + match &self.db.runtime { + StorageRuntime::Sqlite(runtime) => Ok(runtime.conn.lock().unwrap()), + StorageRuntime::Postgres(_) => Err(anyhow::anyhow!( + "SQLite connection not initialized for ERC721 storage" + )), + } + } + + #[cfg(test)] + fn with_sqlite_conn( + &self, + f: impl FnOnce(&Connection) -> Result, + ) -> Result { + let conn = self.sqlite_conn()?; + Ok(f(&conn)?) + } + /// Insert multiple transfers and update ownership in a single transaction pub async fn insert_transfers_batch(&self, transfers: &[NftTransferData]) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_insert_transfers_batch(transfers).await; } if transfers.is_empty() { return Ok(0); } - let mut conn = self.conn.lock().unwrap(); + let mut conn = self.sqlite_conn()?; let tx = conn.transaction()?; let mut inserted = 0; @@ -602,14 +670,14 @@ impl Erc721Storage { &self, approvals: &[OperatorApprovalData], ) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_insert_operator_approvals_batch(approvals).await; } if approvals.is_empty() { return Ok(0); } - let mut conn = self.conn.lock().unwrap(); + let mut conn = self.sqlite_conn()?; let tx = conn.transaction()?; let mut inserted = 0; @@ -657,14 +725,14 @@ impl Erc721Storage { cursor: Option, limit: u32, ) -> Result<(Vec, Option)> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self .pg_get_transfers_filtered( wallet, from, to, tokens, token_ids, block_from, block_to, cursor, limit, ) .await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let mut query = String::new(); let mut params_vec: Vec> = Vec::new(); @@ -781,10 +849,10 @@ impl Erc721Storage { /// Get current owner of a specific NFT pub async fn get_owner(&self, token: Felt, token_id: U256) -> Result> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_owner(token, token_id).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let result: Result, _> = conn.query_row( "SELECT owner FROM nft_ownership WHERE token = ? AND token_id = ?", @@ -807,12 +875,12 @@ impl Erc721Storage { cursor: Option, limit: u32, ) -> Result<(Vec, Option)> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self .pg_get_ownership_by_owner(owner, tokens, cursor, limit) .await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let mut query = String::from( "SELECT id, token, token_id, owner, block_number FROM nft_ownership WHERE owner = ?", @@ -890,7 +958,7 @@ impl Erc721Storage { let page_fetch = page_limit as i64 + 1; let facet_limit = facet_limit.clamp(1, 1000) as i64; - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { let resolved_filters = self.pg_resolve_facet_filters(token, &filters).await?; let Some(resolved_filters) = resolved_filters else { return Ok(TokenAttributeQueryResult { @@ -912,7 +980,7 @@ impl Erc721Storage { .await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let resolved_filters = self.sqlite_resolve_facet_filters(&conn, token, &filters)?; let Some(resolved_filters) = resolved_filters else { return Ok(TokenAttributeQueryResult { @@ -936,10 +1004,10 @@ impl Erc721Storage { /// Get transfer count pub async fn get_transfer_count(&self) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_transfer_count().await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let count: i64 = conn.query_row("SELECT COUNT(*) FROM nft_transfers", [], |row| row.get(0))?; Ok(count as u64) @@ -947,10 +1015,10 @@ impl Erc721Storage { /// Get unique token contract count pub async fn get_token_count(&self) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_token_count().await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let count: i64 = conn.query_row( "SELECT COUNT(DISTINCT token) FROM nft_transfers", [], @@ -961,10 +1029,10 @@ impl Erc721Storage { /// Get unique NFT count pub async fn get_nft_count(&self) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_nft_count().await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let count: i64 = conn.query_row("SELECT COUNT(*) FROM nft_ownership", [], |row| row.get(0))?; Ok(count as u64) @@ -972,10 +1040,10 @@ impl Erc721Storage { /// Get latest block number indexed pub async fn get_latest_block(&self) -> Result> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_latest_block().await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let block: Option = conn .query_row("SELECT MAX(block_number) FROM nft_transfers", [], |row| { row.get(0) @@ -989,10 +1057,10 @@ impl Erc721Storage { /// Check if metadata exists for a token pub async fn has_token_metadata(&self, token: Felt) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_has_token_metadata(token).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let token_blob = felt_to_blob(token); let count: i64 = conn.query_row( "SELECT COUNT(*) FROM token_metadata @@ -1012,11 +1080,11 @@ impl Erc721Storage { if tokens.is_empty() { return Ok(HashSet::new()); } - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_has_token_metadata_batch(tokens).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let mut existing = HashSet::with_capacity(tokens.len()); for chunk in tokens.chunks(SQLITE_TOKEN_BATCH_SIZE) { @@ -1054,12 +1122,12 @@ impl Erc721Storage { symbol: Option<&str>, total_supply: Option, ) -> Result<()> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self .pg_upsert_token_metadata(token, name, symbol, total_supply) .await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let token_blob = felt_to_blob(token); let supply_blob = total_supply.map(u256_to_blob); conn.execute( @@ -1079,10 +1147,10 @@ impl Erc721Storage { &self, token: Felt, ) -> Result, Option, Option)>> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_token_metadata(token).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let token_blob = felt_to_blob(token); let result = conn .query_row( @@ -1103,7 +1171,7 @@ impl Erc721Storage { pub async fn get_all_token_metadata( &self, ) -> Result, Option, Option)>> { - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let mut stmt = conn.prepare_cached("SELECT token, name, symbol, total_supply FROM token_metadata")?; let rows = stmt.query_map([], |row| { @@ -1132,10 +1200,10 @@ impl Erc721Storage { Vec<(Felt, Option, Option, Option)>, Option, )> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_token_metadata_paginated(cursor, limit).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let fetch_limit = limit.clamp(1, 1000) as usize + 1; let mut out = if let Some(cursor_token) = cursor { @@ -1196,10 +1264,10 @@ impl Erc721Storage { /// Returns true if a token URI row exists for `(token, token_id)`. pub async fn has_token_uri(&self, token: Felt, token_id: U256) -> Result { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_has_token_uri(token, token_id).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let token_blob = felt_to_blob(token); let token_id_blob = u256_to_blob(token_id); let count: i64 = conn.query_row( @@ -1219,11 +1287,11 @@ impl Erc721Storage { if tokens.is_empty() { return Ok(HashSet::new()); } - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_has_token_uri_batch(tokens).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let mut existing = HashSet::with_capacity(tokens.len()); for chunk in tokens.chunks(SQLITE_TOKEN_PAIR_BATCH_SIZE) { @@ -1256,10 +1324,10 @@ impl Erc721Storage { &self, token: Felt, ) -> Result, Option)>> { - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_token_uris_by_contract(token).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let token_blob = felt_to_blob(token); let mut stmt = conn.prepare_cached( "SELECT token_id, uri, metadata_json @@ -1284,11 +1352,11 @@ impl Erc721Storage { if token_ids.is_empty() { return Ok(Vec::new()); } - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { return self.pg_get_token_uris_batch(token, token_ids).await; } - let conn = self.conn.lock().unwrap(); + let conn = self.sqlite_conn()?; let token_blob = felt_to_blob(token); let mut rows_out = Vec::new(); for chunk in token_ids.chunks(SQLITE_TOKEN_BATCH_SIZE) { @@ -1319,11 +1387,12 @@ impl Erc721Storage { } async fn pg_client(&self) -> Result> { - let conn = self - .pg_conn - .as_ref() - .ok_or_else(|| anyhow::anyhow!("PostgreSQL connection not initialized"))?; - Ok(conn.lock().await) + match &self.db.runtime { + StorageRuntime::Postgres(runtime) => Ok(runtime.conn.lock().await), + StorageRuntime::Sqlite(_) => { + Err(anyhow::anyhow!("PostgreSQL connection not initialized")) + } + } } fn pg_next_param( @@ -2329,6 +2398,21 @@ impl Erc721Storage { } } +fn db_pool_url(db_path: &str) -> String { + if db_path == ":memory:" || db_path == "sqlite::memory:" { + return "sqlite::memory:".to_owned(); + } + + if db_path.starts_with("postgres://") + || db_path.starts_with("postgresql://") + || db_path.starts_with("sqlite:") + { + return db_path.to_owned(); + } + + format!("sqlite://{db_path}?mode=rwc") +} + #[async_trait::async_trait] impl TokenUriStore for Erc721Storage { async fn store_token_uris_batch(&self, results: &[TokenUriResult]) -> Result<()> { @@ -2342,14 +2426,14 @@ impl TokenUriStore for Erc721Storage { .collect::>(); let token_blobs = results .iter() - .map(|result| felt_to_blob(result.contract)) + .map(|result| result.contract.to_be_bytes_vec()) .collect::>(); let token_id_blobs = results .iter() - .map(|result| u256_to_blob(result.token_id)) + .map(|result| result.token_id.to_big_endian().to_vec()) .collect::>(); - if self.backend == StorageBackend::Postgres { + if self.is_postgres() { let mut client = self.pg_client().await?; let mut changed_indexes = Vec::new(); for (idx, result) in results.iter().enumerate() { @@ -2429,7 +2513,7 @@ impl TokenUriStore for Erc721Storage { return Ok(()); } - let mut conn = self.conn.lock().unwrap(); + let mut conn = self.sqlite_conn()?; let mut changed_indexes = Vec::new(); for (idx, result) in results.iter().enumerate() { if !sqlite_token_uri_state_matches(&conn, result, &expected_attributes[idx])? { @@ -2942,8 +3026,8 @@ fn sqlite_token_uri_state_matches( result: &TokenUriResult, expected_attributes: &[(String, String)], ) -> Result { - let token_blob = felt_to_blob(result.contract); - let token_id_blob = u256_to_blob(result.token_id); + let token_blob = result.contract.to_be_bytes_vec(); + let token_id_blob = result.token_id.to_big_endian().to_vec(); let row = conn.query_row( "SELECT uri, metadata_json FROM token_uris WHERE token = ?1 AND token_id = ?2", params![&token_blob, &token_id_blob], @@ -2984,8 +3068,8 @@ async fn pg_token_uri_state_matches( result: &TokenUriResult, expected_attributes: &[(String, String)], ) -> Result { - let token_blob = felt_to_blob(result.contract); - let token_id_blob = u256_to_blob(result.token_id); + let token_blob = result.contract.to_be_bytes_vec(); + let token_id_blob = result.token_id.to_big_endian().to_vec(); let row = client .query_opt( "SELECT uri, metadata_json FROM erc721.token_uris WHERE token = $1 AND token_id = $2", @@ -3033,6 +3117,16 @@ fn sanitize_metadata_text(input: &str) -> Option { mod tests { use super::*; + fn raw_contract() -> starknet_types_raw::Felt { + starknet_types_raw::Felt::from_hex_unchecked( + "0x046da8955829adf2bda310099a0063451923f02e648cf25a1203aac6335cf0e4", + ) + } + + fn primitive_token_id(value: u64) -> primitive_types::U256 { + primitive_types::U256::from(value) + } + fn temp_db_path(test_name: &str) -> String { let nanos = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -3078,10 +3172,8 @@ mod tests { let db_path = temp_db_path("token-uri-noop"); let storage = Erc721Storage::new(&db_path).await.expect("create storage"); let result = TokenUriResult { - contract: Felt::from_hex_unchecked( - "0x046da8955829adf2bda310099a0063451923f02e648cf25a1203aac6335cf0e4", - ), - token_id: U256::from(1u64), + contract: raw_contract(), + token_id: primitive_token_id(1), uri: Some("ipfs://beasts/1".to_owned()), metadata_json: Some( r#"{"name":"Beast #1","attributes":[{"trait_type":"Class","value":"Wolf"}]}"# @@ -3094,38 +3186,45 @@ mod tests { .await .expect("insert token uri"); - { - let conn = storage.conn.lock().unwrap(); - conn.execute( - "UPDATE token_uris SET updated_at = '123' WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], - ) + storage + .with_sqlite_conn(|conn| { + conn.execute( + "UPDATE token_uris SET updated_at = '123' WHERE token = ?1 AND token_id = ?2", + params![ + result.contract.to_be_bytes_vec(), + result.token_id.to_big_endian().to_vec() + ], + )?; + Ok(()) + }) .expect("set sentinel updated_at"); - } storage .store_token_uri(&result) .await .expect("store unchanged token uri"); - let (updated_at, attr_count): (String, i64) = { - let conn = storage.conn.lock().unwrap(); - let updated_at: String = conn - .query_row( + let (updated_at, attr_count): (String, i64) = storage + .with_sqlite_conn(|conn| { + let updated_at: String = conn.query_row( "SELECT updated_at FROM token_uris WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], + params![ + result.contract.to_be_bytes_vec(), + result.token_id.to_big_endian().to_vec() + ], |row| row.get(0), - ) - .expect("read updated_at"); - let attr_count: i64 = conn - .query_row( + )?; + let attr_count: i64 = conn.query_row( "SELECT COUNT(*) FROM token_attributes WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], + params![ + result.contract.to_be_bytes_vec(), + result.token_id.to_big_endian().to_vec() + ], |row| row.get(0), - ) - .expect("count attributes"); - (updated_at, attr_count) - }; + )?; + Ok((updated_at, attr_count)) + }) + .expect("read sqlite assertions"); assert_eq!(updated_at, "123"); assert_eq!(attr_count, 1); @@ -3138,10 +3237,8 @@ mod tests { let db_path = temp_db_path("token-uri-backfill"); let storage = Erc721Storage::new(&db_path).await.expect("create storage"); let result = TokenUriResult { - contract: Felt::from_hex_unchecked( - "0x046da8955829adf2bda310099a0063451923f02e648cf25a1203aac6335cf0e4", - ), - token_id: U256::from(2u64), + contract: raw_contract(), + token_id: primitive_token_id(2), uri: Some("ipfs://beasts/2".to_owned()), metadata_json: Some( r#"{"name":"Beast #2","attributes":[{"trait_type":"Class","value":"Bear"}]}"# @@ -3154,29 +3251,36 @@ mod tests { .await .expect("insert token uri"); - { - let conn = storage.conn.lock().unwrap(); - conn.execute( - "DELETE FROM token_attributes WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], - ) + storage + .with_sqlite_conn(|conn| { + conn.execute( + "DELETE FROM token_attributes WHERE token = ?1 AND token_id = ?2", + params![ + result.contract.to_be_bytes_vec(), + result.token_id.to_big_endian().to_vec() + ], + )?; + Ok(()) + }) .expect("delete attributes"); - } storage .store_token_uri(&result) .await .expect("repair missing attributes"); - let attr_count: i64 = { - let conn = storage.conn.lock().unwrap(); - conn.query_row( - "SELECT COUNT(*) FROM token_attributes WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], - |row| row.get(0), - ) - .expect("count attributes") - }; + let attr_count: i64 = storage + .with_sqlite_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM token_attributes WHERE token = ?1 AND token_id = ?2", + params![ + result.contract.to_be_bytes_vec(), + result.token_id.to_big_endian().to_vec() + ], + |row| row.get(0), + ) + }) + .expect("count attributes"); assert_eq!(attr_count, 1); @@ -3188,10 +3292,8 @@ mod tests { let db_path = temp_db_path("token-uri-facet-backfill"); let storage = Erc721Storage::new(&db_path).await.expect("create storage"); let result = TokenUriResult { - contract: Felt::from_hex_unchecked( - "0x046da8955829adf2bda310099a0063451923f02e648cf25a1203aac6335cf0e4", - ), - token_id: U256::from(3u64), + contract: raw_contract(), + token_id: primitive_token_id(3), uri: Some("ipfs://beasts/3".to_owned()), metadata_json: Some( r#"{"name":"Beast #3","attributes":[{"trait_type":"Class","value":"Wolf"},{"trait_type":"Color","value":"Red"}]}"# @@ -3204,45 +3306,45 @@ mod tests { .await .expect("insert token uri"); - { - let conn = storage.conn.lock().unwrap(); - conn.execute( - "UPDATE facet_values SET token_count = '0' WHERE id IN ( - SELECT facet_value_id FROM facet_token_map WHERE token = ?1 AND token_id = ?2 - )", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], - ) - .expect("zero facet counts"); - conn.execute( - "DELETE FROM facet_token_map WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], - ) + storage + .with_sqlite_conn(|conn| { + conn.execute( + "UPDATE facet_values SET token_count = '0' WHERE id IN ( + SELECT facet_value_id FROM facet_token_map WHERE token = ?1 AND token_id = ?2 + )", + params![result.contract.to_be_bytes_vec(), result.token_id.to_big_endian().to_vec()], + )?; + conn.execute( + "DELETE FROM facet_token_map WHERE token = ?1 AND token_id = ?2", + params![result.contract.to_be_bytes_vec(), result.token_id.to_big_endian().to_vec()], + )?; + Ok(()) + }) .expect("delete facet mappings"); - } storage .store_token_uri(&result) .await .expect("repair missing facets"); - let (facet_map_count, positive_counts): (i64, i64) = { - let conn = storage.conn.lock().unwrap(); - let facet_map_count: i64 = conn - .query_row( + let (facet_map_count, positive_counts): (i64, i64) = storage + .with_sqlite_conn(|conn| { + let facet_map_count: i64 = conn.query_row( "SELECT COUNT(*) FROM facet_token_map WHERE token = ?1 AND token_id = ?2", - params![felt_to_blob(result.contract), u256_to_blob(result.token_id)], + params![ + result.contract.to_be_bytes_vec(), + result.token_id.to_big_endian().to_vec() + ], |row| row.get(0), - ) - .expect("count facet mappings"); - let positive_counts: i64 = conn - .query_row( + )?; + let positive_counts: i64 = conn.query_row( "SELECT COUNT(*) FROM facet_values WHERE CAST(token_count AS INTEGER) > 0", [], |row| row.get(0), - ) - .expect("count positive facet values"); - (facet_map_count, positive_counts) - }; + )?; + Ok((facet_map_count, positive_counts)) + }) + .expect("read facet assertions"); assert_eq!(facet_map_count, 2); assert_eq!(positive_counts, 2); @@ -3276,8 +3378,8 @@ mod tests { for (token_id, metadata_json) in fixtures { storage .store_token_uri(&TokenUriResult { - contract, - token_id: U256::from(token_id), + contract: contract.into(), + token_id: primitive_types::U256::from(token_id), uri: Some(format!("ipfs://beasts/{token_id}")), metadata_json: Some(metadata_json.to_owned()), }) diff --git a/crates/torii-erc721/src/synthetic.rs b/crates/torii-erc721/src/synthetic.rs index 0857f716..667a422f 100644 --- a/crates/torii-erc721/src/synthetic.rs +++ b/crates/torii-erc721/src/synthetic.rs @@ -5,12 +5,27 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt, U256}; +use starknet::core::types::U256; use starknet::macros::selector; +use starknet_types_raw::Felt; use torii::etl::extractor::{ExtractionBatch, SyntheticExtractor}; +use torii::etl::StarknetEvent; + +use crate::conversions::core_to_raw_felt; const EXTRACTOR_NAME: &str = "synthetic_erc721"; +fn raw_selector(name: &str) -> Felt { + core_to_raw_felt(match name { + "Transfer" => selector!("Transfer"), + "Approval" => selector!("Approval"), + "ApprovalForAll" => selector!("ApprovalForAll"), + "MetadataUpdate" => selector!("MetadataUpdate"), + "BatchMetadataUpdate" => selector!("BatchMetadataUpdate"), + _ => unreachable!(), + }) +} + /// Configuration for the synthetic ERC721 workload. #[derive(Debug, Clone)] pub struct SyntheticErc721Config { @@ -210,65 +225,60 @@ impl SyntheticErc721Extractor { let token_id = self.token_id_for(block_number, tx_index); let event = match self.event_type_for(tx_index) { - Erc721EventType::Transfer => EmittedEvent { - from_address: token, - keys: vec![ - selector!("Transfer"), + Erc721EventType::Transfer => StarknetEvent::new( + token, + vec![ + raw_selector("Transfer"), from, to, Felt::from(token_id.low()), Felt::from(token_id.high()), ], - data: vec![], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - }, - Erc721EventType::Approval => EmittedEvent { - from_address: token, - keys: vec![ - selector!("Approval"), + vec![], + block_number, + tx_hash, + ), + Erc721EventType::Approval => StarknetEvent::new( + token, + vec![ + raw_selector("Approval"), from, to, Felt::from(token_id.low()), Felt::from(token_id.high()), ], - data: vec![], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - }, - Erc721EventType::ApprovalForAll => EmittedEvent { - from_address: token, - keys: vec![selector!("ApprovalForAll"), from, to], - data: vec![Felt::from(1u64)], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - }, - Erc721EventType::MetadataUpdate => EmittedEvent { - from_address: token, - keys: vec![selector!("MetadataUpdate")], - data: vec![Felt::from(token_id.low()), Felt::from(token_id.high())], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - }, + vec![], + block_number, + tx_hash, + ), + Erc721EventType::ApprovalForAll => StarknetEvent::new( + token, + vec![raw_selector("ApprovalForAll"), from, to], + vec![Felt::from(1u64)], + block_number, + tx_hash, + ), + Erc721EventType::MetadataUpdate => StarknetEvent::new( + token, + vec![raw_selector("MetadataUpdate")], + vec![Felt::from(token_id.low()), Felt::from(token_id.high())], + block_number, + tx_hash, + ), Erc721EventType::BatchMetadataUpdate => { let to_id = token_id + U256::from(100u64); - EmittedEvent { - from_address: token, - keys: vec![selector!("BatchMetadataUpdate")], - data: vec![ + StarknetEvent::new( + token, + vec![raw_selector("BatchMetadataUpdate")], + vec![ Felt::from(token_id.low()), Felt::from(token_id.high()), Felt::from(to_id.low()), Felt::from(to_id.high()), ], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), - transaction_hash: tx_hash, - } + block_number, + tx_hash, + ) } }; @@ -376,11 +386,11 @@ mod tests { let mut extractor = SyntheticErc721Extractor::new(cfg).unwrap(); let batch = extractor.extract(None).await.unwrap(); - let transfer_selector = selector!("Transfer"); - let approval_selector = selector!("Approval"); - let approval_for_all_selector = selector!("ApprovalForAll"); - let metadata_update_selector = selector!("MetadataUpdate"); - let batch_metadata_update_selector = selector!("BatchMetadataUpdate"); + let transfer_selector = raw_selector("Transfer"); + let approval_selector = raw_selector("Approval"); + let approval_for_all_selector = raw_selector("ApprovalForAll"); + let metadata_update_selector = raw_selector("MetadataUpdate"); + let batch_metadata_update_selector = raw_selector("BatchMetadataUpdate"); let mut has_transfer = false; let mut has_approval = false; diff --git a/crates/torii-log-sink/Cargo.toml b/crates/torii-log-sink/Cargo.toml index 536984b5..ea6dd97e 100644 --- a/crates/torii-log-sink/Cargo.toml +++ b/crates/torii-log-sink/Cargo.toml @@ -4,35 +4,36 @@ version = "0.1.0" edition = "2021" [dependencies] -torii = { path = "../../" } -starknet = "0.17" +torii.workspace = true # Async -tokio = { version = "1", features = ["full"] } -tokio-stream = { version = "0.1", features = ["sync"] } -async-trait = "0.1" +tokio.workspace = true +tokio-stream.workspace = true +async-trait.workspace = true # gRPC and protobuf -tonic = "0.12" -prost = "0.13" -prost-types = "0.13" +tonic.workspace = true +prost.workspace = true +prost-types.workspace = true # HTTP (use torii's re-exported axum) -serde = { version = "1", features = ["derive"] } -serde_json = "1" +serde.workspace = true +serde_json.workspace = true # Logging -tracing = "0.1" +tracing.workspace = true # Utils -anyhow = "1" -chrono = "0.4" +anyhow.workspace = true +chrono.workspace = true + +starknet-types-raw.workspace = true [build-dependencies] -tonic-build = "0.12" +tonic-build.workspace = true [dev-dependencies] -tokio = { version = "1", features = ["full"] } +tokio.workspace = true [lints] workspace = true diff --git a/crates/torii-log-sink/README.md b/crates/torii-log-sink/README.md index 3cfe148d..a6c9aa88 100644 --- a/crates/torii-log-sink/README.md +++ b/crates/torii-log-sink/README.md @@ -1,174 +1,120 @@ -# Torii Log Sink +# torii-log-sink -A demonstration sink for Torii that collects and stores log entries from blockchain events. +A minimal, **in-memory** sink that turns decoded events into log entries, +publishes them on the EventBus, streams them over gRPC, and serves recent +entries over REST. Intended as a learning reference and a smoke-test sink +in the `multi_sink_example`. Data lives in a bounded `VecDeque` — old +entries fall off the back. -## Features +## Role in Torii -This sink demonstrates all three Torii extension points: +`LogSink` is the smallest possible real sink. It does not touch a database, +so binaries can drop it into a pipeline with zero infrastructure and still +observe the gRPC + EventBus + HTTP extension points. Use it alongside any +sink (`torii-sql-sink`, `torii-erc20`, etc.) to get a human-readable audit +log of events without provisioning a database. -1. **EventBus** - Publishes to central topic-based subscriptions (via `torii.Torii/Subscribe`) -2. **gRPC Service** - Provides `torii.sinks.log.LogSink` service with: - - `QueryLogs` - Query recent logs - - `SubscribeLogs` - Real-time streaming subscription -3. **REST HTTP** - Exposes: - - `GET /logs?limit=N` - Get recent logs (default: 5) - - `GET /logs/count` - Get total log count - -## Usage - -```rust -use std::sync::Arc; -use torii::{ToriiConfig, run}; -use torii_log_sink::{LogDecoder, LogSink}; -use tonic::transport::Server; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // 1. Create sink - let log_sink = LogSink::new(100); // Max 100 logs in memory - - // 2. Get the gRPC service implementation - let log_grpc_service = log_sink.get_grpc_service_impl(); - - // 3. Build gRPC router with log sink service - let grpc_router = { - use torii_log_sink::proto::log_sink_server::LogSinkServer; - - Server::builder() - .accept_http1(true) - .add_service(tonic_web::enable(LogSinkServer::new((*log_grpc_service).clone()))) - }; - - // 4. Create decoder - let log_decoder = Arc::new(LogDecoder::new(None)); // No filter - - // 5. Configure and run Torii - let config = ToriiConfig::builder() - .port(8080) - .add_sink_boxed(Box::new(log_sink)) - .add_decoder(log_decoder) - .with_grpc_router(grpc_router) - .build(); +## Architecture - run(config).await -} +```text ++---------------------------------------------------------+ +| LogSink | +| | +| +-------------+ +----------------+ +------------+ | +| | LogDecoder |-->| Envelope |-->| process() | | +| | log.entry | | LogEntry | +-----+------+ | +| +-------------+ +----------------+ | | +| v | +| +-----------------+--------+| +| | || +| v v| +| grpc_service.log_store() EventBus +| .add_log() publish_protobuf +| (VecDeque "logs" +| in Arc>) | +| | v| +| v gRPC topic +| LogSinkService "logs" +| (QueryLogs, | +| SubscribeLogs) | +| | | +| v | +| HTTP: /logs, /logs/count | ++---------------------------------------------------------+ ``` -## Protobuf Code Generation +## Deep Dive -The gRPC service definitions are in `proto/log.proto`. The Rust code is automatically generated during build. +### Public API -### Generated Files +| Item | File | Line | Purpose | +|---|---|---|---| +| `LogSink` | `src/lib.rs` | 36 | Zero-config in-memory sink | +| `LogSink::new(max_logs)` | `src/lib.rs` | 49 | Bounded ring buffer sized at construction | +| `LogSink::get_grpc_service_impl` | `src/lib.rs` | 90 | Returns `Arc` for `with_grpc_router` | +| `LogDecoder` | `src/decoder.rs` | — | Turns arbitrary events into `LogEntry` envelopes; optional filter | +| `LogEntry` | `src/decoder.rs` | — | In-memory body: `message`, `block_number`, `event_key` | +| `LogSinkService` | `src/grpc_service.rs` | — | Implements `QueryLogs`, `SubscribeLogs`; holds the `VecDeque` | +| `ProtoLogEntry`, `LogUpdate` | `src/proto` (generated) | — | Wire types from `proto/log.proto` | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs` | 11 | Re-exported bytes for reflection composition | -- **Location**: `src/generated/torii.sinks.log.rs` -- **Generated by**: `build.rs` (runs automatically with `cargo build`) -- **Includes**: Service trait, message types, client/server stubs +### Internal Modules -### How to Regenerate +- `lib.rs` — `Sink` impl; holds the counter and the gRPC service. +- `decoder.rs` — `LogDecoder` + `LogEntry` body; supports optional event-key filter. +- `grpc_service.rs` — `LogSinkService` with `VecDeque` storage and a `tokio::sync::broadcast` channel; server impl for `QueryLogs` / `SubscribeLogs`. +- `api.rs` — Axum handlers: `GET /logs?limit=N`, `GET /logs/count`. +- `generated/` — `torii.sinks.log.rs` + `log_descriptor.bin` from `proto/log.proto`. -The protobuf code is automatically regenerated when: +### Sink trait wiring -1. **Building the crate**: `cargo build -p torii-log-sink` -2. **Proto file changes**: Any modification to `proto/log.proto` triggers regeneration +| Method | Behavior | +|---|---| +| `name` | `"log"` | +| `interested_types` | `[TypeId::new("log.entry")]` | +| `process(envelopes, _batch)` | Stores via `log_store().add_log()` (evicts oldest when full), publishes on EventBus topic `"logs"`, broadcasts via gRPC channel | +| `topics` | One `TopicInfo { name: "logs", available_filters: [] }` | +| `build_routes` | Axum router: `GET /logs`, `GET /logs/count` | +| `initialize` | Stores the `event_bus` handle | -### Manual Regeneration +### Storage -If you need to force regeneration: +- In-memory `VecDeque` wrapped in `Arc>`. +- Configured size at construction (`LogSink::new(max_logs)`); overflow evicts oldest. +- No persistence — restart loses all state. -```bash -# Clean and rebuild -cargo clean -p torii-log-sink -cargo build -p torii-log-sink +### Interactions -# Or touch the proto file to trigger rebuild -touch crates/torii-log-sink/proto/log.proto -cargo build -p torii-log-sink -``` +- **Upstream (consumers)**: `examples/multi_sink_example`; any tutorial that wants visible side-effects without a DB. +- **Downstream deps**: `torii`, `tonic`, `prost`, `prost-types`, `tokio-stream`, `chrono`, `anyhow`, `tracing`. +- **Workspace deps**: `torii` only. -### Build Process - -The `build.rs` script: - -1. Creates `src/generated/` directory if it doesn't exist -2. Configures `tonic-build` to generate server code (no client) -3. Compiles `proto/log.proto` into Rust code -4. Outputs to `src/generated/torii.sinks.log.rs` - -See `build.rs` for implementation details. - -## Architecture - -``` -┌─────────────────────────────────────────────────┐ -│ LogSink │ -├─────────────────────────────────────────────────┤ -│ │ -│ ┌────────────────┐ ┌─────────────────┐ │ -│ │ LogDecoder │─────▶│ LogEntry │ │ -│ │ │ │ (in-memory) │ │ -│ │ Events → Logs │ │ VecDeque │ │ -│ └────────────────┘ └─────────────────┘ │ -│ │ │ -│ ┌─────────────┼──────────┐ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌────────┐ ┌────┐│ -│ │ EventBus │ │ gRPC │ │HTTP││ -│ │ (central) │ │ APIs │ │API ││ -│ └──────────────┘ └────────┘ └────┘│ -│ │ -└─────────────────────────────────────────────────┘ -``` - -## Storage - -- **In-Memory**: Uses `VecDeque` with configurable max size -- **Thread-Safe**: Wrapped in `Arc>` for concurrent access -- **Circular Buffer**: Oldest logs are evicted when max size is reached - -## gRPC Service Registration - -Due to Rust's type system limitations, gRPC services must be registered by the user: +### gRPC registration pattern ```rust -// User code (e.g., main.rs or examples/) +use torii::{ToriiConfig, run}; +use torii_log_sink::{LogDecoder, LogSink}; use torii_log_sink::proto::log_sink_server::LogSinkServer; use tonic::transport::Server; +use std::sync::Arc; -let log_sink = LogSink::new(100); -let service = log_sink.get_grpc_service_impl(); +let sink = LogSink::new(500); +let service = sink.get_grpc_service_impl(); let grpc_router = Server::builder() .accept_http1(true) .add_service(tonic_web::enable(LogSinkServer::new((*service).clone()))); -// Pass router to Torii let config = ToriiConfig::builder() - .add_sink_boxed(Box::new(log_sink)) + .add_sink_boxed(Box::new(sink)) + .add_decoder(Arc::new(LogDecoder::new(None))) .with_grpc_router(grpc_router) .build(); +run(config).await?; ``` -This pattern keeps all gRPC services on the same port while maintaining type safety. - -## Testing - -```bash -# Build -cargo build -p torii-log-sink - -# Run tests -cargo test -p torii-log-sink - -# Test gRPC (with server running) -grpcurl -plaintext localhost:8080 torii.sinks.log.LogSink/QueryLogs - -# Test HTTP -curl http://localhost:8080/logs?limit=5 -curl http://localhost:8080/logs/count -``` - -## See Also +### Extension Points -- **Example**: `examples/multi_sink_example` - Multi-sink usage (SqlSink + LogSink) -- **SQL Sink**: `crates/torii-sql-sink` - Similar architecture with SQLite storage +- Persist logs → replace `VecDeque` with a DB-backed store in `grpc_service.rs`; keep the `add_log` API shape so `Sink::process` stays untouched. +- Structured filters → widen `TopicInfo.available_filters` and add a filter fn to `EventBus::publish_protobuf` (pattern: see `torii-sql-sink::matches_filters`). +- New REST endpoints → add routes in `build_routes` (re-use the existing `LogSinkService` state). diff --git a/crates/torii-log-sink/src/decoder.rs b/crates/torii-log-sink/src/decoder.rs index 85452a52..316d247c 100644 --- a/crates/torii-log-sink/src/decoder.rs +++ b/crates/torii-log-sink/src/decoder.rs @@ -1,11 +1,9 @@ use anyhow::Result; use async_trait::async_trait; -use starknet::core::types::EmittedEvent; +use starknet_types_raw::Felt; use std::any::Any; -use torii::etl::{ - envelope::{Envelope, TypeId, TypedBody}, - Decoder, -}; +use torii::etl::envelope::{Envelope, TypeId, TypedBody}; +use torii::etl::{Decoder, EventContext}; /// Decoded log entry #[derive(Debug, Clone)] @@ -50,10 +48,15 @@ impl Decoder for LogDecoder { "log" } - async fn decode_event(&self, event: &EmittedEvent) -> Result> { + async fn decode( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> Result> { // Apply key filter if specified if let Some(ref filter) = self.key_filter { - let matches = event.keys.iter().any(|k| { + let matches = keys.iter().any(|k| { let key_hex = format!("{k:#x}"); key_hex.contains(filter) }); @@ -64,29 +67,29 @@ impl Decoder for LogDecoder { // Extract log message from event data. // For this example, we'll convert the first data field to a string representation. - let message = if event.data.is_empty() { + let message = if data.is_empty() { "Empty event data".to_string() } else { - format!("Event data: {:#x}", event.data[0]) + format!("Event data: {:#x}", data[0]) }; - let event_key = if event.keys.is_empty() { + let event_key = if keys.is_empty() { "no-key".to_string() } else { - format!("{:#x}", event.keys[0]) + format!("{:#x}", keys[0]) }; let log_entry = LogEntry { message, - block_number: event.block_number.unwrap_or(0), + block_number: context.block_number, event_key, }; // Create envelope with unique ID let envelope_id = format!( "log_{}_{}", - event.block_number.unwrap_or(0), - format!("{:#x}", event.transaction_hash) + context.block_number, + format!("{:#x}", context.transaction_hash) ); let envelope = Envelope::new( envelope_id, @@ -101,18 +104,18 @@ impl Decoder for LogDecoder { #[cfg(test)] mod tests { use super::*; - use starknet::core::types::Felt; + use starknet_types_raw::Felt; + use torii::etl::StarknetEvent; #[tokio::test] async fn test_decode_logs() { let decoder = LogDecoder::new(None); - let event = EmittedEvent { + let event = StarknetEvent { from_address: Felt::from(1u64), keys: vec![Felt::from(0x1234u64)], data: vec![Felt::from(0x5678u64)], - block_hash: None, - block_number: Some(100), + block_number: 100, transaction_hash: Felt::from(0xabcdu64), }; diff --git a/crates/torii-runtime-common/Cargo.toml b/crates/torii-runtime-common/Cargo.toml index 423d5e70..9cbc5ffd 100644 --- a/crates/torii-runtime-common/Cargo.toml +++ b/crates/torii-runtime-common/Cargo.toml @@ -10,6 +10,7 @@ tokio.workspace = true tokio-postgres = "0.7" tracing.workspace = true torii.workspace = true +torii-sql.workspace = true [lints] workspace = true diff --git a/crates/torii-runtime-common/README.md b/crates/torii-runtime-common/README.md index 853e155d..e0101b33 100644 --- a/crates/torii-runtime-common/README.md +++ b/crates/torii-runtime-common/README.md @@ -1,28 +1,84 @@ # torii-runtime-common -Shared runtime setup helpers for Torii binaries. +Shared **runtime** helpers used by every Torii binary. Where +`torii-config-common` validates CLI input, this crate resolves it into +concrete database URLs, initialises sinks, and handles destructive tasks +like dropping Postgres schemas for synthetic profilers. -## Database helpers +## Role in Torii -- `backend_from_url_or_path(...)` to classify Postgres vs SQLite. -- `resolve_single_db_setup(db_path, database_url)` for single-storage binaries. -- `resolve_token_db_setup(db_dir, engine_database_url, storage_database_url)` for multi-storage token binaries. +Each binary produces a `clap`-parsed `Config`, then calls helpers from this +crate to produce a `SingleDbSetup` / `TokenDbSetup`, wire up a throwaway +`EventBus` + `CommandBus` during `Sink::initialize`, and optionally reset +Postgres schemas between profiling runs. The resolved URLs are then fed to +`ToriiConfigBuilder::engine_database_url` and the sink-specific constructors. -## Sink helpers +## Architecture -- `initialize_sink(sink, database_root)` - - Standard sink initialization with EventBus and SinkContext. -- `drop_postgres_schemas(database_url, schemas, logging_target)` - - Shared schema reset utility used by synthetic profilers. +```text +CLI parsed Config (clap) + | + v ++---------------------------------+ +| database.rs | +| - backend_from_url_or_path |-- DbBackend::{Sqlite,Postgres} +| - resolve_single_db_setup |-- SingleDbSetup +| - resolve_token_db_setup |-- TokenDbSetup (engine+erc20+erc721+erc1155) +| - validate_uniform_backends | ++---------------------------------+ + | + v ++---------------------------------+ +| sink.rs | +| - initialize_sink |-- EventBus + SinkContext +| - initialize_sink_with_ |-- + CommandBus with user handlers +| command_handlers | +| - drop_postgres_schemas |-- schema reset for -synth bins ++---------------------------------+ + | + v ++---------------------------------+ +| token_support.rs | +| - InstalledTokenSupport |-- which token sinks to install +| - resolve_installed_token_... | ++---------------------------------+ + | + v + torii::ToriiConfigBuilder::build() → torii::run(...) +``` -## Example +## Deep Dive -```rust -use torii_runtime_common::database::resolve_token_db_setup; +### Public API -let db_setup = resolve_token_db_setup( - std::path::Path::new("./torii-data"), - config.database_url.as_deref(), - config.storage_database_url.as_deref(), -)?; -``` +| Item | File | Line | Purpose | +|---|---|---|---| +| `backend_from_url_or_path` | `database.rs` | 7 | Classify URL as `DbBackend::Postgres` or `DbBackend::Sqlite` | +| `validate_uniform_backends` | `database.rs` | 15 | Reject mixed Postgres/SQLite targets in one binary | +| `SingleDbSetup` | `database.rs` | 40 | `{ storage_url, engine_url, database_root }` | +| `resolve_single_db_setup` | `database.rs` | 46 | Derive engine path next to storage DB when not explicit | +| `TokenDbSetup` | `database.rs` | 71 | Engine + erc20 + erc721 + erc1155 URLs + each backend | +| `resolve_token_db_setup` | `database.rs` | 82 | Multi-DB resolver; rejects Postgres-engine + SQLite-storage mixes | +| `DEFAULT_SQLITE_MAX_CONNECTIONS` | `database.rs` | 5 | 500 — used by token sinks for connection pools | +| `initialize_sink` | `sink.rs` | 18 | `EventBus` + default `SinkContext`, drops its command bus immediately | +| `initialize_sink_with_command_handlers` | `sink.rs` | 24 | Keeps the `CommandBus` alive; returns `InitializedSinkRuntime` for shutdown | +| `drop_postgres_schemas` | `sink.rs` | 46 | `DROP SCHEMA ... CASCADE` helper for synthetic profilers | +| `InstalledTokenSupport` | `token_support.rs` | 1 | `{erc20, erc721, erc1155}` flags plus `any()` | +| `resolve_installed_token_support` | `token_support.rs` | 14 | Compose explicit targets with `--include-external-contracts` | + +### Internal Modules + +- `database` — URL classification + multi-target DB resolution with cross-backend validation. +- `sink` — builds a short-lived `EventBus`/`CommandBus` for `Sink::initialize` so binaries don’t duplicate that plumbing. +- `token_support` — little helper deciding which of the three token sinks the binary installs. + +### Interactions + +- **Upstream (consumers)**: `bins/torii-arcade`, `bins/torii-introspect-bin`, `bins/torii-tokens`, the three `-synth` profilers, `crates/torii-controllers-sink`, `crates/torii-ecs-sink` (all need `initialize_sink` or DB-setup helpers). +- **Downstream deps**: `torii`, `torii-sql` (for `DbBackend`), `tokio-postgres` (schema reset), `anyhow`, `tracing`. +- **Workspace deps**: `torii`, `torii-sql`. + +### Extension Points + +- New shared DB layout → add a `resolve_*_db_setup` returning a typed struct. Keep `validate_uniform_backends` as the one place that enforces "don’t mix backends". +- New global runtime chore (e.g., temp-dir prep, seccomp hook) belongs here, not in individual binaries. diff --git a/crates/torii-runtime-common/src/database.rs b/crates/torii-runtime-common/src/database.rs index ef7aeec0..ef409403 100644 --- a/crates/torii-runtime-common/src/database.rs +++ b/crates/torii-runtime-common/src/database.rs @@ -1,26 +1,21 @@ -use anyhow::{bail, Result}; +use anyhow::{bail, Result as AnyResult}; use std::path::{Path, PathBuf}; +use torii_sql::connection::DbBackend; pub const DEFAULT_SQLITE_MAX_CONNECTIONS: u32 = 500; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum DatabaseBackend { - Postgres, - Sqlite, -} - -pub fn backend_from_url_or_path(value: &str) -> DatabaseBackend { +pub fn backend_from_url_or_path(value: &str) -> DbBackend { if value.starts_with("postgres://") || value.starts_with("postgresql://") { - DatabaseBackend::Postgres + DbBackend::Postgres } else { - DatabaseBackend::Sqlite + DbBackend::Sqlite } } pub fn validate_uniform_backends( named_urls: &[(&str, &str)], mixed_backend_message: &str, -) -> Result { +) -> AnyResult { let Some((_, first_url)) = named_urls.first() else { bail!("at least one database URL must be provided"); }; @@ -78,17 +73,17 @@ pub struct TokenDbSetup { pub erc20_url: String, pub erc721_url: String, pub erc1155_url: String, - pub engine_backend: DatabaseBackend, - pub erc20_backend: DatabaseBackend, - pub erc721_backend: DatabaseBackend, - pub erc1155_backend: DatabaseBackend, + pub engine_backend: DbBackend, + pub erc20_backend: DbBackend, + pub erc721_backend: DbBackend, + pub erc1155_backend: DbBackend, } pub fn resolve_token_db_setup( db_dir: &Path, engine_database_url: Option<&str>, storage_database_url: Option<&str>, -) -> Result { +) -> AnyResult { let engine_url = engine_database_url.map_or_else( || db_dir.join("engine.db").to_string_lossy().to_string(), ToOwned::to_owned, @@ -119,10 +114,10 @@ pub fn resolve_token_db_setup( if engine_database_url .map(backend_from_url_or_path) - .is_some_and(|backend| backend == DatabaseBackend::Postgres) - && (erc20_backend != DatabaseBackend::Postgres - || erc721_backend != DatabaseBackend::Postgres - || erc1155_backend != DatabaseBackend::Postgres) + .is_some_and(|backend| backend == DbBackend::Postgres) + && (erc20_backend != DbBackend::Postgres + || erc721_backend != DbBackend::Postgres + || erc1155_backend != DbBackend::Postgres) { bail!( "Engine is configured for Postgres but one or more token storages resolved to SQLite. Set --storage-database-url to the same Postgres URL." @@ -161,8 +156,8 @@ mod tests { fn resolves_sqlite_defaults() { let db_dir = Path::new("./torii-data"); let setup = resolve_token_db_setup(db_dir, None, None).unwrap(); - assert_eq!(setup.engine_backend, DatabaseBackend::Sqlite); - assert_eq!(setup.erc20_backend, DatabaseBackend::Sqlite); + assert_eq!(setup.engine_backend, DbBackend::Sqlite); + assert_eq!(setup.erc20_backend, DbBackend::Sqlite); assert!(setup.engine_url.ends_with("engine.db")); assert!(setup.erc20_url.ends_with("erc20.db")); } @@ -174,8 +169,9 @@ mod tests { db_dir, Some("postgres://localhost/torii"), Some("./torii-data"), - ) - .expect_err("expected mixed backend validation error"); + ); + println!("{err:?}"); + let err = err.expect_err("expected mixed backend validation error"); assert!(err .to_string() .contains("Engine is configured for Postgres")); @@ -190,8 +186,8 @@ mod tests { Some("postgres://localhost/torii"), ) .unwrap(); - assert_eq!(setup.engine_backend, DatabaseBackend::Postgres); - assert_eq!(setup.erc721_backend, DatabaseBackend::Postgres); + assert_eq!(setup.engine_backend, DbBackend::Postgres); + assert_eq!(setup.erc721_backend, DbBackend::Postgres); } #[test] @@ -204,7 +200,7 @@ mod tests { "mixed backends", ) .unwrap(); - assert_eq!(backend, DatabaseBackend::Postgres); + assert_eq!(backend, DbBackend::Postgres); } #[test] diff --git a/crates/torii-sql-sink/Cargo.toml b/crates/torii-sql-sink/Cargo.toml index 97443de8..5343a2ab 100644 --- a/crates/torii-sql-sink/Cargo.toml +++ b/crates/torii-sql-sink/Cargo.toml @@ -5,38 +5,43 @@ edition = "2021" [dependencies] # Torii core -torii = { path = "../.." } +torii.workspace = true # Async -tokio = { version = "1.35", features = ["full"] } -async-trait = "0.1" +tokio.workspace = true +async-trait.workspace = true # Database -sqlx = { version = "0.8", features = ["runtime-tokio-rustls", "sqlite", "postgres", "any"] } +sqlx = { workspace = true, features = [ + "runtime-tokio-rustls", + "sqlite", + "postgres", + "any", +] } # gRPC -tonic = "0.12" -tonic-web = "0.12" -prost = "0.13" -prost-types = "0.13" -async-stream = "0.3" -futures = "0.3" +tonic.workspace = true +tonic-web.workspace = true +prost.workspace = true +prost-types.workspace = true +async-stream.workspace = true +futures.workspace = true # Web framework -axum = { version = "0.7", features = ["json"] } -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +axum = { workspace = true, features = ["json"] } +serde.workspace = true +serde_json.workspace = true # Blockchain -starknet = "0.17" +starknet-types-raw.workspace = true # Utilities -anyhow = "1.0" -tracing = "0.1" -chrono = "0.4" +anyhow.workspace = true +tracing.workspace = true +chrono.workspace = true [build-dependencies] -tonic-build = "0.12" +tonic-build.workspace = true [lints] workspace = true diff --git a/crates/torii-sql-sink/README.md b/crates/torii-sql-sink/README.md index 1dac68ed..ca859fb2 100644 --- a/crates/torii-sql-sink/README.md +++ b/crates/torii-sql-sink/README.md @@ -1,217 +1,147 @@ -# Torii SQL Sink +# torii-sql-sink -A demonstration sink for Torii that processes SQL operations and stores them in SQLite or PostgreSQL. +The reference sink. Persists decoded envelopes as SQL operations into SQLite +or PostgreSQL, publishes updates on the EventBus, serves a +`torii.sinks.sql.SqlSink` gRPC service, and exposes `/sql/query` and +`/sql/events` REST endpoints. Every new sink author should read this one +first — it is the canonical implementation of the three-extension-point +pattern. -## Features +## Role in Torii -This sink demonstrates all three Torii extension points: +Torii's core library (`src/etl/sink/mod.rs:74`) defines the `Sink` trait but +does not persist anything. `torii-sql-sink` is the first real consumer: it +receives `Envelope`s decoded by a paired `SqlDecoder`, writes a row per +envelope, and fans the write out to three downstream channels (EventBus, +sink-specific gRPC broadcast, SQL-queryable table). It is used as the +primary demo sink in `examples/simple_sql_sink` and `examples/multi_sink_example`. -1. **EventBus** - Publishes to central topic-based subscriptions (via `torii.Torii/Subscribe`) -2. **gRPC Service** - Provides `torii.sinks.sql.SqlSink` service with: - - `Query` - Execute SQL queries - - `StreamQuery` - Stream large result sets - - `GetSchema` - Get database schema - - `Subscribe` - Real-time operation updates -3. **REST HTTP** - Exposes: - - `POST /sql/query` - Execute SQL queries - - `GET /sql/events` - List all SQL operations - -## Usage - -```rust -use std::sync::Arc; -use torii::{ToriiConfig, run}; -use torii_sql_sink::{SqlDecoder, SqlSink}; -use tonic::transport::Server; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // 1. Create sink - let sql_sink = SqlSink::new("postgres://torii:torii@localhost:5432/torii").await?; - - // 2. Get the gRPC service implementation - let sql_grpc_service = sql_sink.get_grpc_service_impl(); - - // 3. Build gRPC router with SQL sink service - let grpc_router = { - use torii_sql_sink::proto::sql_sink_server::SqlSinkServer; - - Server::builder() - .accept_http1(true) - .add_service(tonic_web::enable(SqlSinkServer::new((*sql_grpc_service).clone()))) - }; - - // 4. Create decoder - let sql_decoder = Arc::new(SqlDecoder::new(Vec::new())); // No filters - - // 5. Get sample events - let sample_events = SqlSink::generate_sample_events(); - - // 6. Configure and run Torii - let config = ToriiConfig::builder() - .port(8080) - .add_sink_boxed(Box::new(sql_sink)) - .add_decoder(sql_decoder) - .with_grpc_router(grpc_router) - .with_sample_events(sample_events) - .build(); +## Architecture - run(config).await -} +```text ++----------------------------------------------------------------+ +| SqlSink | +| | +| +-------------+ +---------------+ +------------------+ | +| | SqlDecoder |--->| Envelope |--->| Sink::process | | +| | (sql.insert, | sql.insert/ | | INSERT INTO | | +| | sql.update) | sql.update | | sql_operation | | +| +-------------+ +---------------+ | (SQLite/Postgres)| | +| +------------------+ | +| | | +| +---------------------------+----------+ | +| | | | | +| v v v | +| EventBus.publish_ grpc_service HTTP| +| protobuf("sql", ...) .update_tx.send() api| +| with matches_filters (broadcast chan) ↓ | +| | | /sql/ *| +| v v | +| gRPC clients subscribed SqlSinkService | +| to torii.Torii/Subscribe (tonic) | ++----------------------------------------------------------------+ ``` -## Protobuf Code Generation - -The gRPC service definitions are in `proto/sql.proto`. The Rust code is automatically generated during build. - -### Generated Files - -- **Location**: `src/generated/torii.sinks.sql.rs` -- **Generated by**: `build.rs` (runs automatically with `cargo build`) -- **Includes**: Service trait, message types, client/server stubs - -### How to Regenerate - -The protobuf code is automatically regenerated when: - -1. **Building the crate**: `cargo build -p torii-sql-sink` -2. **Proto file changes**: Any modification to `proto/sql.proto` triggers regeneration - -### Manual Regeneration - -If you need to force regeneration: - -```bash -# Clean and rebuild -cargo clean -p torii-sql-sink -cargo build -p torii-sql-sink - -# Or touch the proto file to trigger rebuild -touch crates/torii-sql-sink/proto/sql.proto -cargo build -p torii-sql-sink +## Deep Dive + +### Public API + +| Item | File | Line | Purpose | +|---|---|---|---| +| `SqlSink` | `src/lib.rs` | 44 | The `Sink` impl — holds `Arc>`, the gRPC service, and the EventBus handle | +| `SqlSink::new(database_url)` | `src/lib.rs` | 94 | Detects SQLite vs Postgres, runs the embedded schema | +| `SqlSink::get_grpc_service_impl` | `src/lib.rs` | 83 | Returns `Arc` for registration via `with_grpc_router` | +| `SqlSink::generate_sample_events` | `src/lib.rs` | 54 | Test fixture events | +| `SqlDecoder` / `SqlInsert` / `SqlUpdate` | `src/decoder.rs` | — | Pair decoder; emits `sql.insert` / `sql.update` envelopes | +| `SqlSinkService` | `src/grpc_service.rs` | 18 | Implements `Query`, `StreamQuery`, `GetSchema`, `Subscribe` | +| `ProtoSqlOperation` / `SqlOperationUpdate` | `src/proto` (generated) | — | Wire types from `proto/sql.proto` | +| `FILE_DESCRIPTOR_SET` | `src/lib.rs` | 12 | Re-exported bytes for reflection composition | + +### Internal Modules + +- `lib.rs` — `Sink` impl. `process` handles both envelope types, broadcasts to EventBus + gRPC, writes the row. +- `decoder.rs` — `SqlDecoder`, `SqlInsert`, `SqlUpdate` + the `matches_filters` helper used by `EventBus::publish_protobuf`. +- `grpc_service.rs` — tonic service with a `tokio::sync::broadcast::Sender` fanning out live updates to subscribed RPC clients. +- `api.rs` — Axum handlers for `/sql/query` (arbitrary SQL) and `/sql/events` (recent rows). +- `samples.rs` — sample events for tutorials. +- `generated/` — `torii.sinks.sql.rs` + `sql_descriptor.bin` produced by `build.rs` from `proto/sql.proto`. + +### Sink trait wiring + +| Method | Behavior | +|---|---| +| `name` | `"sql"` | +| `interested_types` | `[TypeId::new("sql.insert"), TypeId::new("sql.update")]` | +| `process(envelopes, batch)` | For each envelope: run an `INSERT INTO {sql_operation}` via `QueryBuilder`, then encode to `ProtoSqlOperation`, publish on EventBus topic `"sql"` with filter fn `matches_filters`, and send on the gRPC broadcast channel | +| `topics` | One `TopicInfo { name: "sql", available_filters: [table, operation, value_gt, value_lt, value_gte, value_lte, value_eq] }` | +| `build_routes` | Axum router: `POST /sql/query`, `GET /sql/events` | +| `initialize(event_bus, ctx)` | Stores the `event_bus` handle so subsequent `process` calls can publish | + +### Storage + +Single table, same shape on both backends: + +```sql +CREATE TABLE sql_operation ( + id INTEGER PRIMARY KEY AUTOINCREMENT, -- SERIAL on Postgres + table_name TEXT NOT NULL, + operation TEXT NOT NULL, + value INTEGER NOT NULL, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); ``` -### Build Process +Postgres uses the `sql_sink.sql_operation` schema-qualified name; SQLite uses +the bare table name. `SqlSink::table_name` (`src/lib.rs:87`) picks between +them. -The `build.rs` script: +### Filtering (EventBus topic `"sql"`) -1. Creates `src/generated/` directory if it doesn't exist -2. Configures `tonic-build` to generate server code (no client) -3. Compiles `proto/sql.proto` into Rust code -4. Outputs to `src/generated/torii.sinks.sql.rs` +| Filter key | Semantics | +|---|---| +| `table` | Exact match on `table_name` | +| `operation` | Exact match on `"insert"` / `"update"` | +| `value_gt`, `value_lt`, `value_gte`, `value_lte`, `value_eq` | Numeric comparisons on the integer `value` | -See `build.rs` for implementation details. - -## Architecture - -``` -┌─────────────────────────────────────────────────┐ -│ SqlSink │ -├─────────────────────────────────────────────────┤ -│ │ -│ ┌────────────────┐ ┌─────────────────┐ │ -│ │ SqlDecoder │─────▶│ SqlOperation │ │ -│ │ │ │ (SQLite) │ │ -│ │ Events → SQL │ │ Database │ │ -│ └────────────────┘ └─────────────────┘ │ -│ │ │ -│ ┌─────────────┼──────────┐ │ -│ │ │ │ │ -│ ▼ ▼ ▼ │ -│ ┌──────────────┐ ┌────────┐ ┌────┐│ -│ │ EventBus │ │ gRPC │ │HTTP││ -│ │ (central) │ │ APIs │ │API ││ -│ └──────────────┘ └────────┘ └────┘│ -│ │ -└─────────────────────────────────────────────────┘ -``` +Filter logic lives in `SqlSink::matches_filters` (`src/lib.rs:~200`) and is +passed by reference to `EventBus::publish_protobuf` so the central bus can +decide per-client routing without re-decoding the protobuf for each filter +evaluation. -## Storage - -- **Database**: Stores SQL operations (insert, update, etc.) in SQLite or PostgreSQL. -- **Schema**: - ```sql - CREATE TABLE sql_operation ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - table_name TEXT NOT NULL, - operation TEXT NOT NULL, - value INTEGER NOT NULL, - created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ) - ``` -- **Thread-Safe**: Uses `Arc>` for concurrent access - -## Filtering - -The SqlSink supports filtering via EventBus subscriptions: - -- `table` - Filter by table name (e.g., "user", "order") -- `operation` - Filter by operation type (e.g., "insert", "update") -- `value_gt` - Value greater than -- `value_lt` - Value less than -- `value_gte` - Value greater than or equal -- `value_lte` - Value less than or equal -- `value_eq` - Value equal to - -Example subscription: -```json -{ - "topics": ["sql"], - "filters": { - "sql": { - "table": "user", - "operation": "insert", - "value_gt": "100" - } - } -} -``` +### Interactions -## gRPC Service Registration +- **Upstream (consumers)**: `examples/simple_sql_sink`, `examples/multi_sink_example`. +- **Downstream deps**: `torii`, `sqlx` (sqlite/postgres/any), `tonic`, `tonic-web`, `prost`, `prost-types`, `axum`, `chrono`, `hex`, `anyhow`, `tracing`. +- **Workspace deps**: `torii` only (this sink pre-dates `torii-sql`; it uses raw `sqlx::Any`). -Due to Rust's type system limitations, gRPC services must be registered by the user: +### gRPC registration pattern ```rust -// User code (e.g., main.rs or examples/) +use torii::{ToriiConfig, run}; +use torii_sql_sink::{SqlDecoder, SqlSink}; use torii_sql_sink::proto::sql_sink_server::SqlSinkServer; use tonic::transport::Server; +use std::sync::Arc; -let sql_sink = SqlSink::new("postgres://torii:torii@localhost:5432/torii").await?; -let service = sql_sink.get_grpc_service_impl(); +let sink = SqlSink::new("postgres://torii:torii@localhost:5432/torii").await?; +let service = sink.get_grpc_service_impl(); let grpc_router = Server::builder() .accept_http1(true) .add_service(tonic_web::enable(SqlSinkServer::new((*service).clone()))); -// Pass router to Torii let config = ToriiConfig::builder() - .add_sink_boxed(Box::new(sql_sink)) + .add_sink_boxed(Box::new(sink)) + .add_decoder(Arc::new(SqlDecoder::new(Vec::new()))) .with_grpc_router(grpc_router) .build(); -``` - -This pattern keeps all gRPC services on the same port while maintaining type safety. - -## Testing - -```bash -# Build -cargo build -p torii-sql-sink - -# Run tests -cargo test -p torii-sql-sink - -# Test gRPC (with server running) -grpcurl -plaintext localhost:8080 torii.sinks.sql.SqlSink/Query -grpcurl -plaintext localhost:8080 torii.sinks.sql.SqlSink/GetSchema -# Test HTTP -curl -X POST http://localhost:8080/sql/query -d '{"query":"SELECT * FROM sql_operation"}' -curl http://localhost:8080/sql/events +run(config).await?; ``` -## See Also +### Extension Points -- **Example**: `examples/simple_sql_sink` - Basic SqlSink usage -- **Example**: `examples/multi_sink_example` - Multi-sink usage (SqlSink + LogSink) -- **Log Sink**: `crates/torii-log-sink` - Similar architecture with in-memory storage +- New envelope types → widen `interested_types` + add a branch to `process` + add a new `TypeId::new("sql.*")`. +- New filter keys → add a key to `TopicInfo.available_filters` and extend `matches_filters`. +- New REST endpoints → add routes in `build_routes` (state stays in `api::SqlSinkState`). +- **Testing**: `grpcurl -plaintext localhost:8080 torii.sinks.sql.SqlSink/Query` and `curl http://localhost:8080/sql/events`. diff --git a/crates/torii-sql-sink/src/decoder.rs b/crates/torii-sql-sink/src/decoder.rs index ba03c262..6a50db92 100644 --- a/crates/torii-sql-sink/src/decoder.rs +++ b/crates/torii-sql-sink/src/decoder.rs @@ -2,15 +2,17 @@ //! //! It demonstrate how to decode Starknet events into envelopes based on the event content. use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; -use starknet::core::utils::parse_cairo_short_string; -use starknet::macros::selector; +use starknet_types_raw::event::EmittedEvent; +use starknet_types_raw::Felt; use std::any::Any; use std::collections::HashMap; use torii::etl::decoder::Decoder; use torii::etl::envelope::{Envelope, TypeId, TypedBody}; +use torii::etl::EventContext; +const INSERT_SELECTOR: Felt = Felt::selector("insert"); +const UPDATE_SELECTOR: Felt = Felt::selector("update"); /// SqlInsert event type - represents a SQL insert operation. /// By deriving TypedBody, it allows the envelope to be downcast to this type. #[derive(Debug, Clone)] @@ -88,35 +90,18 @@ impl SqlDecoder { self.contract_filters.contains(&event.from_address) } -} -/// Implementation of the Decoder trait for generic usage (not specific to the SQL sink). -#[async_trait] -impl Decoder for SqlDecoder { - fn decoder_name(&self) -> &'static str { - "sql" - } - - async fn decode_event(&self, event: &EmittedEvent) -> anyhow::Result> { + pub async fn decode(&self, event: &EmittedEvent) -> anyhow::Result> { if !self.is_interested(event) { return Ok(Vec::new()); } - let insert_selector = selector!("insert"); - let update_selector = selector!("update"); - - // We could add additional checks for example length of keys etc.. - // In this case, we're going to assume they are present already. let selector = match event.keys.first() { Some(s) => s, None => return Ok(Vec::new()), }; - let table_name = match event - .keys - .get(1) - .and_then(|k| parse_cairo_short_string(k).ok()) - { + let table_name = match event.keys.get(1).and_then(|k| k.as_short_ascii_str().ok()) { Some(name) => name, None => return Ok(Vec::new()), }; @@ -126,18 +111,18 @@ impl Decoder for SqlDecoder { None => return Ok(Vec::new()), }; - let (body, operation): (Box, &str) = if *selector == insert_selector { + let (body, operation): (Box, &str) = if *selector == INSERT_SELECTOR { ( Box::new(SqlInsert { - table: table_name.clone(), + table: table_name.to_string(), value, }), "insert", ) - } else if *selector == update_selector { + } else if *selector == UPDATE_SELECTOR { ( Box::new(SqlUpdate { - table: table_name.clone(), + table: table_name.to_string(), value, }), "update", @@ -151,12 +136,10 @@ impl Decoder for SqlDecoder { return Ok(Vec::new()); }; - // Create metadata, they are optional, but currently they can give more context to the envelope - // without adding this information to the envelope body. let mut metadata = HashMap::new(); metadata.insert("source".to_string(), "starknet".to_string()); metadata.insert("operation".to_string(), operation.to_string()); - metadata.insert("table".to_string(), table_name.clone()); + metadata.insert("table".to_string(), table_name.to_string()); metadata.insert("value".to_string(), value.to_string()); metadata.insert( "from_address".to_string(), @@ -170,3 +153,29 @@ impl Decoder for SqlDecoder { )]) } } + +/// Implementation of the Decoder trait for generic usage (not specific to the SQL sink). +#[async_trait] +impl Decoder for SqlDecoder { + fn decoder_name(&self) -> &'static str { + "sql" + } + + async fn decode( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> anyhow::Result> { + let event = EmittedEvent { + from_address: context.from_address, + keys: keys.to_vec(), + data: data.to_vec(), + block_hash: None, + block_number: Some(context.block_number), + transaction_hash: context.transaction_hash, + }; + + SqlDecoder::decode(self, &event).await + } +} diff --git a/crates/torii-sql-sink/src/lib.rs b/crates/torii-sql-sink/src/lib.rs index 32739823..e166b980 100644 --- a/crates/torii-sql-sink/src/lib.rs +++ b/crates/torii-sql-sink/src/lib.rs @@ -12,21 +12,17 @@ pub mod proto { pub const FILE_DESCRIPTOR_SET: &[u8] = include_bytes!("generated/sql_descriptor.bin"); use async_trait::async_trait; -use axum::{ - routing::{get, post}, - Router, -}; +use axum::routing::{get, post}; +use axum::Router; use prost::Message; use prost_types::Any as ProtoAny; -use sqlx::{any::AnyPoolOptions, Any as SqlxAny, QueryBuilder}; +use sqlx::any::AnyPoolOptions; +use sqlx::{Any as SqlxAny, QueryBuilder}; use std::sync::Arc; - -use starknet::core::types::EmittedEvent; -use torii::etl::{ - envelope::{Envelope, TypeId}, - extractor::ExtractionBatch, - sink::{EventBus, Sink, TopicInfo}, -}; +use torii::etl::envelope::{Envelope, TypeId}; +use torii::etl::extractor::ExtractionBatch; +use torii::etl::sink::{EventBus, Sink, TopicInfo}; +use torii::etl::StarknetEvent; use torii::grpc::UpdateType; pub use decoder::{SqlDecoder, SqlInsert, SqlUpdate}; @@ -55,7 +51,7 @@ pub struct SqlSink { impl SqlSink { /// Generates sample events for testing the SQL sink. - pub fn generate_sample_events() -> Vec { + pub fn generate_sample_events() -> Vec { samples::generate_sample_events() } diff --git a/crates/torii-sql-sink/src/samples.rs b/crates/torii-sql-sink/src/samples.rs index 9611e129..23fc8bef 100644 --- a/crates/torii-sql-sink/src/samples.rs +++ b/crates/torii-sql-sink/src/samples.rs @@ -5,12 +5,11 @@ //! SampleExtractor to generate test data. //! The decoder is where the events are decoded into envelopes based on the event content. -use starknet::core::types::{EmittedEvent, Felt}; -use starknet::core::utils::cairo_short_string_to_felt; -use starknet::macros::{felt, selector}; +use starknet_types_raw::Felt; +use torii::etl::StarknetEvent; const DUMMY_CONTRACT_ADDRESS: Felt = - felt!("0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7"); + Felt::from_hex_unchecked("0x049d36570d4e46f48e99674bd3fcc84644ddd6b96f7c741b1562b82f9e004dc7"); /// Generate sample events for testing the SQL sink /// @@ -20,52 +19,47 @@ const DUMMY_CONTRACT_ADDRESS: Felt = /// - Processed by SqlSink and inserted into the events table /// - Published to gRPC subscribers /// -/// Returns a vector of EmittedEvent objects. The remaining default fields are automatically filled by the extractor -/// if not provided. -pub fn generate_sample_events() -> Vec { +/// Returns a vector of sample Starknet events. +pub fn generate_sample_events() -> Vec { vec![ - EmittedEvent { + StarknetEvent { from_address: DUMMY_CONTRACT_ADDRESS, keys: vec![ - selector!("insert"), - cairo_short_string_to_felt("user").unwrap(), + Felt::selector("insert"), + Felt::from_short_ascii_str_unchecked("user"), ], data: vec![Felt::from(100u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: DUMMY_CONTRACT_ADDRESS, keys: vec![ - selector!("update"), - cairo_short_string_to_felt("user").unwrap(), + Felt::selector("update"), + Felt::from_short_ascii_str_unchecked("user"), ], data: vec![Felt::from(200u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: DUMMY_CONTRACT_ADDRESS, keys: vec![ - selector!("insert"), - cairo_short_string_to_felt("order").unwrap(), + Felt::selector("insert"), + Felt::from_short_ascii_str_unchecked("order"), ], data: vec![Felt::from(150u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: DUMMY_CONTRACT_ADDRESS, keys: vec![ - selector!("update"), - cairo_short_string_to_felt("order").unwrap(), + Felt::selector("update"), + Felt::from_short_ascii_str_unchecked("order"), ], data: vec![Felt::from(300u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, ] diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml new file mode 100644 index 00000000..d552c211 --- /dev/null +++ b/crates/types/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "torii-types" +version = "0.1.0" +edition = "2021" +description = "Common types for Torii" + +[dependencies] +thiserror.workspace = true +starknet-types-raw = { workspace = true, features = ["serde"] } +starknet = { workspace = true, optional = true } + +[lints] +workspace = true + +[features] +field = ["dep:starknet"] diff --git a/crates/types/README.md b/crates/types/README.md new file mode 100644 index 00000000..094355a5 --- /dev/null +++ b/crates/types/README.md @@ -0,0 +1,71 @@ +# torii-types + +The lowest-level shared-types crate. Holds the three primitives that every +decoder, extractor, and sink agrees on: `StarknetEvent`, `EventContext`, and +`BlockContext`. Nothing here is async; nothing here does I/O. + +## Role in Torii + +`StarknetEvent` is the atomic unit produced by every `Extractor` and consumed +by every `Decoder`. `EventContext` is the identity of an event +(`from_address` + `block_number` + `tx_hash`) and is what decoders receive in +`Decoder::decode(keys, data, context)`. `BlockContext` is the dedup-key +value in `ExtractionBatch.blocks`. Because every other Torii crate depends +on the root `torii` crate, this crate is implicitly on every compile path — +keeping it tiny and dependency-free is the point. + +## Architecture + +```text ++-----------------------------------------------------------+ +| torii-types | +| | +| event.rs block.rs | +| +-------------------+ +-----------------------+ | +| | StarknetEvent | | BlockContext | | +| | from_address | | number | | +| | keys | | hash | | +| | data | | parent_hash | | +| | block_number | | timestamp | | +| | transaction_hash| +-----------------------+ | +| +-------------------+ | +| | | +| v | +| EventContext <-- .context() | +| from_address | block_number | transaction_hash | +| | +| (optional) feature "field": TryFrom`, dropping failures (typically events without a block number) | +| `EventContext` | `src/event.rs` | 13 | Copy-hashable identity used as a key in decoder code | +| `MissingBlockNumber` | `src/event.rs` | 19 | Error returned when converting from `starknet::core::types::EmittedEvent` without a confirmed block | +| `BlockContext` | `src/block.rs` | 5 | Dedupe-key for `ExtractionBatch.blocks` | + +### Internal Modules + +- `event` — `StarknetEvent`, `EventContext`, `MissingBlockNumber`, optional `feature = "field"` conversion from `starknet::core::types::EmittedEvent`. +- `block` — one struct. + +### Interactions + +- **Upstream (consumers)**: the root `torii` crate re-exports both types from `src/etl/mod.rs:18` and `src/etl/extractor/mod.rs:33`. All sinks/decoders use them through that re-export. +- **Downstream deps**: `starknet-types-raw`; optional `starknet` (feature `field`) for the `EmittedEvent` conversion. +- **Feature flags**: `field` (opt-in) — adds the `TryFrom` impl. Enabled by the root crate via `torii-types = { workspace = true, features = ["field"] }`. + +### Extension Points + +- Anything cross-cutting that isn’t a `StarknetEvent` or a `BlockContext` does **not** belong here — keep it pure. Transaction / class / deployment contexts live in `src/etl/extractor/mod.rs` with the rest of the batch types. diff --git a/crates/types/src/block.rs b/crates/types/src/block.rs new file mode 100644 index 00000000..19efc047 --- /dev/null +++ b/crates/types/src/block.rs @@ -0,0 +1,10 @@ +use starknet_types_raw::Felt; + +/// Block context information +#[derive(Debug, Clone, Default)] +pub struct BlockContext { + pub number: u64, + pub hash: Felt, + pub parent_hash: Felt, + pub timestamp: u64, +} diff --git a/crates/types/src/event.rs b/crates/types/src/event.rs new file mode 100644 index 00000000..00a05ad2 --- /dev/null +++ b/crates/types/src/event.rs @@ -0,0 +1,89 @@ +use starknet_types_raw::Felt; + +#[derive(Debug, Clone)] +pub struct StarknetEvent { + pub from_address: Felt, + pub keys: Vec, + pub data: Vec, + pub block_number: u64, + pub transaction_hash: Felt, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct EventContext { + pub from_address: Felt, + pub block_number: u64, + pub transaction_hash: Felt, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct MissingBlockNumber; + +impl std::fmt::Display for MissingBlockNumber { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Event is missing block number") + } +} + +impl std::error::Error for MissingBlockNumber {} + +impl std::fmt::Display for EventContext { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "from={:#066x} block={} tx={:#066x}", + self.from_address, self.block_number, self.transaction_hash + ) + } +} + +#[cfg(feature = "field")] +mod field { + use super::{MissingBlockNumber, StarknetEvent}; + use starknet::core::types::EmittedEvent as SnEmittedEvent; + impl TryFrom for StarknetEvent { + type Error = MissingBlockNumber; + + fn try_from(event: SnEmittedEvent) -> Result { + Ok(Self { + from_address: event.from_address.into(), + keys: event.keys.into_iter().map(Into::into).collect(), + data: event.data.into_iter().map(Into::into).collect(), + block_number: event.block_number.ok_or(MissingBlockNumber)?, + transaction_hash: event.transaction_hash.into(), + }) + } + } +} + +impl StarknetEvent { + pub fn new( + from_address: Felt, + keys: Vec, + data: Vec, + block_number: u64, + transaction_hash: Felt, + ) -> Self { + StarknetEvent { + from_address, + keys, + data, + block_number, + transaction_hash, + } + } + pub fn context(&self) -> EventContext { + EventContext { + from_address: self.from_address, + block_number: self.block_number, + transaction_hash: self.transaction_hash, + } + } + + pub fn filter_pending(events: Vec>) -> Vec { + events + .into_iter() + .filter_map(|e| e.try_into().ok()) + .collect() + } +} diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs new file mode 100644 index 00000000..b4c01961 --- /dev/null +++ b/crates/types/src/lib.rs @@ -0,0 +1,2 @@ +pub mod block; +pub mod event; diff --git a/examples/eventbus_only_sink/main.rs b/examples/eventbus_only_sink/main.rs index 49888c46..92f6d6ae 100644 --- a/examples/eventbus_only_sink/main.rs +++ b/examples/eventbus_only_sink/main.rs @@ -14,13 +14,14 @@ use anyhow::Result; use axum::Router; use prost::Message; use prost_types::Any; -use starknet::core::types::EmittedEvent; +use starknet_types_raw::Felt; use std::sync::Arc; use torii::etl::envelope::{Envelope, TypeId, TypedBody}; use torii::etl::extractor::ExtractionBatch; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; -use torii::etl::Decoder; +use torii::etl::{Decoder, EventContext}; use torii::{async_trait, run, ToriiConfig, UpdateType}; +use torii_types::event::StarknetEvent; // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ // 1. DEFINE EVENT TYPE @@ -70,24 +71,20 @@ impl BroadcastSink { Self { event_bus: None } } - pub fn generate_sample_events() -> Vec { - use starknet::core::types::Felt; - + pub fn generate_sample_events() -> Vec { vec![ - EmittedEvent { + StarknetEvent { from_address: Felt::from_hex("0x1234567890abcdef").unwrap(), keys: vec![Felt::from_hex("0x1").unwrap()], data: vec![Felt::from_hex("0x100").unwrap()], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: Felt::from_hex("0xfedcba0987654321").unwrap(), keys: vec![Felt::from_hex("0x2").unwrap()], data: vec![Felt::from_hex("0x200").unwrap()], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, ] @@ -189,15 +186,20 @@ impl Decoder for BroadcastDecoder { "broadcast" } - async fn decode_event(&self, event: &EmittedEvent) -> Result> { + async fn decode( + &self, + _keys: &[Felt], + _data: &[Felt], + context: EventContext, + ) -> Result> { let broadcast_event = BroadcastEvent { - event_id: format!("{:#x}", event.transaction_hash), - from_address: format!("{:#x}", event.from_address), - block_number: event.block_number.unwrap_or(0), + event_id: format!("{:#x}", context.transaction_hash), + from_address: format!("{:#x}", context.from_address), + block_number: context.block_number, }; Ok(vec![Envelope::new( - format!("broadcast_{:#x}", event.transaction_hash), + format!("broadcast_{:#x}", context.transaction_hash), Box::new(broadcast_event), Default::default(), )]) diff --git a/examples/http_only_sink/main.rs b/examples/http_only_sink/main.rs index 7c542cb5..7fd8c97a 100644 --- a/examples/http_only_sink/main.rs +++ b/examples/http_only_sink/main.rs @@ -12,19 +12,18 @@ use anyhow::Result; use serde::{Deserialize, Serialize}; -use starknet::core::types::EmittedEvent; +use starknet_types_raw::Felt; use std::sync::{Arc, RwLock}; -use torii::axum::{ - extract::State, - http::StatusCode, - routing::{get, post}, - Json, Router, -}; +use torii::axum::extract::State; +use torii::axum::http::StatusCode; +use torii::axum::routing::{get, post}; +use torii::axum::{Json, Router}; use torii::etl::envelope::{Envelope, TypeId, TypedBody}; use torii::etl::extractor::ExtractionBatch; use torii::etl::sink::{EventBus, Sink, SinkContext, TopicInfo}; -use torii::etl::Decoder; +use torii::etl::{Decoder, EventContext}; use torii::{async_trait, run, ToriiConfig}; +use torii_types::event::StarknetEvent; // ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ // 1. DEFINE EVENT TYPE @@ -109,32 +108,27 @@ impl HttpSink { } } - pub fn generate_sample_events() -> Vec { - use starknet::core::types::Felt; - + pub fn generate_sample_events() -> Vec { vec![ - EmittedEvent { + StarknetEvent { from_address: Felt::from_hex("0xaabbccdd").unwrap(), keys: vec![Felt::from_hex("0x1").unwrap()], data: vec![Felt::from_hex("0x100").unwrap()], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: Felt::from_hex("0x11223344").unwrap(), keys: vec![Felt::from_hex("0x2").unwrap()], data: vec![Felt::from_hex("0x200").unwrap()], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: Felt::from_hex("0x55667788").unwrap(), keys: vec![Felt::from_hex("0x3").unwrap()], data: vec![Felt::from_hex("0x300").unwrap()], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, ] @@ -270,15 +264,20 @@ impl Decoder for HttpDecoder { "http" } - async fn decode_event(&self, event: &EmittedEvent) -> Result> { + async fn decode( + &self, + _keys: &[Felt], + _data: &[Felt], + context: EventContext, + ) -> Result> { let id = self .counter .fetch_add(1, std::sync::atomic::Ordering::SeqCst); let stored_event = StoredEvent { id, - from_address: format!("{:#x}", event.from_address), - block_number: event.block_number.unwrap_or(0), + from_address: format!("{:#x}", context.from_address), + block_number: context.block_number, timestamp: chrono::Utc::now().timestamp(), }; diff --git a/examples/introspect/restart.rs b/examples/introspect/restart.rs index 1221bd2d..3387bd23 100644 --- a/examples/introspect/restart.rs +++ b/examples/introspect/restart.rs @@ -1,32 +1,69 @@ use itertools::Itertools; -use sqlx::{postgres::PgPoolOptions, PgPool}; -use std::sync::Arc; +use starknet::core::types::Felt; +use torii::etl::decoder::Decoder; use torii_dojo::decoder::DojoDecoder; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::DojoToriiError; -use torii_introspect_postgres_sink::IntrospectPgDb; -use torii_test_utils::{resolve_path_like, EventIterator, FakeProvider}; +use torii_dojo::store::DojoStoreTrait; +use torii_introspect::events::{IntrospectBody, IntrospectMsg}; +use torii_introspect_sql_sink::IntrospectDb; +use torii_sql::{DbPool, PoolConfig}; +use torii_test_utils::{resolve_path_like, FakeProvider, MultiContractEventIterator}; +use torii_types::event::StarknetEvent; -const DB_URL: &str = "postgres://torii:torii@localhost:5432/torii"; -// const CHAIN_DATA_PATH: &str = "~/tc-tests/pistols-2"; -// const SCHEMA_NAME: &str = "pistols"; -const CHAIN_DATA_PATH: &str = "~/tc-tests/blob-arena-2"; -const SCHEMA_NAME: &str = "blob_arena"; -const BATCH_SIZE: usize = 10000; +// const DB_URL: &str = "postgres://torii:torii@localhost:5432/torii"; +const DB_URL: &str = "sqlite://sqlite-data.db?mode=rwc"; +const EVENT_PATHS: [&str; 2] = ["~/tc-tests/blob-arena/events", "~/tc-tests/pistols/events"]; +const MODEL_CONTRACTS_PATH: &str = "~/tc-tests/model-contracts"; +const BATCH_SIZE: usize = 2000; +const PISTOLS_ADDRESS: Felt = + Felt::from_hex_unchecked("08b4838140a3cbd36ebe64d4b5aaf56a30cc3753c928a79338bf56c53f506c5"); +const BLOB_ARENA_ADDRESS: Felt = + Felt::from_hex_unchecked("2d26295d6c541d64740e1ae56abc079b82b22c35ab83985ef8bd15dc0f9edfb"); + +// const SCHEMA_MAP: [(Felt, &str); 2] = [ +// (PISTOLS_ADDRESS, "pistols"), +// (BLOB_ARENA_ADDRESS, "blob_arena"), +// ]; + +const ADDRESSES: [Felt; 2] = [PISTOLS_ADDRESS, BLOB_ARENA_ADDRESS]; + +fn clone_introspect_msg(msg: &IntrospectMsg) -> IntrospectMsg { + match msg { + IntrospectMsg::CreateTable(value) => IntrospectMsg::CreateTable(value.clone()), + IntrospectMsg::UpdateTable(value) => IntrospectMsg::UpdateTable(value.clone()), + IntrospectMsg::RenameTable(value) => IntrospectMsg::RenameTable(value.clone()), + IntrospectMsg::RenamePrimary(value) => IntrospectMsg::RenamePrimary(value.clone()), + IntrospectMsg::RetypePrimary(value) => IntrospectMsg::RetypePrimary(value.clone()), + IntrospectMsg::RenameColumns(value) => IntrospectMsg::RenameColumns(value.clone()), + IntrospectMsg::RetypeColumns(value) => IntrospectMsg::RetypeColumns(value.clone()), + IntrospectMsg::AddColumns(value) => IntrospectMsg::AddColumns(value.clone()), + IntrospectMsg::DropTable(value) => IntrospectMsg::DropTable(value.clone()), + IntrospectMsg::DropColumns(value) => IntrospectMsg::DropColumns(value.clone()), + IntrospectMsg::InsertsFields(value) => IntrospectMsg::InsertsFields(value.clone()), + IntrospectMsg::DeleteRecords(value) => IntrospectMsg::DeleteRecords(value.clone()), + IntrospectMsg::DeletesFields(value) => IntrospectMsg::DeletesFields(value.clone()), + } +} async fn run_events( - events: &mut EventIterator, + events: &mut MultiContractEventIterator, provider: FakeProvider, - pool: Arc, + pool: DbPool, end: Option, event_n: &mut u32, success: &mut u32, ) -> bool { - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - let db = IntrospectPgDb::new(pool.clone(), SCHEMA_NAME); - decoder.store.initialize().await.unwrap(); + println!("Starting event processing run"); + let decoder = DojoDecoder::new(pool.clone(), provider); + let db = IntrospectDb::new(pool, ADDRESSES); + decoder.initialize().await.unwrap(); decoder.load_tables(&[]).await.unwrap(); - db.initialize_introspect_pg_sink().await.unwrap(); + let errors = db.initialize_introspect_sql_sink().await.unwrap(); + if !errors.is_empty() { + for err in errors { + println!("Error loading table: {err}"); + } + panic!(""); + } let mut running = true; let mut this_run = 0; while running { @@ -38,18 +75,38 @@ async fn run_events( }; *event_n += 1; this_run += 1; - match decoder.decode_raw_event(&event).await { - Ok(msg) => { - msgs.push(msg); - } - Err(DojoToriiError::UnknownDojoEventSelector(_)) => { - println!("Unknown event selector, skipping event"); + let event = StarknetEvent { + from_address: event.from_address, + keys: event.keys, + data: event.data, + block_number: event.block_number.unwrap_or_default(), + transaction_hash: event.transaction_hash, + }; + match decoder.decode_event(&event).await { + Ok(envelopes) => { + for envelope in envelopes { + let Some(body) = envelope.downcast_ref::() else { + continue; + }; + let metadata = body.context; + match &body.msg { + IntrospectMsg::CreateTable(msg) => { + let mut msg = msg.clone(); + msg.append_only = true; + msgs.push((IntrospectMsg::CreateTable(msg), metadata).into()); + } + other => { + msgs.push((clone_introspect_msg(other), metadata).into()); + } + } + } } Err(err) => { println!("Failed to decode event: {err:?}"); } }; } + let msgs_ref = msgs.iter().collect_vec(); for res in db.process_messages(msgs_ref).await.unwrap() { match res { @@ -72,19 +129,21 @@ async fn run_events( #[tokio::main] async fn main() { - let chain_path = resolve_path_like(CHAIN_DATA_PATH); - let events_path = chain_path.join("events"); - let contracts_path = chain_path.join("model-contracts"); - let provider = FakeProvider::new(contracts_path); - let mut event_iterator = EventIterator::new(events_path); - let pool = Arc::new(PgPoolOptions::new().connect(DB_URL).await.unwrap()); + let event_paths = EVENT_PATHS.map(resolve_path_like).to_vec(); + let provider = FakeProvider::new(resolve_path_like(MODEL_CONTRACTS_PATH)); + let mut event_iterator = MultiContractEventIterator::new(event_paths); + let pool = PoolConfig::new(DB_URL.to_string()) + .max_connections(5) + .connect_any() + .await + .unwrap(); let mut event_n = 0; let mut success = 0; while run_events( &mut event_iterator, provider.clone(), pool.clone(), - Some(500000), + Some(20000), &mut event_n, &mut success, ) diff --git a/examples/introspect/simple.rs b/examples/introspect/simple.rs deleted file mode 100644 index cbe2b205..00000000 --- a/examples/introspect/simple.rs +++ /dev/null @@ -1,65 +0,0 @@ -use itertools::Itertools; -use sqlx::postgres::PgPoolOptions; -use std::sync::Arc; -use torii_dojo::decoder::DojoDecoder; -use torii_dojo::store::postgres::PgStore; -use torii_dojo::DojoToriiError; -use torii_introspect_postgres_sink::IntrospectPgDb; -use torii_test_utils::{resolve_path_like, EventIterator, FakeProvider}; - -const DB_URL: &str = "postgres://torii:torii@localhost:5432/torii"; -const CHAIN_DATA_PATH: &str = "~/tc-tests/pistols"; -const SCHEMA_NAME: &str = "pistols"; -// const CHAIN_DATA_PATH: &str = "~/tc-tests/blob-arena"; -// const SCHEMA_NAME: &str = "blob_arena"; -const BATCH_SIZE: usize = 1000; - -#[tokio::main] -async fn main() { - let chain_path = resolve_path_like(CHAIN_DATA_PATH); - let events_path = chain_path.join("events"); - let contracts_path = chain_path.join("model-contracts"); - let provider = FakeProvider::new(contracts_path); - let mut event_iterator = EventIterator::new(events_path); - - let pool = Arc::new(PgPoolOptions::new().connect(DB_URL).await.unwrap()); - let decoder = DojoDecoder::, _>::new(pool.clone(), provider); - let db = IntrospectPgDb::new(pool.clone(), SCHEMA_NAME); - decoder.store.initialize().await.unwrap(); - db.initialize_introspect_pg_sink().await.unwrap(); - - let mut event_n = 0; - let mut success = 0; - let mut running = true; - while running { - let mut msgs = Vec::with_capacity(BATCH_SIZE); - for _ in 0..BATCH_SIZE { - let Some(event) = event_iterator.next() else { - running = false; - break; - }; - event_n += 1; - match decoder.decode_raw_event(&event).await { - Ok(msg) => { - msgs.push(msg); - } - Err(DojoToriiError::UnknownDojoEventSelector(_)) => { - println!("Unknown event selector, skipping event"); - } - Err(err) => { - println!("Failed to decode event: {err:?}"); - } - }; - } - let msgs_ref = msgs.iter().collect_vec(); - for res in db.process_messages(msgs_ref).await.unwrap() { - match res { - Err(err) => println!("Failed to process message: {err:?}"), - Ok(()) => success += 1, - } - } - println!( - "Processed batch of events, total events processed: {event_n}, successful: {success}" - ); - } -} diff --git a/examples/pathfinder/main.rs b/examples/pathfinder/main.rs index 57bf5370..11b0c826 100644 --- a/examples/pathfinder/main.rs +++ b/examples/pathfinder/main.rs @@ -9,7 +9,7 @@ fn main() { let mut current_block = 6000000; for _ in 0..100 { let (blocks, events) = conn - .get_emitted_events_with_context(current_block, current_block + BATCH_SIZE - 1) + .get_events_with_context(current_block, current_block + BATCH_SIZE - 1) .expect("failed to fetch events with context"); println!( "Fetched {} blocks and {} events for blocks {} to {}", diff --git a/sqlite-data.db-journal b/sqlite-data.db-journal new file mode 100644 index 00000000..51827734 Binary files /dev/null and b/sqlite-data.db-journal differ diff --git a/src/README.md b/src/README.md new file mode 100644 index 00000000..f4987833 --- /dev/null +++ b/src/README.md @@ -0,0 +1,108 @@ +# torii (root library) + +The `torii` library is the heart of the indexer. It wires a three-stage ETL +pipeline (Extract → Decode → Sink) to an EventBus, a CommandBus, a gRPC +service, and an HTTP server, and exposes a single `run(ToriiConfig)` +orchestrator that binaries call. + +## Role in Torii + +Every binary in `bins/` builds a `ToriiConfig`, registers its own sinks, +decoders, and extractor, and hands the config to `torii::run`. This crate +owns the event loop, the cursor-commit contract, contract auto-identification, +graceful shutdown, and the gRPC/HTTP surface. Sinks and decoders plug in via +traits defined under `src/etl/`. + +## Architecture + +```text + +----------------------+ + | ToriiConfigBuilder | src/lib.rs (builder) + +-----------+----------+ + | + v + +---------------- torii::run(ToriiConfig) ----------------+ src/lib.rs:613 + | | + | tokio task #1 (producer) | + | Extractor.extract(cursor, engine_db) ----> mpsc --- | + | | | + | tokio task #2 (identifier, optional) | | + | ContractIdentifier.identify_contracts <------+ | + | | + | tokio task #3 (consumer loop) | + | DecoderContext.decode_events --> Vec | + | MultiSink.process(envelopes, batch) | + | | | | | + | v v v | + | EventBus gRPC svc HTTP routes (per sink) | + | | | | | + | +---> SubscriptionManager -> connected clients | + | | + | CommandBus: async fire-and-forget side channel | + | | + | EngineDb: cursor + stats + contract→decoder cache | + +----------------------------------------------------------+ + | + axum + tonic on a single TCP port (+ optional TLS) +``` + +## Deep Dive + +### Public API + +| Type / Trait | File | Line | Purpose | +|---|---|---|---| +| `ToriiConfig` / `ToriiConfigBuilder` | `src/lib.rs` | 96 / 233 | All server config: sinks, decoders, extractor, filters, TLS, concurrency | +| `ToriiTlsConfig` | `src/lib.rs` | 202 | Optional TLS acceptor config with ALPN | +| `EtlConcurrencyConfig` | `src/lib.rs` | 78 | Prefetch batch cap (`max_prefetch_batches`) | +| `run(config)` | `src/lib.rs` | 613 | Single entry point for every binary | +| `Sink` (trait) | `src/etl/sink/mod.rs` | 74 | `name` / `interested_types` / `process` / `topics` / `build_routes` / `initialize` | +| `Decoder` (trait) | `src/etl/decoder/mod.rs` | 96 | `decoder_name` + `decode(keys, data, context)` | +| `Extractor` (trait) | `src/etl/extractor/mod.rs` | 299 | `extract(cursor, engine_db)` + `commit_cursor` | +| `ContractIdentifier` (trait) | `src/etl/identification/registry.rs` | 32 | Batch ABI lookup + shared cache | +| `IdentificationRule` (trait) | `src/etl/identification/rule.rs` | 52 | ABI pattern → `Vec` | +| `EventBus` | `src/etl/sink/mod.rs` | 188 | Topic pub/sub via `SubscriptionManager` | +| `CommandBus` / `CommandHandler` | `src/command.rs` | 148 / 35 | Sinks dispatch fire-and-forget commands | +| `SubscriptionManager` | `src/grpc.rs` | 42 | Client registration + filter routing | +| `GrpcState` / `ToriiService` | `src/grpc.rs` | 134 / 156 | Core `torii.Torii` service | +| `EngineDb` / `EngineStats` | `src/etl/engine_db.rs` | 33 / (below) | Cursor + head + contract-decoder persistence | +| `Envelope` / `TypedBody` / `EventMsg` | `src/etl/envelope.rs` | 58 / 30 / 118 | Envelope model flowing through the pipeline | +| `DecoderContext` | `src/etl/decoder/context.rs` | 32 | Router from event → decoders (mapping > registry > all) | +| `MultiSink` | `src/etl/sink/multi.rs` | 16 | Runs sinks concurrently via `join_all`, aggregates topics & routes | +| `ContractFilter` / `DecoderId` | `src/etl/decoder/mod.rs` | 236 / 179 | Whitelist + blacklist + deterministic decoder IDs | + +### Internal Modules + +- `command.rs` — `Command`, `CommandHandler`, `CommandBus`, `CommandBusSender`; a bounded-queue MPSC that routes typed commands to matching handlers. +- `grpc.rs` — core `torii.Torii` service (`GetVersion`, `ListTopics`, `SubscribeToTopics[Stream]`) + `SubscriptionManager` + `ClientSubscription`. +- `http.rs` — minimal Axum router with `/health` and `/metrics`. Sinks merge their routes onto this via `MultiSink::build_routes`. +- `metrics.rs` — Prometheus recorder bootstrapped from `TORII_METRICS_ENABLED`; rendered at `/metrics`. +- `etl/mod.rs` — public re-exports (`Decoder`, `Extractor`, `Sink`, `MultiSink`, `EngineDb`, `Envelope`, `StarknetEvent`, `EventContext`, …). +- `etl/envelope.rs` — pipeline data unit: `Envelope { id, type_id, body: Box, metadata, timestamp }`. `EventMsg` is the sugar trait decoders most often implement via `to_envelope(ctx)`. +- `etl/engine_db.rs` — `sqlx::Pool` wrapper with embedded SQLite + PostgreSQL schema (see `sql/engine_schema*.sql`). Persists head, cursor, stats, and contract→decoder mappings. +- `etl/decoder/` — `Decoder` trait, `DecoderId`, `ContractFilter`, `DecoderContext` (routing logic). +- `etl/extractor/` — `Extractor` trait + several implementations: `SampleExtractor`, `BlockRangeExtractor`, `EventExtractor`, `GlobalEventExtractor`, `CompositeExtractor`, `SyntheticExtractorAdapter`, `SyntheticErc20Extractor`, plus `RetryPolicy` and starknet RPC helpers. +- `etl/identification/` — `IdentificationRule` trait, `ContractRegistry` (`ContractIdentifier` impl) with bounded positive/negative caches, batched JSON-RPC fetches (`MAX_BATCH_SIZE = 500`). +- `etl/sink/` — `Sink` trait, `EventBus`, `SinkContext`, `TopicInfo`; `multi.rs` drives per-sink concurrency with `futures::future::join_all`. + +### Interactions + +- **Upstream** (binaries): every `bins/*` calls `torii::run` with its own `ToriiConfigBuilder` assembly. +- **Downstream**: every `torii-*-sink`, `torii-dojo`, `torii-introspect`, `torii-erc20/721/1155` depends on this crate for the `Sink`, `Decoder`, `Extractor`, `Envelope`, `EventBus`, and `ContractIdentifier` types. +- **Workspace deps**: `torii-common` (indirect), `torii-types` (re-exports `StarknetEvent` / `EventContext` / `BlockContext`), `torii-sql` (not a direct dep — `engine_db` uses raw `sqlx::Any`). + +### Key Invariants + +- **Cursor safety**: the cursor is committed (`extractor.commit_cursor`) *only after* `multi_sink.process` returns `Ok` — see `src/lib.rs:1113-1127`. A crash between extract and sink processing replays the last batch; it never loses it. +- **Decoder determinism**: `DecoderContext::decoder_ids` sorts `DecoderId` values so the decode order is reproducible across restarts (`src/etl/decoder/context.rs:197`). +- **Decoder priority**: blacklist → explicit mapping → registry cache → fallback to all decoders. Stale cache entries pointing at unknown `DecoderId`s are evicted automatically (`src/etl/decoder/context.rs:341-360`). +- **Multi-sink isolation**: one sink failing logs & increments `torii_sink_failures_total` but does *not* abort the batch. Other sinks still run (`src/etl/sink/multi.rs:51-66`). +- **Shutdown**: SIGINT/SIGTERM cancels a `CancellationToken`. The HTTP/gRPC server has 15 s to drain; the ETL loop gets `config.shutdown_timeout` seconds (default 30 s) to finish its current batch. + +### Extension Points + +- Add a sink → implement `Sink` (see `crates/torii-sql-sink/src/lib.rs` as the reference). +- Add a decoder → implement `Decoder` (see `crates/torii-erc20/src/decoder.rs`). +- Add a custom extractor → implement `Extractor` (RPC-backed: `etl/extractor/event.rs`; deterministic: `SyntheticExtractor` under `etl/extractor/synthetic.rs`). +- Add auto-identification → implement `IdentificationRule` and register it on a `ContractRegistry`, then plug the registry in via `ToriiConfigBuilder::with_contract_identifier`. +- Add a background worker → implement `CommandHandler` and register via `ToriiConfigBuilder::with_command_handler`. diff --git a/src/etl/decoder/context.rs b/src/etl/decoder/context.rs index 8f850c38..9550e6d7 100644 --- a/src/etl/decoder/context.rs +++ b/src/etl/decoder/context.rs @@ -11,23 +11,16 @@ //! - Unmapped contracts with no registry fall back to all decoders //! - Deterministic ordering: decoders are always called in sorted DecoderId order +use super::{ContractFilter, Decoder, DecoderId}; +use crate::etl::decoder::EventContext; +use crate::etl::engine_db::EngineDb; +use crate::etl::envelope::Envelope; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; +use starknet_types_raw::Felt; use std::collections::HashMap; use std::sync::Arc; use tokio::sync::RwLock; -use super::{ContractFilter, Decoder, DecoderId}; -use crate::etl::engine_db::EngineDb; -use crate::etl::envelope::Envelope; - -fn event_preview(event: &EmittedEvent) -> String { - format!( - "contract={:#x} tx={:#x}", - event.from_address, event.transaction_hash - ) -} - /// DecoderContext manages multiple decoders with contract filtering. /// /// Routes events to decoders based on: @@ -215,39 +208,38 @@ impl DecoderContext { /// Decode an event using specific decoders async fn decode_with_decoders( &self, - event: &EmittedEvent, + keys: &[Felt], + data: &[Felt], + context: EventContext, decoder_ids: &[DecoderId], ) -> anyhow::Result> { let mut all_envelopes = Vec::new(); for decoder_id in decoder_ids { if let Some(decoder) = self.decoders.get(decoder_id) { - match decoder.decode_event(event).await { + match decoder.decode(keys, data, context).await { Ok(envelopes) => { if !envelopes.is_empty() { tracing::trace!( target: "torii::etl::decoder_context", "Decoder '{}' decoded event from {:#x} into {} envelope(s)", decoder.decoder_name(), - event.from_address, + context.from_address, envelopes.len() ); } all_envelopes.extend(envelopes); } Err(e) => { - let selector = event - .keys + let selector = keys .first() .map_or_else(|| "".to_string(), |felt| format!("{felt:#x}")); - let preview = event_preview(event); tracing::warn!( target: "torii::etl::decoder_context", - contract = %format!("{:#x}", event.from_address), + contract = %format!("{:#x}", context.from_address), selector = %selector, - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number, - event = %preview, + tx_hash = %format!("{:#x}", context.transaction_hash), + block_number = context.block_number, "Decoder '{}' failed: {}", decoder.decoder_name(), e @@ -259,7 +251,7 @@ impl DecoderContext { target: "torii::etl::decoder_context", "Decoder ID {:?} not found for contract {:#x}", decoder_id, - event.from_address + context.from_address ); } } @@ -270,37 +262,36 @@ impl DecoderContext { /// Decode an event using all registered decoders (fallback) async fn decode_with_all_decoders( &self, - event: &EmittedEvent, + keys: &[Felt], + data: &[Felt], + context: EventContext, ) -> anyhow::Result> { let mut all_envelopes = Vec::new(); for decoder in self.decoders.values() { - match decoder.decode_event(event).await { + match decoder.decode(keys, data, context).await { Ok(envelopes) => { if !envelopes.is_empty() { tracing::trace!( target: "torii::etl::decoder_context", "Decoder '{}' decoded event from {:#x} into {} envelope(s)", decoder.decoder_name(), - event.from_address, + context.from_address, envelopes.len() ); } all_envelopes.extend(envelopes); } Err(e) => { - let selector = event - .keys + let selector = keys .first() .map_or_else(|| "".to_string(), |felt| format!("{felt:#x}")); - let preview = event_preview(event); tracing::warn!( target: "torii::etl::decoder_context", - contract = %format!("{:#x}", event.from_address), + contract = %format!("{:#x}", context.from_address), selector = %selector, - tx_hash = %format!("{:#x}", event.transaction_hash), - block_number = event.block_number, - event = %preview, + tx_hash = %format!("{:#x}", context.transaction_hash), + block_number = context.block_number, "Decoder '{}' failed: {}", decoder.decoder_name(), e @@ -319,21 +310,28 @@ impl Decoder for DecoderContext { "context" } - async fn decode_event(&self, event: &EmittedEvent) -> anyhow::Result> { + async fn decode( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> anyhow::Result> { // 1. Check blacklist first - if !self.contract_filter.allows(event.from_address) { + if !self.contract_filter.allows(context.from_address) { return Ok(Vec::new()); } // 2. Check explicit mappings (highest priority) - if let Some(decoder_ids) = self.contract_filter.get_decoders(event.from_address) { - return self.decode_with_decoders(event, decoder_ids).await; + if let Some(decoder_ids) = self.contract_filter.get_decoders(context.from_address) { + return self + .decode_with_decoders(keys, data, context, decoder_ids) + .await; } // 3. Check registry cache (if registry is configured) if self.has_registry { let cache = self.registry_cache.read().await; - if let Some(decoder_ids) = cache.get(&event.from_address) { + if let Some(decoder_ids) = cache.get(&context.from_address) { if decoder_ids.is_empty() { // Contract was identified but no decoders match - skip silently return Ok(Vec::new()); @@ -350,55 +348,37 @@ impl Decoder for DecoderContext { drop(cache); { let mut cache = self.registry_cache.write().await; - cache.remove(&event.from_address); + cache.remove(&context.from_address); } tracing::debug!( target: "torii::etl::decoder_context", - contract = %format!("{:#x}", event.from_address), + contract = %format!("{:#x}", context.from_address), invalid_decoder_ids = ?invalid_ids, "Evicted stale decoder mapping from registry cache; falling back to all decoders" ); - return self.decode_with_all_decoders(event).await; + return self.decode_with_all_decoders(keys, data, context).await; } // Clone to release lock before async decode let decoder_ids = decoder_ids.clone(); drop(cache); - return self.decode_with_decoders(event, &decoder_ids).await; + return self + .decode_with_decoders(keys, data, context, &decoder_ids) + .await; } // Not in registry cache = not yet identified, try all decoders // This enables auto-discovery: decoders can identify events they understand tracing::trace!( target: "torii::etl::decoder_context", - contract = %format!("{:#x}", event.from_address), + contract = %format!("{:#x}", context.from_address), "Contract not in registry cache, trying all decoders" ); drop(cache); - return self.decode_with_all_decoders(event).await; + return self.decode_with_all_decoders(keys, data, context).await; } // 4. No registry: try all decoders (fallback for non-block-range extractors) - self.decode_with_all_decoders(event).await - } - - async fn decode(&self, events: &[EmittedEvent]) -> anyhow::Result> { - let mut all_envelopes = Vec::new(); - - for event in events { - let envelopes = self.decode_event(event).await?; - - all_envelopes.extend(envelopes); - } - - tracing::debug!( - target: "torii::etl::decoder_context", - "Decoded {} events into {} envelopes across {} decoders", - events.len(), - all_envelopes.len(), - self.decoders.len(), - ); - - Ok(all_envelopes) + self.decode_with_all_decoders(keys, data, context).await } } @@ -439,15 +419,20 @@ mod tests { "ordered_decoder" } - async fn decode_event(&self, event: &EmittedEvent) -> anyhow::Result> { - if event.from_address != self.contract { + async fn decode( + &self, + _keys: &[Felt], + _data: &[Felt], + context: EventContext, + ) -> anyhow::Result> { + if context.from_address != self.contract { return Ok(Vec::new()); } Ok(vec![Envelope::new( - format!("evt-{}", event.block_number.unwrap_or_default()), + format!("evt-{}", context.block_number), Box::new(TestBody { - seq: event.block_number.unwrap_or_default(), + seq: context.block_number, }), HashMap::new(), )]) @@ -472,17 +457,16 @@ mod tests { let context = DecoderContext::new(vec![decoder], engine_db, ContractFilter::new()); let events = (0..600_u64) - .map(|seq| EmittedEvent { + .map(|seq| crate::etl::StarknetEvent { from_address: contract, keys: Vec::new(), data: Vec::new(), - block_hash: None, - block_number: Some(seq), + block_number: seq, transaction_hash: Felt::from(seq + 1), }) .collect::>(); - let envelopes = Decoder::decode(&context, &events).await.unwrap(); + let envelopes = Decoder::decode_events(&context, &events).await.unwrap(); let actual = envelopes .iter() .map(|envelope| envelope.downcast_ref::().unwrap().seq) diff --git a/src/etl/decoder/mod.rs b/src/etl/decoder/mod.rs index e6b74268..07f8a73f 100644 --- a/src/etl/decoder/mod.rs +++ b/src/etl/decoder/mod.rs @@ -1,11 +1,12 @@ pub mod context; -use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; -use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; - use super::envelope::Envelope; +use async_trait::async_trait; +use starknet_types_raw::Felt; +use std::collections::{HashMap, HashSet}; +use std::hash::Hash; +use torii_types::event::{EventContext, StarknetEvent}; +use xxhash_rust::const_xxh3::xxh3_64; pub use context::DecoderContext; @@ -43,16 +44,17 @@ pub use context::DecoderContext; /// ```rust,ignore /// use crate::etl::decoder::Decoder; /// use crate::etl::envelope::{Envelope, TypeId, TypedBody}; -/// use starknet::core::types::EmittedEvent; +/// use crate::etl::decoder::StarknetEvent; /// use async_trait::async_trait; /// use std::collections::HashMap; +/// use starknet_types_raw::Felt; /// /// pub struct MyDecoder { -/// contract_filters: Vec, +/// contract_filters: Vec, /// } /// /// impl MyDecoder { -/// fn is_interested(&self, event: &EmittedEvent) -> bool { +/// fn is_interested(&self, event: &StarknetEvent) -> bool { /// // Filter logic here /// true /// } @@ -64,7 +66,7 @@ pub use context::DecoderContext; /// "my_decoder" /// } /// -/// async fn decode_event(&self, event: &EmittedEvent) -> anyhow::Result> { +/// async fn decode_event(&self, event: &StarknetEvent) -> anyhow::Result> { /// if !self.is_interested(event) { /// return Ok(Vec::new()); /// } @@ -86,7 +88,7 @@ pub use context::DecoderContext; /// /// # Performance /// -/// **Zero-copy filtering**: Decoders receive `&EmittedEvent` (reference), allowing: +/// **Zero-copy filtering**: Decoders receive `&StarknetEvent` (reference), allowing: /// - Process events without cloning. /// - Only extract data for events the decoder is interested in. /// - Multiple decoders process the same events without memory duplication. @@ -109,17 +111,38 @@ pub trait Decoder: Send + Sync { /// ``` fn decoder_name(&self) -> &str; - /// Decode a single event into typed envelopes + /// Decode event data into typed envelopes /// /// This is the primary method that decoders should implement. /// Returns an empty Vec if the decoder is not interested in this event. /// /// # Arguments + /// * `keys` - Event keys (selectors and other identifiers). + /// * `data` - Event data fields. + /// * `context` - from_address, block_number, transaction_hash + /// + /// # Returns + /// Vector of envelopes produced from this event (empty if not interested). + async fn decode( + &self, + keys: &[Felt], + data: &[Felt], + context: EventContext, + ) -> anyhow::Result>; + + /// Decode a single event into envelopes (convenience method) + /// + /// Default implementation calls `decode()` with event fields. + /// Override only if you want to change the input type or add pre-processing. + /// + /// # Arguments /// * `event` - Reference to the event to decode. /// /// # Returns /// Vector of envelopes produced from this event (empty if not interested). - async fn decode_event(&self, event: &EmittedEvent) -> anyhow::Result>; + async fn decode_event(&self, event: &StarknetEvent) -> anyhow::Result> { + self.decode(&event.keys, &event.data, event.context()).await + } /// Decode multiple events into typed envelopes (convenience method) /// @@ -131,11 +154,10 @@ pub trait Decoder: Send + Sync { /// /// # Returns /// Vector of all envelopes produced from all events. - async fn decode(&self, events: &[EmittedEvent]) -> anyhow::Result> { + async fn decode_events(&self, events: &[StarknetEvent]) -> anyhow::Result> { let mut all_envelopes = Vec::new(); for event in events { - let envelopes = self.decode_event(event).await?; - all_envelopes.extend(envelopes); + all_envelopes.extend(self.decode_event(event).await?); } Ok(all_envelopes) } @@ -150,18 +172,16 @@ pub trait Decoder: Send + Sync { /// # Example /// /// ```rust,ignore -/// let erc20_decoder_id = DecoderId::new("erc20"); -/// let erc721_decoder_id = DecoderId::new("erc721"); +/// const ERC20_DECODER_ID = DecoderId::new("erc20"); +/// const ERC721_DECODER_ID = DecoderId::new("erc721"); /// ``` #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct DecoderId(u64); impl DecoderId { /// Creates a DecoderId from a decoder name (deterministic) - pub fn new(name: &str) -> Self { - let mut hasher = DefaultHasher::new(); - name.hash(&mut hasher); - DecoderId(hasher.finish()) + pub const fn new(type_name: &str) -> Self { + Self(xxh3_64(type_name.as_bytes())) } /// Creates a DecoderId from a u64 value (for deserialization) diff --git a/src/etl/engine_db.rs b/src/etl/engine_db.rs index 94215a7d..b86e1271 100644 --- a/src/etl/engine_db.rs +++ b/src/etl/engine_db.rs @@ -4,8 +4,10 @@ //! This will be enhanced with actual Torii features in the future. use anyhow::{Context, Result}; -use sqlx::{any::AnyPoolOptions, sqlite::SqliteConnectOptions, Any, ConnectOptions, Pool, Row}; -use starknet::core::types::Felt; +use sqlx::any::AnyPoolOptions; +use sqlx::sqlite::SqliteConnectOptions; +use sqlx::{Any, ConnectOptions, Pool, Row}; +use starknet_types_raw::Felt; use std::collections::HashMap; use std::str::FromStr; diff --git a/src/etl/envelope.rs b/src/etl/envelope.rs index 9ca2df8b..2fac8d3b 100644 --- a/src/etl/envelope.rs +++ b/src/etl/envelope.rs @@ -1,8 +1,8 @@ //! This module contains the envelope for the ETL pipeline. -use starknet::core::types::{EmittedEvent, Felt}; use std::any::Any; use std::collections::HashMap; +use torii_types::event::EventContext; use xxhash_rust::const_xxh3::xxh3_64; /// Type identifier based on a string hash @@ -109,67 +109,61 @@ impl std::fmt::Debug for Envelope { } } -#[derive(Debug, Clone)] -pub struct MetaData { - pub block_number: Option, - pub transaction_hash: Felt, - pub from_address: Felt, -} - #[derive(Debug, Clone)] pub struct EventBody { - pub metadata: MetaData, + pub context: EventContext, pub msg: T, } pub trait EventMsg: Send + Sync + 'static { fn event_id(&self) -> String; fn envelope_type_id(&self) -> TypeId; - fn to_body(self, raw: &EmittedEvent) -> EventBody + fn to_body(self, context: EventContext) -> EventBody where Self: Sized, { - EventBody { - metadata: raw.into(), - msg: self, - } + EventBody { context, msg: self } + } + fn to_envelope(self, context: EventContext) -> Envelope + where + Self: Sized, + { + Envelope::new( + self.event_id(), + Box::new(self.to_body(context)), + HashMap::new(), + ) + } + fn to_envelopes(self, context: EventContext) -> Vec + where + Self: Sized, + { + vec![self.to_envelope(context)] } - fn to_envelope(self, raw: &EmittedEvent) -> Envelope + fn to_ok_envelopes(self, context: EventContext) -> Result, E> where Self: Sized, { - Envelope::new(self.event_id(), Box::new(self.to_body(raw)), HashMap::new()) + Ok(self.to_envelopes(context)) } } -impl From> for (T, MetaData) { +impl From> for (T, EventContext) { fn from(value: EventBody) -> Self { - (value.msg, value.metadata) + (value.msg, value.context) } } -impl From<(T, MetaData)> for EventBody { - fn from(value: (T, MetaData)) -> Self { - let (msg, meta_data) = value; - EventBody { - msg, - metadata: meta_data, - } +impl From<(T, EventContext)> for EventBody { + fn from(value: (T, EventContext)) -> Self { + let (msg, context) = value; + EventBody { context, msg } } } -impl<'a, T> From<&'a EventBody> for (&'a T, &'a MetaData) { +impl<'a, T> From<&'a EventBody> for (&'a T, &'a EventContext) { fn from(value: &'a EventBody) -> Self { - (&value.msg, &value.metadata) - } -} -impl From<&EmittedEvent> for MetaData { - fn from(value: &EmittedEvent) -> Self { - MetaData { - block_number: value.block_number, - transaction_hash: value.transaction_hash, - from_address: value.from_address, - } + (&value.msg, &value.context) } } diff --git a/src/etl/event.rs b/src/etl/event.rs deleted file mode 100644 index a75b8088..00000000 --- a/src/etl/event.rs +++ /dev/null @@ -1,32 +0,0 @@ -use std::collections::HashMap; - -use starknet::core::types::{EmittedEvent, Felt}; - -pub trait EmittedEventExt { - fn metadata(&self) -> HashMap; - fn split_keys(&self) -> Option<(&Felt, &[Felt])>; -} - -impl EmittedEventExt for EmittedEvent { - fn metadata(&self) -> HashMap { - let mut metadata = HashMap::new(); - metadata.insert( - "from_address".to_string(), - self.from_address.to_fixed_hex_string(), - ); - metadata.insert( - "tx_hash".to_string(), - self.transaction_hash.to_fixed_hex_string(), - ); - if let Some(block_hash) = self.block_hash { - metadata.insert("block_hash".to_string(), block_hash.to_fixed_hex_string()); - } - if let Some(block_number) = self.block_number { - metadata.insert("block_number".to_string(), block_number.to_string()); - } - metadata - } - fn split_keys(&self) -> Option<(&Felt, &[Felt])> { - self.keys.split_first() - } -} diff --git a/src/etl/extractor/composite.rs b/src/etl/extractor/composite.rs index 9ad93d84..da5bf920 100644 --- a/src/etl/extractor/composite.rs +++ b/src/etl/extractor/composite.rs @@ -167,15 +167,15 @@ impl Extractor for CompositeExtractor { mod tests { use super::*; use crate::etl::extractor::SampleExtractor; - use starknet::core::types::{EmittedEvent, Felt}; + use starknet_types_raw::Felt; + use torii_types::event::StarknetEvent; fn make_sample_extractor() -> SampleExtractor { - let events = vec![EmittedEvent { + let events = vec![StarknetEvent { from_address: Felt::from(1u64), keys: vec![Felt::from(100u64)], data: vec![Felt::from(1000u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }]; SampleExtractor::new(events, 1) diff --git a/src/etl/extractor/event.rs b/src/etl/extractor/event.rs index 5f8d8511..c4976b42 100644 --- a/src/etl/extractor/event.rs +++ b/src/etl/extractor/event.rs @@ -46,20 +46,18 @@ //! .build(); //! ``` +use crate::etl::engine_db::EngineDb; +use crate::etl::extractor::{event_common, ExtractionBatch, Extractor, RetryPolicy}; use anyhow::{Context, Result}; use async_trait::async_trait; -use starknet::core::types::{ - requests::GetEventsRequest, BlockId, EmittedEvent, EventFilter, EventFilterWithPage, Felt, - ResultPageRequest, -}; +use starknet::core::types::requests::GetEventsRequest; +use starknet::core::types::{BlockId, EventFilter, EventFilterWithPage, ResultPageRequest}; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::collections::{HashMap, HashSet}; use std::sync::Arc; - -use crate::etl::engine_db::EngineDb; -use crate::etl::extractor::event_common; -use crate::etl::extractor::{ExtractionBatch, Extractor, RetryPolicy}; +use torii_types::event::StarknetEvent; const EXTRACTOR_TYPE: &str = "event"; @@ -471,7 +469,7 @@ impl EventExtractor { event_filter: EventFilter { from_block: Some(BlockId::Number(state.current_block.max(self.start_block))), to_block: Some(BlockId::Number(range_end)), - address: Some(state.address), + address: Some(state.address.into()), keys: None, }, result_page_request: ResultPageRequest { @@ -500,16 +498,16 @@ impl EventExtractor { } fn filter_events_by_tx_hashes( - events: Vec, + events: Vec, successful_transaction_hashes: &HashSet, - ) -> Vec { + ) -> Vec { event_common::filter_events_by_tx_hashes(events, successful_transaction_hashes) } /// Build ExtractionBatch from events with block context. async fn build_batch( &self, - events: Vec, + events: Vec, engine_db: &EngineDb, ) -> Result { event_common::build_batch( @@ -622,7 +620,7 @@ impl Extractor for EventExtractor { ); // Process responses and update state - let mut all_events = Vec::new(); + let mut all_events: Vec = Vec::new(); let mut any_advanced = false; for (address, response) in addresses.iter().zip(responses) { @@ -633,7 +631,7 @@ impl Extractor for EventExtractor { if let ProviderResponseData::GetEvents(events_page) = response { let event_count = events_page.events.len(); - all_events.extend(events_page.events); + all_events.extend(StarknetEvent::filter_pending(events_page.events)); tracing::debug!( target: "torii::etl::event", @@ -903,20 +901,18 @@ mod tests { let keep = Felt::from(1u64); let drop = Felt::from(2u64); let events = vec![ - EmittedEvent { + StarknetEvent { from_address: Felt::ZERO, keys: Vec::new(), data: Vec::new(), - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: keep, }, - EmittedEvent { + StarknetEvent { from_address: Felt::ZERO, keys: Vec::new(), data: Vec::new(), - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: drop, }, ]; @@ -948,12 +944,11 @@ mod tests { .unwrap(); let tx_hash = Felt::from(42u64); - let event = EmittedEvent { + let event = StarknetEvent { from_address: Felt::from(7u64), keys: Vec::new(), data: Vec::new(), - block_hash: None, - block_number: Some(123), + block_number: 123, transaction_hash: tx_hash, }; diff --git a/src/etl/extractor/event_common.rs b/src/etl/extractor/event_common.rs index 87d28871..1fc6028b 100644 --- a/src/etl/extractor/event_common.rs +++ b/src/etl/extractor/event_common.rs @@ -1,17 +1,16 @@ +use crate::etl::engine_db::EngineDb; +use crate::etl::extractor::{BlockContext, ExtractionBatch, RetryPolicy, TransactionContext}; +use crate::etl::StarknetEvent; use anyhow::{Context, Result}; use futures::stream::{self, StreamExt}; -use starknet::core::types::{ - requests::{GetBlockWithTxHashesRequest, GetTransactionReceiptRequest}, - BlockId, EmittedEvent, ExecutionResult, Felt, MaybePreConfirmedBlockWithTxHashes, -}; +use starknet::core::types::requests::{GetBlockWithTxHashesRequest, GetTransactionReceiptRequest}; +use starknet::core::types::{BlockId, ExecutionResult, MaybePreConfirmedBlockWithTxHashes}; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::collections::{HashMap, HashSet}; use std::sync::Arc; -use crate::etl::engine_db::EngineDb; -use crate::etl::extractor::{BlockContext, ExtractionBatch, RetryPolicy, TransactionContext}; - const RECEIPT_LOOKUP_BATCH_SIZE: usize = 200; pub(crate) fn resolved_rpc_parallelism(configured: usize) -> usize { @@ -153,7 +152,7 @@ pub(crate) async fn fetch_successful_transaction_hashes( .iter() .map(|tx_hash| { ProviderRequestData::GetTransactionReceipt(GetTransactionReceiptRequest { - transaction_hash: *tx_hash, + transaction_hash: tx_hash.into(), }) }) .collect(); @@ -235,9 +234,9 @@ pub(crate) async fn fetch_successful_transaction_hashes( } pub(crate) fn filter_events_by_tx_hashes( - events: Vec, + events: Vec, successful_transaction_hashes: &HashSet, -) -> Vec { +) -> Vec { events .into_iter() .filter(|event| successful_transaction_hashes.contains(&event.transaction_hash)) @@ -248,15 +247,15 @@ pub(crate) async fn build_batch( provider: Arc>, retry_policy: &RetryPolicy, rpc_parallelism: usize, - events: Vec, + events: Vec, engine_db: &EngineDb, ) -> Result { let block_numbers: Vec = events .iter() - .filter_map(|e| e.block_number) + .map(|e| e.block_number) .collect::>() .into_iter() - .collect(); + .collect::>(); let timestamps = fetch_block_timestamps( provider, @@ -283,18 +282,16 @@ pub(crate) async fn build_batch( let mut transactions = HashMap::new(); for event in &events { - if let Some(block_number) = event.block_number { - transactions - .entry(event.transaction_hash) - .or_insert_with(|| { - Arc::new(TransactionContext { - hash: event.transaction_hash, - block_number, - sender_address: None, - calldata: Vec::new(), - }) - }); - } + transactions + .entry(event.transaction_hash) + .or_insert_with(|| { + Arc::new(TransactionContext { + hash: event.transaction_hash, + block_number: event.block_number, + sender_address: None, + calldata: Vec::new(), + }) + }); } Ok(ExtractionBatch { diff --git a/src/etl/extractor/global_event.rs b/src/etl/extractor/global_event.rs index d9b115bd..8e20c7cc 100644 --- a/src/etl/extractor/global_event.rs +++ b/src/etl/extractor/global_event.rs @@ -5,13 +5,14 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -use starknet::core::types::{ - requests::GetEventsRequest, BlockId, EventFilter, EventFilterWithPage, Felt, ResultPageRequest, -}; +use starknet::core::types::requests::GetEventsRequest; +use starknet::core::types::{BlockId, EventFilter, EventFilterWithPage, ResultPageRequest}; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::collections::HashSet; use std::sync::Arc; +use torii_types::event::StarknetEvent; use crate::etl::engine_db::EngineDb; use crate::etl::extractor::event_common::{ @@ -313,7 +314,7 @@ impl Extractor for GlobalEventExtractor { _ => anyhow::bail!("Unexpected response type for global event request"), }; - let mut all_events = events_page.events; + let mut all_events: Vec = StarknetEvent::filter_pending(events_page.events); let mut any_advanced = false; if let Some(token) = events_page.continuation_token { diff --git a/src/etl/extractor/mod.rs b/src/etl/extractor/mod.rs index 541fab4a..624a6521 100644 --- a/src/etl/extractor/mod.rs +++ b/src/etl/extractor/mod.rs @@ -13,9 +13,10 @@ pub mod synthetic_adapter; pub mod synthetic_erc20; use crate::etl::engine_db::EngineDb; +use crate::etl::StarknetEvent; use anyhow::Result; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; +use starknet_types_raw::Felt; use std::collections::HashMap; use std::sync::Arc; @@ -29,16 +30,7 @@ pub use starknet_helpers::ContractAbi; pub use synthetic::SyntheticExtractor; pub use synthetic_adapter::SyntheticExtractorAdapter; pub use synthetic_erc20::{SyntheticErc20Config, SyntheticErc20Extractor}; - -/// Block context information -#[derive(Debug, Clone, Default)] -pub struct BlockContext { - pub number: u64, - pub hash: Felt, - pub parent_hash: Felt, - pub timestamp: u64, -} - +pub use torii_types::block::BlockContext; /// Transaction context information #[derive(Debug, Clone, Default)] pub struct TransactionContext { @@ -48,13 +40,6 @@ pub struct TransactionContext { pub calldata: Vec, } -#[derive(Debug, Clone, Default)] -pub struct EventContext { - pub from_address: Felt, - pub transaction: Arc, - pub block: Arc, -} - /// Declared class information #[derive(Debug, Clone)] pub struct DeclaredClass { @@ -76,7 +61,7 @@ pub struct DeployedContract { pub struct BlockData { pub block_context: BlockContext, pub transactions: Vec, - pub events: Vec, + pub events: Vec, pub declared_classes: Vec, pub deployed_contracts: Vec, } @@ -110,7 +95,7 @@ pub struct BlockData { #[derive(Debug, Clone)] pub struct ExtractionBatch { /// Events extracted (may contain duplicates from same block/tx) - pub events: Vec, + pub events: Vec, /// Block context (deduplicated by block_number for memory efficiency) pub blocks: HashMap>, @@ -240,29 +225,28 @@ impl ExtractionBatch { ); } // Add an event to the batch - pub fn add_event(&mut self, event: EmittedEvent) { + pub fn add_event(&mut self, event: StarknetEvent) { self.events.push(event); } // Add an event with transaction context (block_number and sender_address) to the batch // Returns None if block_number is missing from event and does not updated, since we need it to add transaction context pub fn add_event_with_tx_context( &mut self, - event: EmittedEvent, + event: StarknetEvent, sender_address: Option, calldata: Vec, - ) -> Option<()> { + ) { self.add_transaction_context( event.transaction_hash, - event.block_number?, + event.block_number, sender_address, calldata, ); self.events.push(event); - Some(()) } // Add multiple events to the batch - pub fn add_events(&mut self, events: Vec) { + pub fn add_events(&mut self, events: Vec) { self.events.extend(events); } // add a declared class to the batch @@ -308,17 +292,6 @@ impl ExtractionBatch { pub fn remove_chain_head(&mut self) { self.chain_head = None; } - - pub fn get_event_context(&self, tx_hash: &Felt, from_address: Felt) -> Option { - let transaction = self.transactions.get(tx_hash)?.clone(); - let block = self.blocks.get(&transaction.block_number)?.clone(); - - Some(EventContext { - from_address, - transaction, - block, - }) - } } /// Extractor trait for fetching enriched event batches diff --git a/src/etl/extractor/sample.rs b/src/etl/extractor/sample.rs index 54797cb5..ae98e592 100644 --- a/src/etl/extractor/sample.rs +++ b/src/etl/extractor/sample.rs @@ -5,17 +5,19 @@ use anyhow::Result; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; -use std::{collections::HashMap, sync::Arc}; +use starknet_types_raw::Felt; +use std::collections::HashMap; +use std::sync::Arc; use crate::etl::engine_db::EngineDb; +use crate::etl::StarknetEvent; use super::{BlockContext, ExtractionBatch, Extractor, TransactionContext}; /// Simple extractor that cycles through predefined events pub struct SampleExtractor { /// Sample events to cycle through - events: Vec, + events: Vec, /// Current index in the events array current_index: usize, /// Number of events to return per extraction @@ -30,7 +32,7 @@ impl SampleExtractor { /// # Arguments /// * `events` - Predefined events to cycle through /// * `batch_size` - Number of events to return per extraction - pub fn new(events: Vec, batch_size: usize) -> Self { + pub fn new(events: Vec, batch_size: usize) -> Self { Self { events, current_index: 0, @@ -40,7 +42,7 @@ impl SampleExtractor { } /// Generate the next batch of events (cycling through the predefined list) - fn next_batch(&mut self) -> Vec { + fn next_batch(&mut self) -> Vec { if self.events.is_empty() { return Vec::new(); } @@ -52,8 +54,7 @@ impl SampleExtractor { let mut event = self.events[self.current_index].clone(); // Update block number to current block - event.block_number = Some(self.current_block); - event.block_hash = Some(Felt::from(self.current_block)); + event.block_number = self.current_block; // Generate unique transaction hash based on position event.transaction_hash = Felt::from(2000 + self.current_index as u64); @@ -101,20 +102,18 @@ impl Extractor for SampleExtractor { // Create block context (deduplicated) let mut blocks = HashMap::new(); for event in &events { - if let Some(block_num) = event.block_number { - blocks.entry(block_num).or_insert_with(|| { - Arc::new(BlockContext { - number: block_num, - hash: event.block_hash.unwrap_or(Felt::ZERO), - parent_hash: if block_num > 0 { - Felt::from(block_num - 1) - } else { - Felt::ZERO - }, - timestamp: 1700000000 + block_num, // Realistic timestamp - }) - }); - } + blocks.entry(event.block_number).or_insert_with(|| { + Arc::new(BlockContext { + number: event.block_number, + hash: event.block_number.into(), + parent_hash: if event.block_number > 0 { + Felt::from(event.block_number - 1) + } else { + Felt::ZERO + }, + timestamp: 1700000000 + event.block_number, // Realistic timestamp + }) + }); } // Create transaction context (deduplicated) @@ -125,7 +124,7 @@ impl Extractor for SampleExtractor { .or_insert_with(|| { Arc::new(TransactionContext { hash: event.transaction_hash, - block_number: event.block_number.unwrap_or(0), + block_number: event.block_number, sender_address: Some( Felt::from_hex( "0x0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcd", @@ -133,7 +132,7 @@ impl Extractor for SampleExtractor { .unwrap(), ), calldata: vec![ - Felt::from(1), // selector + Felt::from(0), // selector Felt::from(self.current_block), // param1 Felt::from(42), // param2 ], @@ -169,20 +168,18 @@ mod tests { async fn test_sample_extractor_cycling() { // Create sample events let events = vec![ - EmittedEvent { + StarknetEvent { from_address: Felt::from(1u64), keys: vec![Felt::from(100u64)], data: vec![Felt::from(1000u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, - EmittedEvent { + StarknetEvent { from_address: Felt::from(2u64), keys: vec![Felt::from(200u64)], data: vec![Felt::from(2000u64)], - block_hash: None, - block_number: None, + block_number: 0, transaction_hash: Felt::ZERO, }, ]; diff --git a/src/etl/extractor/starknet_helpers.rs b/src/etl/extractor/starknet_helpers.rs index a2dd9696..1132a86a 100644 --- a/src/etl/extractor/starknet_helpers.rs +++ b/src/etl/extractor/starknet_helpers.rs @@ -1,17 +1,18 @@ +use crate::etl::StarknetEvent; + +use super::{BlockContext, BlockData, DeclaredClass, DeployedContract, TransactionContext}; use anyhow::{Context, Result}; use starknet::core::types::contract::{AbiEntry, TypedAbiEvent}; -use starknet::core::types::requests::GetClassAtRequest; -use starknet::core::types::LegacyContractAbiEntry; +use starknet::core::types::requests::{GetBlockWithReceiptsRequest, GetClassAtRequest}; use starknet::core::types::{ - requests::GetBlockWithReceiptsRequest, BlockId, ContractClass, DeclareTransactionContent, - DeployAccountTransactionContent, EmittedEvent, ExecutionResult, Felt, InvokeTransactionContent, + BlockId, ContractClass, DeclareTransactionContent, DeployAccountTransactionContent, + ExecutionResult, InvokeTransactionContent, LegacyContractAbiEntry, MaybePreConfirmedBlockWithReceipts, TransactionContent, TransactionReceipt, }; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use std::collections::HashSet; -use super::{BlockContext, BlockData, DeclaredClass, DeployedContract, TransactionContext}; - #[inline] fn is_execution_succeeded(execution_result: &ExecutionResult) -> bool { matches!(execution_result, ExecutionResult::Succeeded) @@ -88,7 +89,7 @@ where .map(|&class_hash| { ProviderRequestData::GetClassAt(GetClassAtRequest { block_id: BlockId::Tag(starknet::core::types::BlockTag::Latest), - contract_address: class_hash, + contract_address: class_hash.into(), }) }) .collect(); @@ -153,8 +154,8 @@ pub fn block_into_contexts(block: MaybePreConfirmedBlockWithReceipts) -> Result< let block_context = BlockContext { number: block_with_receipts.block_number, - hash: block_with_receipts.block_hash, - parent_hash: block_with_receipts.parent_hash, + hash: block_with_receipts.block_hash.into(), + parent_hash: block_with_receipts.parent_hash.into(), timestamp: block_with_receipts.timestamp, }; @@ -178,7 +179,8 @@ pub fn block_into_contexts(block: MaybePreConfirmedBlockWithReceipts) -> Result< TransactionReceipt::Declare(r) => r.transaction_hash, TransactionReceipt::Deploy(r) => r.transaction_hash, TransactionReceipt::DeployAccount(r) => r.transaction_hash, - }; + } + .into(); if !is_receipt_succeeded(&receipt) { skipped_reverted += 1; @@ -242,20 +244,20 @@ pub fn block_into_contexts(block: MaybePreConfirmedBlockWithReceipts) -> Result< } }, }; - + let sender_address = sender_address.map(Into::into); // Build transaction context transaction_contexts.push(TransactionContext { hash: tx_hash, block_number: block_with_receipts.block_number, sender_address, - calldata, + calldata: calldata.into_iter().map(Into::into).collect(), }); // Extract declared classes from Declare transactions if let Some((class_hash, compiled_class_hash)) = declare_info { declared_classes.push(DeclaredClass { - class_hash, - compiled_class_hash, + class_hash: class_hash.into(), + compiled_class_hash: compiled_class_hash.map(Into::into), transaction_hash: tx_hash, }); } @@ -264,15 +266,15 @@ pub fn block_into_contexts(block: MaybePreConfirmedBlockWithReceipts) -> Result< match &receipt { TransactionReceipt::Deploy(deploy_receipt) => { deployed_contracts.push(DeployedContract { - contract_address: deploy_receipt.contract_address, + contract_address: deploy_receipt.contract_address.into(), class_hash: Felt::ZERO, // Old Deploy doesn't have class_hash accessible easily transaction_hash: tx_hash, }); } TransactionReceipt::DeployAccount(deploy_account_receipt) => { deployed_contracts.push(DeployedContract { - contract_address: deploy_account_receipt.contract_address, - class_hash: deploy_account_class.unwrap_or(Felt::ZERO), + contract_address: deploy_account_receipt.contract_address.into(), + class_hash: deploy_account_class.map_or(Felt::ZERO, Into::into), transaction_hash: tx_hash, }); } @@ -288,17 +290,15 @@ pub fn block_into_contexts(block: MaybePreConfirmedBlockWithReceipts) -> Result< TransactionReceipt::DeployAccount(r) => r.events, }; - // Convert to EmittedEvent format - move ownership to avoid cloning - for event in events { - all_events.push(EmittedEvent { - from_address: event.from_address, - keys: event.keys, - data: event.data, - block_hash: Some(block_with_receipts.block_hash), - block_number: Some(block_with_receipts.block_number), - transaction_hash: tx_hash, - }); - } + all_events.extend(events.into_iter().map(|e| { + StarknetEvent::new( + e.from_address.into(), + e.keys.into_iter().map(Into::into).collect(), + e.data.into_iter().map(Into::into).collect(), + block_with_receipts.block_number, + tx_hash, + ) + })); } tracing::debug!( diff --git a/src/etl/extractor/synthetic_adapter.rs b/src/etl/extractor/synthetic_adapter.rs index 5d8dca21..0cae363f 100644 --- a/src/etl/extractor/synthetic_adapter.rs +++ b/src/etl/extractor/synthetic_adapter.rs @@ -155,8 +155,8 @@ mod tests { batch.set_cursor(Self::make_cursor(block)); batch.add_block_context( block, - starknet::core::types::Felt::ZERO, - starknet::core::types::Felt::ZERO, + starknet::core::types::Felt::ZERO.into(), + starknet::core::types::Felt::ZERO.into(), 0, ); Ok(batch) diff --git a/src/etl/extractor/synthetic_erc20.rs b/src/etl/extractor/synthetic_erc20.rs index b418ff7d..2673fae2 100644 --- a/src/etl/extractor/synthetic_erc20.rs +++ b/src/etl/extractor/synthetic_erc20.rs @@ -6,12 +6,12 @@ use anyhow::{Context, Result}; use async_trait::async_trait; -use starknet::core::types::{EmittedEvent, Felt}; -use starknet::macros::selector; +use starknet_types_raw::Felt; use std::collections::HashMap; use std::sync::Arc; use crate::etl::engine_db::EngineDb; +use crate::etl::StarknetEvent; use super::{BlockContext, ExtractionBatch, Extractor, TransactionContext}; @@ -210,21 +210,19 @@ impl SyntheticErc20Extractor { let amount_low = self.amount_low_for(block_number, tx_index); let event = if self.is_approval(tx_index) { - EmittedEvent { + StarknetEvent { from_address: token, - keys: vec![selector!("Approval"), from, to], + keys: vec![Felt::selector("Approval"), from, to], data: vec![amount_low, Felt::ZERO], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), + block_number, transaction_hash: tx_hash, } } else { - EmittedEvent { + StarknetEvent { from_address: token, - keys: vec![selector!("Transfer"), from, to], + keys: vec![Felt::selector("Transfer"), from, to], data: vec![amount_low, Felt::ZERO], - block_hash: Some(Felt::from(0x0300_0000_u64 + block_number)), - block_number: Some(block_number), + block_number, transaction_hash: tx_hash, } }; diff --git a/src/etl/identification/registry.rs b/src/etl/identification/registry.rs index a7856810..07d57b1c 100644 --- a/src/etl/identification/registry.rs +++ b/src/etl/identification/registry.rs @@ -10,20 +10,20 @@ use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; const MAX_BATCH_SIZE: usize = 500; use std::sync::Arc; +use super::IdentificationRule; +use crate::etl::decoder::DecoderId; +use crate::etl::engine_db::EngineDb; +use crate::etl::extractor::ContractAbi; use anyhow::{Context, Result}; use async_trait::async_trait; use futures::stream::{self, StreamExt}; use starknet::core::types::requests::{GetClassHashAtRequest, GetClassRequest}; -use starknet::core::types::{BlockId, BlockTag, Felt}; +use starknet::core::types::{BlockId, BlockTag}; use starknet::providers::jsonrpc::{HttpTransport, JsonRpcClient}; use starknet::providers::{Provider, ProviderRequestData, ProviderResponseData}; +use starknet_types_raw::Felt; use tokio::sync::RwLock; -use super::IdentificationRule; -use crate::etl::decoder::DecoderId; -use crate::etl::engine_db::EngineDb; -use crate::etl::extractor::ContractAbi; - /// Trait for contract identification (object-safe). /// /// This trait allows type-erased contract identification, so that `ToriiConfig` @@ -297,7 +297,7 @@ impl ContractRegistry { .map(|&addr| { ProviderRequestData::GetClassHashAt(GetClassHashAtRequest { block_id: BlockId::Tag(BlockTag::Latest), - contract_address: addr, + contract_address: addr.into(), }) }) .collect(); @@ -332,7 +332,7 @@ impl ContractRegistry { // Map contract → class_hash, track failures for (addr, response) in chunk_addresses.iter().zip(class_hash_responses) { if let ProviderResponseData::GetClassHashAt(class_hash) = response { - contract_to_class.insert(*addr, class_hash); + contract_to_class.insert(*addr, class_hash.into()); } else { tracing::debug!( target: "torii::etl::identification", @@ -375,7 +375,7 @@ impl ContractRegistry { .map(|&class_hash| { ProviderRequestData::GetClass(GetClassRequest { block_id: BlockId::Tag(BlockTag::Latest), - class_hash, + class_hash: class_hash.into(), }) }) .collect(); diff --git a/src/etl/identification/rule.rs b/src/etl/identification/rule.rs index 07dbc8ae..87f7b1bf 100644 --- a/src/etl/identification/rule.rs +++ b/src/etl/identification/rule.rs @@ -4,7 +4,7 @@ //! can be identified by inspecting the ABI. use anyhow::Result; -use starknet::core::types::Felt; +use starknet_types_raw::Felt; use crate::etl::decoder::DecoderId; use crate::etl::extractor::ContractAbi; diff --git a/src/etl/mod.rs b/src/etl/mod.rs index 0bc9ca41..89c8db8f 100644 --- a/src/etl/mod.rs +++ b/src/etl/mod.rs @@ -1,18 +1,18 @@ pub mod decoder; pub mod engine_db; pub mod envelope; -pub mod event; pub mod extractor; pub mod identification; pub mod sink; pub use decoder::{Decoder, DecoderContext}; pub use engine_db::{EngineDb, EngineStats}; -pub use envelope::{Envelope, EventBody, EventMsg, MetaData, TypeId, TypedBody}; +pub use envelope::{Envelope, EventBody, EventMsg, TypeId, TypedBody}; pub use extractor::{ - BlockContext, ContractAbi, EventContext, ExtractionBatch, Extractor, SampleExtractor, - SyntheticErc20Config, SyntheticErc20Extractor, SyntheticExtractor, SyntheticExtractorAdapter, - TransactionContext, + BlockContext, ContractAbi, ExtractionBatch, Extractor, SampleExtractor, SyntheticErc20Config, + SyntheticErc20Extractor, SyntheticExtractor, SyntheticExtractorAdapter, TransactionContext, }; pub use identification::{ContractRegistry, IdentificationRule}; pub use sink::{MultiSink, Sink}; + +pub use torii_types::event::{EventContext, StarknetEvent}; diff --git a/src/lib.rs b/src/lib.rs index 5173bdaa..1a96f3b4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,10 +17,10 @@ pub mod proto { } // Re-export commonly used types for external sink authors +use crate::etl::StarknetEvent; pub use async_trait::async_trait; -pub use axum; -pub use tokio; -pub use tonic; + +pub use {axum, tokio, tonic}; // Re-export UpdateType for sink implementations pub use grpc::UpdateType; @@ -47,6 +47,7 @@ use etl::sink::{EventBus, Sink}; use etl::{Decoder, DecoderContext, MultiSink, SampleExtractor}; use grpc::{create_grpc_service, GrpcState, SubscriptionManager}; use http::create_http_router; +use starknet_types_raw::Felt; // Include the file descriptor set generated at build time. // This is also exported publicly so external sink authors can use it for reflection. @@ -126,7 +127,7 @@ pub struct ToriiConfig { pub events_per_cycle: usize, /// Sample events for testing (provided by sinks). - pub sample_events: Vec, + pub sample_events: Vec, /// Extractor for fetching blockchain events. /// @@ -163,13 +164,8 @@ pub struct ToriiConfig { /// contract→decoder mappings for contracts not in explicit mappings. /// The cache is typically populated by a ContractRegistry running batch /// identification before decoding. - pub registry_cache: Option< - Arc< - tokio::sync::RwLock< - std::collections::HashMap>, - >, - >, - >, + pub registry_cache: + Option>>>>, /// Optional contract identifier for runtime identification. /// @@ -243,19 +239,14 @@ pub struct ToriiConfigBuilder { custom_reflection: bool, cycle_interval: Option, events_per_cycle: Option, - sample_events: Vec, + sample_events: Vec, extractor: Option>, database_root: Option, engine_database_url: Option, contract_filter: Option, identification_rules: Vec>, - registry_cache: Option< - Arc< - tokio::sync::RwLock< - std::collections::HashMap>, - >, - >, - >, + registry_cache: + Option>>>>, contract_identifier: Option>, shutdown_timeout: Option, etl_concurrency: Option, @@ -364,7 +355,7 @@ impl ToriiConfigBuilder { } /// Adds sample events for testing. - pub fn with_sample_events(mut self, events: Vec) -> Self { + pub fn with_sample_events(mut self, events: Vec) -> Self { self.sample_events.extend(events); self } @@ -419,11 +410,7 @@ impl ToriiConfigBuilder { /// /// Events from this contract will ONLY be tried with the specified decoders. /// This provides O(k) performance where k is the number of mapped decoders. - pub fn map_contract( - mut self, - contract: starknet::core::types::Felt, - decoder_ids: Vec, - ) -> Self { + pub fn map_contract(mut self, contract: Felt, decoder_ids: Vec) -> Self { self.contract_filter .get_or_insert_with(ContractFilter::new) .mappings @@ -434,7 +421,7 @@ impl ToriiConfigBuilder { /// Add contract to blacklist (fast discard). /// /// Events from this contract will be discarded immediately (O(1) check). - pub fn blacklist_contract(mut self, contract: starknet::core::types::Felt) -> Self { + pub fn blacklist_contract(mut self, contract: Felt) -> Self { self.contract_filter .get_or_insert_with(ContractFilter::new) .blacklist @@ -445,7 +432,7 @@ impl ToriiConfigBuilder { /// Add multiple contracts to blacklist. /// /// Events from these contracts will be discarded immediately (O(1) check). - pub fn blacklist_contracts(mut self, contracts: Vec) -> Self { + pub fn blacklist_contracts(mut self, contracts: Vec) -> Self { self.contract_filter .get_or_insert_with(ContractFilter::new) .blacklist @@ -504,11 +491,7 @@ impl ToriiConfigBuilder { /// ``` pub fn with_registry_cache( mut self, - cache: Arc< - tokio::sync::RwLock< - std::collections::HashMap>, - >, - >, + cache: Arc>>>, ) -> Self { self.registry_cache = Some(cache); self @@ -839,9 +822,8 @@ pub async fn run(config: ToriiConfig) -> Result<(), Box> let queue_depth = Arc::new(AtomicUsize::new(0)); let (identify_tx, identify_handle) = if let Some(identifier) = contract_identifier.clone() { - let (tx, mut rx) = tokio::sync::mpsc::channel::>( - prefetch_capacity.saturating_mul(2).max(8), - ); + let (tx, mut rx) = + tokio::sync::mpsc::channel::>(prefetch_capacity.saturating_mul(2).max(8)); let handle = tokio::spawn(async move { while let Some(contract_addresses) = rx.recv().await { if contract_addresses.is_empty() { @@ -903,7 +885,7 @@ pub async fn run(config: ToriiConfig) -> Result<(), Box> let new_cursor = batch.cursor.clone(); if let Some(ref identify_tx) = producer_identify_tx { - let contract_addresses: Vec = batch + let contract_addresses: Vec = batch .events .iter() .map(|event| event.from_address) @@ -1095,7 +1077,7 @@ pub async fn run(config: ToriiConfig) -> Result<(), Box> } // Transform the events into envelopes. - let envelopes = match etl_decoder_context.decode(&batch.events).await { + let envelopes = match etl_decoder_context.decode_events(&batch.events).await { Ok(envelopes) => envelopes, Err(e) => { tracing::error!(target: "torii::etl", "Decode failed: {}", e);