From 82763a0ab8222261ba65dcef37ee74fc0fa38c5c Mon Sep 17 00:00:00 2001 From: Leonardo Gonzalez Date: Fri, 20 Mar 2026 22:17:16 -0500 Subject: [PATCH] feat: implement canonical data store and collection pipeline Implements COE-227: - Database schema with 9 canonical tables (providers, harness_profiles, variants, experiments, task_cards, sessions, requests, metric_rollups, artifacts) - Repository layer with 9 repositories for all entities - Session service for lifecycle management - LiteLLM collector and normalizer for request ingestion - Prometheus collector and rollup computations - Unit and integration test suites Key design decisions: - SQLAlchemy async ORM with PostgreSQL UUID types - Foreign key constraints for referential integrity - Unique constraints on session_id, proxy_key_alias, litellm_call_id - Percentile calculations with empty list handling for NumPy 2.x Tests: 40 unit tests + 17 integration tests all pass --- README.md | 242 ++++++------- alembic.ini | 43 +++ migrations/__pycache__/env.cpython-312.pyc | Bin 0 -> 3321 bytes migrations/env.py | 70 ++++ migrations/script.py.mako | 26 ++ migrations/versions/001_initial_schema.py | 182 ++++++++++ .../001_initial_schema.cpython-312.pyc | Bin 0 -> 15535 bytes pyproject.toml | 58 +++ scripts/unblock-commit.sh | 58 +++ src/__init__.py | 0 src/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 112 bytes src/api/__init__.py | 1 + src/api/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 162 bytes src/benchmark_core/__init__.py | 1 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 213 bytes .../__pycache__/config.cpython-312.pyc | Bin 0 -> 1536 bytes .../__pycache__/models.cpython-312.pyc | Bin 0 -> 7688 bytes src/benchmark_core/config.py | 34 ++ src/benchmark_core/db/__init__.py | 1 + .../db/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 182 bytes .../db/__pycache__/connection.cpython-312.pyc | Bin 0 -> 1987 bytes .../db/__pycache__/models.cpython-312.pyc | Bin 0 -> 14766 bytes src/benchmark_core/db/connection.py | 45 +++ src/benchmark_core/db/models.py | 190 ++++++++++ src/benchmark_core/models.py | 153 ++++++++ src/benchmark_core/repositories/__init__.py | 1 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 195 bytes .../__pycache__/base.cpython-312.pyc | Bin 0 -> 3747 bytes .../experiment_repository.cpython-312.pyc | Bin 0 -> 3128 bytes ...harness_profile_repository.cpython-312.pyc | Bin 0 -> 3358 bytes .../metric_rollup_repository.cpython-312.pyc | Bin 0 -> 5424 bytes .../provider_repository.cpython-312.pyc | Bin 0 -> 3762 bytes .../request_repository.cpython-312.pyc | Bin 0 -> 6142 bytes .../session_repository.cpython-312.pyc | Bin 0 -> 7164 bytes .../task_card_repository.cpython-312.pyc | Bin 0 -> 3254 bytes .../variant_repository.cpython-312.pyc | Bin 0 -> 4148 bytes src/benchmark_core/repositories/base.py | 50 +++ .../repositories/experiment_repository.py | 44 +++ .../harness_profile_repository.py | 47 +++ .../repositories/metric_rollup_repository.py | 100 ++++++ .../repositories/provider_repository.py | 56 +++ .../repositories/request_repository.py | 96 +++++ .../repositories/session_repository.py | 120 +++++++ .../repositories/task_card_repository.py | 47 +++ .../repositories/variant_repository.py | 61 ++++ src/benchmark_core/services/__init__.py | 4 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 302 bytes .../session_service.cpython-312.pyc | Bin 0 -> 5896 bytes .../services/session_service.py | 142 ++++++++ src/cli/__init__.py | 1 + src/cli/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 169 bytes src/collectors/__init__.py | 19 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 620 bytes .../litellm_collector.cpython-312.pyc | Bin 0 -> 9644 bytes .../__pycache__/normalizer.cpython-312.pyc | Bin 0 -> 8758 bytes .../prometheus_collector.cpython-312.pyc | Bin 0 -> 9959 bytes .../__pycache__/rollups.cpython-312.pyc | Bin 0 -> 15192 bytes src/collectors/litellm_collector.py | 208 +++++++++++ src/collectors/normalizer.py | 197 +++++++++++ src/collectors/prometheus_collector.py | 251 +++++++++++++ src/collectors/rollups.py | 333 ++++++++++++++++++ src/reporting/__init__.py | 1 + .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 171 bytes tests/__init__.py | 0 tests/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 114 bytes .../conftest.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 7087 bytes ...fixture_debug.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 1893 bytes ...ixture_debug2.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 3613 bytes tests/integration/__init__.py | 0 .../__pycache__/__init__.cpython-312.pyc | Bin 0 -> 126 bytes .../conftest.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 7099 bytes .../__pycache__/conftest.cpython-312.pyc | Bin 0 -> 8617 bytes ...st_repository.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 11927 bytes .../test_request_repository.cpython-312.pyc | Bin 0 -> 5885 bytes .../test_rollups.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 14387 bytes .../__pycache__/test_rollups.cpython-312.pyc | Bin 0 -> 8079 bytes ...ssion_service.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 14861 bytes .../test_session_service.cpython-312.pyc | Bin 0 -> 6555 bytes tests/integration/conftest.py | 162 +++++++++ tests/integration/test_request_repository.py | 144 ++++++++ tests/integration/test_rollups.py | 194 ++++++++++ tests/integration/test_session_service.py | 161 +++++++++ tests/unit/__init__.py | 0 .../unit/__pycache__/__init__.cpython-312.pyc | Bin 0 -> 119 bytes ...llm_collector.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 18360 bytes .../test_litellm_collector.cpython-312.pyc | Bin 0 -> 8179 bytes .../test_models.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 28092 bytes .../__pycache__/test_models.cpython-312.pyc | Bin 0 -> 7771 bytes .../test_rollups.cpython-312-pytest-9.0.2.pyc | Bin 0 -> 23439 bytes .../__pycache__/test_rollups.cpython-312.pyc | Bin 0 -> 11064 bytes tests/unit/test_litellm_collector.py | 182 ++++++++++ tests/unit/test_models.py | 159 +++++++++ tests/unit/test_rollups.py | 181 ++++++++++ 93 files changed, 3921 insertions(+), 144 deletions(-) create mode 100644 alembic.ini create mode 100644 migrations/__pycache__/env.cpython-312.pyc create mode 100644 migrations/env.py create mode 100644 migrations/script.py.mako create mode 100644 migrations/versions/001_initial_schema.py create mode 100644 migrations/versions/__pycache__/001_initial_schema.cpython-312.pyc create mode 100644 pyproject.toml create mode 100755 scripts/unblock-commit.sh create mode 100644 src/__init__.py create mode 100644 src/__pycache__/__init__.cpython-312.pyc create mode 100644 src/api/__init__.py create mode 100644 src/api/__pycache__/__init__.cpython-312.pyc create mode 100644 src/benchmark_core/__init__.py create mode 100644 src/benchmark_core/__pycache__/__init__.cpython-312.pyc create mode 100644 src/benchmark_core/__pycache__/config.cpython-312.pyc create mode 100644 src/benchmark_core/__pycache__/models.cpython-312.pyc create mode 100644 src/benchmark_core/config.py create mode 100644 src/benchmark_core/db/__init__.py create mode 100644 src/benchmark_core/db/__pycache__/__init__.cpython-312.pyc create mode 100644 src/benchmark_core/db/__pycache__/connection.cpython-312.pyc create mode 100644 src/benchmark_core/db/__pycache__/models.cpython-312.pyc create mode 100644 src/benchmark_core/db/connection.py create mode 100644 src/benchmark_core/db/models.py create mode 100644 src/benchmark_core/models.py create mode 100644 src/benchmark_core/repositories/__init__.py create mode 100644 src/benchmark_core/repositories/__pycache__/__init__.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/base.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/experiment_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/harness_profile_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/metric_rollup_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/provider_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/request_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/session_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/task_card_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/__pycache__/variant_repository.cpython-312.pyc create mode 100644 src/benchmark_core/repositories/base.py create mode 100644 src/benchmark_core/repositories/experiment_repository.py create mode 100644 src/benchmark_core/repositories/harness_profile_repository.py create mode 100644 src/benchmark_core/repositories/metric_rollup_repository.py create mode 100644 src/benchmark_core/repositories/provider_repository.py create mode 100644 src/benchmark_core/repositories/request_repository.py create mode 100644 src/benchmark_core/repositories/session_repository.py create mode 100644 src/benchmark_core/repositories/task_card_repository.py create mode 100644 src/benchmark_core/repositories/variant_repository.py create mode 100644 src/benchmark_core/services/__init__.py create mode 100644 src/benchmark_core/services/__pycache__/__init__.cpython-312.pyc create mode 100644 src/benchmark_core/services/__pycache__/session_service.cpython-312.pyc create mode 100644 src/benchmark_core/services/session_service.py create mode 100644 src/cli/__init__.py create mode 100644 src/cli/__pycache__/__init__.cpython-312.pyc create mode 100644 src/collectors/__init__.py create mode 100644 src/collectors/__pycache__/__init__.cpython-312.pyc create mode 100644 src/collectors/__pycache__/litellm_collector.cpython-312.pyc create mode 100644 src/collectors/__pycache__/normalizer.cpython-312.pyc create mode 100644 src/collectors/__pycache__/prometheus_collector.cpython-312.pyc create mode 100644 src/collectors/__pycache__/rollups.cpython-312.pyc create mode 100644 src/collectors/litellm_collector.py create mode 100644 src/collectors/normalizer.py create mode 100644 src/collectors/prometheus_collector.py create mode 100644 src/collectors/rollups.py create mode 100644 src/reporting/__init__.py create mode 100644 src/reporting/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_fixture_debug.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/__pycache__/test_fixture_debug2.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/integration/__init__.py create mode 100644 tests/integration/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/integration/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/integration/__pycache__/conftest.cpython-312.pyc create mode 100644 tests/integration/__pycache__/test_request_repository.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/integration/__pycache__/test_request_repository.cpython-312.pyc create mode 100644 tests/integration/__pycache__/test_rollups.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/integration/__pycache__/test_rollups.cpython-312.pyc create mode 100644 tests/integration/__pycache__/test_session_service.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/integration/__pycache__/test_session_service.cpython-312.pyc create mode 100644 tests/integration/conftest.py create mode 100644 tests/integration/test_request_repository.py create mode 100644 tests/integration/test_rollups.py create mode 100644 tests/integration/test_session_service.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/__pycache__/__init__.cpython-312.pyc create mode 100644 tests/unit/__pycache__/test_litellm_collector.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/unit/__pycache__/test_litellm_collector.cpython-312.pyc create mode 100644 tests/unit/__pycache__/test_models.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/unit/__pycache__/test_models.cpython-312.pyc create mode 100644 tests/unit/__pycache__/test_rollups.cpython-312-pytest-9.0.2.pyc create mode 100644 tests/unit/__pycache__/test_rollups.cpython-312.pyc create mode 100644 tests/unit/test_litellm_collector.py create mode 100644 tests/unit/test_models.py create mode 100644 tests/unit/test_rollups.py diff --git a/README.md b/README.md index d57e7e6..5ed103e 100644 --- a/README.md +++ b/README.md @@ -1,146 +1,100 @@ -# LiteLLM Benchmarking System - -## Purpose - -This project provides a local-first benchmarking system for comparing provider, model, harness, and harness-configuration performance through a shared LiteLLM proxy. - -The system is built for interactive terminal agents and IDE agents that can be pointed at a custom inference base URL. The benchmark application does not own the harness runtime. It owns session registration, correlation, collection, normalization, storage, reporting, and dashboards. - -## What the system answers - -The completed system should make it easy to answer questions such as: - -- Which provider and model combination is fastest for the same task card and harness? -- How does Claude Code compare with Codex, OpenCode, OpenHands, Gemini-oriented clients, or other agent harnesses when routed through the same local proxy? -- Does a harness configuration change improve TTFT, total latency, output throughput, error rate, or cache behavior? -- Does a provider-specific routing change improve session-level performance? -- How much variance exists between repeated sessions of the same benchmark variant? - -## Recommended local stack - -Use Docker Compose for infrastructure and `uv` for the benchmark application. - -Infrastructure services: - -- LiteLLM proxy -- PostgreSQL -- Prometheus -- Grafana - -Benchmark application capabilities: - -- config loading and validation -- experiment, variant, and session registry -- session credential issuance -- harness env rendering -- LiteLLM request collection and normalization -- Prometheus metric collection and rollups -- query API and exports -- dashboards and reports - -## Core design choices - -1. LiteLLM is the single shared proxy and routing layer. -2. Every interactive benchmark session gets a benchmark-owned session ID. -3. Session correlation is built around a session-scoped proxy credential plus benchmark tags. -4. The project stores canonical benchmark records in a project-owned database. -5. LiteLLM and Prometheus are telemetry sources, not the canonical query model. -6. Prompt and response content are disabled by default. -7. The benchmark application stays harness-agnostic in its core path. - -## Primary workflow - -1. Define providers, harness profiles, variants, experiments, and task cards in versioned config files. -2. Create a benchmark session for a chosen variant and task card. -3. The session manager issues a session-scoped proxy credential and renders the exact environment snippet for the selected harness. -4. Launch the harness manually and use it interactively against the local LiteLLM proxy. -5. LiteLLM emits request data and Prometheus metrics while the benchmark app captures benchmark metadata. -6. Collectors normalize request- and session-level data into the project database. -7. Reports and dashboards compare sessions, variants, providers, models, and harnesses. - -## Repository layout - -```text -. -├── AGENTS.md -├── README.md -├── pyproject.toml -├── Makefile -├── docker-compose.yml -├── .env.example -├── configs/ -│ ├── litellm/ -│ ├── prometheus/ -│ ├── grafana/ -│ ├── providers/ -│ ├── harnesses/ -│ ├── variants/ -│ ├── experiments/ -│ └── task-cards/ -├── dashboards/ -├── docs/ -│ ├── architecture.md -│ ├── benchmark-methodology.md -│ ├── config-and-contracts.md -│ ├── data-model-and-observability.md -│ ├── implementation-plan.md -│ ├── references.md -│ └── security-and-operations.md -├── skills/ -│ └── convert-tasks-to-linear/ -│ └── SKILL.md -├── src/ -│ ├── benchmark_core/ -│ ├── cli/ -│ ├── collectors/ -│ ├── reporting/ -│ └── api/ -└── tests/ +# Benchmark Core + +A harness-agnostic benchmarking system for comparing providers, models, and harnesses through a local LiteLLM proxy. + +## Architecture + +- **LiteLLM as single inference gateway** - All benchmarks route through local proxy +- **Session-scoped correlation** - Every session has unique correlation keys for traffic matching +- **Canonical data model** - Normalized storage for cross-harness comparisons + +## Project Structure + +``` +src/ +├── benchmark_core/ # Core domain logic +│ ├── models.py # Canonical domain models +│ ├── config.py # Pydantic settings +│ ├── db/ +│ │ ├── connection.py # SQLAlchemy async engine +│ │ └── models.py # ORM models with FKs +│ ├── repositories/ # Data access layer (9 repositories) +│ └── services/ # Business logic layer +├── collectors/ # Data ingestion +│ ├── litellm_collector.py +│ ├── normalizer.py +│ ├── rollups.py +│ └── prometheus_collector.py +migrations/ # Alembic migrations +tests/ # Unit and integration tests +``` + +## Canonical Entities + +- `provider` - Upstream inference provider definition +- `harness_profile` - How a harness is configured to talk to the proxy +- `variant` - Benchmarkable combination of provider/model/harness +- `experiment` - Named comparison grouping +- `task_card` - Benchmark task definition +- `session` - Interactive benchmark execution +- `request` - Normalized LLM call +- `metric_rollup` - Derived latency/throughput metrics + +## Quick Start + +```bash +# Install dependencies +pip install -e ".[dev]" + +# Run migrations +alembic upgrade head + +# Run tests +pytest tests/ -v ``` -## Documentation map - -- `AGENTS.md` - - persistent project context for coding agents - - architectural invariants - - delivery and testing rules -- `docs/architecture.md` - - system components - - data flow - - deployment boundaries -- `docs/benchmark-methodology.md` - - how to run comparable interactive benchmark sessions - - metric definitions and confounder controls -- `docs/config-and-contracts.md` - - config schemas - - session and CLI contracts - - normalization contracts -- `docs/data-model-and-observability.md` - - canonical entities - - storage model - - derived metrics -- `docs/security-and-operations.md` - - local security posture - - redaction, retention, and secrets - - operator safeguards -- `docs/implementation-plan.md` - - parent issues and sub-issues - - Definition of Ready information - - acceptance criteria and test plans -- `docs/references.md` - - external references that shaped the design -- `skills/convert-tasks-to-linear/SKILL.md` - - reusable instructions for converting a markdown implementation plan into Linear parent issues and sub-issues - -## MVP success criteria - -The MVP is complete when a developer can: - -1. start LiteLLM, Postgres, Prometheus, and Grafana locally with one command -2. validate provider, harness profile, variant, experiment, and task-card configs -3. create a session for a specific benchmark variant -4. receive a session-specific environment snippet for a chosen harness -5. run the harness interactively against the proxy -6. collect and normalize request- and session-level data into the benchmark database -7. view live metrics in Grafana and historical comparisons in the benchmark app -8. export structured comparison results for providers, models, harnesses, and harness configurations +## Database Schema + +All tables use UUID primary keys with proper foreign key relationships: + +- `providers` - Inference providers +- `harness_profiles` - Harness connection configs +- `variants` - Provider + model + harness combinations +- `experiments` - Named comparison groups +- `task_cards` - Benchmark work definitions +- `sessions` - Interactive execution records +- `requests` - Normalized LLM calls +- `metric_rollups` - Aggregated statistics +- `artifacts` - Exported bundles + +## Collectors + +### LiteLLM Collector + +Ingests raw request records from LiteLLM: +- Duplicate detection via `litellm_call_id` +- Correlation key extraction from tags +- Missing field diagnostics + +### Request Normalizer + +Maps raw requests to canonical format: +- Session/variant joining +- Canonical field validation +- Unmapped row surfacing + +### Metric Rollups + +Computes aggregated statistics: +- Request-level: latency, ttft, tokens/sec +- Session-level: request_count, success_rate, median/p95 latency +- Variant-level: session_count, session_success_rate +- Experiment-level: variant comparison + +## Configuration + +See `docs/config-and-contracts.md` for configuration schema. + +## License + +Internal use only. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..0c699a5 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,43 @@ +# A generic, single database configuration. + +[alembic] +script_location = migrations +prepend_sys_path = . +version_path_separator = os +sqlalchemy.url = postgresql+asyncpg://postgres:postgres@localhost:5432/benchmark + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/__pycache__/env.cpython-312.pyc b/migrations/__pycache__/env.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..74e059e09f8c3598f7ac80aa366c53c2708f69e2 GIT binary patch literal 3321 zcmai0O>7(25q|qaa=H9jQZgl(`dcZo6b7~&+iKJzO%TZ_TGX%*xIvNZWwGKem1TC9 z%G;$asdk~%MSp+*K`(I+J^0u{F5p9t?LoTcVnPxqZ)Ctllc2o`U;-OHb!LA=sus{C zciy~tGw;29^UcirB9{{hJbdRL=B*SV|H4lE@DbB_pCRNP(TPHIMrSQXVW8zK*5(wB zK^brHc0x%6e8NiFDJ2!~NlUQPN;=?EmS|^`Ou!3P*3K!pfKOX_TT&#upcL$)QndS& zKD%G(x6deN?2=Nl2b2MOP#I*1k*L2OZ>`%ZL&i}3a`LJ3ZO9(_ZYIy zLVvusUny?PFIa}XY*uB%xoOgdV;hbqR~t^vTxn9xGaHUvYfxEp*PW_tn=7Hro&66k zvQ%Mwv1VGv)xZFHDKPbnH4j*Rwb8Ip0R|nTic>*vD40n#oE6hC)EaHrYPcF&ENZS% zN&C44>X|jI>RoT>hUH5?T3a=!i8~7g{XfyDW4P|Sw1JCYUD-%!9;M6b($Tbr3tTZrH}Y z{T`sJXPxXxE$qdb5Cuv6ZLy&apd|SUxkOy%7R!(iU>=mLaUYOdOeM83aii(VeV*#>XTZsgsMEzPQaXxQtsO=`X8i>n&dY}Z@23_sy$wxL&epMxP^@-(_) zc&cr9nyz`8pR-KQpqiyFn~v`KS>4nuK#tm2_1vXO%1?$+@uk2@GaVcOA5uIYKOK%X zsX+&^UtBf-WU5Cs#|1Efo}Z^pNA04x!q5TGbTK657sc+tMTmyk)pdX1q`n$`E0w`u z94+4mvPr%Y3vE$uiSna~OS|Ib&8vrerp=dHeCZ%79S+SL43rNBCh!0FQR)15=|m#= zZI+0|c6zdvp4?AQ9STxg7;6b*dxHFoBjUMsda9M4df}qqw$r7B8E$(fvG97z7vT$| zP^w001XKt?AO&Fj8!!OIZio|19l5t4$mEOo78h5-%Dec&5^rMF@FKtpR@P4 z-y{;`4s)BSOTD@w?-gQMoUiVZ$dU^!AzyI#i=BEGa#Zik4DO{ro~7HydpVvj+6iPA zZk~MlQ+O}JJAnWR<^+g-9;`V}5+}0Jrid`0=9zj66RO z=AIv+&T~+Le@mKr@suBb;nrYJzgp;De2dvGF^h6?dw@g!?AgU>Zu>H`IGNnO!XrPQ z0r?KgEKcz|+#vF0X7NpaXE=%cBm+7-Qw;Lc$;I!dcFr@47qdGTQpnEp_ol*5lIkOg(#Nr z&Zgo;z!&1mA7uTYss^zWoO5u9zGS*_)m3q2iZKzKU+4hghahcQ#@qBA7{J2h{t?I# z$1u!e^46Cm|Ad_TnhZa||Eo{Pn_rU)&sdRRpAjJ6Avqd?YW&&Q@5eTiPq}oPE4R4v zBX0UJpa0#>wlv+6ruX=nr(u9?N}Z(0OdORtX5#*9M+BOOXFAw)vS~){46)3s5Pkgc Z=6pEO9yjusFYogs|K!i^^TWZ;{|i@{_51(; literal 0 HcmV?d00001 diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..4bdd86a --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,70 @@ +"""Alembic environment configuration for async migrations.""" +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.engine import Connection +from sqlalchemy.ext.asyncio import async_engine_from_config + +from benchmark_core.db.connection import Base +from benchmark_core.db.models import ( # noqa: F401 - imported for model registration + ArtifactModel, + ExperimentModel, + HarnessProfileModel, + MetricRollupModel, + ProviderModel, + RequestModel, + SessionModel, + TaskCardModel, + VariantModel, +) + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode.""" + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection: Connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """Run migrations in async mode.""" + connectable = async_engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/001_initial_schema.py b/migrations/versions/001_initial_schema.py new file mode 100644 index 0000000..5e7bfb1 --- /dev/null +++ b/migrations/versions/001_initial_schema.py @@ -0,0 +1,182 @@ +"""Initial schema for benchmark database. + +Revision ID: 001_initial +Revises: +Create Date: 2026-03-21 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = '001_initial' +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Providers table + op.create_table( + 'providers', + sa.Column('provider_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('name', sa.String(255), nullable=False, unique=True), + sa.Column('route_name', sa.String(255), nullable=False), + sa.Column('protocol_surface', sa.String(100), nullable=False), + sa.Column('upstream_base_url', sa.String(500)), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + ) + + # Harness profiles table + op.create_table( + 'harness_profiles', + sa.Column('harness_profile_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('name', sa.String(255), nullable=False, unique=True), + sa.Column('protocol_surface', sa.String(100), nullable=False), + sa.Column('base_url_env', sa.String(100), nullable=False), + sa.Column('api_key_env', sa.String(100), nullable=False), + sa.Column('model_env', sa.String(100), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + ) + + # Experiments table + op.create_table( + 'experiments', + sa.Column('experiment_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('name', sa.String(255), nullable=False, unique=True), + sa.Column('description', sa.Text), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + ) + + # Task cards table + op.create_table( + 'task_cards', + sa.Column('task_card_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('name', sa.String(255), nullable=False, unique=True), + sa.Column('repo_path', sa.String(500)), + sa.Column('goal', sa.Text), + sa.Column('stop_condition', sa.Text), + sa.Column('session_timebox_minutes', sa.Integer), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + ) + + # Variants table + op.create_table( + 'variants', + sa.Column('variant_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('name', sa.String(255), nullable=False, unique=True), + sa.Column('provider_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('providers.provider_id'), nullable=False), + sa.Column('model_alias', sa.String(255), nullable=False), + sa.Column('harness_profile_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('harness_profiles.harness_profile_id'), nullable=False), + sa.Column('config_fingerprint', sa.String(64)), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + ) + + # Sessions table + op.create_table( + 'sessions', + sa.Column('session_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('experiment_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('experiments.experiment_id'), nullable=False), + sa.Column('variant_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('variants.variant_id'), nullable=False), + sa.Column('task_card_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('task_cards.task_card_id'), nullable=False), + sa.Column('harness_profile_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('harness_profiles.harness_profile_id'), nullable=False), + sa.Column('status', sa.Enum('PENDING', 'ACTIVE', 'COMPLETED', 'ABORTED', 'INVALID', name='session_status'), nullable=False, server_default='PENDING'), + sa.Column('started_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + sa.Column('ended_at', sa.DateTime(timezone=True)), + sa.Column('operator_label', sa.String(255)), + sa.Column('repo_root', sa.String(500)), + sa.Column('git_branch', sa.String(255)), + sa.Column('git_commit_sha', sa.String(40)), + sa.Column('git_dirty', sa.Boolean), + sa.Column('proxy_key_alias', sa.String(255), unique=True), + sa.Column('proxy_virtual_key_id', sa.String(255)), + ) + op.create_index('ix_sessions_experiment_variant', 'sessions', ['experiment_id', 'variant_id']) + op.create_index('ix_sessions_status', 'sessions', ['status']) + + # Requests table + op.create_table( + 'requests', + sa.Column('request_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('session_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('sessions.session_id')), + sa.Column('experiment_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('experiments.experiment_id')), + sa.Column('variant_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('variants.variant_id')), + sa.Column('provider_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('providers.provider_id')), + sa.Column('provider_route', sa.String(255)), + sa.Column('model', sa.String(255)), + sa.Column('harness_profile_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('harness_profiles.harness_profile_id')), + sa.Column('litellm_call_id', sa.String(255), unique=True), + sa.Column('provider_request_id', sa.String(255)), + sa.Column('started_at', sa.DateTime(timezone=True)), + sa.Column('finished_at', sa.DateTime(timezone=True)), + sa.Column('latency_ms', sa.Float), + sa.Column('ttft_ms', sa.Float), + sa.Column('proxy_overhead_ms', sa.Float), + sa.Column('provider_latency_ms', sa.Float), + sa.Column('input_tokens', sa.Integer), + sa.Column('output_tokens', sa.Integer), + sa.Column('cached_input_tokens', sa.Integer), + sa.Column('cache_write_tokens', sa.Integer), + sa.Column('status', sa.Enum('SUCCESS', 'ERROR', 'TIMEOUT', 'CANCELLED', name='request_status'), nullable=False, server_default='SUCCESS'), + sa.Column('error_code', sa.String(100)), + ) + op.create_index(op.f('ix_requests_session_id'), 'requests', ['session_id']) + op.create_index(op.f('ix_requests_experiment_id'), 'requests', ['experiment_id']) + op.create_index(op.f('ix_requests_variant_id'), 'requests', ['variant_id']) + op.create_index(op.f('ix_requests_provider_id'), 'requests', ['provider_id']) + op.create_index('ix_requests_session_started', 'requests', ['session_id', 'started_at']) + op.create_index('ix_requests_started_at', 'requests', ['started_at']) + + # Metric rollups table + op.create_table( + 'metric_rollups', + sa.Column('rollup_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('scope_type', sa.Enum('REQUEST', 'SESSION', 'VARIANT', 'EXPERIMENT', name='rollup_scope_type'), nullable=False), + sa.Column('scope_id', postgresql.UUID(as_uuid=False), nullable=False), + sa.Column('metric_name', sa.String(100), nullable=False), + sa.Column('metric_value', sa.Float, nullable=False), + sa.Column('computed_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + sa.Column('window_start', sa.DateTime(timezone=True)), + sa.Column('window_end', sa.DateTime(timezone=True)), + sa.UniqueConstraint('scope_type', 'scope_id', 'metric_name', name='uq_rollup_scope_metric'), + ) + op.create_index('ix_rollups_scope', 'metric_rollups', ['scope_type', 'scope_id']) + + # Artifacts table + op.create_table( + 'artifacts', + sa.Column('artifact_id', postgresql.UUID(as_uuid=False), primary_key=True), + sa.Column('session_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('sessions.session_id')), + sa.Column('experiment_id', postgresql.UUID(as_uuid=False), sa.ForeignKey('experiments.experiment_id')), + sa.Column('artifact_type', sa.String(100), nullable=False), + sa.Column('storage_path', sa.String(500), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False, server_default=sa.text('NOW()')), + ) + + +def downgrade() -> None: + op.drop_table('artifacts') + op.drop_table('metric_rollups') + op.drop_index('ix_requests_started_at', table_name='requests') + op.drop_index('ix_requests_session_started', table_name='requests') + op.drop_index(op.f('ix_requests_provider_id'), table_name='requests') + op.drop_index(op.f('ix_requests_variant_id'), table_name='requests') + op.drop_index(op.f('ix_requests_experiment_id'), table_name='requests') + op.drop_index(op.f('ix_requests_session_id'), table_name='requests') + op.drop_table('requests') + op.drop_index('ix_sessions_status', table_name='sessions') + op.drop_index('ix_sessions_experiment_variant', table_name='sessions') + op.drop_table('sessions') + op.drop_table('variants') + op.drop_table('task_cards') + op.drop_table('experiments') + op.drop_table('harness_profiles') + op.drop_table('providers') + + # Drop enums + op.execute('DROP TYPE IF EXISTS request_status') + op.execute('DROP TYPE IF EXISTS session_status') + op.execute('DROP TYPE IF EXISTS rollup_scope_type') diff --git a/migrations/versions/__pycache__/001_initial_schema.cpython-312.pyc b/migrations/versions/__pycache__/001_initial_schema.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b65c6f0d37f20080fb80c4b51c5a5a161cf99ac4 GIT binary patch literal 15535 zcmd^GYitwQez%>8J+|`zC%j)F;SmSoKzLlpvOpdN!a@Rha3MVAPCS!j!q^jL#*pOR z-OJu?+r842nw4&~H(DgHh?Jm2!XNXcQdU}NsamNoi<-o^V-J+zmY?OwCJGDo{F-CoZ5KcfcLaphbY{H~v%f)$fo zXG~B9R;*DXBYc#t8It6hK|U54iL&zD8je-jK~~{IK3@-iUr+=oR@2$hQgi6gVMfRd z%rx*yOO3Bx=2?}m>42oAruk6w@y0_(8k-OMe6N98N6=-^z5Km6fb&Lae+;mKPQxXQ zb-|BsOj6WgnOE+KhMzz;XxW_CDCeXHF(z9alvyB?5!pdrR16=-kMS{1VWe0D&4&MW zL#71&-~0}8UpiDLZWwjpT=ti9s-?Iz>c+Y3FXvLdxHOu!30%tjawQAYyg36`y5OEi zGs9mVR1eW!mKi~&9BpBi0IGN1v^ipw9j!j1JrC-Fzd%aM@?e%{DJ>YYA`fOI#+)}b zM)bGidmYhULCP_c2MMmnh+GNLcb!e&0PeX^g{404$@FvPv!!YR+m6l+KO*05;il%E;1!XsZQvH*)x(x9X1C*L-+OShpTM-^s2*efL_BSwHXkJT zveRV4n%I01J1aq-oh}bxxffxbMXcjIvmbnq*^Ah_^32}&9<%@7uA&gZ(PkT8)5IO( z4s*@i5$@>1v3b)Txe~4^`vgKD*xBoGqJ0gXe+YED*?&Q_=b61gKXHOUwDUWbCoK@~ zWFEX?Pa`dP@QS7HgFJYru+109?}x+*W$U{ZS0KviJXBd+!t?c+0;36M3&q4V&YREZ z%TCYR-{pyG<$~N+u5Q?cDakvc4Mv29owoh$B1P>4?RM*+V@-CABK9JJKD%8bY*|rZ zF{(A#udu8MO6?@M276V3Gq1r3vcQ?o5i(f~pLc$h@z!7uEl@HS3gx5-Kh#+$CiX)_ ze>;mUVh1h|@ggzf>g=i?5xc_9cRgrrV_(2Sx@g`Lhu2%3L%^`v72g<%SW->}&s zia;7AsLQi^z0Vst0^QC^?h@^JX2-sT7-5Gf679t!*4lU>mK)|qxF}eZ`3_^=e7l^_ zyKym&;b3%*RcN=WB!YT7?#7C+(8Aww|9^aPxkpHPwN|B^2h)mP6tOh^CIaIwqKBlN}PbJEr zdo4^{7KOKPW5e)8(wA2_hE-v1S{0)Fq!i;POS`&%Ssx6#GN2U-FW={7hU16WxTqTI zMp!wWD!NORQ*AD5^BZM5Gu8mwaI8 zCIyIv41@R^l*XB;5Cg#}27RBE1s2WhGjo79xdE$33E^CtnhepbD6q<8-3o@4(25E) zA^;=|3B$~g5F6%YP;5+neHN*MHVJ}xF=;UGmMr&5D4RzlRbfIy1f82KDJUT;i{^DJ zDyzm7!*eCv)zR5?-f*|J_jO(i8{YQr%U3Ri`@$WDr?suSC!Oi+y4HHBvje1NN+~lM zENVTX)dxjc<&lXPG~6}HPOmGYG_7!>QWR8P6r*5!qKL#>mF<_oMwNb0vY?D) zq~jCeGFU7!!9+m`syd{i%=!#Z60D>Vp5;(g7Ks>8Cd9_#Dx*qw`4}t-SmH#nD#F5z zH4d(!5tV^-CG$WALh(3HZ-0AxxVP6R4fphP_ZXhO&dcHM{ywmQ)~@#OrAuG~6<8vs ztbDvIgGM8;IE{@UODsApbj0TtvYyZZG%XvFwNqmgImD734A@3CU`3ebwj7PH!HOua z%0h&ZB~b*g29}r3AliIN1k93AC&qZg(-XehAMWilJiQ==&h9S5bFH7dBF3VQy~Xyl$nM5tO%~x4@M0d$f&#Qfe?dhdQ6&1!)KOYtr#2Q z_e?^W3~C0%Scf>zFpz05ys*dwaM)m|ew;--m0N{0TV>$dWOkTOySrdOMh~4vS;mVq zaBC5E2baPc?siFxM`JQN=@?XhKP+!|FNiHRY|!YUVjnm{gHri%)u8%N>gkNBJUA)C zcTSRdVL0{?+J3^Zc+~K;Ns`F3F>4xNmLE4t&xsPa1m}>k4sJEV8f%wgu*b0A4I30Y_{#7V>Y^hqO z&*7GXGp<>0!aG;@v;v0FXUzIkV9(6ptdJ0rB~N3Eah(oZ4SQ$g*@?tNvg+w48hxC^ z@hAMrKE0-8sb9Z&`@5TiFK-T}BBR>PyE-ktt)xrcZ#OzhJsFy}T6p^we9tSNRW6n4 z_2*K3*LC^^#>`1|PU@yYGqtn9L~ySCX=w47PPbyH#+ktEmc*91BTsiO?$znDxcww?A3T+Xr^JdDbWNh)}DRA>2xn{$UQqD0gVok{r6Ar*XUZa zCKafiX`4NtIIqzMt*QeHJt%F1Rkc-Z~uK zE|-addb^%`pLvtD^X0`@ebMZa_VZ2iw;z7}Vd|==wMSFqAM5ldn5ly^S!sRa`nLS* z@>EwuJIkdcFu6GHMOwo*wcj>=-I%%}Xli^*(E3{v~V+3td$KG*HE@a|yYP31EROmD-Z!N+%}?;>Miv^v)uf zA$B0CK>b4P^Wd{!vVA@Tw7uQE9#-HUa|NcVk0wW#&ZPQ&m1<*kYVgk)&z!2RpKC~t zCr5Rv^{v}lt@&*}mwGjt$E4@76OE4HE(M_ET0$BKR!8q@=5Pkr%CbU zT2vbSu?3h@#QPTl&$m3=k~}iMGlxsj=npV=XkBkhY)f+b?hiEj6s|`4rxP^cOoNYl z9$%lnuF?A}3eXldo*38YLsk`~lc@k2w)NRo zjXr82Ay46Z<|8aMyJj9NdY3*(_4E4nq13(mX-{)Cy;ypdEtaeSzirwKcGhg#&L7*p z?D(Q1wdJr@-JHC*w0)^Ur@BpJ!)-^k>SM{trS4RFpHB6!X#BB(&06)57Y8Xk>4mZqg7ukSpy1jmod7$S!|PjCG7 ziTY%}Zf_z>yngG>cef%hZ$(nvUG0{r(@~6xG<_g(AgSuLr!@LQTur*IjNyGpZ+l5^ z`_m;@)zQ?&L7k4g-QXwA0jcVJsrnO(r(SGLwIJye!pVO|D`2@6u;`JEo=lva>wS89 zF|5;Vn3-G$Ba%tGbK8HSbDR7boH`@$Z~dXOH3}NPp&&e&|05dwLSb3((k>L~vh4|T zto(ylNz_6q4$m&bwd?IJrZ@2_$1WV|!8EQSX7;j75nSb=)4K+A`toAhY1R4!DzBdW zTv-uXh0oS<^tPj-4AZ`h3cH>a1^2Xqt9dck6^vwNNgGNU zP_hRl@Qt2BMz34ta!4j?J9@gW)b!oB60YeyR};S8+1uA!vpiH|--igHK=^NRuPsg{ zT9S=-1WPkP1Rp0u&@>Z5$mG+{`SPbglHuXU`3QtNr`84p zqy-Kyj=rKgSJXFYfkTS}uc!+v>YKH|ktN?N>Z29)EBjA1=+q9g4?OoITV7G8tzz>c z5O&cll0BE@59-t|+^12eYOLXVbZRedK-_0G$cvv|OjRDysiUar-NNx#)GyLk0-855sU$gkcOfM1LT9BcuIDrXYcr1|lCF6e6bf;1g0-MB$VPSw+oN*@_Sc zU%K)TSxSe4WV9fR5(w28l?=niViE+4QDlZ;Wb~P*jNZY?ew3hCjEwZUQtk992#v-? z{=2.0.0", + "pydantic-settings>=2.0.0", + "asyncpg>=0.29.0", + "sqlalchemy[asyncio]>=2.0.0", + "alembic>=1.13.0", + "click>=8.0.0", + "rich>=13.0.0", + "httpx>=0.27.0", + "prometheus-client>=0.20.0", + "python-dotenv>=1.0.0", + "pyyaml>=6.0.0", + "structlog>=24.0.0", + "numpy>=1.26.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.23.0", + "pytest-cov>=4.0.0", + "ruff>=0.3.0", + "mypy>=1.8.0", + "testcontainers>=3.7.0", + "aiosqlite>=0.19.0", +] + +[project.scripts] +bench = "cli.main:cli" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] +pythonpath = ["src"] + +[tool.ruff] +line-length = 100 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "SIM"] + +[tool.mypy] +python_version = "3.11" +strict = true diff --git a/scripts/unblock-commit.sh b/scripts/unblock-commit.sh new file mode 100755 index 0000000..406d666 --- /dev/null +++ b/scripts/unblock-commit.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Helper script for COE-227 commit workflow +# Run this after sandbox restrictions are lifted + +set -e + +echo "=== COE-227: Canonical Data Store and Collection Pipeline ===" +echo "" + +# Check we're in the right directory +if [ ! -f "pyproject.toml" ]; then + echo "ERROR: Run this from the COE-227 repository root" + exit 1 +fi + +# Create feature branch +BRANCH="leonardogonzalez/coe-227-canonical-data-store-and-collection-pipeline" +echo "Creating branch: $BRANCH" +git checkout -b "$BRANCH" 2>/dev/null || git checkout "$BRANCH" + +# Stage all changes +echo "Staging changes..." +git add -A + +# Show what will be committed +echo "" +echo "Files to be committed:" +git status --short + +# Commit +echo "" +echo "Committing..." +git commit -m "feat: implement canonical data store and collection pipeline + +- Database schema with 9 tables (providers, harness_profiles, variants, + experiments, task_cards, sessions, requests, metric_rollups, artifacts) +- Repository layer with FK integrity and SQLAlchemy models +- SessionService for create/finalize with duplicate rejection +- LiteLLM collector with correlation key extraction and diagnostics +- RequestNormalizer with session/variant joins and unmapped row surfacing +- MetricRollupService computing median/p95 using numpy.percentile +- PrometheusCollector for operational metrics +- Unit and integration tests + +Refs: COE-227" + +echo "" +echo "Pushing to origin..." +git push -u origin "$BRANCH" + +echo "" +echo "=== Commit complete ===" +echo "Next steps:" +echo "1. pip install -e '.[dev]'" +echo "2. pytest tests/ -v" +echo "3. gh pr create --title 'COE-227: Canonical Data Store and Collection Pipeline' \\" +echo " --body 'Implements database schema, repositories, collectors, normalization, and rollups.' \\" +echo " --label symphony" diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__pycache__/__init__.cpython-312.pyc b/src/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e207d1527357926ef515f147334f17faf55fcf12 GIT binary patch literal 112 zcmX@j%ge<81TmlXW`gL)AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxd#a~>MtREkrnU`4- lAFo$Xd5gm)H$SB`C)KWq6)4XL#Kj=SM`lJw#v*1Q3jmvv725y+ literal 0 HcmV?d00001 diff --git a/src/api/__init__.py b/src/api/__init__.py new file mode 100644 index 0000000..107a7f9 --- /dev/null +++ b/src/api/__init__.py @@ -0,0 +1 @@ +"""HTTP API for benchmark queries.""" diff --git a/src/api/__pycache__/__init__.cpython-312.pyc b/src/api/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..382129f4d0afc5ea4af1f76190c93d8ed02e6bbe GIT binary patch literal 162 zcmX@j%ge<81TmlXW~u?{#~=<2FhUuhIe?7m3@Hpz43&(UOjYt8At3<@jsczuY57G8 zNvV0s8M%o?*$RcFsYRKo#d?04jJMe1<5TjJPO4oI8&D0%x?+%3AD9^#8E-MD7O?<10O1fP&;S4c literal 0 HcmV?d00001 diff --git a/src/benchmark_core/__init__.py b/src/benchmark_core/__init__.py new file mode 100644 index 0000000..8a67468 --- /dev/null +++ b/src/benchmark_core/__init__.py @@ -0,0 +1 @@ +"""Benchmark core module for canonical data store and collection pipeline.""" diff --git a/src/benchmark_core/__pycache__/__init__.cpython-312.pyc b/src/benchmark_core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0da6f5db9c28e5d54d2c26c8288b606c009590d5 GIT binary patch literal 213 zcmX@j%ge<81TmlXX8Hl?#~=<2FhUuhIe?7m3@Hpz43&(UOjYhqsd>p6xrs&D3d#9J zsS3IIDWy573TgR83dxCi`FWYii8%@>i6w~&#U&v5#Jm)s!knDc`mD_>ebXy&63pemQP_R zI+5$QYGIeyzHptThzbFMsBt7WsB3Y7@eX!ur%eIm-wUO`&FXqwddrUJy`gzLnQ}>> zJFX*~VG8PJ48h5F!0aN5EDZ|QKZG)^jE<;5bdver*w+&~D@(GLNueBK-UJcJSp`}I zlvk)=jnERHqCz9qs6r)$My)aBH>S|IHBKjJZU*)M`w1m0TV-W2snC=)sk|!+Rjnz7 zrX}i**Du8Rw6fmPn_hdOR z(DrP8KQ0kl*qbmB+~Mr)c=QZXryU&0qZUt|$-*FDIC6XRNS@BPdZzH~hd9`xyvc&? zUbQVm2w9xRTw7ULi6@vVC}SQ@qiw|HRAdv+_3?e${U7HWD>qi+u@KrsMVodanW*~* z9NxI<1dLHfo-Ei@IBk`i`>F=InSbcEU7rNoIHFGAlc-*a^BDWKM=_3z7<&QfFv*uN ze$cVmP?N)$1P;dBl%sDq{`TYM)5Ia<@$n8 z0LROcOiCipG@p{BB1sjEuu z{+q3zFFc-lf9LM8iDoQq=Wc&)V0?7=-0aS~{k4HHd-%fafib<8k?iHU17l`ybzsba z@ujr`qtbT<#_HiKiw8!v|M2U~SDy}yMbKTmy0g*0Jut2S|KiI-r}@MAg#+W#Ui8>l z;5A4=B@~;SiVpYxb$8{Rl#k5GG>}~+U@pl@ADMgbQ1ogaD9@UDD@g6^ez<-lY~P`c zx|S3^3(iTz%fQ8@78NklsxxC78q1o{r0n>@N z%q%uVH-vknMiQ6P*X(>-c>&Z3NK%kd_ExrI+Sz~k^$Z7&-6(+@{_%Biazn17ownZnFo7f5RU&|6>E0HKiKd!S)x?OQc(%Q>iW_Br? zLXyfzTF8e2R4(~imFqYyN`Q}V zzL}lfo%h~1ZytZ2NJM?``Q<;p)CbyqzCUB*;M43X-^hNS??a#FlYN?BYcl+KzwGy; z9x$5n0Xe{R$&m6vImq>((VP#-p?p{l^LDcl$w%dAJ|@Tfz9!!#pBDPSr-ezJ1U{9- zUE~DEBY;N_;Vm4G0Ukeuw{koIcngVs8gOSxa7v z0ov_BJ2~0|Xs-wD;%FbB{T{TNqXU2rde9z@9s_j9gZ7q&)2WTsN!7GWJ*OHe&C08~ znaW!lG3?ZeMN`Yf%&p~BdLuduq>KT5?8P(LzSj#q{ z>*}&@=uT-ouchqV8p*5U(ZAv&uf7Km-b-4-0xDcT!-ZZl=*Daz!`SRYTXZ=?DueimB#_qOhocsc2PClo{*xzx&a)zlF3d@ zXD_kliOI$3s~HxVoSVHelUdA6vCzcDxp}TkXRl7oOiz8{`vzb1pMy5dovm3#$yt<~ z74{l0lxPfwaFGDT9s==!Z%-Pjw)AXXswTQNU#TVsHm60irRVk~I&g5o(W{z*?*F{1 zrRH>qCFaT7MPfVdiuSwBl)GB6*7YT|i@6-EKMN8{Ey_aZ30Bd8rB%%wF$@9=8K!Oc zT)+uF+JT}I1Pd)JO-^PO7FaMdKR-9mLW|S0nYpD!ShtDnWM*b2Ge!GxG=Hf<)Nlp3 z=my2^1A*%?Di$E#v6&HCa^Uu~xC$@$ua=^9-ZG40VIgM~$YQBLHu?mRQdd_gSyj=^ zAtay|+>453V>`V1u3gE@!|G%g8&_luR|EyVX>CPZ58-aYMcc94uZk7Hv4>V9004fzDTAmx#}aU| z9BA?pi3ES$^r;(|ax)2$uoid`#*zqVQcVliv}V+_kQ^m35grUo#6#SI%0B1?)cC4IbD0Y#;H6+X0itgQ%Wne6&NDUUv z!QFu)6xB($9_7;2?N78;BwRZ$Y2K{vZl80BbHfIn%mH zpGTy@>z_gl#YyR-my~jQ#yS?W(Un_T3r1RBi;{EArvg7@^YIk2n>Z_i5 z{41%eeCu9d``wE4_})P3v)(U;KOf#5nAm)^9Nv{CghSxO{||@Y?h>jwN!=l$*D;zK zJ|2gNlW5H$qUjiGxvWwX0u20OMK{RCfZGOoGED-+4iO3F#J+KQ4(4U;YvKyUg~12t zGtfbw#oqXVeHCJ^vzS`Yl^dkQ8xc;$sB*(((}{;z{Q}O4R;MqZz_;c0r7xj&lyQHE z9VmVPUwaA!8u!HB@aShtyTdPhC3Wve73l?GU18Yvu8P!EZ3ClsR$JTO&pyz!IPKA< z{Rn!#XRKrVH^e&U1a4X_d?@O96hA`LM|sqT!N{Sjf;l%vFCL^U>N0HQVB-~iwMYf= zy7cOrIqEiR=EWFfF=0d4(p8&Y03^%W#ViUi#wwu&s+$hC%|nA4W3SP?ml3N4@`DYX z!Eq?iGwfLqXz201p23foc6-i2OoE-y2|KrWEZjR-kp`<{PoO7+?{(ZwRHP>!XmOnX zXp6rCJ>SpbkZgIGoU{8Q*p&N4DF}&LG*jCIH~O?+W40Sw|AGg03$ z5A2y`F^$+cs`KRXyIJftcz2EcPmnLxES$ruEoiYHqURN=YMA&Fj?HQ`HIjC$_Wp~`4`5u0M_bm()Dmn zUVjW8sNv(`@4=8z^_d}2*>Cf`nb5;_w?pWd9)%uEgWNbWOI7!)=uCKY{Dbe z{-MpOt+DdCJ7+4=P<3={Gh4pAv#>o|k;bZLp8QH0*tvEuclWId?CuZpA-w0&K6IHM ztq;yj-ywlRDi;Y{ozzj|AzmkS61k2Ed&5VDhSdLEj>|B2PGlV@D5s6tW9WJAAj`zIOrUWX<9j%A)X#Cr!vAvn&iJQm(QVI77g; zW>LjZm*I3QVuD&0oE}znM_HzDpju-ssKY@l4?p&r$|Bg-bn28?E2KBKN<4YtyB+Hk zz&iB8d6jqQTDt!s(efjBSWmR@GiVoL*HF|W;|*-eD0mqC7&Tlx`X-9EQ2Ydi0)k1) zmSxbNVp~PgkD`9CyNXQ|IH~DY#?cC=w6^kFzlcizLk z6Y0&l^0l4ZHbiW?dV2gTsds1W-nqMHcBOIgcKA+5N9&*bkH+{=_lQ6y9CH1cr(l4^ zJeW9mNx=Q}af16L;Ql%$xL*S9j~?cJm=8)_HH`B+U~QgjMXX9P6u)ZYM6613hX)g} zD#=g|BVFz@i&&LpD2CNA5v!8i16Z#+PQvS7aR6q-7c2$U_FSDq1K1F=l!srPKX?w?0)foWzzRRJvLMfCc{a;h zp)dz^mON~!h5<9Mj{0CxkHCHQ4R{_F!y~f2Cio_5z){xBm6W^eG2Hi0fvd=qQeLgP1l;z(Ly6?8vE5BZTeUJEon3%1sJS>t4qR1xh1WI08!3 zNczM>s0p`P4>j?n;^cF9z32*#Lle+d6c3irg`uI*O5C(z`jt!rg+N5NbMC&KzdaQkOC z(c@>uyeIAsSERFecAoFUSj(4R|!K%2d zN3-g0py&JL(LCP~2V1N+h#V*e8fVZ)IPeH|jU38w1d~R1hI1(EDMiw87M~>!)pMe< zxY08OmBMvki9APBb}Jh)Go0@e~;pqsAx-G~@((Mlk!<(?ko-o9cqKGyJwR(o|4mRIK!GrWY)Ec5> z9WfNYfUo@~2#k{Bd))&cXLh@vg3Z>cNKaLh;8VlZ7ASVOb&qz;1oj4xe>V2T+0W1J z4qimJtwc#!;+57E~gFlYGf8gG`Qd)Qq-tK7c`3QO*#d~1y`A&N>FKKc=;#xgt zPFx>^TQ+<==d4ze01p(!Pzj5n6V%bMrn%7%hiU$YMGDVO)JfsLJ4I8g6}@7RWb&bh zcNLbX;oNcJaMGgcD&eOuahys%#QeA6c|GQDU|~+Xbsu;h2R9gf6EPIOg0GDkvN%6A z6f_Uh-(hO(;MpIqggx-2gm=iM`}n^N+$)(x;QuKq0nVD??V;MhJ(EfJ@055YEZ(wk zp`=0CUeD1mG(}aHzZXx2Ir(#^$J!|`-toc?uNClfAx~vF`E%dq@7k~kEc_yD;_!#k z%XAB7f$hO(e+lAm0l(k>_ojef`ZnzI4}I Settings: + """Return cached settings instance.""" + return Settings() diff --git a/src/benchmark_core/db/__init__.py b/src/benchmark_core/db/__init__.py new file mode 100644 index 0000000..0d94c4b --- /dev/null +++ b/src/benchmark_core/db/__init__.py @@ -0,0 +1 @@ +"""Database layer for benchmark storage.""" diff --git a/src/benchmark_core/db/__pycache__/__init__.cpython-312.pyc b/src/benchmark_core/db/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95db4eba8e25b3034b826d6db48630d261391335 GIT binary patch literal 182 zcmX@j%ge<81TmlXX6gXx#~=<2FhUuhIe?7m3@Hpz43&(UOjW8bi6x0iiN&c3If<32 zMG9&8MG8r&dC3{MiAC88#U=SgiRr0&ewvK8*yH0<@{{A^S2BDCne|JtxF}g4t~5S5 zzbI8dB}qR%J~J<~BtBlRpz;=nO>TZlX-=wL5gSkk$O*+D>pw6vGBVy`&@N&DasX3` BFl_(; literal 0 HcmV?d00001 diff --git a/src/benchmark_core/db/__pycache__/connection.cpython-312.pyc b/src/benchmark_core/db/__pycache__/connection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..171dc10c3324279f957ead14fb2e6f3cbb664ef8 GIT binary patch literal 1987 zcmZWqO>Epm6rQoYwzJvwCTUV=)20q7EiSMD38|OZ+vvlq+p%%-OPNyko; zU4j%f5dwkW*7R6FLPcpMZuEjsE?l^5RHSUJ^aK**7T85p6$jqf-lhp7&6_v#-n{X9 z-+RwL7mIlWXKZV3^Ysw-k@tI|ajD-W}q^bnO6NEa=VJk>}y%mPuWc~V@;RZ65l z1uDT_*asJ@0@0SFN)PEJC0KiFiqTg;bk>TjNh_qf9r!-ABPZ~6&+@G)>QO%`Z}Jxn z9BX4?&9~1}pE4^7m?6f6e->8pps)Lm<)T;s0E!Wja#}O|e z)D6qyh7_x&>02H(%{XtGa6aR5zF?YHW-K?8D5gmQ+cdvLE&leSk&xM=lhn5_c^12D z+5w}ZWO8&nJmu*c>jKS$sKA7rETQ|-i%m&orH*JkE_A-hoMCP(QTtL5cTER@Ylcpp z#s$xsGlQ#?Ro!4VmKlv^m~TV=JRUM|9P`#VkYyBgqJ!qqJf0Wk#oKr#)3)eZk z#~ini3?keXJNaWtHOa`;M_>1y$gy1bmhI@JFzJ?0^bAbBYS{pJt!#*_4{WhLob&@Z zX9%&JpkT&5bZ**V)C_zGnddptxRGQ0^;k|#GX8mte{DRlBU#MMz9F$*uCnuukl)Z@ zFrE>y;=nhXWReZSj;n_+0a-w88EL~!wZB>HZT1Z|2Zq~SIZa+T*Dj!gqZ_KWQ5tOS z8)_@kL3vHumw>b*?-Z~iLfBg00rLjJJlepeW1&0BDi0wdtYj!$!`pbQWSXy_MNIGq zpnY9f7C*}6(1(E49SL?gscVp~WaunHRzInUQ4JqQpY4aKjG&iMA_woUi`>$2ltUq& zg^n=+U&nRl#0fNu?})S55bIA(IMgNEFlN}5$LphyE8Gtvorf=loRsx5yl%39K4<7v zhlNqu5ZHe3CN8`+<42B1UtNyH0Yo#L(8!&-QcmIZ`z{)j! zY%l%_pOn8Af%!%r!{Y63;J(MFrMogPKVl%OSRUJ>tSWdcUszQY&X*K8-vKZOa1mzf zJs`UQW(T*yexGaZ8Xb5SpX5qXsWy+35@sQSRqb04Qfn- zUD_{_+*WOVnJ4!%ATVHtBS4Z+{w*U&Vy*uR{}cb7Hngq|t*Jv>A{;i=p+{i>^sWsV z_$EFoeW@6b5$85}y>0UU61+PA%ec|Uim=9VRGW7EDaIX*_l8$o%e602uU4jWQ8_71 zCrAo4E!m`qwI|d9<}qF`X&2-!r5PiJ5sRfs=x?#;_)$`#NmeBO{LkAHX+u86o(Bs& zsX_*1OT-x8N6$V$!@r}8Yv|$wbgV54xa;H7Z3N3!9_6(Kx%u>g1{(NNE@Jt{$QFX- v<~zx9*G`tkyBE`SvfAR(Ej@?#-Z*vZg}YMY*on2llPv@X?H&mqN)Y%D*_q~T literal 0 HcmV?d00001 diff --git a/src/benchmark_core/db/__pycache__/models.cpython-312.pyc b/src/benchmark_core/db/__pycache__/models.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95a89b7274597d9113ab46c413dae3d161692df3 GIT binary patch literal 14766 zcmd5@drTYYb{~TQgKeI+F>eS2Fb|XWW;Y3W0f8jIZh+j4_c3cc0~q{(8Jj?M@4c+< z?QOPCXnT8!s;Z>7kD#P2=}Ik8REbrkDyvqNtr=s((c4AKQkjEA2n+Ip283 zGq%aH(%#%VlF!VX`Cjulzw_ z<)Ih@lM^rm4IYC5k8=Y#!CX%+IW`82!8}hMInE2@2Mas}L6gT6Ec6rxi#$a^v&S5? zcq~Dy#~Lj56bDN@CBafpDJh>HC<~T*%E@s-pdx7V*n)PCJ?QW_h|UzK3_3kda$Fdw z3RZingEgL-V6CUtK;=-ED5mI#6k}$c_jBbjd+LbR0$Qs^TTisbpe@m88;G_Pv}GD? zBhi+FwnC$AB3c`0?HcU?qIH0_Qlo7qS|@0$G};!Ttp;t4Mtg9%)>U__d+d#?=L5bO zHn`k4dTqEd7-rZ2-#8WK8YkJ1ZzkyF<{BAq#5?KbS@$!%SC>IDfrgFvgDf78&PV*= zkT(FwdC{n!Ip!*o3ND1h0oEImOnp#t97;;Y{!lb1Z{!j$EF-}Yj^|^AT{9*6> zJj+PdAlcKta3C5CNfwR`cyUbpjDH@BUx2YgaqBe}TH+&P5pN{QOC{IBfk1SA%om<# z$Cu|>$vVdJJXDjf`;y-x*{L7qo8-e}DUSg@&YTJOBt5xIj>niov3c;zXL27H??V~f zm^>!`htz`t^%E&zO>AKmTw#kC(`nczYnhoswN5UR_n=6vVbRnu?^WZ0<$ezKM^PTD zro2^GzBtu`rhKuce2K1nsZw6qdrF{esXYGVJ(Ou`l)<>mm9k7OTRz(=k4Y;ZRH%1m zgVxHE?bxuLWxLBESzqVEi++aXhVeVRRj+=M)eI%wv3F^CmPWTwTLpfD2_nR)I2 zsGvvlJm&|KTc+p0FbWuU${P(tP%A$b+Rr@uKe?nS6b%IMDM!qrgESgA9pwV>AWdl6x56QI zGl#Us;iHp^d{8dJGPE}`;<8I7nzT;SQV~tV6Nm;_IJVIALev|;e&MWurkSu04y-gy zUYV@t%Aq~36om^#8Hz>_QVC6aL!mGjD4H)#b5^|MMsWzm5fsNz;A`ayP~dBXhY<0F z!Z*2-sA)ohPn2s$fnOOBc)*)O}I|s%3!IjIa z`K$bzSuhSJS`I&K-LCk!^GTlAG7>Y|R;Sj-*5(A`NTTQTqw6~tAHOB^48@H0)y4Ih z4gO9rUfCrWhZ4<)9y)i9iH8ScV{gVLXFr=zOb9?)jwA~e`0mQiu7H2uN=R|Bv6TL3n$9@^m=P^4K&3+QguW*M}KU1+O( z61(O?VmKm%7sa`o@>hvyFO{c_8}N$jz`i2{|XAUZ$KaJRXlkO#VHi{wz)GXQqa(PgOJ9hEdrP#!cxbLRex)d`y*7Mi-JAe#J zFG7>OL8GyIb82hs{@hN#*m!y6>T2tH#To$62FRS)z*63cA z={nPfshaHxxFN-Cr~KkI6q&7#?z`Mgdsz*&Vp|8}JQa!k)R9QwTHK<_K;J|}sR?OPa(2~Zc_11PU+A^hIFj~hf`i-;=LXBKm z<#9r5j$}I(X#C4IV5in`FF$wD%Q3RaHt79s1TNA(xT;!gQA+i)4A!9=qH3<^;1(2f zWoJZxNbt{#?1IM2TGX*-S}S)Hs!N3&J0GU!y^$HoI34x|q+&i2o`-lP#Q2d}OO?DF z>e9&cC&Np0&>w<$fzPn-s{PpaHRyA1`yRuq#I%W}kK^eC3JZ$sAf#M~>-Xb*N3jM9 ze9!#9fk4A|J+(E0;e)4SwvUVM^G_o2cNWA(F6Aw`7ukK=v4_{UFMj-%&~~19%b9h4 zEf_Cp5sc>(2YR-Q!hz$2_&WhZ>f%KWg7J8w>(%WRq3i6*$m+oQ$&HJ5&ctml!FV=t z{KC#F!tpCHW6k>Yjlp|kcZcJ3M+D=QZvY~AdpZ!w2L*1=W4|6CqRVjrs94kJyJDDH z`mTZn0U=GcLlYio>AMOWP(HIiH8(-R3#Grk=)9Rxp+|ug4+2%W4)h3gKyR4EsQ?JF zDP;kIY_mvW)3yh86@nbl8i6z54UQ!LsAMXcoCi)d_;EtJl}wdRSG8R2s*x<$y`0~x zkc&D!QZ35YRGcy?ikF2_cJ1g?kV1F;9L*Wh7``cSIv`AkL`a8_pqp@tqU1m`A?ho; zlFs1W$`O_#K>@}Xl38Z{-hkiBf7P58Q}FII1=j6@YR}GL-EsiADgQJ*1t~<9gLDcY zi#~iHsQ^qDO?rKE96Lqm{D`Z1KS=iubg?(mVPad(XwZ0sgW@Jp5cJCyT~xqT$)r35 zsU+>z(qpt6*cb|A8hioZ0pefVQ(HaYA7mu@w(Vlq*(c-ix58rmd@3t3|04WTGa#RI z#D*E+0vnr;iVcgYkYaJ~SVN6?KCO<8FAGDrzNDyuoGUrI6xDV$M~!}-Q&is7oW!9s zkEV96J)RW~K>`E=qosSfcfK2UcJC_zA{e)CWxs}0$<3OzWxvr)h&LE!f)9z}7z71+Ot})9}4Y09$*K@7XoMo~|7Qya_w3lFKLS zC;*K>U(Hl1C6&FW2HLOExdx}EmujX;DXHu|wHY<4bu}{ksLQBPqpMM?w5{wt^%*tl zlp1VLMnK@f240{2b>xzj@RHFHx~(u~!MlT$CWmz%~Jl_tEBnjD$IQxrGg z$DhvGKc!F;EuUPDjm`>vbFszu#ikEZ*_ID*l0>E?pH#=LdWG|ou^=bb^C@D&C)B9z zlh7{4SoE~jF*<-|858IVeC+4`<IiDO8UwvKZd}C$s_SEW|>oakCTfDekFunmH;G@=^ipQNo*Fahr zIFNWzNZEe;;neon$8$pa1$k~~BX{k)@zR5W@q*lD`Qr|ut3R#H{zUunt!bgXHyNTv z)+6FXV|Rw*_71@~o;Y&$ll&e2pUuLNt1+W< zy>-KRula6Wyt+p)Uj3r1E>_>OT`iXNtz1qNSFiVOy(Sj-u3So#)~?fAvtlU(#2T8l zGWtLHRAF(f*u7PZS6q^vQIq z&a-Q$)HB&sij8SoK{lmtMa`Ficgk!f#mTg-Ag9u|vR`JVZ3RPKeJfgiR+c63EzhW- zX=6_%j*Xb(>H>s!!rFioGpN-EV__KaFu9;T_K^yk4a zT;ETXhR0Mw>(xqGWl#8_m95e+CM^@oUZ43J{aqN(4Y1On+#6gq1Cu_7P zYjiO#V45B3x7H2g=wdo`IycdE>2y6r*R9hXB09HDcNla>m>yuRHOk0|<`A5Nhs6z; z>(S-It|O8qxhg3ubLHd}ODX26C|(+K)s9YOt{=b+aBrg^L_$vZB!`B~wL;-z?i8Jc zg;;rhT)pw@dz_ONv^5nMQE~^Mam;-A?`gaKq%KuIFd>N{sOGUX-Y_XgfLe#tWLtpQ%Ed=3f$rjyKFa_BCqbbiEjY(K>JN7zX= z;!9o~A+9M3OkNn_u^(Zz-$FrN&u`<=@1P(Ym@r?+!%*Dsq9{c1dmyB|DO?lC{XQyI zQLLe`p!fq6-$C(*DE1mCK)A zj(sO2Tn)!=zAw5yNSo07;DyW;mLWN|b3d*Tn$N8C-#)S0zJ7GAJ8o_ijAt-?$2>VM z_P!Axdj|$h8~So!gyV*O+#McBru&qf9u9sW=WFQc;oe7!JHE&73x{FOq;lP{VY*jw z*BY;Kt|1?s$9SG-rL)! z1=Aqm`mOgYg6X)9>!Yps;HPf+$)o50VfZC*eKVV4<+%?E6>mxwzFLxAzDUChbbK79 zi&8bTSy62qP@JXR0%Xt8$v3!i_5P?vt!IH7863#mSI{ z;S|dEfOMh<3PYL#sRl24D8JCU1n88|nNq7*&7{BY&#Bd+@W`1lQ|obKa^S9Rcp zyD0t~1@Y|oaAc4D7kG3BMOtF(FH!ebC zC${jiuZfpI9*M^#y>CL|<;=LyGm-QJ?eVJ4csZ=;op`YW!(2+$dhN!otuwJpv}k&d zAZ+7C*H)2WI;F!{d`8(ZcH@_Uu{z8I+trYW}W(*$4&M zmHdZ;u|w47P{&pYcj_S1{wbYv+3Bj1tmnCi9~Rvw7Z9fXtpRfF*EOIdcN%zGq-~IP@Cd8&IsZ{$F%%f*tC;YX4EWmkn z!W2)5#b+V;D>KC!v8?~Y%ktghjgPom>1z!H%#ADQM@xJ3Rj0!!J{ zF5F=*D8Kn~Yy;n(RIC4cj+>cu!-|j)GzIJ4I2#tRrxhV-1)jrbk|WL!XK_-2wmkDR zin<#p$Vd7EJR%N^d=>gH1^xyEZNeX7dbw>Vhd9Rn4Tvvu4F*GkI{GMbIpz3; z(QL^11*M3eJ1N7VpHaPWs`tOB?w?VkacXooztF%NR{M4--0wPzK(4p!Qn=r(F2HNd zE`|GD8|tv6e9dYYF{~cjrEtI7SZm0~6WHzcw#dh@+Z{2K8uHhTyA None: + """Initialize database engine and session factory.""" + global engine, async_session_factory + engine = create_async_engine( + database_url, + echo=echo, + pool_size=pool_size, + max_overflow=max_overflow, + ) + async_session_factory = async_sessionmaker( + engine, + class_=AsyncSession, + expire_on_commit=False, + ) + + +async def get_session() -> AsyncGenerator[AsyncSession, None]: + """Yield database session.""" + if async_session_factory is None: + raise RuntimeError("Database not initialized. Call init_db() first.") + async with async_session_factory() as session: + yield session + + +async def close_db() -> None: + """Close database connections.""" + global engine + if engine: + await engine.dispose() diff --git a/src/benchmark_core/db/models.py b/src/benchmark_core/db/models.py new file mode 100644 index 0000000..a8f6832 --- /dev/null +++ b/src/benchmark_core/db/models.py @@ -0,0 +1,190 @@ +"""SQLAlchemy ORM models for benchmark database.""" +from datetime import datetime +from typing import Optional +from uuid import uuid4 + +from sqlalchemy import ( + Boolean, + DateTime, + Enum, + Float, + ForeignKey, + Index, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from benchmark_core.db.connection import Base +from benchmark_core.models import RequestStatus, RollupScopeType, SessionStatus + + +class ProviderModel(Base): + """ORM model for providers table.""" + __tablename__ = "providers" + + provider_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + route_name: Mapped[str] = mapped_column(String(255), nullable=False) + protocol_surface: Mapped[str] = mapped_column(String(100), nullable=False) + upstream_base_url: Mapped[Optional[str]] = mapped_column(String(500)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + + +class HarnessProfileModel(Base): + """ORM model for harness_profiles table.""" + __tablename__ = "harness_profiles" + + harness_profile_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + protocol_surface: Mapped[str] = mapped_column(String(100), nullable=False) + base_url_env: Mapped[str] = mapped_column(String(100), nullable=False) + api_key_env: Mapped[str] = mapped_column(String(100), nullable=False) + model_env: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + + +class ExperimentModel(Base): + """ORM model for experiments table.""" + __tablename__ = "experiments" + + experiment_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + description: Mapped[Optional[str]] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + + +class TaskCardModel(Base): + """ORM model for task_cards table.""" + __tablename__ = "task_cards" + + task_card_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + repo_path: Mapped[Optional[str]] = mapped_column(String(500)) + goal: Mapped[Optional[str]] = mapped_column(Text) + stop_condition: Mapped[Optional[str]] = mapped_column(Text) + session_timebox_minutes: Mapped[Optional[int]] = mapped_column(Integer) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + + +class VariantModel(Base): + """ORM model for variants table.""" + __tablename__ = "variants" + + variant_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + name: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + provider_id: Mapped[str] = mapped_column(UUID(as_uuid=False), ForeignKey("providers.provider_id"), nullable=False) + model_alias: Mapped[str] = mapped_column(String(255), nullable=False) + harness_profile_id: Mapped[str] = mapped_column(UUID(as_uuid=False), ForeignKey("harness_profiles.harness_profile_id"), nullable=False) + config_fingerprint: Mapped[Optional[str]] = mapped_column(String(64)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + + provider = relationship("ProviderModel", backref="variants") + harness_profile = relationship("HarnessProfileModel", backref="variants") + + +class SessionModel(Base): + """ORM model for sessions table.""" + __tablename__ = "sessions" + + session_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + experiment_id: Mapped[str] = mapped_column(UUID(as_uuid=False), ForeignKey("experiments.experiment_id"), nullable=False) + variant_id: Mapped[str] = mapped_column(UUID(as_uuid=False), ForeignKey("variants.variant_id"), nullable=False) + task_card_id: Mapped[str] = mapped_column(UUID(as_uuid=False), ForeignKey("task_cards.task_card_id"), nullable=False) + harness_profile_id: Mapped[str] = mapped_column(UUID(as_uuid=False), ForeignKey("harness_profiles.harness_profile_id"), nullable=False) + status: Mapped[SessionStatus] = mapped_column(Enum(SessionStatus), nullable=False, default=SessionStatus.PENDING) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + ended_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + operator_label: Mapped[Optional[str]] = mapped_column(String(255)) + repo_root: Mapped[Optional[str]] = mapped_column(String(500)) + git_branch: Mapped[Optional[str]] = mapped_column(String(255)) + git_commit_sha: Mapped[Optional[str]] = mapped_column(String(40)) + git_dirty: Mapped[Optional[bool]] = mapped_column(Boolean) + proxy_key_alias: Mapped[Optional[str]] = mapped_column(String(255), unique=True) + proxy_virtual_key_id: Mapped[Optional[str]] = mapped_column(String(255)) + + experiment = relationship("ExperimentModel", backref="sessions") + variant = relationship("VariantModel", backref="sessions") + task_card = relationship("TaskCardModel", backref="sessions") + harness_profile = relationship("HarnessProfileModel", backref="sessions") + + __table_args__ = ( + Index("ix_sessions_experiment_variant", "experiment_id", "variant_id"), + Index("ix_sessions_status", "status"), + ) + + +class RequestModel(Base): + """ORM model for requests table.""" + __tablename__ = "requests" + + request_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + session_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("sessions.session_id"), index=True) + experiment_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("experiments.experiment_id"), index=True) + variant_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("variants.variant_id"), index=True) + provider_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("providers.provider_id"), index=True) + provider_route: Mapped[Optional[str]] = mapped_column(String(255)) + model: Mapped[Optional[str]] = mapped_column(String(255)) + harness_profile_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("harness_profiles.harness_profile_id")) + litellm_call_id: Mapped[Optional[str]] = mapped_column(String(255), unique=True) + provider_request_id: Mapped[Optional[str]] = mapped_column(String(255)) + started_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + latency_ms: Mapped[Optional[float]] = mapped_column(Float) + ttft_ms: Mapped[Optional[float]] = mapped_column(Float) + proxy_overhead_ms: Mapped[Optional[float]] = mapped_column(Float) + provider_latency_ms: Mapped[Optional[float]] = mapped_column(Float) + input_tokens: Mapped[Optional[int]] = mapped_column(Integer) + output_tokens: Mapped[Optional[int]] = mapped_column(Integer) + cached_input_tokens: Mapped[Optional[int]] = mapped_column(Integer) + cache_write_tokens: Mapped[Optional[int]] = mapped_column(Integer) + status: Mapped[RequestStatus] = mapped_column(Enum(RequestStatus), nullable=False, default=RequestStatus.SUCCESS) + error_code: Mapped[Optional[str]] = mapped_column(String(100)) + + session = relationship("SessionModel", backref="requests") + experiment = relationship("ExperimentModel", backref="requests") + variant = relationship("VariantModel", backref="requests") + provider = relationship("ProviderModel", backref="requests") + harness_profile = relationship("HarnessProfileModel", backref="requests") + + __table_args__ = ( + Index("ix_requests_session_started", "session_id", "started_at"), + Index("ix_requests_started_at", "started_at"), + ) + + +class MetricRollupModel(Base): + """ORM model for metric_rollups table.""" + __tablename__ = "metric_rollups" + + rollup_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + scope_type: Mapped[RollupScopeType] = mapped_column(Enum(RollupScopeType), nullable=False) + scope_id: Mapped[str] = mapped_column(UUID(as_uuid=False), nullable=False) + metric_name: Mapped[str] = mapped_column(String(100), nullable=False) + metric_value: Mapped[float] = mapped_column(Float, nullable=False) + computed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + window_start: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + window_end: Mapped[Optional[datetime]] = mapped_column(DateTime(timezone=True)) + + __table_args__ = ( + UniqueConstraint("scope_type", "scope_id", "metric_name", name="uq_rollup_scope_metric"), + Index("ix_rollups_scope", "scope_type", "scope_id"), + ) + + +class ArtifactModel(Base): + """ORM model for artifacts table.""" + __tablename__ = "artifacts" + + artifact_id: Mapped[str] = mapped_column(UUID(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + session_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("sessions.session_id")) + experiment_id: Mapped[Optional[str]] = mapped_column(UUID(as_uuid=False), ForeignKey("experiments.experiment_id")) + artifact_type: Mapped[str] = mapped_column(String(100), nullable=False) + storage_path: Mapped[str] = mapped_column(String(500), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=datetime.utcnow) + + session = relationship("SessionModel", backref="artifacts") + experiment = relationship("ExperimentModel", backref="artifacts") diff --git a/src/benchmark_core/models.py b/src/benchmark_core/models.py new file mode 100644 index 0000000..16985e9 --- /dev/null +++ b/src/benchmark_core/models.py @@ -0,0 +1,153 @@ +"""Canonical domain models for benchmark entities. + +Based on docs/data-model-and-observability.md schema. +""" +from datetime import datetime +from enum import Enum +from typing import Optional +from uuid import UUID, uuid4 + +from pydantic import BaseModel, Field + + +class SessionStatus(str, Enum): + """Session lifecycle status.""" + PENDING = "pending" + ACTIVE = "active" + COMPLETED = "completed" + ABORTED = "aborted" + INVALID = "invalid" + + +class RequestStatus(str, Enum): + """Request completion status.""" + SUCCESS = "success" + ERROR = "error" + TIMEOUT = "timeout" + CANCELLED = "cancelled" + + +class RollupScopeType(str, Enum): + """Rollup aggregation scope.""" + REQUEST = "request" + SESSION = "session" + VARIANT = "variant" + EXPERIMENT = "experiment" + + +class Provider(BaseModel): + """Upstream inference provider definition.""" + provider_id: UUID = Field(default_factory=uuid4) + name: str + route_name: str + protocol_surface: str + upstream_base_url: Optional[str] = None + created_at: datetime = Field(default_factory=datetime.utcnow) + + +class HarnessProfile(BaseModel): + """Harness connection profile.""" + harness_profile_id: UUID = Field(default_factory=uuid4) + name: str + protocol_surface: str + base_url_env: str + api_key_env: str + model_env: str + created_at: datetime = Field(default_factory=datetime.utcnow) + + +class Variant(BaseModel): + """Benchmarkable configuration variant.""" + variant_id: UUID = Field(default_factory=uuid4) + name: str + provider_id: UUID + model_alias: str + harness_profile_id: UUID + config_fingerprint: Optional[str] = None + created_at: datetime = Field(default_factory=datetime.utcnow) + + +class Experiment(BaseModel): + """Named comparison grouping.""" + experiment_id: UUID = Field(default_factory=uuid4) + name: str + description: Optional[str] = None + created_at: datetime = Field(default_factory=datetime.utcnow) + + +class TaskCard(BaseModel): + """Benchmark task definition.""" + task_card_id: UUID = Field(default_factory=uuid4) + name: str + repo_path: Optional[str] = None + goal: Optional[str] = None + stop_condition: Optional[str] = None + session_timebox_minutes: Optional[int] = None + created_at: datetime = Field(default_factory=datetime.utcnow) + + +class Session(BaseModel): + """Interactive benchmark execution.""" + session_id: UUID = Field(default_factory=uuid4) + experiment_id: UUID + variant_id: UUID + task_card_id: UUID + harness_profile_id: UUID + status: SessionStatus = SessionStatus.PENDING + started_at: datetime = Field(default_factory=datetime.utcnow) + ended_at: Optional[datetime] = None + operator_label: Optional[str] = None + repo_root: Optional[str] = None + git_branch: Optional[str] = None + git_commit_sha: Optional[str] = None + git_dirty: Optional[bool] = None + proxy_key_alias: Optional[str] = None + proxy_virtual_key_id: Optional[str] = None + + +class Request(BaseModel): + """Normalized LLM call.""" + request_id: UUID = Field(default_factory=uuid4) + session_id: Optional[UUID] = None + experiment_id: Optional[UUID] = None + variant_id: Optional[UUID] = None + provider_id: Optional[UUID] = None + provider_route: Optional[str] = None + model: Optional[str] = None + harness_profile_id: Optional[UUID] = None + litellm_call_id: Optional[str] = None + provider_request_id: Optional[str] = None + started_at: Optional[datetime] = None + finished_at: Optional[datetime] = None + latency_ms: Optional[float] = None + ttft_ms: Optional[float] = None + proxy_overhead_ms: Optional[float] = None + provider_latency_ms: Optional[float] = None + input_tokens: Optional[int] = None + output_tokens: Optional[int] = None + cached_input_tokens: Optional[int] = None + cache_write_tokens: Optional[int] = None + status: RequestStatus = RequestStatus.SUCCESS + error_code: Optional[str] = None + + +class MetricRollup(BaseModel): + """Derived metric summary.""" + rollup_id: UUID = Field(default_factory=uuid4) + scope_type: RollupScopeType + scope_id: UUID + metric_name: str + metric_value: float + computed_at: datetime = Field(default_factory=datetime.utcnow) + window_start: Optional[datetime] = None + window_end: Optional[datetime] = None + + +class Artifact(BaseModel): + """Exported report or bundle.""" + artifact_id: UUID = Field(default_factory=uuid4) + session_id: Optional[UUID] = None + experiment_id: Optional[UUID] = None + artifact_type: str + storage_path: str + created_at: datetime = Field(default_factory=datetime.utcnow) diff --git a/src/benchmark_core/repositories/__init__.py b/src/benchmark_core/repositories/__init__.py new file mode 100644 index 0000000..5a35edc --- /dev/null +++ b/src/benchmark_core/repositories/__init__.py @@ -0,0 +1 @@ +"""Repository layer for canonical entities.""" diff --git a/src/benchmark_core/repositories/__pycache__/__init__.cpython-312.pyc b/src/benchmark_core/repositories/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb30cc0a957b2701eaec53a613c4cd6959a34762 GIT binary patch literal 195 zcmX@j%ge<81TmlXX6gg!#~=<2FhUuhIe?7m3@Hpz43&(UOjQ~|sRjAPnI-u}l?pkD zm8nGvY57G8$%%RSd6~(HISQ$HC7C6esl|GJnvA#D?i)P#PkYgiX?fsHq`;MOh(@);r^5-Ps-Y z&KPGGSB;zqC8DUw3r=~mD1ua>A`+DdNQmrHgLql9mDU-hq6jL{w?!@@ig@6hJG1_) z$r~fhx#ym9?%BELJLlZHf9mQA6L{>^-cgk zrrs~*y94AH(Suir9x?=@`za9L7JOY#HJpjmo<3Bt4V4))mR)izR#oRp&IPq-mCIH| z)eBBx67|-M!3s{vs@U02aCt^{V>!E8DIPN_sIi!(APKmm7Ma;{(U?@fq<^j}91#}stAsuBw zLjtWdCF>!Og}E$jL_ikx{D39`nl3%gbx|V*y7-!|>xz``-mt0%>|X3nD?;3-_{7F#s`_}?X2tPIqf)$3F0fa$qQ#8ywUj6s_BbX%cBbkEHLX-BIht1M z+q{)*M}fa#hk#rnE9o6q#UF)NVM2<2Mj}t3E{Fg>y?O_j7YT(Ac2LfrOA*Jas6n-E{tn=H z+)_JlIGwtv?c9pN1||Bib3pz<9$g}TkEo5v$iu`yW7oHviIa`!$%o02R&w-%!2Dd!QQ?;n;C@37ig#sT z{y>4$shmqHb%o}_(RxrpKCXbAVV${xH91{e!xkjSUxt?#u9pY%bb1YE(D={%S{_hH z?dgp^e2Y#_T!xOh8E3GUI)2v2b&uPu2%IL67rWpMa>N6*)L~{6978Rr6=M#%B~*x> zoyZ8TP|$T(#&%h9T!k4^%&;$H1P&qJHQNb4YU3BHs_KC#7rC=qdJBA>cn8`a0>Xj& zC>Vb;axHTG@CU*Em00@C%C$-}HqZ(VG=c*-x2{&E)4z9!#SHo?Igmyhz z2@XECzW{C)hH?@53z}2JTm8V*3C)GXI#rMlQJ`+1q;G-+k0}nzpMiN2EbE0jPo|-G zwmzNWoyLOS@@&6iy3@-ySkDJu8G&QVPm9jD6}H!ed0{%TVVTc%5`#mU^5N{k0`^c; zC#&l5BY>ssD#k^lIO`Y;PjKEJU1{!u!3>6RbQ5;5U>2BWRSeByS_Q_8xB+Nx1kKr= zH60tH0f766hp;dwt6E8~^=|=vO@qtr2_PQseFKZH-+29J-@B#%YWkh&=J5X3@WJKb zgZISu!}aj{UG=WU$N$H=UWun$@sZ{DNHad#2#&S|l6aXuw(&q% z$i;HQwBAi~Lt?!r4ZJIvB}my#xMP|8|8ZX{<^PSOgXaf-o{Z^qSI2IB?BLz;&T${} zao>r!V<+~ox4p{_j|60Z!ghn7b@Ume9dShyuQQ4Vz6}L$nwwxpBlblgTQFA;Sun2k zZi6_;`H~Z`?#Xyd&EccXzGJPPV^`&e@#OWRixW2{nyEdl)W~vbq?sCR#>X1LF+}^D zfOZ@lF6;w)xgncM$umeKk-Opb77XSA{psQ50*bBXDnoX1uw8&m4ueI}E#WWw<4Qo(;fc6I57Y zUO(Pco@^ZCpXtV)lQB@)CxLh{7Cs7-~g_ z8qpzN(o77tqJxd-;7YP@C7x{-R3ElbjlrHMPw-%B;V^1||-qpJi2?U*Db zmrk^SPu{8W;odpaB-=4T8e5ug6BzC$`EalQer7p$W)(&M3=7iUrSGDA@7>*exG&y6 z_s2MD_qM}|lt#BO+$r+muKp>`yzj?PpU5QGvoN}{Q=KVQ&U=Bu7fK6rT0|`X_c091;?697vVeiJT%K=%Gy&NE#|aDuIxwMZ0pm6XLM;hS^QQ zZV(kjRgfwHQq^$e&_k=xQ+q&_DiIDBXd!xknIre!Pd%vlK|71KkUEty!yaSJ%lh?!2s zPh-;r9jtvdHzy4ZQ)5?66ffp{HC=D7XXxv&8)48w*-fF78vrK<(L@-+^Dg>SpX zN?CLrX^szJ=bt&2`Hw{r&II-?+wE%}cX)$8jgFvCy5Uj2;xBO_KGSiG;FyJjv0w^c z3f96J2?=&nZK)33pO@^FAhV9tAX%RvXLYgU|TO_Sa z5z{nViW|vf!m?$8rwKM^;4=aMe44naorl_qNeX6@MlJ|r^EH#iho*2UK9h`*3u@dT zc!*T$DS^$Qx@}hF>Bw9fKE}3}=^CrTrG~ahN$G{pnDan>LoYuHxPx=v-n`PguJmsB{7asKh@9OoH=Y zJ1PsS{T;wRMBGuQ@Q{-!@8p28i0+K>Z^13abWDQ!s}F(oSs|oG1!ue=z5DtBNN=+nMb=k1cg7 z4Y#uin1;*wgyyn>o*cyeE|PgzgRH!{gcT_}UxU@Y$T_ z-6HSkaJ@i|R@ahLkWlmPystm&>(BYZ>q@vFgM70{OXAvyW2B2)6}XW-!m8K?Jgb@i zX`0)&r>vgqjLN?{bY|gf6gDy`3BaNfdY?OPjZ_k?H$(+j`VO}b0gitH-o@UhPY=Mh(kVGQB?{wRkdBJ z3is8#PI-^2UYw8VWsR(=nh{r3LjNKN{b^twjC4|N7bUbElOQDrfY^K@Wf8dI5uwNV z5*2|W)~*QsxnP9$HS-FPMf6BGR1n-^+uex*g6aM&o7U7KT@^Y-ap3Mk0m1a(7@O9P zubU?Sc^kBBjyr{?Y_zX^_S!vt5Yk~;AY(--9cQ#?L_}Cp=Z}^Bd z6)F7(l-hnQR2dxZLaLOoB>z9(qt TDLV874Ze_ixWPXV5a!}fm?FOh literal 0 HcmV?d00001 diff --git a/src/benchmark_core/repositories/__pycache__/harness_profile_repository.cpython-312.pyc b/src/benchmark_core/repositories/__pycache__/harness_profile_repository.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c60c6be7dec046a93722596f571b29fce4aa12e8 GIT binary patch literal 3358 zcmds3O>7%Q6rT0|`X_dpU#H~9Z48O?vjs#260}m%e?imIf`qI}%f>rtoUXlLcGK1& zs8y*@Da^4opmfy zNX%IzaZo3DY}-ePHJ3GU0_UBqQ&8{IIjMjryppvYPdT#aWUahzrZ`Wsv57MdaFHpy zkk1^4$)JOAXY*2rEd$dD(PnYR-;Ga|+MwzHvx+cEb8z^8!KHa4fCYocq9I^uPBO%c zLRw}r2`eBAVyO@2fJO#Qh-s7-O%OC;qmAi8I0CcU)@Ow;is@(}l8Soq=KVF?v#d7w z&0MwT*z0=})*>c4b&joivhDlAlNWt&dJ@6T63J)y+Bs<#gzsJWswX&vUPc_8+X2pR z&bsAFV41_m#Z6}L$IJFwbIpN9!rql?8RN0Doj#ULJZJ5>(%SW`=zQ$gk+ZdGt&B z@ZF*9*Pbs8?{bHC6^C}0x5QWbulAP*lBI#2?!eA+U!v4E>Gn;Q<3pwRxEmiY_Y9PJ zM%VTAH5VZYtdLsBkkFr~Ec517w;?Q1oCM9_?K8rI;4kjWj=4YpPW*!sF z^3&_EGrFl0&B|lVB3d5$lt8vpTb6!bbj-j@zA26(b|1$fO&Yysb`^)Yn|~wY4{9U3In?$yE73JoG0D zbc8NR6$!QXlwzZ9Y_u2~TT{j=GRPlRDX=a~1*V?hz7@EsKH*!j8+eAi|7pnk9}{v~ z$7s+=0$kDvIN3&rtP*Vpt|`7uItPfoD5g`8*;Qg+;|5nmf zXNr*{|02GGF6@bB7HVWuJZVBR2!996e+l80KpmL%{!xHB^j2r>@1eTTR<~>Pbr{!I zXK)^`!MTsdRyWO7NZSxtxizp*OOPj+^vNdsK(^sdG0mnBQzNoTh-DgLFkWtX5;h(| zYRzH-RUsJ{_JnDFrduY?k@xn7x+qf^dsUhLZ3E*Z@+i zci)mp`G}^ywWOOhjjU;gmC-apOF5y>AXeST0Ofipq4k>3n}Uo0@%U`sAv_&U0{IkG zfg)DQ2z@RwLTjFV9>_&>M|hzkM8)oFuT&6pH+HjbZAP!sVI?7ocrvEL>!>7&-6`tkpl1p;=BZ;K`EQyk6fs|w_ivN<@wJWQRW3-kHNeNO0=!WL5Vk-Pm zc2|+5DoEiNg$)FV4HT7ost*O~z$xlO554pt1qcfCB7z3UEL_Au5wyLrrGXea^}X38 zDN(eOp3(t0^JdhZV;l;2kWoc@Q0kiTQ6Rya$jz6HVtkx7EcjBLv= zSzE%EWfLreeKx~o`2Vb_$5lF~sEr?k~vYEIEK&_Ta#_d*x)vXX%X#NC62&wR>UD!-)6uP!Mm zb7N-hwmQ|%LD(P)Nig6Go6ICw*`{zZtMD?X*ca?Ff1OJ>n#SxfCeSg5;*ceG(5jH-jMg7!u0HGKMaG9)Fk>{iyIEjbhBYs={Z5G=*wy@#ad3o1|jw#{HoU( zLViRTaH0;9S@gQ)x^1xrxtbE%`<5kW?Yo98anEdFzkb(j3iQ1eu%@_o1bZ;*Z8_?1 zIoiH|^i$@(wU_tVb!IWR-(sDu?TjH{(YfE^zBTkAk!?Q%AN|U7SX^K)>He4FALeha2 zhG5Dt>NclHxnx!`M62^&GPA50?o>Xzw5%(#l++DzC7qM=E0U%sRo!q|JtZg4#5}44 z-oaosT~*PwY8zJQPPGH809NQbHG~!V$Y7JQY}nxe*|csrRAoU`v@0>r;DG%LDjuc^ zVT>4j7J05Ap2ARePH1Xs;#tsYc9bccJjqy55U2lR_f7Y$vk!%yvN!l~ z?q;s&?Jo)a+d@C~^EdN%l%h9M5+d6|r0fga5Gy=!4m>RjksU1po4j_WXPPiu2beP_ zxvjn~l#eoJPH|hu94Mb+px#4bJS$4#&+8;!CyR9&gVrkR3=qhQ?6om$b-vbXP30OP zK--GQR%iflGrQ>k1Y^||nA3{hthD=?t*FjQ^`HW4w4yyL;hWO}7Og@6E7kW4)I{hu zm0eWcvv*46=LU!7w}lx%W$=58fPsFmWz^<*l>Qq){}b?wz|Td)fNgES3ImXCmzQJ= z2qqC~p}UAnv~ncPq2WJ-FItzGb{&!?=~gC@OL4L}v+)B`9ts46H|A(y9f zZ`awX_Jf6CUtZD_RevVyO!Xj`YM7WQ>nZ~2=`VF3E_5F*iygOKF9k;m!I4kdd$0WA z;wKl2(UX5#D4n`gICZJY+In1H6I;M_L-@Cyh=Fosv=ljBh#bH1>O*m`JRB!bDv3ke;t=*HZcp5e7Td>4;@Gx0R_^G%aj_ii z-gxu&n|C|!jTD2ECEw(>Fj?t@$3Lz3iD&p}SsZ)%*f+VOy$Y{#?ez2@v&k^i-Q1>a zst-C_1I%x~U?0A`8m4Fu1KMkhz)TmPf!yd&t|}?WZz}rJ%whb> z6-8B={B5*rsbnUpO8J~3Fn9O7>7;5wao;0Bd&~vH>VYLSA)Di5~QDkr#{3vpmsMFxCl*z>S zmUsj15DnqmfSDoghSJ*bx4_#pu9Ahygqin$hgp?p5m-C|i$`ER5s5!=>y?d}+cU*L zq!fr20?}e%wCEe#7RD+bu&IG8aE4PrODs}*mW;|x7S&N{@NMjWXhpt&JbDCPtF=e1 zMZU?Smf0Jyj*5|0Jq(rMQLe&wa5@WrOnle=>EDOAT)45jCLg)ZzH#a7?D-34o80u$ zJ~y>lz-uhE7JWYU8kRrB)zDeZ@PljFFQrm&q7a;T&%Hi>4$QE=hv#x#f%kvM5RKE+;dXjiKj|BsrgwBo&jUikBLikm*%Z9mXzFO{LdV z%tADos>m;sh?6KC$La)BpO7ll#7w(xl(52|b^tQeb@BywtirYN!E)ztg~j&J%d~y$ z?5Xn5uZ;Nk7pW3kR6FxZ^>`Bhxl8BiX&PO7zt`jP<+%A(Z{E3rO=?R=F+dNU9^emfTXb*qqe3`o w0_l``4itFn&=4gRmSLDLNbE~8_!a5<8=3fu9Q%@-{@VT%hH?FeU`-eKFIz31Jpcdz literal 0 HcmV?d00001 diff --git a/src/benchmark_core/repositories/__pycache__/provider_repository.cpython-312.pyc b/src/benchmark_core/repositories/__pycache__/provider_repository.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04a96f390dd48896403aaa8163017475f79eb346 GIT binary patch literal 3762 zcmds4O>7&-6`tkpk}Gm4O0-1%SfnUL4oTUdV;6N&Cow?Tacn11oH#}cCdHQGjv~rS zt}?rfLsdlz$7p3BKq84l93Xah7iwhRzMr@oox zlA@%vJ@nK8_~y-fZ{E(l@4cDf7kzDVye8zuw8(ftbLTy25938m%J-yu81L4+d0*Pcc#kIM{b@gkTH$1?--8tKv-|?O`%EmO$2n-6JpuG#=1+nMT^9=>d&T?PWSI zmSL8^J#lUp*JqmoUlD02xWkW7u)9 zq@`9!EjPhtkrcALyV$Te5~M5(cZq-nsoX80Bars)m)_;>w7&KWeuZ1?-*2(PJ1gT5SRC4Kai=Bv0I9B@ z!!Cd8D)YY*Mc6;s5AL_`csyVRe+4~_J{*Ui^cjDhOYqB~vjk^M9L+=v_E@qM5+^{lQbMv=^Cve@b`E*U0x%PVsk11m+MS>7!3t|}RG zHsK>Q$QBPqqz^bk<3WOy(0!8uO6ZO(KBKCZJFDmOIn(kGJWsH(kPs{Z3TU3t@Do3^ z6Z5oO42pLY+RiD1WRK6`LUtjaA=i|wPVjN!RBa9$$J<+KOH0Y65`2m+b9v-)D*-#X zo3vE{_=NFmAb&@Xe~JFq7k)T=`1Xa`*i)6Ur>esz>ir|-@LIS&G+rB;t_)4r!$)f2 zQUh|PA&M(0Y=i|3NFkhvcG9?n9d61r78>BQ>` zm|zwRD`;dhT81ck0V_ID3eYD6GD>Y(YSG-JCy-Ip%yw>(F)HnO8=J~p2`XZF?0{Tp zYa?K6yaI%!%U45@^80J=Ka_*NdA=%7)Z|!2j@|Xw;;BkJb^pyu{Bl)(t;xHFz26~M z(7WnxxKUrI7Km2@@oHdtTbgcoK)%zYwQ?inN*(4l1uiurY>I=xvpV`eg?Ml;#PsZ; zAd_*3MF(IKr9*b0OaRw~CzDQsbPt%m7F^Wja^(Brr1~Rt`hN!WbR~XzL#@PTs`AWLaw^a1_gA3V10QAW-`CNQBg%xGG>W(`2nvlq!2xU>X~KB7E!r5bP{Xj&Jh)QB8_sfS_;#>=DMgD2>Y!Cd14 zkln@k`*-i=tFh;5v6)J2=Kc?>v5VEAOSS$>tL}$^Lm!_n&#ui@2c~KR@ybBFIxt-g z9Nm_VHh2_#iQ6&g?Qr8nYJ&R%ml_xD9RsclPI`PYIPU?|f)gEdgy($TH_-#(WJBl{I>Sd@{xq8&_!+0qZ#F64@Otd;LbqWw5Onv=Z)r!Kc2+u@2#_DZz;c_VrCi}{dy{ncmQXC_ zR6_I9@*3}G87;ej^Cfn!TcM61iQhAm8Ty|or?Yrk!`;ukq&kI?=*K#PyxuoOR z^V*>eh6EedG%MzGD9}FiVO=+@*vo(c1Cpl}*o$QcP)f8&fUQ9H##IFxj28*F=h#?L#O$6^1 zBKY2S7^`K&Oa4?~x&N#nJ#fNWy+luFD{6(!igoNpe$wnj>;)U^$IyfR0>5g zXt=G%qzeJr{twl{uxo00YkZN?^Ib3CAim9m(@_6c}B@8 z3v%9gCW8j+LhZM}tP+uA8Q2nsz+|0*Lv#sFkriB`d&VuWS6o?-KIay_Fy|B9U2qQ| z9zgs$;#qxHGoY zl8UP>^}p5F)#bqkzh#GypMCZTt~@W&9PXo>Q9ZJ;ZK{)s7gaYEl?7VLJL|iFgBO0U z!moB8W={|Xc1K|lz)R>XfU78*`YVn()3FQ%oxWe!{{=W0Xt?3g~(WXfX4M#Eyq19l)(trz@bFCbCt zJRyMi0Eul!ENk0nSl0G~o!0GkV1)o6?Y2k|kWO191W30n5(cE#7Ks4TZ;M0$;cSr> zKz7<9F+h^GNF0z|wn!@=L$*j8Aj7suJ0NLWqyvzQ9rAPLN6r;yZfrZY;xtc%A)v8u zyT<$VsaJ{M{2@f%FCDASpRg=MCg|_quHUd|bBFU8GC_WpfS>nw&P68GwcK``ija4? z99I$-Imil_nlq}u&hZ67_1BXI@Tb*~cp35uWa*NkW0!KYkTdweJS|@;2qHxwyeZJ~ zf+DJH5!168o6XS@$rotQq!FH(4D^FB8`8;PldnDCEF(Ii=S)sHAIEpbz<47>> zXd4oYGunX!CE(etpCv&9^}0RMuDqtqiGh!sq0{kICPnyiCoE<4?Xda5G=+AFcF{-s#@+(f9++ zgSeNB?zOqshnSy?jsG$7hsaybo5$Zi^X8d5o#|WW9%uo?f}}T5?b*HFvs(+H947Jh z)zpoY7C|XWjviyGljqha&%GtTv#&aEs4{TqPU7%d_Js#p3u?qjA6M-iUGE*$;wZP0 z9my&;vCd6sZ78>s&YsnmZoH&*pwvmadf$%xDx!6v)NNutDD{%I&ean)PH257^^?TT zYX83V{(V{kWv*d&2g*B1cVD$DwceG|22f6tp8jh0$a?pP_87{$8v27M4-syln%KXd z*srBf9wx0FtA}qK)<#fDn@_SEq^qp9hxnqe1+E3IKY7R7eK*|pTIpJ8Bivv0_OE&S zQ7&IA-xN2($*MQG=1ty>v|bHrEb%2C-1R0O%IJ%)Onav@%!htvdf4@0pcDB4W;*Tq zFzG=)%>dP7VoUFVeZZ^mtNjUx(R)CS+w4AIvVJwX2L-)Xzsx*3=P_p*V?)cXi&$}7 zfEr-lZ;p^XndW>88QeJ+XxKD4Ge7H@wN=lmj^)_2sF=x=JHNy|{Y2(;%B_0D%VHj~ zA;o`cWCwO}R-~f77ivs~449*Qxg_$CMoKWIToh}K5wn;?uxXPQBxN&f>293$y(lXD z`6Wn{%RSrUa1+GpB%cDJhvNN?{?(UXfBBPO?2RKEL9QCyRSE8TccnTwSs9$XHC!1y zxe+{7b2>WxUlB*lf7K0v7;UdchANSvjmYqtcUbel{KFb1wpXScQ$5UW7ctpS=oN9ah;4B4l zdFYWxY|+I2Kt2ORKX1sBoeBFBKVBI;e(UAR;P*Cy&;1)GPSLPl&+O0lbAbQ%0$AA$ zUkGKaf|kn+yj*cU_MW z`!dvSN!k+ide>m%9O1fi)E8Mc->aMI?^caTSWW&ukbmV2j|%@UA^cmfKec-L_0t=z z$!hCRrFCedb$BB(vgREzQh{uyg1vew*aN8`J++sC9um7Z9$)WOuUHZcj+F|~q2$@7 zz22+=PeI=Que#9ljT-PD)DY(d=-N!@u&Wa*E)zA!Z54)9Hl1>X{W>R{!%ASezSco^ zHl-_>TP!HEg%Y&fVTp^2Vae$wwx#Caq~OBwq$%_-)jZ#O{S+*PeajNHsoOi4XsJt`a}E z5jwPWa<&pbuo0TjC$I1S>GL%oi#3hKn)YSz>zq0SIuGkkJ$7sdb9*0iEaAF6o(A4N z^{vB$epA1-`$l4V$(#=iTCS#k#^zhaH|S?J{jITp=|=*y!01V?y`JQDADmiKah@RH8?r{Ds0brwcvArFT94=C0?1APd&sFA+!3tg2ez8=&_ z!+TGA_dfz*m~QFQSQw$jZ0plJ$azV;NAn>E2N2g#3*OlLVdM4?iky#~&=9}xkL>u2^nFT3KOLzsu&n$Qu+vLtH zV0Tk0KowP^Hb_;LD6PszK3b7}wIB4Qf=Z%D{aM$Q?2L$l)I>`D+$>U3+K--dXU4`} zL#ULmDkJ&abI&>V-kE!z_xi6spNGKn#`>R9T!4^&V8?m{6IuT;Age?oaUyY&P2tj9 zoZ}F;E4H*fZcp=Zp7nV}NIT*Vh6{=_?TWh??oixmPu#w9D=?eW?CU-gIjo7=vYIsB(tw=VS#JYmmB=K{!8Y0?E^g1+ zB6huVcKL*oO2R9ebFjtobBDx~DodeL$K*^%)HFE_ubep=(z2l>l|ecrhE(%=Po=bD zA<3F1l#*}*7E>~fE<||Ul}Kd7w46xjo? zos8#^y)epG9o2GPh_^D1SN7L&d@!nwaa!4YwK0C+1Q@5SffE2uka6(saK@nAe%D)i zq;`<#$aO@z^nj`7n(^izuinTlwxYN$ccR9*th1x{3VnhMc*M05w0 zwPl(~nm&YEOu)CvIryyyf%-JzV81oXIEhx3T(K?jC|>_uF(vEd9Q88TYbu9^(C{>s zV{xcnu&JE4UIu%X9K3z&MlMj%`x+jg1{+Aw1!_lAYFATgPg81NQ|hLs)WN3I&_?Q; z+!en;eaXJUEsbuJSg~9Cz!5hAnemM>S4`45BH6wRT7T8PYX5;CfIWbGc%yu6&L$)5 zhsh4|@(}!7qePpqMvoDC=p`QJo3tpkwr+i}v{!jte+R)gknDHa7jJ#bNuv5BzBz~X zY7bj;O9JSwiS52ler~hoTH(*y&cc_!#D9-J#hoIj?cXG)xL?~(aS=!UM|-WvnvqXg z(KYG`nNJmAp;?`&_L1K-iqC}R7p-{`DJcXw>IBW6&Lxh^xrC^sL^UKTu%?s?$)_PP zskOx-e%)gQ*_5RF`aw2N0m+&>;jN1j7hM5TIw!F$8G;^dSUj*=%hZ0Zg}x66}ldIl7a|i&R#R zQ4CbNi)Iz&h?qRiGFe2xkg6=wZkzzUT^AsU!ai}biscbA4vHZI=41Ag?=iow7S*E$ zrxT>qCm@u5L_YgIx$Px^ft%fZm$v_6&ue>%-IF)_1};gZ!7YWsEycdAb)4Q!m%dRN zh!zH-#op!UFG)f)m;~NmAm^_&s;oH?(AKC;o=MB{-IKTw9p?d zclE7iE@mulBY9J(v}sFW)0Xn^cxia1Fg$a6s4d{RAXG+(yX_|}KWuqr?(eR?ip}E= zmV>>k(TmYyaHQlNS@VuyfBNF|KWZHI^6cK}eNOH#+uCO%&`;Bewb9?M>3EZ5^@s?vJ;(G+3zs=i0p4nPm z`ov^+4u0#u2VkU62yoQ|3Pkg%1uCr?jI8jpPZa%|*Ib(`PN3ge#~gn)W{dT6*Lg11%U>5d0oMfrRI z2!Lt4;vzjm&2_%F5Z-%j`GIv#(Uf#SNkOuz(=Y~X=EQFU@ou*N!nWmkn zKM>CMHuqm(&BIW2KY+Wv_l=WpD#h@wQaDx!$F7YR!~2Wf^QFN21;K6NQe6V#ns9e{#(=S+SGgQ`{XDi`}zZW5e8UxY!W?+bO_UrmxdzH7tJw(qE)T zznHMBRf96<{62bJGAn!di0X4+op+?uNhuOrw4az~_1NOd&jTb+HhhEqdqfj^ei=ig(=8GU_Ax zHWtyhKt!L4ZR03bhF{u7#Figyw2j)~!ASPaO@vj}_c0O1Z(ggO4#OVskeI|$zaFFQDOXv+XxUeG7 z*ou8cNK{V+D?Ic}dAM5Qn$`bSm~WhqUC2~R2J2N}7Qre4nv4(*lN<-vG5g^+0>2is zL>S+fg6r;7e#5o#BGPNgL!ghsO2&bjou#L+i&rdAG1eFRcL{9)zKWR`YksThcd@Bq z$$a0&XkhVDI8p{+Y|NUO9ug~@XFXvL20)HG{vxdX3E72HGX0ucWh zvu}iS0W+ zPh#u#W)gm(<5pmxB0vjgyw0AA6EPPF3|8ETc}RPIr3EoB@dVlH-}XV**gf31g-E-h zDcoM&q2*4%UnzzrP(^St_M##x$zyUl$4*3gSB(&rPis*T{~1bUnWpvDI%d+iHR`GV zwzUys3cRgnOeY2ZB3qvr-s94)o z>>S6vOUB+K1MidJ56QrLWcoc4`H*aVpFDcoVdLDF=5G^dKJ_eeT*oH_A={k)0#5J@ A6951J literal 0 HcmV?d00001 diff --git a/src/benchmark_core/repositories/__pycache__/task_card_repository.cpython-312.pyc b/src/benchmark_core/repositories/__pycache__/task_card_repository.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e6714c2cdf1524233889af640530e8efed536c8 GIT binary patch literal 3254 zcmds3O>7fK6rT0&+H1$LlaL<*Byqqrb`p#rR8)voN@&V|Dgl+KOSRf~2jZ~yhS^QQ zX%H2l3Z$xlR5ctq^w0`=Y7eMVCE!p+>Lr#dIjlgf)FPFeTO)xu^}Vq-PC{sV=&2+5 z&71e$j%VKYzPI~heSLtyu*%=3MIRx*;=&zLRVg0>WrpY^NpwN?7(&LA^kl@ODBxN& zq>P-Dxh@&rj4$cqx@;&Jf6~u&uMx=9CF=y@A-jq0yGV59rl+!dGMMwn15;gx=!9vd zZIk69$4wSFqFE<*X-tpMtev)ViN_cn_c+plv}HTW+Y@%$%xXqlaJ&{ZXvzjHIB4av zsl(K=zz6GkcawvrPBBz*4XMh1SEN#1lxEJ6CF(uy)#Zy) z(#LIHs({T;z3uP}czoaqaF4Rg;|EWjUeA328ic3n7oMuSC?`X?U_9hBxhJcoVJcSB zm!@Rbp?BVlm=ly~m;@`~_^isE9ls4}Q&W&P$IGZa$+9W2dOMu_@VfxN@-cys{X~G< z+Mrd2y%0I=8F!&~oiN|5MuOM7$Xr#y>gI8g6Rf5i>HOSIMud`QO%|iEs~|DBZx}`aY@8|PH+JwHLW{RR?ASw&!GO*3C$jJ zq!XrQI3de6C)AXg)zgTw(^kb;wbL0oYMxOu>FlIUEhmuTf26CLJrWNx%!?yhHe+GX z7$%Q3qG&>anPSZ-Fe8qr=};*tGm}Z%j*rpfj9O!H$&r9l$5|c5AyST!I_M1#tU!)r>V8McEbg8E-{gV|Kf+8YClrtcdf<~6&Kck^(lzo z$+M5hZw(y}+Sgrwx7aa|?-(ewZ<}viGt+skbH25!*c#8b#@+SWXmRa8e(k_~bW<@p zoR1FAw?&F=efhS&Ct;;Aa7ivT5r4y#y32J}hksJm%!gX8WG`n6q0XYxIj3~u+PrLD zr-e|os6^+K=zO^OQm`Zwf7i2lCHmZo!jaAod58Lidx|g=lkWLfquwnH^-A~F`%v!{ zK=QONVI8uCv+g@kDy)MFTi_iStE0w1dx3v$6^~cz-HpY$7PT_>PR44ib@zeV6&6=J z@JAd+pqCAo9lWlac};t!ns-yXR?3mlTx9Q?iIKS1@zFCh1=Yx8*rrl~^64?kD9?t| zWTiAiW2%{@s>#$W%yEgW!B}2nSgF{stVzROC^EJVt^O0#R!4I{l&N(q)Xajn6N0US zAb8pQ($+b1?%KHr!KTk%F9f@b!Jd4u=T?0&Hk6MI-7)g9LxtesvglbIcuG7?flJ}fiWhD<)%kKf(Z3NN|7S#Zi}yZxa(ga%8Sjt+88&E!UoX6Ku@2N)QD7Hl8&JFg!V%M1n~Au@>wHDZAPH|%48L-?zCOP3W?!2-qUHCGmeE!Uf%tMAcicW) z2|GHQ$qygn?mt4J+<)U(iNJJw1D|H4+4u8%-g|;3w#_}yNW7kHfraC>a}(+8iOTu$ zVotz1WoSlfjAnA2)lO^8kf3MmgodBfX_M#2S-tdipU_7Wm5+i-yKhQ*Tu0|C4~=Tl@t` C?9opE literal 0 HcmV?d00001 diff --git a/src/benchmark_core/repositories/__pycache__/variant_repository.cpython-312.pyc b/src/benchmark_core/repositories/__pycache__/variant_repository.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ba2d4e682c9c7b6b36536c5259ab986305b81bb GIT binary patch literal 4148 zcmds4O>7(25q`_vC6|AqL`fEH$s!#orYO;r-MUcR$U><&wUyc`U^hX)8eP!bm9@;0 zRNk&^lckZsF$xO}niw!DqnG-SLkqj7_8=4};OI~o=tT<#pt4Zl0BwNg#+EJ;=+t>{ zx%`vj_Rv!wzV3ORSmO?jv4$$3*=2074KB)M)QNqFe6c2D_= z?wF@EdQn~}Xo_CI#mG_tN8U(dC7stJa$Z;TV&Y3;9CMi5c}3Gr;fE`_Qpl&(7-Kp$ zS(P(7aK5Bg%x5ming%-Z+}PtnL6V83tMQz%{@9}M&UDm&0n<8?QHp{0bVy8!l^ilB zu`(}lvUABP@wd5@i^`m`05Z4iJP2z*;{uI`YJ@J08#G=iKy@D32dn(eRo>fts)uTP zbX7~^hgE@MPpsGMYaF@doKm#KcR1>mi?5%F6jo%MCa%#Erc1LtVY;qazBHX!)>m;p z!&U^xhgIx`KvNs z!Af2?JsJ9El9<*PV?IoLW3rl#aR4|>9Eke}2@*oQg$D>BUNPCUB$>`kA(vBh(}m?F zENja#&g38;m#~-Ek*JwGJ(xz&6M?b&9SvuuugLk#axRVE5;FxXPh&gx6I$*TUn5>Gz0|NuYmj=ef=)_Tkpuu(4qS?)!}2t@UhBJV!JQ2K5}Cwd)if3h zd08xAF%NT$!-t41g&F}Ysc+hwJy^ZC%| z`a5^u+3^MM%~pJos_&@bJNi+%8l5wubB`yD==qB8LY;MlJYOM4&~wLGbE4h@)xd-i zn5YCMw}i=>3*@_Xk~+7Nj^r@&F~=kiav$@9z|-9NKMn9eSAa>aA|b|Ma78P;I6@w2 zjX45bM>MJQ1(0@y(%Xm@*|K!_8-q#IFOt>&Ih-#U(U&&P7}4)le98YJoS0;=$*T$! zy^in^)YAO?Gcf9= z5Jy*c9HX$A$*H6-b6Huc!R4T;9T6o)c%O*6NJPQ7ba4M@f+`M6wR1q)6Z4~MkJL)^ z6dRn@1*3Q--Jrh5nEC28^w^nRv8;o+yO=y9k4wLA2 z4ZT@w^=4hQeY{_-Y*}TW)9hr)(Ywa;S7ylt-ZjdON6j{;c_y~Y2p+q!0m$Pz>OQA8 zTc|Skyk=>?#;AMuHstZ8>9!)Gk!zwA9=I_g`Z6x8UR{nfT3juG$=3w*_<{sDRPy9| z03QPpv^Azjds|JbnPoXw#M2<_0TIdozvGSRIO8&#`Xs|Tud#a6R?YR69|R+zXar-=(>oW`uo;?bT3{FMvY*! z;-7fx53ap_Z?@V$Vf0T_`{PD`eB<<1f4tJaQ1QR^)IYfP=Fd}gH&5;W``VA3ZD|e{i{mVA>d?(-ZE=W#e3`PDC}2kB@H;jn!B( zpPZrd=WhQU)o>3#<3WgL?GrD?{P+|+m`=U80{yr3n)LmT6B~ja^FRxx3HgSeNRzKw zr9k~>hB`augmfig_0?9ZG}xtX=&0AJ-(DtvL(fBdB4O5w+8j=Tg0ztuxrA#h!!Vzs e@u%p>7wGV3X!;9u;xiQc%K0rO{!aviZt-uazI*8a literal 0 HcmV?d00001 diff --git a/src/benchmark_core/repositories/base.py b/src/benchmark_core/repositories/base.py new file mode 100644 index 0000000..4e9d548 --- /dev/null +++ b/src/benchmark_core/repositories/base.py @@ -0,0 +1,50 @@ +"""Base repository with common database operations.""" +from typing import AsyncGenerator, Generic, List, Optional, Type, TypeVar +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.connection import async_session_factory + +ModelType = TypeVar("ModelType") + + +class BaseRepository(Generic[ModelType]): + """Base repository with common CRUD operations.""" + + def __init__(self, model: Type[ModelType]) -> None: + self.model = model + + async def _get_session(self) -> AsyncGenerator[AsyncSession, None]: + """Get database session.""" + if async_session_factory is None: + raise RuntimeError("Database not initialized. Call init_db() first.") + async with async_session_factory() as session: + yield session + + async def create(self, session: AsyncSession, obj: ModelType) -> ModelType: + """Create a new record.""" + session.add(obj) + await session.commit() + await session.refresh(obj) + return obj + + async def get_by_id(self, session: AsyncSession, id: str) -> Optional[ModelType]: + """Get record by ID.""" + result = await session.execute(select(self.model).where(self.model.id == id)) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[ModelType]: + """Get all records with limit.""" + result = await session.execute(select(self.model).limit(limit)) + return list(result.scalars().all()) + + async def delete(self, session: AsyncSession, id: str) -> bool: + """Delete record by ID.""" + obj = await self.get_by_id(session, id) + if obj: + await session.delete(obj) + await session.commit() + return True + return False diff --git a/src/benchmark_core/repositories/experiment_repository.py b/src/benchmark_core/repositories/experiment_repository.py new file mode 100644 index 0000000..793541b --- /dev/null +++ b/src/benchmark_core/repositories/experiment_repository.py @@ -0,0 +1,44 @@ +"""Repository for Experiment entity.""" +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import ExperimentModel +from benchmark_core.models import Experiment + + +class ExperimentRepository: + """Repository for Experiment CRUD operations.""" + + async def create(self, session: AsyncSession, experiment: Experiment) -> ExperimentModel: + """Create a new experiment.""" + model = ExperimentModel( + experiment_id=str(experiment.experiment_id), + name=experiment.name, + description=experiment.description, + created_at=experiment.created_at, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def get_by_id(self, session: AsyncSession, experiment_id: str) -> Optional[ExperimentModel]: + """Get experiment by ID.""" + result = await session.execute( + select(ExperimentModel).where(ExperimentModel.experiment_id == experiment_id) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, session: AsyncSession, name: str) -> Optional[ExperimentModel]: + """Get experiment by name.""" + result = await session.execute( + select(ExperimentModel).where(ExperimentModel.name == name) + ) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[ExperimentModel]: + """Get all experiments.""" + result = await session.execute(select(ExperimentModel).limit(limit)) + return list(result.scalars().all()) diff --git a/src/benchmark_core/repositories/harness_profile_repository.py b/src/benchmark_core/repositories/harness_profile_repository.py new file mode 100644 index 0000000..e63d78a --- /dev/null +++ b/src/benchmark_core/repositories/harness_profile_repository.py @@ -0,0 +1,47 @@ +"""Repository for HarnessProfile entity.""" +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import HarnessProfileModel +from benchmark_core.models import HarnessProfile + + +class HarnessProfileRepository: + """Repository for HarnessProfile CRUD operations.""" + + async def create(self, session: AsyncSession, profile: HarnessProfile) -> HarnessProfileModel: + """Create a new harness profile.""" + model = HarnessProfileModel( + harness_profile_id=str(profile.harness_profile_id), + name=profile.name, + protocol_surface=profile.protocol_surface, + base_url_env=profile.base_url_env, + api_key_env=profile.api_key_env, + model_env=profile.model_env, + created_at=profile.created_at, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def get_by_id(self, session: AsyncSession, profile_id: str) -> Optional[HarnessProfileModel]: + """Get harness profile by ID.""" + result = await session.execute( + select(HarnessProfileModel).where(HarnessProfileModel.harness_profile_id == profile_id) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, session: AsyncSession, name: str) -> Optional[HarnessProfileModel]: + """Get harness profile by name.""" + result = await session.execute( + select(HarnessProfileModel).where(HarnessProfileModel.name == name) + ) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[HarnessProfileModel]: + """Get all harness profiles.""" + result = await session.execute(select(HarnessProfileModel).limit(limit)) + return list(result.scalars().all()) diff --git a/src/benchmark_core/repositories/metric_rollup_repository.py b/src/benchmark_core/repositories/metric_rollup_repository.py new file mode 100644 index 0000000..741aadb --- /dev/null +++ b/src/benchmark_core/repositories/metric_rollup_repository.py @@ -0,0 +1,100 @@ +"""Repository for MetricRollup entity.""" +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import MetricRollupModel +from benchmark_core.models import MetricRollup, RollupScopeType + + +class MetricRollupRepository: + """Repository for MetricRollup CRUD operations.""" + + async def create( + self, session: AsyncSession, rollup: MetricRollup + ) -> MetricRollupModel: + """Create a new metric rollup.""" + model = MetricRollupModel( + rollup_id=str(rollup.rollup_id), + scope_type=rollup.scope_type, + scope_id=str(rollup.scope_id), + metric_name=rollup.metric_name, + metric_value=rollup.metric_value, + computed_at=rollup.computed_at, + window_start=rollup.window_start, + window_end=rollup.window_end, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def upsert( + self, session: AsyncSession, rollup: MetricRollup + ) -> MetricRollupModel: + """Create or update metric rollup.""" + existing = await self.get_by_scope_and_name( + session, rollup.scope_type, str(rollup.scope_id), rollup.metric_name + ) + if existing: + existing.metric_value = rollup.metric_value + existing.computed_at = rollup.computed_at + existing.window_start = rollup.window_start + existing.window_end = rollup.window_end + await session.commit() + await session.refresh(existing) + return existing + return await self.create(session, rollup) + + async def get_by_scope_and_name( + self, + session: AsyncSession, + scope_type: RollupScopeType, + scope_id: str, + metric_name: str, + ) -> Optional[MetricRollupModel]: + """Get rollup by scope and metric name.""" + result = await session.execute( + select(MetricRollupModel).where( + MetricRollupModel.scope_type == scope_type, + MetricRollupModel.scope_id == scope_id, + MetricRollupModel.metric_name == metric_name, + ) + ) + return result.scalar_one_or_none() + + async def get_by_scope( + self, + session: AsyncSession, + scope_type: RollupScopeType, + scope_id: str, + ) -> List[MetricRollupModel]: + """Get all rollups for a scope.""" + result = await session.execute( + select(MetricRollupModel).where( + MetricRollupModel.scope_type == scope_type, + MetricRollupModel.scope_id == scope_id, + ) + ) + return list(result.scalars().all()) + + async def get_by_session( + self, session: AsyncSession, session_id: str + ) -> List[MetricRollupModel]: + """Get rollups for a session.""" + return await self.get_by_scope(session, RollupScopeType.SESSION, session_id) + + async def get_by_variant( + self, session: AsyncSession, variant_id: str + ) -> List[MetricRollupModel]: + """Get rollups for a variant.""" + return await self.get_by_scope(session, RollupScopeType.VARIANT, variant_id) + + async def get_by_experiment( + self, session: AsyncSession, experiment_id: str + ) -> List[MetricRollupModel]: + """Get rollups for an experiment.""" + return await self.get_by_scope( + session, RollupScopeType.EXPERIMENT, experiment_id + ) diff --git a/src/benchmark_core/repositories/provider_repository.py b/src/benchmark_core/repositories/provider_repository.py new file mode 100644 index 0000000..a384655 --- /dev/null +++ b/src/benchmark_core/repositories/provider_repository.py @@ -0,0 +1,56 @@ +"""Repository for Provider entity.""" +from typing import List, Optional +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import ProviderModel +from benchmark_core.models import Provider + + +class ProviderRepository: + """Repository for Provider CRUD operations.""" + + async def create(self, session: AsyncSession, provider: Provider) -> ProviderModel: + """Create a new provider.""" + model = ProviderModel( + provider_id=str(provider.provider_id), + name=provider.name, + route_name=provider.route_name, + protocol_surface=provider.protocol_surface, + upstream_base_url=provider.upstream_base_url, + created_at=provider.created_at, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def get_by_id(self, session: AsyncSession, provider_id: str) -> Optional[ProviderModel]: + """Get provider by ID.""" + result = await session.execute( + select(ProviderModel).where(ProviderModel.provider_id == provider_id) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, session: AsyncSession, name: str) -> Optional[ProviderModel]: + """Get provider by name.""" + result = await session.execute( + select(ProviderModel).where(ProviderModel.name == name) + ) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[ProviderModel]: + """Get all providers.""" + result = await session.execute(select(ProviderModel).limit(limit)) + return list(result.scalars().all()) + + async def delete(self, session: AsyncSession, provider_id: str) -> bool: + """Delete provider by ID.""" + model = await self.get_by_id(session, provider_id) + if model: + await session.delete(model) + await session.commit() + return True + return False diff --git a/src/benchmark_core/repositories/request_repository.py b/src/benchmark_core/repositories/request_repository.py new file mode 100644 index 0000000..a04cd15 --- /dev/null +++ b/src/benchmark_core/repositories/request_repository.py @@ -0,0 +1,96 @@ +"""Repository for Request entity.""" +from datetime import datetime +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import RequestModel +from benchmark_core.models import Request, RequestStatus + + +class RequestRepository: + """Repository for Request CRUD operations.""" + + async def create(self, session: AsyncSession, req: Request) -> RequestModel: + """Create a new request.""" + model = RequestModel( + request_id=str(req.request_id), + session_id=str(req.session_id) if req.session_id else None, + experiment_id=str(req.experiment_id) if req.experiment_id else None, + variant_id=str(req.variant_id) if req.variant_id else None, + provider_id=str(req.provider_id) if req.provider_id else None, + provider_route=req.provider_route, + model=req.model, + harness_profile_id=str(req.harness_profile_id) if req.harness_profile_id else None, + litellm_call_id=req.litellm_call_id, + provider_request_id=req.provider_request_id, + started_at=req.started_at, + finished_at=req.finished_at, + latency_ms=req.latency_ms, + ttft_ms=req.ttft_ms, + proxy_overhead_ms=req.proxy_overhead_ms, + provider_latency_ms=req.provider_latency_ms, + input_tokens=req.input_tokens, + output_tokens=req.output_tokens, + cached_input_tokens=req.cached_input_tokens, + cache_write_tokens=req.cache_write_tokens, + status=req.status, + error_code=req.error_code, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def get_by_id(self, session: AsyncSession, request_id: str) -> Optional[RequestModel]: + """Get request by ID.""" + result = await session.execute( + select(RequestModel).where(RequestModel.request_id == request_id) + ) + return result.scalar_one_or_none() + + async def get_by_litellm_call_id( + self, session: AsyncSession, litellm_call_id: str + ) -> Optional[RequestModel]: + """Get request by LiteLLM call ID.""" + result = await session.execute( + select(RequestModel).where(RequestModel.litellm_call_id == litellm_call_id) + ) + return result.scalar_one_or_none() + + async def get_by_session( + self, session: AsyncSession, session_id: str + ) -> List[RequestModel]: + """Get all requests for a session.""" + result = await session.execute( + select(RequestModel).where(RequestModel.session_id == session_id) + ) + return list(result.scalars().all()) + + async def get_by_time_window( + self, + session: AsyncSession, + start_time: datetime, + end_time: datetime, + limit: int = 1000, + ) -> List[RequestModel]: + """Get requests within a time window.""" + result = await session.execute( + select(RequestModel) + .where(RequestModel.started_at >= start_time) + .where(RequestModel.started_at <= end_time) + .limit(limit) + ) + return list(result.scalars().all()) + + async def exists_by_litellm_call_id( + self, session: AsyncSession, litellm_call_id: str + ) -> bool: + """Check if request with LiteLLM call ID exists.""" + result = await session.execute( + select(RequestModel.request_id).where( + RequestModel.litellm_call_id == litellm_call_id + ) + ) + return result.scalar_one_or_none() is not None diff --git a/src/benchmark_core/repositories/session_repository.py b/src/benchmark_core/repositories/session_repository.py new file mode 100644 index 0000000..ac5feec --- /dev/null +++ b/src/benchmark_core/repositories/session_repository.py @@ -0,0 +1,120 @@ +"""Repository for Session entity.""" +from datetime import datetime +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.exc import IntegrityError + +from benchmark_core.db.models import SessionModel +from benchmark_core.models import Session, SessionStatus + + +class DuplicateSessionError(Exception): + """Raised when attempting to create a session with duplicate identifier.""" + pass + + +class SessionRepository: + """Repository for Session CRUD operations with duplicate rejection.""" + + async def create(self, session: AsyncSession, sess: Session) -> SessionModel: + """Create a new session. + + Raises: + DuplicateSessionError: If session_id or proxy_key_alias already exists. + """ + model = SessionModel( + session_id=str(sess.session_id), + experiment_id=str(sess.experiment_id), + variant_id=str(sess.variant_id), + task_card_id=str(sess.task_card_id), + harness_profile_id=str(sess.harness_profile_id), + status=sess.status, + started_at=sess.started_at, + ended_at=sess.ended_at, + operator_label=sess.operator_label, + repo_root=sess.repo_root, + git_branch=sess.git_branch, + git_commit_sha=sess.git_commit_sha, + git_dirty=sess.git_dirty, + proxy_key_alias=sess.proxy_key_alias, + proxy_virtual_key_id=sess.proxy_virtual_key_id, + ) + session.add(model) + try: + await session.commit() + await session.refresh(model) + return model + except IntegrityError as e: + await session.rollback() + if "session_id" in str(e) or "proxy_key_alias" in str(e): + raise DuplicateSessionError( + f"Session with identifier already exists: {sess.session_id}" + ) from e + raise + + async def get_by_id(self, session: AsyncSession, session_id: str) -> Optional[SessionModel]: + """Get session by ID.""" + result = await session.execute( + select(SessionModel).where(SessionModel.session_id == session_id) + ) + return result.scalar_one_or_none() + + async def get_by_proxy_key_alias( + self, session: AsyncSession, alias: str + ) -> Optional[SessionModel]: + """Get session by proxy key alias.""" + result = await session.execute( + select(SessionModel).where(SessionModel.proxy_key_alias == alias) + ) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[SessionModel]: + """Get all sessions.""" + result = await session.execute(select(SessionModel).limit(limit)) + return list(result.scalars().all()) + + async def get_by_status( + self, session: AsyncSession, status: SessionStatus + ) -> List[SessionModel]: + """Get sessions by status.""" + result = await session.execute( + select(SessionModel).where(SessionModel.status == status) + ) + return list(result.scalars().all()) + + async def get_by_experiment( + self, session: AsyncSession, experiment_id: str + ) -> List[SessionModel]: + """Get sessions by experiment ID.""" + result = await session.execute( + select(SessionModel).where(SessionModel.experiment_id == experiment_id) + ) + return list(result.scalars().all()) + + async def get_by_variant( + self, session: AsyncSession, variant_id: str + ) -> List[SessionModel]: + """Get sessions by variant ID.""" + result = await session.execute( + select(SessionModel).where(SessionModel.variant_id == variant_id) + ) + return list(result.scalars().all()) + + async def finalize( + self, + session: AsyncSession, + session_id: str, + status: SessionStatus, + ended_at: datetime, + ) -> Optional[SessionModel]: + """Finalize a session with final status.""" + model = await self.get_by_id(session, session_id) + if model: + model.status = status + model.ended_at = ended_at + await session.commit() + await session.refresh(model) + return model + return None diff --git a/src/benchmark_core/repositories/task_card_repository.py b/src/benchmark_core/repositories/task_card_repository.py new file mode 100644 index 0000000..36802d5 --- /dev/null +++ b/src/benchmark_core/repositories/task_card_repository.py @@ -0,0 +1,47 @@ +"""Repository for TaskCard entity.""" +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import TaskCardModel +from benchmark_core.models import TaskCard + + +class TaskCardRepository: + """Repository for TaskCard CRUD operations.""" + + async def create(self, session: AsyncSession, task_card: TaskCard) -> TaskCardModel: + """Create a new task card.""" + model = TaskCardModel( + task_card_id=str(task_card.task_card_id), + name=task_card.name, + repo_path=task_card.repo_path, + goal=task_card.goal, + stop_condition=task_card.stop_condition, + session_timebox_minutes=task_card.session_timebox_minutes, + created_at=task_card.created_at, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def get_by_id(self, session: AsyncSession, task_card_id: str) -> Optional[TaskCardModel]: + """Get task card by ID.""" + result = await session.execute( + select(TaskCardModel).where(TaskCardModel.task_card_id == task_card_id) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, session: AsyncSession, name: str) -> Optional[TaskCardModel]: + """Get task card by name.""" + result = await session.execute( + select(TaskCardModel).where(TaskCardModel.name == name) + ) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[TaskCardModel]: + """Get all task cards.""" + result = await session.execute(select(TaskCardModel).limit(limit)) + return list(result.scalars().all()) diff --git a/src/benchmark_core/repositories/variant_repository.py b/src/benchmark_core/repositories/variant_repository.py new file mode 100644 index 0000000..d45963a --- /dev/null +++ b/src/benchmark_core/repositories/variant_repository.py @@ -0,0 +1,61 @@ +"""Repository for Variant entity.""" +from typing import List, Optional + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import VariantModel +from benchmark_core.models import Variant + + +class VariantRepository: + """Repository for Variant CRUD operations.""" + + async def create(self, session: AsyncSession, variant: Variant) -> VariantModel: + """Create a new variant.""" + model = VariantModel( + variant_id=str(variant.variant_id), + name=variant.name, + provider_id=str(variant.provider_id), + model_alias=variant.model_alias, + harness_profile_id=str(variant.harness_profile_id), + config_fingerprint=variant.config_fingerprint, + created_at=variant.created_at, + ) + session.add(model) + await session.commit() + await session.refresh(model) + return model + + async def get_by_id(self, session: AsyncSession, variant_id: str) -> Optional[VariantModel]: + """Get variant by ID.""" + result = await session.execute( + select(VariantModel).where(VariantModel.variant_id == variant_id) + ) + return result.scalar_one_or_none() + + async def get_by_name(self, session: AsyncSession, name: str) -> Optional[VariantModel]: + """Get variant by name.""" + result = await session.execute( + select(VariantModel).where(VariantModel.name == name) + ) + return result.scalar_one_or_none() + + async def get_all(self, session: AsyncSession, limit: int = 100) -> List[VariantModel]: + """Get all variants.""" + result = await session.execute(select(VariantModel).limit(limit)) + return list(result.scalars().all()) + + async def get_by_experiment( + self, session: AsyncSession, experiment_id: str + ) -> List[VariantModel]: + """Get variants by experiment ID through sessions.""" + from sqlalchemy import distinct + + result = await session.execute( + select(VariantModel) + .join(VariantModel.sessions) + .where(VariantModel.sessions.any(experiment_id=experiment_id)) + .distinct() + ) + return list(result.scalars().all()) diff --git a/src/benchmark_core/services/__init__.py b/src/benchmark_core/services/__init__.py new file mode 100644 index 0000000..0c2e512 --- /dev/null +++ b/src/benchmark_core/services/__init__.py @@ -0,0 +1,4 @@ +"""Service layer for benchmark operations.""" +from benchmark_core.services.session_service import SessionService + +__all__ = ["SessionService"] diff --git a/src/benchmark_core/services/__pycache__/__init__.cpython-312.pyc b/src/benchmark_core/services/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..362171939ce8617a36b0c67dc0cca8ca1e03ed62 GIT binary patch literal 302 zcmX@j%ge<81QWjO&Flx#k3k$5V1hC}O92_v8B!Rc7*ZHhm~t3%nWC5&8B&<_QQ;QVR@{1IbQuC5CaubWP74i#GixNvR^Ye=JUV=>2WW2=} zoLXEA6oG0hVg?HPX|mj6kB?8uPmYhjr4H8~pPXNms#gp#zZl2>YmA2qfOUejCFbPB z$FF4g46@*tdT~**KB@`&Fcb9S<1_OzOXB183My}L*yQG?l;)(`6@fy55r~V0fy4)9 aMn=Y)47zt2oE~sXU*M8yWG~_XiUR;O0a&sC literal 0 HcmV?d00001 diff --git a/src/benchmark_core/services/__pycache__/session_service.cpython-312.pyc b/src/benchmark_core/services/__pycache__/session_service.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ff047e9ecd1b779ad2bc29c2384e57f59c70c47 GIT binary patch literal 5896 zcmdT|O>7&-6`oy^;y?9AkrpY5W6QF{SfS)Nj#DCy!&r{(RF>T+P73xwthp;`8U84< zOWRZ#BuI?H1p>6b)HYfm>Zu8w0`8$dmjX4~TQ4Srg3Lk&0<=Zj8&erb;Zxt+AO4vR z(nAj&g0pYl&+P2H_rA9~fAaf11loy*zmwk#67m-uSj}z{dwC#k5|N~d$ccQOD{yI! zL)w<-3%0aPr|o%r!I5_8v?D)IaHgF)J&<=5+-Y~gllJIiXWm=zrF{i|+RqW5oF<~{ z8WG)hc;jyAK*baHZbeolMUhKInwQt4Y$cnQs3Nh;a#l+G6BXl}<`M-}Qssh#^!ZH{ zw1qs-_KO$KF2U88QYyu)IZvF|oF=LHO=49Q)UpDSv8D26Ue3aFrsNc3C8iCVqt~R( zk|L`mR>>M`H2SDH*t-wJO(Kyr2V2C8T-qk`l6{RAZP)B+hnsZ%#IWd)2E+mQJK^tw zzZ?D@$tk(UVII-jqI@mN-=YFQxplq47H_CU4NIQ2h&Xc1k@mWYxKF>+sN@rapyAi8 zhJo@RwdNAXKrf){w7GUM1Zy6w*yH0`z}hBb-?!qc20_x7cNP~>_$OgZ)LJ2CM}pDqyV8WKV!wzW<_&lGICK?GnuXMsv$L> zH(?1zU}6Q5guF)@;i>EP4}H56Pwz}DT(|$(HNFRYvZ8scnPpM)OIKk}AV7<128N!? z0+R)U_NjsbYZ91U-tkvSPRFLU4N|$zb($j3G_5*yt9}bpJ|H6hJqWNL^EYkZcR1jA zs{U1SoP0R&3)>YgZWAOBo@5dL02OFaxdo^CZ1kl;18R zSQ+^ti{O2EO_o@fMOS1b>8VzrP<2CATGrtBQgL0OYN^Y*9Y{&~h7nXWTPm)}>t!Pa z^%U)#smr`G_LB5hTy&(lbWfQZsI;T%=`1>N4vN z<0Q3!np%RV!T2(4lSuVI?=o)1e3G_7m|_ypO+nr0(rqV2lAbIx2K!EdM~y^zOtqae z-RcIPuAWIfW}>H6lD@ei!3Hski=!KYLd!+NNQGy}ZV1J7N$Ilf$-^W~VLF;Vl5q2` zRw$%tAEWMEP10pqm6n&!QIn5W^z2eQHeb_oxhFi0$QD~sC6)>Tc&bnmrF@5}AVbi0 z2X^1On51XdT60~I)eWi$1qqfxp)T}1E0wOwirU_ecz}&TjcA9|WA2tOt~m=}iCLJJr-we+?oSG!9~w-|Rt zxeDDs$iLqs4`1a-c&ZWJ--yg&KR(ll9oZeF_0hT7=-h5NcJtjE@9u=>8qs(o6hjSz zm^j?nf9&DpVAylr+dM!9N9%!DEfA{*4%GsOw$-0ke^$Nw-JL+H?n>Qvr5b~g`>sgS zM%*uRj}%<=`>E+vg1a}vrRMGT4mp8l)A%edsR&sLUdeWXJ?`TN@ZZZrF)&I+)%ISn zT`qF1A^}=vv%voy8$fsGY^FTdDjk*uA)wi(jHq5qqsrYeOB9x;JLaqa{ah#Xzhkf3 z*#oszMAX5Sy*-C<($lOtyYz6EdRkvSd{^jcef7}qJ*}@E=1Nbq^s4q8=1NZkF6K>7 z11{!JPXjLPVZji%2l{dk_T?Vx%bn}XeX1{a{&DW7{2kLvtE9p~+5W3pwx0rrzaqn1 zzr2No+p4)bCP66^^3;HouU)*&?1Xwtb$giC)0x)BbV;h;)SR5oojiZ;!t$xtPc6;U z)XV2z(+PmWvny|;mf=gJYX&Qapyn4vieE1RG&<~e?u>8~MTe+VseEi(q=l5)U*z+af zhdtLFUkni6@a|}2HyYb^)ahc4F78B+>`oonK31Q8t~UML&eZW9+2sE1nfifeYX_d) znLM^TJ^QhvKKD{>?xmgSR3kibvwEZ2h|+rWa4mYcF+TOt)a|K8>|i~1v=%$sh{Qfx zxV_Mp_X_AtogS;vV~v@o>N6*5GbbL#hJt}68S)39R15{1hhf!^lu>y6_fy`bXSsWR zZs{5Oy}-#C7<7HkwSfPHHt>@i81}ZDt(T*Nc^}wse=)SdyyG3d8_4-84?ym==I$s2 ze9mWa+gA%HyY%{eHT2c{s>0ANKz#123PY#fR}_XSpX*Z&fZGTBf!h!C`M#jf_XT~v zFX;1qL7&g{1ARW%5A^w5AJBgikZ+Bgmedw}U&2?~#0tYo5FazJ#l0@UCvOEljEysZ z1jzn3;9~0lmj$?sg3$|rV!(+P*J0L~))|#`PS7E%pye)fI*4=t#*td+$lZg#ng8Yd zA8o&X^$&wPp$qk)3)dZ=d58XwP-2!0e;eKwF)iLdx-`N4id%}^k2dYRV{zNJN8r*7k@(Y% z>Dk7?xyIZ)_ApF5^F{QyBl>x8U(*5C<^b6@(R3o^B4guCH&PxFjyAnW`N-t{rXQ&Q z8Q;G>dfVR|#L*B5PTV?qdS;HEOU6O=|sv!Mx(c$uZUOV>6L!^zJ!%bU>53=L9c^LQFfLhsv z69I;f(Cp}lFKj{A(z2W0XA ei9R4>e Session: + """Create a new benchmark session. + + Args: + session_id: Unique session identifier + experiment_id: Experiment this session belongs to + variant_id: Variant configuration for this session + task_card_id: Task card defining the work + harness_profile_id: Harness profile in use + operator_label: Optional operator-provided label + repo_root: Repository root path + git_branch: Current git branch + git_commit_sha: Current git commit SHA + git_dirty: Whether repo has uncommitted changes + proxy_key_alias: Alias for the proxy key + proxy_virtual_key_id: LiteLLM virtual key ID + + Returns: + Created session domain model + + Raises: + DuplicateSessionError: If session with same ID or proxy_key_alias exists + """ + new_session = Session( + session_id=session_id, + experiment_id=experiment_id, + variant_id=variant_id, + task_card_id=task_card_id, + harness_profile_id=harness_profile_id, + status=SessionStatus.PENDING, + started_at=datetime.utcnow(), + operator_label=operator_label, + repo_root=repo_root, + git_branch=git_branch, + git_commit_sha=git_commit_sha, + git_dirty=git_dirty, + proxy_key_alias=proxy_key_alias, + proxy_virtual_key_id=proxy_virtual_key_id, + ) + + await self.session_repo.create(self.session, new_session) + return new_session + + async def finalize_session( + self, + session_id: UUID, + status: SessionStatus, + ended_at: Optional[datetime] = None, + ) -> Optional[Session]: + """Finalize a session with final status. + + Args: + session_id: Session to finalize + status: Final status (COMPLETED, ABORTED, or INVALID) + ended_at: End timestamp (defaults to now) + + Returns: + Updated session model or None if not found + """ + if ended_at is None: + ended_at = datetime.utcnow() + + model = await self.session_repo.finalize( + self.session, str(session_id), status, ended_at + ) + if model: + return Session( + session_id=UUID(model.session_id), + experiment_id=UUID(model.experiment_id), + variant_id=UUID(model.variant_id), + task_card_id=UUID(model.task_card_id), + harness_profile_id=UUID(model.harness_profile_id), + status=model.status, + started_at=model.started_at, + ended_at=model.ended_at, + operator_label=model.operator_label, + repo_root=model.repo_root, + git_branch=model.git_branch, + git_commit_sha=model.git_commit_sha, + git_dirty=model.git_dirty, + proxy_key_alias=model.proxy_key_alias, + proxy_virtual_key_id=model.proxy_virtual_key_id, + ) + return None + + async def get_session(self, session_id: UUID) -> Optional[Session]: + """Get session by ID.""" + model = await self.session_repo.get_by_id(self.session, str(session_id)) + if model: + return Session( + session_id=UUID(model.session_id), + experiment_id=UUID(model.experiment_id), + variant_id=UUID(model.variant_id), + task_card_id=UUID(model.task_card_id), + harness_profile_id=UUID(model.harness_profile_id), + status=model.status, + started_at=model.started_at, + ended_at=model.ended_at, + operator_label=model.operator_label, + repo_root=model.repo_root, + git_branch=model.git_branch, + git_commit_sha=model.git_commit_sha, + git_dirty=model.git_dirty, + proxy_key_alias=model.proxy_key_alias, + proxy_virtual_key_id=model.proxy_virtual_key_id, + ) + return None diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 0000000..b1e1432 --- /dev/null +++ b/src/cli/__init__.py @@ -0,0 +1 @@ +"""CLI commands for benchmark operations.""" diff --git a/src/cli/__pycache__/__init__.cpython-312.pyc b/src/cli/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..71b1f2c0b84ceb7fc407a141e5dc78b8b558d096 GIT binary patch literal 169 zcmX@j%ge<81TmlXX6gdz#~=<2FhUuhIe?7m3@Hpz43&(UOjT;mKAsB6`MJ4?c`3yT zY57G8NvV0s8M%o?*$VjusYQt;nfZCedVZRWx7g$3Q}UDJ<5x0#1{wBCxVR`;KRG8; vKR!M)FS8^*Uaz3?7Kcr4eoARhs$CHqPz}h|Vvvm=m>C%vZ!zc;u>d&$i25rz literal 0 HcmV?d00001 diff --git a/src/collectors/__init__.py b/src/collectors/__init__.py new file mode 100644 index 0000000..9997d55 --- /dev/null +++ b/src/collectors/__init__.py @@ -0,0 +1,19 @@ +"""Collectors for LiteLLM and Prometheus data.""" +from collectors.litellm_collector import ( + LiteLLMCollector, + MissingFieldError, + UnmappedRowError, +) +from collectors.normalizer import NormalizationDiagnostics, RequestNormalizer +from collectors.prometheus_collector import PrometheusCollector +from collectors.rollups import MetricRollupService + +__all__ = [ + "LiteLLMCollector", + "MissingFieldError", + "UnmappedRowError", + "NormalizationDiagnostics", + "RequestNormalizer", + "PrometheusCollector", + "MetricRollupService", +] diff --git a/src/collectors/__pycache__/__init__.cpython-312.pyc b/src/collectors/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d41a5caec29a99f9be3f55874c71f7abe65422b1 GIT binary patch literal 620 zcmZ8dF>ezw6t?d!xl2N%h6)LxO1*{QKuQHOf}y~aL@20>7wdF3!IFJGo*huT@ejIn z@5Ybd7w`+P)QO1=C{iUR>{EM*`Xqn$`~3O6@A=z!JVZWD|9%yZB7}Z~ztfPXQeHtKZA=Rc01Ri?x~y1g)yRDy%t=u zR{$E^&QYz(rr~UB z!IDv8jDV*uO2)`BlE5%3s*6!$RE!>@WBM_D%rIt9;B?d&GmV*)wmxc(S;j0%+Yq(J zY-6@q#aM;%Zj9PvjxmSQHbpCARby2IA`wSS?$Rc+@qCzXj=4y50-_ zSd9X;0@PLnbt}*cfZ89Dc}&kN>37T>8H|YB;NTgC=iZs-1d$0RqERj^CU|BX{v+`T z07A|1p_u{>&xI2_D+E5rq=x*2WMo4kCq`nNpOSPv@mY!Li-bjq8jJ{{WIQ`1MiTK* z6cDM=(SbgAT6=`qczA>p1nBXTl3`f=ykyHaM#PXfEdXX^9x%*JC4>mfYZe3ZBR7Cy zbyaFj6i<(UY4I6iSa!bRDBo=@;|FAawx0>?5l7r8hS zQdSz_;e8yVFcE=?sY94?J`qzk1C|^L3_YMEV=x#G#kgQlG6#dP1UntYZEG<2&U7f6 z?=b{}Y$6;CK0xbuZeKSGe7K{q)j~%UP9YkN1q-0S)GTiYOr@%XhG2y1ee@~SouiC= zjn?&lTDM~~9t%xPaqMtnMqT#4l4U1mm~bc_Pl(I~j!|(LF##JQs7DnT;_;sH)woh7 z;l?emR$)+Krd@@Z;{HuD#f!mZy#~%|^9fBLmi%L4jDn{Qo-|8w`f-w_->1e5te!J+ zCeA!gVv4MRHI7rP33@E0Jqn(cGhsXx8!*j)X)__#1tNp#Z0cvN+B<8|TH|`w_C7sU z!B%Jx=(7VQJ5X|L`KFbtRK7>C4(P99D>-NJeAPK~E*M`0{nbEgOFyjC34JxQMt`;B z$O~t$CX;h}3(XSoEEQvgJAu{h1Fp;uDu6EXU&xLw-y+uTb!Dg4RMv2mkADL3JA?i6ja_OD!Vq zQHrL-ptmquG?eM3G}r_JTumjTS@eJy^6U9#=;p0ZN!Gmn3i6;Qc$)&!7KK5Qg^h$J z;t5b@VF45lC2-MkWxRlAz%V>!WGu^(cti{a=e(tRAJDpX0HV+e)m4;pH!M)st&i*W zuGMucP#L4QsF~~Bzd-%Q=uzZEGJ*mP;!~0IJSR@`@u5u{hZmu2+yzA-CeS>YM3Vq4 zEq&;1uvGK9q%KKZCImffu?{oV_lfJaBsp2w;*D`h-NU>>B+&In3MEN?mL*x;WMMmt za7hjK1N2AaJOy*mZ}e)da$jMoBq^FSj3hB>(b^kN?bx+%fbFDxs1MzsZ|K9i2nv&L zyAXN{4rHF5*C%NapQBQe`l2y*_cwQIm;CoNohq5*5W3l?H1O6C|A=@``=%8+bqT@_R6fAwL(1R=z2jIBej8 zF=0X?E=j~?0m~Vq7M1C}!OQur0O@&W8rwZ}A;R45tx~WZE|v)4uc5k%o>aLO47qA= zw%VVm_Am73?5@R=X?x4Eai#i!J>7OR=WySyxK*(@{Ts(lnMO6cpBjHEdN zM+2)9YcC#8?LU$7oLm^lSshtxUD{fgwKk`%%{fnN&bQ-nci+cv{LObBeP^wIbglcf z)a&O{Z=6qek7b(Qg!(V3RbsVsspn6G<<6g8US^ktrR4pysqV4AZ+wVA&*@)E$A(UEj$Ph4vm0-?9{0GJ-VMZl2!Ir65F_tu6HP zd$(0B4a%J1%XyVz0yRTlMgnoYqG*n^+OLURVRj5wE+fdVn``aABJ!ayb|-nBi-xcR z$Xw!Pg;M_sHhO0l5T<%`7YQ2$s7T14GdO1%t;gr zo+YUM4Th&MT}cPEL<4h9bNn$Fq@3VpF6Y!afs<93QxK0W$K~7M9>S#m04m^ed*uR^ zbGoz6wv4lFp*QEMyFGYoaLKvm+Ntn-%FaBlYh0o~ICYb{Ir1aZlR8Fi$QI=C%MRr5 z+(=$aE*@Vxwq|c#pdMSQZ?bp$GL{`VcWu_)o^iL|)WHW9b+-+*fAtVuK**At79BCMa0V58!`& z6{^#SQ0(?(-edr%P*_4cah^^>5QKST1e=p1z)uA`UO-1>4NO^sk|Z_W{Q-D?--X%z zA$eQ(yELsBI^LSpO%}L#gZ>DbB@d$O&Og)55WsVXhdCh;z07I8s>*6bd!aHQL^aB8 z;5SCujR4;WM2Gw|-v>_~D>A7Dm@jx?HdwN8s>gxR{gldrr}2-3T(V{?dvng_rNOkbGiB|3 zsz=t^T+^;>)BbeR{%q5!bknJv!*$zw%et6Y*OAQ?Um~);V&M!RI_h)omaMxY?e0i9 zI-b~_vJJYv7OLPo@AZ&9KH?EY^mwR8vM?5(jZC_Hkm`PC3F zaD@n2-LVp72i|}|m=1@*;!Ct*l_diXniJEaWP-3O3?U%Lf-$$iR3yX5Xm4--$cRMu z4-cOmmJH_x&h(!hJtvuZdxm=Z2M7E66n>THXkv!r{W_H&zm3OY?;cJp2oaosMZLg= zw%`<=XZ}Rz8n*IaaX)t56$S)lXQ@tE>K{89&~|BOTguw@OpiP}K})6G?T@RQvej+r z>bA$2wG@bVSJt~P?cE2qw#K|*maWL)yYcR|cNZtK4S{q+V69=_pYP4Sa3cM}i8cGl zl<}mZx;C*GPR%HK>4#t)^Lh!D(@V*MULwFcs(MLrpUWKD|A$WEJsUQ)985qw zu?GQK3iM+cHrtrD;wKc`R#ns@nufX)>zEyCs z3*Xd>IL=oJTNi58m(iE*n((%c^cqx zl%-+P@JeVX5x&-!r(xcPhFQZapYWYK8Adac{wSc`CM+swtL;qboVJt@2P z0)%!C-QaC?DBf0m z=uNQu14?$72{AaC8U^dOB}W`Eo3RgsOAkLGbQRt(M2m&HnBk&`lulO|Ghgx5=pHy+Ec-NjPg^x4%G;$EPzi=Nq{cnVQ1+_rel1AHz>5 zII8cdR}H@6$np#f<#*t!2`Z_gkhSB%)8KeWA0esgVu+7}BZ3{=@d#wBA*1KqKvc8w z0OTPeM3QA3vSPwTC0!_)A)Mpl;aNy2fd3+ni?~s#I?9R59Dk7uu^6;Lu4sTY5}%qD zgJR+m7Z)TOOsMG984iWgS9DZt#LKHfDZ1-;Dvgppv}Y6-cKD*w2q@33xS4s`&YmKAM=A;CP80C`An& zs^IwH5+;bnC67DTyv1?biWCJ>f(hI|HGHT;CR^6HD{b7hbR}cl_r%wfYJM^8JFsAv z^{A>QTiKMZY|2(1Nmm|$l*^65YlDj&5IaA7sch(>gb7$&kkqv@W})NA2P4||tN)$d)7rR$GmJg=lC zaNP!00#xM#g%U3xp?*)O{>$g>H{i$OkAU9aROycyRX zxfX-!P-E-8=$&Y4&$luSN9B49YuNnFMhs&RoMGLax}|u=dqCcSK~1Q(86WG?_cFeN zax(_CY#zB2!*(I2?OyUuGS%LlY3z|(0rs>FxqPx8_wL>_%RK;EFvu?;NPN_^rfXWW zHGyO8jA(kmNKW#gBKs;qj7tN)O7FyudlgD|7_j6JUqzvt>C?-3pPB}*sykOIKZiB9q( zdruer2<_`RKtI|?1MtxS8bi8t&<^9%&TkUkmO3R9{+PiY)qkxRy_2YBv_#rn{(yrnzc>Va?C(WN+Nxkvr-_EB_ zy_N2NdmRB(Zb!QVPYqSH>xr*nQCK>;JaYd`su1kCL`Ql-Mg+lW#0fnyH{K*;nn>s7e8u{5q|&p`l z;Vtrel~a;TaJ@Di7QqMPG2cmgxXHsZt(=c?TzK<&d`J>CJq@m=;s@3U??gjU@MB}M z0k}m7gzzt2A_?WFrRoddIbV#0_)GW}f(xjrV%3XOvVMUQ1bla`{Ol#d2?5QkEdXcL z%kOU-q-CXx24P(b6-y+9nsQw>O`xi9Qw=rc43e{k;Rh6QidRG&80ubCRJf`3O6 drbQDR0bzP(hU(_5(*DvsO%lu(2-nKd{14LJ?kE5N literal 0 HcmV?d00001 diff --git a/src/collectors/__pycache__/normalizer.cpython-312.pyc b/src/collectors/__pycache__/normalizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7d716b1615ec8ef1079092b52bf60aca42eb671 GIT binary patch literal 8758 zcmcIpZ)_V!cAw>cQlvzQl&Jq&$%<`C7G+zK|DO{@vYgm4)7s9($6m_P+?7O{LApkGAE#VXqbXW)8h?;9PtIQKsF zz5PduUKwqH4y4(6^WMCfc{BXIc{BeU2>1vj>FM8Pe%(&UuaGd3LoaOqBNUbiM-qf% zI9rZMFz~hKYzz4I-kb2MWmnFZuS?XaWp~b> z40>VAe$zY|;9~V;A{%=Xg7gQ0(J{1;sI$NlS|3jf^NMo;RoVx^>yK zK5zA7uFo+bmkCc`T}Z;lF$p_oyF(HV&dxihZJgtdBjJLwoA>bEXw1}G7 z-*t*KJ)9PDXnBE(!^XgQ8oR!rc#_FXAtNP|i@m$eJZ6Z5L7xae404-n4Gt|i9(lUV z@5tDa<6k`O+c-DHMR{qC7Iqi{7f3Y(2Zd!KnbR(ktT}0Ht)-ic*v^2lvc|$14EO@i zuku_~m3r}0UGF7Ao1hAIrsJw0 zIU$`)Qru3o8?zqFoRBGYQKA&ro%TZZ2~jnx>qLWCi&-Q5iw7XPL%wz#{?1`{9x3}6 zXM$Oo+$Q)fyPQtnN_Zu`HnK9i+WnLOsVTzdJFr7VeW=|2E68?Y#|+_YKVxdp!^yij z7vhJC9uCfp=;0+?kn{3&yq^#7^%}AupYwqv)&;8STztc{i>tfiOa#3IS~hZikZM%r z17^wfyf$c653QP@{}A9-1N0nT@I;%GMjaH5Rpsg80kbT!G&Rps4ev!(5~{Hs@lJ|G zLyAuqNM<<2r=yKp)^P^Q0siK7o&w|-BqZyuQ{hI2<0%sLCP9TciC3Jk_5kKhvng5t z>`DTAIsqD$iO=QoN$@T?tO!-wYNNzTpD3lM1gMshl)7nvDREXKd^vDDh4ex) zFDhZg|D5FD%8M^o|L6pyGY_-|mDT-g&|x3^7(P zL}#GhoD8(kllTYLP?WWR=B(FfRry-?W`HMYT1)TCW!iFU6^yUfy}xBu}h?Yqv>LhEOa$}A;Be>HKP8NxXw1Q<10 zYcxjmrwr$^wjpiap~FyD^)+-97|soPJfH`+a80VwV_?>c}`8=77{gm*Mpw^=k-7j4ZzuJ$q53- zB6ff_jldbOY|{iBbg?xF+Ju0E2ilqhoG@?>)>H!toMzx0w&b(`XVj7t0nQOiPAhQm zs9%$ypElr}sHp}LIPJhWWy$FP&S@*opD>rmU)blFXxHKwraNU*ES|`7yx~$=)zew} zRiBU5(>R?GPnZ-|>acu*y;$+ix~gd^Qdd(VU(s<NT_0Uf~%mz|DbhRaNkLZ&prS>WbEQnSw6*~Z?7B=DxRO>^3&s(;d0ZHnBnp& zKAuv6Cev{Ipm>0dvg&*^&-*fpA#N)!44Dd?;)0NOhNp^q9u6w7uG&0s@QPcXrWQG% zr=jRFwdwt@XGUMewiB4`Hg-A*T;nY83Y4qQ0!m{+4^H`{zIPV)=xb_j0vdZxXdLyR zqW>VzZj*AD_(O7C*G65}+RQJ6p9x$3pzQA~`8#F*P{}_e`$tRu(a+zO$Ih0<&OY+L z_Km;kC-J-St>EZZN1xm=RO%Qq3uQM6bUY=FKz-T&|DyJ(p9CY{kx0<@O{8PxtlSwZ zb;ce=hRb%SdOUJM9yzlya^_>HiuKj#v$7MoWfwUy&dA5(8^`1K`&Z`Wp3zdz=%eOcLc=jHYE%(HSMiZqd~ zez|jGqjRJjLOM)(_si_b4fbTY8R-_%+9OAz8 zq`7nX&5zzJcOlhHdiKiQM>o2UmV1z9E82UJ?j!BJa@){G+faEA()|P$xP5q|eYpG* z(t9iN`;gvG*nM)(@r|D2q^EekQK? zAq+8guBbE`F$P>dLG%qb7-{`{22BXThVR4w_P;@m4GUS%6$V`%V z{*aJcj-n%LUMPULo;aJ~Id3_OPUY$+uW84tgnZ2PGd~O$zY5oOphcWZgVz(M=W@9PRXfj_Rxx!@fJ&k?)byx&EhSPAV5dsyS(p(< zwBV>zLjh)^SHr8XlqeKbhw~fUl6W^d!%I~4JL=uj8LV|_;odnY)AMRW?FLV=!*4{5 zs}0TSF9;2#M}$gK1>lW)zs3F+xdB5;f_}sgA=CUoA886Lxwblo0DV4hTKBa5D%7(y zvDM!7VBhM#^}dnK_R*#DUx&Ka2G_@?N_|sX!LS@;OF>o+#!A7M92_eJ$F}wj{9^v6 z^XtQ}Z0;N1>OT6!X+PTZl-L_X0IrQ~fWf_oKOZcyrakse2(C&G$y!9wS-gI&NktPi--ox8KZf=EGuyKL z$4)0;`V0Zoc4W;Mw$jwNU;*_1jF2Bft6Q#H?pvOsGiye;73reODgrIKtyZt-v9dPE zc2@KmC6Kj<0HHV5kf?m2wy#1NC8+H;%9StF?yXQp$trpSNsEY=wp-U{@laor3`;_^ z4zF4Fea0Gs*ZNEFOwLQ>DEZ)@)CkC)s)vnoPJp;VKYZ@=BdG$%8ZQ#+3rPwde-^~d zi+zWB#i#znPq0&&oCGj!YEMlgDWO9{G%)cQq0j(^=UZLn;qi*EYG$^x^bPn}g`Ob` z6H}c5xK&PM=QGl*;b7RRsj->Dv|wtCM}3v3M~ipV$VUwg)Og1XYt+z0jd!9g6mP<4 z9cK93rg*ibIB*17pmG3d=lm=LXA#J`I#NusFOVPQoaG@4#b+OtUz zMd9kye4H|4M)MRyW3@8Nphr9fdr5UePrFGV^zo^mcYapf^q=}=pz-eIj}QFZFSAEW z?9snHC7--lfz_an(y|q9TR#8cc|*rvpD3|sHp5d(XTGir)$)#& z*on>XK)nY>3i^_)gNuObv|fYZ8NwX-49N#olGCU`Mo7Q4^#wGY8n zm(#H*qn%$6i>sf_8sKCD&q(%y^O^?lXy*NA+HD$s+&vrYQ6w?LF^K;F89p~1`OXn;b*($X zQp~dh!Lfy;wTOkb&6NzQHH%0nicdT)9#T z?=3rlfVQEIm1Cu*{xb5gP5lygZ)C-O@7Uc!$=kK<#p)}J&B?C!9)AksWVPAhoT9Co zRlV@sHkX#*7DP3P;*u8donL!_bAq5|N3^Ztn48OR>KzE2)85Uca`38>Ur_N(X{l7i z_?uEJg-`q$L0u!I^(x#q&E`||8r;iJKE_>*sh4JAEetl-2783@CHeM1WEA)krFA`GAW4?CDD4A)Wfb!GnTDRVk$NQW0xc-5TJIK zl8AtcM^Q3R#~sO&PDDm>1^;R2+U+n+I^}fgsmMt?n#mu+@RaaYGf^i$TK|JWWgNTt z(R1!T0Fn^x#OY**ynFZFbKm!V-#O>v7jCzMg6I2>K8WsZrl?=xhw-p_Wc30>Zc+j@ zL`5xqa;<<>t|y)=$rq+s0CkVT+}7{Ma4!xn+L$_{aYPvGIr3s#X8SP??eHf*Vf z!o#d!gB-h^5~@kAU33Tz_;+rQbO^4i+>jF_YXrCGf_mJgHR-i^K*nRIK*~>~D#QvR z1${vY{MUQn(tE%Ql~AU#q+aZwk;@8IkX!YoYBa{g3tlMWo#KMEs;{^<$BZROZZ)H1 zb&4X!@^z}733E+DlTs|AuQ-`+215K_9fsg0m7>RB_SZiG{hpqhqC#fA!WnX7yb&wD z1V#4Tv7Q5dBb*NWid8P1Z5miAn~BE$of0x*^m7D((GS4S3#75 z5s1p96d#P369uU_#ovYh)iV%0PtiEl)ELCqKVvAl?lF$-cj4fF8}ff>xygK=J zUv45u7M7PAA^0Eaw{K7nJl^Z)uAN))1hSq$&eOVLYyHsH`nl`^fh)~B=nUP?zQgaJ zRqp(xC{3-|Yc`M}lduQc*CxUKdrSjNn7NCfCjGeulk&Dju%zfH4LeTX+b~runJPw` zs%cBgGG;Z(6yL_w(iG&|i|Izp$bXrFiQd*mDFnYUcMM0o?%7639n*}0`-VJYRYt7% z7Pu6x;3AMGWuB4u`SQH@`)TGfEbB$;67vdmktR!QO@8mhB{4Foh=FKaR>E;5 zKqk4<;RxtYpEM%(nqQbmBzgn5yr&1WuK*sg_(-wTOJv8cD|i~$FCdBXWK0PNfZk@c zNq1LxBHDFfcNeK#YJj1R0W!uTNk>m5(JxMxc%WrZX^@AWOEjR=W+l zJdudYqDqT0ZWx>eve!v|`JGM%;s;^*CIn=s7rbt}H{Z1N&e_?scS5tFysvh}*PivY zFMB&O)613n@^vkln&sM#0!KAJwMsF~o=+*J&XazwV58hME3WNX*Y=NG9eH2F&BJdU z{=v~@S8Kru0-rr{QI$2C)+&FW9IoS79l5_S8ykiXUsgoMhfTxc(QAZ?-Xya--eUTU*6wQAFuN zIQEC5hHnjOgvdbQ0-LC{(Rj`e*&;+J+nAR}++^Qm#BlfZ>Gf@Djvmqit# zM4%I^_R@9- zu(1)a@je!8YCzz#NB**4)Afa5^E7lBz@~@3+UMyzNG*ElzWwZ?uMOipn7+85#`r;B ze>bz_YVGf2mO43%cQcTydM-xeLgHdb79)wcAg|E`kZFk~Aem^RKumkaO7&Ac%{Z;d z<3+tQMZa%qrb%GtX$EwJGAs|;0h6*A{0h1OQ>sQyvm?|rmtxDIh`<`BSm4gVW{|%5 z7&$&l8M;T=daNU6{&DKn{m@swJT3{g6koOmd$EPbNH~gZ2+rH4Wp+R?G2m(TnT820VSbIRkWrmlN+=!%tCx?3Ul(JtbPzUeke5)MQ~60H5>H%|P!W+3 z3J6qzK0Yx4;A`Md7iJLnBS6@LC9S&ot+Ft%@r@u#d;lSzTl#KHEC3^a7_pu-J9fYXLD&Y}1qHEM3P@>`NfmF@<_>!2zq?k=G z2lNv_4~ukkb1mmMb)CO%jW8or#PTZmI=%@gH_cC5Q#@EOOv(x&`!@TQ<;@C~nzmt% zzzJoMAZ0T!DG9?8tQbynfEygtVNcNzI^O1{?SO^Ii0ht|-6#z>=v;$?;1!v6rW|92 zC#IyN;5JGL9)g3 zG;&h3P;;Ao8TKYkMVM*pG=C8+-W&9rF0gyA6VC7_e_?Qj4zb`EoCF(G$Ei)N&@aUp zx{gI&czk;UhNCcmpGYmk(HN+qf!C+LfL_qs2#lnR0;3Q*(E3^=7HK!^AC*d44!5c_ z{C!)ccdGPsvQ3^G-w};RBrz;+;1oHeB$RMWrO$rO%E%)BZR}mC8)DL44EAAw2ri)> zqjH2RK0)m!8QiS$5V4B&Cctg#A#E9SCaWHjU;M7riOWopkVS%a!aLF->;RIE$`Sei zQ#uiiYsxk%%PKbxmImg&BCAfV zn?&JNT}Gb?hjQ9%Sv5)pQEddes4*NO%4>rsSYEmfFs|v`euKdbekYqxPCl_2+pU$I zgrUj*2rHqHUE>#!-IlcLV`oKv?}2|h^^;RS8~A8T-%keKJwDercYY>3^K!( zy|(NRrJd+n7@8fLbLN`%uGH>LyB<{f(p=tIyW(ufIvZx5%Q?5t)qm*RllQl-_}jDo z_RNJjDd*p_;_u1&d*(0Q_2&G?)6aff;lE$kG&7P3&yL-R&&KDzKCdjCo{#35pSfF= zZ9bN(JAMPnyS|^kyYp`Iz4ObbL(4CIYx!HkvR^FMjjHN2|A%#>zIotmv_PPU3vOsE&YM_q>Epy*bmVqo<8Yim#lq9 z;N#~UeX@rAIp6PwkAJVBAzWj1pk+X;jvvCw{Jk`QFR(h$`-m+5HZ#qZw?I;+9btMV zbtf(`H%DxNGTukHok84yN3iOwTh`Yh*h*NpTs`&@*0rXrz`CYpkg^I6!n)A3nzRvDuAwrzV_OeVyN`#=tI*Xl^{tWi+pmYkt$1c?8QhGt)`IW>DKlgu>&Y3xZmA5?hLY~Ja=e)QVWnFBfJwmJKU&fR(c)|-j95}Dz- z*K_{eEB<|1|GxRryOx~)n8q%(jWb`*oSq%J^V;ldbI;BT3w`sigX?0EpUco^eHm&Z|;`RWFq?U0fSVu;A8W0Z1&j8RFb&0~m3^B^By@ z z6709xh~*U+?KF28oKV_(odevFR881Rfa_Kq72|UBf@l~Q0YHZFur&46(XZr)6072m zVq}D4G2>SfqG0sdgv>#36v3?&u|s6Ia7A1vfRo&Il3a|TWupTHxv0~Bf1!^VE^Py_ zG{l?;G6e9XK}`4>28gYY8pb$cq4YW?u+k{TqZkmiK7grKDTE1l)&F>xI9~#YeITiN z)@sqJdksKoE^_<w~=Q{LEU2ggvyO52fa+mVIK z_mo`QnU$t9>A`$!`%3GfZ0n(g*K(}`D~$u`7vN+!J(;g>&$sRZXHm7MK<>`0RP&$H zzs)xeeD(-DM*5YT&+aj=+v4Jp*nWt+dJ|*f>2SD7ra_++wi;N3HaXP z1&y?A_x<|DnbWuY59{64p5IVzw+AlhpxWQbxFX;)ucxop^wHETeVF}G9Yp?tKFa>o z4iWHB)zcr)5c^jQ7HMCb zyav>$5z$9;QX(}<2yD_b7$8`b?41~VzDr4XU#eZKf+vBJFhDGkpMefur9Nh#EU*@? zCnK*?@LIK5xVD1L&v`Sx0tK(R>bb~#_uS~a&8zrX=%s45Jalg3yueZg4&Jb)m30Lx z#%z?wU$A4$K|wA`!NNE^>Bx=l^yszbRX3)*NP~6pkZa*;AJjDz@C{XfkZ_}WhPiRz zT0CoSTD4;GNtDb7pL%HB#W{#Dyg|64_J%TZEaRJb?ybvNPkRBgq4AQ2?3gmDHMB!7 z3vaCJe`G(oX6P$;%0q=5cU~*|2kL-}5O)gGp&@D;uQtfcjJsDBp?psr9h;h#= zQxkBNudN|jO|ocYz0fbJI^cFaq~Dj5yDzd-GGrB(PqiJ=_RSILB4ohoQF1o~1%{^S xkEsLyNmc%m+VM+jXO7zWE6Vl2-juFiu{XhN=IkwR@SpM%4DI?ge!g>Fy5)9O8i0uSb_98)9>AI_0IXX};#o8d&epREyHH>On9b=n!(Hs*A z`g)rOhe<8#~LJ1B+p#Yk3UoM_rTjAe&nt)!^izkkP8$fg?{NrXiM zOULLi2l3V(b~F(lklzgXU2+xsr5f%^Lb0N?pFTE1vz+pz@JdgR!CII>v?;s;(5w+w zbjallgpp;Wkr;-p!QioxP)tcN27^>G91LC|X0bBAt74gOXBeacVoNe?r`&~X$M7iQ zfpm$KLx42_d6BrMdm^hdG1Y3)oaPxJ7TxTRfXr>)5J74Fkqqd-*MhGHUkAQ{($mI4 z4P|&y7cfyq+Du!jATMR2EtHuqlb;3+lm*hQ1?l-T9Sl=lzy<@=PT6P&Wv1-F?|?Wb z<)mFuikWs}nS(mY1+nFnyC}9CVm*|nD7FG(D@N_Dm122e#}3fU3z0BAwpa;B<69o@ zk;ITcUN{ffj#i6kW8^^=3@1ktoM>Z5!eJPc%3~Rgy{tUAV6?;R2*zLnB*LS?I4jzR z*R7iKtgvw0AcyI0F3E*r!AJt;TaZg0r4y`JF2^NDxPr(ACF7jfAWXj?ONWyQN_2!n z;UjeLNQ4VAICEPKP*-{ODAn_*k7^Tuj~3N})kJDU%AN?uBZ(1?W|>BiJ+lDGLLjYr z(WWc{O0i0H)%>!Ul*l=<%a;jA8W@@zVG`k7 z)u99f{G~>2g8(*?B}Yk4T~~;FGow0SC~_bYjkNy;<#`TEzQ8X3o?O%XTvFL2XnzacG1SrSU$;t{ zN-*AA^RMie1W@$Wf?}i!_+f|XOD1T4WYACJ`kAW$bg8Hxj3q;ys7VYnt3Vvk$xndC ztVDtlEJ_pFB6nyRP6Hai7ta~1*f~-+HbDTok%;EH5F!+p0);{CE~%UTUhO7%GJl3H%896?gm+2n zkldZ~OZOubWGl(w=x){I#X4t(7=vVR5$&aU*7}_684h_K`{`?5%r;>?g4#qyij0|)B z(9jS=55Y3ZcI0|wtXwWYUVxNxK)yKIm`y-%R2!rs8BrHW3?@arG!N9FonxFkv^~Ki zQX_)W90|(x8*3@qEhRV)z?<1cKu!@4EQHzda@k8|S!-j~+nDh-^WNrcMcs^%Fnj)o z(3vf2hbj9%}~nM;Xiv26oxMmq&xv2qS%${HBQy92@cmn$z4?@PZUcoa(S+( zq_fXW=4|S7id0KE+CmU&SoOaSy82gO41G&;PW#t-Ju$A~9J!pDs7np!eRZ3OqViPa z)HJp9acxS=w!tD$QN#$3T%{Tp^{Jt}U#kAgW)ts)=5#xHuS*onsqJrR2DXT<^QkG$!vg+q8t3--YmT{i4{PUJo zB1I|7G*4Lz<}Is4ic*$&9F*O03(D;9vx?kHw1S;i?vPSfx<@F@(XB8W$UYvP2g@+NM;QIwDEp^m96bYgTzE-HXyldSv8U3wZSN@Yo7?4D;~V z3wVt4@HkQqx#f&LWluSq z1aSy-=yA_@MXCZ;3NyDxWq$lx2imJARgo&c=D4ct0ppd>+O=x#ydPbk533>IRD%V6 zs#29<@d{58DR;>^QmIM@=ExAt5fqT*qPRI1Im%gTg)SJSZnd;)uDs5Xat{z|h%?&L ztHK%**6nfKaiD)#Xbo{X_|MwoWUG7Z2dcUlip5l=B`yI}R?@XYs$u4_O1=urB2<@u zK;F;`)px~SfCYYn-manq87+f8RH%CR!CXPdSPQ}mbbSostjKR*&kYx<#;C`wfdA|v zAgbQx5j7plM3R}K?wY%Y=meOxn2lo_3!A4{3p$jx1L&44VnNl~(j7~N zLos$^M=t-1@G=&6m){Wody1H<*?;lGWaIVaH`m@+`@`|frhWXTeZq=;=X)-mcsPru znXfHqC;!RVPX2kM#h}6s3g}MoWI5@3`}=qIGm>hLU7#Q=W;cXCt8D%X$|#Uc)!vH- zC&}xso4y;qA0EzZ*u!twBP`!@zVYJ0hs++}EiID(q~u1>!udy#v_qF1TNI(}!+wfX z#?N9HKN@NLXr&R%?0~oQfqAf8vae8cOf_GObkZmt+_~kE-pGS2|tui-DbRs)0D7Sn*l zj|3JTfUx*c$fx5qLv+rGq@>> z+8CW+HXs9<2pC+tMZ*XeP9%?u`UpqISq4EAgPH{-46`)DF+CWw8Az*FHYrF=1ydM1 z5(CS>Vi9DSKFnm6YZw8;nI0?1G7-#~Hs6<`C1(L;MW<{- zJs#nX$TL>zU(qP{7AqQ+gTD|GZR5Rd(^mI4y9KL1V{PKCO_x%c=8b&w#ybmdN8jq?oA(OV z{#mWYXZbCmaa+=c2L{69n=o9kzhW1BEg4@s?`s!)9YT3$+Vo3@H*0lftc!W;;!DX) z(+0k2!w=cpT|YSazIAuDydu5hzJX|8DR}l=SvwiJe(YxIMv7mv`^?^RhrV&>^4hd{ z%CqO*qISW#OL@HThIpKjqwZ?};wK7r4=KdGD>z8_Cy`{KjV| zTo)R?({Q~dZJY81?k!s@cn{n#U0!~x=Z(JC`}po>Cddo+@7S-K($*>OfqQG86nulP zo31av-ILk+ERR2W!j-A&5UM(EnXrt(83R$*FlDOxxvMJOlXW;>KKXYiCzrq4{k`t% zJvX1e@$~KG!pf~#ukYN+vnMaRUaP%Q%X?Sd^6-@#&SEvOdam( z2Ueo0mdBl0sM#<0_NQ&rl~p|MMnctIp>l7!41@|XxB?44N~oaHmg$;$9#+tXn|8&*j>4uI>!)m@^b+)cK>tB>@Y|8o7Ly?4Q zEBQP2+c2HJ)>F~9gnZLW_E~jr`g&^N;on!f`W(8q40fa*1H`;l?dtRB-fFia?ZTKl zgsZPgcgJl<>X%}gTz&PrJ8SJg-!@>(+g4ZKLfzZ-cBJk2njdMbNF5HO4ZV%v{e zg7>b8>}xT+Ywpp(!%rR6Kt+psa8hJX0~{>d)Akwj8ciu(_%HUf&%vHn5=VZ5WUHZv zB&VXil9tLGSF~2;&-(?~xP-+AaSWHw@6YE`Ls8UtirgG~o*gt`v<;$4Q{=#+<E!P4G8U8nCJvz>;tbYFw5oi=u{APTRfDXInd7DnbkvmJ500`nh%JfIQk=03dx5KQ`J8;{i*fHCo z*YM=B$T907z65`1?lI68qd6N-z9`jlL+R1pHf~LsOSU6tQ3Jb)u9Sr+iYIPGjxxM} z=sTIRP{uJ(RmUx-N3gw&J!Q#Th4PlXoW;+^ty7CC#i#aI$_Cap&A1(GbZBiWWfjU< z=~8w|i>4glLzT72rHqtC9rM6hCUJtDaGp|gmO|jwNxbHHdCj@n+GQ&qjBy8-vn55J zEUF4Ts(muX~`9=C_XhV?}N3?m2#(?*K9d!TRF6L zTb@G=MG@i_Ip8dc!N*j&D#POS0!EFa1#Wqr*2V*0!yA8yi^PG^Z)BrS(a+H0Ril6);xXj;3tP{!o}nvT)`H_BO@B0|$E+ z6$F$zK$osV1<|(ljC?ifPi9+_uT$WzFeC|r=z9|Iw;5qcjOJ)cvRp85a)F1txQH^X zNCHUEF2yWC@*I!?q^b%+K|zr7Byu4nZf%yYS9}>GV6h<>*l`FZghXOOFvEnAP)KMb z1*r6yp8g#@eft=+R>)|25JNaoL~SGyl<+Nc2;&YT!2u|m0FlxZplQ(pMmz?Zg<+eF z?&O;beErhO`Z;QQ=LSvrczf zpY>Ei$XAuN{mR{t-kf!~UjFJ!U!7`P`JQ9tE%zG@|JsmsR%M)i-szuozuIuI;REOL ztgHN7{n`4dg)0Qtid&jD96xYmofYS-XRQVNJRl>?YGTfW_<>Ab*f-AoavjlNvi-ap{09S6&z{8#D^1gcaouzIJ zZz2&D5x$K*#_P_)tgs=_Y)`J!WQyf0H#fbcYS~hTgac*8;~{uR3t@P z{x&V_Cvz@bnBe_ihyUz0{a*kg$btAGFI?n-4_AZlLPu#zA8D)V$heNHlQ}_2(&b|H zrPZiU0Gl~@sFqU_Z05K=*K09`eCtz&&kXyFDI;87F%?{1F#(M7tCGt5(uI?fDY?rS zCT^k1pFd*H#}&x`(TfAQoCRDt;GjyG(w^4>Ff~h1W3l?Kyr0S|`W6Kg5K%leC~_$? zrB~Ht07eFc4*&_u@>dd~sCgARRpkeOW+`>8Kus@z1U3Lhb_o)6s)c-GDLHrF#l@GR1y;8XIt*d z%QbTWBq)<0f#&oE00QLcxB?7J$KhCBfdrPZ4;}{z5(OVO6v6^Ue|kJDn6r|1`FnGp zN&G3m!JNZV>>KRMSw!!-!%{d1pYOO`IR>5%RU_#LC8~pNSrbvk!mrFNq+-GEK^nZ2nDcYBoTpcV1nwf6fJPT2AAIP zu#Exx8X=vt$>`+qV8aQ_g&+gNGSH|-NC_vB!ACoBtmD&lI(+*?uvp>k8Y zY`VH`;;2x)B<%nwQ0u?Y{YrPHrjxJfeDpM~RPAc(wSL=j@YYM~njXFtpw|+|9{nMWJtm7Nj zW$TuFzUMuyDCf9P-9GPu&ruQ%d>+o!f;4~su@#SfW|qmPJpC|IK^x!Mh}!s04{GCX zc(}8K-09WbY3qdW+tsd})o?~*N7}X74pDc_Jx1`}agsY54DYylbnx(>kRNEZ0cdL_ zgG&#C*L4{@{bkUoCOVYE^x&@Eef>MO42TZJj**LS=T`tVjw!?Zxb89rB$7qKh@mbd z8<4n=NNbB^BS95kwoQB$LoXwlKvIRI8VPEn3~qc9!P>Du1A^alb$)Cp^BPX={LZO`o(-!m{P zJf5uL`Yp=hAR&vVGeH0OIHvW z+y_Jhd`18tY%u_c6d|L0Mcv3qgkn&bqCPPahmYV;rvT&8u~;Yu-$%tq88hZU`;}Ok zkL{pOa2+B1;xLkwdQbE{58rSdiHDe@@SPJwcTmrFD5f6CfF-T)XaEx{iWn6x2fhXu zRrylJ$|5fOE=l^p5bi#7$hQ)d%Vg3gOY*me@~2FeA({)y-wQ~agJ|wO5vHXtDCJN5 zq%kpfoQS4ws0VyMxRH4k>Hy0w!9ELQMoW_9J>sdK6V4BbmJf+$o@o9BQSmFn{UNdN z7ewc;iGH5w|B&e8iN0SD2VkwR)TgU5<~rV7Czu;f!8HR7X+AUhfB^5emJuE4{g^;1 Hb=&^|!+ None: + """Validate that required fields are present.""" + missing = [] + for field in self.REQUIRED_FIELDS: + if field not in raw_data or raw_data[field] is None: + missing.append(field) + + if missing: + error_msg = f"Missing required fields: {', '.join(missing)}" + self.diagnostics.append({ + "type": "missing_field", + "fields": missing, + "raw_data_sample": {k: str(v)[:100] for k, v in list(raw_data.items())[:5]}, + "message": error_msg, + }) + raise MissingFieldError(error_msg) + + def _extract_correlation_keys(self, raw_data: Dict[str, Any]) -> Dict[str, Optional[str]]: + """Extract correlation keys from raw data.""" + keys = {} + for key in self.CORRELATION_KEYS: + value = raw_data.get(key) + if value is not None: + keys[key] = str(value) + else: + keys[key] = None + + # Try to extract from tags if present + tags = raw_data.get("tags", {}) + if isinstance(tags, dict): + for key in self.CORRELATION_KEYS: + if keys[key] is None and key in tags: + keys[key] = str(tags[key]) + + return keys + + async def _resolve_session( + self, + correlation_keys: Dict[str, Optional[str]], + proxy_key_alias: Optional[str] = None, + ) -> Optional[str]: + """Resolve session from correlation keys or proxy key alias.""" + # First try direct session_id + if correlation_keys.get("session_id"): + return correlation_keys["session_id"] + + # Try to find session by proxy_key_alias + if proxy_key_alias: + session_model = await self.session_repo.get_by_proxy_key_alias( + self.session, proxy_key_alias + ) + if session_model: + return session_model.session_id + + return None + + def _parse_status(self, raw_status: Optional[str]) -> RequestStatus: + """Parse request status from raw data.""" + if raw_status is None: + return RequestStatus.SUCCESS + + status_map = { + "success": RequestStatus.SUCCESS, + "error": RequestStatus.ERROR, + "timeout": RequestStatus.TIMEOUT, + "cancelled": RequestStatus.CANCELLED, + } + return status_map.get(raw_status.lower(), RequestStatus.SUCCESS) + + async def ingest_raw_request(self, raw_data: Dict[str, Any]) -> Optional[Request]: + """Ingest a single raw request record. + + Args: + raw_data: Raw request data from LiteLLM + + Returns: + Normalized Request model or None if duplicate + + Raises: + MissingFieldError: If required fields are missing + UnmappedRowError: If row cannot be mapped to session (when required) + """ + # Validate required fields + self._validate_required_fields(raw_data) + + litellm_call_id = raw_data["litellm_call_id"] + + # Check for duplicate + if await self.request_repo.exists_by_litellm_call_id( + self.session, litellm_call_id + ): + logger.debug( + "Skipping duplicate request", + litellm_call_id=litellm_call_id, + ) + return None + + # Extract correlation keys + correlation_keys = self._extract_correlation_keys(raw_data) + proxy_key_alias = raw_data.get("proxy_key_alias") + + # Resolve session + session_id = await self._resolve_session(correlation_keys, proxy_key_alias) + + # Map status + status = self._parse_status(raw_data.get("status")) + + # Build normalized request + request = Request( + session_id=UUID(session_id) if session_id else None, + experiment_id=UUID(correlation_keys["experiment_id"]) if correlation_keys.get("experiment_id") else None, + variant_id=UUID(correlation_keys["variant_id"]) if correlation_keys.get("variant_id") else None, + provider_id=UUID(correlation_keys["provider_id"]) if correlation_keys.get("provider_id") else None, + provider_route=raw_data.get("provider_route"), + model=raw_data.get("model"), + harness_profile_id=UUID(correlation_keys.get("harness_profile_id")) if correlation_keys.get("harness_profile_id") else None, + litellm_call_id=litellm_call_id, + provider_request_id=raw_data.get("provider_request_id"), + started_at=raw_data.get("started_at"), + finished_at=raw_data.get("finished_at"), + latency_ms=raw_data.get("latency_ms"), + ttft_ms=raw_data.get("ttft_ms"), + proxy_overhead_ms=raw_data.get("proxy_overhead_ms"), + provider_latency_ms=raw_data.get("provider_latency_ms"), + input_tokens=raw_data.get("input_tokens"), + output_tokens=raw_data.get("output_tokens"), + cached_input_tokens=raw_data.get("cached_input_tokens"), + cache_write_tokens=raw_data.get("cache_write_tokens"), + status=status, + error_code=raw_data.get("error_code"), + ) + + # Persist + model = await self.request_repo.create(self.session, request) + + logger.info( + "Ingested request", + request_id=model.request_id, + litellm_call_id=litellm_call_id, + session_id=session_id, + ) + + return request + + async def ingest_batch(self, raw_requests: List[Dict[str, Any]]) -> int: + """Ingest multiple raw request records. + + Args: + raw_requests: List of raw request data + + Returns: + Number of successfully ingested records + """ + ingested = 0 + for raw_data in raw_requests: + try: + result = await self.ingest_raw_request(raw_data) + if result: + ingested += 1 + except (MissingFieldError, UnmappedRowError) as e: + logger.warning( + "Failed to ingest request", + error=str(e), + litellm_call_id=raw_data.get("litellm_call_id"), + ) + return ingested + + def get_diagnostics(self) -> List[Dict[str, Any]]: + """Get accumulated diagnostics.""" + return self.diagnostics diff --git a/src/collectors/normalizer.py b/src/collectors/normalizer.py new file mode 100644 index 0000000..87a0f19 --- /dev/null +++ b/src/collectors/normalizer.py @@ -0,0 +1,197 @@ +"""Request normalization logic for canonical field mapping.""" +import structlog +from datetime import datetime +from typing import Any, Dict, List, Optional +from uuid import UUID + +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import RequestModel, SessionModel, VariantModel +from benchmark_core.models import Request + + +logger = structlog.get_logger() + + +class NormalizationDiagnostics: + """Container for normalization diagnostics.""" + + def __init__(self): + self.missing_sessions: List[str] = [] + self.missing_variants: List[str] = [] + self.unmapped_rows: List[Dict[str, Any]] = [] + + def has_issues(self) -> bool: + return bool(self.missing_sessions or self.missing_variants or self.unmapped_rows) + + def to_dict(self) -> Dict[str, Any]: + return { + "missing_sessions": self.missing_sessions, + "missing_variants": self.missing_variants, + "unmapped_rows": self.unmapped_rows, + } + + +class RequestNormalizer: + """Normalizes raw requests to canonical format.""" + + CANONICAL_FIELDS = [ + "request_id", + "session_id", + "variant_id", + "experiment_id", + "provider_id", + "provider_route", + "model", + "harness_profile_id", + "litellm_call_id", + "provider_request_id", + "started_at", + "finished_at", + "latency_ms", + "ttft_ms", + "input_tokens", + "output_tokens", + "cached_input_tokens", + "cache_write_tokens", + "status", + ] + + def __init__(self, session: AsyncSession): + self.session = session + self.diagnostics = NormalizationDiagnostics() + + async def normalize_request(self, request: RequestModel) -> Optional[Request]: + """Normalize a single request model to canonical format. + + Args: + request: Raw request model from database + + Returns: + Normalized Request domain model + """ + # Check join integrity + if request.session_id and not request.session: + self.diagnostics.missing_sessions.append(request.session_id) + logger.warning( + "Request references missing session", + request_id=request.request_id, + session_id=request.session_id, + ) + + if request.variant_id and not request.variant: + self.diagnostics.missing_variants.append(request.variant_id) + logger.warning( + "Request references missing variant", + request_id=request.request_id, + variant_id=request.variant_id, + ) + + return Request( + request_id=UUID(request.request_id), + session_id=UUID(request.session_id) if request.session_id else None, + experiment_id=UUID(request.experiment_id) if request.experiment_id else None, + variant_id=UUID(request.variant_id) if request.variant_id else None, + provider_id=UUID(request.provider_id) if request.provider_id else None, + provider_route=request.provider_route, + model=request.model, + harness_profile_id=UUID(request.harness_profile_id) if request.harness_profile_id else None, + litellm_call_id=request.litellm_call_id, + provider_request_id=request.provider_request_id, + started_at=request.started_at, + finished_at=request.finished_at, + latency_ms=request.latency_ms, + ttft_ms=request.ttft_ms, + proxy_overhead_ms=request.proxy_overhead_ms, + provider_latency_ms=request.provider_latency_ms, + input_tokens=request.input_tokens, + output_tokens=request.output_tokens, + cached_input_tokens=request.cached_input_tokens, + cache_write_tokens=request.cache_write_tokens, + status=request.status, + error_code=request.error_code, + ) + + async def normalize_unmapped( + self, + raw_requests: List[Dict[str, Any]], + session_alias_map: Optional[Dict[str, str]] = None, + ) -> int: + """Normalize raw requests that lack session correlation. + + This method attempts to map requests to sessions using + proxy_key_alias lookups. + + Args: + raw_requests: Raw request data lacking session_id + session_alias_map: Optional mapping of proxy_key_alias to session_id + + Returns: + Number of successfully mapped requests + """ + mapped = 0 + session_alias_map = session_alias_map or {} + + for raw in raw_requests: + proxy_alias = raw.get("proxy_key_alias") + if not proxy_alias or proxy_alias not in session_alias_map: + self.diagnostics.unmapped_rows.append({ + "litellm_call_id": raw.get("litellm_call_id"), + "reason": "no_matching_proxy_alias", + }) + continue + + # Inject session_id from mapping + raw["session_id"] = session_alias_map[proxy_alias] + mapped += 1 + + return mapped + + async def join_to_sessions( + self, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None, + ) -> List[Dict[str, Any]]: + """Join requests to sessions and variants for analysis. + + Args: + start_time: Filter requests after this time + end_time: Filter requests before this time + + Returns: + List of joined records with session and variant info + """ + query = ( + select(RequestModel, SessionModel, VariantModel) + .join(SessionModel, RequestModel.session_id == SessionModel.session_id) + .join(VariantModel, SessionModel.variant_id == VariantModel.variant_id) + ) + + if start_time: + query = query.where(RequestModel.started_at >= start_time) + if end_time: + query = query.where(RequestModel.started_at <= end_time) + + result = await self.session.execute(query) + + joined = [] + for req, sess, var in result.all(): + joined.append({ + "request_id": req.request_id, + "session_id": sess.session_id, + "variant_id": var.variant_id, + "experiment_id": sess.experiment_id, + "provider_id": var.provider_id, + "model": req.model, + "latency_ms": req.latency_ms, + "ttft_ms": req.ttft_ms, + "status": req.status.value, + "started_at": req.started_at.isoformat() if req.started_at else None, + }) + + return joined + + def get_diagnostics(self) -> NormalizationDiagnostics: + """Get normalization diagnostics.""" + return self.diagnostics diff --git a/src/collectors/prometheus_collector.py b/src/collectors/prometheus_collector.py new file mode 100644 index 0000000..e7ea0bd --- /dev/null +++ b/src/collectors/prometheus_collector.py @@ -0,0 +1,251 @@ +"""Prometheus metrics collector for operational data.""" +import structlog +from datetime import datetime, timedelta +from typing import Any, Dict, List, Optional + +import httpx + +logger = structlog.get_logger() + + +class PrometheusCollector: + """Collector for Prometheus metrics.""" + + def __init__(self, prometheus_url: str): + self.prometheus_url = prometheus_url.rstrip("/") + self.client = httpx.AsyncClient(timeout=30.0) + + async def close(self) -> None: + """Close HTTP client.""" + await self.client.aclose() + + async def query(self, query: str) -> Dict[str, Any]: + """Execute instant query. + + Args: + query: PromQL query string + + Returns: + Query result dict + """ + try: + response = await self.client.get( + f"{self.prometheus_url}/api/v1/query", + params={"query": query}, + ) + response.raise_for_status() + return response.json() + except httpx.HTTPError as e: + logger.error( + "Prometheus query failed", + query=query, + error=str(e), + ) + raise + + async def query_range( + self, + query: str, + start: datetime, + end: datetime, + step: str = "15s", + ) -> Dict[str, Any]: + """Execute range query. + + Args: + query: PromQL query string + start: Start time + end: End time + step: Query step interval + + Returns: + Query result dict + """ + try: + response = await self.client.get( + f"{self.prometheus_url}/api/v1/query_range", + params={ + "query": query, + "start": start.timestamp(), + "end": end.timestamp(), + "step": step, + }, + ) + response.raise_for_status() + return response.json() + except httpx.HTTPError as e: + logger.error( + "Prometheus range query failed", + query=query, + error=str(e), + ) + raise + + async def collect_litellm_latency( + self, + window_seconds: int = 300, + ) -> List[Dict[str, Any]]: + """Collect LiteLLM latency metrics. + + Args: + window_seconds: Time window to query + + Returns: + List of latency metric records + """ + end = datetime.utcnow() + start = end - timedelta(seconds=window_seconds) + + # Query for latency histogram + result = await self.query_range( + 'histogram_quantile(0.50, rate(litellm_request_duration_seconds_bucket[1m]))', + start, + end, + ) + + metrics = [] + if result.get("status") == "success": + for item in result.get("data", {}).get("result", []): + metrics.append({ + "metric_name": "latency_p50_seconds", + "labels": item.get("metric", {}), + "values": item.get("values", []), + }) + + return metrics + + async def collect_request_counts( + self, + window_seconds: int = 300, + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, int]: + """Collect request count metrics. + + Args: + window_seconds: Time window to query + tags: Optional tags to filter by + + Returns: + Dict with request counts + """ + end = datetime.utcnow() + start = end - timedelta(seconds=window_seconds) + + # Build label filter + label_filter = "" + if tags: + label_parts = [f'{k}="{v}"' for k, v in tags.items()] + label_filter = "{" + ",".join(label_parts) + "}" + + # Query for total requests + total_query = f'sum(increase(litellm_requests_total{label_filter}[{window_seconds}s]))' + total_result = await self.query(total_query) + + total = 0.0 + if total_result.get("status") == "success": + results = total_result.get("data", {}).get("result", []) + if results: + total = float(results[0].get("value", [None, 0])[1]) + + # Query for error requests + error_query = f'sum(increase(litellm_request_errors_total{label_filter}[{window_seconds}s]))' + error_result = await self.query(error_query) + + errors = 0.0 + if error_result.get("status") == "success": + results = error_result.get("data", {}).get("result", []) + if results: + errors = float(results[0].get("value", [None, 0])[1]) + + return { + "total_requests": int(total), + "error_requests": int(errors), + "success_requests": int(total - errors), + "window_seconds": window_seconds, + } + + async def collect_cache_metrics( + self, + window_seconds: int = 300, + ) -> Dict[str, int]: + """Collect cache hit/miss metrics. + + Args: + window_seconds: Time window to query + + Returns: + Dict with cache metrics + """ + end = datetime.utcnow() + start = end - timedelta(seconds=window_seconds) + + # Query for cache hits + hits_query = f'sum(increase(litellm_cache_hits_total[{window_seconds}s]))' + hits_result = await self.query(hits_query) + + hits = 0.0 + if hits_result.get("status") == "success": + results = hits_result.get("data", {}).get("result", []) + if results: + hits = float(results[0].get("value", [None, 0])[1]) + + # Query for total requests with potential caching + total_query = f'sum(increase(litellm_requests_total[{window_seconds}s]))' + total_result = await self.query(total_query) + + total = 0.0 + if total_result.get("status") == "success": + results = total_result.get("data", {}).get("result", []) + if results: + total = float(results[0].get("value", [None, 0])[1]) + + return { + "cache_hits": int(hits), + "total_requests": int(total), + "cache_hit_ratio": hits / total if total > 0 else 0.0, + "window_seconds": window_seconds, + } + + async def collect_summary( + self, + window_seconds: int = 300, + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + """Collect comprehensive metrics summary. + + Args: + window_seconds: Time window to query + tags: Optional tags to filter by + + Returns: + Dict with all collected metrics + """ + try: + request_counts = await self.collect_request_counts(window_seconds, tags) + cache_metrics = await self.collect_cache_metrics(window_seconds) + + return { + "requests": request_counts, + "cache": cache_metrics, + "window_seconds": window_seconds, + } + except httpx.HTTPError as e: + logger.error( + "Failed to collect Prometheus summary", + error=str(e), + ) + # Return empty window handling + return { + "requests": { + "total_requests": 0, + "error_requests": 0, + "success_requests": 0, + }, + "cache": { + "cache_hits": 0, + "total_requests": 0, + "cache_hit_ratio": 0.0, + }, + "window_seconds": window_seconds, + "error": str(e), + } diff --git a/src/collectors/rollups.py b/src/collectors/rollups.py new file mode 100644 index 0000000..1cdbbbf --- /dev/null +++ b/src/collectors/rollups.py @@ -0,0 +1,333 @@ +"""Metric rollup computation for sessions, variants, and experiments.""" +import structlog +from datetime import datetime, timedelta +from typing import Dict, List, Optional +from uuid import UUID + +import numpy as np +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.db.models import MetricRollupModel, RequestModel, SessionModel +from benchmark_core.models import MetricRollup, RequestStatus, RollupScopeType +from benchmark_core.repositories.metric_rollup_repository import MetricRollupRepository + + +logger = structlog.get_logger() + + +class EmptyWindowError(Exception): + """Raised when a rollup window has no data.""" + pass + + +class MetricRollupService: + """Service for computing metric rollups.""" + + # Metric names that must be computed + SESSION_METRICS = [ + "request_count", + "success_count", + "error_count", + "median_latency_ms", + "p95_latency_ms", + "median_ttft_ms", + "total_input_tokens", + "total_output_tokens", + "median_output_tokens_per_second", + "cache_hit_ratio", + ] + + VARIANT_METRICS = [ + "session_count", + "session_success_rate", + "median_session_duration_minutes", + "median_latency_ms", + "p95_latency_ms", + "median_ttft_ms", + ] + + EXPERIMENT_METRICS = [ + "variant_count", + "total_session_count", + "total_request_count", + "median_latency_ms", + "p95_latency_ms", + ] + + def __init__(self, session: AsyncSession): + self.session = session + self.rollup_repo = MetricRollupRepository() + + def _percentile(self, values: List[float], percentile: float) -> Optional[float]: + """Compute percentile safely. + + Args: + values: List of values + percentile: Percentile to compute (e.g., 95 for p95) + + Returns: + Percentile value or None if empty + """ + if not values: + return None + return float(np.percentile(values, percentile)) + + def _median(self, values: List[float]) -> Optional[float]: + """Compute median safely. + + Args: + values: List of values + + Returns: + Median value or None if empty + """ + return self._percentile(values, 50) + + def _handle_empty_window(self, scope_type: RollupScopeType, scope_id: str) -> None: + """Handle empty rollup window safely. + + Empty windows are logged but don't corrupt aggregates. + """ + logger.info( + "Empty rollup window", + scope_type=scope_type.value, + scope_id=scope_id, + ) + + async def compute_session_rollups(self, session_id: UUID) -> List[MetricRollup]: + """Compute all rollup metrics for a session. + + Args: + session_id: Session to compute rollups for + + Returns: + List of computed MetricRollup models + """ + # Fetch all requests for this session + result = await self.session.execute( + select(RequestModel).where(RequestModel.session_id == str(session_id)) + ) + requests = list(result.scalars().all()) + + if not requests: + self._handle_empty_window(RollupScopeType.SESSION, str(session_id)) + return [] + + # Extract values for computation + latencies = [r.latency_ms for r in requests if r.latency_ms is not None] + ttfts = [r.ttft_ms for r in requests if r.ttft_ms is not None] + output_tokens = [r.output_tokens for r in requests if r.output_tokens is not None] + input_tokens = [r.input_tokens for r in requests if r.input_tokens is not None] + cached_tokens = [r.cached_input_tokens for r in requests if r.cached_input_tokens is not None] + + # Compute token-per-second for each request + tokens_per_second = [] + for r in requests: + if r.output_tokens and r.latency_ms and r.latency_ms > 0: + tps = (r.output_tokens / r.latency_ms) * 1000 + tokens_per_second.append(tps) + + # Count successes/errors + success_count = sum(1 for r in requests if r.status == RequestStatus.SUCCESS) + error_count = sum(1 for r in requests if r.status == RequestStatus.ERROR) + + # Cache hit ratio (requests with cached tokens / total requests with input) + cache_hits = sum(1 for r in requests if r.cached_input_tokens and r.cached_input_tokens > 0) + total_with_input = sum(1 for r in requests if r.input_tokens and r.input_tokens > 0) + cache_hit_ratio = cache_hits / total_with_input if total_with_input > 0 else 0.0 + + # Build metrics + metrics = { + "request_count": float(len(requests)), + "success_count": float(success_count), + "error_count": float(error_count), + "total_input_tokens": float(sum(input_tokens)) if input_tokens else 0.0, + "total_output_tokens": float(sum(output_tokens)) if output_tokens else 0.0, + "cache_hit_ratio": cache_hit_ratio, + } + + if latencies: + metrics["median_latency_ms"] = self._median(latencies) + metrics["p95_latency_ms"] = self._percentile(latencies, 95) + + if ttfts: + metrics["median_ttft_ms"] = self._median(ttfts) + + if tokens_per_second: + metrics["median_output_tokens_per_second"] = self._median(tokens_per_second) + + # Persist rollups + rollups = [] + now = datetime.utcnow() + for name, value in metrics.items(): + if value is not None: + rollup = MetricRollup( + scope_type=RollupScopeType.SESSION, + scope_id=session_id, + metric_name=name, + metric_value=value, + computed_at=now, + ) + await self.rollup_repo.upsert(self.session, rollup) + rollups.append(rollup) + + logger.info( + "Computed session rollups", + session_id=str(session_id), + metric_count=len(rollups), + ) + + return rollups + + async def compute_variant_rollups(self, variant_id: UUID) -> List[MetricRollup]: + """Compute rollup metrics for a variant across all sessions. + + Args: + variant_id: Variant to compute rollups for + + Returns: + List of computed MetricRollup models + """ + # Get all sessions for this variant + result = await self.session.execute( + select(SessionModel).where(SessionModel.variant_id == str(variant_id)) + ) + sessions = list(result.scalars().all()) + + if not sessions: + self._handle_empty_window(RollupScopeType.VARIANT, str(variant_id)) + return [] + + # Get session-level rollups + session_rollups = await self.session.execute( + select(MetricRollupModel).where( + MetricRollupModel.scope_type == RollupScopeType.SESSION, + MetricRollupModel.scope_id.in_([s.session_id for s in sessions]), + ) + ) + rollup_models = list(session_rollups.scalars().all()) + + # Aggregate session-level metrics + latencies = [] + ttfts = [] + durations = [] + success_count = 0 + + for rm in rollup_models: + if rm.metric_name == "median_latency_ms" and rm.metric_value: + latencies.append(rm.metric_value) + elif rm.metric_name == "median_ttft_ms" and rm.metric_value: + ttfts.append(rm.metric_value) + + for s in sessions: + if s.ended_at and s.started_at: + duration = (s.ended_at - s.started_at).total_seconds() / 60.0 + durations.append(duration) + if s.status.value in ["completed"]: + success_count += 1 + + success_rate = success_count / len(sessions) if sessions else 0.0 + + metrics = { + "session_count": float(len(sessions)), + "session_success_rate": success_rate, + } + + if latencies: + metrics["median_latency_ms"] = self._median(latencies) + metrics["p95_latency_ms"] = self._percentile(latencies, 95) + + if ttfts: + metrics["median_ttft_ms"] = self._median(ttfts) + + if durations: + metrics["median_session_duration_minutes"] = self._median(durations) + + # Persist + rollups = [] + now = datetime.utcnow() + for name, value in metrics.items(): + if value is not None: + rollup = MetricRollup( + scope_type=RollupScopeType.VARIANT, + scope_id=variant_id, + metric_name=name, + metric_value=value, + computed_at=now, + ) + await self.rollup_repo.upsert(self.session, rollup) + rollups.append(rollup) + + logger.info( + "Computed variant rollups", + variant_id=str(variant_id), + metric_count=len(rollups), + ) + + return rollups + + async def compute_experiment_rollups(self, experiment_id: UUID) -> List[MetricRollup]: + """Compute rollup metrics for an experiment across all variants. + + Args: + experiment_id: Experiment to compute rollups for + + Returns: + List of computed MetricRollup models + """ + # Get all sessions for this experiment + result = await self.session.execute( + select(SessionModel).where(SessionModel.experiment_id == str(experiment_id)) + ) + sessions = list(result.scalars().all()) + + if not sessions: + self._handle_empty_window(RollupScopeType.EXPERIMENT, str(experiment_id)) + return [] + + # Get variant IDs + variant_ids = list(set(s.variant_id for s in sessions)) + unique_variants = len(variant_ids) + + # Get all requests for this experiment + req_result = await self.session.execute( + select(RequestModel).where(RequestModel.experiment_id == str(experiment_id)) + ) + requests = list(req_result.scalars().all()) + + # Aggregate + latencies = [r.latency_ms for r in requests if r.latency_ms is not None] + + metrics = { + "variant_count": float(unique_variants), + "total_session_count": float(len(sessions)), + "total_request_count": float(len(requests)), + } + + if latencies: + metrics["median_latency_ms"] = self._median(latencies) + metrics["p95_latency_ms"] = self._percentile(latencies, 95) + + # Persist + rollups = [] + now = datetime.utcnow() + for name, value in metrics.items(): + if value is not None: + rollup = MetricRollup( + scope_type=RollupScopeType.EXPERIMENT, + scope_id=experiment_id, + metric_name=name, + metric_value=value, + computed_at=now, + ) + await self.rollup_repo.upsert(self.session, rollup) + rollups.append(rollup) + + logger.info( + "Computed experiment rollups", + experiment_id=str(experiment_id), + metric_count=len(rollups), + ) + + return rollups diff --git a/src/reporting/__init__.py b/src/reporting/__init__.py new file mode 100644 index 0000000..5aa298d --- /dev/null +++ b/src/reporting/__init__.py @@ -0,0 +1 @@ +"""Reporting and comparison services.""" diff --git a/src/reporting/__pycache__/__init__.cpython-312.pyc b/src/reporting/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0dfe6e0813dee8f6f972631e832e805e7da7273f GIT binary patch literal 171 zcmX@j%ge<81TmlXW@-ZI#~=<2FhUuhIe?7m3@Hpz43&(UOjSxjsRj8(C7F5Y3W<3s z3d#Ao1&KwO#rb&(#i>PQnaQcedVZRWx7g$3Q}UDJ<5x0#1{w5AvbZQ&zX+yPKR!M) rFS8^*Uaz3?7Kcr4eoARhs$CHqP$kITVvwC5m>C%vZ!u^Tu>d&$JJT%T literal 0 HcmV?d00001 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/__pycache__/__init__.cpython-312.pyc b/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1cedebc630ca421986d64aaaeb18050b81bd6bd3 GIT binary patch literal 114 zcmX@j%ge<81TmlXW`gL)AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdC0LSLTvDtbAD@|* nSrQ+wS5SG2!zMRBr8Fniu80+=f)R*|L5z>gjEsy$%s>_Z-QE_+ literal 0 HcmV?d00001 diff --git a/tests/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc b/tests/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c830052984eebd2fbd2e72a978f9cc2ea2a9eee GIT binary patch literal 7087 zcmb^#X>1hNd1iKIcD=W+HF$lDZLTHQ4ipL)9AdISLcpA9!=}}0*csbnW_C7j#_^Gb zI*>L(O-m%Iga}tj)yol%l>V#wqY}ZYD)rBpR{bLLol|zpl`4mCvdv>TmeN{dpbW-e!iP-lHTcMM<>8sC0@UZ;!@i z7@E?!6i?d}Z^}z3zLXE%9<@sIr~HJ*ssSyS3TmNLNDHUJ1jnfnEt-mI)v0Q&CRIal zyjrW(rRoUmRqM4_Dn?+R+MumTtNw|qy(B` zDA_x_-yOojTdJDz-=OVpDexwB!y%>GSK$xJK{+Zh1I$|B9AKo-n`}ju*PuT^TinQqB5hB2 ztZ<((p3NMRvobL1$YPU9#NBS3GtL{7@)$c%^+Dc<+O+;(sFiC$;wuZ{bkrZ zns`n|pq@45F%!S{i-w%2vO>p^{+c4m=%_Bqs#W{Eh_WC7*a1bAZDi!YSWZTYCTC3> z5jrgz=bjUhWPb*~AtFU|i9yFIhjvnabp%Fee^noqO{8Q_>Z&@Db9D6~QyCC54u{vV z7te%|(Q|Uf*_`ABhmY!j=iV4JFH$rdMbp{H6g7;g%YN=Q9p;u1;M1j@r}EUh^m{Z7 z?_u|#FJgE3Xlk7P*fUBen2Gl1$b<{>YcMB*s_QwjtYpTW1?^9;mX{zD-Krmy%`{<7 zW9D>DSC!1T70$||i_jLAQFT}qSRKq|;E@Rr_jSKy$jIo{#6jKY&cHHt8{=ATNY9RU zj_T-~krOkr(f!=91D)Hp?d&Go3D~j&80yN6TmFTPQ46SC1CP-O%|EFJlhl2l3f9gC zHWdS#<^r4N1G|fX-E)CGlgW}leDQFJ^}TiYjlnGF?Q=t^mghJEpe{{$GgmXp6_uSb)L@CD=nc} zSuTlpJ(~NzL^(PQ9quaDv3;G@*x@T0iZdyqK%cf;9;mzbrapeYmL ziv7z|W{nHO>@YLl8+d}@cb;LWJo8`X!jt#>ALhdIF$=Q`oBn^<3~NKnNpX}xk3z&e1BaNV-u}Uu=L|JU1NW=& zoPg(F2U98CKpN}?&A1wSV^PDL11%pq{TotmuaXk0jT;)r6(U48+6 zq_L$9!FHir1Q1tcV~ZeWrHT$wRY8TIXJyx?+X^O{j8|1q_mrsUHq_JI-Q8n%)pPK4 zg0WaRGo)MmzzB{6{S&d{C;JW^?Zd7^RtB@_bMm;+d#oSaJD*WS!$@0Ec`Tq$;e!Pi0+D4&5J{6Lg79`A?f}mlYQ_ru(%G~#$*3C; z0%rL)G`5ckQsK6eZ$l|mQ;O8y_p)Jr@?bekMPpa;m-AOHynW%rx>C62ih5a{+FlTL z&Bb@mg?r|GJq2G6#%q_gspJPo-#X_Xr}Ilv**uqtBjR<^0DUWJd^k2**quJ zPjROxh?N)N36lsIoA~%SJ2i!PkyB+;Ch4eveLqAnC_`4PKKzvkczQ^I62-1ngbYz- z2sq>+$m1atNe~-N#Z;h>fgorO$wEd)NY0poj-ZMlib#~rAxPel8`}hkZJFoaudzA@4!6F^vnH2y%Sy4%6hC~zMQ^4g-E0V=#MNPK+gfUIv zD3DD(qpN9S1VLFLPlSlBA-k^Wv?9ljAL#FUA-%uv)PeL%Cy!WR2hn%@h4hOD&R9X` z`{=Rcfg=gts&Pcqpc&Uh&@b8!Mxw7nV+C=3ksd)RlwGe`0WqiGFit+IG+a_)a3V@< zfMD*FQJoX(Ps6x{1{`m04+6bcVcU&)?eRvRV_tYf5!dP`^&&j0@M4aPkb0Dv_4%3Z7tR9 zffTBSpFCXlyTU!U*%lD4Zn;yxW~%;r!_{w21?}*S=F|=`MtxE`^#w zxSF4QVL^EJZMM10Q-S(0c z@lt&Ae7v_9@4X-3BmCqcNb#I0+IgFe|J7f&tOMw@z8Qo{F`-o7QmSpd8{mU{nc_LV z+)Q!J4&EY$&Q1e3vmVE+Ji03iTU` zfsF-rHbVH&{|+yo$j4$0N`4bEW>^8 z6D7-e+TOY=CCES{SF^)7crN<^&4ai6aOIj~fdObNp{;Q7F~e1o7@W{X7NlGVE-9RH zfZUw@_k@jKny}>=lDP2X70oupcin1Bg;x$=tWF80!_kW2NE0IUZ^RYd}*q%~t)3r0j+KuzI+lsZ@z{_2Gy;%R$eEqIs{jO4cU19y#i}9WF z@x#UV;T4@`=@F+{3e-&YL&1q1BR>BZ`+D|s^v%8Weh+)I*9&lBi{(wHB|Vc)TfVa+ zifSrZ!wNo|8&6lh)~0jhUb+kJ8DzMD)w5kl9z}3}0apwz%JRB6KM0fXDI$xvL4@xS zEf!x_+u@rx$8$<{5MfV4O+=8nzy(Q{bhZm$NkFOz#@g2!YaV4GDT+Zdi)C4aoj#C_fu{}Ok^$k+ zV0oQ4A7aTz@h5kf^|4haHLTxEhTmC`4T%=z9GZp_k)%gYErORT*`fHyI{i@}2{ONE5R6fW!>~|TOZgU6#!JUTPAI5L0;5RM7 vcU15lOYrR#eESl-P{9ioe3=Q;wWVlN*#mD#5%`+Vy#Nu;yO@SFnAraSKVwz} literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_fixture_debug.cpython-312-pytest-9.0.2.pyc b/tests/__pycache__/test_fixture_debug.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0197a2d24c0df6a3a356bfd2a5c162e566807a15 GIT binary patch literal 1893 zcmZuyO>7%Q6rPz~du=C9)5I;UipmC&rUo@3=})9WE0B|@2xz5JK#GJ`o857;$^Mv` zaT6O`Ds2Uos03VSBtW7P5<(RhF7$+i#Epxi7PX8(9Js)VmK=bdc;ofP4(iCBd4Jz~ zZ{ECT|B%n;5UlC^x89!^q2CmfEYcx1Lm;jp7tJFVyTr%yq9?0gm@YAhELXWesocb85mSHFyIbeQ@68Y!mglQVRox@Zc@g()Jl>@8LqHKVEk za!n|ju2H8dyeKpy=2ABPhjMVbh@~#*nsDGcTgw3I5_~t0!KGeA7_QSm%e@vIQ=ugo z7vJdUlk*aC$y=cRh+Na&Nu^K|H}G;QvG1PAMP|_Z#375wA#XtZCTS2?+Y4qNlB-!A zHMJ%YDzEl(^@jE_`jjNGprEOy-T>)&)J!!}{A-a-Y?s?lmT(<>x{)el*HDNuxuM6> z9!&psd&M>MB$}HMI}-i!U_y1z4K4A>dMzIoeakY$ZpO`iLh5O# zQo6_gLz>6UZLgEtSBKDUe+LOq-XTwM`P0owZ+-tb0H&Zn-jjrBT z&FzpTSw&xItGJl2A1N(SXT|i&rmz=$%Ao^2=uW#VC^Ov!i&M^3J0=avsv#e*Prfqy z(roFR8QK9g=gyva+2n8fo}gA>a|L|;)a=<=Q`9Q-#8}(!*lDIluReG#j^Z$C3+f&> zWglIuQ04_R6dd}TcwrIOhei)pYLi87dZDSv!$mIBae_r6HPET;aJ|rR7e>O|a#!a=5mk4m!higwo%riA1&CvrAh?e$`K1}@Z8QT z;EQtL`8El)on@4M&`P2Ak&QUsxPF*qE)8GO!lcfVxLdSN)`oS88&oUAk>z_87#12|&CB+wbU~3+P@N753j59^Duo{oK8qd1g~1 zW9dyq3dY}v3>X*Yz+hzTiyhZ@+%_j~<)=2x$@SqY&Icp!m0RZIg}GLK>SsOM>VI_o zwe?f2+~{rnz}>z*ExmN*#QI#T@9{f&2|ho@e9YWZOiXU=09c>fQYRz`HuO^K!10^M zZXRqsp*;QXbFUAQeDGHq4IKWLkHDoa?w{Fh|Iyj*}k(xgIe6p zdSJ5^Y1q6LI$qSS{Fi7GAc*Z(z|6BjDF6Dng h```gc=G}d7@4JHw8@O8JK literal 0 HcmV?d00001 diff --git a/tests/__pycache__/test_fixture_debug2.cpython-312-pytest-9.0.2.pyc b/tests/__pycache__/test_fixture_debug2.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ac348a60a47f411454f0ace81c82ad9d055e803 GIT binary patch literal 3613 zcmc&0TWB0r^v+{e0NhvV4B|@K$;*HUqu(2@N~S)r$UvU$Bdc9lZs`SmWnk?Q)aWv)*>+m zuT#F`Db5isW7*P~mvmKv{~7pi&4&>>hB)xrM)MpcZT~ER;h4MFq_g=9!u)BVzr&vs zUJHd#mACo%5aVyn%-_e*n}W`tL^}T>*sltn;3~R<0v87oKN2$at ze^K;S12XaL{1mgpEHWIkIv9O^2ZLHO%lJ#k+!fH-+2UZ3Ei#=X$l1=i&spGb49n;F z3iM7jg5x-0N3xuYMXg;-bHHw!KDMLGE}LJ09ka|;!&S+mP2qmFU(>3Dlhv3l*|Agm z!K!S_=C1(@?K0e9%aGw6RHl_-yz`bb*Bj8^CUaH~^O8kCf)Z^q_)iJQ$VuwJovuZo(I1e7N(x=ypRKcj0b3-t3LUE%!!(X<~D;Hzeo} z8M=VOyJ(Bh{WgC)t3qq0`_B%+KBQgpX~64tT@rD-WH*&)bxA;`T}J$UcoRN9bN_#h z@onb?Z-?R@+-pahHA&ubO_Hn@Y-alvY9dwTGM_8UW^%^BaYApq%8q9!W-6Wme7{n;;~%Pw z<&r_Nd81&NmBd5~xnFVQ01d8DK3hm}mCpYCrSeG1RKQURj-^Z|>{>NbydwaO4r7(1 zW|{`ET%JR~1B-{Q(;GC!{mGAygQnr8c_IStG__!>858)M5?U4r-8kDUWIPL*RML@* zWkX+tGSHWT6PYzEeNM~eoKT5m3zidmq*QjFCK_>esya|qVw8wp%$GD`I8oKqW(~UH z#DPaCM})J-sS$OIup;&iFAMUvMn z6@t!b1Rm%Ae|RVy&ifqW*iy2@}Iy%vp{=ic5sPKnQ)rI5PFeE8=hv8TAm) z*u6m{9b`KcM7FVIf-?67k;5jv)|KtwpGSwDg-$Yu^4iIwIiwe{F=UqWrMY6Ed~m5q zPMIZ5H_V|UCngS#j%J2vV5U2&&6l8x=?igmuvB(-yJ7pWGY)bP?80@9%pTY{KcU}O z(2X$a-m}@&-{|W95MPZvuqE&V;Vr~>OTQsLDXmNb!(9U(bzJD!P)61hqYY)G*0rjy z-Sx(7T^U)KtS3gl6QlL`9ku6bkJn@U8{)oeU3W5t^#f0=?>oBQb!;vCURRB8(_)aAa~LPJ(IMf$taT<|b| zY>3L)F%MgBfj;?bQCU6hlNq?->hhL>qqgAl`v54rDtE8F(2)1m#Wamn9%#q|EQwc{>Rb13n)rqj(PUv;+^s3%bG3aeNdh`=xiNHbcMExa4YOWFbs&vAhh!cYw zLCfiLM!xJFL}FeeryNN$%LP4Kbo=`gv;`uF+(QK;59W(_F=sqP?gNyb%jPf)Hw2F3 szD0X}KwPcxZF1cWuKQ;`$z`?##P4z;au|gE27H?h{Qv*} literal 0 HcmV?d00001 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__pycache__/__init__.cpython-312.pyc b/tests/integration/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5d5128288942d8f9b3f3609078495c035491f796 GIT binary patch literal 126 zcmX@j%ge<81TmlXW`gL)AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdC0mkOTvDu`nOBmU zUX)mpnV+X0AD@|*SrQ+wS5SG2!zMRBr8Fniu80+=nh}VLL5z>gjEsy$%s>_ZZO$8~ literal 0 HcmV?d00001 diff --git a/tests/integration/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc b/tests/integration/__pycache__/conftest.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..684d633c2e5e9402507e2c308b3dbe17d2727cf1 GIT binary patch literal 7099 zcmds6U2Gf2wVqincd6yCM3K~=WlENmFtw=0v6{q5o65AC#IY5}cH4w)LD1ZlM43yn zyDQsLWka!ZZxA$q+lMACP_%jIg9|4>9{LmXp@8Ivdx5@4kWp^84H}>=+P*QiFXHQ` zd(P}G|03(Q_ui+w7SEiSIcH{e=Q}?${!K6_5qRDV{<`?vZbJTq7551`%$+fgkhh6U zvP7mbSE5;tl^&HZaFnRttU&D^F)OkjU)Beur_`lNS&8-WCBGWT2Gn3SsD`p3*5)mR z)krqNc%js-_GEh)FP5TeZ?;#BWn*eQ8&?zAgxZ(wQ~R_1>Ogit9n20=!VyJWe8OF7 zvy^@BcHN+MU6x)WH=M5Qo(IVL6-f!m+#J^rl5?CKc#Uu93_n082pWE+caE1sEz)~; zm5zaQxJ5d-t8@aSBQ4S+yGr+gba#vN=q}PdO0OJMVzB=BoKNm;(b~6*R!m9AaV0J% zls-qhe=Z>RwP=qmbE*E7iIdBQq8rJ%;zeUgQ}pCqMN1aThBB|^jbf#o#9lr9ciffP z6tzM#`f|DOtWs8>zoMl)R)DpfR&*W2t)4vg=XATO=Fcgb6)k8=-cWLOmr|ZDmK7LE zI&Hudg_D&^32OY|yso5tR`8@&IbW0&?L=o_yjF(6pmnZTQtZaSSMpjhUpDMo z_~^x|q7_xeZ3v#p>*q3gO?IT6wK=s@$~Twbo$Olo3B}Ngg;SMMX{l;AiO$+#Ohh`k+;i_JxEM+a2hzDs14yhY!p6v{>M!E%%= z)Au|VXo_1I&9HBpRL;X|Crgz|mF*n+#Lk}0rg%$a4aJHTn^%k+6V9P1-1dU1Eq9)v*i63roOBznm(oG=PUYD0rq7|UskINmGbiBg^G4gujUJi zK9za#=;VO|k54hgOtoMLRLXPMnyxNeQgad701O|8pFRoXZ{*$@xhs%BwB~o zkyXwG+zxNyS@pcf!#e>8|GU}X3q|cwTd6yCWA)DW0nmCMCvScaesl19c^`R{us47& z{uX<~$BCEd^a6yYSHPesd2`{S=M{2+g7gXa9r_0}jx{u%WCkLx{JUcy%)7~XBYzhB zOVY9JVB7;2@L60*sPZl_Y?= ztQ44!Eb2*3S<)4m*&vg!{oj5z*f~Y17R&QYwK->FP5DysJdUeXmgW~KO9qo;=EAgB zhV*ZiV0E9#7b`Y*XliQekiGIlR(D}RDV)pUl5#MHl9#`PPT5&qF0ak&R#z2#Y@w9b zb&FRD3zgXvX9=vHvm(mHYEe^im2$37QPrZ6k~9JKRYN1Pyk`~g)|O9OD(BF#YY1w~ zrz%Ds*KbL#YcG{FbZ{D)ktNDSy^4N>w*=eWTRbkvinc|E919N`q%nlqGk$#yqq-%@ zXWbB@eFhqaQRLr%*iIruLL;Vc!i=Y`PjAK^{X@@?8SJ^+;L*Bb~`P$_-F5j=mLE+wJc^>-a2k+}Yfp@v`NahLpF3oVldmiB4 zpUn6?>w9R17uJV)7Rf0e&M)Gkb)M2#Rk+h0KU&qaS59#%N|QzcHRb z{%M-|9KZ2-0r;Cdg>h~Q6nUSJ84z!VXr?=GGb|z>_hI{9);l6(9usdqL^F>BZayp` zKZU*Q>lYdU4pxo3d=-9oegR}>-wNK;^*R)Mg~Nk*<5{IQoH1F*+a)x9+MDbgBAX-b zpg~U)@FRQS=VRVMSV=hE0nDaaQj%3C=SZ&v(eNhS`ijJ=V3~#jsZAM;v|23WR50)S zyrP4p@nx_)OJX89#z#Qgs1z!t95{?Q2#pYkwJvyf%afO7%R5(E(ibdIQ|2JwT1av1 z8(ThiP943xYY1U`fjPHRn}*J3;HQ5Bh#N8^d(7aF8Qp94PXc@+!rF6niFm`W&%QeQ zohNSbL#EVQlLoe=fo*B1#t&^eEkj$<(Dz^7mL_Wa#3nys`lD+v*1g2L=bp(=+`lsd zGhBIWrl0Fr&~OMREYiV?bOWiOwS1a1?iNv0pK{}VXz|1_LBH9FzsUFmO!6NFsn zz>;1GGbA-zR>seO2TC>@c0Q58ZX;|gzpUs5tysl81c8BRfic@a(d~euIhU3J zb@VKH7Kn=n|9}}BHhZ7C>-G4BwG4(Z+qXP+i%;wZgVWp6NR1!a6$9Dt4sck7)1;l$NAy}2rc$lNNoi0-1}0SDoBvXCEAe(*oVV{c+&JS@oos{!0@!BT{@u(VSClQk9V6gC zk=L!B4qp!fvnfHtwQ9$Z97oa-LwJ^qG&jW(ow+*_^pJxar&pVWvG7JjF9C6rbjYV6 zj`Z%U^-gW|PMN()bMVpH;E}DtBao}JfD&*$$^Sf{jKPFgW_&Ygx-QX~NBDJrAM*R? z%zl1-OhkS^1wFt~fm~!8=~`jcj2kTmo=YMiJZAH)bX9Ca zc%rq>=35ZncE7W~jKJQvg$q%A>bV^|WDnxV+KyfK>!TCuZAxv-$-HE?tlO-VCb z(Y2E>zxEOm+zag#lG8}eAVC|k#FIy7k32v7tR*ZNg>vOWr_uOqqruqO1Q_#a&Cx9$ z-&jY);4v2;`>5+$lJh~duYtt3;irEW2(zUB=po*NAMzuA_9x8vL$&zyR(#rwj?|+2 zx1#&a*mx~=U@LZ@VOwT+$UbYD(ZRQ$`R+5@(XqQhF(9lRfupe#xDVdq`*!o$XSSu$ z8b7+pkDC7Q+6%7F9=)$Kb34o(ewc3X^zazJ;q60y4?R4=Zw!mbPf(z#aZAkQ~m(OXqm{2qfZD~!pput^-)zxUigOc4Qfieb3xYMu% zo7H;ZBUys6jQ1-V1`sO_cO>}|T#Bg6Y2~5;W5cyYvBEAU>?;bi6f4{!hPNa3b;nQ# zM&z?;=OP6bXWxDFeOkmp5_%CkF=sxOWnwH8GhpEjorZ>OiOyX+yZ-F(x3-tU7;Ga6 z(eP7p2Pm=O;(HoqTPi7E(9qN3AzJ?t(7K0G`a81!kA%ENt_FWkLLZWmzmk`?$jcv+ ziNBD^Pq;41eL{fTK~nD{Tx2c&`tYm6-znar$&WdncHi+3ZlsP}Jw!MkcGkH)bnyG> z>qGAwo2jR^2A;l6paBIVtgHS-_@5m|Qlh=@l1!jS?F*WQ>i^v)_y8TufIQ-&H-hFYc!)2JC4 zs(YaPC^lO6LRlBcp3l}rAwIn*MN@z literal 0 HcmV?d00001 diff --git a/tests/integration/__pycache__/conftest.cpython-312.pyc b/tests/integration/__pycache__/conftest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..880752a56096555389170c6adab6910d4411ec39 GIT binary patch literal 8617 zcmd5>TWlLwdOl}3oEhFF@ghp1BubX;$gw5avV4!Tl<1AGir4mb6ZXQ=?&CL!=?y7->v5MViv?h&%0xc+%d8H|>k~(#?@((6dDR9FghxE@Oh#JYik1 z-KFK{g#Cx)u3i;sePs(rO$%pDi?&yc;F5V+N^oOb3-mq4B^rLnS4IiEqD>>TaLEm0 ze4?pFyM06LM$mTGXm@O=?FMa6jdtgT+8)sM)@TPe(DumWA;R#rgire*V{h5<@8ieR&&9zkZCCzbK$zDz-WaU&gA*VF=(U_8f zuAp=*nUc$e_CJa#$yg>=esUeTIw32`v}_bO&c)PABQYhRYwI)9pU%i1Ov3AxpBhie zIVBlClTD>2Cs@&RK>Z+<%*k)YlG(C!pufNWKw3^`m8k=osJ|D?*sR906LMS!JIjEo zWE0%0S0On;C=fTN6N8X(tgX8Y5RaSdD&WzmnkGx|BDkdKS6)}v&Keqp*od%5ZE z5w_kz`7$g@FqO?tuusgEU;ntHy}Tx}f@D_nUXXK9rW{4(=tMS^j8AFKjC`f0STn~{ zS@=lsu~19JHw%8#&cM=N1E2nQHZz8$eG^lfwc4%H1`Y65cR=!=K8s&S@;}w!>$E-S1-lo-Dp{Nlbc|7KQs~Y8TYyAFU)2#OK0e~#ptP;=a@Ce z=W%JYeV{dWWSZ+QqgW1z0Wik^*3#lN2G$}3TV%V`}HEd)mrAY|| zSQrCIOeEC_Bps&-87hF+Ag z=NI$W=C9ql_6^PmiwEn91!8XdM*Ri6%8hN|Zn{86wh3RDK>BjW$bf0FosMi57CU&9 zyKPXsxPy-L@r!-!Am6jmuvxfgNA-KI9bud4-XR(mg?n%DAm68`f8S2S7XH4Y8RZ@t z4)XUm3n*`=(C7U=it>OE=EVDZ=*WKi{k+(V06Z^i4C}nJjY7O$$ ziKHS&vzcf-n@%TlqrIFWz?&$z)%BVfQZ>gKXF+L#Diz;2Xh7Qv(AK+G`k-PDY8-~d zz_xd5QEGhXYAd#MJQI1ZFn8*Soiw!1U;pI#ovlTu_pclX#?JZcx30hV%j+-ZucP!E z^$>J%qa);@cj!UkZ%rWlo$$8l?`G~A3T3|LM!^xX=pk>`*C`?<~qmrG>$>2z4au>l4CgagGc!04BvI@>7 zpNXYq&B|1wOm2YIxokX}imH>!SS&7sXJa#Zj3yHb9$Az@Om;&u3Qnf?yqt+&OvjW< z08T~jOQWx;Dg&s4o496DbBZ#=q*x*Wpk@%(L`5D0fB7Q77s1FHXp%A3TI9OQFCR5( z6@=Uf-0DY=F!=sA*th=2uE0Z2^KI!)_E+1NVv{TW%g+gE8R4F=bd?Zqm@7I0MR#Yh zWydp%xj~paTCx+f>*LXnMnB!R!Uu|0Pr=%jx3(=?+Y5aA65q~B0(ooTuism?ZYl6v zmiR42n|tovl9`yn(DGZpQF~w(H$uM9A-d?Np?&;fiy!5kG<1Mp+$ExXfI_+kO}bMKxg`vyJ$4v`c}YheQmJUx2OlZZMtJE*e{k3EC_l$u~abc zTzit4wlk>f+R9FNL#-(^dhY0%cTXQ1i4N<&gVtCr3!Od|J#plH%~dTr^=|mc$zF3A z;)rg|jvh*MQb~bFa9Oj(CX&%h@)UcpFqos*D`-vn$cRszXN4%(i!zJ}f>jAa0+dnE z@G59{6~utD7n38H?89U~CI=ufn1Py_sQ0~=7(SgCbrVqDfhOnRR(}jhg&#wY!Fy@z zxP7hI*ja4f2^=W;-z@sJKC=ly>vKD?wFAso9YUa^*qXt z?P9E`D*c=?Y)@}K2m3^@TE2eB0`><-RmXs|W=qIwTuDygv9W^XWAITF+#1VRK7rEu zSVm3F1pThXaifmo>VV29Xr|#-KY*kH;hm4IEf0Myx3?{AJGjz%sL*;m-+Fx6ccSR% zD0&Wp*J2aqM!*HuP5R&p-@Fk@&n{cL3VhcR-&M3Z=Z+gF?J`j64(+FlUK$$W7kz$| z`)Ft{zt}ILyq7|{K1yG}oc}N=O$ZDB|9uw!TAbz}h7^oL5UB!F$p|Y=TNp@DLpctM zHf>?xypWBhrrk_ba}aTXV_G7KkOP0jj>=K&t^!IP^))A|f)k>vR?vAG`c<5mFiZPA zl-5TlYHD0Az80JfIyfuiD&GgaS-4ddk_tBaA6tA6z5d(&J0GsJ3>I3xlW+OXvNu$8 z`0>AaJ2)IxVQ#qUaO_&)eH)?k%(69D;DbwiuxNA49W&4wG|=e@9iWRo8rs7zHv3WT zXU@lfi1I#gJ}7gP*lrLv!BmY4{W4}vPrvMXm|p2$n-V4n)~w@GC-@m|yb`c61|ej@ zY)OV$gg%v5>=pS`{XU+1$+ z)4sL%zg!NnR?si%F<0Gh(W`ZbojDf0eAx|QZnLfIYs6B?n5v+k&}-JLdLx4Z%)wUP zgG#1>5ftWS`~XWZf>y>p_Qw>I_fe{2fw}C7qjSwLqhM=I)Q3{h*@VNSah!DOw9u=S zKZa^_H`Jd(QYM9ynEM_(TOYc8Ke>WFhkt9qKbZFq7X3ZN&Y?o*(R}AoV1?iM+)8ZT zTiaG`W{(f}0ltT4Q^C`p_w*M%!D8FaLR&cB7JlZnF(<@tnD^h)2^l8CTj)KG4tMkS zOn#Kx=x`5zFCe1aLm_3`5PLO7`nc|J!y6J(VbSB4S!7Ogv*n+U@Mk5{tO%^aQ!Tr4 z==QM-POWC()H)$fU=*6Hx@*QXzE1FPh9Ae%e%%G0kAAit_taIvqrtvE)aWz9xbNjr zXT)**)v4|>*a-GQKhMufz-$a}*IhG`F>l7~_PT0#H2MOb3NLwJoiFQGdt9f5RL5+w zp4mpUrhFLZH1Vwd%Z)=9&gNpdNfnRj5SWy6@XHEN`vgp*OhC35F-nV?RqqTby^e+^ zsH`v0%f*zOoQTFC4%ASl`JZBse}+jE6SRpnt{CHbWhmv(p;i-59~liF8+}I;CUfyj z_Da1iu#{~9dSO-gm#xCu!89J{ts=OYad_E4=dL^d9cYQhBjCLon`SNk4V=evgtp_dwKHSBhvHlWXE&PLb>Mzl2uGft;9RHL>eL5&Na;WKJNIaWTvrJtIOkTX$5&|A6V653u)4d1QmKb<61FWt@4h>Szuf*SerZ=Yzv;+R z7{rH)4XmoVKhHR$pVnd*0g8XZ5cgYMntWNtt zNkmB^Leos{Pp($0m8g{zi;|Z6&}tv4q*%5rL6Rd|52G{UC4B1yJB4|<6w?6lIIE|iFX~eKe29-=Y>3u_mr@j{6YSk*X=R#USI(&kwV;4yyPk2@Yz2$!ncvJq z%}M$KR66y<>c4_=ok=k<7DgCGHs(xmvt(B6<_t6ITbss|d&H@C|!svbcmD2xZwME;QJ1I zS0$cdPBAQulgi=)yo@)lI90Pu*)ePUYN-*C_-j?&M#ijQXx81ROL9t_#NTt>)vYUB z**WXA>g^Cf}JG%ICVQMzsQhb#+2EAw~tv6StWn~dky z-LlNF(7x5{R<`0qC{ni*wCI>3DXkf*@1{6;|hAmsX4(C#VCpcj{^xsNy#d)nusWvIRVW% z4jsC~mL{IkRMY!GdxvO0pe^X-=+latuf4WZyNoC(BOu8gC=l#4S3ynY@|QFhohD7R zUQ!dEpD*In#1pdCPKO$pkDSbB=M!>D^TFfzg+e-`reRQBnLJE}qHT&RWZvUw8lTDJ zFC;SYTmpGfTuDr)Fk1^8A>#r}=p|Xs%UWj~=v6kM##3+4XA(Jr^fJPejCd3AYMheF zOv`@kaX?9A=QAk-=QR#$%xiAF-((^*FLsTN{fIp=W>4&=3EJ|wJu#^uREMqWact-{ zm`G(Po0!QfLrM6t4JpO!{Kb5(ICv>9&nfc>_*DXy$u0BOL_)PnX)GAT`160-BK5c zG0lE})tUCD0@lme;?7!dXH}TE9RyJg?z|nW1t%H;ewOqMnlfcgn7|g%qHd{+#F%D3 z!0JqUQvqwzB7Mimt?{Z5!FL?51tSdsKTCQBO_{PLM6gA)s9WkHF{arM1gOrmHx)2h z@i6TZpDFm$zI-Hba+tl}#-1GH?zf-lhr}ArihoqXS^pOPs9%h;-gDl?KRaB6hw7Y^ z=1|$hLpeJSWf|bExOOyMWmaNs+;vTv3NLXLSD7!nfTOz0ZskX+$$ndLT8Rg4%1We! zjib74h}44FuHmRw{kXa{T^92Mj%r!9;Jub5a8&m*Y)y_T2{#>tZ3>b{63c?lJw0lV z(Ggost=O)|;GD)CxF-j^(~YAzyVvwMu83uKStPvETlRt1<~@os&$ibDk zupIa|j2uC+2gP0#`%ny`h@!w3i1_6p)Df-aVH6`MMo}CDVaMQ6dB2%Mfp|Y|GiB^aD1GE&Iyc+EbP&Mwk((pPDKf#% zAI|+??#7;5Gxb2UAw;V}bj1aD+6ax*LL=|*sUCi%9(uLW_9{T?mT*-FFBEPZtp>vl z0Y6JLO%tZ9VJ=P8Ep<@{-#BX8mX0v7<~amK}@u*zQX@A!U@k21+6-HmjvnvG8vJ=eJ z_X(y2OWO?({$eUQ7f!vMR?wkNg_C(%P9;_J-zl1rM~aDN@*lxkaQZw%tsQCfRy%AM zhAQHhi2`sJKtJ3JY$h2;m=drN^Uk*5J+S2GbQ0qDH{%rMy zbM>C{jrQ{kuI1kTs?fji){U-(KWzy8@L8fsnlNPzb7`t>sf$AYjV{y1RKOVUVN1Bt zH&yGK`e0A>*thC^uQ$41Ul5o32dcurLg~h9)!;xwz|Rs*(}XE&m`hW2OI;KOZoFpN zn92iTV5MD$rHqR~2;N9?sm)EGtxPzn1W6%Msn zD~vskA@octrwMu_Rk_2N4)BP47@9;^czlg3JW1p3laU}H4;n8gax*Dg9Gq~7fbJ9O zeU<+h>e{H+8}YSb4;q`MKJT}%DkKng$M*p7aP2|o343Bn!C$@Iar?D)Z$OQTw=(+ahdz%)H{3_JH+!tx|jn(?bmIsF3XKyRj@c!jcKlr;n zfraBMBGcJ(eeb(_Z_M6)uHG@)@Qv1dqu>a3Y+mVLHg9e8?yL3gyB+&rYh&tcZR%{j z_gtg%+=B3%V8^w|zo=A&?YCGE3zb`JEx5fQ;Ae@ZX~L8>Vf$5urs|ezK7Jr@o0MM~{Xe z@y{X)at&nQ&i^~`UHyC5P2Wgtw$*8ZyHEjM03T*V7M5_VyXnGn2W1YTxcc5boD5hG z0_@PS#$zH;Tyw#<8oPZUf4m@XlOnyU~KRSOT>0us@E_ z@+a`t6k0G~M6On65a8qjLW>8>*q}wUB6+hd7E)te;p9TEZOLyDLsiX~vatXHv{*gD z&|1^=WC>W2@l&_d$b6k4R<6{hSuYMs6DsXL6~Wi$R-@t6I|UrTLL`-aEYK7TF7SH&jk1*qk27{${Maj5`5pNB19 zS%Fh+bOSz4&cO+(oJ6z;?06qOv~jdQOt-oL?I{cs0DT==l40FoWW=$gzD9w{Fi17_ z!*})TAn0mbX5N5bdW9{svyj((Neq(P_t@(!%Utc|m?BHY$RSUI<>?4Z!`KeZ5gpc8 zSwS*xM4UYbXHpQHn3oTi4m|^QaS-DApdO^V6{3qJj z`$6^u@UgZ7p#PL-Px!f?x{mXZ_!mD5@^nO$PvURo6pEKod=tfK6tAFo6$OdGNT|Jq z;tUEB@0djGydh$sD435Cn4L`#m@a-mkF5-tv6X=*Vk>?8%PcWNMUSqaN}I!$B0Z#w znqVBDAscI2Z=8mYqV(Sm2V~&X(Ipv9J82#>3lF=HJnEp3w@Zkb`Q`CIj=-3JIe;ag>F6 zSDTs&Nf(TYflbl~7ai%fqpt}EX<+TJ{5`P4)m_2We8tJK?1#+IuW>fs@&B6H{Sg!T mcV_56n8}Zrmp@`gK5?C7*&V-SP<|3~jI%GZzhh96q5fY&Gi~7j literal 0 HcmV?d00001 diff --git a/tests/integration/__pycache__/test_request_repository.cpython-312.pyc b/tests/integration/__pycache__/test_request_repository.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b63c1d7f75cdbad09a79c694628b8feebf929fc GIT binary patch literal 5885 zcmd5=U2NOd6~2^6ijrm1mTW1q<3x7S*0N*Sa-1LcC#b!y#hTV@;uKib6@nr%j%@un zB!yI(-F9ivsaSv}7*GdTH{{VpGN6yeejc2sVSSO}4pO-mE3gh*_N3Ms+U05IT#~XR z+gOVZ*e<}s!^?B-y}b83fBc}m-9@17`{(bHA9ND(7ksc9YciWfU{;AlqC{dOTZ)M? zaJQ#yXP)+$F8b_9wWZxrH|2#?d%7dq z!4MmHib&4OL~`A<8M8)3+Sdks?s{Km-WF;vgua*2V}iWn zv@w|8ZZdOPT+68-*=2T|uFrz%X6Lh67&&2lfI;o%!{#4=StT-oy(3YZ#6;~fyFmB8 zQJjzkck?qoNjBMW&L-I}vp3C`QQ;eFvkltLZ>-HVXuI-!$f0}Bz-F7@{T)T7r%~v` z&gSDtnD4^6OFA1_ZduwfyeM=(mI7Y4<`IY@56A}ss=FVqxDJe*I zQ?6rO%_S1Bqvvv|R6bm2PsCHHQM8EB@$txgZhlD{ot&q0JpHrB?@GYk{XGykJ(qb9 zv81GnsiY>SQt23~V+WT;9D|G&S2S6Q#WmfP0t3n<^0Bn4yOWuvoEFovi*iQQ+p{@s z>v1;ZRPa0%+*-wh1dmPUb6O&ky{PlX2k0)vFff=-@uHLuNQl>2RZg8#+EJxLji;AV zvIQBPMZ?w|rlQgfGAa)wNZb8G)Dw&)w|v7G_Qb4-8r;HCUiZ>>#$qgy!b4}aLpr>J zARe5070BPoSFe$K9i($$LmVu!pLqgR{{t2Oc-cQ*-8)kC1U~lfUw!r3s~i5Is=vP) z*jw%Gzb6V_SJ6@HCjP$F#cPW`F7TJ%YM_eNoqlL+k=74)?z_yqXjVptPZi#(v~5*G+14e4q9E^!glbUjTV+? z!G0#;+DmPSD?DV{(79wrs$0<6e6R)n;etr*D3?h>KtW4VL|Edxb_327tq8u^5Z?;* zUg*|+^RgB@n~${&OJNQwdqIW*TfV3d@9#6cz4cHw8?A~**6sRsDdf!@nbMgA{jrom^a7750gm8@q_*Q(8vM=)9 zaOuSNH+;`kI-jEmE{VaC5CmGsZfR)hjyP2krpn?}t(`c>z5?~GQ_OvJ0G4v;`1E1s z*Ua=7`)p(BgkA6HLq5U)-5I`}-#S?QZ{RC&IV%M027i86`18xW7 zA#O*Q2aExBKzhirJIHxMOLt|mTFl^8^c#i~4;ql)M>uat!T;wymey}?S~RJm{J{o&G+KiKGd zq0;k0(Nz@#*PkzogC$`QXsye%1kdx#S2cn7qF@k*VTqTH&+uRnGXx9*I53DA8yLiX zPCrhp80YO)21Nhgzh3+yQf} z161mB82x}UX{{pxOud=Gh-_YguK%B8JI_I4<7vR`shq4NWwp?Yea+c2L>zn-rV*ma z@!`?D4%cSy*K~XkML`nV6Xd01gsiX{#(vr$gU0b%nq*sMe@=b@mBuZ$9@)9wqVzJ? z;HJ3d$bMyt#pu3sPQByYyN$KNYlt=4mU-)on&LbM?(Y$}{L}^Vg$GR+2q7>C8LbuS zjg5{6&CEA^7vW;&%x2#xjr4{q1TTikiOwnU%)H!?&(a{@Va8WeQMae$%+7vKXOc#I z7tBml-R|&=Y_xgE!h_opVV>!W6sJG-^;i2smA=Vx-(+=g z>^){pEd>u(ef<#P{T;=pYA)jSuZ~RLdTSoiyRQ;BR1O?k zi@v|FGIhE1xxw5K2TH;~SsbXfk@mwF-Mi0#Gdl>Yx-{*a zX=B#gm>CDZ?tTFI2s7hk*C%|y-*z%Un{bYG8^!s;*TnfwwV+k$0p{yjDPVqwnh>kT zWfoEav!a9yb2r>J;=L&=rI~CUrLcapVfL$G?noQ0I8BQzK<>KhJCMHRuo*$##SXd2SOls$o z@S6qxngE}XvKPbmJ7|uO%#0cx)M!F%OM0yIz%pj<;s@iO7ARj875=OkMP-csixK3E zM7e~yEp*{%mKX*%9|mF+D9hv}_^1_TnOUGB%|s$tG{4QPG7P!W!;(BhKggy$4sx?0 z#>l*Fx-C4eGm45on1W`e;e2#nhD>KkIZ=4*KbRY?IA-p^qp2)xfqEibSI25l#e|+( zVaaoHedzsDS0feg!Ls*YY3TSLy~o#w{-q+wE(Q9Skju!a@LQDtlC64~`WKIBN=x51`69fj$gytc>5%U;jh098jk#1AT6Hfvku=|**CKr z9s7%RC@jGFY!$_aps=X=d+#|ppR4GqxwmSIg^i9#MTnGy2)GaQIpgKPc-8F#>33dO zJN04zSlK^T?b-9r)LP`jzR_~eXw5--Bb$Wnb$vOl_Eze1{kRP>~JxJ97;i-l$|<`15FXWodXds>vFv(~^^&l?#g-KOTP z(}hQkoc4q=2Xg2zR1~Y(8HV|YjQxqU|CtPZLVSNEV}B#ZKOxV2LLy)CPczJc&k52m Qqqd{WGt3tRDcz%g123IkdH?_b literal 0 HcmV?d00001 diff --git a/tests/integration/__pycache__/test_rollups.cpython-312-pytest-9.0.2.pyc b/tests/integration/__pycache__/test_rollups.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d4a5719c4e6352ad73ba9b6c63dcfac545e150f GIT binary patch literal 14387 zcmeHOeQ*>HfXfPyf=;;OF34`uWeY$6_4!@0ehh%V4fw z0p<-(MJcDG6TIdg8_SBDfe8-C z6NSuSNtQuh!mW7^NvFm@tJYvJhn2K4CTqd{^Z|#cel4uu4`+&_(vgW#Noz1P=+f}q zpizB)5gw@pjbfUDr?%O(9g;?ia#ksl2`Fi{mmHSJ>1;;I=pCg~r;)gR4wyGMi5uc! zlwov-oRVvV&J4Ra$BhK`eCgUQTW#NjJK@oqjzG^Dud4Tu zJQ>G4eT-zim)GZ{(^AeHD$Oq*OBqEO_IB7RZ1*ehcU=@X?&}y%o3!J%Lxr ztGY&_I&Y^cuX@?{@(L#gUxNPpzT*w&cinF2zjKO7eN3w47x_{^^{auCyP(}cH7NhLvW&@%#2IJZ#RfHaoEJk_=eXmdv7(7# zWhIjt`5NP_I~(ECoZN2HzuJ;uoYKiS)<$Blv4)je_q%Hpcrh$CK5&a#@82S;w}@S` zPD?CFXhEY*Ydu*%d%s2G0!&Vfyy=>Rucy=?Hi^w@LzbthkkZZi!lr2*%d_rkh-qT? zmw7SxB46^uU$7KVdYCL5tx{MGk8EVQ)?MY5O)Pc)S$MD6UTRbucX6*pGLB&`<9Hzi zZxvA^Bb%94>;AgqT%(ICHI*V_OQ~6HR+~<9CBN#Ie_(stxFydxu~qfMJZ-}o);zV< zsW&FgtY^gNLz_KKOy{Dr$CQNG6BT1`I_UVei0$GMwZ$0i)(0N#R`y(WH%7Y|zWI)? zdb0Fzdb0DYo_shxIT7sR&O6WjA*Ig&b)0KKESzzaBIDfK&T&4`HJLbkqBxcl1@Z}zGD>bDNf5qX;I)VW^ehzj2||&{pbkaXXVSUc`rGc|QDr^iO>aJ1 z0LAjJ^kw8*{h#y#_lbxEx4ML&`HhFBvZ5BwWfdux%coEcX1MbdQ2J#hO%zE?r4`Me z1BhG5Or-JvQ?mtVcS8GVKTb2ZCS=W-lM0$we=+%jOLKxOl8s0AWN-9AqtY3c(HN4$WJ1&0Dkv zc`TCwT&Zi;0umubVmzipOCE!cIgwA3laNKEBwjp*VGHESpoZFp8dB5k_CuVkwzUPHZqq4d;r- z(mC2qnlB}%k4u=Xg$8K%gE6xkdqQhYfpwHmD=F#4(OkMf%_ai7A*Y3gB_(xif;CWc z$_mi}w%$*)X)amH9VZA?wUC_7kLDyMPP?l49GsCc*yf6dMa3POi%>cfQ zJ}Y|di7ocTW<8-dw$GmUJuRU39UUkw%)SnzLp7cdL=bjm8|?M((7aV{1KD`p>-K@3 zqtI}9EIjfgKAn%c9_ z!ZkdSv#IN>gt&*jo`=G8MJm8OkT-uc+FYOJpu>w70u-S$j* z+cTBep=#5iDeuSOrt^K@Q|E+b(`g`6>U6psURD+GcTG?036qwEW#?o)Rk2vJ_=>RX ze4km&*sG{Y&jo?$v?*>fpzDT*i>|Mm0g_)pP z%viG+n5)9Ng_T^xrdtaF*L;}2wGf1U`{FZ!i}mxjpmaE%K%UlPl01gAgT~YFZA#b`8 z;2n$$40)f;wDHy$@{fZdAF*T^07tWBq?zSg_xqbrV93|#S8yMsmuaX4)IfcQg90u{ z{Tf-qIO}e63Dk9|u$GGgGxTd)QwoaS61W7^hB3YrQbUWHWj-|oW|<#rSZ0|RcxdKU zkhOVHbIX(jb1SfjVc5X5ib2CH3yTfl1PL2vS>pqnWsOV|yMtNg1p_>Uy*QGz#)fgF zkQfHzIHX2Sg2`HHQk&$rZLN(nIqNPqg2~H^w9fBPADiw8+uPMM(j%OT@@le4YvJ zNgpINpHvvjOK^~2NB57RAg;#*7I#hu%B>72qqjemE{GI#6LhtM4FCqf-UEZ=DHPv< zx49(|a9E7V-^DJ0VT> zWAPx8BS`)L$pIjC7`%}@huPmk@=Q(HCObslCgc362a3rM*6X0KlmYKM=fY(2UNE-4 zJa&cr58!#Xq1b;5$M3?hT1(%Q@8h;*GrnK6tv&nH)WDQ{KQs$~_C~-%;c2KYJdJSC z?i+13cp7T0hSrxu>t_qquqbsV>ZRO}Tz=R(JJF3Bz<>1N> zg5mR_YGb0@n5YET&IgxW_qk&L5Zz7wDbGTPTe@OSSUz=n`p8^(c~!vQH9f5-Oj^cV zJyo&TtHSc>Bc_bWfbd7*)@pb~ISlH%XCxqV;T1DdIow?p@ORC~&{HNY3*Gn>l&Dzj zRVu7!UqN-IyvcxwiR65H0m%vQf?vqnVEA4hNbFt8zuV34ZS}mnt_S!_9sJ&y>r&@{ z2NIW8@OwL4msjrI1c`r&@kpV3DLQV zfj@fwJI`0$v9de1;8^2c|4~c)GdJ9?J2_7~%~?3YbDp4GY?XT*6@xn!`_us(d5JwD z1B`j0(*UpE#$&lCexbYbqsqjy*@9Rce;U@XT(&T5Kbw=VI6h9x<0lM(Bwdv5z}a4S zM~buGflpY+*$Qy_Wjj)Gs?HJCC#)FmfJD)OQggB05wIU`N4h1~FjsP`F4awuiAVJS z`ok8`YXmWW%_TZi46Psu&O_i^VgQhXsy z_8Rzt4z=ay9M38zU{jA9o*X$2C+!*O_*f1+o`d8VJPr10>b3(XA=`<;ZAU#lBn{=( z2A>Gnc2bHh9&mq5_-c1Vy3sv?+HsE~MI`3lCZly`ZeIG1wTA9kbW$^q6}XR-F;b9I zdamI8>*sDV4^a@kPWUdIE3tlw5Z(qkL`SRMGOZd0=oiW$jPp$*@t1lTk z#kj2R^Z(bEd^$}4*ekG<>tC{g!s>j<VqW_A2I)Fi3Pw?F+ISGu6OKUx}6m)k{7zB3%EdjUCSp$~* zh7{Zlumqg`EPTxG3|MN!{ooJ{^Ex_;Cs~urF-MPJ(l?DbM3~_b*D>it`ZVCt`@y8G z;2^ApNeph3oQesNAT-Cg>cuYsY^n!#AR6i|Cv%*YTmZQOOpA2~SE9@2N_5=;uppAD z9v~`tL4@i(45FOMJ5m?PbBzY{S5D5#1k(>aM2vau* zL6k+E9t2kl>MVj&Qu-xOO%J>i1ZvS8x;o(D^qV8Eq4FXUbXky5B-Dd(3OE_(w+Z#) zaUJYM>8tBaN$H2MatblK%FUXG9a?x=9>C$%V;IMJmB5c&cEy9cST+A z`gevPQ6Kkhhck%Z-ld7>{?Ch}+qO8m@3}a-@3}a7piTqyXw3z|Y32e{e_>{y_B}O& zWe34&f)Shs3nLA|thcQ4xB$!ixn@;>K(#gqRP(`UlW7fF0f7*7jqll zCSn9ysy;Di>X^0lnL&=il zkD>d?_mG?eGKd>2Tu1vqLH`-`tt9x#4R{W+cubaY1_D7@dO%T-Zh?kRgTMm7k@ZND zNC1Lzpqd`B)KLv#UiUa+0rd%|6@f)C<~PxNPdGr%vjz~1t z#Ro>%*+v$B)~h3hM^ODYk^BXc14s@cq3Gcda)*&T3q-~R)`lJ^t3wDqQpW%W`e=fq z$QaJa_4maQwcY)_@5(S7=^#A2`wMpYkpQxp4DqC(oXoUOO{f31J)wgo!+0 zO%sTWSE7B@NFN+WfCs9p8tX2{x@Y(O)cMX}b^Fos_M??;->k%*tu{S7<$ZtU+Ddf) zkNT$5v+2LfU&y~xswBVhm;6j0hPCX+P@V0SaC}Z!H8TKYE*zg3D2G>71^itzGW3*5 z%fc#*)9==X^c47Mi%GZMUEco1f#q z(4zZq(eng=SNl2bzN?CA#HzZ zD`fq1gh$#!Y19q;moeo33}^B*k{UyP3iFWQ@?skD-8IJ^5qCE=4DnXP+gr{w;k-@k|`afA1C~BNP>#M6hTr`ALn=)XyKumCvKu9UIVPxI! zP0ymJi-iu)okOMx+x640$I0J-7Tf~p^jdK8JpVq|{Y&)4ywdPXZq0{W+kbF9H(egy x@i_ZEPFoeSeO;Bah^frx#14BgvxdIW_e{$FFi@CzcwiPTE*mB)e(ZrVPzH%ar*d zKXu=$%DIk4+b z`~RQ2BPm95({3N~0QZ0X&-?rD|8oEC_j?J1ZJ+*g>Qso3|HOnHc#BzkADGKTAu*zG ziX+X%IEbBTN5&CzWSlW4hxvS(&j>Mr@j}{_amU<@ccsOQC+5j`W8RD}=F3PiDdUg% zS(!T>$jC7{(-Lc8X))cJX^XWn-ji<6Y>92*h=Uv>iuWy|_^vt38O1tSo&l2y!7FluLGZOSpIoNw{htiqW1 zd$;BrT6+pY#AURdfSt3JYVMFWAI3cMj5KqXhtleKHC;O@!>gQ%TSZBGbvP@W_nYvo z{S*9nKW7H=%ca3#2<4pMa5voG#(0cbX<1+sUX1qUDA1? zZ?$U|IeXS6u_(@T+4=Q&<6W>@HTPfg=skAZqQ|bKaA(H|(HvhR7dXAwF0Y-3x9FYe zvvcclk<)kBsmIS^v(=V-Mc){CTS_{nNz(CZ0M1t`N;7|C_gatNbzExUNy%T56o1iw z-kPftXyD~KUyz2$rN4)-0N?(kgM;2n(j57jbB+UaicFo!<PZy#T;qNMEv|2z>M}fO|mFFcB%7&7Obb9EKFg>deAzryNsVsD?{kOS*Z;jmD z3*21=2||2CG`!ZJ@swi7>6EUf)0sGSgBe1G0g_kK6I54~ctSV4X@JOVvJlSz%%!q0 z@3@{jt7bLBpUdkFNf$H%ESxa>)C9?RGMCQ+RBHKT65h6z^r@8QsF{eg8_`qH`I$_D zo`o!`MyU;VDOH06L)&;Qvp_pw6%=v6aOL%6HaBMoG?ATFsT&LMicwrB#iwAnr%pci z-1yX#A&gH>zBFmLl2nD&P<)+6&^iShUE=1JYKAkdW+_|=Vg#mDJ$|ZS*D#!#P7R+a zOyb!@MvVjvUQ^SjDZW=Dpd~W1Y1OXI=4$XbdBce_W zz{&q155G@74G_;s?w*UZ^i|~T3;gXs=i9l9xk{jS!TE7(=R4qs@(_5-3Kb6gKJJlpZ5WAba@x#dm?%2 zXeD^`>KjXaSGd3P|H!}e#)7mQJh~!pTO2IQ(Pc5ZEJvGD-HTFL9$Xfo#^CA>;@|gh zRV3|G+{0A~x_$JCNJ1mrZ?zX-n=gE&YrKWK-ouRxuIt-&1AoKMjl209fl(JEZf@bm zJ^alM5&53>aUXy4sZQYk)y*OGFzOS4UnfNIl`tXd{eKjq(?oIpgad(D6Q;$Q2n1nz zgplvRXeD<^{2U_HZHv^R2!hlS<0TIWxSEKCHCO=zoTmZu-Aw}SSrBlo^}4nYDxsKZ zxAW`q<3$SyxTgMf#3mMM0u02=#)4P}1>6iXPuN^NZV;6lsv7pp74aH^9faoDpg8+J z6RRNBwwZMrsp2`i%ND!^HwdeD;Y-4|jfpDn{H`WZ1G`_>;S>>!b=l$ z52kGFJ)cPD)rg<&!#)u7>3$?kDjh)XAP~c2&J#o!9mcd1()17(e;LVP%yXGb()^HE z2jlc87I!h+wWYxN<Dn7p~KHmVh!R^)H zp>psL!0m0Rt19g%OFM2$@;iZQOQhTqsYv}RQtz5a2m$sAZQcdfYJjwKF3Ww(Vjs{u zaz|C(UY55ns<-5UWpSV^57eO5^8!Gt2(2DzcrF)4BV$hPg8}ZjXIvi)?gsvP7x&yz z{(ASQ3lcZBb7LHTV@DVAM>(K3IiwR2uOY_SRf)$9Z#)i4Qy#>A+>qk&bNNIXH3LYw z<8dXI1jaR6z|%B%RC5e>0&U7vj-n+(N059S$W^iil(5sMh*FH;yVCv`$Xnz?zWLCmV%a{xzA*h3_N zsRh)mFnkg1#~|MN@yx7VcqNrpa&ynagH5Nh(~U-?ipAkMRvtcMRfw{ex=!%nI+@@@ zE6lVWb$B3VXs~^pMdysYCw4K!AQ~*-ZO9}xECU#GjiRT-7x`(@0Jt0gt;B%VF~8RW zS~l6bNidhv&xEtNu%$0*VQ^|B)zkSj7zYy(X9JQUY65sRArb?cr=dtgnaKdg1eRS4 zSlB+oAz|2tVw8TvCTD@su>D3rDhWLdAtAPa z6u8yyCLHxzwIH@|6gYD%93}~0E!ddxMa0n-eAkoA>O46MW~e^jhdHJf&V2k~@RCA7 z_>_7kaUMj~1_+A#{w92D{|2NPL2C-IQ@3}5fP2Xb8~NY>QPCJ_3h2>$&uJ^UfTPYC zUsDxGg4)kFsQvsp>;-S!hTb|y10M5$$DIt1h53G4@nx=pa8{kO+%sYDD4$MIO}E`Q zQNzcu)HKUYZI_IXT;;*X!W2p!eZQi~rxc2=7Cc>kwYX0w+7pK zRx#pghIQ024VQUH5UW}~t!sD`rV437e2`7c>?dxzJhV70WPVrk$(vCA*_wX=J!-E3 zv5~vw$%_1?1^(mKp4-7EtHEeF7+vYuQSFG9JEAM0ohw^=?t5%)EYyAZ^><%ie0u48 zC3vvfdT_y2SIU%Ia{sc}UzYn<1=2SBNJAa^!l-+6JNJQy8x8Xxc!R*Nlhw7G_C(D{ zt;^~!&Q0UU|M{*94R>AG^sWn=-gOtWL3^SpfWNe6EN}eIY|UlUp4~+^_)A60UkX|& z^9r{#4FpN>=%QiCU1~3i;5+RE->C--{dT)%gYVZ>e}P4h;zrwX2KC*hpa}JP1$?Rh z1m7HdFIxUp(ekf~^8?n~&ClY0ov`}7l%|7%*zkbCy=wYiqx2MvLz74pAQP66dUE70 z>VeGv8u4xVOet2UPa#43(0tpwk;4ZLlB-D_WAD~-_3g2JetT&OTcF6MC^HR-JzPuw z($EYwpEXkRX`B8#>%p4K1hvgt=lsxP-*`9^jIs9K$2{|P?}B?( zB69oNXD^<;(!V%e2}G;l-x8yrvtI?|uT2qrVb7{^Q{*iKWCpWUgl3FIA#1{x!EajBd8$=)gNtk;99l zx8$A6;?8Ax=Zf5QW%S}3RT(Y7CH|jGj|MY?mtYo2lLqq|<{|k3d^HVbpve?$6$Y3F1F@yS0Jr*NlLpf(>}Rzgwlo;v zObrG{h=dw6ER@^Km2U9(HnPT`Fj7t{~NOFBhvX_WcLH!#W_AD tKpq4LH^O~HUipaZec(F5af9~=(g#^b57%>#0C_ON4RiZHAwbxU|1W|Dbu0h? literal 0 HcmV?d00001 diff --git a/tests/integration/__pycache__/test_session_service.cpython-312-pytest-9.0.2.pyc b/tests/integration/__pycache__/test_session_service.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..51bafd1552596755d200c54706c2d0e071876efd GIT binary patch literal 14861 zcmeHOU2GiJb)MPZ*?<23n$%J@6?AVfN*HU13Q+vDI8A?m-a+#T- zB{I8qD;q5h1aN~CwFCG;4}P#BI}d%Vn*xg8TA<*HB$gS(t%0^^3-nMY+|BMNCc}(W^Pl0(` zkcFfmi?S;xCSBz2&U-R0QOJ9fK9N50C;jlmos;rXQX+YtTp%Az1_|%Yh4SHKnDD+_ zBp*#i^RZ-%r2V;;d~33m@KUZV-=1vGcO*MVI*{wkcO|<-!6iH?$iX)RIdsEC`x!PuE5Lh~m$UK&FyWKh%3|gmEt7B=ev>sKCUZ_t>&qHQMp*`}7i2q6EidJ=8K~8K zbXrvls?lnuo>!I%T2?QpE7*>i`Ak7pavAz=l)BBt?Y{%&Z9x%|B1{BKK+>&v7Rcn; z#RXv@T${#}>{9&mF4_Ht=Z5((DShSA9$V?aS1#?fl@6}>5`LroJWPuD-ZQE~CTg`G zGizrmV?G--g7RESqjQn*+1^qD{&o0&dlhoe2qKJ=F0$uE@rKEld^#IVfsI4SU-T__ znXGj$`irgwdmHAXqI&^btGkMhGg0(9&V~4a>xNnL>w@h5KJ@7?U2nVp)aw;WQc+rn z)>RJ4QlJPXPk8} zMPN1~TmwG0wZF3lgs{v#sR7}pHK5lv5)Qg*jVhZ7)n39u5AQAk8WFa&;!vg-VYSPV zcRf^BnHRf)+>%4_I2kxKcnCOEH-37}t;wZ9U#uLt3!R zy{I9!w{x107#lsg)%=oMI5X#PY(GyLWQwFT7dy4!GrIFfNB6JJKt8ek}$ zR5JzQkU0e{ooUn(0ZUdZP>gm67_&pr@aS`=XHPvf`(&`B`mtq?iZiVCBe@R=F1R(# zAEnJ7h32OkG+#S7RyQUS+I6?p2)@Z=P5slf{^>-AikrWRJE{?-bIbsVikq8?Uz*yE z1i-XFM$QO6wzPsXo=OAwgXzoVQmV3~W(xVGw5k|kc$}JB&gS&&qGtGV1(48;o|Hy3 ziHN2yy9x1@)4)l!othZ06b12gk>!$8h$vv8-iJkQ_GiaHpok(u&bfCc0KL zQX2jGSI5pnJ?AKXonfH1K><`D?LPs${=4wS9}BtcD|^0O-t+D4{{7X6 z&^vZp@b?Dq2>$lqn!nm6v~*WuL*>}ex-_;P8@iS($HpoW{YJ~wOH=O+REDO?LsP)6y}Ax;Y^oyRZ_CV}DNf6f z4Lw1L4U64GVJ<@%S~-)1b_ffS&`O;sB%w0sR3$cCjt#F%N8jHAWIZ=N04A7>j^>FZyZ%S>5&r#r*k-m_Q?YhFxM9e?UocE1*-htTnYsjagZmNkbwqzZ z8^7CZtau6Be~G~T*y?=@?x&aZWmSn^$m;RDqNl-?m>yjX&x2D7vI$x+!s)rp;qfD~ zGCwgnbv_|cXykwzjUZH$np4w@nTtjgd4P&}xM>&DMgY@tR@GN#RdnL)g6*xEYTzzG zWGd>FioOPQnAX$?JlFQ4rnc*%;mU#!boFop@Xh-TMY9^Q8US}E+9;w%o0h<8M;Lty zpFW5LJYa&FK(Zgn0VEG0If!Hw$!R1{07*pD!^k7LRmYJWK{A1463G-2T-}j@- zO`*Q-;qCs!c2{4um0-UBV!r^veqA=~cjVg3>(UVffG?M0M=BEjw#*Eg;dE*i9l#vu|N_T%I!^ zQtgM~@RCc6=m`@i72{y@B8B1gb+E<27B1w96D zLaqh$7t~!kcR?{X$|Shy|F>Q54*hpi$2-KCT=f0ka9JdKfSbE~RAoWQ=y90^eVFqH{4%2n?~beeY$9km6;!egI_(KQ>O(Mm zYB~#k88rX{40}iu3mnddhHA#vF}j59B}*9ZEjMPfGPoS@O%!=`YRC1fVY#QU#eV>@ zligbN31D4C?=MI1Z_;wDhJ@JgZNV3VrOOuxR+0R=O^9~9_0pR!ZT279iX7f<>#ekn zl-owO`-is&$0~y-%7Z6h;j8pdl=~;P2M$&Sj+Y0HZ|^%+*>|?Q?<`c=o`5Rbqflie z3ROm;cicjBd_8iw+79)9QSE@NFIe7xX}CdLJ=s0eE#B-EXZ)U_}mkRhdr8z+@Af~i!#ATNW88$FDQt0+1o5EgS!%e|$Z3+wMldQX#abNO4)Dr8gyTIr-&2sw#oyU}&qL)Pv zETU##>ozhDadY!Pl^z&gc3HB-K(Pugj@^g0;{{&Tah@W zwdpuSG>cxg*?`GzNje$#ToPhE-d$vUtXA3gu8YKaB#amqr7VpS26S{)lgm~UdamoP zA~)(;jSyRKff??{QCx5^7EAudvjfpZe=$g0Cg2*-KDV{+uqn%dVhF~cl(~n-pF3jY zP}7>yYa0m%O|eFm%>+aV8*0qmB|vjRjms2+tadrfql6JT%A+R0k5yLX~q;|A4tsVB&!9hEU9yZFmt|MF$w4sH^9-}1o z7&X_2aMQEwbc%qk44mre7KLYVDpQ>ofIsLke1j;Z*cBCdo2FAR)59(!J ziDw%40%%0a@*(UH2b_8Kndi=)K7aa@83|PPVEu6AhxK!h^2dx z;86||N=&p;0FJww;ak=-i-pTZ3mZuWjOobXE@}jL$C`FZ;E0p96EO?3^Y|JIv=WB| z>ETP*jxAy~PFxiB*wzH~Qap?m6Q{*-KA$r-4;AOfb%_V!8KnsEDY{lmxk6}dFMR%CMpvCwrH9rI4xr?O>J20rZn`T`FaCZV_&>I0QF>i#)f;VEVR3}L*XOhqk zVbO09$I^8_;;iU=>$NvuTbK5)y>@M+9NS-!@V7Za-ne)*s$15l;tv%f%>k2 zE5npztX&97Ra;@mncE_EHYVO|70-q}H`|_m7!uA1^*u008n`XofILmK2-?9fK%NeG zRR^?7FDG0IxYIoJc^B^V0Pge_0Y?Jv^c8)8J43uCr_+#bz?+CWp+wEjX6^V6`(X|4 zwCd-M4c4L{)a)0Qqyv_rctm{}$sZxH1HG@|z}LygpMAa&%$sNq=GhP{ zqS3nWR*a&&IqW}qM}p=2X4}6Tl2c`@mB2SLQVJ$XP-z}H5>IkMYA5h12{i2p*9M$X zmpNgplV1(2nZ>R`yV^A;sP?HdJR2#uk5s(`o4FC2xd}E4e)HID60zB2IX<~AO(GPV zEXO7*68^T#44UG!4B3E*P-4SkH&K|&PzGv5KxRrZ79^pSI#EbMWe6B4HoGQMfF`fk z0yNojOpEw8;+cr&XHT~Rn=x?^JhJ%;Z^3jn zKw>VyTnX7k@H7@Za3sN7^frOQyaa_oiEm_;ck?~9bC(sFOE!VR zxP{MAn2(?^pNYc!O|RbEkHz|FqA+jCLxN8~A?tQv^?+&Hs|p+@Sd7mW78UZBOpCx= z$!a>B6lP2LsG0?pm{p$y3g&T}&6Y$ z1gZzX1nt0cg>ZIoV2dVcg3~hQ($t2!>WA^66>q~G|_nFqzc7H5V% zA3c5mSO^uVvpC|`Ov$s;+BghXE+L+%AeP=d#U5_R#Q zR__54!-2aQ=>e|#KwYFKW=g@`lmf20w@xbT9VB(Zok)G=0*>|wJCo=ouVP5-&@)jj zqc=rEoTK!NFdS+gRh7$XR)-_cJj?K#c~mjUXyB@5cS3LLA!h#Gi2RAm$*a7s>7(Ym20q#vvkFc0@y$K=_`X8dG~(K=~3mQDYt zFRNH3c;*TT+o~8~F?#As%5$tQA?yo@;*qJVolZmg77ZFcqFw7&-6`o!0a!HEhN}?p{Cz7bxN<`bC{3pqB^8;CS5<7_!BnE;oNQo796;tMq zGP{gTm2wg%XzZeZTcoJ3E|5b5r zzpgZ&;p04qc|zKi5#j>V#k81l$K6bKr=^T1?#X!LUY3^9zD!rVi|L-UKNE-tGQoI| zrM+o6(;e^T2u~g(s_!kLc3tPK5yX2~o*(i8^}OB!9}TYGcQR{eFH^-x<+2e&(+xec zlA{q_({;?zHTr5Qsm1<*4J}1E)1xYeW~4G2=wd#fQV)aTU(gHLI#>ps469wR?LVGhOQ(|1s$Fz~(i}B=?9{W`T24peDoKtb6iwd-}T&?Tt2Nv*dAyN^5m0A*oT#TI$emZ2asJ4r_n@O zS=Q1^Q9s4ir|5sopfxB55wr(Oa2aR_1ug@{l``ErBPo?IAw6?q>G;W|$7nyKqFz(b zwe$+bH8*>7C9?({H%4p*?JRrLX1ATLkK7!vbJ|94O16Wsh5#w%c62Xv6mh=WQRbJX zQw>iwjXBPg;Ma0eYXx(J6|_z{pq$L+khFnDYicVn14HX+;? zS&$&n;sVeSG8eG#8(aX;z3v2{j{O;R7vMOC{~Y~QaK!Hm*KIcfEI~V;LH(d?#kOvq#y#x=NQS|>-RV$SJsR?)gmvaj7UZ^6gUu-*t&lO zTr*@dWSV|uIeFmVAyr#BJTrSb>S6HiL_O07%_Nqol1-j5gQzESnGF2sXB5+mX*ETS z!cx>{Q5wAx<8`OT=x!{XL~$Po=ynKoWGx*=F@oYA6r(7{P>h4nag`mq(Fi2W$xggD zmom;Ia8S;?*XQq+QroJ=u-~77*ha3K4y!&5-}BD7yW&-NvN{s24h+|N7`wd4E-z!( zK!aW1-INcNJcmm1p$3b_Hs!gJXRajAvAjrGj$AulmZwUdDUh`hXymqz7(r(BF=q7w zvyOn?LM&WBMyy|c1F`-W&er$2>o$>rR(vz9IAX{!6ZUl?PviV_qJaL1l%vcn6c!%>jiR-WAt{_&v2@(YDpuaI#H=} zn!`aGi;8z7a-E@g=u$@04kxYYUMnE6OgB|haM)2i=?Ogz zD=%Gs=?^3O{}?z>?H#K0PL+G7sw0!tvFXa#L*=oDfZdgm!{w2~)zJqkqesi5N2|N$ zE4!X7?|KrN3`L>IP!O671fj`5@Ro}N50(N4Y9VO<>slXVeeLA^OMeKaba5fP80Kya zaf_00V|Wzxxdnc)%k|+rx9D|$cvL|BVILHHkJarT#g=!T;ZVE8IkuM^jlPom#i< zT;o{KdgoecDvz;qHt8%3t`u%DI~2xtJ``MyLtz!&N89JbS{p*qwTjVs+Xpe29VImv z81+kUr|VTw04HjXwfzv-5R2mL)+I{p6l3k&a#M6Ab~-|eLc@YN=Xu)!PulU7y{L_A zl%hu$~Kx`APRcNuDTqCO}r@{wo`oH{K0YB6H=)TvJrkf918y zuU(tkl=qZ8d&=^jS`SorVj}rifVJpy}vx7$z-9j&_Em-T(T` zw*#bI?ILzm0I>~@H#&6FcEoAOM_GgFdh5SM_zQXPFV zRKBfakb3dx!VLEtZsCyd!5nBwd*h*mkUI2!1c@D>#{V0XE?L+Xr%OBYI> z(aRTVU5!0Hvf$=^#VrWJuYI6F;6?XAH}nA%c!Q+VC@{XF`%xS~!9pvH^5_DJMHB~7 zuvjL7+KeO4f>MvJqHWPtkGPjbMDT0JO`z58Ym1v?+X{YdE6{59wpsbbG1eCR+R+tg z)`)O0(NNP=5R#^Vuj|$u-hlo}T1h9*XqiGxJ8#6AV@R`qS%b#|3@+iQp;`=rQ7x@g z^pn{QNLI1$GcB5gol+&BbxtCuHBBNZ>Ov`WYGcE1CWqIr14fc8hm${8t{r Z9lYg!p5ua_6O^|MewG{if`DMl`fudwvylJ* literal 0 HcmV?d00001 diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py new file mode 100644 index 0000000..0931f3d --- /dev/null +++ b/tests/integration/conftest.py @@ -0,0 +1,162 @@ +"""Pytest fixtures for integration tests.""" +import pytest +import asyncio +from typing import AsyncGenerator + +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import StaticPool + +from benchmark_core.db.connection import Base +from benchmark_core.db.models import ( + ProviderModel, + HarnessProfileModel, + VariantModel, + ExperimentModel, + TaskCardModel, + SessionModel, + RequestModel, + MetricRollupModel, +) + + +@pytest.fixture(scope="session") +def event_loop(): + """Create event loop for async tests.""" + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + + +@pytest.fixture +async def db_session() -> AsyncGenerator[AsyncSession, None]: + """Create test database session with in-memory SQLite. + + Uses StaticPool to ensure the same connection is reused for the session, + keeping the in-memory database alive throughout the test. + """ + engine = create_async_engine( + "sqlite+aiosqlite:///:memory:", + connect_args={"check_same_thread": False}, + poolclass=StaticPool, # Critical: keeps single connection alive + echo=False, + ) + + # Create all tables + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + + # Create session factory + async_session_maker = async_sessionmaker( + engine, + class_=AsyncSession, + expire_on_commit=False, + ) + + # Yield session for test + async with async_session_maker() as session: + yield session + + # Cleanup + await engine.dispose() + + +@pytest.fixture +async def sample_provider(db_session: AsyncSession) -> ProviderModel: + """Create sample provider.""" + provider = ProviderModel( + name="test-provider", + route_name="test-route", + protocol_surface="anthropic_messages", + ) + db_session.add(provider) + await db_session.flush() + await db_session.refresh(provider) + return provider + + +@pytest.fixture +async def sample_harness_profile(db_session: AsyncSession) -> HarnessProfileModel: + """Create sample harness profile.""" + profile = HarnessProfileModel( + name="test-harness", + protocol_surface="anthropic_messages", + base_url_env="ANTHROPIC_BASE_URL", + api_key_env="ANTHROPIC_API_KEY", + model_env="ANTHROPIC_MODEL", + ) + db_session.add(profile) + await db_session.flush() + await db_session.refresh(profile) + return profile + + +@pytest.fixture +async def sample_experiment(db_session: AsyncSession) -> ExperimentModel: + """Create sample experiment.""" + exp = ExperimentModel( + name="test-experiment", + description="Test experiment for integration tests", + ) + db_session.add(exp) + await db_session.flush() + await db_session.refresh(exp) + return exp + + +@pytest.fixture +async def sample_task_card(db_session: AsyncSession) -> TaskCardModel: + """Create sample task card.""" + tc = TaskCardModel( + name="test-task", + goal="Test goal", + stop_condition="Test stop condition", + ) + db_session.add(tc) + await db_session.flush() + await db_session.refresh(tc) + return tc + + +@pytest.fixture +async def sample_variant( + db_session: AsyncSession, + sample_provider: ProviderModel, + sample_harness_profile: HarnessProfileModel, +) -> VariantModel: + """Create sample variant.""" + variant = VariantModel( + name="test-variant", + provider_id=sample_provider.provider_id, + model_alias="test-model", + harness_profile_id=sample_harness_profile.harness_profile_id, + ) + db_session.add(variant) + await db_session.flush() + await db_session.refresh(variant) + return variant + + +@pytest.fixture +async def sample_session( + db_session: AsyncSession, + sample_experiment: ExperimentModel, + sample_variant: VariantModel, + sample_task_card: TaskCardModel, + sample_harness_profile: HarnessProfileModel, +) -> SessionModel: + """Create sample session.""" + from benchmark_core.models import SessionStatus + from datetime import datetime + + sess = SessionModel( + experiment_id=sample_experiment.experiment_id, + variant_id=sample_variant.variant_id, + task_card_id=sample_task_card.task_card_id, + harness_profile_id=sample_harness_profile.harness_profile_id, + status=SessionStatus.PENDING, + started_at=datetime.utcnow(), + ) + db_session.add(sess) + await db_session.flush() + await db_session.refresh(sess) + return sess diff --git a/tests/integration/test_request_repository.py b/tests/integration/test_request_repository.py new file mode 100644 index 0000000..0d0e301 --- /dev/null +++ b/tests/integration/test_request_repository.py @@ -0,0 +1,144 @@ +"""Integration tests for request repository.""" +import pytest +from datetime import datetime, timedelta +from uuid import uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.models import Request, RequestStatus +from benchmark_core.repositories.request_repository import RequestRepository +from benchmark_core.db.models import RequestModel + + +class TestRequestCreation: + """Tests for request creation.""" + + @pytest.mark.asyncio + async def test_create_request( + self, + db_session: AsyncSession, + sample_session, + ): + """Should create request successfully.""" + repo = RequestRepository() + + request = Request( + session_id=sample_session.session_id, + litellm_call_id="call-test-001", + model="gpt-4", + started_at=datetime.utcnow(), + latency_ms=1234.5, + input_tokens=100, + output_tokens=200, + ) + + model = await repo.create(db_session, request) + + assert model.request_id is not None + assert model.litellm_call_id == "call-test-001" + assert model.latency_ms == 1234.5 + + @pytest.mark.asyncio + async def test_get_by_litellm_call_id( + self, + db_session: AsyncSession, + sample_session, + ): + """Should find request by litellm_call_id.""" + repo = RequestRepository() + + # Create request + request = Request( + session_id=sample_session.session_id, + litellm_call_id="call-unique-001", + model="gpt-4", + ) + await repo.create(db_session, request) + + # Find by call ID + found = await repo.get_by_litellm_call_id(db_session, "call-unique-001") + + assert found is not None + assert found.litellm_call_id == "call-unique-001" + + @pytest.mark.asyncio + async def test_exists_by_litellm_call_id( + self, + db_session: AsyncSession, + sample_session, + ): + """Should check existence correctly.""" + repo = RequestRepository() + + # Create request + request = Request( + session_id=sample_session.session_id, + litellm_call_id="call-exists-001", + model="gpt-4", + ) + await repo.create(db_session, request) + + # Check exists + exists = await repo.exists_by_litellm_call_id(db_session, "call-exists-001") + assert exists is True + + # Check non-exists + not_exists = await repo.exists_by_litellm_call_id(db_session, "call-nonexistent") + assert not_exists is False + + +class TestRequestQueries: + """Tests for request queries.""" + + @pytest.mark.asyncio + async def test_get_by_session( + self, + db_session: AsyncSession, + sample_session, + ): + """Should get all requests for a session.""" + repo = RequestRepository() + + # Create multiple requests + for i in range(3): + request = Request( + session_id=sample_session.session_id, + litellm_call_id=f"call-session-{i}", + model="gpt-4", + ) + await repo.create(db_session, request) + + # Query by session + requests = await repo.get_by_session(db_session, sample_session.session_id) + + assert len(requests) == 3 + + @pytest.mark.asyncio + async def test_get_by_time_window( + self, + db_session: AsyncSession, + sample_session, + ): + """Should get requests within time window.""" + repo = RequestRepository() + + now = datetime.utcnow() + + # Create request in window + request_in = Request( + session_id=sample_session.session_id, + litellm_call_id="call-in-window", + model="gpt-4", + started_at=now, + ) + await repo.create(db_session, request_in) + + # Query window + requests = await repo.get_by_time_window( + db_session, + start_time=now - timedelta(hours=1), + end_time=now + timedelta(hours=1), + ) + + assert len(requests) >= 1 + assert any(r.litellm_call_id == "call-in-window" for r in requests) diff --git a/tests/integration/test_rollups.py b/tests/integration/test_rollups.py new file mode 100644 index 0000000..93e3d1d --- /dev/null +++ b/tests/integration/test_rollups.py @@ -0,0 +1,194 @@ +"""Integration tests for metric rollups.""" +import pytest +from datetime import datetime, timedelta +from uuid import uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.models import ( + Request, + RequestStatus, + MetricRollup, + RollupScopeType, + SessionStatus, +) +from benchmark_core.db.models import RequestModel, SessionModel +from benchmark_core.repositories.metric_rollup_repository import MetricRollupRepository +from collectors.rollups import MetricRollupService + + +class TestSessionRollups: + """Tests for session-level rollups.""" + + @pytest.mark.asyncio + async def test_compute_session_rollups( + self, + db_session: AsyncSession, + sample_session, + ): + """Should compute session rollups correctly.""" + from benchmark_core.repositories.request_repository import RequestRepository + + # Create some requests + repo = RequestRepository() + now = datetime.utcnow() + + for i in range(5): + request = Request( + session_id=sample_session.session_id, + litellm_call_id=f"rollup-call-{i}", + model="gpt-4", + started_at=now - timedelta(minutes=i), + latency_ms=100.0 + i * 50, + input_tokens=100, + output_tokens=50, + status=RequestStatus.SUCCESS if i < 4 else RequestStatus.ERROR, + ) + await repo.create(db_session, request) + + # Compute rollups + service = MetricRollupService(db_session) + rollups = await service.compute_session_rollups(sample_session.session_id) + + assert len(rollups) > 0 + + # Verify metrics exist + rollup_repo = MetricRollupRepository() + session_rollups = await rollup_repo.get_by_session( + db_session, str(sample_session.session_id) + ) + + metric_names = [r.metric_name for r in session_rollups] + assert "request_count" in metric_names + assert "success_count" in metric_names + assert "error_count" in metric_names + + @pytest.mark.asyncio + async def test_session_median_latency( + self, + db_session: AsyncSession, + sample_session, + ): + """Should compute median latency correctly.""" + from benchmark_core.repositories.request_repository import RequestRepository + + repo = RequestRepository() + now = datetime.utcnow() + latencies = [100.0, 200.0, 300.0, 400.0, 500.0] + + for i, lat in enumerate(latencies): + request = Request( + session_id=sample_session.session_id, + litellm_call_id=f"median-call-{i}", + model="gpt-4", + started_at=now, + latency_ms=lat, + ) + await repo.create(db_session, request) + + # Compute rollups + service = MetricRollupService(db_session) + await service.compute_session_rollups(sample_session.session_id) + + # Get median + rollup_repo = MetricRollupRepository() + median = await rollup_repo.get_by_scope_and_name( + db_session, + RollupScopeType.SESSION, + str(sample_session.session_id), + "median_latency_ms", + ) + + assert median is not None + # Median of [100, 200, 300, 400, 500] is 300 + assert median.metric_value == 300.0 + + +class TestEmptyWindowHandling: + """Tests for empty window handling.""" + + @pytest.mark.asyncio + async def test_empty_session_returns_empty_rollups( + self, + db_session: AsyncSession, + sample_session, + ): + """Should handle session with no requests gracefully.""" + service = MetricRollupService(db_session) + + # Session with no requests + rollups = await service.compute_session_rollups(sample_session.session_id) + + # Should return empty list, not raise + assert rollups == [] + + @pytest.mark.asyncio + async def test_empty_variant_returns_empty_rollups( + self, + db_session: AsyncSession, + sample_variant, + ): + """Should handle variant with no sessions gracefully.""" + service = MetricRollupService(db_session) + + rollups = await service.compute_variant_rollups(sample_variant.variant_id) + + assert rollups == [] + + +class TestRollupUpsert: + """Tests for rollup upsert behavior.""" + + @pytest.mark.asyncio + async def test_upsert_creates_new( + self, + db_session: AsyncSession, + sample_session, + ): + """Should create new rollup on first compute.""" + repo = MetricRollupRepository() + + rollup = MetricRollup( + scope_type=RollupScopeType.SESSION, + scope_id=sample_session.session_id, + metric_name="test_metric", + metric_value=42.0, + ) + + result = await repo.upsert(db_session, rollup) + + assert result.metric_value == 42.0 + + @pytest.mark.asyncio + async def test_upsert_updates_existing( + self, + db_session: AsyncSession, + sample_session, + ): + """Should update existing rollup on recompute.""" + repo = MetricRollupRepository() + + # Create first + rollup1 = MetricRollup( + scope_type=RollupScopeType.SESSION, + scope_id=sample_session.session_id, + metric_name="test_metric", + metric_value=42.0, + ) + await repo.upsert(db_session, rollup1) + + # Update with same scope + rollup2 = MetricRollup( + scope_type=RollupScopeType.SESSION, + scope_id=sample_session.session_id, + metric_name="test_metric", + metric_value=84.0, + ) + result = await repo.upsert(db_session, rollup2) + + # Should update, not create duplicate + rollups = await repo.get_by_session(db_session, str(sample_session.session_id)) + test_rollups = [r for r in rollups if r.metric_name == "test_metric"] + + assert len(test_rollups) == 1 + assert test_rollups[0].metric_value == 84.0 diff --git a/tests/integration/test_session_service.py b/tests/integration/test_session_service.py new file mode 100644 index 0000000..f5c8ba2 --- /dev/null +++ b/tests/integration/test_session_service.py @@ -0,0 +1,161 @@ +"""Integration tests for session service.""" +import pytest +from datetime import datetime +from uuid import uuid4 + +from sqlalchemy.ext.asyncio import AsyncSession + +from benchmark_core.models import Session, SessionStatus +from benchmark_core.services.session_service import SessionService +from benchmark_core.repositories.session_repository import ( + DuplicateSessionError, + SessionRepository, +) +from benchmark_core.db.models import SessionModel + + +class TestSessionCreation: + """Tests for session creation.""" + + @pytest.mark.asyncio + async def test_create_session_success( + self, + db_session: AsyncSession, + sample_experiment, + sample_variant, + sample_task_card, + sample_harness_profile, + ): + """Should create session successfully.""" + service = SessionService(db_session) + + session_id = uuid4() + session = await service.create_session( + session_id=session_id, + experiment_id=sample_experiment.experiment_id, + variant_id=sample_variant.variant_id, + task_card_id=sample_task_card.task_card_id, + harness_profile_id=sample_harness_profile.harness_profile_id, + operator_label="test-operator", + ) + + assert session.session_id == session_id + assert session.status == SessionStatus.PENDING + assert session.operator_label == "test-operator" + + @pytest.mark.asyncio + async def test_create_session_with_git_metadata( + self, + db_session: AsyncSession, + sample_experiment, + sample_variant, + sample_task_card, + sample_harness_profile, + ): + """Should capture git metadata.""" + service = SessionService(db_session) + + session = await service.create_session( + session_id=uuid4(), + experiment_id=sample_experiment.experiment_id, + variant_id=sample_variant.variant_id, + task_card_id=sample_task_card.task_card_id, + harness_profile_id=sample_harness_profile.harness_profile_id, + git_branch="feature/test", + git_commit_sha="abc123def456", + git_dirty=True, + ) + + assert session.git_branch == "feature/test" + assert session.git_commit_sha == "abc123def456" + assert session.git_dirty is True + + @pytest.mark.asyncio + async def test_duplicate_session_id_rejected( + self, + db_session: AsyncSession, + sample_experiment, + sample_variant, + sample_task_card, + sample_harness_profile, + ): + """Should reject duplicate session_id.""" + service = SessionService(db_session) + + session_id = uuid4() + + # Create first session + await service.create_session( + session_id=session_id, + experiment_id=sample_experiment.experiment_id, + variant_id=sample_variant.variant_id, + task_card_id=sample_task_card.task_card_id, + harness_profile_id=sample_harness_profile.harness_profile_id, + ) + + # Attempt to create duplicate + with pytest.raises(DuplicateSessionError): + await service.create_session( + session_id=session_id, + experiment_id=sample_experiment.experiment_id, + variant_id=sample_variant.variant_id, + task_card_id=sample_task_card.task_card_id, + harness_profile_id=sample_harness_profile.harness_profile_id, + ) + + +class TestSessionFinalization: + """Tests for session finalization.""" + + @pytest.mark.asyncio + async def test_finalize_session_completed( + self, + db_session: AsyncSession, + sample_session, + ): + """Should finalize session as completed.""" + service = SessionService(db_session) + + ended_at = datetime.utcnow() + result = await service.finalize_session( + session_id=sample_session.session_id, + status=SessionStatus.COMPLETED, + ended_at=ended_at, + ) + + assert result is not None + assert result.status == SessionStatus.COMPLETED + assert result.ended_at == ended_at + + @pytest.mark.asyncio + async def test_finalize_session_aborted( + self, + db_session: AsyncSession, + sample_session, + ): + """Should finalize session as aborted.""" + service = SessionService(db_session) + + result = await service.finalize_session( + session_id=sample_session.session_id, + status=SessionStatus.ABORTED, + ) + + assert result is not None + assert result.status == SessionStatus.ABORTED + assert result.ended_at is not None + + @pytest.mark.asyncio + async def test_finalize_nonexistent_session( + self, + db_session: AsyncSession, + ): + """Should return None for nonexistent session.""" + service = SessionService(db_session) + + result = await service.finalize_session( + session_id=uuid4(), + status=SessionStatus.COMPLETED, + ) + + assert result is None diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/__pycache__/__init__.cpython-312.pyc b/tests/unit/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..72acd60e10a87c93d23e7f22f757db34074c8219 GIT binary patch literal 119 zcmX@j%ge<81TmlXW`gL)AOanHW&w&!XQ*V*Wb|9fP{ah}eFmxdC03GJTvDuGnwMFk rA0M9yq~hcC3My}L*yQG?l;)(`6|n-9F#>Thi1Cq`k&&^88OQF?)z1ur}oFslrB6XBVktc~?67}K8vMp1TqgbL+iE3op&gFc&q)xgI zdUh#^mpjK!9E464Shic3u7pI1jaICa#svzr`J+Gz7-)e0bH|U6-pGj2259}CBnEY@ zzxv+HejT|Yr6y?VSc|)NZ)V@j&hE~<-+OOn?q4e^f*c$l{`Q{|iF%IvcTAXvU@(jS z2F#nB%nfleFME>wkca+yQ$pOsb18Ai$Fnp3AwQh)CZ&`#B+)z}8At_(f|M7NWvTL^ z@>Io8MXGYBlIHo6q12k8HL0~jYiZh_T$idEs-nDJpua4$bKwhPlS5NaokhjLlYoK{+A#a_Pw|?9csmeB=N+-0irl^`49?p>P z(S)WPJ$fu0&m@ydT+5KokMa5g5nh+%n5JlnlmfgsHkOckBVIjlP#sUlk7eTL^uV#$ zNP@BAXiSTriFkDX3FY}Qs9diwn3Gye8&jc%wMNDXWi+EE;6me=c~Vi;L?&%##@XF7 z${UHr&A`0LDclebuLxevkXQD+%nb>$7pN!;Kz-w4#HTk5!o`mzpwjfn;e?Wu4=Ebl z7F;5`8BejBPBDS-FiM1FQ=~I)UJ$SAhp!jDi$8~4=o`Fdojq&3BNQ`_cMV4NT=nGn z5l;60h|ha+oHM? zmYbaYYu?j&POWj&3HPz|$O&eh#3|`^bxShM%aR

dC@=<^6I{F3b57JWWa3TJs8y z>#j5N?BASZUf2BnCNB#Y`Mj9-jnlCk_vF17xJY?+>&Y{jG3#}Q%|4KX0gz0lqVZTV z8BNHYcf^rVt)mzEvY5)qN>UH1T8wCl9F1vHUOlu#o~(h!5Nz!LsRBC1>z+hrQ1%RTZL9q1_+^nY*Wg1iPN@8`I+H%mqF8 z*yuQp*=UUDYgto?GD_l^)M$(-dLXLCh84`w%Rw^w+*l&1CDN+yOJ?A{RJ}f`(wmQ> zOmrlfc`lZWrei5ZuRKVv4g;j0kPOkQqM#L0F)gZG98Jd3F&sKm0bNj)S=6J14qM!Gln{Qn8VY+7*X~ z-=&VHM$cr@;~f_=Kyp2;&4{=&MG3!!j9+Fl5SuRT!+ZJ(F$GiPM5 z)C^5A-l9-=dIF<>RNgUaLWFwMBByB)S2Q2A^EPY+c zNTCJ=a`n%a8aum6`#|Ga7D;E7_@$@>{iwFRkp;=*KGA=ErC;b)Ge zSz-ngjI}5fm~697o2+#jm|3aW7)oR(w9kD2O@b|e)`9G~NBh8g)%&~H2R0tsn41^D zYVd&#;5*ljSeo;z|D~b*LXWd9C=7Ok->@5ew%u?a_CSN9eiwT{Gc9Y*#AV_vfuOnM zSYCqJ2lrv2*mvyGBKv89gJx(QmmWIYf}?#ViBpMA{s*%v9Jwhw7S(T9+1LJ~%TegN5mqy5kpN2B^+idua2rbv>~Vip+=3zm+e_a z?=3Has#RS8@*;QB$Az~o)NH)5e)A8y-s+lJzx|e9s18m@w}V{uBll!f+z73^y!X#? z1t~oFJdlapv7`cgc{KX`SS*P~DWv?-sGNyMqofC( z4cU$4Ado3;5hzDr1F1j`33@In4waX=4}=}J1yS5_yHXT4FIEP`X7Yq7=I&Y|!rk4q z92X*@z7bz*UxpARN$ue$mGS?r`m()%Cd^7nH0 z74+^aKihW?F_loZ%YYE0H-}XgvjJy0JWMjFu$CF^NGj)*BtsJih(Nq3XiGEG69IQh zAWy=NUI9*~LLkDF)}S^$cs@oFF~*mTlFa#ptPnUcK&8oEEa`Bp4V>wXs8b#aEQl$+ zCH^#E5ZMpK$T3H?)+pt^WImK;;a4y@AQ9q%F?Ru1!ucaujUM=b+lLydtoO%!X4M<12J<&JAn`G`Tt}0hXuIyOY6af@6v8SBdc$zOr^$Vf36T+{mwoLRb zRBwFq=xax>Rn1hlPV_H?s=0Yot1V_v~V}{p8?(cCTDBVyX)#akJ>ut2pHa7 z7IeBt7QiKd5TPuBUiX4t7jjtLjUVeiWcIg#q;0O(|oo0gaT3C;}j=6$xl%lR}* zFPrDd7FstLSAYmmGwL=7K?cxyfG{6_snWhf&|UPrsTr-7grOyZEK$7(&PI;6FyXD0 z&2>3IwPFyd@dKs^awU7BvM&#z&|KLVAP~h_yM-=6S~*0Y z=^89?|IgIXUbnk1GoO>Z*0}R>d7WI9D>rbqLc7;HuP|#ee*tIdol~4G8}~)3v)h&+ zjYGSlJq1P<#hciO>I4}Ng9lXc5w8t0x}bp|ISSbW2M$+Ji(wBW0CKnt+Ulc_C17ks)= zd}!HS_l_!7p~v7F>K7@R{HCVgb=S|i+4Ga{sv8em{p@P(nSahfm>YkVIGiPM5)C^5A zep&)~#yaV&)J?HyAmSsB<9qBwatKL3lCL6>ksL-sog9qEks~I-{-UBQk6uyNZDF0* zvDjWO_Apr#b9ac)hrYW*#OB&$5oX-{FlPPwTI?!pmha0kg7a?KX3Sz95e6C{5r*Cm z(|}=^tw4!VZwR#;`aO%4Fk}E>fn|r=G@Iir4zMMMBIT*H?^ zumL>q^}?4<-oh`2GJ}z_tU0rRszV8@V{w=RR1JO7IVI?RBTE+*I)UlO6$_a$O%KEo zaA8%0OcKxt(|vpuLHwbJ%7U1{7(vXPW)?7ZT(c8g!C;<7H^EEXK_P#yDG7`r3G=p{ zCr|bD^`AT$%sXPBJz!R_3n6C+_rR>MEluZ_K{rNIu~8C$hE7T33-E)^>`rn5TaFi zr(x@SLwB*E`&K}#4Ne5ng4_{$&DiWLurILOX}qK*&KTALS}zoW#!wat zK|`!k(17K|pn+yFlf`0vMk|*TfNL-Jhd&=^?P6P$POv>dYZFmFb693AKyo0Jn~n1T zt-1R~>-E-jBce4$WD~2&8AOh=NHCHM{`B|$e?T#X#$cX;8ih@g!A6cDIS*ujTtGGl z#1)U$XyD;Q8Pd=@W=joAd?F_rQ5iHz)ZYz@1}8@f(%?J!`L>6PZ4UzrSTs^-dw3>v z021#v&L12s9vlR=02#oC2InRG{L;)}NsAVxLBt{ynX%bfU;&FTcUpo=GS;H31!f^= z3}vAZG=%X`3L0P)U<_-ZSq$206+39Oa!CQG<@1U~ZocLN&k@FtdE_2=j^?A_-ZeOg zfjd3jKb&v;hw~gG1{TT6aBM88g~!t8U@dLhj74b~)0^z1(4_kf0SZqf1rPFn4GCKv zkYLmrHJYJDwT$Jq>|4=oS)n9`Daf~xVC1i4!U~rG&~REOcsi?v(=gzrzC>5AVL5uZ zturs_R_LWNNtaOB@&F2iTxs(yTbo7|)c8BAAm(YO~5THf{EWJdxo{Rz+hy<9SaS?ND zkDD%ZYQelxw;QH4sN0=~Q_Qbs?(!}bU$hm?-{Ri+l&kcaeM# z$&yI-Fyl=iSNHU_C-5sQ!$V0Q^fu){6b4gqHYmhcEzq9ovauezefQNK&b zJ!g$X2sqm#PEbA}_4Ba5geprYkCfM#0M>yAURk!I+3%nefB7i~4ymvhC z6Ym8+63n(6Fhi8Hur?!Xza48s0xtW5oh5W}jzO-fal#L?d+TZ^dLaB<&01zJAKN8En^Ylwk&x`d%vHq4v5}OvioUh>~ayP3vU%B(F zPuxV$-tqv)5EN@HK~Pf{-}@#_I%Pb7+su-QEL~BX4cqW$P8p}20;My?IDWnGT`Y%- zv&|O?0618u%F`7zbhZ{ji>^yTpi4Rwy$ShHfqA$CouiCkO4`3RB&{N6!;xr>HXm-; zwBQKuhaQfa+~Cp;jwFC1oI68Ek|Kd@gE897$ESp0(Xx_`k9X=pgNvrrk>LE>qX)k9 z#$(Lc`7&I6$ugFKDbWEp%_z`{WGfH@U)Yiqy8PrBJVgPvhGF zMHrCbScK7PFTyag-4d zLWZ^VvJl$L_-T}PZop(bY{R7b(!J}ffeysqHYJ`OcRFQ08&T?z{+OPMOE`Wp# zYk?!2X8bhDJC`|HB2Z(Yz5^Onkb2((mDyVeb-uT`80wvu@H1y*u+$7qF@7558SA98 zQZLn(uszJp_9gG&8~JM_KSi>{(nW)SAh`-(Ya`e;+z8f#8^LzqMzGz?d_}Ry24+!9 z=Kmy6Z(47NG7Gzth4$c&4?2l*fx!Pm=m9HB_ZxZ4IJ$zDMr%M9@E=mx{*em&Qv&QG zohkUA6oSDL;zNQKkzVs0SYv0ybhW(8#6UnW$7_8wcb)44Kqe{lOgt}GBabm~b#91dafkM)>shyOzj#FX>!j>{DI}>MXcsF!CPXx%b@Hd7m?XYG?=&cs}^&Zxe|QLjH;`-Xpq=)pw!sD$z-l=$y_cxhMz! zg(RQiqdbRwVp2$nQIWN!q?GbSeJOv`pOT|8>+>Z8sbDmis*BdKcYm@z)evo9Z8_PP zQlbh+c=8a@1J4sZ_$Ke{9BpEKbf6Nn8lbPS)Yr`V6zFRz^@Uj9Cg^J}^|eg% z;ZUyUWIAD~mSI|^dOAbZ(S&7;jviCvnPk$4TNxVp5ceMobGEFM38yJd#jN<5FmL-$7*A$lal657j9W1)Yr+bf+>R5*RK`re zM$_0iZkT2wlP-70o!dK*b6-}sLE}|o5FkE@@;Vn4bpCk~6?Fliq>BK3(^A-HcTT|O z#}cq;dh%$(Na`bo1=oTta@%moxpc}wP){R1HNumos;%4|fKSEv-;`}6u^G`GqG9%=LJa?WmeR(b~bi$lxxfdJq z!pr=NO(K~QX9Qi!i|2{SKSG}4ro}kOSe%~`E|9PfIsg+cJ$^&_E<&&;lkWsG-~3Y1{7}!$f+i_U#A_%Ek~V}*t+ZoF$fl<4 zE;i)OE~7Y?$bAmEb*vU6BX^poz#+`X09+(DxAgw%OTRw;%j4Jg9Vv#6&dPS^zFGOb z?j6^g$A55Ue$&;Sx9%~uR*Nx5FemL}lq3fG>t@y>3;H-QrNLmivmAc(fn&&?Mz5M)> z3yNBh)uN)hOf}IvzutJ*FTN!L7z_JtS<}+7l%Z*MP}4x9vPo>$Yuc09SkfKwYnq;k zYZ}F4(jEl80Irc$fP_ghZNL_Sdia?r@XwR?#KBvlBn{qbl%#E|jRC2L?)1=VdnHtg zjlI&e7?$iVB<^sAQX?5dFZ8G}edH;N#vGT_hog>5ikETJbH=o4l(4G362>LWEB%3O zUY2PKd)ec8qLTE1?CYLXi|CaS=!d{FJrM?vB<8uZC7JcsAfzn4Gg4l9Rw?h4m%Nq1 zmx%{q%8PyAwY>H&k>my4hrTV(IsePj4e$R~rgrY#G#$KV+|tU2&K1_$z^% zU8>Vqt|KkI%Z;R|Wo{H~?Uhqklf}LR3(A3le4wBlSW-fB$BRmDK^ZK_gGFVK=}VOr z_QTKVp++rBd*f;#^h= zTeJX?4#5^<^Z>vLG>P7W9Vqp59{@X;fB+hz#&q1UMLiL>!hS~&P_LLW(EZr@9D>gy zz!gm-o+JFO^tV+>KOR@ZQ{?Wsd#Nu?dH{EL8^A^Kx7Lo!t*^Cz|Le2Cn@rkob`RX> z>HYD^H@^Bt*NP;xHm>;n%nLQI7ypi~*^#->cOPCBP`Z7jspF;mrTqNSD-T{*`e(%( zb**#y<&i(v^}OHSKPN1O+UJFrzfcVIUJbn^{`509)a_SBu7+NJ_>IQ7k@?Uo4=+ok zV_;b>DM@re87jy_1!btLAA1V&o}#jc>BktDls>ox-GQJV0mKG`egVND1Ynej<9g7+ zP}Fh;u-`=AkG^W?me?%qULEL=_ByzTR@+VC@*BG;CwbB(NTK}GKJc-dkT*`+_E?N! zB9-gGF2_${WCAZL2nXbK>Xd^~?W53}Fs=Uw+t~u>Jhk$sicyrmHBZGTTGlmB#b`-a z%P2Z`S)Ad(yVaHZycR@*UX#>1m8bD{#wu&?v=A0^UfhjDF;p`fkAo|7DHXXU+J3jo zmJCWW)b?X?oyl5uAdZ>_3Aj#&;aFqWd^7<6ZU6Ym;o&3W~(F3k%so0c>k)i6Up%b{XV`=Uga=q)=?0B0hEDZ8i zzXE`Hn2x^1_Q7KN;EmR8i>(92)`1)Cy>3tEj>XPB#m+q|0jVuG8$_qE`K6~XJw2cJ z*_Ordp;qd?pqu_RUAIGpqwhmr;5s{^$z2IQ*>$w_8QUITuK8_beMQ}f_1rSKX_fa@0~4+ zhbM}MCl-{6f;>@FCjMVoi)yy!S;N|}%Uazy9m^&yHJd&MWtFsNL9L8uDI$#1w%*DWz1eT}=-7LZOd z_%&{|jhFh^ZVbK3nR{ZkdJ*PVJ99%k%gWd=Q1P6tuXu5rSTpsK!>IpL%!4XxX`amT z+BPKeNOmfjh(nRMTx9LWekW4Uz0r*dtazvIz;9Ir@F?NnUOETiKaXWoozHp&6XEB= z1rFTnfU8Km14E2V1AR)+4BBiG;8v8o%-djU2v@p}It|AJykkc76(`XT0QZkzQjDXsG@m!2-j zTP{8Q(JH!#=jt1|Ds}s`6x!xeC|d{M5B1dDr$5xY5+=! z4$gu=hds6V?$Dvuj8|}V6@WtlUx@!#(UzxO{wmM$De(CQR*m49V@&n zb*~D<*SU54c+61e7e(s|qz-(QuDvXyp}hyYL!SH*=+;2J#Q#q$XxTtb*9kIp4o5@R+c% zd~CkHsPtWJUr;LZrge((X^7CajtCPIM<*f>fm^37ygnR-kl%gFy*@nyZ&mkp64??B zHFsWa;1e(HV*UhIEZ?|oE?afuKvCIt#aQq@Ixumox-r@Zc4LhG9JcEMt9>BrStUZ} z%B=S)ZO` zK#sPkOYpb7&#i=k@onmuZ!9YPSMOO+A_X~8P$El8`@B+A2CfcQWbN&Rj@?Bi^47Km zWnV$wS5)>fc0m2PmTjUKNYf+&7C5c3LDKK_|hMg)wi z?8q7f>V1k$7*IiS5_@oY-uGFGVa1&%d4e%u@r-*-u z<^T_^sflsavcPfNd!+w;((nP<@&Rf43)x>J`~Obv{eXN2|6mgU4YWy=@Z78T!QJKk*LICA2xBfGwBD{M>9nvqR~6qT8g zV{xe@yPFnH8z9@dK(@^mHHx$=!$}_&+t5(R724@JNKoVijX zlIz&sETRE*=0Au3xy+p5obUh7caHw8r6umek!k+tY`oLw`!9-^N5~MCBYvOneV^nT z_DOyzkn;}**g2RFjRgF@e0UfNEG0S|#W|Q0^1`sd(n7gdK0X|0ayZwNZys*Ww+y%B zTZdco;;@)+8*aR$4bpizBVcO54QJnvvFGr9H*cT9GDNX-~7XHl(#%X`5ME2huvNv@KJC zkTROA zW%DxR(4|YKUraXZ!a1dIJuAtI-uiMz8JATR38UGZtVb_rlx${P)8m6TCS(QKAPW~W z>Z>O+ilj#`KsQ?$*Q4j<*Cx>vz4;YcQ?euH3%T6ngf5&=wCreRMAKUg%?nyan^g4{ zL*IysT>rvIVM4w*H6f1}J_S4oZQyeZwIsh%h`AN{03J zi)e(as^Vr!;~8l|yg%;<76PUmUA%>7q-ZIW^|M%5Yd2fu zD5=9tv(KflS(17FpmFNY=Q1VqQ|zJoDZbWEkFB3Jn7w14(N9ee z)lbc9{q)%SX`|ggri6ZKd8mGBUF)aE)=!(v-m%Zc+fU?j z2^k=2M3be2%3zSJCe&+%$()2QvZ|KSqZ9JDlpVi%$NwPk+;d5Pv46+ziKzogHU51= zNIaJy>6xUOIyX4<;_0E6;?s%ec;exts?dOaC#VOZLcswF4nJtksH&`JEbEb^N{L5? z^ay}i_Bw!IF;%0Q6DKd8zC7rvMURaX@)J23S8~T+++QQ}$+NGVJ2QB3@I`n2Xy!^m zLGC(Nc;bb#=U3*Hl`pRTBr)9@gJ(4RxL)AB?d8aYe^wnJ9N+y>c&*Wu2mR2*PGL@jW zo?v$z*mzJ;3X0yD21d_kv@{-pTxL9@5o(U`d#lH++oVU0Yu2O21>A|~Ayv+e>P^%b z-jDm8kwebNK_20C4LBo5RYIf*c072g{}P@DwLhP^T2T8(U^4WpQ~8N&h4HDr*9*$4 z>O^KlR{Kw$9qc=B;7C6+GWsWBZ!j^ zSt`!bw*+Xj8h2L2on_(h%w7a#ap%n5igXWMxUA;_WpSL~t6IgZi?YWYF%yczSQN}S1OB2=|Y&T}-g-RjyUo^55 z*)ax7vHF0W7h-lE*?UequgTOL^RV7rcAj0nhXrQaJElDZJI~J1X4Yb#SLR@LUd=oX zJFm8$W#`pyLn&ys7WQC>kFd-h3}?AL7}4PP;fJFn4~wo8F-tPf4tp>Fn=mvTw6A!~ z+t!FpM}c#qtWN`_0Bm>rD!uzOfIcOgF!~MX)2Ml8r6$967o=vXr6d@A6>hk%YT53A z=_s&KYq6KRflM%TMFu`n0E7DA!MYCIlD(3XQ_6Zg8wzzMP{Vf!a~v{p5d;Gl)q_eY zK!-`$1|tAN3=$+m3XMSu;b=lfhm_5bicZ{gjN_(bL&{dtOi)1PkwR`Mrk-qOWhvW{ zpodvaN-ur6gMytD?4khfBA=3^U=Ia*DdJHuA%w} zM(IUyPet5Q7LLqh5tPL}GueuGWKp2cf|0;uR#*{^(2zzNbGA57%Dp^!mSwc$nJgEb zApMX+54MU^JPNAvdSfuT5rhgbERhrqfbkB%M}{B9n3g z!5tqffc1?+E-x0)!>x)4-uB%K?fN1V2_IN)3x)@{E~$AtMk$|{V{|H*3~`sJ%iZd9 ziMqb1R5A$Iwl6b zF2AQWm#FoWI!uJxQ$|nhy$3icDr7R~QFU?zJ`B}FjI}&z7cQMVIe6iM4GYPWM&oc5 zz#G!TGI?_VK>KPqy#~*pKYQL)m>#7W*TSTRoV6Nq@$@T$XD?lJ=Z%eI#z*8_j=9Tf zI@?YRog6%KrpnotY+;vzxmtdejY3jA%&tR^8jXUN$}C@b2KUC{PvS6!zzz5MxIo3~ zULQig>c3*D(gh`-q}OL#Mch^v`fguEP!_k{zFHCc76tk&7zsRPg%zQX04r(C+2TA? z=E<{EoTYCG(EbhiV4j?x$%e&YkkY%LO^f$x4 z%b~Cdd+E!EfW4h$)Kt+5id!j>a{;QpT7$k`{DCz#fX@eS99a#_fE&HeNrrXuUOsXD^ob#NWAr#psdHW;vLC30%76dd;CbNdsuBQ? z>4Lyu+qq|BPbGA6olzH+j6=&&N0#poVvNvk}L;AT@ zq2>#eMgdI~Sg1KksQFnAuIWo})Z74zwwYkE$q6oY+o0J@<1Ax0c~(G;LHsSB!$cZ9 z#Rdfx018}fDfm(7E`5t*+K-7=FGB6giPn!D{vY_Cs}Cv6CR!sjIqR5|i#ANCUGFI0 zV@lS3$tzAxFHe+3iUnkjI!g7@wY(v9^y~>!!I*>#OECaZd&0C)0kUW0(xgq*;W>R>zc->x zEULiuEbARu$a768Ok_vWd7$;oRas32^$?>j#+eGIF}5MMRv0M&rcNqYy(62Z!oC{t zno4oQdckZ&wXpzpYb2{C#tY;QKT@Mivw(C~LeV@6^9O1) zpZAoM&S)!I4<;(jm?#njy%f~vi0Dm@Mk`~KiQ1>f?Rt`N&LFu;UlKOtq>w`-k=?*) zBMv6%0p)j)?Bt0aVQ|u7j~Mikb@Ujec|~cn8O0t#hUbh2k=rX(D$k>IRp7+V&Zm~z z6H8s&mp1SJG8$SJzbSkc_qA<&Z~C2SVqS^}Zcg78oy1FuTf|8&fOmP*U{!)iX^P-o z-Za>lM7ylcgsf0r61yy-=gc8a^u*as^qe`=XoaAOa#hs0B9~F)rfrLxQk6|9a6I2F zmN%t-5t%^O+`E!huEA0j;^ikMcDp$jD0Oij`(X5A6BMhrXt41cF;nDx`;1t0zPI!&>zrN)V zvn+?W)bxlfhi2Pym}*Y8fWF5^rVJ)rC$vOig2?TeTw-zpEHgQiPh2{GCZ%NP{d0xh zHf``qjc2mp&#MzyqAIJ!u4|e$p+4K+pP9&}U&J8Mbo3B$) zApPHr22jKk2pYnVkpwli2PB|dewoG?6rm%%w}@5u)}M$FbwrzDa2UK z;Zj)rmDXZud%RGjP$`VLtuWvYC5X`ljEnb;}xqfQqDTU0^CBfjpN0>_|lI62& zQH^NG>~PfXVTzS3lhrKIFy4ZAIfwBS1jfkGS9MGW;nES%+X7rV^n85msKx8~XlBnx z3+4E2d_Ejk=k?6l#&fGNOvd1>Xe-5x-iSBc8*#H2%(JUE1oXz1s;)rvA$y~Jtv4L~ z>h;^G-y4aVz2R%hdQ>?uGv3iP-!n@DoAhMh{Vu2YdH)^Uw*ZbAku?QViJ6FmNI4e-d9LD-1V!`T4W zfxR48xr_$#g}9DYCCqD}*;=rOnp9Ld<$E>h|01I+R{B3^W`r3MaoC%W%8-zPQb-Da6uC=g3Jz=pv5Gk%1*=Skk8Artuz?Sm3~@e2-ZogM^`n>W|j7{`kuM;Q=K9(aD&?vp#8pLi?GdA+G9> zME2`wTkDOl)*DkT$@bz?CMe8IXp@SZxQhMmup(HKZ)h9^Dn;CWJw*H7VKb8}Bl`~= zya*p$38~671#eLBCMm^{IekUJtj;yPm1KAq^Ej#3GJ1^SQdZHXOgLDJM`jmfLqb|F z!Z1j+6S4LLk-Hu4;(mly1BKOE?_o8Yqd^mfwJLv1mu;ai_3<(?pXn(R@Kd-aP@7>4ZP)B9ZD^A zZC-k6`_hI@Uy1_5R2{%nCVbg@`*K;>TNZn6U#^JQAA>#%MgosnVMW+WLpy2A+2TA? z=E<`HinH`B0on|{?5K!4Xy=lRZ0C|4GaKDImmtB~dV~wuxrAz@y-lzei7B$3OLp*N z+PS1U7uRMrCMsg0EbPAhB7(A*z*)px5q%bn1Rk@(im;oiA&og(oM*~Bd6tT^^eq7@ zHGQ~$Mvu;4P*CN=y+LUdd>4P#0_SaCgj&M8m$!)FBzNJG##eLU8Viq{E?gt2rVIB| z;4kOQlZjSVpKsDaXe&FsowTmGs9!nWg|VWjQ!td0G;9c6l8Kl`qlZZtSy?eW&56)SU~<<1Q?J@PcL5lN#M&x=<_8&lf6 zIvNnF4)E#(Y(5ewMJ-;PeT`nvM~pomamwLv!v5AgAGLeXc4V)z)ksWhv^Tu?Om=VZ ztwyRGO$Ha<+tEa*9z*aW@C{iDOqEM0lP@iOi=-b=@IwR*`G1VoHsb$jQOLca{0RkY zCF>AL4T1nYZr5Z|N?5}WP$6y?i}I(G#!$gwk_HfXQb=5$6|;P|XXQL!nc(uQRBIBX zQ`;CEfLrBO@@5pH;5XiT0td@V&w@2Y`9jtB#u?cSGApV#zglBf;0Xk){V_8uR^?;| z9*XHw&B|V5y0Bif15Z}Ao9WV8+)`zcVpTtOQA)t9!Jg-8>MASlJGY+ag6|JZVVxli z+md18AeLi&VWGlM*+k(go~ybxqDll+ z_r2IlR-|jt@X&f2<*EOR0DPE^ZZZ_tbuVq{SxW45`rn_tt@z%X@4QKFXPAq-ZoYXN zhNJCurnu!b=EAB@=bKn{L^;XlT%8HYrq3CBRe2I`-p8F${)U1oqwOy#jRJBcIE}VY zIJMjy3v*kLX?PoL1fN#jm8y-p&iTIroSRZP(`y=TKd<33*CME&9bS7hI@vhnEO(F$ z$yM$F!@<0%Sz?vjwjT2)!W@OsDo3HvFrUTAr}5yk*v|1^i+LkBOgZYms>5MAIhGmJ z;Rrc-G#)Lr@2bP-vkgj*R;B`@}kqd(x< z@2uu~DD2N{AY zK1XBi3Tbx14ck!4q57p%?R82NjW);J04=ZMZ5$@)?ftaMpbC^Zz+uenxC>(yzg2LHHW$9gMFFY%66L4 z1#{DWw(G9mo#xy8r}&aFtWHQN^6N?#YvN*7k{&hE*pEPu@GJ^kCqDk82|?1=UxDRz zEuT?9D=R2+if@I_mkJO(RLFc$XhKXOr!J#GadV0N+z6v^HWYKpkq?}2|N1gX7br+m zKvN+MTC>Zlx*59gExyt0x0Qc{9-exisv!6*==b~Y`40XkapZs0@=M?5U;5VH54HFM z_kCut%%nuCf8cW;g8SP9k`UbAAo!mp>Dl`my8JKsKlf31|4@%V`MD3l{g?e6{@tJZ Q5ZvD*`uo1{AzN)q$EHlfPkhTw1 zN7}P@&YXMa?mgfA&beoQ*VdL2;Ck(!zcEOcAp8YyyesB5R{sHwPXtZK3Yw@z@?utm z?`S?!h-4!o>SFn5A(oA?c03<1B(jM@GMg+&S*g&HZ7HO(sltZrhC*w$wa}JrD{Rbe zERZZIv}fB39oY_1hzKVIEpbiIl6NBft=UedlR($f&~-6g3UnJ9x=l>i3c9w2ZZp$u z1RZJUwlG~g=sFs@Zl>!5T~|Z5bt#hGR2`Ww8kTJ7re(?tB`RyBf@&1yLP^u}rmPk< zIaey076qkcX8w+Io=J<2q^Xu}83i5MvHAJwKowu2zynz?RtlM%H#)uq9UFzu>gOOn5iF4Wy6C#x zyT@-$e0xw3%*ZRkPeh-k2T7l)774XX_)OfU77ewOLM&sUmTap8t+n`L%cVp*`DkDw zpO!3Az#9>Lkxh%^aVeIq4)OCg-NlO*t)-r(ZpDYHDukxRadt?&(?9n4UQ| zaccVbZ2+LD=NC+Lf&8e4L3y+Ss2;_p!jYUQFFEXyqUQ68H(N#17mT8=RfnH&c!qTj zGwy-+m+fyTZZVlJT zL0dX#lY>kb*4MAAk^QUiCLw??#1_NX2hjK?d|eDfk{@$P93L+q`a+O$$3?gwg-g+N zth$SDKF{&zb-)4zmyZB%T+(6Vp3mzUx*1+TyFtvP zrRA=nTG!AY$?&cB>g1>`9j%e0bO3q+J{?I>ifTbu6ep!1v?_UQw<^jz6*ccZNh*p~ z$|(xn37zO(5Vr+30Q*U_6SXL|z{eyYt_k;IdmqLU@#k0Dqw#TWixxTfG%nWx!YcL) zBQSIar>+1k`l&1M;Azx#m%YOWiT`QpT8aa8Y=S8}Nwbp6K{RpVI;AD*Br!&nDi#or zTFmMBybd%Ir?~16cIGE1r_P>r;#0G;r)Qny-1Mtcr|0K@ZYE|Xr%s)kI^K*&=2FfU zfu&rZEr#Gt<}3&w?wZ+RXN~Nh zw+~wgU5>mAwoo7@O(c1SywzAdKE}=P;lBa8F5HPt!MECnN**zr9{qm-ui77sgqVFi z5eKQ8*d$Fa)w$%QL>NPPN`%O42M<6VH1*p2)Y&Vu0 z7Jxc4*9=aln~^1pS!@!)@Ydu6A7lf@$dL`e_AV0!x^$f!u%!bwIdGqR3thTK(zo8M zlOwitq(+WB6|j@=UfKlMBdA01JWk33Y#IT}DVaSC2kbMXd98?0FKMmzJjSc1<7sd( zyNknAqVQS$6$psF9xbyYgBHCTyMs^Rx+19g=vt>}KuqyN0zWLq?=l@{sC3YC|-Qcuyat*ewMv)tyer zv$>bP} zVIp{dn!BD`>5x~t2Ud3M|2i4ll)5f`l@i*wethMlD>q3ak6R(#Jhy|K&dcn2emS_j zEmPZ;`E~41UT(+ikkxIHv87CnWEk`VSsdo=3`fVI@g$D4*xmRY7aQv$$(OxjP7b*k z_pAyrux!btvIVCqH7{4nkT>aSL7tyIm7yo$h4f_*PCHOlQ8g5*n`JoV>1MTi(Xz_s z(a}-0Y-IGyDkke0$l22|<}&&{?0X9Pc6fs-=T%czDl|Wnj&SGBqtO`<0XOwAH+lY1 z5N%3h4Cv1DxuD0^x5hb`qaXs#8(rx=^mR*|`L7lHw@i?N?2!|7a>ABQ)W`{!gA$=Z z6XP^JhoXsyrcj6CCHR=g(HJ=g9>!Af(bcU|{G`a^Cz>Fl8kffo&vNSAfQWSum({I! z6F6^%kr7e^1JU200hJ4^S_lR4iYN*e?lX2pg!jce-=cjt+5leU{m8|T1LkAu38R|F z2Sw=J>b}_b#iQIov+@Km@F!xBBp-2^QNQ_r1hV}wv z$Zd}UF5?|H5M#U5f|7)igPH{1buRKM2676PF+ewyZgFBb5d8s+N|9%s1V3;1PBkfa zK5Te+ilmr&*@Wp`(EF!Pv3>z(v`ZN{0eqU_s1W6^ucF4mixzQ z{bO}9W=mr>8CxM+mPvn&^h0LK^V4-SeKpc&lfAaIw?_6p$#*R<5eC)YLgT*$RV}K; zwD{e`9XD29SqHHGiU-vllFA&R$9C}@;|&dPZTv2W4Yh;3lp0(_Z^ zs?&n^nnA5444yy2J`L>Ai+KAsiaZJ(L=Z~73`!m{80X@61>I7if~Qu8o&eLjwgwK4 zFMJxDR=RhrJU6hiW!u+8Vz}xATygx|JXa^fwlr*$VTPdHHM0BG)<)fg>Es%bZyv9a zL0cLG$xwtpdk}wJ(+LMlFtmW8iLBp19g5fC(>P%7d>Cts53cr-c$y8%EdDdDc8`F5 zu69*nT#Y=$HUNc&Kj3@Zyk3~M+%!NC!noMSwqh8t-Na8!N19pFJ(Gm8CLTpLz&xpn z0*}k|1rU(k@q-Ldo12u;anJ_>eSp3^`yhR}%M_rmW=pJYnFl~XgzxORk-nY;h&&5L z)yT+`^wUVU5H|kR$C!yU@(k&guWS~ck_U=on2_~0kpxsDpnN66_JiuSnYjRS5rkK6 z9ODh)`(9&{+ZelX3?f!44|#eqx1y&94vJ@(s{^4pUn%=2>XPuCVRvPu`kfim@&vPCb1*A6OeCNg51S^^9>l$Tm-=*WXF#C-Fue1Gqvu_z0KQhyuPw^c%`Qg_C#mudIA$JNUSj6 zeRg_z*LZE$c%6*f(zs2=SIA~J<=TC?P7d4BVVfLgN>(XcsFQuRw68|?x$!Z)0{Bz> zkxhFAw+Kb?bNDn0DtlLBt#Qto>|wfx)exOTc?nm7V$|ZR8-td!`8<%(VnL;lDwn99 z;s3DkqaR|KVlqZC$D)|@u`HQ$5c5kL>JV2vtYD(gYvNdZWCa~|(qi$M?OHacreESn zFY@a9cj(Wc8y>UFB8ab|qA1=I4*XeY`$E|9g|K-o)+R>Q1W&B8mb_6Me;|NZ+bM}- y4+IcvTO{!)wvMiC=@w6l4+NBJFZ7G)2Lgz-m&HzT@PPngZHR~?4+Rix{r?3Czby#> literal 0 HcmV?d00001 diff --git a/tests/unit/__pycache__/test_rollups.cpython-312-pytest-9.0.2.pyc b/tests/unit/__pycache__/test_rollups.cpython-312-pytest-9.0.2.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d18fa62ea712dd45f8f765b5a7f08e96ad2ab824 GIT binary patch literal 23439 zcmeHPeQ+Dcb-%;s0R#wABqdV(5G3jgwnT{^vZUCOs7Q2dWkg$&?ZmWI6vCY-Quv_m zj*Z{oP0W5hUIdSHGU z0+%?E8{kAFuDFBelI{TyPp^0fyl};t5R$@xkn|1sNV+TGPgV_7k+3@v zNCpRj$?AdXWN09y=c^g0g?ygGie%kDU9x_lo}_yd4at=QE0e1RR`HyJ`w}M#&v2se zH3zNNKqE=>Ls}I}Ya(d@NDH#G)g-MN(n2h)dCU>1$+SP58dAfmq^L^xWLgf(=|p05 zBpi<=;-iU}I+RW+9XIiA{gVz&5M!#O4kabchg6gjYAnKQ?$Oa9u_xlxeEXHLRQzx{ zep>S#jtvgcpnD{y#!p2YTJWfJW)$vpOpU3diWWLb?sY7l9+4gz8W82b4@z=eN~uE$>44=~Gvpd=iV?;UPR7(z9dYBXcyllOo$#N32~sa{gYfph&F6TP z@u+x}9K*&}WF){j3ty4(CSKF?^VfBV?XM;5o^Q zZp?Gi@tS@g(W6!wJi`GaWS(EsgJ(IV!BT#;$zg=D4x?_em9xgUZk|of6nw>&Qtf!1 zMyaCrvLoYxXLXB$=*zl?coOrdbwYEV5DP7En(SUHs*O^D$WQaCvziU}!vHkKHbIwJ1(X!-B&(*sLt1=t54o zhNS!B>X@QPvPyV%01qVY=%1|8+$R(1n5sEaBbwj*1T}x}$QX9hXiU~>qtHJSQCS+1 zj18uhZE@)I+mx~7$fM zh)b@8r*!~`1?O({Iwf*5uq&@V|mpLImuxeTWrQw z7@?gDayvvAS+X!jpN4mkbBXR8j=gRY^JHDJBkNIqu=MCF0$6jNeV!Q8K=pNWgCVWc>-ocaziZ?u-+9o=}0;ZH&gE@3NDO=w8t; zR%N~V_%5icj4E47!tcB6$O=YD#u>)<0F3WW9OL2d$+^b@kzi)+gS&TFM)a5(-n|0` zdUN>fxQQU^ri3Jlz7n8->^@|9P8==}MVQpT%@JJ!F7DD4PkPDBD z%MhIv^q$-0b>7~n2^x>%G7$Pb@8tL}%6u zP~U~QEKuKV=|D?E{U%HKHXU5mCl4HF)Ka$Quo;XO<^a*){sfZ`Yx#CHV$Hkk8ZF4N z44A+Ko9JD3TUsYWHZ{-ktj1POn>I8#p{8ET8?xhN13(2n_MKJWV`91jADd)}!exPo zbC|4&{;Z&jbKkP3Yd)i0j5CPyDiGrii%54v*qifI>}`S1d)w!|4LiKqhbk&5Sik(d zw?QvM0Hyb~u_^#j+f9`iPEwSi)S#gjv0y9p6rd}rE72z?eZ6G5LS`uuK9fPEM<*yi z{7)F6Fa=YPwi%NE8qsSI&@X|KgeE~iPXZLyVFEaRj${QGwuNX+0E9*SIrtfBmQKiN z!B&WqB7)ouCwVUt#3S-PB)v$!2t;#XMG5WBge>ZqPSnTX#U1_f7l1KRWdUqq%xHrfV^;g;;yS* z#fA+Qb*-fxIfn4@leL9e+5}Xq(nccV@TI7>&8ynjr*Wp*6CB1g6#nPj!vA` zmUlQ(l_*aBqNEN==AKW`>lWn3d7y&Io~O9?{aj6bzUo=^`Ksru@>O|q#xTanE9;N= zG$Bfqz0s)Vk4BSeaWo-8I1r7V8I2`S4Gb}FG%BX!(Wu-F1{EWJnnV3YFzGJ^8vTJo@T`iarm1ZE16Arg<5YEYC8gD+*v7Mn6;9WZxK^Y3g&f}IP*&s8D2QZJet#46%A#)&gdOFQ0{S(E6w?4o8U z(JKmBk1ngc%bw+U4H;#eVV2|5XE^ShYs`lx<2ZaN=eUSJvq?93hKa?(09s%ujU}iM zqeiQaoA_A~j5Wz;w(Ewaa?E5PERbgwXarbEb3!qiAX5XZGU+NuM!i`2ek6#D`T_AA z#0Y+wGIrf%BOZZ>N{x@oB4)D%Jw5V$cw(C+@e>N>Z_wC<#dt=(#(#< zIzKh&ERt;p-s<~6p5lHH>9~64dyU1quF1xU{TJ{5i!Wbo%nMU>UGIceT-f!E9H@^b zcvLgxCU~ZKx=!O|`lEud_Bn;7fg;KXGNx%zU+rR-jUXtZ@?bVXeHJiG@Qj@Wfa9ZE zPOiRN!FT@WwFkRgCAx1kp`iep|0YnhJzMS0R-twLOQ@JyD6G)REILkuxRYy4HO{cj zz?%wLuG@&ItTSqgJAN2Pg|Y*bVY;;Ex##Y8aVic5NcWDH&kd~D_>G84pwqhpy}FoL+1`k7c{kyV;D^wWrtBWVkq za0+)5yLL%m$9|5;Z2oZ|U*>q|?^*8ndITX^ei+iq4;6YfJ?#8Wu(v4ub8AKJCH@i* zJ#wRq8{@z4JjYKu7Q?BN3k`i>m~-vvjnU!k`Qa<)C)Qr;e0|ps zcl{(cyYHdGzK4q44_)4W<@`tUD2|?9w}t64f*w zYeC~35U5dZM!t!knpS0bKam;>2*8g)P9r&s1Rq)R#70J>l&CqC(d4oLg02!lbfED= zniG0}d=85oLqhPvHVkb?(h0;?xzVViFO2a(mP%)cY1?jgKmmeH7F6*lO+=;fzK5${ znHRwA&y+nOn1P5z1TSRu4#R>0qN8mAFP3z$k{MKd@ftQn6n9u}Au)+|&4Fbwj~ zt_m0g{1EIh2uyLDQ6{YbkvIt=0UJBQdboxkBsoCRp`EhqC_xrIfGp5#M-KG$TLf8{ zJl;ikU~(q9PeV78-C(JAXCMdEM}Se%ne#%5mmm%yE5Inp@Xm0xac}bs9&G|34nO9x zOyR05qf{m9y;;7U_1=m&OimjVQ-V0EEVZ`dWz!cipnK=}#GqK6_30BKzl!E(a#%{T z8*lvvqBG7g5emUXNXUVYEVLycvQONWh~i97$C}&LmJCM#;#j&Z87F)dH5GV2owlrn zw)Ff4Z3%fSZMovMwI#z@72A?=LR+r*EVSiVC{mY+F!Yuj8C4~4hK`?-!l%GbT1IC` zz;ZgO>yQzF-Nd0xK92oAVj>PWrC=uirtBC4iOODSa$WYW7db|VEvp#eaR2@PO_+m6DMDVu;$+}0>w zxo`Z;HGXz`UvYciTWeSou3|k1^x!yu;k56u%IC3radg@21w`%{T?H}8w1|BQk|Jda-lbb)1 zp8>x2;7<+2wq#H+5+eP1d5iwh9raJ z2_#rM>WZ`hLmQFcJEfjUrctYkYQxEApT+ArBzYvyBKcD!c0djXUqjaeXREsrdocQm zWww5zFOq1Io*@YCMqPsqep|CHe_$riGPh;xHU4k9&Uy#Jl%wgSGaV%y`z=EtXm#<|Y@Z|vi!l+G&2BVDai(4 zW*NLN>Inb~mp`DE8J9n7E*r?wqbdvqVLB9`cVt+R;^~ywG4x9(#|=#Zs2?Ox*88Nz z=~%0!8KRION;8xTAF#A9S9s|9;UKMxJ}hZ|SZgGu;$zXIBL5jw!B%|WD>B3fl^yHI zEO0^fN*ovOURshYnHk3)1!W^x+&o-UYd$#F(3JPzXuad=K%sT#8$E@VeN#fyjrEZ! zAu<&TPxKW+ky!zM*YyM%W3)h$X|!kw&j^u;KE}lY(?Vp4?i7Z%cDvRdnV*`eFSgSl z+4@C0!7thcu8*d9>t%~s#-?jQr^T-8!8{fWdiz}Pn289wE_~MXsJNz}k_(>Mbp?HH z!K?hbY0NKQd1$Q{J_85U2zpykhEoWnHUV2^Wp#FDln$O zuh^E16U1Q1(gJpdxlg4nK^t3>*`$j>>AWh(;%b=6njr}c#b8!4Rj?5^m44i##!9at zeWll%CyaosU+PWkUQb_u7cPGj2~MWTDh6ry?Y{-bz*zXbz)9% zq9)5ZK%k<#x8`GYl`mrPFCnpmRtse#)sPu-H8waXOM_%BS0xRZO%{ZD%D)4FWl{Az z^S)oOWl@h_a}-x~7i)XK52(8C;ukL+e&O&$1lCH`Z!cDN=AAd12r@c5-e5&W5M?lj z1`rv+S}BB1h>Y|gBBOFn-~i?uD+zc}E=X4J0#SxUXaKpVWE{zHBt1w#+r`Pa!hjHrF)xYETP`Q~M)-s7 z@U6CxyXDq8!F|Vkz0ci3vtq6|{MlhZj`BhfC*qNq4C~+2MSUX9((9kd zCqf@$7Bw+~Wz}B2XAUFG*a#{<5Dc#GF%?RQ0+1hOf|@k}XCf;wR_p{NpU=3qxRNi+ z%hg$nr-bQuq$~5h;DNu;e8^Cjtujv_=0hJ{yu-|g{@bQBVdAi&(u8qBTly;~O_+j3 ztkMT8K@5mNi|_Vwm3emN!)h3?(0mwLU_M-4Tb7y+KkLf8Po*u#sw1_T^<}tS_hky} zwhL54NgX#`&=ZK-N%P?<;Dn?~qUMdANXrnwW%(_Rzx5aeZ=sPN*NDBEclr&|$Q znGR~5p@RZr*k5C{s43AwEo0w;l!!w<11F7F7TU%75TV=2F5O~7J-L{%hdlj4(;>oi zB@9G29Ud`Fhp+Gb;odhND|SCf42Q(!sjPsJ2;6-Q&B~dV8o@GF*Ye`s`GfDQZhxcpjUzv9n7s4N z`)5}7=MO&jWkaiS@Ooe^n#5M?CNX#;#pWZ$#v}RQ+q-%%h*>SZHYM&MGcb&#*gwX<~(pb?9&Is+}D&t{+X`y|IiP?XH4jA73)TbhVIIo_)~_}j9K(p($`0eni0;WO-_Bj9%FyWm9xyq> zWd%Ys%Qy)hSeiFrl*;fxHNgYbx^XY0wi|b_+;zQ?W}dSlqa@=Dx+lZ~!CU~wun?Pc z;n0Le6gnjWg%jD*B`l5-oph>$+AIKNa%}pwUY~4nJ;pq2uaR=l2;AQbBb+xml!CGq zT|oEiQ?laf>hg1)nV3!C+n~%Y{cbg(rc(D-Mqc)L=1Mscm-=xYt&bX>W`^8AxVA}FRsv= zRg5w&`3hbTjJQ0Bb+8*Wa58QV8Z>0}yflUmqogQpKk9i|d6+KC_Q!BHrIrjJ?+M#) zgxd1wpMUb%CySwQ-Zh6Kiq`T4P@(<2ZhiiMKBmmIt%G6Yfw`9OmEG5l&9v_>wCsk2 z!zJU2kE`GCet>HWPYJ94wH918=0Y{kXTFgck9{lgSBbfn)^8np`OwwcAGA!i6k2x7 zHXkfBA1pTC`&NCS`S1l{u4dhAO{7p0xw`p!O}8;fHHTkyd^dO{IMvbn=DK3Iuekc& zd}wan#wlTA{>h2;h0w-X0e{zNoJJTeU@DCj&ESl%abi8=VS#C3BN;#XBOcqS=6CSz z{Rm0f-109m4ao@n6_lRnfe&-bfP4LXOM`n8olGJ}9RBQpH$aYg{6H6p{-LmPGhDMhy)WU0FjG^(fI)IwjHdH8$TYok>*4eHrelpdBBuE_i|Y z-rlwUVuQZ@57l$c)b_lo;s!viQ$j0j092pwc0Gndqr%mt+ZXNb(Q~zRzIl=h~y93E%!7nb0($)sCS@27f zrFLrc-}~ z=d;`N#h3Gr`67YLHY%hH6lvYbO<&wc`nbN{OVQCK(afKKPmeBTc?ClflKn`&2&6x= zLSO%zO4BXbU?~k*BLk}7IC8ZkS&w7`l0}^(^nF^$J&0ht7ugY^nu}g!;^6ufNVD62 zWvfNf8Ca}D(or#{?|zmc+7r?&rNgvUO0x{+n6Ht}aT!Be(;tqLkl4ogWkTgKQr6v>Jb+D==Sh6(F;`09!xK5_)29U)4 zxAe_3VZR5^9PdSreLzYUZR0*%y^z>%mKE{JVMr;L*r3y~Zly(St7Ls5s1WFqV7U)@ zzJvs&wsn>F4kEaT*4UUfQ%Krw5!A-R5kZZQPnWnT8RC9YK|C*(8K>~dYBF~w%2(J( ztF8Uj?uqlS>?yW(&b5Z;n%h3`y6XISFHD7NR$X|q5ZW{)bX@HQG8Ni%wYw04?G5mE zT~DB~B8k!PH1gA+l>*CLXytFA#2k3CO$l4B)&ZFcZMa%j2yK}a@ONELps^x}(eO0# z)1Z~k2wU_?wAXeL{RY0yA0zn(BxN({zr!>nFT-C!O(D*t@4&U}ZMR_Yddqxuz`d1D zr;#nvLae}=U~IEWkbu?{r5m$$PyhkjsCLNGIeAEhO;%Zw=GD^(j?g?bi$(9`!abB_ z58Orz;tr?yX>}xFCuJGOEV>RCO`n=OHJTh5Bm0qP4JW{7>Qpi&pN6kZmVh6&O;*TC zBpJoM)>y_Mo-lY&^2=zYqf7eH_Dr-2q(RDBN)zs(n+NQb{|Wd|lb}2RJc-vgdRxfLF>U|c#IK(MZ{i^7Rfd^$UMEaY%(6E8)?Km#O)pn zO=cG2^^dVlVkl#yWJy+JNd;o3f|DP1r8X&2sY*GOADPkMLhdDH61&RgpN(AQcz@-b z+uifh2-ew6wpBWF`}TeHdEM`v+yCbE))2V<`RjiOpJ*ZEQ@pV*yIENH01B6gOh$>! z$$W$x<>1#A;iLR0AGM9zIIOov>`}+4gOwc-XVf+7iVCAb)II8Eb$p~|bQ!dBMm$mP zs5iQNbUCYcMZ{?BXl=A^w2mV@d4kBoD@1m`!yB`Yu3&XFP`9j5SI_D^Q0FbwHL$wn zP$w4Z8mD<*ZL<4lEUZbIqH3yiB2Fb5k3)+|%9%F{ekfcb3IVQ@QC{Xo zZ8HA~8MVtcpbps%)G0fFx@0F%VcO*r^i@Y-5eF0+Qes*-qUDbN*567QIO(tL`SZ70_g zqVoI5%iOfx=ScSMSLARoCdE%kaarCRQDS4-xD*Minxu}$6A@WDu1L|aEJqaSrC=nX z_RV8vl zcd{u|-ANTS5z$n9M@f=?xr!;!BMH!@N3a+$)JXMDPV0@V0<;BQBLMFZ2*ndIExBcB zA^5BIsD-&yU=BG;{;O_7#(Af%=_0+9zL1`M=GvkB$}Rc2-i&izY@DfoEu9tGUQI7R zTk=~G+M`9GDK9CpC8QR7Nuj|p1x0HLP@9VTeRf(0>!vG^pjc=Fl13n=%kR1f)Uvk8+e-hY3!T9l4b)Z3UgqfAo=Sp@S*txyLfP$K|b352o zQRV&j&?@gKNCHTH56O!_Mtlz4js~bZXfQUW(3Mb0n~}gKQB5ihOe&K*w+BkxYVwhX z$)ut~8w{@g0LWSLwUex9x~R+y&Zuuj^L0H-i07_YcmA1MV&@E<6Faj)XU?Jzkky~S z1hAcPW51>|lewZJt0ws+oZ=#RmX5<5;+CT0M$1vdu`$c_l?YTD2ay{7(6pN20cJyV zExe#%KR%we!VP7N#T`|W3@d6gnIQhk=|WcULr>Zvvr- z4HpkxI(FgM?Dwv1%{Th<;v+Y<&51j*!j7!C;{i`|;*JN1@D|L0A_O!-(p-w0DC|V= zz`Wiuf4373_M(Y`!CB7<0n;+~C5}AU3}NrHTV|N%OWHE@R2&gYDc%~@*yG$p=JC(! z&yi`qoAeTuo8me09IOs&PTQU%QylDu+scKnL?^ZBvdxeN!zy@Gz%h!JpfPDAIO3-$ z+Y#Lv9*f1P!nDBW)P<=aje+v0`tlIslR)9#qlTlC5#>ZUqA9d!*t#RE#)2`Q&Cr@H zFeq&YQtp8}D|LshF@XLez!s+_ANvjpw5az6oN+&pv*c4@<-E7?wznte?fJ;NcHYx) z+tZozbk457<=J?raV68Nt+O=WxFPfSXZ4Mj`Y-g)JbtUbJL~Dr)pvj6Cd=2|_O8u& z*Zxvn1|z<;d$7ayPaTfIb*>SgTNeU>STL#t0(wm#5RJ=;2$ns8z^OzqVm@&N0&+YQ z2+-BgpLQYHh-5R8Z9uLQrl`!WsED;lyzsB$hV=^h*xqyBZg;fb_b}pgbltDDJNg$q zE=P~i6+49Bzi?R)>wgFSnvwe^urvVZenq3Qsx3-YvW*~ zG&es&w=sDaJZa&1b_Tyeht_4F)P*tx#FZzBzTsvY|MQQcO18PdQ?CWI!?4Fuz*K)h5C$DRNI@QVeI zwgL3!PlBVMwHl7bt_cMB6<16wd0q{Z*aAfnaucZO6<4anv7Xy&CmJd^yD(@;aYMvX z3rm60_L4UCCE7Src6bB8$A)7~IRN<-?xqp*Y&~oQ452usN=S=LTUwI{g+Sk>U`&?4 z;W6jQ;xvf-(1qVa5LU)-%4!9&QTR7u=a2F zV`EB8IXy{tC5IlYp{nuf0ruL4dix)X!~v$MyZnVwhGA?Ko!b}Wzt56v>(MKxXFG57 zzTf{||IgF6cOS{^K9cV{a(VE|>2DWMC0_2@#QlY96ZhNU8GRg<0G17^P7H22h7TQj z`Vid<6{UQlzY7(=84lr;B^+uwc;(qy?nd4F&F?k;{Mha9AIN?GK)(0D<<2Y5eoJxl zx`!Gx;uDJgh^>@93DxvzB)EFr8JwI{VzO>i6H%Yj*mno;0TQK@$$)clbTY}-ns&sh_##Vm}Dw)t!8NlQ#_34>CA9>pkTBtF9COym+di5cEB?npIu<^ZmV4UEpu|)z{w%NU>D|LmT+># zHm)VwIG9@-f|Y=~OF5Fz1J)y?o#6aRGAGcgB>YG@Wp{}ytnn>N`U8E>O)%Ro|^Z*i!!x^9~<2Z7IImA&Y6dYpr(hgDO&ec|Tl`=UA z?+4E%nKRwt8gPd^bB8uIwdoT%y`HUG4uoUy76C1OQi-WcnEAki9U&Vi01N<-3Jj>? z1dqU*sq26kPOw?N^5o2^8{F+J!}%@4A9mi}7s%}kWRHjP`$CuNt~~ke0$?5Ta@|Jm zqPUT}>e{%3H~hb47=0cV0U1iq{|ASTB8NccCWks+xW0b&;Emw>vA>IbSoiN8|Jsoq zeLlbU`OBxSum6@FgLa?8KfA{I*p=Ts0$#2F(J1B}C~mRzAd*8se6FGwq=&H<<)YUV z~HLPzGzytsOH>zueIE3C;Vd6pe8eL7p)oUQ%uEavqfvBq2 z8N&r&xPUPnL%t|JV>jz!yV;^vWf`VNA1uiSc1E>RHvwTgTuXb#JKxxnskz&};o4}f zea8pea&5b_Ld#uInjM}KeObYm6Mc)>jGHiH^HnJPrqblUPCkhteS2{`00BEr!poW| zcf`r-OaUvWH5v?Qk|CC`0-+&9K9E^3!LiR_$dx>2rPae`TK(p6t3b_11M!md6Fmvc zpb;QuBFfYUcVQ;oY2?dw8zccMByZptOILzpV^kSq2Q?4FF;%*pfHkOy3n8c8up{IC ztdLWG?gpP<*_W@|25GA0^%s9|Y5#@&vpz_)H*Co-@6FimLZZFVw7a_ytSP0I2I&L3 z9pCYLNHE@`{YVCp{1AzZB#dMd30ehH9wSlkDBdE$6ha|U(0ku*gY-a+*CE~S5FKsz zy9CFEg$B2y&FG39LhxU>3Wo0CDQ^g8PC5*c6V30Pfh|by$&M8A#(M(8E_;Wk75P0I>pCTN;R^IJ!J#FG99dA(deQ zE?0VSv?&yAYTE6qN%mB@Ni(G)nP?h;mg2u>hvhP0rMp9M%(`icth<88;}i;bw8(O< z)v*@td=*`~==i0fF*9_|Z9a zaTqm>zP6hFm@$NrP%4BU?ClpvnrRqbm4KL$=D}j5`Tnl=cHMkF-*<{144r$*0uMtU zc{(wIT4hF1z^;7j!F=<b3r#rRP>|yBW%ODr2bD8EsDN&I;W*vHJlV z*FUlt%E+ciA;A&l(vX4UF>8*>TXDT(-luI~@^;9K+u&3fqLv|Y3=Q2`%#IgYx>B~3 zD^M6KWrMN#l=EG?W#ma16IvA0O7^O0C70D65-4EkFu#zp!@S^c)9#de0uv_{7mGsN z6R1lS7g|R3K2TeweG*2WC?v9qgIRAWvsf)%lMhFOcK}L&!1XsDL71DQxaN=2lL& zS-bo$t7gb5>>U7C<%))5Ft%zM;)uE7pMk+fe9I^bn4%1IXF?0b;!{Q%wg%E#=<5P- z>MlU8gOwQG*S!XiH{oB!F{9681Omlj?Z<LkIR_U{E7DY&im}alq(oB!?gF5i7d=7-mxc1;|KAp8rfv=ESwkv0zjO*1rf2p|_{6BdNeY6zdp)f{JE^2PJ@iJdU1)wnoP~1Mpyr z>VT&Uo-7S`jL%}G9dHV^5*_w19;5v*9FyZyPeKwn0;gh02_G1TA;u}&6nid>n+?iO zK?*Sc88$J(9RLxlzXMR`D(x764#s2r9~j@l@Nn9(i7nK`s;%oEBE((jQa%9&7MymzPlQuI1GtTSS8KN49#bb(bYFC9*#{T}0-hT!&s~Bm* zCI{I<_s9K%7aPs5R}7?mt8Uv(jeWn8`B7GAfAvR;wGj zeUBLIvz%V(H$>)XDSHd^xJxz9O?{jJ8qC9T^e?68N8en+JL@&%3sX?KVktMLZ0DG;y1yBHJ z@lmbD#2s6PEh_~!O zjX^bv21nmSg2;{CB4%BQHw^5Vx<3FFutgb1nB`}D-6ayVW?&M;H5i8?YLev=9wN7; zIwJ^BG2vxO3qD`z=)T%Fd-`YJ&$su^w@dS_9bdWZ^)(q6xaiuI=g;KCb=UglM1NNB zXGQ 0) + total_with_input = sum(1 for r in requests if r.input_tokens and r.input_tokens > 0) + ratio = cache_hits / total_with_input if total_with_input > 0 else 0.0 + + assert cache_hits == 3 + assert total_with_input == 10 + assert ratio == 0.3 + + def test_tokens_per_second_calculation(self): + """Should compute output tokens per second.""" + # Request with 1000 output tokens, 2000ms latency + output_tokens = 1000 + latency_ms = 2000 + + tokens_per_second = (output_tokens / latency_ms) * 1000 + + assert tokens_per_second == 500.0 + + def test_latency_aggregation(self): + """Should extract latencies from requests.""" + latencies = [100.0, 200.0, 150.0, 300.0, 250.0] + + median = float(np.percentile(latencies, 50)) + p95 = float(np.percentile(latencies, 95)) + + assert median == 200.0 + # P95 of these 5 values + assert p95 >= 250.0 + + +class TestVariantRollupMetrics: + """Tests for variant-level rollup metrics.""" + + def test_session_success_rate(self): + """Should compute session success rate.""" + sessions = [] + for i in range(4): + sess = MagicMock() + sess.status = MagicMock(value="completed" if i < 3 else "aborted") + sessions.append(sess) + + success_count = sum(1 for s in sessions if s.status.value == "completed") + success_rate = success_count / len(sessions) if sessions else 0.0 + + assert success_count == 3 + assert success_rate == 0.75 + + def test_session_duration_median(self): + """Should compute median session duration.""" + from datetime import datetime, timedelta + + base = datetime.utcnow() + sessions = [] + durations_minutes = [10.0, 20.0, 30.0, 40.0, 50.0] + + for dur in durations_minutes: + sess = MagicMock() + sess.started_at = base + sess.ended_at = base + timedelta(minutes=dur) + sessions.append(sess) + + computed_durations = [] + for s in sessions: + if s.ended_at and s.started_at: + duration = (s.ended_at - s.started_at).total_seconds() / 60.0 + computed_durations.append(duration) + + median_duration = float(np.percentile(computed_durations, 50)) + assert median_duration == 30.0 + + +class TestEmptyWindowHandling: + """Tests for empty window handling.""" + + def test_empty_latency_list_returns_none(self): + """Empty latency list should handle gracefully.""" + latencies = [] + result = float(np.percentile(latencies, 50)) if latencies else None + assert result is None + + def test_empty_session_list_returns_empty_rollups(self): + """Empty session list should return empty rollups.""" + sessions = [] + assert len(sessions) == 0 + # Verify we don't attempt calculation + assert not sessions # Evaluates to True (empty) + + def test_empty_request_list_zero_counts(self): + """Empty request list should have zero counts.""" + requests = [] + + metrics = { + "request_count": float(len(requests)), + "success_count": 0.0, + "error_count": 0.0, + } + + assert metrics["request_count"] == 0.0 + assert metrics["success_count"] == 0.0 + assert metrics["error_count"] == 0.0