diff --git a/asap-dropin/.env b/asap-dropin/.env new file mode 100644 index 0000000..79e0f40 --- /dev/null +++ b/asap-dropin/.env @@ -0,0 +1,21 @@ +# ── User's existing Prometheus ──────────────────────────────────────────────── +# URL of your running Prometheus instance, reachable from inside Docker. +# - Docker Desktop (Mac/Windows): http://host.docker.internal:9090 +# - Linux (host networking): http://172.17.0.1:9090 (default Docker bridge gateway) +# - Prometheus in another compose: use a shared Docker network and the service name +PROMETHEUS_URL=http://host.docker.internal:9090 + +# Scrape interval configured in your Prometheus (in seconds). +PROMETHEUS_SCRAPE_INTERVAL=15 + +# ── Exposed ports ──────────────────────────────────────────────────────────── +# Port on the host where the remote-write receiver listens. +# Add this to your prometheus.yml: +# remote_write: +# - url: http://localhost:${REMOTE_WRITE_PORT}/receive +REMOTE_WRITE_PORT=9091 + +# Port on the host where the ASAPQuery query engine listens. +# Point your Grafana Prometheus datasource here: +# http://localhost:${QUERY_ENGINE_PORT} +QUERY_ENGINE_PORT=8088 diff --git a/asap-dropin/README.md b/asap-dropin/README.md new file mode 100644 index 0000000..bb16ec9 --- /dev/null +++ b/asap-dropin/README.md @@ -0,0 +1,79 @@ +# ASAPQuery Drop-in for Existing Prometheus + Grafana Stacks + +A self-contained single-container Docker Compose that adds ASAPQuery to an existing Prometheus and Grafana deployment. + +On startup, all queries are forwarded transparently to your upstream Prometheus. After one observation window (default 10 min), the engine automatically plans and activates sketch-based acceleration based on the real queries it observed from Grafana. + +## Prerequisites + +- Docker and Docker Compose +- A running Prometheus instance +- A running Grafana instance (with a Prometheus datasource) + +## Quick Start + +### 1. Configure environment + +Edit `.env`: + +| Variable | Default | Description | +|---|---|---| +| `PROMETHEUS_URL` | `http://host.docker.internal:9090` | URL of your Prometheus, reachable from inside Docker | +| `PROMETHEUS_SCRAPE_INTERVAL` | `15` | Your Prometheus scrape interval in seconds | +| `REMOTE_WRITE_PORT` | `9091` | Host port for the remote-write receiver | +| `QUERY_ENGINE_PORT` | `8088` | Host port for the ASAPQuery query engine | + +**Finding the right `PROMETHEUS_URL`:** +- **Docker Desktop (Mac/Windows):** `http://host.docker.internal:9090` (default) +- **Linux (Prometheus on host):** `http://172.17.0.1:9090` (default Docker bridge gateway) +- **Prometheus in another Docker Compose:** create a shared external network + +### 2. Start ASAPQuery + +```bash +docker compose up -d +``` + +### 3. Add remote_write to your Prometheus + +Add this to your `prometheus.yml` and reload Prometheus: + +```yaml +remote_write: + - url: http://localhost:9091/receive + queue_config: + batch_send_deadline: 1s + sample_age_limit: 5m +``` + +### 4. Point Grafana at ASAPQuery + +Change your Grafana Prometheus datasource URL from your Prometheus address to: + +``` +http://localhost:8088 +``` + +ASAPQuery speaks the Prometheus query API. Queries it can accelerate are answered from sketches; all others are transparently forwarded to your upstream Prometheus. + +## Architecture + +``` +Your Prometheus ──remote_write──▸ ASAPQuery (:9091/receive) + │ + ▼ +Your Grafana ◂──query──── ASAPQuery Query Engine (:8088) + │ + ▼ (fallback / passthrough) + Your Prometheus +``` + +The query engine embeds the planner and runs it automatically after observing real Grafana queries for one observation window. No separate planner container, no Kafka, no Arroyo. + +## Development + +To build from local source instead of pulling pre-built images: + +```bash +docker compose -f docker-compose.yml -f docker-compose.dev.yml up +``` diff --git a/asap-dropin/docker-compose.dev.yml b/asap-dropin/docker-compose.dev.yml new file mode 100644 index 0000000..2a99784 --- /dev/null +++ b/asap-dropin/docker-compose.dev.yml @@ -0,0 +1,13 @@ +name: asapquery-dropin + +# Development override: builds ASAP services from local source instead of +# pulling pre-built images from ghcr. +# +# Usage: +# docker compose -f docker-compose.yml -f docker-compose.dev.yml up + +services: + queryengine: + build: + context: .. + dockerfile: asap-query-engine/Dockerfile diff --git a/asap-dropin/docker-compose.yml b/asap-dropin/docker-compose.yml new file mode 100644 index 0000000..8ce572b --- /dev/null +++ b/asap-dropin/docker-compose.yml @@ -0,0 +1,53 @@ +name: asapquery-dropin + +# Self-contained ASAPQuery stack for existing Prometheus + Grafana deployments. +# +# Usage: +# 1. Adjust PROMETHEUS_URL in .env to point at your Prometheus. +# 2. docker compose up -d +# 3. Add remote_write to your prometheus.yml -> http://localhost:${REMOTE_WRITE_PORT}/receive +# 4. Point your Grafana datasource URL -> http://localhost:${QUERY_ENGINE_PORT} +# +# The query engine starts with an empty plan and forwards all queries to Prometheus. +# After the observation window (default 10 min), it automatically generates a plan +# based on real query patterns and begins precomputing sketches. + +networks: + asap-network: + driver: bridge + +services: + queryengine: + image: ghcr.io/projectasap/asap-query-engine:v0.3.0 + container_name: asap-dropin-queryengine + hostname: queryengine + networks: + - asap-network + ports: + - "${QUERY_ENGINE_PORT:-8088}:8088" + - "${REMOTE_WRITE_PORT:-9091}:9091" + environment: + - RUST_LOG=INFO + - RUST_BACKTRACE=1 + volumes: + - ./output/queryengine:/app/outputs + command: + - "--prometheus-server=${PROMETHEUS_URL:-http://host.docker.internal:9090}" + - "--prometheus-scrape-interval=${PROMETHEUS_SCRAPE_INTERVAL:-15}" + - "--streaming-engine=precompute" + - "--prometheus-remote-write-port=9091" + - "--delete-existing-db" + - "--log-level=INFO" + - "--output-dir=/app/outputs" + - "--query-language=PROMQL" + - "--lock-strategy=per-key" + - "--forward-unsupported-queries" + - "--enable-query-tracker" + - "--tracker-observation-window-secs=600" + healthcheck: + test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/8088' 2>/dev/null || exit 1"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 15s + restart: unless-stopped diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index 3ffe683..863f4d4 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -104,9 +104,10 @@ fn main() -> anyhow::Result<()> { (query logs have no metrics hint to fall back on)" ) } - _ => anyhow::bail!( - "exactly one of --input_config or --query-log must be provided for PromQL mode" - ), + (None, None, _) => { + anyhow::bail!("provide one of --input_config or --query-log") + } + _ => unreachable!("clap conflicts_with prevents this combination"), }; controller.generate_to_dir(&args.output_dir)?; } diff --git a/asap-query-engine/src/main.rs b/asap-query-engine/src/main.rs index 78385c6..92aade4 100644 --- a/asap-query-engine/src/main.rs +++ b/asap-query-engine/src/main.rs @@ -7,11 +7,15 @@ use tracing::{error, info, warn}; use sketch_core::config::{self, ImplMode}; -use query_engine_rust::data_model::enums::{InputFormat, LockStrategy, StreamingEngine}; +use asap_types::streaming_config::StreamingConfig; +use query_engine_rust::data_model::enums::{ + CleanupPolicy, InputFormat, LockStrategy, StreamingEngine, +}; use query_engine_rust::drivers::AdapterConfig; use query_engine_rust::precompute_engine::config::LateDataPolicy; use query_engine_rust::precompute_engine::PrecomputeWorkerDiagnostics; use query_engine_rust::utils::file_io::{read_inference_config, read_streaming_config}; +use query_engine_rust::InferenceConfig; use query_engine_rust::{ HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver, OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, PrecomputeEngineHandle, Result, @@ -29,13 +33,13 @@ struct Args { #[arg(long, value_enum)] input_format: Option, - /// Configuration file path + /// Inference config file path (optional; starts with empty config when omitted, requires --enable-query-tracker) #[arg(long)] - config: String, + config: Option, - /// File path for streaming_config + /// Streaming config file path (optional; starts with empty config when omitted, requires --enable-query-tracker) #[arg(long)] - streaming_config: String, + streaming_config: Option, /// Streaming engine to use #[arg(long, value_enum, default_value = "arroyo")] @@ -182,26 +186,34 @@ async fn main() -> Result<()> { let _log_guard = setup_logging(&args.output_dir, &args.log_level)?; info!("Starting Query Engine Rust"); - info!("Config file: {}", args.config); info!("Output directory: {}", args.output_dir); - // Read config (equivalent to utils.file_io.read_inference_config) - let inference_config = read_inference_config(&args.config, args.query_language)?; + let inference_config = match &args.config { + Some(path) => { + info!("Config file: {}", path); + read_inference_config(path, args.query_language)? + } + None => { + info!("No config file provided; starting with empty inference config"); + InferenceConfig::new(args.query_language, CleanupPolicy::NoCleanup) + } + }; info!( "Loaded inference config with {} query configs", inference_config.query_configs.len() ); - info!("Inference config: {:?}", inference_config); - let streaming_config = Arc::new(read_streaming_config( - &args.streaming_config, - &inference_config, - )?); + let streaming_config = Arc::new(match &args.streaming_config { + Some(path) => read_streaming_config(path, &inference_config)?, + None => { + info!("No streaming config file provided; starting with empty streaming config"); + StreamingConfig::default() + } + }); info!( "Loaded streaming config with {} entries", streaming_config.get_all_aggregation_configs().len() ); - info!("Streaming config: {:?}", streaming_config); // Shared config refs — passed to QueryTracker so it can populate ControllerConfig // with the current configs as context for the planner. The applier task updates