feat: self-contained docker compose for existing Prometheus-Grafana stacks

asap-dropin/.env

@@ -0,0 +1,21 @@
+# ── User's existing Prometheus ────────────────────────────────────────────────
+# URL of your running Prometheus instance, reachable from inside Docker.
+#   - Docker Desktop (Mac/Windows): http://host.docker.internal:9090
+#   - Linux (host networking):      http://172.17.0.1:9090   (default Docker bridge gateway)
+#   - Prometheus in another compose: use a shared Docker network and the service name
+PROMETHEUS_URL=http://host.docker.internal:9090
+
+# Scrape interval configured in your Prometheus (in seconds).
+PROMETHEUS_SCRAPE_INTERVAL=15
+
+# ── Exposed ports ────────────────────────────────────────────────────────────
+# Port on the host where the remote-write receiver listens.
+# Add this to your prometheus.yml:
+#   remote_write:
+#     - url: http://localhost:${REMOTE_WRITE_PORT}/receive
+REMOTE_WRITE_PORT=9091
+
+# Port on the host where the ASAPQuery query engine listens.
+# Point your Grafana Prometheus datasource here:
+#   http://localhost:${QUERY_ENGINE_PORT}
+QUERY_ENGINE_PORT=8088

asap-dropin/README.md

@@ -0,0 +1,79 @@
+# ASAPQuery Drop-in for Existing Prometheus + Grafana Stacks
+
+A self-contained single-container Docker Compose that adds ASAPQuery to an existing Prometheus and Grafana deployment.
+
+On startup, all queries are forwarded transparently to your upstream Prometheus. After one observation window (default 10 min), the engine automatically plans and activates sketch-based acceleration based on the real queries it observed from Grafana.
+
+## Prerequisites
+
+- Docker and Docker Compose
+- A running Prometheus instance
+- A running Grafana instance (with a Prometheus datasource)
+
+## Quick Start
+
+### 1. Configure environment
+
+Edit `.env`:
+
+| Variable | Default | Description |
+|---|---|---|
+| `PROMETHEUS_URL` | `http://host.docker.internal:9090` | URL of your Prometheus, reachable from inside Docker |
+| `PROMETHEUS_SCRAPE_INTERVAL` | `15` | Your Prometheus scrape interval in seconds |
+| `REMOTE_WRITE_PORT` | `9091` | Host port for the remote-write receiver |
+| `QUERY_ENGINE_PORT` | `8088` | Host port for the ASAPQuery query engine |
+
+**Finding the right `PROMETHEUS_URL`:**
+- **Docker Desktop (Mac/Windows):** `http://host.docker.internal:9090` (default)
+- **Linux (Prometheus on host):** `http://172.17.0.1:9090` (default Docker bridge gateway)
+- **Prometheus in another Docker Compose:** create a shared external network
+
+### 2. Start ASAPQuery
+
+```bash
+docker compose up -d
+```
+
+### 3. Add remote_write to your Prometheus
+
+Add this to your `prometheus.yml` and reload Prometheus:
+
+```yaml
+remote_write:
+  - url: http://localhost:9091/receive
+    queue_config:
+      batch_send_deadline: 1s
+      sample_age_limit: 5m
+```
+
+### 4. Point Grafana at ASAPQuery
+
+Change your Grafana Prometheus datasource URL from your Prometheus address to:
+
+```
+http://localhost:8088
+```
+
+ASAPQuery speaks the Prometheus query API. Queries it can accelerate are answered from sketches; all others are transparently forwarded to your upstream Prometheus.
+
+## Architecture
+
+```
+Your Prometheus ──remote_write──▸ ASAPQuery (:9091/receive)
+                                      │
+                                      ▼
+Your Grafana ◂──query──── ASAPQuery Query Engine (:8088)
+                                      │
+                                      ▼ (fallback / passthrough)
+                               Your Prometheus
+```
+
+The query engine embeds the planner and runs it automatically after observing real Grafana queries for one observation window. No separate planner container, no Kafka, no Arroyo.
+
+## Development
+
+To build from local source instead of pulling pre-built images:
+
+```bash
+docker compose -f docker-compose.yml -f docker-compose.dev.yml up
+```

asap-dropin/docker-compose.dev.yml

@@ -0,0 +1,13 @@
+name: asapquery-dropin
+
+# Development override: builds ASAP services from local source instead of
+# pulling pre-built images from ghcr.
+#
+# Usage:
+#   docker compose -f docker-compose.yml -f docker-compose.dev.yml up
+
+services:
+  queryengine:
+    build:
+      context: ..
+      dockerfile: asap-query-engine/Dockerfile

asap-dropin/docker-compose.yml

@@ -0,0 +1,53 @@
+name: asapquery-dropin
+
+# Self-contained ASAPQuery stack for existing Prometheus + Grafana deployments.
+#
+# Usage:
+#   1. Adjust PROMETHEUS_URL in .env to point at your Prometheus.
+#   2. docker compose up -d
+#   3. Add remote_write to your prometheus.yml  ->  http://localhost:${REMOTE_WRITE_PORT}/receive
+#   4. Point your Grafana datasource URL        ->  http://localhost:${QUERY_ENGINE_PORT}
+#
+# The query engine starts with an empty plan and forwards all queries to Prometheus.
+# After the observation window (default 10 min), it automatically generates a plan
+# based on real query patterns and begins precomputing sketches.
+
+networks:
+  asap-network:
+    driver: bridge
+
+services:
+  queryengine:
+    image: ghcr.io/projectasap/asap-query-engine:v0.3.0
+    container_name: asap-dropin-queryengine
+    hostname: queryengine
+    networks:
+      - asap-network
+    ports:
+      - "${QUERY_ENGINE_PORT:-8088}:8088"
+      - "${REMOTE_WRITE_PORT:-9091}:9091"
+    environment:
+      - RUST_LOG=INFO
+      - RUST_BACKTRACE=1
+    volumes:
+      - ./output/queryengine:/app/outputs
+    command:
+      - "--prometheus-server=${PROMETHEUS_URL:-http://host.docker.internal:9090}"
+      - "--prometheus-scrape-interval=${PROMETHEUS_SCRAPE_INTERVAL:-15}"
+      - "--streaming-engine=precompute"
+      - "--prometheus-remote-write-port=9091"
+      - "--delete-existing-db"
+      - "--log-level=INFO"
+      - "--output-dir=/app/outputs"
+      - "--query-language=PROMQL"
+      - "--lock-strategy=per-key"
+      - "--forward-unsupported-queries"
+      - "--enable-query-tracker"
+      - "--tracker-observation-window-secs=600"
+    healthcheck:
+      test: ["CMD-SHELL", "bash -c 'echo > /dev/tcp/localhost/8088' 2>/dev/null || exit 1"]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 15s
+    restart: unless-stopped

asap-planner-rs/src/main.rs

@@ -104,9 +104,10 @@ fn main() -> anyhow::Result<()> {
                          (query logs have no metrics hint to fall back on)"
                     )
                 }
-                _ => anyhow::bail!(
-                    "exactly one of --input_config or --query-log must be provided for PromQL mode"
-                ),
+                (None, None, _) => {
+                    anyhow::bail!("provide one of --input_config or --query-log")
+                }
+                _ => unreachable!("clap conflicts_with prevents this combination"),
             };
             controller.generate_to_dir(&args.output_dir)?;
         }

asap-query-engine/src/main.rs

@@ -7,11 +7,15 @@ use tracing::{error, info, warn};
 
 use sketch_core::config::{self, ImplMode};
 
-use query_engine_rust::data_model::enums::{InputFormat, LockStrategy, StreamingEngine};
+use asap_types::streaming_config::StreamingConfig;
+use query_engine_rust::data_model::enums::{
+    CleanupPolicy, InputFormat, LockStrategy, StreamingEngine,
+};
 use query_engine_rust::drivers::AdapterConfig;
 use query_engine_rust::precompute_engine::config::LateDataPolicy;
 use query_engine_rust::precompute_engine::PrecomputeWorkerDiagnostics;
 use query_engine_rust::utils::file_io::{read_inference_config, read_streaming_config};
+use query_engine_rust::InferenceConfig;
 use query_engine_rust::{
     HttpServer, HttpServerConfig, KafkaConsumer, KafkaConsumerConfig, OtlpReceiver,
     OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, PrecomputeEngineHandle, Result,
@@ -29,13 +33,13 @@ struct Args {
     #[arg(long, value_enum)]
     input_format: Option<InputFormat>,
 
-    /// Configuration file path
+    /// Inference config file path (optional; starts with empty config when omitted, requires --enable-query-tracker)
     #[arg(long)]
-    config: String,
+    config: Option<String>,
 
-    /// File path for streaming_config
+    /// Streaming config file path (optional; starts with empty config when omitted, requires --enable-query-tracker)
     #[arg(long)]
-    streaming_config: String,
+    streaming_config: Option<String>,
 
     /// Streaming engine to use
     #[arg(long, value_enum, default_value = "arroyo")]
@@ -182,26 +186,34 @@ async fn main() -> Result<()> {
     let _log_guard = setup_logging(&args.output_dir, &args.log_level)?;
 
     info!("Starting Query Engine Rust");
-    info!("Config file: {}", args.config);
     info!("Output directory: {}", args.output_dir);
 
-    // Read config (equivalent to utils.file_io.read_inference_config)
-    let inference_config = read_inference_config(&args.config, args.query_language)?;
+    let inference_config = match &args.config {
+        Some(path) => {
+            info!("Config file: {}", path);
+            read_inference_config(path, args.query_language)?
+        }
+        None => {
+            info!("No config file provided; starting with empty inference config");
+            InferenceConfig::new(args.query_language, CleanupPolicy::NoCleanup)
+        }
+    };
     info!(
         "Loaded inference config with {} query configs",
         inference_config.query_configs.len()
     );
-    info!("Inference config: {:?}", inference_config);
 
-    let streaming_config = Arc::new(read_streaming_config(
-        &args.streaming_config,
-        &inference_config,
-    )?);
+    let streaming_config = Arc::new(match &args.streaming_config {
+        Some(path) => read_streaming_config(path, &inference_config)?,
+        None => {
+            info!("No streaming config file provided; starting with empty streaming config");
+            StreamingConfig::default()
+        }
+    });
     info!(
         "Loaded streaming config with {} entries",
         streaming_config.get_all_aggregation_configs().len()
     );
-    info!("Streaming config: {:?}", streaming_config);
 
     // Shared config refs — passed to QueryTracker so it can populate ControllerConfig
     // with the current configs as context for the planner.  The applier task updates