docker-compose.yml

# ==============================================================================
# Foundry: Docker Compose
# ==============================================================================
# Quick start (uses default Qwen3.5 MoE model):
#   docker compose up
#
# Choose a different model:
#   FOUNDRY_MODEL=hermes-4.3-36b docker compose up
#
# With monitoring (Prometheus + Grafana + GPU metrics):
#   docker compose --profile monitoring up
#
# With explicit profile:
#   FOUNDRY_PROFILE=rtx5090 docker compose up
#
# With HF token (create .env file with HF_TOKEN=hf_...):
#   docker compose up
# ==============================================================================

services:
  # ============================================================================
  # eBPF Exporter (Linux Kernel metrics)
  # ============================================================================
  ebpf-exporter:
    image: ghcr.io/cloudflare/ebpf_exporter:latest
    profiles:
      - monitoring
    privileged: true
    pid: host
    network_mode: host
    volumes:
      - /sys/kernel/debug:/sys/kernel/debug:ro
      - /sys/kernel/tracing:/sys/kernel/tracing:ro
      - /lib/modules:/lib/modules:ro
    command: ["--config.file=/examples/biolatency.yaml"]

  # ============================================================================
  # Prometheus (Scrapes metrics)
  # ============================================================================
  inference:
    image: ghcr.io/infernet-org/foundry/${FOUNDRY_MODEL:-qwen3.5-9b}:latest
    build:
      context: models/${FOUNDRY_MODEL:-qwen3.5-9b}/
    ports:
      - "${FOUNDRY_PORT:-8080}:8080"
    volumes:
      - ~/.cache/foundry:/models
    env_file:
      - path: .env
        required: false
    environment:
      - FOUNDRY_PROFILE=${FOUNDRY_PROFILE:-auto}
      - FOUNDRY_PORT=8080
      - HF_TOKEN=${HF_TOKEN:-}
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: 1
              capabilities: [gpu]
    shm_size: '2gb'
    # NOTE: Network tuning (BBR, busy_poll) and memory locking are applied at the
    # host level by scripts/host-setup.sh. Run it once before deploying:
    #   sudo ./scripts/host-setup.sh
    # Container-level sysctls for these are omitted because they fail on rootless
    # Docker and are less effective than host-level settings.
    sysctls:
      - net.core.somaxconn=4096
      - net.ipv4.tcp_keepalive_time=60
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/health"]
      interval: 10s
      timeout: 5s
      retries: 5
    networks:
      - default
      - monitoring

  # ============================================================================
  # Monitoring Stack (activated with: --profile monitoring)
  # ============================================================================

  prometheus:
    image: prom/prometheus:latest
    profiles:
      - monitoring
    extra_hosts:
      - "host.docker.internal:host-gateway"
    ports:
      - "9090:9090"
    volumes:
      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - prometheus-data:/prometheus
    networks:
      - monitoring
    depends_on:
      inference:
        condition: service_started

  grafana:
    image: grafana/grafana:11.6.0
    profiles: [monitoring]
    container_name: foundry-grafana
    restart: unless-stopped
    environment:
      - GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER:-admin}
      - GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD:-admin}
      - GF_USERS_ALLOW_SIGN_UP=false
    ports:
      - "${GF_PORT:-3000}:3000"
    volumes:
      - grafana-data:/var/lib/grafana
      - ./monitoring/grafana/provisioning/datasources:/etc/grafana/provisioning/datasources:ro
      - ./monitoring/grafana/provisioning/dashboards:/etc/grafana/provisioning/dashboards:ro
      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
    networks:
      - monitoring
    depends_on:
      - prometheus

  node-exporter:
    image: prom/node-exporter:v1.9.1
    profiles: [monitoring]
    container_name: foundry-node-exporter
    restart: unless-stopped
    command:
      - --path.procfs=/host/proc
      - --path.rootfs=/rootfs
      - --path.sysfs=/host/sys
      - --collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    pid: host
    networks:
      - monitoring

  nvidia-gpu-exporter:
    image: utkuozdemir/nvidia_gpu_exporter:1.4.1
    profiles: [monitoring]
    container_name: foundry-nvidia-gpu-exporter
    restart: unless-stopped
    ports:
      - "9835:9835"
    volumes:
      - /usr/bin/nvidia-smi:/usr/bin/nvidia-smi:ro
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu, utility]
    networks:
      - monitoring

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.51.0
    profiles: [monitoring]
    container_name: foundry-cadvisor
    restart: unless-stopped
    privileged: true
    ports:
      - "8081:8080"
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    devices:
      - /dev/kmsg:/dev/kmsg
    networks:
      - monitoring

# ==============================================================================
# Volumes
# ==============================================================================
volumes:
  prometheus-data:
    driver: local
  grafana-data:
    driver: local

# ==============================================================================
# Networks
# ==============================================================================
networks:
  default:
    driver: bridge
  monitoring:
    driver: bridge