docker-compose.yml

services:
  # Kafka in KRaft mode (no Zookeeper needed!)
  kafka:
    image: confluentinc/cp-kafka:7.5.0
    hostname: kafka
    container_name: kafka
    user: "0:0"  # Run as root to avoid permission issues
    ports:
      - "9092:9092"
      - "9093:9093"
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: 'broker,controller'
      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
      KAFKA_LISTENERS: 'PLAINTEXT://kafka:29092,CONTROLLER://kafka:29093,PLAINTEXT_HOST://0.0.0.0:9092'
      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:29092,PLAINTEXT_HOST://localhost:9092'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_LOG_DIRS: '/tmp/kraft-combined-logs'
      CLUSTER_ID: 'MkU3OEVBNTcwNTJENDM2Qk'
    volumes:
      - kafka-data:/tmp/kraft-combined-logs
    networks:
      - lambda-network
    healthcheck:
      test: ["CMD-SHELL", "kafka-broker-api-versions --bootstrap-server localhost:9092"]
      interval: 10s
      timeout: 10s
      retries: 5

  # Schema Registry for Avro
  schema-registry:
    image: confluentinc/cp-schema-registry:7.5.0
    hostname: schema-registry
    container_name: schema-registry
    depends_on:
      kafka:
        condition: service_healthy
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka:29092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
    networks:
      - lambda-network
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8081/ || exit 1"]
      interval: 10s
      timeout: 5s
      retries: 5

  # Cassandra for Speed Layer storage
  cassandra:
    image: cassandra:4.1
    hostname: cassandra
    container_name: cassandra
    ports:
      - "9042:9042"
    environment:
      CASSANDRA_CLUSTER_NAME: 'SpeechCluster'
      CASSANDRA_DC: 'dc1'
      CASSANDRA_ENDPOINT_SNITCH: 'GossipingPropertyFileSnitch'
      MAX_HEAP_SIZE: '512M'
      HEAP_NEWSIZE: '128M'
    volumes:
      - cassandra-data:/var/lib/cassandra
      - ./docker/schema.cql:/schema.cql
    networks:
      - lambda-network
    healthcheck:
      test: ["CMD-SHELL", "cqlsh -e 'describe cluster'"]
      interval: 15s
      timeout: 10s
      retries: 10

  # HDFS NameNode for Batch Layer storage
  namenode:
    image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8
    container_name: namenode
    hostname: namenode
    ports:
      - "9870:9870"
      - "9000:9000"
    environment:
      CLUSTER_NAME: 'speech-cluster'
      CORE_CONF_fs_defaultFS: 'hdfs://namenode:9000'
      CORE_CONF_hadoop_http_staticuser_user: 'root'
      HDFS_CONF_dfs_webhdfs_enabled: 'true'
      HDFS_CONF_dfs_permissions_enabled: 'false'
      HDFS_CONF_dfs_namenode_datanode_registration_ip___hostname___check: 'false'
    volumes:
      - namenode-data:/hadoop/dfs/name
    networks:
      - lambda-network
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:9870/ || exit 1"]
      interval: 15s
      timeout: 10s
      retries: 5

  # HDFS DataNode
  datanode:
    image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8
    container_name: datanode
    hostname: datanode
    depends_on:
      namenode:
        condition: service_healthy
    environment:
      CORE_CONF_fs_defaultFS: 'hdfs://namenode:9000'
      CORE_CONF_hadoop_http_staticuser_user: 'root'
      HDFS_CONF_dfs_webhdfs_enabled: 'true'
      HDFS_CONF_dfs_permissions_enabled: 'false'
    volumes:
      - datanode-data:/hadoop/dfs/data
    networks:
      - lambda-network

  # Spark Master
  spark-master:
    image: bde2020/spark-master:3.3.0-hadoop3.3
    container_name: spark-master
    hostname: spark-master
    ports:
      - "8080:8080"
      - "7077:7077"
      - "4040:4040"
    environment:
      SPARK_MODE: 'master'
      SPARK_RPC_AUTHENTICATION_ENABLED: 'no'
      SPARK_RPC_ENCRYPTION_ENABLED: 'no'
      SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED: 'no'
      SPARK_SSL_ENABLED: 'no'
    volumes:
      - spark-logs:/spark/logs
    networks:
      - lambda-network
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8080/ || exit 1"]
      interval: 15s
      timeout: 10s
      retries: 5

  # Spark Worker
  spark-worker:
    image: bde2020/spark-worker:3.3.0-hadoop3.3
    container_name: spark-worker
    hostname: spark-worker
    depends_on:
      spark-master:
        condition: service_healthy
    environment:
      SPARK_MODE: 'worker'
      SPARK_MASTER_URL: 'spark://spark-master:7077'
      SPARK_WORKER_MEMORY: '2G'
      SPARK_WORKER_CORES: '2'
      SPARK_RPC_AUTHENTICATION_ENABLED: 'no'
      SPARK_RPC_ENCRYPTION_ENABLED: 'no'
    volumes:
      - ./spark-processor/target:/app
      - spark-logs:/spark/logs
    networks:
      - lambda-network

  # Dashboard (Spring Boot)
  dashboard:
    build:
      context: ./dashboard
      dockerfile: Dockerfile
    container_name: dashboard
    hostname: dashboard
    depends_on:
      cassandra:
        condition: service_healthy
    ports:
      - "8090:8090"
    environment:
      SPRING_DATA_CASSANDRA_CONTACT_POINTS: 'cassandra'
      SPRING_DATA_CASSANDRA_PORT: '9042'
      SPRING_DATA_CASSANDRA_KEYSPACE: 'speech_analytics'
      SERVER_PORT: '8090'
    networks:
      - lambda-network

networks:
  lambda-network:
    driver: bridge

volumes:
  kafka-data:
  cassandra-data:
  namenode-data:
  datanode-data:
  spark-logs: