Optimize Docker images for size and 4G deploy bandwidth

.dockerignore

@@ -0,0 +1,16 @@
+.venv
+.git
+.github
+.mypy_cache
+.ruff_cache
+__pycache__
+*.pyc
+*.egg-info
+dist
+build
+.env
+data
+*.md
+!requirements.txt
+tests
+.pytest_cache

.github/workflows/build-push-image.yml

@@ -1,24 +1,62 @@
-name: Build and Push Docker Image
+name: Build and Push Docker Images
 
 on:
   push:
     branches:
       - develop
+      - main
+    tags:
+      - 'v*'
 
 jobs:
-  build_and_push_image:
-    name: Build and Push Docker Image
+  build-and-push:
+    name: Build and Push ${{ matrix.image }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - image: pyronear/pyro-engine
+            context: .
+            dockerfile: Dockerfile
+          - image: pyronear/pyro-camera-api
+            context: pyro_camera_api
+            dockerfile: pyro_camera_api/Dockerfile
 
     steps:
-    - name: Checkout repository
-      uses: actions/checkout@v4
+      - name: Checkout repository
+        uses: actions/checkout@v4
 
-    - name: Log in to Docker Hub
-      run: echo "${{ secrets.DOCKERHUB_PW }}" | docker login -u "${{ secrets.DOCKERHUB_LOGIN }}" --password-stdin
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
 
-    - name: Build Docker image
-      run: make build-app
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
 
-    - name: Push Docker image
-      run: docker push pyronear/pyro-engine:latest
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_LOGIN }}
+          password: ${{ secrets.DOCKERHUB_PW }}
+
+      - name: Extract metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ matrix.image }}
+          tags: |
+            type=raw,value=latest,enable={{is_default_branch}}
+            type=ref,event=branch
+            type=semver,pattern={{version}}
+            type=semver,pattern={{major}}.{{minor}}
+
+      - name: Build and push
+        uses: docker/build-push-action@v6
+        with:
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha,scope=${{ matrix.image }}
+          cache-to: type=gha,mode=max,scope=${{ matrix.image }}

.github/workflows/docker.yml

@@ -11,8 +11,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - name: Build docker image
+      - name: Build engine image
         run: make build-app
       - name: Verify engine starts successfully
         run: |
           docker run --rm pyronear/pyro-engine:latest python -c "from pyroengine import SystemController; print('OK')"
+      - name: Build camera API image
+        run: docker build -f pyro_camera_api/Dockerfile pyro_camera_api -t pyronear/pyro-camera-api:latest
+      - name: Verify camera API starts successfully
+        run: |
+          docker run --rm pyronear/pyro-camera-api:latest python -c "from pyro_camera_api.main import app; print('OK')"

.github/workflows/release.yml

@@ -103,28 +103,3 @@ jobs:
           conda install -c pyronear pyroengine
           python -c "import pyroengine; print(pyroengine.__version__)"
 
-
-  dockerhub-publish:
-    if: "!github.event.release.prerelease"
-    name: Push Docker image to Docker Hub
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out the repo
-        uses: actions/checkout@v4
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v4
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v4
-      - name: Login to Docker Hub
-        uses: docker/login-action@v4
-        with:
-          username: ${{ secrets.DOCKERHUB_LOGIN }}
-          password: ${{ secrets.DOCKERHUB_PW }}
-      - name: Push to Docker Hub
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          platforms: linux/amd64,linux/arm64
-          repository: pyronear/pyro-engine
-          tag_with_ref: true

Dockerfile

@@ -1,43 +1,50 @@
+# ---- Builder: only used for git-based deps (needs git) ----
+FROM python:3.11.13-slim-bullseye AS git-deps
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends git && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY ./requirements-git.txt /tmp/requirements-git.txt
+RUN pip install --no-cache-dir --default-timeout=500 --target=/tmp/git-packages -r /tmp/requirements-git.txt
+
+# ---- Runtime ----
 FROM python:3.11.13-slim-bullseye
 
-# set environment variables
-ENV PATH="/usr/local/bin:$PATH" \
-    LANG="C.UTF-8" \
+ENV LANG="C.UTF-8" \
     PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1
-# set work directory
-WORKDIR /usr/src/app
-
-# install git
-RUN apt-get update && apt-get install git -y
 
+# Layer 1: System libs (~100MB, almost never changes)
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-        git \
         ffmpeg \
         libsm6 \
         libxext6 \
-    || apt-get install -y --fix-missing && \
-    apt-get clean && \
+        libglib2.0-0 \
+        libgl1 \
+    && apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-
-RUN pip install --no-cache-dir --upgrade pip setuptools wheel
+# Layer 2: Stable pip deps (~400MB, changes only on version bumps)
 COPY ./requirements.txt /tmp/requirements.txt
-
-RUN pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements.txt && \
+RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \
+    pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements.txt && \
     rm -f /tmp/requirements.txt
 
+# Layer 3: Git-based deps (~5MB, changes when API clients are updated)
+COPY --from=git-deps /tmp/git-packages /usr/local/lib/python3.11/site-packages/
+
+# Layer 4: Local packages (~1MB, changes on every deploy)
 WORKDIR /opt/pyroengine_src
 COPY ./pyro-predictor ./pyro-predictor
 COPY ./pyroengine ./pyroengine
 COPY ./setup.py ./setup.py
+RUN pip install --no-cache-dir ./pyro-predictor \
+    && pip install --no-cache-dir .
 
-RUN pip install --no-cache-dir -e ./pyro-predictor \
-    && pip install --no-cache-dir -e . \
-    && rm -rf /root/.cache/pip
-
+# Layer 5: Entrypoint scripts (~few KB, rarely changes)
 WORKDIR /usr/src/app
-
 COPY ./src/run.py /usr/src/app/run.py
-COPY ./src/control_reolink_cam.py /usr/src/app/control_reolink_cam.py
+COPY ./src/control_reolink_cam.py /usr/src/app/control_reolink_cam.py

Makefile

@@ -20,7 +20,10 @@ single-docs:
 # Update requirements.txt for the main project
 lock:
 	poetry lock
-	poetry export -f requirements.txt --without-hashes --output requirements.txt
+	poetry export -f requirements.txt --without-hashes --output requirements-all.txt
+	grep 'git+' requirements-all.txt > requirements-git.txt || true
+	grep -v 'git+' requirements-all.txt > requirements.txt
+	rm requirements-all.txt
 
 # Generate requirements and build camera API Docker image
 build-api:
@@ -40,10 +43,10 @@ build-optional-lib:
 	pip install -e .[docs]
 	pip install -e .[dev]
 
-# Build both images and run the stack
+# Pull latest images and run the stack
 run:
-	docker build . -t pyronear/pyro-engine:latest
-	docker build -f pyro_camera_api/Dockerfile pyro_camera_api -t pyronear/pyro-camera-api:latest
+	docker pull pyronear/pyro-engine:latest
+	docker pull pyronear/pyro-camera-api:latest
 	docker compose up -d
 
 # Get log from engine wrapper

pyro_camera_api/.dockerignore

@@ -0,0 +1,16 @@
+.venv
+.git
+.mypy_cache
+.ruff_cache
+__pycache__
+*.pyc
+*.egg-info
+dist
+build
+.env
+data
+*.md
+!requirements.txt
+tests
+.pytest_cache
+client

pyro_camera_api/Dockerfile

@@ -1,32 +1,29 @@
 FROM python:3.9.16-slim
 
-WORKDIR /usr/src/app
-
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-ENV PYTHONPATH=/usr/src/app
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PYTHONPATH=/usr/src/app
 
+# Layer 1: System libs (~100MB, almost never changes)
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
         curl \
-        git \
         ffmpeg \
         libsm6 \
         libxext6 \
         libgl1 \
-        build-essential \
         libglib2.0-0 \
     && apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Layer 2: Pip deps (~400MB, changes only on version bumps)
 COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv
-
 COPY requirements.txt /tmp/requirements.txt
-# if you have a lock file uncomment this
-# COPY uv.lock /tmp/uv.lock
-
-RUN uv pip install --no-cache --system -r /tmp/requirements.txt
+RUN uv pip install --no-cache --system -r /tmp/requirements.txt && \
+    rm /bin/uv /tmp/requirements.txt
 
+# Layer 3: Source code (~few KB, changes on every deploy)
+WORKDIR /usr/src/app
 COPY pyproject.toml ./pyproject.toml
 COPY pyro_camera_api ./pyro_camera_api
 

pyro_camera_api/requirements.txt

@@ -18,7 +18,6 @@ ncnn==1.0.20240410 ; python_version >= "3.9" and python_version < "4.0"
 numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0"
 onnxruntime==1.19.2 ; python_version >= "3.9" and python_version < "4.0"
 opencv-python-headless==4.11.0.86 ; python_version >= "3.9" and python_version < "4.0"
-opencv-python==4.11.0.86 ; python_version >= "3.9" and python_version < "4.0"
 packaging==25.0 ; python_version >= "3.9" and python_version < "4.0"
 pillow==10.4.0 ; python_version >= "3.9" and python_version < "4.0"
 portalocker==3.2.0 ; python_version >= "3.9" and python_version < "4.0"

requirements-git.txt

@@ -0,0 +1,2 @@
+pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0"
+pyroclient @ git+https://github.com/pyronear/pyro-api.git@9cba4afdf1d096436ca875bfde104f1a9bc1df24#subdirectory=client ; python_version >= "3.11" and python_version < "4.0"

requirements.txt

@@ -12,14 +12,12 @@ mpmath==1.3.0 ; python_version >= "3.11" and python_version < "4.0"
 ncnn==1.0.20240410 ; python_version >= "3.11" and python_version < "4.0"
 numpy==2.2.6 ; python_version >= "3.11" and python_version < "4.0"
 onnxruntime==1.22.1 ; python_version >= "3.11" and python_version < "4.0"
-opencv-python==4.12.0.88 ; python_version >= "3.11" and python_version < "4.0"
+opencv-python-headless==4.12.0.88 ; python_version >= "3.11" and python_version < "4.0"
 packaging==25.0 ; python_version >= "3.11" and python_version < "4.0"
 pillow==11.0.0 ; python_version >= "3.11" and python_version < "4.0"
 portalocker==3.2.0 ; python_version >= "3.11" and python_version < "4.0"
 protobuf==6.33.1 ; python_version >= "3.11" and python_version < "4.0"
 pyreadline3==3.5.4 ; python_version >= "3.11" and python_version < "4.0" and sys_platform == "win32"
-pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0"
-pyroclient @ git+https://github.com/pyronear/pyro-api.git@119ff76266eee72ffeb06141a85d420506322fa8#subdirectory=client ; python_version >= "3.11" and python_version < "4.0"
 python-dotenv==1.1.0 ; python_version >= "3.11" and python_version < "4.0"
 pywin32==311 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows"
 pyyaml==6.0.3 ; python_version >= "3.11" and python_version < "4.0"