From 025bbd3a20d0d8676bc36a3e8e29a0cfd17d6ba7 Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 2 Apr 2026 16:59:25 +0200 Subject: [PATCH 1/7] optimize docker images for size and 4G deploy bandwidth - Split requirements into stable (pip) and git-based for layered caching - Multi-stage build for engine to isolate git dependency - Remove duplicate opencv-python, use headless variant - Remove build-essential and git from camera API runtime - Add .dockerignore files - Update Makefile lock target to auto-split requirements --- .dockerignore | 16 ++++++++++ Dockerfile | 52 +++++++++++++++++++------------- Makefile | 5 ++- pyro_camera_api/.dockerignore | 16 ++++++++++ pyro_camera_api/Dockerfile | 21 ++++++------- pyro_camera_api/requirements.txt | 1 - requirements-git.txt | 2 ++ requirements.txt | 4 +-- 8 files changed, 79 insertions(+), 38 deletions(-) create mode 100644 .dockerignore create mode 100644 pyro_camera_api/.dockerignore create mode 100644 requirements-git.txt diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..0dec9dae --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.venv +.git +.github +.mypy_cache +.ruff_cache +__pycache__ +*.pyc +*.egg-info +dist +build +.env +data +*.md +!requirements.txt +tests +.pytest_cache diff --git a/Dockerfile b/Dockerfile index d239c9a2..c94941d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,53 @@ +# ---- Builder: only used for git-based deps (needs git) ---- +FROM python:3.11.13-slim-bullseye AS git-deps + +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY ./requirements-git.txt /tmp/requirements-git.txt +RUN pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements-git.txt + +# ---- Runtime ---- FROM python:3.11.13-slim-bullseye -# set environment variables -ENV PATH="/usr/local/bin:$PATH" \ - LANG="C.UTF-8" \ +ENV LANG="C.UTF-8" \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 -# set work directory -WORKDIR /usr/src/app - -# install git -RUN apt-get update && apt-get install git -y +# Layer 1: System libs (~100MB, almost never changes) RUN apt-get update && \ apt-get install -y --no-install-recommends \ - git \ ffmpeg \ libsm6 \ libxext6 \ - || apt-get install -y --fix-missing && \ - apt-get clean && \ + libglib2.0-0 \ + libgl1 \ + && apt-get clean && \ rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir --upgrade pip setuptools wheel +# Layer 2: Stable pip deps (~400MB, changes only on version bumps) COPY ./requirements.txt /tmp/requirements.txt - -RUN pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements.txt && \ +RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ + pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements.txt && \ rm -f /tmp/requirements.txt +# Layer 3: Git-based deps (~5MB, changes when API clients are updated) +COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyroclient /usr/local/lib/python3.11/site-packages/pyroclient +COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyroclient-*.dist-info /usr/local/lib/python3.11/site-packages/ +COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyro_camera_api_client /usr/local/lib/python3.11/site-packages/pyro_camera_api_client +COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyro_camera_api_client-*.dist-info /usr/local/lib/python3.11/site-packages/ + +# Layer 4: Local packages (~1MB, changes on every deploy) WORKDIR /opt/pyroengine_src COPY ./pyro-predictor ./pyro-predictor COPY ./pyroengine ./pyroengine COPY ./setup.py ./setup.py +RUN pip install --no-cache-dir ./pyro-predictor \ + && pip install --no-cache-dir . -RUN pip install --no-cache-dir -e ./pyro-predictor \ - && pip install --no-cache-dir -e . \ - && rm -rf /root/.cache/pip - +# Layer 5: Entrypoint scripts (~few KB, rarely changes) WORKDIR /usr/src/app - COPY ./src/run.py /usr/src/app/run.py -COPY ./src/control_reolink_cam.py /usr/src/app/control_reolink_cam.py \ No newline at end of file +COPY ./src/control_reolink_cam.py /usr/src/app/control_reolink_cam.py diff --git a/Makefile b/Makefile index 695d88c2..5699b65d 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,10 @@ single-docs: # Update requirements.txt for the main project lock: poetry lock - poetry export -f requirements.txt --without-hashes --output requirements.txt + poetry export -f requirements.txt --without-hashes --output requirements-all.txt + grep 'git+' requirements-all.txt > requirements-git.txt || true + grep -v 'git+' requirements-all.txt > requirements.txt + rm requirements-all.txt # Generate requirements and build camera API Docker image build-api: diff --git a/pyro_camera_api/.dockerignore b/pyro_camera_api/.dockerignore new file mode 100644 index 00000000..609d7703 --- /dev/null +++ b/pyro_camera_api/.dockerignore @@ -0,0 +1,16 @@ +.venv +.git +.mypy_cache +.ruff_cache +__pycache__ +*.pyc +*.egg-info +dist +build +.env +data +*.md +!requirements.txt +tests +.pytest_cache +client diff --git a/pyro_camera_api/Dockerfile b/pyro_camera_api/Dockerfile index 77bb9afe..71612d1c 100644 --- a/pyro_camera_api/Dockerfile +++ b/pyro_camera_api/Dockerfile @@ -1,31 +1,28 @@ FROM python:3.9.16-slim -WORKDIR /usr/src/app - -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV PYTHONPATH=/usr/src/app +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/usr/src/app +# Layer 1: System libs (~100MB, almost never changes) RUN apt-get update && \ apt-get install -y --no-install-recommends \ - git \ ffmpeg \ libsm6 \ libxext6 \ libgl1 \ - build-essential \ libglib2.0-0 \ && apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Layer 2: Pip deps (~400MB, changes only on version bumps) COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv - COPY requirements.txt /tmp/requirements.txt -# if you have a lock file uncomment this -# COPY uv.lock /tmp/uv.lock - -RUN uv pip install --no-cache --system -r /tmp/requirements.txt +RUN uv pip install --no-cache --system -r /tmp/requirements.txt && \ + rm /bin/uv /tmp/requirements.txt +# Layer 3: Source code (~few KB, changes on every deploy) +WORKDIR /usr/src/app COPY pyproject.toml ./pyproject.toml COPY pyro_camera_api ./pyro_camera_api diff --git a/pyro_camera_api/requirements.txt b/pyro_camera_api/requirements.txt index 2cba2353..046bffdc 100644 --- a/pyro_camera_api/requirements.txt +++ b/pyro_camera_api/requirements.txt @@ -18,7 +18,6 @@ ncnn==1.0.20240410 ; python_version >= "3.9" and python_version < "4.0" numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0" onnxruntime==1.19.2 ; python_version >= "3.9" and python_version < "4.0" opencv-python-headless==4.11.0.86 ; python_version >= "3.9" and python_version < "4.0" -opencv-python==4.11.0.86 ; python_version >= "3.9" and python_version < "4.0" packaging==25.0 ; python_version >= "3.9" and python_version < "4.0" pillow==10.4.0 ; python_version >= "3.9" and python_version < "4.0" portalocker==3.2.0 ; python_version >= "3.9" and python_version < "4.0" diff --git a/requirements-git.txt b/requirements-git.txt new file mode 100644 index 00000000..96f8c22b --- /dev/null +++ b/requirements-git.txt @@ -0,0 +1,2 @@ +pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0" +pyroclient @ git+https://github.com/pyronear/pyro-api.git@9cba4afdf1d096436ca875bfde104f1a9bc1df24#subdirectory=client ; python_version >= "3.11" and python_version < "4.0" diff --git a/requirements.txt b/requirements.txt index f665c286..94912f9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,14 +12,12 @@ mpmath==1.3.0 ; python_version >= "3.11" and python_version < "4.0" ncnn==1.0.20240410 ; python_version >= "3.11" and python_version < "4.0" numpy==2.2.6 ; python_version >= "3.11" and python_version < "4.0" onnxruntime==1.22.1 ; python_version >= "3.11" and python_version < "4.0" -opencv-python==4.12.0.88 ; python_version >= "3.11" and python_version < "4.0" +opencv-python-headless==4.12.0.88 ; python_version >= "3.11" and python_version < "4.0" packaging==25.0 ; python_version >= "3.11" and python_version < "4.0" pillow==11.0.0 ; python_version >= "3.11" and python_version < "4.0" portalocker==3.2.0 ; python_version >= "3.11" and python_version < "4.0" protobuf==6.33.1 ; python_version >= "3.11" and python_version < "4.0" pyreadline3==3.5.4 ; python_version >= "3.11" and python_version < "4.0" and sys_platform == "win32" -pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0" -pyroclient @ git+https://github.com/pyronear/pyro-api.git@9cba4afdf1d096436ca875bfde104f1a9bc1df24#subdirectory=client ; python_version >= "3.11" and python_version < "4.0" python-dotenv==1.1.0 ; python_version >= "3.11" and python_version < "4.0" pywin32==311 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows" pyyaml==6.0.3 ; python_version >= "3.11" and python_version < "4.0" From 28105797e80f354c7851d14ab9be374bfcd1692a Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 2 Apr 2026 17:01:50 +0200 Subject: [PATCH 2/7] add multi-platform docker CI for engine and camera API - Build and push both images for linux/amd64 and linux/arm64 - Trigger on develop, main, and version tags - GHA layer cache for faster rebuilds - Test both images in docker.yml CI - Remove redundant dockerhub-publish from release.yml --- .github/workflows/build-push-image.yml | 60 +++++++++++++++++++++----- .github/workflows/docker.yml | 7 ++- .github/workflows/release.yml | 25 ----------- 3 files changed, 55 insertions(+), 37 deletions(-) diff --git a/.github/workflows/build-push-image.yml b/.github/workflows/build-push-image.yml index 97980f41..3e5333ee 100644 --- a/.github/workflows/build-push-image.yml +++ b/.github/workflows/build-push-image.yml @@ -1,24 +1,62 @@ -name: Build and Push Docker Image +name: Build and Push Docker Images on: push: branches: - develop + - main + tags: + - 'v*' jobs: - build_and_push_image: - name: Build and Push Docker Image + build-and-push: + name: Build and Push ${{ matrix.image }} runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: pyronear/pyro-engine + context: . + dockerfile: Dockerfile + - image: pyronear/pyro-camera-api + context: pyro_camera_api + dockerfile: pyro_camera_api/Dockerfile steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Log in to Docker Hub - run: echo "${{ secrets.DOCKERHUB_PW }}" | docker login -u "${{ secrets.DOCKERHUB_LOGIN }}" --password-stdin + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - - name: Build Docker image - run: make build-app + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Push Docker image - run: docker push pyronear/pyro-engine:latest + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_PW }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ matrix.image }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.image }} + cache-to: type=gha,mode=max,scope=${{ matrix.image }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d8471c8d..18c996eb 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,8 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build docker image + - name: Build engine image run: make build-app - name: Verify engine starts successfully run: | docker run --rm pyronear/pyro-engine:latest python -c "from pyroengine import SystemController; print('OK')" + - name: Build camera API image + run: make build-api + - name: Verify camera API starts successfully + run: | + docker run --rm pyronear/pyro-camera-api:latest python -c "from pyro_camera_api.main import app; print('OK')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4a1b7589..edebced5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,28 +103,3 @@ jobs: conda install -c pyronear pyroengine python -c "import pyroengine; print(pyroengine.__version__)" - - dockerhub-publish: - if: "!github.event.release.prerelease" - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v4 - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_LOGIN }} - password: ${{ secrets.DOCKERHUB_PW }} - - name: Push to Docker Hub - uses: docker/build-push-action@v4 - with: - context: . - platforms: linux/amd64,linux/arm64 - repository: pyronear/pyro-engine - tag_with_ref: true From a0b65f01142600bbd0597e3618118f2111a446a5 Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 2 Apr 2026 17:10:46 +0200 Subject: [PATCH 3/7] fix docker CI: build camera API without poetry --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 18c996eb..01dbf5f2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -17,7 +17,7 @@ jobs: run: | docker run --rm pyronear/pyro-engine:latest python -c "from pyroengine import SystemController; print('OK')" - name: Build camera API image - run: make build-api + run: docker build -f pyro_camera_api/Dockerfile pyro_camera_api -t pyronear/pyro-camera-api:latest - name: Verify camera API starts successfully run: | docker run --rm pyronear/pyro-camera-api:latest python -c "from pyro_camera_api.main import app; print('OK')" From 8cb6cdaec439ec8245d64b54566c2e7b59505e4c Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 2 Apr 2026 17:20:08 +0200 Subject: [PATCH 4/7] fix engine Dockerfile: copy git deps with dist-info intact --- Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index c94941d2..ae1ec5b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* COPY ./requirements-git.txt /tmp/requirements-git.txt -RUN pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements-git.txt +RUN pip install --no-cache-dir --default-timeout=500 --target=/tmp/git-packages -r /tmp/requirements-git.txt # ---- Runtime ---- FROM python:3.11.13-slim-bullseye @@ -34,10 +34,7 @@ RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ rm -f /tmp/requirements.txt # Layer 3: Git-based deps (~5MB, changes when API clients are updated) -COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyroclient /usr/local/lib/python3.11/site-packages/pyroclient -COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyroclient-*.dist-info /usr/local/lib/python3.11/site-packages/ -COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyro_camera_api_client /usr/local/lib/python3.11/site-packages/pyro_camera_api_client -COPY --from=git-deps /usr/local/lib/python3.11/site-packages/pyro_camera_api_client-*.dist-info /usr/local/lib/python3.11/site-packages/ +COPY --from=git-deps /tmp/git-packages /usr/local/lib/python3.11/site-packages/ # Layer 4: Local packages (~1MB, changes on every deploy) WORKDIR /opt/pyroengine_src From 07006ab6a07fef1c2a51f10d77ee89e6da251cdc Mon Sep 17 00:00:00 2001 From: Mateo Date: Thu, 2 Apr 2026 17:26:35 +0200 Subject: [PATCH 5/7] update make run to pull images instead of building locally --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 5699b65d..2bad0fe6 100644 --- a/Makefile +++ b/Makefile @@ -43,10 +43,10 @@ build-optional-lib: pip install -e .[docs] pip install -e .[dev] -# Build both images and run the stack +# Pull latest images and run the stack run: - docker build . -t pyronear/pyro-engine:latest - docker build -f pyro_camera_api/Dockerfile pyro_camera_api -t pyronear/pyro-camera-api:latest + docker pull pyronear/pyro-engine:latest + docker pull pyronear/pyro-camera-api:latest docker compose up -d # Get log from engine wrapper From 5adaf63db31125e1a2be2773efec7eb42e5f2696 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 3 Apr 2026 17:22:55 +0200 Subject: [PATCH 6/7] fix merge issue --- pyro_camera_api/Dockerfile | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyro_camera_api/Dockerfile b/pyro_camera_api/Dockerfile index aad6bd8b..1b5ef2f4 100644 --- a/pyro_camera_api/Dockerfile +++ b/pyro_camera_api/Dockerfile @@ -7,11 +7,7 @@ ENV PYTHONDONTWRITEBYTECODE=1 \ # Layer 1: System libs (~100MB, almost never changes) RUN apt-get update && \ apt-get install -y --no-install-recommends \ -<<<<<<< HEAD -======= curl \ - git \ ->>>>>>> develop ffmpeg \ libsm6 \ libxext6 \ From 49cfd0e892a56883af50a9558d4cd116e5551640 Mon Sep 17 00:00:00 2001 From: Mateo Date: Fri, 3 Apr 2026 17:23:48 +0200 Subject: [PATCH 7/7] put back --- requirements.txt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index 8e7de242..94912f9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,11 +18,6 @@ pillow==11.0.0 ; python_version >= "3.11" and python_version < "4.0" portalocker==3.2.0 ; python_version >= "3.11" and python_version < "4.0" protobuf==6.33.1 ; python_version >= "3.11" and python_version < "4.0" pyreadline3==3.5.4 ; python_version >= "3.11" and python_version < "4.0" and sys_platform == "win32" -<<<<<<< HEAD -======= -pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0" -pyroclient @ git+https://github.com/pyronear/pyro-api.git@119ff76266eee72ffeb06141a85d420506322fa8#subdirectory=client ; python_version >= "3.11" and python_version < "4.0" ->>>>>>> develop python-dotenv==1.1.0 ; python_version >= "3.11" and python_version < "4.0" pywin32==311 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows" pyyaml==6.0.3 ; python_version >= "3.11" and python_version < "4.0"