diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..0dec9dae --- /dev/null +++ b/.dockerignore @@ -0,0 +1,16 @@ +.venv +.git +.github +.mypy_cache +.ruff_cache +__pycache__ +*.pyc +*.egg-info +dist +build +.env +data +*.md +!requirements.txt +tests +.pytest_cache diff --git a/.github/workflows/build-push-image.yml b/.github/workflows/build-push-image.yml index 97980f41..3e5333ee 100644 --- a/.github/workflows/build-push-image.yml +++ b/.github/workflows/build-push-image.yml @@ -1,24 +1,62 @@ -name: Build and Push Docker Image +name: Build and Push Docker Images on: push: branches: - develop + - main + tags: + - 'v*' jobs: - build_and_push_image: - name: Build and Push Docker Image + build-and-push: + name: Build and Push ${{ matrix.image }} runs-on: ubuntu-latest + strategy: + matrix: + include: + - image: pyronear/pyro-engine + context: . + dockerfile: Dockerfile + - image: pyronear/pyro-camera-api + context: pyro_camera_api + dockerfile: pyro_camera_api/Dockerfile steps: - - name: Checkout repository - uses: actions/checkout@v4 + - name: Checkout repository + uses: actions/checkout@v4 - - name: Log in to Docker Hub - run: echo "${{ secrets.DOCKERHUB_PW }}" | docker login -u "${{ secrets.DOCKERHUB_LOGIN }}" --password-stdin + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 - - name: Build Docker image - run: make build-app + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 - - name: Push Docker image - run: docker push pyronear/pyro-engine:latest + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_LOGIN }} + password: ${{ secrets.DOCKERHUB_PW }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ matrix.image }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.image }} + cache-to: type=gha,mode=max,scope=${{ matrix.image }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d8471c8d..01dbf5f2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,8 +11,13 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Build docker image + - name: Build engine image run: make build-app - name: Verify engine starts successfully run: | docker run --rm pyronear/pyro-engine:latest python -c "from pyroengine import SystemController; print('OK')" + - name: Build camera API image + run: docker build -f pyro_camera_api/Dockerfile pyro_camera_api -t pyronear/pyro-camera-api:latest + - name: Verify camera API starts successfully + run: | + docker run --rm pyronear/pyro-camera-api:latest python -c "from pyro_camera_api.main import app; print('OK')" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5b404ee9..e1e70b1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -103,28 +103,3 @@ jobs: conda install -c pyronear pyroengine python -c "import pyroengine; print(pyroengine.__version__)" - - dockerhub-publish: - if: "!github.event.release.prerelease" - name: Push Docker image to Docker Hub - runs-on: ubuntu-latest - steps: - - name: Check out the repo - uses: actions/checkout@v4 - - name: Set up QEMU - uses: docker/setup-qemu-action@v4 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v4 - - name: Login to Docker Hub - uses: docker/login-action@v4 - with: - username: ${{ secrets.DOCKERHUB_LOGIN }} - password: ${{ secrets.DOCKERHUB_PW }} - - name: Push to Docker Hub - uses: docker/build-push-action@v4 - with: - context: . - platforms: linux/amd64,linux/arm64 - repository: pyronear/pyro-engine - tag_with_ref: true diff --git a/Dockerfile b/Dockerfile index d239c9a2..ae1ec5b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,50 @@ +# ---- Builder: only used for git-based deps (needs git) ---- +FROM python:3.11.13-slim-bullseye AS git-deps + +RUN apt-get update && \ + apt-get install -y --no-install-recommends git && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +COPY ./requirements-git.txt /tmp/requirements-git.txt +RUN pip install --no-cache-dir --default-timeout=500 --target=/tmp/git-packages -r /tmp/requirements-git.txt + +# ---- Runtime ---- FROM python:3.11.13-slim-bullseye -# set environment variables -ENV PATH="/usr/local/bin:$PATH" \ - LANG="C.UTF-8" \ +ENV LANG="C.UTF-8" \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 -# set work directory -WORKDIR /usr/src/app - -# install git -RUN apt-get update && apt-get install git -y +# Layer 1: System libs (~100MB, almost never changes) RUN apt-get update && \ apt-get install -y --no-install-recommends \ - git \ ffmpeg \ libsm6 \ libxext6 \ - || apt-get install -y --fix-missing && \ - apt-get clean && \ + libglib2.0-0 \ + libgl1 \ + && apt-get clean && \ rm -rf /var/lib/apt/lists/* - -RUN pip install --no-cache-dir --upgrade pip setuptools wheel +# Layer 2: Stable pip deps (~400MB, changes only on version bumps) COPY ./requirements.txt /tmp/requirements.txt - -RUN pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements.txt && \ +RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ + pip install --no-cache-dir --default-timeout=500 -r /tmp/requirements.txt && \ rm -f /tmp/requirements.txt +# Layer 3: Git-based deps (~5MB, changes when API clients are updated) +COPY --from=git-deps /tmp/git-packages /usr/local/lib/python3.11/site-packages/ + +# Layer 4: Local packages (~1MB, changes on every deploy) WORKDIR /opt/pyroengine_src COPY ./pyro-predictor ./pyro-predictor COPY ./pyroengine ./pyroengine COPY ./setup.py ./setup.py +RUN pip install --no-cache-dir ./pyro-predictor \ + && pip install --no-cache-dir . -RUN pip install --no-cache-dir -e ./pyro-predictor \ - && pip install --no-cache-dir -e . \ - && rm -rf /root/.cache/pip - +# Layer 5: Entrypoint scripts (~few KB, rarely changes) WORKDIR /usr/src/app - COPY ./src/run.py /usr/src/app/run.py -COPY ./src/control_reolink_cam.py /usr/src/app/control_reolink_cam.py \ No newline at end of file +COPY ./src/control_reolink_cam.py /usr/src/app/control_reolink_cam.py diff --git a/Makefile b/Makefile index 695d88c2..2bad0fe6 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,10 @@ single-docs: # Update requirements.txt for the main project lock: poetry lock - poetry export -f requirements.txt --without-hashes --output requirements.txt + poetry export -f requirements.txt --without-hashes --output requirements-all.txt + grep 'git+' requirements-all.txt > requirements-git.txt || true + grep -v 'git+' requirements-all.txt > requirements.txt + rm requirements-all.txt # Generate requirements and build camera API Docker image build-api: @@ -40,10 +43,10 @@ build-optional-lib: pip install -e .[docs] pip install -e .[dev] -# Build both images and run the stack +# Pull latest images and run the stack run: - docker build . -t pyronear/pyro-engine:latest - docker build -f pyro_camera_api/Dockerfile pyro_camera_api -t pyronear/pyro-camera-api:latest + docker pull pyronear/pyro-engine:latest + docker pull pyronear/pyro-camera-api:latest docker compose up -d # Get log from engine wrapper diff --git a/pyro_camera_api/.dockerignore b/pyro_camera_api/.dockerignore new file mode 100644 index 00000000..609d7703 --- /dev/null +++ b/pyro_camera_api/.dockerignore @@ -0,0 +1,16 @@ +.venv +.git +.mypy_cache +.ruff_cache +__pycache__ +*.pyc +*.egg-info +dist +build +.env +data +*.md +!requirements.txt +tests +.pytest_cache +client diff --git a/pyro_camera_api/Dockerfile b/pyro_camera_api/Dockerfile index 466bfb73..1b5ef2f4 100644 --- a/pyro_camera_api/Dockerfile +++ b/pyro_camera_api/Dockerfile @@ -1,32 +1,29 @@ FROM python:3.9.16-slim -WORKDIR /usr/src/app - -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV PYTHONPATH=/usr/src/app +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/usr/src/app +# Layer 1: System libs (~100MB, almost never changes) RUN apt-get update && \ apt-get install -y --no-install-recommends \ curl \ - git \ ffmpeg \ libsm6 \ libxext6 \ libgl1 \ - build-essential \ libglib2.0-0 \ && apt-get clean && \ rm -rf /var/lib/apt/lists/* +# Layer 2: Pip deps (~400MB, changes only on version bumps) COPY --from=ghcr.io/astral-sh/uv:0.5.13 /uv /bin/uv - COPY requirements.txt /tmp/requirements.txt -# if you have a lock file uncomment this -# COPY uv.lock /tmp/uv.lock - -RUN uv pip install --no-cache --system -r /tmp/requirements.txt +RUN uv pip install --no-cache --system -r /tmp/requirements.txt && \ + rm /bin/uv /tmp/requirements.txt +# Layer 3: Source code (~few KB, changes on every deploy) +WORKDIR /usr/src/app COPY pyproject.toml ./pyproject.toml COPY pyro_camera_api ./pyro_camera_api diff --git a/pyro_camera_api/requirements.txt b/pyro_camera_api/requirements.txt index 2cba2353..046bffdc 100644 --- a/pyro_camera_api/requirements.txt +++ b/pyro_camera_api/requirements.txt @@ -18,7 +18,6 @@ ncnn==1.0.20240410 ; python_version >= "3.9" and python_version < "4.0" numpy==1.26.4 ; python_version >= "3.9" and python_version < "4.0" onnxruntime==1.19.2 ; python_version >= "3.9" and python_version < "4.0" opencv-python-headless==4.11.0.86 ; python_version >= "3.9" and python_version < "4.0" -opencv-python==4.11.0.86 ; python_version >= "3.9" and python_version < "4.0" packaging==25.0 ; python_version >= "3.9" and python_version < "4.0" pillow==10.4.0 ; python_version >= "3.9" and python_version < "4.0" portalocker==3.2.0 ; python_version >= "3.9" and python_version < "4.0" diff --git a/requirements-git.txt b/requirements-git.txt new file mode 100644 index 00000000..96f8c22b --- /dev/null +++ b/requirements-git.txt @@ -0,0 +1,2 @@ +pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0" +pyroclient @ git+https://github.com/pyronear/pyro-api.git@9cba4afdf1d096436ca875bfde104f1a9bc1df24#subdirectory=client ; python_version >= "3.11" and python_version < "4.0" diff --git a/requirements.txt b/requirements.txt index f21d96d7..94912f9b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,14 +12,12 @@ mpmath==1.3.0 ; python_version >= "3.11" and python_version < "4.0" ncnn==1.0.20240410 ; python_version >= "3.11" and python_version < "4.0" numpy==2.2.6 ; python_version >= "3.11" and python_version < "4.0" onnxruntime==1.22.1 ; python_version >= "3.11" and python_version < "4.0" -opencv-python==4.12.0.88 ; python_version >= "3.11" and python_version < "4.0" +opencv-python-headless==4.12.0.88 ; python_version >= "3.11" and python_version < "4.0" packaging==25.0 ; python_version >= "3.11" and python_version < "4.0" pillow==11.0.0 ; python_version >= "3.11" and python_version < "4.0" portalocker==3.2.0 ; python_version >= "3.11" and python_version < "4.0" protobuf==6.33.1 ; python_version >= "3.11" and python_version < "4.0" pyreadline3==3.5.4 ; python_version >= "3.11" and python_version < "4.0" and sys_platform == "win32" -pyro-camera-api-client @ git+https://github.com/pyronear/pyro-engine.git@0f3ff6836d226334847af63e365e8849c2bced22#subdirectory=pyro_camera_api/client ; python_version >= "3.11" and python_version < "4.0" -pyroclient @ git+https://github.com/pyronear/pyro-api.git@119ff76266eee72ffeb06141a85d420506322fa8#subdirectory=client ; python_version >= "3.11" and python_version < "4.0" python-dotenv==1.1.0 ; python_version >= "3.11" and python_version < "4.0" pywin32==311 ; python_version >= "3.11" and python_version < "4.0" and platform_system == "Windows" pyyaml==6.0.3 ; python_version >= "3.11" and python_version < "4.0"