From c4c8719c84a20a8d3d686f0860b029b05a292aba Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 17:24:29 +0000 Subject: [PATCH 01/34] Image ARM64 --- airflow_dir/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow_dir/Dockerfile b/airflow_dir/Dockerfile index bbb6c3f..8246146 100755 --- a/airflow_dir/Dockerfile +++ b/airflow_dir/Dockerfile @@ -4,7 +4,8 @@ # BUILD: docker build --rm -t puckel/docker-airflow . # SOURCE: https://github.com/puckel/docker-airflow -FROM python:3.7-slim-buster +# FROM python:3.7-slim-buster +FROM arm64v8/python LABEL maintainer="Puckel_" # Never prompt the user for choices on installation/configuration of packages From 6d8ff2909f8ef943207ffe8013516a0b147bf3ef Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 17:40:54 +0000 Subject: [PATCH 02/34] github action for deploying docker image --- .github/workflows/docker-publish.yml | 38 ++++++++++++++++++++++++++++ dev.docker-compose.yml | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..105c9e3 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,38 @@ +name: Docker + +on: + push: + # Publish `main` as Docker `latest` image. + branches: + - main + + # Publish `v1.2.3` tags as releases. + tags: + - v* + + + +jobs: + + # Push image to GitHub Packages. + # See also https://docs.docker.com/docker-hub/builds/ + push: + + runs-on: ubuntu-latest + if: github.event_name == 'push' + + steps: + - uses: actions/checkout@v2 + + - name: Build image + run: docker-compose -f dev-docker-compose.yml build + + - name: Log into GitHub Container Registry + env: + docker_login: ${{ secrets.DOCKER_USERNAME }} + docker_password: ${{ secrets.DOCKER_PASSWORD }} + run: docker login -u $docker_login --password $docker_password + + - name: Push image to GitHub Container Registry + run: | + docker-compose -f dev-docker-compose.yml push diff --git a/dev.docker-compose.yml b/dev.docker-compose.yml index 49aed17..a399e70 100644 --- a/dev.docker-compose.yml +++ b/dev.docker-compose.yml @@ -2,5 +2,5 @@ version: '2.1' services: airflow: - image: datafuel/airflow:latest + image: youssubuntu/airflow:latest build: ./airflow_dir \ No newline at end of file From 609b2a46dd141d3192f6a9b73b5a12044ff8c871 Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 17:52:55 +0000 Subject: [PATCH 03/34] Test github action --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 105c9e3..2083300 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 - name: Build image - run: docker-compose -f dev-docker-compose.yml build + run: cd ../.. && docker-compose -f dev-docker-compose.yml build - name: Log into GitHub Container Registry env: @@ -35,4 +35,4 @@ jobs: - name: Push image to GitHub Container Registry run: | - docker-compose -f dev-docker-compose.yml push + cd ../.. && docker-compose -f dev-docker-compose.yml push From 2a0bb6660d2965dc21fde345a5f5ea6b5800c047 Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 18:05:04 +0000 Subject: [PATCH 04/34] Revert "Test github action" This reverts commit 609b2a46dd141d3192f6a9b73b5a12044ff8c871. --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 2083300..105c9e3 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 - name: Build image - run: cd ../.. && docker-compose -f dev-docker-compose.yml build + run: docker-compose -f dev-docker-compose.yml build - name: Log into GitHub Container Registry env: @@ -35,4 +35,4 @@ jobs: - name: Push image to GitHub Container Registry run: | - cd ../.. && docker-compose -f dev-docker-compose.yml push + docker-compose -f dev-docker-compose.yml push From b72553510a103472cc004190882ea6a4cc950cbc Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 18:26:03 +0000 Subject: [PATCH 05/34] test action github 3 --- .github/workflows/docker-publish.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 105c9e3..8cb5a09 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -23,6 +23,9 @@ jobs: steps: - uses: actions/checkout@v2 + + - name: list element + run: pwd && ls - name: Build image run: docker-compose -f dev-docker-compose.yml build From caa3f8ce3d3f2b7442b94cc59b25b03e446704c7 Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 18:28:53 +0000 Subject: [PATCH 06/34] Test github action 4 --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 8cb5a09..65692f1 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,7 +28,7 @@ jobs: run: pwd && ls - name: Build image - run: docker-compose -f dev-docker-compose.yml build + run: docker-compose -f dev.docker-compose.yml build - name: Log into GitHub Container Registry env: @@ -38,4 +38,4 @@ jobs: - name: Push image to GitHub Container Registry run: | - docker-compose -f dev-docker-compose.yml push + docker-compose -f dev.docker-compose.yml push From b90e582348c8341e68d2ba2c43d00287b6ff943a Mon Sep 17 00:00:00 2001 From: Youss Ubuntu Date: Sun, 14 Mar 2021 18:36:45 +0000 Subject: [PATCH 07/34] tags image change 3.7 --- airflow_dir/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow_dir/Dockerfile b/airflow_dir/Dockerfile index 8246146..3a9fee2 100755 --- a/airflow_dir/Dockerfile +++ b/airflow_dir/Dockerfile @@ -5,7 +5,7 @@ # SOURCE: https://github.com/puckel/docker-airflow # FROM python:3.7-slim-buster -FROM arm64v8/python +FROM arm64v8/python:3.7.10-buster LABEL maintainer="Puckel_" # Never prompt the user for choices on installation/configuration of packages From 87467643ac1567d2901ee26389b1f26b89968f21 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:06:34 +0000 Subject: [PATCH 08/34] Docker BuildX ARM (Plateform) --- .github/workflows/docker-publish.yml | 58 +++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 65692f1..50ad21d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -13,9 +13,17 @@ on: jobs: + build: + env: + # DOCKER_REGISTRY: kemansel.azurecr.io + DOCKER_IMAGE: youssubuntu/airflow:latest + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_TARGET_PLATFORM: linux/arm/v7 # Push image to GitHub Packages. # See also https://docs.docker.com/docker-hub/builds/ + push: runs-on: ubuntu-latest @@ -27,15 +35,45 @@ jobs: - name: list element run: pwd && ls - - name: Build image - run: docker-compose -f dev.docker-compose.yml build + # - name: Build image + # run: docker-compose -f dev.docker-compose.yml build - name: Log into GitHub Container Registry - env: - docker_login: ${{ secrets.DOCKER_USERNAME }} - docker_password: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u $docker_login --password $docker_password - - - name: Push image to GitHub Container Registry - run: | - docker-compose -f dev.docker-compose.yml push + # env: + # docker_login: ${{ secrets.DOCKER_USERNAME }} + # docker_password: ${{ secrets.DOCKER_PASSWORD }} + run: docker login -u $DOCKER_USERNAME --password $DOCKER_PASSWORD + + # - name: Push image to GitHub Container Registry + # run: | + # docker-compose -f dev.docker-compose.yml push + + - name: Set up Docker Buildx + uses: crazy-max/ghaction-docker-buildx@v1 + with: + version: latest + + - name: Prepare + if: success() + id: prepare + run: | + echo ::set-output name=docker_platform::${DOCKER_TARGET_PLATFORM} + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${GITHUB_RUN_NUMBER} + + - name: Docker Login + if: success() + run: | + echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin + + - name: Run Buildx (push image) + if: success() + run: | + docker buildx build \ + --platform ${{ steps.prepare.outputs.docker_platform }} \ + --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ + --file ./Dockerfile \ + --output type=image,push=true . + + + \ No newline at end of file From f1594d27de9cbcfbdb293ad8115962adb2e70481 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:12:36 +0000 Subject: [PATCH 09/34] Docker buildX - Remove Build --- .github/workflows/docker-publish.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 50ad21d..a25ee36 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -11,16 +11,14 @@ on: - v* +env: + # DOCKER_REGISTRY: kemansel.azurecr.io + DOCKER_IMAGE: youssubuntu/airflow:latest + DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} + DOCKER_TARGET_PLATFORM: linux/arm/v7 jobs: - build: - env: - # DOCKER_REGISTRY: kemansel.azurecr.io - DOCKER_IMAGE: youssubuntu/airflow:latest - DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} - DOCKER_TARGET_PLATFORM: linux/arm/v7 - # Push image to GitHub Packages. # See also https://docs.docker.com/docker-hub/builds/ From 8735294830da646501798107c9671a0f2ae255d0 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:25:42 +0000 Subject: [PATCH 10/34] Docker_buildX --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a25ee36..c1542e0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,7 +71,8 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./Dockerfile \ - --output type=image,push=true . + #--output type=image,push=true . + --push \ No newline at end of file From dea4442448c74eabb0daf02ab1eb8ba72f735a21 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:27:49 +0000 Subject: [PATCH 11/34] docker_build --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index c1542e0..b289733 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -73,6 +73,7 @@ jobs: --file ./Dockerfile \ #--output type=image,push=true . --push + \ No newline at end of file From ce9933cad78e00260126b57785b67ad85d167abf Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:30:18 +0000 Subject: [PATCH 12/34] Docker_BuildX2 --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index b289733..96efe52 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,9 +71,9 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./Dockerfile \ - #--output type=image,push=true . --push - + #--output type=image,push=true . + \ No newline at end of file From d8fa0470983ecd6e4562d6d3606300ebc783a835 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:33:12 +0000 Subject: [PATCH 13/34] Buildx3 --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 96efe52..a50d053 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,6 +71,7 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./Dockerfile \ + --output type=image --push #--output type=image,push=true . From 88817426ff34311828b40c98c12e85eef865fa29 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:36:51 +0000 Subject: [PATCH 14/34] BuildX5 --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a50d053..7670282 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,8 +71,8 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./Dockerfile \ - --output type=image - --push + --output=type=registry + #--output type=image,push=true . From 1b9cebbd4aeb23a9f4b7df10231dbe20112c955f Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 24 Mar 2021 20:40:18 +0000 Subject: [PATCH 15/34] BuildX6 --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 7670282..91e55a1 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -70,9 +70,9 @@ jobs: docker buildx build \ --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ - --file ./Dockerfile \ + --file ./airflow_dir/Dockerfile \ --output=type=registry - + #--output type=image,push=true . From 6cabbcc6b896144f789fd101f7522540864c1e82 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 15:27:43 +0000 Subject: [PATCH 16/34] LIBPQC --- airflow_dir/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow_dir/Dockerfile b/airflow_dir/Dockerfile index 3a9fee2..04ea38e 100755 --- a/airflow_dir/Dockerfile +++ b/airflow_dir/Dockerfile @@ -51,6 +51,7 @@ RUN set -ex \ rsync \ netcat \ locales \ + libpq5 \ && sed -i 's/^# en_US.UTF-8 UTF-8$/en_US.UTF-8 UTF-8/g' /etc/locale.gen \ && locale-gen \ && update-locale LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 \ From 8f23ac7f1b9424760c282601af926dc69a9dab43 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 15:27:58 +0000 Subject: [PATCH 17/34] Change to Youssubuntu --- datafuel.docker-compose.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/datafuel.docker-compose.yml b/datafuel.docker-compose.yml index 25f08c9..6de9a96 100644 --- a/datafuel.docker-compose.yml +++ b/datafuel.docker-compose.yml @@ -17,7 +17,7 @@ services: # - ./pgdata:/var/lib/postgresql/data/pgdata webserver: - image: datafuel/airflow:latest + image: youssubuntu/airflow:latest container_name: webserver restart: always depends_on: @@ -56,7 +56,7 @@ services: # retries: 3 flower: - image: datafuel/airflow:latest + image: youssubuntu/airflow:latest container_name: flower restart: always depends_on: @@ -73,7 +73,7 @@ services: command: airflow celery flower scheduler: - image: datafuel/airflow:latest + image: youssubuntu/airflow:latest container_name: scheduler restart: always depends_on: @@ -101,7 +101,7 @@ services: worker: - image: datafuel/airflow:latest + image: youssubuntu/airflow:latest container_name: worker restart: always depends_on: From 69ede8b3d88e83729603ef03314b5961851091f9 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 16:09:45 +0000 Subject: [PATCH 18/34] Buildx10 --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 91e55a1..0c890af 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,7 +71,7 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./airflow_dir/Dockerfile \ - --output=type=registry + --output type=image,push=true . #--output type=image,push=true . From 228246c4b5b7efe5a6c3208d66ab833f29332174 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 16:13:48 +0000 Subject: [PATCH 19/34] BuildX11 --- .github/workflows/docker-publish.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0c890af..e26cfca 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,7 +71,8 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./airflow_dir/Dockerfile \ - --output type=image,push=true . + --output type=image + --push=true . #--output type=image,push=true . From bac50e9f96627033dd4c3ad170ceaaeea5d52b11 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 18:07:58 +0000 Subject: [PATCH 20/34] BuildX13 --- .github/workflows/docker-publish.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e26cfca..0c890af 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,8 +71,7 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./airflow_dir/Dockerfile \ - --output type=image - --push=true . + --output type=image,push=true . #--output type=image,push=true . From f6f40f97dc253f089e41e43dfe4e3142f9bdfa84 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 18:11:03 +0000 Subject: [PATCH 21/34] BuildX14 --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0c890af..6b95812 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,7 +71,7 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./airflow_dir/Dockerfile \ - --output type=image,push=true . + --output type=image, push=true . #--output type=image,push=true . From 979f4c2669e9ac5cb6bf0877ed428733556f91d4 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 18:55:01 +0000 Subject: [PATCH 22/34] BuildX output=true=image --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6b95812..1ccd603 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -71,7 +71,7 @@ jobs: --platform ${{ steps.prepare.outputs.docker_platform }} \ --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./airflow_dir/Dockerfile \ - --output type=image, push=true . + --output=type=image #--output type=image,push=true . From 30ecdd36df0fedd53ca9c6daf66ba44725ba5606 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:00:21 +0000 Subject: [PATCH 23/34] New_version crazyMax --- .github/workflows/docker-publish.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 1ccd603..65bdc97 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -46,10 +46,13 @@ jobs: # run: | # docker-compose -f dev.docker-compose.yml push - - name: Set up Docker Buildx - uses: crazy-max/ghaction-docker-buildx@v1 - with: - version: latest + + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Prepare if: success() From f227b3b607e6484c2dfb0acdd4117f0001572382 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:03:01 +0000 Subject: [PATCH 24/34] Push --- .github/workflows/docker-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 65bdc97..0785070 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -75,6 +75,7 @@ jobs: --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ --file ./airflow_dir/Dockerfile \ --output=type=image + --push #--output type=image,push=true . From acbb43a6a6eaac327baa4f06b52abce34cd49734 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:16:58 +0000 Subject: [PATCH 25/34] docker buildx install --- .github/workflows/docker-publish.yml | 110 +++++++++++++-------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 0785070..cbdd062 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,63 +22,63 @@ jobs: # Push image to GitHub Packages. # See also https://docs.docker.com/docker-hub/builds/ - push: - + buildx: runs-on: ubuntu-latest - if: github.event_name == 'push' - steps: - - uses: actions/checkout@v2 - - - name: list element - run: pwd && ls - - # - name: Build image - # run: docker-compose -f dev.docker-compose.yml build - - - name: Log into GitHub Container Registry - # env: - # docker_login: ${{ secrets.DOCKER_USERNAME }} - # docker_password: ${{ secrets.DOCKER_PASSWORD }} - run: docker login -u $DOCKER_USERNAME --password $DOCKER_PASSWORD - - # - name: Push image to GitHub Container Registry - # run: | - # docker-compose -f dev.docker-compose.yml push - - - # https://github.com/docker/setup-qemu-action - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - # https://github.com/docker/setup-buildx-action - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - - name: Prepare - if: success() - id: prepare - run: | - echo ::set-output name=docker_platform::${DOCKER_TARGET_PLATFORM} - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=version::${GITHUB_RUN_NUMBER} - - - name: Docker Login - if: success() - run: | - echo "${DOCKER_PASSWORD}" | docker login --username "${DOCKER_USERNAME}" --password-stdin - - - name: Run Buildx (push image) - if: success() - run: | - docker buildx build \ - --platform ${{ steps.prepare.outputs.docker_platform }} \ - --tag ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ - --file ./airflow_dir/Dockerfile \ - --output=type=image - --push - - #--output type=image,push=true . - + - + name: Checkout + uses: actions/checkout@v2 + - + name: Prepare + id: prepare + run: | + DOCKER_IMAGE=crazymax/diun + DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/386,linux/ppc64le,linux/s390x + VERSION=edge + + if [[ $GITHUB_REF == refs/tags/* ]]; then + VERSION=${GITHUB_REF#refs/tags/v} + fi + if [ "${{ github.event_name }}" = "schedule" ]; then + VERSION=nightly + fi + + TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" + if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" + fi + + echo ::set-output name=docker_image::${DOCKER_IMAGE} + echo ::set-output name=version::${VERSION} + echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ + --build-arg VERSION=${VERSION} \ + --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ + --build-arg VCS_REF=${GITHUB_SHA::8} \ + ${TAGS} --file ./airflow_dir/Dockerfile ./ + - + name: Set up Docker Buildx + uses: crazy-max/ghaction-docker-buildx@v3 + - + name: Docker Buildx (build) + run: | + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + - + name: Login to DockerHub + if: success() && github.event_name != 'pull_request' + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - + name: Docker Buildx (push) + if: success() && github.event_name != 'pull_request' + run: | + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + - + name: Inspect image + if: always() && github.event_name != 'pull_request' + run: | + docker buildx imagetools inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ No newline at end of file From ee7d15f414cf2163ef9c36be2c58156083d8ced4 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:19:42 +0000 Subject: [PATCH 26/34] True --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index cbdd062..30ce0dd 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -61,7 +61,7 @@ jobs: - name: Docker Buildx (build) run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} - name: Login to DockerHub if: success() && github.event_name != 'pull_request' From a52afecd4d043cc014598df2a74fcb3edd518f6e Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:25:39 +0000 Subject: [PATCH 27/34] Buildx34 --- .github/workflows/docker-publish.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 30ce0dd..ca54870 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -56,8 +56,12 @@ jobs: --build-arg VCS_REF=${GITHUB_SHA::8} \ ${TAGS} --file ./airflow_dir/Dockerfile ./ - - name: Set up Docker Buildx - uses: crazy-max/ghaction-docker-buildx@v3 + # https://github.com/docker/setup-qemu-action + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + # https://github.com/docker/setup-buildx-action + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 - name: Docker Buildx (build) run: | From bce28c44a61a8e395a7d707f806f2f798ac41eb0 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:35:37 +0000 Subject: [PATCH 28/34] False --- .github/workflows/docker-publish.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index ca54870..9044743 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -65,7 +65,7 @@ jobs: - name: Docker Buildx (build) run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} - name: Login to DockerHub if: success() && github.event_name != 'pull_request' @@ -82,7 +82,4 @@ jobs: name: Inspect image if: always() && github.event_name != 'pull_request' run: | - docker buildx imagetools inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} - - - \ No newline at end of file + docker buildx imagetools inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ No newline at end of file From c46770312980f5eea2f6b1872ed245c5cf648468 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Fri, 26 Mar 2021 19:40:14 +0000 Subject: [PATCH 29/34] Cache --- .github/workflows/docker-publish.yml | 60 ++++++++++++---------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 9044743..5920385 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -22,64 +22,54 @@ jobs: # Push image to GitHub Packages. # See also https://docs.docker.com/docker-hub/builds/ - buildx: + buildx: runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - DOCKER_IMAGE=crazymax/diun - DOCKER_PLATFORMS=linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/386,linux/ppc64le,linux/s390x - VERSION=edge - - if [[ $GITHUB_REF == refs/tags/* ]]; then - VERSION=${GITHUB_REF#refs/tags/v} - fi - if [ "${{ github.event_name }}" = "schedule" ]; then - VERSION=nightly - fi - - TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" - if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then - TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" - fi - - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=version::${VERSION} - echo ::set-output name=buildx_args::--platform ${DOCKER_PLATFORMS} \ - --build-arg VERSION=${VERSION} \ - --build-arg BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - --build-arg VCS_REF=${GITHUB_SHA::8} \ - ${TAGS} --file ./airflow_dir/Dockerfile ./ - - - # https://github.com/docker/setup-qemu-action + # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v1 # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 + - + name: Cache Docker layers + uses: actions/cache@v2 + id: cache + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Docker Buildx (build) run: | - docker buildx build --output "type=image,push=false" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build \ + --cache-from "type=local,src=/tmp/.buildx-cache" \ + --cache-to "type=local,dest=/tmp/.buildx-cache" \ + --platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x \ + --output "type=image,push=false" \ + --tag crazymax/diun:latest \ + --file ./Dockerfile-diun ./ - name: Login to DockerHub - if: success() && github.event_name != 'pull_request' uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - name: Docker Buildx (push) - if: success() && github.event_name != 'pull_request' run: | - docker buildx build --output "type=image,push=true" ${{ steps.prepare.outputs.buildx_args }} + docker buildx build \ + --cache-from "type=local,src=/tmp/.buildx-cache" \ + --platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x \ + --output "type=image,push=true" \ + --tag crazymax/diun:latest \ + --file ./Dockerfile-diun ./ - name: Inspect image - if: always() && github.event_name != 'pull_request' run: | - docker buildx imagetools inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} \ No newline at end of file + docker buildx imagetools inspect crazymax/diun:latest \ No newline at end of file From 315c9d4a1351623ef8d6108edb44a417c3cd85ed Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 7 Apr 2021 18:58:59 +0000 Subject: [PATCH 30/34] Verif1 --- .github/workflows/docker-publish.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 5920385..d038f8e 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,10 +50,10 @@ jobs: docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ --cache-to "type=local,dest=/tmp/.buildx-cache" \ - --platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x \ + --platform linux/arm64 \ --output "type=image,push=false" \ - --tag crazymax/diun:latest \ - --file ./Dockerfile-diun ./ + --tag ${{ env.DOCKER_IMAGE }} \ + --file ./airflow_dir ./ - name: Login to DockerHub uses: docker/login-action@v1 @@ -65,11 +65,11 @@ jobs: run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ - --platform linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64,linux/ppc64le,linux/s390x \ + --platform linux/arm64 \ --output "type=image,push=true" \ - --tag crazymax/diun:latest \ - --file ./Dockerfile-diun ./ + --tag ${{ env.DOCKER_IMAGE }} \ + --file ./airflow_dir ./ - name: Inspect image run: | - docker buildx imagetools inspect crazymax/diun:latest \ No newline at end of file + docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }} \ No newline at end of file From c04d2b69136b98301da90368a9ff154921402e2b Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 7 Apr 2021 19:04:27 +0000 Subject: [PATCH 31/34] Verif2 --- .github/workflows/docker-publish.yml | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index d038f8e..e7e7e15 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -25,18 +25,16 @@ jobs: buildx: runs-on: ubuntu-latest steps: - - - name: Checkout + - name: Checkout uses: actions/checkout@v2 - - + # https://github.com/docker/setup-qemu-action - name: Set up QEMU uses: docker/setup-qemu-action@v1 # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - - name: Cache Docker layers + - name: Cache Docker layers uses: actions/cache@v2 id: cache with: @@ -44,8 +42,7 @@ jobs: key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- - - - name: Docker Buildx (build) + - name: Docker Buildx (build) run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ @@ -54,14 +51,12 @@ jobs: --output "type=image,push=false" \ --tag ${{ env.DOCKER_IMAGE }} \ --file ./airflow_dir ./ - - - name: Login to DockerHub + - name: Login to DockerHub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Docker Buildx (push) + - name: Docker Buildx (push) run: | docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ @@ -69,7 +64,6 @@ jobs: --output "type=image,push=true" \ --tag ${{ env.DOCKER_IMAGE }} \ --file ./airflow_dir ./ - - - name: Inspect image + - name: Inspect image run: | docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }} \ No newline at end of file From 962f04a81a0a432d54fb65a6e8c13917101fb0fa Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 7 Apr 2021 19:07:59 +0000 Subject: [PATCH 32/34] Debug --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index e7e7e15..a2f7962 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -44,7 +44,7 @@ jobs: ${{ runner.os }}-buildx- - name: Docker Buildx (build) run: | - docker buildx build \ + pwd && ls && docker buildx build \ --cache-from "type=local,src=/tmp/.buildx-cache" \ --cache-to "type=local,dest=/tmp/.buildx-cache" \ --platform linux/arm64 \ From bc867ee3dd0daf3f06f18735bc3efaa6451180ad Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 7 Apr 2021 19:10:34 +0000 Subject: [PATCH 33/34] Debug2 --- .github/workflows/docker-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index a2f7962..60ef9f1 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,7 +50,7 @@ jobs: --platform linux/arm64 \ --output "type=image,push=false" \ --tag ${{ env.DOCKER_IMAGE }} \ - --file ./airflow_dir ./ + --file airflow_dir ./ - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -63,7 +63,7 @@ jobs: --platform linux/arm64 \ --output "type=image,push=true" \ --tag ${{ env.DOCKER_IMAGE }} \ - --file ./airflow_dir ./ + --file airflow_dir ./ - name: Inspect image run: | docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }} \ No newline at end of file From bcf881acee23455498b37812400a7156000152a6 Mon Sep 17 00:00:00 2001 From: youssef ubuntu Date: Wed, 7 Apr 2021 19:33:54 +0000 Subject: [PATCH 34/34] Debug3 --- .github/workflows/docker-publish.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 60ef9f1..201ee7a 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -34,23 +34,23 @@ jobs: # https://github.com/docker/setup-buildx-action - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + + # - name: Cache Docker layers + # uses: actions/cache@v2 + # id: cache + # with: + # path: /tmp/.buildx-cache + # key: ${{ runner.os }}-buildx-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-buildx- + - name: Docker Buildx (build) run: | pwd && ls && docker buildx build \ - --cache-from "type=local,src=/tmp/.buildx-cache" \ - --cache-to "type=local,dest=/tmp/.buildx-cache" \ --platform linux/arm64 \ --output "type=image,push=false" \ --tag ${{ env.DOCKER_IMAGE }} \ - --file airflow_dir ./ + --file airflow_dir/Dockerfile ./ - name: Login to DockerHub uses: docker/login-action@v1 with: @@ -59,11 +59,10 @@ jobs: - name: Docker Buildx (push) run: | docker buildx build \ - --cache-from "type=local,src=/tmp/.buildx-cache" \ --platform linux/arm64 \ --output "type=image,push=true" \ --tag ${{ env.DOCKER_IMAGE }} \ - --file airflow_dir ./ + --file airflow_dir/Dockerfile ./ - name: Inspect image run: | docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }} \ No newline at end of file