From 8ea45293867f90831828c6ecefc5e50b351188eb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 5 Aug 2025 16:30:04 -0400 Subject: [PATCH 01/50] feat(docker): Add CLP package image for Docker Compose integration. --- .../clp-execution-image-build/action.yaml | 32 +++++- .github/actions/run-on-image/action.yaml | 1 + .../workflows/clp-execution-image-build.yaml | 1 + .../workflows/clp-package-image-build.yaml | 103 ++++++++++++++++++ .../ubuntu-jammy/install-prebuilt-packages.sh | 1 + taskfile.yaml | 11 ++ .../clp-package-ubuntu-jammy/Dockerfile | 22 ++++ .../clp-package-ubuntu-jammy/build.sh | 38 +++++++ .../install-prebuilt-packages.sh | 19 ++++ 9 files changed, 222 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/clp-package-image-build.yaml create mode 100644 tools/docker-images/clp-package-ubuntu-jammy/Dockerfile create mode 100755 tools/docker-images/clp-package-ubuntu-jammy/build.sh create mode 100755 tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index e42c17e7fd..49643e5ef7 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -1,8 +1,14 @@ -name: "clp-execution-image-build" -description: "Builds a container image that contains the dependencies necessary -to run the CLP package." +name: "clp-image-build" +description: "Builds a CLP container image." inputs: + image_type: + description: "Type of image to build" + required: true + type: "choice" + options: + - "execution" + - "package" image_registry: default: "ghcr.io" description: "Container image registry" @@ -50,15 +56,29 @@ runs: uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-execution-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" + /clp-${{inputs.image_type}}-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" + + - name: "Determine Dockerfile Path" + id: "dockerfile" + shell: "bash" + run: | + base_path="./tools/docker-images" + platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" + + if [[ "${{inputs.image_type}}" == "execution" ]]; then + path="$base_path/clp-execution-base-$platform/Dockerfile" + else + path="$base_path/clp-package-$platform/Dockerfile" + fi + + echo "DOCKERFILE=$path" >> "$GITHUB_OUTPUT" - name: "Build and Push" if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" - file: "./tools/docker-images/\ - clp-execution-base-${{inputs.platform_id}}-${{inputs.platform_version_id}}/Dockerfile" + file: "${{steps.dockerfile.outputs.DOCKERFILE}}" push: true tags: "${{steps.meta.outputs.tags}}" labels: "${{steps.meta.outputs.labels}}" diff --git a/.github/actions/run-on-image/action.yaml b/.github/actions/run-on-image/action.yaml index 25df6207a1..6cc28403ef 100644 --- a/.github/actions/run-on-image/action.yaml +++ b/.github/actions/run-on-image/action.yaml @@ -45,6 +45,7 @@ runs: - run: >- docker run --user $(id -u):$(id -g) + --env npm_config_cache=/tmp/.npm --volume "$GITHUB_WORKSPACE":/mnt/repo --workdir /mnt/repo ${{steps.get_image_props.outputs.qualified_image_name}} diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index c2af595c0f..b30e281d92 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -63,6 +63,7 @@ jobs: - uses: "./.github/actions/clp-execution-image-build" with: + image_type: "execution" image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-package-image-build.yaml b/.github/workflows/clp-package-image-build.yaml new file mode 100644 index 0000000000..b3fd1241e4 --- /dev/null +++ b/.github/workflows/clp-package-image-build.yaml @@ -0,0 +1,103 @@ +name: "clp-package-image-build" + +on: + push: + workflow_dispatch: + +env: + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + needs: "filter-relevant-changes" + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-package-image: + name: "ubuntu-jammy-package-image" + needs: "ubuntu-jammy-deps-image" + runs-on: "ubuntu-24.04" + env: + OS_NAME: "ubuntu-jammy" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - name: "Build the package" + uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_type: "package" + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 03a3c26e8c..c0c84d755e 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -25,6 +25,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ python3 \ python3-pip \ python3-venv \ + rsync \ software-properties-common \ unzip diff --git a/taskfile.yaml b/taskfile.yaml index af6f976d60..655a2a1812 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -162,6 +162,17 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + docker-image-package: + vars: + SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package-ubuntu-jammy" + dir: "{{.SRC_DIR}}" + deps: + - "package" + sources: + - "{{.SRC_DIR}}/**/*" + cmds: + - "./build.sh" + clp-s-generate-parsers: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" diff --git a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile new file mode 100644 index 0000000000..529f16a9c9 --- /dev/null +++ b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:jammy AS base + +COPY ./build/clp-package /opt/clp + +WORKDIR /root + +COPY ./tools/docker-images/clp-package-ubuntu-jammy/setup-scripts ./setup-scripts +RUN ./setup-scripts/install-prebuilt-packages.sh +RUN rm -rf ./setup-scripts/ + +# Remove cached files +RUN apt-get clean \ + && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Flatten the image +FROM scratch +COPY --from=base / / + +ENV PATH="/opt/clp/sbin:${PATH}" +ENV PATH="/opt/clp/bin:${PATH}" +ENV PYTHONPATH="/opt/clp/lib/python3/site-packages" +ENV CLP_HOME="/opt/clp" diff --git a/tools/docker-images/clp-package-ubuntu-jammy/build.sh b/tools/docker-images/clp-package-ubuntu-jammy/build.sh new file mode 100755 index 0000000000..d6583cbb9a --- /dev/null +++ b/tools/docker-images/clp-package-ubuntu-jammy/build.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +# Exit on any error +set -e + +# Error on undefined variable +set -u + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" +repo_root=${script_dir}/../../../ + +arch=$(uname -m) + +if [ "$arch" = "x86_64" ]; then + arch_name="x86" +elif [ "$arch" = "aarch64" ]; then + arch_name="arm64" +else + echo "Error: Unsupported architecture - $arch" + exit 1 +fi + +build_cmd=( + docker build + --tag clp-package-${arch_name}-ubuntu-jammy:dev + "$repo_root" + --file "${script_dir}/Dockerfile" +) + +if command -v git >/dev/null && git -C "$script_dir" rev-parse --is-inside-work-tree >/dev/null ; +then + build_cmd+=( + --label "org.opencontainers.image.revision=$(git -C "$script_dir" rev-parse HEAD)" + --label "org.opencontainers.image.source=$(git -C "$script_dir" remote get-url origin)" + ) +fi + +"${build_cmd[@]}" diff --git a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh new file mode 100755 index 0000000000..3e71d6b4d0 --- /dev/null +++ b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +# Exit on any error +set -e + +# Error on undefined variable +set -u + +apt-get update +DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ + ca-certificates \ + checkinstall \ + curl \ + libcurl4 \ + libmariadb-dev \ + libssl-dev \ + python3 \ + rsync \ + zstd From 4d402c179ac54806c6833b33639a6b5d1706fc09 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 03:02:17 -0400 Subject: [PATCH 02/50] Update install-prebuilt-packages.sh --- .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index c0c84d755e..3cd8b708c1 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -23,6 +23,7 @@ DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ openjdk-11-jdk \ pkg-config \ python3 \ + python3-dev \ python3-pip \ python3-venv \ rsync \ From adef439506522029a1446e90fce94379c2a0b856 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 03:33:43 -0400 Subject: [PATCH 03/50] lint --- .github/actions/clp-execution-image-build/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-execution-image-build/action.yaml index 49643e5ef7..c7047cfefc 100644 --- a/.github/actions/clp-execution-image-build/action.yaml +++ b/.github/actions/clp-execution-image-build/action.yaml @@ -64,13 +64,13 @@ runs: run: | base_path="./tools/docker-images" platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" - + if [[ "${{inputs.image_type}}" == "execution" ]]; then path="$base_path/clp-execution-base-$platform/Dockerfile" else path="$base_path/clp-package-$platform/Dockerfile" fi - + echo "DOCKERFILE=$path" >> "$GITHUB_OUTPUT" - name: "Build and Push" From 9aa74ff5ea3d963ab2297734c249f0e80120ceab Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 03:34:25 -0400 Subject: [PATCH 04/50] Rename workflow from 'clp-execution-image-build' to 'clp-image-build' --- .../action.yaml | 0 .github/workflows/clp-execution-image-build.yaml | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename .github/actions/{clp-execution-image-build => clp-image-build}/action.yaml (100%) diff --git a/.github/actions/clp-execution-image-build/action.yaml b/.github/actions/clp-image-build/action.yaml similarity index 100% rename from .github/actions/clp-execution-image-build/action.yaml rename to .github/actions/clp-image-build/action.yaml diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index b30e281d92..b9315af0e7 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -1,14 +1,14 @@ -name: "clp-execution-image-build" +name: "clp-image-build" on: pull_request: paths: - - ".github/actions/clp-execution-image-build/action.yaml" + - "../actions/clp-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" push: paths: - - ".github/actions/clp-execution-image-build/action.yaml" + - "../actions/clp-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" schedule: From 542da81c5aba2894687c63d89a6970439030f6d1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 04:05:05 -0400 Subject: [PATCH 05/50] correct action name --- .github/workflows/clp-execution-image-build.yaml | 2 +- .github/workflows/clp-package-image-build.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index b9315af0e7..1022f0189e 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -61,7 +61,7 @@ jobs: shell: "bash" run: "chown $(id -u):$(id -g) -R ." - - uses: "./.github/actions/clp-execution-image-build" + - uses: "./.github/actions/clp-image-build" with: image_type: "execution" image_registry: "ghcr.io" diff --git a/.github/workflows/clp-package-image-build.yaml b/.github/workflows/clp-package-image-build.yaml index b3fd1241e4..65c327b61b 100644 --- a/.github/workflows/clp-package-image-build.yaml +++ b/.github/workflows/clp-package-image-build.yaml @@ -93,7 +93,7 @@ jobs: run_command: >- CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package - - uses: "./.github/actions/clp-execution-image-build" + - uses: "./.github/actions/clp-image-build" with: image_type: "package" image_registry: "ghcr.io" From ce43d17aece097241f9435ffb9c4ca207f2df8c2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 04:45:51 -0400 Subject: [PATCH 06/50] fix(workflow): Revert workflow name to 'clp-execution-image-build' --- .github/workflows/clp-execution-image-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 1022f0189e..669b59e521 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -1,4 +1,4 @@ -name: "clp-image-build" +name: "clp-execution-image-build" on: pull_request: From 75f88d9324266786d470f1aa2c75da6a68bff92c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 04:48:37 -0400 Subject: [PATCH 07/50] fix(workflow): revert action path in workflow configuration --- .github/workflows/clp-execution-image-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index 669b59e521..bf95f7174b 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -3,12 +3,12 @@ name: "clp-execution-image-build" on: pull_request: paths: - - "../actions/clp-image-build/action.yaml" + - ".github/actions/clp-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" push: paths: - - "../actions/clp-image-build/action.yaml" + - ".github/actions/clp-image-build/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" schedule: From 414e1fb01e567e893156cbf8f2bea5ffbfe5bb75 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 05:00:20 -0400 Subject: [PATCH 08/50] fix(build): add quotes around image tag in build command --- tools/docker-images/clp-package-ubuntu-jammy/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/build.sh b/tools/docker-images/clp-package-ubuntu-jammy/build.sh index d6583cbb9a..d26b5e8b04 100755 --- a/tools/docker-images/clp-package-ubuntu-jammy/build.sh +++ b/tools/docker-images/clp-package-ubuntu-jammy/build.sh @@ -22,7 +22,7 @@ fi build_cmd=( docker build - --tag clp-package-${arch_name}-ubuntu-jammy:dev + --tag "clp-package-${arch_name}-ubuntu-jammy:dev" "$repo_root" --file "${script_dir}/Dockerfile" ) From 0e9138a9f326049abed4664fb4d49d7a22c0bc5c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 6 Aug 2025 05:01:58 -0400 Subject: [PATCH 09/50] fix(docker): combine install and cleanup commands in Dockerfile --- tools/docker-images/clp-package-ubuntu-jammy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile index 529f16a9c9..dafbf23339 100644 --- a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile +++ b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile @@ -5,8 +5,8 @@ COPY ./build/clp-package /opt/clp WORKDIR /root COPY ./tools/docker-images/clp-package-ubuntu-jammy/setup-scripts ./setup-scripts -RUN ./setup-scripts/install-prebuilt-packages.sh -RUN rm -rf ./setup-scripts/ +RUN ./setup-scripts/install-prebuilt-packages.sh && \ + rm -rf ./setup-scripts/ # Remove cached files RUN apt-get clean \ From dcc8a70ef5a726a6072b63134c2f1caa722844c5 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Fri, 8 Aug 2025 18:47:44 -0400 Subject: [PATCH 10/50] add docs --- docs/src/dev-guide/building-package.md | 11 +++++++++++ docs/src/dev-guide/tooling-containers.md | 10 ++++++++++ docs/src/dev-guide/tooling-gh-workflows.md | 4 ++++ 3 files changed, 25 insertions(+) diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index 91134642a5..2506c182ee 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -69,5 +69,16 @@ To clean up all build artifacts, run: task clean ``` +## Building a Docker image + +To build a Docker image containing the CLP package, run: + +```shell +task docker-image-package +``` + +This will create a Docker image named `clp-package--ubuntu-jammy:dev` where `` is `x86` or +`arm64`. + [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [Task]: https://taskfile.dev/ diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index fcef664133..87d997f468 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -133,6 +133,16 @@ environment. tools/docker-images/clp-execution-base-ubuntu-jammy ``` +## clp-package-x86-ubuntu-jammy + +An image containing the CLP package built in an Ubuntu Jammy x86_64 environment. + +* Path: + + ```text + tools/docker-images/clp-package-ubuntu-jammy + ``` + [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 [core-deps-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-ubuntu-jammy [core-ubuntu-jammy]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-x86-ubuntu-jammy diff --git a/docs/src/dev-guide/tooling-gh-workflows.md b/docs/src/dev-guide/tooling-gh-workflows.md index bc38f75257..a99133da6e 100644 --- a/docs/src/dev-guide/tooling-gh-workflows.md +++ b/docs/src/dev-guide/tooling-gh-workflows.md @@ -76,6 +76,10 @@ This workflow builds CLP-core on macOS and runs its unit tests. This workflow builds a container image that contains the dependencies necessary to run the CLP package. +## clp-package-image-build + +This workflow builds a container image that contains the CLP package. + ## clp-lint This workflow runs linting checks on the codebase. From 9db7ca93f4c83c063e7fbd15e89885a7b67825f7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 19 Aug 2025 15:58:53 -0400 Subject: [PATCH 11/50] improve error handling in install-prebuilt-packages.sh --- tools/docker-images/clp-package-ubuntu-jammy/build.sh | 7 ++----- .../setup-scripts/install-prebuilt-packages.sh | 7 ++----- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/build.sh b/tools/docker-images/clp-package-ubuntu-jammy/build.sh index d26b5e8b04..4bcb66e595 100755 --- a/tools/docker-images/clp-package-ubuntu-jammy/build.sh +++ b/tools/docker-images/clp-package-ubuntu-jammy/build.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +set -eu +set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" repo_root=${script_dir}/../../../ diff --git a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh index 3e71d6b4d0..a1bd42828e 100755 --- a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh +++ b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh @@ -1,10 +1,7 @@ #!/usr/bin/env bash -# Exit on any error -set -e - -# Error on undefined variable -set -u +set -eu +set -o pipefail apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ From d609987829515130fb2a7dc4c21d2f6ba32e6380 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 19 Aug 2025 16:04:36 -0400 Subject: [PATCH 12/50] set default user and group to 1000 --- tools/docker-images/clp-package-ubuntu-jammy/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile index dafbf23339..4cbd74e9f9 100644 --- a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile +++ b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile @@ -20,3 +20,5 @@ ENV PATH="/opt/clp/sbin:${PATH}" ENV PATH="/opt/clp/bin:${PATH}" ENV PYTHONPATH="/opt/clp/lib/python3/site-packages" ENV CLP_HOME="/opt/clp" + +USER 1000:1000 From 1b3959d52f95a5cdd8dbb01ffe3ef8612967344d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 19 Aug 2025 16:07:52 -0400 Subject: [PATCH 13/50] update docker image name description --- docs/src/dev-guide/building-package.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index 2506c182ee..7aba4da54b 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -77,8 +77,8 @@ To build a Docker image containing the CLP package, run: task docker-image-package ``` -This will create a Docker image named `clp-package--ubuntu-jammy:dev` where `` is `x86` or -`arm64`. +This will create a Docker image named `clp-package--ubuntu-jammy:dev` where `` is `x86` +or `arm64`. [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [Task]: https://taskfile.dev/ From 1f139541631e7ef74fa711b5d26fcba4d1174e25 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 19 Aug 2025 16:10:21 -0400 Subject: [PATCH 14/50] update CLP package image dependencies --- .github/workflows/clp-package-image-build.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clp-package-image-build.yaml b/.github/workflows/clp-package-image-build.yaml index 65c327b61b..8eba120486 100644 --- a/.github/workflows/clp-package-image-build.yaml +++ b/.github/workflows/clp-package-image-build.yaml @@ -42,6 +42,7 @@ jobs: - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + - "tools/docker-images/clp-package-ubuntu-jammy/**" ubuntu-jammy-deps-image: name: "ubuntu-jammy-deps-image" From c49d8b405d368bcfb2e08a8c80a13da1c9e97e6c Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 10:07:47 -0400 Subject: [PATCH 15/50] remove unneeded prebuilt packages --- .../setup-scripts/install-prebuilt-packages.sh | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh index a1bd42828e..baa75fbf1e 100755 --- a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh +++ b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh @@ -6,11 +6,6 @@ set -o pipefail apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ - checkinstall \ - curl \ libcurl4 \ libmariadb-dev \ - libssl-dev \ - python3 \ - rsync \ - zstd + python3 From 73ca1a0957d84385c6edeec3a020dce89a503286 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 12:47:59 -0400 Subject: [PATCH 16/50] Update MariaDB client package from libmariadb-dev to libmariadb3 --- .../setup-scripts/install-prebuilt-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh index baa75fbf1e..36e1f38793 100755 --- a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh +++ b/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh @@ -7,5 +7,5 @@ apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ libcurl4 \ - libmariadb-dev \ + libmariadb3 \ python3 From a4317ea95f3cc4c9db1db2ea8ae07f044e324d67 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Sun, 24 Aug 2025 21:36:29 -0400 Subject: [PATCH 17/50] docs - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .github/actions/clp-image-build/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/clp-image-build/action.yaml b/.github/actions/clp-image-build/action.yaml index c7047cfefc..8f905e37f8 100644 --- a/.github/actions/clp-image-build/action.yaml +++ b/.github/actions/clp-image-build/action.yaml @@ -1,5 +1,5 @@ name: "clp-image-build" -description: "Builds a CLP container image." +description: "Builds a container image to be used for running CLP." inputs: image_type: From 1887626b482aa22da285c571c9631c5cb3b567bb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 17:58:41 -0400 Subject: [PATCH 18/50] refactor(actions): Rename clp-image-build to clp-build-runtime-image and update references --- .../{clp-image-build => clp-build-runtime-image}/action.yaml | 2 +- .github/workflows/clp-execution-image-build.yaml | 4 ++-- .github/workflows/clp-package-image-build.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) rename .github/actions/{clp-image-build => clp-build-runtime-image}/action.yaml (98%) diff --git a/.github/actions/clp-image-build/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml similarity index 98% rename from .github/actions/clp-image-build/action.yaml rename to .github/actions/clp-build-runtime-image/action.yaml index 8f905e37f8..cd2ec05eb0 100644 --- a/.github/actions/clp-image-build/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -1,4 +1,4 @@ -name: "clp-image-build" +name: "clp-build-runtime-image" description: "Builds a container image to be used for running CLP." inputs: diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index bf95f7174b..f400763844 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -3,12 +3,12 @@ name: "clp-execution-image-build" on: pull_request: paths: - - ".github/actions/clp-image-build/action.yaml" + - "../actions/clp-build-runtime-image/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" push: paths: - - ".github/actions/clp-image-build/action.yaml" + - "../actions/clp-build-runtime-image/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" schedule: diff --git a/.github/workflows/clp-package-image-build.yaml b/.github/workflows/clp-package-image-build.yaml index 8eba120486..10b295b4c0 100644 --- a/.github/workflows/clp-package-image-build.yaml +++ b/.github/workflows/clp-package-image-build.yaml @@ -94,7 +94,7 @@ jobs: run_command: >- CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package - - uses: "./.github/actions/clp-image-build" + - uses: "./.github/actions/clp-build-runtime-image" with: image_type: "package" image_registry: "ghcr.io" From 20dcfb12850f9a3043c9052ea0f115e895a3da86 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 17:59:18 -0400 Subject: [PATCH 19/50] Remove incompatible input options from action.yaml --- .github/actions/clp-build-runtime-image/action.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index cd2ec05eb0..a30d563568 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -5,10 +5,6 @@ inputs: image_type: description: "Type of image to build" required: true - type: "choice" - options: - - "execution" - - "package" image_registry: default: "ghcr.io" description: "Container image registry" From ddab8d138c5e8a184a0bef44642b29779bd3934e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 17:59:58 -0400 Subject: [PATCH 20/50] rename path -> dockerfile_path --- .github/actions/clp-build-runtime-image/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index a30d563568..5bb3fdb301 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -62,12 +62,12 @@ runs: platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" if [[ "${{inputs.image_type}}" == "execution" ]]; then - path="$base_path/clp-execution-base-$platform/Dockerfile" + dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile" else - path="$base_path/clp-package-$platform/Dockerfile" + dockerfile_path="$base_path/clp-package-$platform/Dockerfile" fi - echo "DOCKERFILE=$path" >> "$GITHUB_OUTPUT" + echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" - name: "Build and Push" if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" From 92861b9a157d52094c167a54a07a8c24a7b6df71 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 18:01:11 -0400 Subject: [PATCH 21/50] Add comment to clarify npm cache environment variable usage --- .github/actions/run-on-image/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/run-on-image/action.yaml b/.github/actions/run-on-image/action.yaml index 6cc28403ef..caf49204e3 100644 --- a/.github/actions/run-on-image/action.yaml +++ b/.github/actions/run-on-image/action.yaml @@ -42,6 +42,7 @@ runs: - run: "./tools/scripts/deps-download/init.sh" shell: "bash" + # `--env npm_config_cache`: Override to prevent permission issues with `/.npm`. - run: >- docker run --user $(id -u):$(id -g) From b39644e400b1d1078fb5925fd08f3fc9b8b3c116 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 18:01:47 -0400 Subject: [PATCH 22/50] refactor(install): Remove unnecessary packages and update dependencies as well in the execution image. --- .../setup-scripts/install-prebuilt-packages.sh | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh index 3e71d6b4d0..9ced27320f 100755 --- a/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh +++ b/tools/docker-images/clp-execution-base-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh @@ -9,11 +9,6 @@ set -u apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ - checkinstall \ - curl \ libcurl4 \ - libmariadb-dev \ - libssl-dev \ - python3 \ - rsync \ - zstd + libmariadb3 \ + python3 From c17a78254c7e44b7b1b83ff3d56c290ffb31d548 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 18:14:54 -0400 Subject: [PATCH 23/50] Rename container image name -> clp-package --- .github/actions/clp-build-runtime-image/action.yaml | 4 ++-- .github/workflows/clp-package-image-build.yaml | 2 +- docs/src/dev-guide/building-package.md | 3 +-- docs/src/dev-guide/tooling-containers.md | 6 +++--- taskfile.yaml | 2 +- .../Dockerfile | 2 +- .../build.sh | 13 +------------ .../setup-scripts/install-prebuilt-packages.sh | 0 8 files changed, 10 insertions(+), 22 deletions(-) rename tools/docker-images/{clp-package-ubuntu-jammy => clp-package}/Dockerfile (85%) rename tools/docker-images/{clp-package-ubuntu-jammy => clp-package}/build.sh (69%) rename tools/docker-images/{clp-package-ubuntu-jammy => clp-package}/setup-scripts/install-prebuilt-packages.sh (100%) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 5bb3fdb301..7e66575968 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -59,12 +59,12 @@ runs: shell: "bash" run: | base_path="./tools/docker-images" - platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" if [[ "${{inputs.image_type}}" == "execution" ]]; then + platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile" else - dockerfile_path="$base_path/clp-package-$platform/Dockerfile" + dockerfile_path="$base_path/clp-package/Dockerfile" fi echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/clp-package-image-build.yaml b/.github/workflows/clp-package-image-build.yaml index 10b295b4c0..cb17c752af 100644 --- a/.github/workflows/clp-package-image-build.yaml +++ b/.github/workflows/clp-package-image-build.yaml @@ -42,7 +42,7 @@ jobs: - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - - "tools/docker-images/clp-package-ubuntu-jammy/**" + - "tools/docker-images/clp-package/**" ubuntu-jammy-deps-image: name: "ubuntu-jammy-deps-image" diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index 7aba4da54b..b74e3e6089 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -77,8 +77,7 @@ To build a Docker image containing the CLP package, run: task docker-image-package ``` -This will create a Docker image named `clp-package--ubuntu-jammy:dev` where `` is `x86` -or `arm64`. +This will create a Docker image named `clp-package:dev`. [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [Task]: https://taskfile.dev/ diff --git a/docs/src/dev-guide/tooling-containers.md b/docs/src/dev-guide/tooling-containers.md index 87d997f468..c8c9186151 100644 --- a/docs/src/dev-guide/tooling-containers.md +++ b/docs/src/dev-guide/tooling-containers.md @@ -133,14 +133,14 @@ environment. tools/docker-images/clp-execution-base-ubuntu-jammy ``` -## clp-package-x86-ubuntu-jammy +## clp-package -An image containing the CLP package built in an Ubuntu Jammy x86_64 environment. +An image containing the CLP package. * Path: ```text - tools/docker-images/clp-package-ubuntu-jammy + tools/docker-images/clp-package ``` [core-deps-centos-stream-9]: https://github.com/y-scope/clp/pkgs/container/clp%2Fclp-core-dependencies-x86-centos-stream-9 diff --git a/taskfile.yaml b/taskfile.yaml index 98e3c84a2b..39a6c6928f 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -164,7 +164,7 @@ tasks: docker-image-package: vars: - SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package-ubuntu-jammy" + SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" dir: "{{.SRC_DIR}}" deps: - "package" diff --git a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-package/Dockerfile similarity index 85% rename from tools/docker-images/clp-package-ubuntu-jammy/Dockerfile rename to tools/docker-images/clp-package/Dockerfile index 4cbd74e9f9..1f2fb2c7d1 100644 --- a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -4,7 +4,7 @@ COPY ./build/clp-package /opt/clp WORKDIR /root -COPY ./tools/docker-images/clp-package-ubuntu-jammy/setup-scripts ./setup-scripts +COPY ./tools/docker-images/clp-package/setup-scripts ./setup-scripts RUN ./setup-scripts/install-prebuilt-packages.sh && \ rm -rf ./setup-scripts/ diff --git a/tools/docker-images/clp-package-ubuntu-jammy/build.sh b/tools/docker-images/clp-package/build.sh similarity index 69% rename from tools/docker-images/clp-package-ubuntu-jammy/build.sh rename to tools/docker-images/clp-package/build.sh index 4bcb66e595..39df1a1105 100755 --- a/tools/docker-images/clp-package-ubuntu-jammy/build.sh +++ b/tools/docker-images/clp-package/build.sh @@ -6,20 +6,9 @@ set -o pipefail script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" repo_root=${script_dir}/../../../ -arch=$(uname -m) - -if [ "$arch" = "x86_64" ]; then - arch_name="x86" -elif [ "$arch" = "aarch64" ]; then - arch_name="arm64" -else - echo "Error: Unsupported architecture - $arch" - exit 1 -fi - build_cmd=( docker build - --tag "clp-package-${arch_name}-ubuntu-jammy:dev" + --tag "clp-package:dev" "$repo_root" --file "${script_dir}/Dockerfile" ) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh b/tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh similarity index 100% rename from tools/docker-images/clp-package-ubuntu-jammy/setup-scripts/install-prebuilt-packages.sh rename to tools/docker-images/clp-package/setup-scripts/install-prebuilt-packages.sh From 8ff64f2fdc226e6fa800b2ba3c9ece7e66cbcd35 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 18:16:07 -0400 Subject: [PATCH 24/50] Move && to the next line - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/docker-images/clp-package-ubuntu-jammy/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile index 4cbd74e9f9..7d1896ecf7 100644 --- a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile +++ b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile @@ -5,8 +5,8 @@ COPY ./build/clp-package /opt/clp WORKDIR /root COPY ./tools/docker-images/clp-package-ubuntu-jammy/setup-scripts ./setup-scripts -RUN ./setup-scripts/install-prebuilt-packages.sh && \ - rm -rf ./setup-scripts/ +RUN ./setup-scripts/install-prebuilt-packages.sh \ + && rm -rf ./setup-scripts/ # Remove cached files RUN apt-get clean \ From 1dfe706f159258f8e798473132e1e0f1c310c9dc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 18:16:39 -0400 Subject: [PATCH 25/50] alphabetize ENV definitions in Dockerfile - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- tools/docker-images/clp-package-ubuntu-jammy/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile index 7d1896ecf7..f560eaf6c5 100644 --- a/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile +++ b/tools/docker-images/clp-package-ubuntu-jammy/Dockerfile @@ -16,9 +16,9 @@ RUN apt-get clean \ FROM scratch COPY --from=base / / -ENV PATH="/opt/clp/sbin:${PATH}" -ENV PATH="/opt/clp/bin:${PATH}" -ENV PYTHONPATH="/opt/clp/lib/python3/site-packages" ENV CLP_HOME="/opt/clp" +ENV PATH="${CLP_HOME}/sbin:${PATH}" +ENV PATH="${CLP_HOME}/bin:${PATH}" +ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" USER 1000:1000 From 628ddae46a7dc7c374b5de798dd7262da9be5e01 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:05:50 -0400 Subject: [PATCH 26/50] move job 'ubuntu-jammy-package-image' to clp-core-build.yaml and rename as 'package-image' --- .github/workflows/clp-core-build.yaml | 65 +++++++---- .../workflows/clp-package-image-build.yaml | 104 ------------------ 2 files changed, 46 insertions(+), 123 deletions(-) delete mode 100644 .github/workflows/clp-package-image-build.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 6a04c11798..b39b9fe71a 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -2,27 +2,14 @@ name: "clp-core-build" on: pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" + paths: &monitored_paths + - "**" + - "!.github/ISSUE_TEMPLATE/**" + - "!.github/*" - "!components/core/tools/scripts/lib_install/macos/**" + - "!docs/**" push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" + paths: *monitored_paths schedule: # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - cron: "15 0 * * *" @@ -374,3 +361,43 @@ jobs: .task/checksum/utils-cpp-lint-clang-tidy-* build/lint-clang-tidy key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" + + package-image: + name: "package-image" + if: "!cancelled() && !failure() && ( + needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || + needs.ubuntu-jammy-deps-image.result == 'success' + )" + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-24.04" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - name: "Build the package" + uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package + + - uses: "./.github/actions/clp-build-runtime-image" + with: + image_type: "package" + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-package-image-build.yaml b/.github/workflows/clp-package-image-build.yaml deleted file mode 100644 index cb17c752af..0000000000 --- a/.github/workflows/clp-package-image-build.yaml +++ /dev/null @@ -1,104 +0,0 @@ -name: "clp-package-image-build" - -on: - push: - workflow_dispatch: - -env: - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-24.04" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - - "tools/docker-images/clp-package/**" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - needs: "filter-relevant-changes" - runs-on: "ubuntu-24.04" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-package-image: - name: "ubuntu-jammy-package-image" - needs: "ubuntu-jammy-deps-image" - runs-on: "ubuntu-24.04" - env: - OS_NAME: "ubuntu-jammy" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - name: "Build the package" - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task package - - - uses: "./.github/actions/clp-build-runtime-image" - with: - image_type: "package" - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" From 23f1ef23eb352992f3b794e6cefd9322377152d3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:15:23 -0400 Subject: [PATCH 27/50] fix clp-build-runtime-image action to account for package image name change --- .../clp-build-runtime-image/action.yaml | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 7e66575968..41678eb458 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -47,15 +47,8 @@ runs: echo "REPOSITORY=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]')" \ >> "$GITHUB_OUTPUT" - - name: "Update Metadata" - id: "meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ - /clp-${{inputs.image_type}}-x86-${{inputs.platform_id}}-${{inputs.platform_version_id}}" - - - name: "Determine Dockerfile Path" - id: "dockerfile" + - name: "Compute metadata" + id: "compute-meta" shell: "bash" run: | base_path="./tools/docker-images" @@ -63,18 +56,28 @@ runs: if [[ "${{inputs.image_type}}" == "execution" ]]; then platform="${{inputs.platform_id}}-${{inputs.platform_version_id}}" dockerfile_path="$base_path/clp-execution-base-$platform/Dockerfile" + image_name="clp-${{inputs.image_type}}-$platform" else dockerfile_path="$base_path/clp-package/Dockerfile" + image_name="clp-${{inputs.image_type}}" fi echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" + echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT" + + - name: "Update Metadata" + id: "update-meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ + /${{steps.compute-meta.outputs.IMAGE_NAME}}" - name: "Build and Push" if: "github.event_name != 'pull_request' && github.ref == 'refs/heads/main'" uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" with: context: "./" - file: "${{steps.dockerfile.outputs.DOCKERFILE}}" + file: "${{steps.compute-meta.outputs.DOCKERFILE}}" push: true - tags: "${{steps.meta.outputs.tags}}" - labels: "${{steps.meta.outputs.labels}}" + tags: "${{steps.update-meta.outputs.tags}}" + labels: "${{steps.update-meta.outputs.labels}}" From 63a7ec31f35e8e7ebce0daee29f2e8ff47cb616d Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:22:32 -0400 Subject: [PATCH 28/50] mark clp-build-runtime-image's platform_id and platform_version_id as optional; remove them from the package image invocation --- .github/actions/clp-build-runtime-image/action.yaml | 4 ++-- .github/workflows/clp-core-build.yaml | 2 -- taskfiles/docker-images.yaml | 13 +++++++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 taskfiles/docker-images.yaml diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 41678eb458..b1b15b7132 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -19,11 +19,11 @@ inputs: required: false platform_id: description: "Platform ID of the container (e.g. ubuntu)" - required: true + required: false platform_version_id: description: "Platform VERSION_ID / VERSION_CODENAME of the container (e.g. jammy, focal, etc.)" - required: true + required: false runs: using: "composite" diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index b39b9fe71a..326fbc14fb 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -399,5 +399,3 @@ jobs: image_registry: "ghcr.io" image_registry_username: "${{github.actor}}" image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml new file mode 100644 index 0000000000..34c93717df --- /dev/null +++ b/taskfiles/docker-images.yaml @@ -0,0 +1,13 @@ +version: "3" + +tasks: + package: + vars: + SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" + dir: "{{.SRC_DIR}}" + deps: + - ":package" + sources: + - "{{.SRC_DIR}}/**/*" + cmds: + - "./build.sh" From 61da837cea4e4ef5b7f19859c746de21fb08e374 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:24:52 -0400 Subject: [PATCH 29/50] move package image task into separate task file --- taskfile.yaml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/taskfile.yaml b/taskfile.yaml index 39a6c6928f..cbf0a80f76 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -5,6 +5,7 @@ shopt: ["globstar"] includes: deps: "taskfiles/deps/main.yaml" + docker-images: "taskfiles/docker-images.yaml" docs: "taskfiles/docs.yaml" lint: "taskfiles/lint.yaml" utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" @@ -162,17 +163,6 @@ tasks: CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] - docker-image-package: - vars: - SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" - dir: "{{.SRC_DIR}}" - deps: - - "package" - sources: - - "{{.SRC_DIR}}/**/*" - cmds: - - "./build.sh" - clp-s-generate-parsers: vars: CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" From 227d6dd1f2f1053bc3c7a0cfb83e4e45a22a7fe0 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:26:32 -0400 Subject: [PATCH 30/50] CRLF -> LF --- taskfiles/docker-images.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 34c93717df..4ae6844699 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -1,13 +1,13 @@ -version: "3" - -tasks: - package: - vars: - SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" - dir: "{{.SRC_DIR}}" - deps: - - ":package" - sources: - - "{{.SRC_DIR}}/**/*" - cmds: - - "./build.sh" +version: "3" + +tasks: + package: + vars: + SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" + dir: "{{.SRC_DIR}}" + deps: + - ":package" + sources: + - "{{.SRC_DIR}}/**/*" + cmds: + - "./build.sh" From e98bb8e444436cc6f24f5ba9e0d84c05a16331cb Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:33:40 -0400 Subject: [PATCH 31/50] use `paths-ignore` instead of inverted glob patterns in `paths` --- .github/workflows/clp-core-build.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 326fbc14fb..b7c7550a28 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -2,14 +2,13 @@ name: "clp-core-build" on: pull_request: - paths: &monitored_paths - - "**" - - "!.github/ISSUE_TEMPLATE/**" - - "!.github/*" - - "!components/core/tools/scripts/lib_install/macos/**" - - "!docs/**" + paths-ignore: &ignored_paths + - ".github/ISSUE_TEMPLATE/**" + - ".github/*" + - "components/core/tools/scripts/lib_install/macos/**" + - "docs/**" push: - paths: *monitored_paths + paths-ignore: *ignored_paths schedule: # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - cron: "15 0 * * *" From e6f1af592fefeba731ffaf53da909479b77c970e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:34:33 -0400 Subject: [PATCH 32/50] alphabetize --- .github/workflows/clp-core-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index b7c7550a28..c3fecdf73b 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -3,8 +3,8 @@ name: "clp-core-build" on: pull_request: paths-ignore: &ignored_paths - - ".github/ISSUE_TEMPLATE/**" - ".github/*" + - ".github/ISSUE_TEMPLATE/**" - "components/core/tools/scripts/lib_install/macos/**" - "docs/**" push: From 860e5efd6988662e072a5d24dbc55975ae596e72 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:37:18 -0400 Subject: [PATCH 33/50] use absolute path for `.github/actions` --- .github/workflows/clp-execution-image-build.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index f400763844..bf57dbf01a 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -3,12 +3,12 @@ name: "clp-execution-image-build" on: pull_request: paths: - - "../actions/clp-build-runtime-image/action.yaml" + - ".github/actions/clp-build-runtime-image/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" push: paths: - - "../actions/clp-build-runtime-image/action.yaml" + - ".github/actions/clp-build-runtime-image/action.yaml" - ".github/workflows/clp-execution-image-build.yaml" - "tools/docker-images/**/*" schedule: From 41807399a3f783c11d8bdfd91ca179948d4fb6c9 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:38:24 -0400 Subject: [PATCH 34/50] fix outdated action name --- .github/workflows/clp-execution-image-build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml index bf57dbf01a..9586bbbeb5 100644 --- a/.github/workflows/clp-execution-image-build.yaml +++ b/.github/workflows/clp-execution-image-build.yaml @@ -61,7 +61,7 @@ jobs: shell: "bash" run: "chown $(id -u):$(id -g) -R ." - - uses: "./.github/actions/clp-image-build" + - uses: "./.github/actions/clp-build-runtime-image" with: image_type: "execution" image_registry: "ghcr.io" From 6cfabd7c2ba973f2e4b5b92cb1bb897cccdc3bd1 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:39:26 -0400 Subject: [PATCH 35/50] fix outdated task command ` docker-images:package` --- docs/src/dev-guide/building-package.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/dev-guide/building-package.md b/docs/src/dev-guide/building-package.md index b74e3e6089..e2d20f48a6 100644 --- a/docs/src/dev-guide/building-package.md +++ b/docs/src/dev-guide/building-package.md @@ -74,7 +74,7 @@ task clean To build a Docker image containing the CLP package, run: ```shell -task docker-image-package +task docker-images:package ``` This will create a Docker image named `clp-package:dev`. From f1c566f7c86684f9f6eb29a5770fe2b7224f2a63 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:41:46 -0400 Subject: [PATCH 36/50] order task attrs according to dev guide --- taskfiles/docker-images.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 4ae6844699..3cf7bd970b 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -5,9 +5,9 @@ tasks: vars: SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" dir: "{{.SRC_DIR}}" - deps: - - ":package" sources: - "{{.SRC_DIR}}/**/*" + deps: + - ":package" cmds: - "./build.sh" From 09539ae97956c27f54595ae00843b71f0943ebf3 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Mon, 25 Aug 2025 19:43:24 -0400 Subject: [PATCH 37/50] fix order of ENV defs in Dockerfile --- tools/docker-images/clp-package/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 4b0e61258a..7a82eca905 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -17,8 +17,8 @@ FROM scratch COPY --from=base / / ENV CLP_HOME="/opt/clp" -ENV PATH="${CLP_HOME}/sbin:${PATH}" ENV PATH="${CLP_HOME}/bin:${PATH}" +ENV PATH="${CLP_HOME}/sbin:${PATH}" ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" USER 1000:1000 From b49daba3bb6adc3b613f7a3a0bdb7f0bfdb41a34 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 09:59:52 -0400 Subject: [PATCH 38/50] test ci: to be reverted --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 39d9be5a4f..712e310ef8 100644 --- a/README.md +++ b/README.md @@ -118,4 +118,4 @@ If you would like a feature or want to report a bug, please file an issue and we [re2]: https://github.com/google/re2 [uber-blog]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp [yscope-community-discord]: https://discord.gg/7kZA2m5G87 -[yscope-community-zulip]: https://yscope-clp.zulipchat.com +[yscope-community-zulip]: https://yscope-clp.zulipchat.com From be2560eafecfcdff7f5fba2bbd288215943502f7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 10:06:12 -0400 Subject: [PATCH 39/50] revert the last change --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 712e310ef8..39d9be5a4f 100644 --- a/README.md +++ b/README.md @@ -118,4 +118,4 @@ If you would like a feature or want to report a bug, please file an issue and we [re2]: https://github.com/google/re2 [uber-blog]: https://www.uber.com/en-US/blog/reducing-logging-cost-by-two-orders-of-magnitude-using-clp [yscope-community-discord]: https://discord.gg/7kZA2m5G87 -[yscope-community-zulip]: https://yscope-clp.zulipchat.com +[yscope-community-zulip]: https://yscope-clp.zulipchat.com From 44a2dc50ab751e294ac84119de3fd446442ac185 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 10:08:26 -0400 Subject: [PATCH 40/50] Rename step name "Update Metadata" -> "Extract GitHub Metadata" - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .github/actions/clp-build-runtime-image/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index b1b15b7132..56731da6fe 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -65,7 +65,7 @@ runs: echo "DOCKERFILE=$dockerfile_path" >> "$GITHUB_OUTPUT" echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT" - - name: "Update Metadata" + - name: "Extract GitHub Metadata" id: "update-meta" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: From 2ebbfe0883334b6986e1a97c5b81d6a6a80b8f7e Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 10:10:45 -0400 Subject: [PATCH 41/50] rename action step ID for metadata extraction --- .github/actions/clp-build-runtime-image/action.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/clp-build-runtime-image/action.yaml b/.github/actions/clp-build-runtime-image/action.yaml index 56731da6fe..ecf3153994 100644 --- a/.github/actions/clp-build-runtime-image/action.yaml +++ b/.github/actions/clp-build-runtime-image/action.yaml @@ -66,7 +66,7 @@ runs: echo "IMAGE_NAME=$image_name" >> "$GITHUB_OUTPUT" - name: "Extract GitHub Metadata" - id: "update-meta" + id: "extract-gh-meta" uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" with: images: "${{inputs.image_registry}}/${{steps.sanitization.outputs.REPOSITORY}}\ @@ -79,5 +79,5 @@ runs: context: "./" file: "${{steps.compute-meta.outputs.DOCKERFILE}}" push: true - tags: "${{steps.update-meta.outputs.tags}}" - labels: "${{steps.update-meta.outputs.labels}}" + tags: "${{steps.extract-gh-meta.outputs.tags}}" + labels: "${{steps.extract-gh-meta.outputs.labels}}" From 12548279cb39cf1f82468f6a46523024c4abb4c8 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 10:11:44 -0400 Subject: [PATCH 42/50] update npm cache dir comment - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .github/actions/run-on-image/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/run-on-image/action.yaml b/.github/actions/run-on-image/action.yaml index caf49204e3..4e3203d26b 100644 --- a/.github/actions/run-on-image/action.yaml +++ b/.github/actions/run-on-image/action.yaml @@ -42,7 +42,7 @@ runs: - run: "./tools/scripts/deps-download/init.sh" shell: "bash" - # `--env npm_config_cache`: Override to prevent permission issues with `/.npm`. + # `--env npm_config_cache` overrides the default `/.npm` to avoid permission issues. - run: >- docker run --user $(id -u):$(id -g) From 62cf06ad98d989e5d514ebde3a18cf7b8df784b7 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 10:12:22 -0400 Subject: [PATCH 43/50] use yaml multiline string syntax for long `if` - Apply suggestions from code review Co-authored-by: kirkrodrigues <2454684+kirkrodrigues@users.noreply.github.com> --- .github/workflows/clp-core-build.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index c3fecdf73b..7776dc7706 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -363,10 +363,11 @@ jobs: package-image: name: "package-image" - if: "!cancelled() && !failure() && ( + if: >- + !cancelled() && !failure() && ( needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' || needs.ubuntu-jammy-deps-image.result == 'success' - )" + ) needs: - "filter-relevant-changes" - "ubuntu-jammy-deps-image" From c413a133b356f50f346db3b935ea1e53fd05c1de Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 11:01:19 -0400 Subject: [PATCH 44/50] add checksum file for package task --- taskfile.yaml | 3 ++- taskfiles/docker-images.yaml | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/taskfile.yaml b/taskfile.yaml index cbf0a80f76..0029a14fcb 100644 --- a/taskfile.yaml +++ b/taskfile.yaml @@ -38,6 +38,7 @@ vars: {{default "" (env "CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK")}} # Checksum files + G_PACKAGE_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/package.md5" G_WEBUI_CLIENT_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-client-node-modules.md5" G_WEBUI_LOG_VIEWER_NODE_MODULES_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/webui-log-viewer-node-modules.md5" @@ -93,7 +94,7 @@ tasks: env: NODE_ENV: "production" vars: - CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5" + CHECKSUM_FILE: "{{.G_PACKAGE_CHECKSUM_FILE}}" OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}" sources: - "{{.G_BUILD_DIR}}/package-venv.md5" diff --git a/taskfiles/docker-images.yaml b/taskfiles/docker-images.yaml index 3cf7bd970b..9a546f713c 100644 --- a/taskfiles/docker-images.yaml +++ b/taskfiles/docker-images.yaml @@ -6,6 +6,7 @@ tasks: SRC_DIR: "{{.ROOT_DIR}}/tools/docker-images/clp-package" dir: "{{.SRC_DIR}}" sources: + - "{{.G_PACKAGE_CHECKSUM_FILE}}" - "{{.SRC_DIR}}/**/*" deps: - ":package" From bb36035d05912226a01c02e5b68e7558347f4579 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 11:08:03 -0400 Subject: [PATCH 45/50] reorder COPY instruction in clp-package Dockerfile --- tools/docker-images/clp-package/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 7a82eca905..6ad1367d7d 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -1,7 +1,5 @@ FROM ubuntu:jammy AS base -COPY ./build/clp-package /opt/clp - WORKDIR /root COPY ./tools/docker-images/clp-package/setup-scripts ./setup-scripts @@ -12,6 +10,8 @@ RUN ./setup-scripts/install-prebuilt-packages.sh \ RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +COPY ./build/clp-package /opt/clp + # Flatten the image FROM scratch COPY --from=base / / From ea4ce512ce3edb56ddf044ca7f46d6460d641a2b Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 11:08:26 -0400 Subject: [PATCH 46/50] flatten the image at the last --- tools/docker-images/clp-package/Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 6ad1367d7d..91572082d6 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -12,13 +12,13 @@ RUN apt-get clean \ COPY ./build/clp-package /opt/clp -# Flatten the image -FROM scratch -COPY --from=base / / - ENV CLP_HOME="/opt/clp" ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" USER 1000:1000 + +# Flatten the image +FROM scratch +COPY --from=base / / From 3c8101461691a54d3c520ab1231ffbb5acba17a2 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Tue, 2 Sep 2025 11:10:31 -0400 Subject: [PATCH 47/50] move COPY to later --- tools/docker-images/clp-package/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/docker-images/clp-package/Dockerfile b/tools/docker-images/clp-package/Dockerfile index 91572082d6..9d1bc5c71e 100644 --- a/tools/docker-images/clp-package/Dockerfile +++ b/tools/docker-images/clp-package/Dockerfile @@ -10,8 +10,6 @@ RUN ./setup-scripts/install-prebuilt-packages.sh \ RUN apt-get clean \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -COPY ./build/clp-package /opt/clp - ENV CLP_HOME="/opt/clp" ENV PATH="${CLP_HOME}/bin:${PATH}" ENV PATH="${CLP_HOME}/sbin:${PATH}" @@ -19,6 +17,8 @@ ENV PYTHONPATH="${CLP_HOME}/lib/python3/site-packages" USER 1000:1000 +COPY ./build/clp-package /opt/clp + # Flatten the image FROM scratch COPY --from=base / / From cbea39bcfaf73180425b8e7e90a85071f82ca091 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 15:18:06 -0400 Subject: [PATCH 48/50] docs: rename `clp-core-build` workflow and references -> `clp-artifact-build` for clarity; remove section `clp-package-image-build` --- .github/workflows/clp-core-build.yaml | 12 ++++++------ docs/src/dev-docs/contributing-linting.md | 4 ++-- docs/src/dev-docs/tooling-gh-workflows.md | 6 +----- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml index 1a37f2db0c..fac645c8fa 100644 --- a/.github/workflows/clp-core-build.yaml +++ b/.github/workflows/clp-core-build.yaml @@ -1,4 +1,4 @@ -name: "clp-core-build" +name: "clp-artifact-build" on: pull_request: @@ -60,31 +60,31 @@ jobs: filters: | centos_stream_9_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - "components/core/tools/scripts/lib_install/centos-stream-9/**" manylinux_2_28_x86_64_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-manylinux_2_28-x86_64/**" - "components/core/tools/scripts/lib_install/manylinux_2_28/**" musllinux_1_2_x86_64_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-musllinux_1_2-x86_64/**" - "components/core/tools/scripts/lib_install/musllinux_1_2/**" ubuntu_jammy_image: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - "components/core/tools/scripts/lib_install/*.sh" - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" clp: - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" + - ".github/workflows/clp-artifact-build.yaml" - ".gitmodules" - "components/core/cmake/**" - "components/core/CMakeLists.txt" diff --git a/docs/src/dev-docs/contributing-linting.md b/docs/src/dev-docs/contributing-linting.md index 3492e33f93..0a646bc681 100644 --- a/docs/src/dev-docs/contributing-linting.md +++ b/docs/src/dev-docs/contributing-linting.md @@ -2,7 +2,7 @@ Before submitting a PR, ensure you've run our linting tools and either fixed any violations or suppressed the warning. If you can't run the linting workflows locally, you can enable and run the -[clp-lint] and [clp-core-build] workflows in your fork. +[clp-lint] and [clp-artifact-build] workflows in your fork. ## Requirements @@ -32,7 +32,7 @@ To also apply any automatic fixes: task lint:fix ``` -[clp-core-build]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-core-build.yaml +[clp-artifact-build]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-artifact-build.yaml [clp-lint]: https://github.com/y-scope/clp/blob/main/.github/workflows/clp-lint.yaml [clp-issue-872]: https://github.com/y-scope/clp/issues/872 [feature-req]: https://github.com/y-scope/clp/issues/new?assignees=&labels=enhancement&projects=&template=feature-request.yml diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 698d9f9cb6..71df6a53bd 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -3,7 +3,7 @@ The CLP repo includes several GitHub workflows for automating container image builds, artifact builds, testing, and linting. We briefly describe each workflow below. -## clp-core-build +## clp-artifact-build This workflow is responsible for: @@ -90,10 +90,6 @@ This workflow builds CLP-core on macOS and runs its unit tests. This workflow builds a container image that contains the dependencies necessary to run the CLP package. -## clp-package-image-build - -This workflow builds a container image that contains the CLP package. - ## clp-lint This workflow runs linting checks on the codebase. From a8577015f45dbe71ad1ad6314213f3e25dd17218 Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 15:33:29 -0400 Subject: [PATCH 49/50] docs: update workflows to include CLP package image build --- docs/src/dev-docs/tooling-gh-workflows.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/src/dev-docs/tooling-gh-workflows.md b/docs/src/dev-docs/tooling-gh-workflows.md index 71df6a53bd..418a3f5648 100644 --- a/docs/src/dev-docs/tooling-gh-workflows.md +++ b/docs/src/dev-docs/tooling-gh-workflows.md @@ -7,8 +7,9 @@ builds, testing, and linting. We briefly describe each workflow below. This workflow is responsible for: -1. building (Linux) container images containing CLP-core's dependencies, and -2. building CLP-core and running its unit tests. +1. building (Linux) container images containing CLP-core's dependencies, +2. building CLP-core and running its unit tests, and +3. building a container image containing CLP's package components. To minimize build times, the jobs in the workflow are organized in the directed acyclic graph (DAG) shown below. @@ -40,6 +41,7 @@ flowchart LR manylinux_2_28-x86_64-deps-image --> manylinux_2_28-x86_64-binaries musllinux_1_2-x86_64-deps-image --> musllinux_1_2-x86_64-binaries ubuntu-jammy-deps-image --> ubuntu-jammy-binaries + ubuntu-jammy-deps-image --> package-image ubuntu-jammy-binaries --> ubuntu-jammy-binaries-image ::: @@ -61,6 +63,7 @@ Arrows between jobs indicate a dependency. The jobs are as follows: container and runs core's unit tests. * `musllinux_1_2-x86_64-binaries`: Builds the CLP-core binaries in the built musllinux_1_2 container and runs core's unit tests. +* `package-image`: Builds a container image containing CLP's package components. * `ubuntu-jammy-binaries`: Builds the CLP-core binaries in the built Ubuntu Jammy container and runs core's unit tests. * `ubuntu-jammy-binaries-image`: Builds an Ubuntu Jammy container image containing CLP-core's From 7dfbcc5fc58e0c7c2f90d332e0a595901f530bfc Mon Sep 17 00:00:00 2001 From: Junhao Liao Date: Wed, 3 Sep 2025 16:59:44 -0400 Subject: [PATCH 50/50] ci: rename workflow `clp-core-build` -> `clp-artifact-build` --- .../workflows/{clp-core-build.yaml => clp-artifact-build.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{clp-core-build.yaml => clp-artifact-build.yaml} (100%) diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-artifact-build.yaml similarity index 100% rename from .github/workflows/clp-core-build.yaml rename to .github/workflows/clp-artifact-build.yaml