From 1cc7e762765a3422e537ec4661103a5d0245ed8b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 13:33:17 -0400 Subject: [PATCH 01/42] Install ystdlib as package --- components/core/CMakeLists.txt | 6 ++++-- taskfiles/deps/main.yaml | 23 +++++++++-------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 56726b22cb..27889472ea 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -235,8 +235,10 @@ if(yaml-cpp_FOUND) endif() # Add ystdlib -set(YSTDLIB_CPP_BUILD_TESTING OFF) -add_subdirectory("${CLP_YSTDLIB_SOURCE_DIRECTORY}" "${CMAKE_BINARY_DIR}/ystdlib" EXCLUDE_FROM_ALL) +find_package(ystdlib REQUIRED) +if (ystdlib_FOUND) + message(STATUS "Found ystdlib ${ystdlib_VERSION}") +endif() # Find and setup ZStd Library if(CLP_USE_STATIC_LIBS) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 1bdcdcfcab..843f68d6be 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -373,20 +373,15 @@ tasks: ystdlib: internal: true - vars: - LIB_NAME: "ystdlib" - YSTDLIB_OUTPUT_DIR: "{{.G_DEPS_CORE_DIR}}/{{.LIB_NAME}}-src" run: "once" - deps: - - task: "utils:init" cmds: - - task: "yscope-dev-utils:remote:download-and-extract-tar" + - task: "utils:install-remote-cmake-lib" vars: - CHECKSUM_FILE: "{{.G_DEPS_CORE_CHECKSUMS_DIR}}/{{.LIB_NAME}}.md5" - FILE_SHA256: "36fa0e9d96b7307ca92482343d6ba1091c5576370676e6d423cce32c20e34a3d" - OUTPUT_DIR: "{{.YSTDLIB_OUTPUT_DIR}}" - URL: "https://github.com/y-scope/ystdlib-cpp/archive/d80cf86.tar.gz" - - >- - echo "set( - CLP_YSTDLIB_SOURCE_DIRECTORY \"{{.YSTDLIB_OUTPUT_DIR}}\" - )" > "{{.G_DEPS_CORE_CMAKE_SETTINGS_DIR}}/{{.LIB_NAME}}.cmake" + CMAKE_GEN_ARGS: + - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + - "-Dystdlib_BUILD_TESTING=OFF" + - "-DCMAKE_POLICY_DEFAULT_CMP0144=NEW" + LIB_NAME: "ystdlib" + WORK_DIR: "{{.G_DEPS_DIR}}/ystdlib" + TARBALL_SHA256: "4c027c884506e1775070192f9d1d58238a6d5b078608211fa442477393676738" + TARBALL_URL: "https://github.com/y-scope/ystdlib-cpp/archive/0ae886c.tar.gz" From 8e0e4d5ab11bbee97688e23516c019932d6bc585 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 13:34:28 -0400 Subject: [PATCH 02/42] Bump cmake min version to 3.23 --- components/core/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 27889472ea..1e4cac0e01 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -1,4 +1,4 @@ -cmake_minimum_required(VERSION 3.16.3) +cmake_minimum_required(VERSION 3.23) # Toolchain setup must come before the first project() call in the entire CMake buildsystem. # If CLP is not the top-level project, the following setup has no effect. From aba106f0c3ef2a27846d0693715759bd251eb9a8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 13:37:52 -0400 Subject: [PATCH 03/42] Install cmake 3.23.5 for ubuntu jammy --- .../lib_install/ubuntu-jammy/install-packages-from-source.sh | 4 +++- .../lib_install/ubuntu-jammy/install-prebuilt-packages.sh | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index 130374062e..734c2015ba 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -9,7 +9,9 @@ set -u script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir=$script_dir/.. -# NOTE: boost must be installed first since the remaining packages depend on it +# NOTE: cmake must be installed first since the remaining packages depend on it to build +"$lib_install_scripts_dir"/install-cmake.sh 3.23.5 +# NOTE: boost must be installed second since the remaining packages depend on it "$lib_install_scripts_dir"/install-boost.sh 1.87.0 "$lib_install_scripts_dir"/libarchive.sh 3.5.1 diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh index 03a3c26e8c..b19e4c3c3e 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-prebuilt-packages.sh @@ -10,7 +10,6 @@ apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ ca-certificates \ checkinstall \ - cmake \ curl \ build-essential \ git \ From 71e96ac28f164ad351102a00f0f65bd2edb90efc Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 14:37:28 -0400 Subject: [PATCH 04/42] Remove unnecessary cmake variable --- components/core/CMakeLists.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 845fb48115..713242daca 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -275,7 +275,6 @@ if(CLP_NEED_YAMLCPP) endif() # Add ystdlib -set(ystdlib_BUILD_TESTING OFF) find_package(ystdlib REQUIRED) if (ystdlib_FOUND) message(STATUS "Found ystdlib ${ystdlib_VERSION}") From 430f91cd960f7551867dc8c174da93493914b22d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 16:37:45 -0400 Subject: [PATCH 05/42] Use homebrew llvm in macos GH runner --- .github/workflows/clp-core-build-macos.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..dcb326a850 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,6 +71,16 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" + - name: "Set environment to use homebrew llvm" + run: | + LLVM_PREFIX=$(brew --prefix llvm) + echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV + echo "PATH=$LLVM_PREFIX/bin:$PATH" >> $GITHUB_ENV + echo "CC=clang" >> $GITHUB_ENV + echo "CXX=clang++" >> $GITHUB_ENV + echo "LDFLAGS=-L$LLVM_PREFIX/lib" >> $GITHUB_ENV + echo "CPPFLAGS=-I$LLVM_PREFIX/include" >> $GITHUB_ENV + - run: "./tools/scripts/deps-download/init.sh" shell: "bash" From 2f1b08d822a8600a86f133b9fa425ad27466bb55 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 16:48:15 -0400 Subject: [PATCH 06/42] Revert "Use homebrew llvm in macos GH runner" This reverts commit 430f91cd960f7551867dc8c174da93493914b22d. --- .github/workflows/clp-core-build-macos.yaml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index dcb326a850..51eb6985d0 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,16 +71,6 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" - - name: "Set environment to use homebrew llvm" - run: | - LLVM_PREFIX=$(brew --prefix llvm) - echo "LLVM_PREFIX=$LLVM_PREFIX" >> $GITHUB_ENV - echo "PATH=$LLVM_PREFIX/bin:$PATH" >> $GITHUB_ENV - echo "CC=clang" >> $GITHUB_ENV - echo "CXX=clang++" >> $GITHUB_ENV - echo "LDFLAGS=-L$LLVM_PREFIX/lib" >> $GITHUB_ENV - echo "CPPFLAGS=-I$LLVM_PREFIX/include" >> $GITHUB_ENV - - run: "./tools/scripts/deps-download/init.sh" shell: "bash" From f324686f083c890770d81e523e858cc2ddcc4849 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 17:37:19 -0400 Subject: [PATCH 07/42] Temporarily remove other workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From a51b899e3c0b0f4d253c967959610acbf4d4bd7b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 17:37:37 -0400 Subject: [PATCH 08/42] Add setting of environment variables --- .github/workflows/clp-core-build-macos.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..79ba380a0c 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,6 +71,18 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" + - name: "Set environment to use Homebrew's llvm" + run: | + LLVM_PREFIX=$(brew --prefix llvm) + echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" + { + echo "LLVM_PREFIX=$LLVM_PREFIX" + echo "Cc=clang" + echo "CXX=clang++" + echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "CPPFLAGS=-I$LLVM_PREFIX/include" + } >> "$GITHUB_ENV" + - run: "./tools/scripts/deps-download/init.sh" shell: "bash" From 04cc82ac9bacde7501ce0f555d62b3d1477e7b95 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 17:52:03 -0400 Subject: [PATCH 09/42] Revert "Temporarily remove other workflows" This reverts commit f324686f083c890770d81e523e858cc2ddcc4849. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From 0e2949958788a9f8a515f7f5c9a40bbef0ea1636 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 18:15:23 -0400 Subject: [PATCH 10/42] Bug fix --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 79ba380a0c..4b644b6144 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -77,7 +77,7 @@ jobs: echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" { echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "Cc=clang" + echo "CC=clang" echo "CXX=clang++" echo "LDFLAGS=-L$LLVM_PREFIX/lib" echo "CPPFLAGS=-I$LLVM_PREFIX/include" From cfad209be24e966a2883bc34eb32d2df6e80389e Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 18:41:29 -0400 Subject: [PATCH 11/42] Temporarily remove other GH workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From 753e55588db3b687f5672543b949e713d249cc03 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 18:43:22 -0400 Subject: [PATCH 12/42] Add check on PATH variable --- taskfiles/deps/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 16f999df94..d0169e2680 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -74,6 +74,8 @@ tasks: - task: "utfcpp" - task: "yaml-cpp" - task: "ystdlib" + cmds: + - echo "PATH:$PATH" absl: internal: true From 15f3097324c444e9f6f98e61e3fd54c5e81ccffd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:12:50 -0400 Subject: [PATCH 13/42] Revert "Add check on PATH variable" This reverts commit 753e55588db3b687f5672543b949e713d249cc03. --- taskfiles/deps/main.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index d0169e2680..16f999df94 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -74,8 +74,6 @@ tasks: - task: "utfcpp" - task: "yaml-cpp" - task: "ystdlib" - cmds: - - echo "PATH:$PATH" absl: internal: true From 55516f561460a7598f9142c22a265c21a9c61e26 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:20:22 -0400 Subject: [PATCH 14/42] Use llvm --- .github/workflows/clp-core-build-macos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 4b644b6144..ab8cdf10ee 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -77,8 +77,8 @@ jobs: echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" { echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=clang" - echo "CXX=clang++" + echo "CC=$LLVM_PREFIX/bin/clang" + echo "CXX=$LLVM_PREFIX/bin/clang++" echo "LDFLAGS=-L$LLVM_PREFIX/lib" echo "CPPFLAGS=-I$LLVM_PREFIX/include" } >> "$GITHUB_ENV" From 2f7963d32d472f98c955e8956b10dc471513135a Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:24:21 -0400 Subject: [PATCH 15/42] Bug fix --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index ab8cdf10ee..1a4310bf9b 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -73,7 +73,7 @@ jobs: - name: "Set environment to use Homebrew's llvm" run: | - LLVM_PREFIX=$(brew --prefix llvm) + LLVM_PREFIX=$(brew --prefix llvm@16) echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" { echo "LLVM_PREFIX=$LLVM_PREFIX" From 067930318596089a3c39602535ee21ec26093abd Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 20:52:33 -0400 Subject: [PATCH 16/42] Add llvm library --- .github/workflows/clp-core-build-macos.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 1a4310bf9b..c3ab824a8e 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,7 +79,7 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "LDFLAGS=-L$LLVM_PREFIX/lib -Wl,-rpath,$LLVM_PREFIX/lib" echo "CPPFLAGS=-I$LLVM_PREFIX/include" } >> "$GITHUB_ENV" From 71c0a05e9a6fd734761d076d96f88ffd4b3b3973 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 21:18:48 -0400 Subject: [PATCH 17/42] Remove ld flags --- .github/workflows/clp-core-build-macos.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index c3ab824a8e..654fe0b930 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,8 +79,6 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "LDFLAGS=-L$LLVM_PREFIX/lib -Wl,-rpath,$LLVM_PREFIX/lib" - echo "CPPFLAGS=-I$LLVM_PREFIX/include" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 7d534bf12cbd5fabc0cb8dabc94112973bc5c9da Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 26 Jun 2025 21:46:53 -0400 Subject: [PATCH 18/42] Revert "Temporarily remove other GH workflows" This reverts commit cfad209be24e966a2883bc34eb32d2df6e80389e. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From cf2c37b97314433e4167b94b45b619002ed31ee4 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 10:55:51 -0400 Subject: [PATCH 19/42] Experiment with extra clang-tidy flags --- taskfiles/lint.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index b07c4cad31..aef2da5d32 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -177,6 +177,8 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "-Drsize_t=size_t" + - "-Derrno_t=int" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" @@ -764,6 +766,8 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "-Drsize_t=size_t" + - "-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From fa5dfc2d2f6d4d99106b0c3f2cda968f33e57a12 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:19:57 -0400 Subject: [PATCH 20/42] Temporarily remove other GH workflows --- .github/workflows/clp-core-build.yaml | 376 ------------------ .github/workflows/clp-docs.yaml | 57 --- .../workflows/clp-execution-image-build.yaml | 70 ---- .github/workflows/clp-lint.yaml | 42 -- .github/workflows/clp-pr-title-checks.yaml | 31 -- .../clp-s-generated-code-checks.yaml | 42 -- 6 files changed, 618 deletions(-) delete mode 100644 .github/workflows/clp-core-build.yaml delete mode 100644 .github/workflows/clp-docs.yaml delete mode 100644 .github/workflows/clp-execution-image-build.yaml delete mode 100644 .github/workflows/clp-lint.yaml delete mode 100644 .github/workflows/clp-pr-title-checks.yaml delete mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml deleted file mode 100644 index ad0ea8fbf2..0000000000 --- a/.github/workflows/clp-core-build.yaml +++ /dev/null @@ -1,376 +0,0 @@ -name: "clp-core-build" - -on: - pull_request: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - push: - paths: - - ".github/actions/clp-core-build-containers/action.yaml" - - ".github/actions/run-on-image/action.yaml" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/**" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - "!components/core/tools/scripts/lib_install/macos/**" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -env: - BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" - DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted - # publishing of container images. - cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - clp_changed: "${{steps.filter.outputs.clp}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - # Consider changes between the current commit and `main` - # NOTE: If a pull request changes one of the images, then we need to (1) build the image - # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull - # request doesn't change an image, then we don't need to rebuild the image; instead we can - # use the published image which is based on `main`. So when determining what files have - # changed, we need to consider the delta between the current commit and `main` (rather - # than the current and previous commits) in order to detect if we need to rebuild the - # image (since it would be different from the published image). - base: "main" - filters: | - centos_stream_9_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" - - "components/core/tools/scripts/lib_install/centos-stream-9/**" - ubuntu_jammy_image: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - "components/core/tools/scripts/lib_install/*.sh" - - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" - - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" - clp: - - ".github/actions/**" - - ".github/workflows/clp-core-build.yaml" - - ".gitmodules" - - "components/core/cmake/**" - - "components/core/CMakeLists.txt" - - "components/core/src/**" - - "components/core/tests/**" - - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" - - "taskfile.yaml" - - "taskfiles/**" - - "tools/scripts/deps-download/**" - - centos-stream-9-deps-image: - name: "centos-stream-9-deps-image" - if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - ubuntu-jammy-deps-image: - name: "ubuntu-jammy-deps-image" - if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/clp-core-build-containers" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - docker_context: "components/core" - docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ - /Dockerfile" - push_deps_image: >- - ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} - token: "${{secrets.GITHUB_TOKEN}}" - - centos-stream-9-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "centos-stream-9-deps-image" - - "filter-relevant-changes" - strategy: - matrix: - use_shared_libs: [true, false] - name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "centos-stream-9" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - ubuntu-jammy-binaries: - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - strategy: - matrix: - include: - - use_shared_libs: true - upload_binaries: false - - use_shared_libs: false - upload_binaries: true - env: - OS_NAME: "ubuntu-jammy" - name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" - continue-on-error: true - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "./.github/actions/run-on-image" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core - && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py - ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} - --source-dir /mnt/repo/components/core - --build-dir /mnt/repo/components/core/build - --num-jobs $(getconf _NPROCESSORS_ONLN) - - - if: "matrix.upload_binaries == true" - id: "copy_binaries" - run: |- - output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" - - mkdir -p "${output_dir}" - cd "$GITHUB_WORKSPACE/components/core/build" - tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable - shell: "bash" - - - if: "matrix.upload_binaries == true" - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{steps.copy_binaries.outputs.output_dir}}" - retention-days: 1 - - ubuntu-jammy-binaries-image: - name: "ubuntu-jammy-binaries-image" - # Run if the ancestor jobs were successful/skipped and building clp was successful. - if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" - needs: "ubuntu-jammy-binaries" - runs-on: "ubuntu-latest" - env: - OS_NAME: "ubuntu-jammy" - TMP_OUTPUT_DIR: "/tmp" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" - with: - name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - - - name: "Untar binaries" - working-directory: >- - ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} - run: |- - tar xf clp.tar - rm clp.tar - - - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" - with: - registry: "ghcr.io" - username: "${{github.actor}}" - password: "${{secrets.GITHUB_TOKEN}}" - - - name: "Sanitize the repo's name" - id: "sanitize_repo_name" - run: |- - # Docker doesn't support repository names with uppercase characters, so we convert to - # lowercase here. - lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') - echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" - shell: "bash" - - - id: "core_image_meta" - uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" - with: - images: >- - ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} - tags: "type=raw,value=${{github.ref_name}}" - - # Only publish the image if this workflow was triggered by a push to `main`. - # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. - - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" - uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" - with: - context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" - file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" - push: true - tags: "${{steps.core_image_meta.outputs.tags}}" - labels: "${{steps.core_image_meta.outputs.labels}}" - - ubuntu-jammy-lint: - name: "ubuntu-jammy-lint" - # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. - if: >- - success() - || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') - needs: - - "filter-relevant-changes" - - "ubuntu-jammy-deps-image" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. - fetch-depth: 0 - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full - # on all files. - - if: "'schedule' != github.event_name" - name: "Restore lint:check-cpp-static-full cache" - id: "cache-restore-lint-check-cpp-static-full" - uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - - # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy - # violations. - key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" - - - uses: "./.github/actions/run-on-image" - env: - OS_NAME: "ubuntu-jammy" - with: - image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" - use_published_image: >- - ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' - || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} - # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. - # run_command: >- - # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} - run_command: >- - CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) - task lint:check-cpp-full - - # Cache the source file checksums and the generated files (logs) for the - # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. - # NOTE: If we don't cache the generated files, the task will re-run to generate them. - - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" - name: "Update lint:check-cpp-static-full cache" - uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" - with: - path: | - .task/checksum/lint-check-cpp-static-full - .task/checksum/utils-cpp-lint-clang-tidy-* - build/lint-clang-tidy - key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml deleted file mode 100644 index 82fc4a0e90..0000000000 --- a/.github/workflows/clp-docs.yaml +++ /dev/null @@ -1,57 +0,0 @@ -name: "clp-docs" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - build: - name: "build" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - lfs: "true" - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.10" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Build docs" - shell: "bash" - run: "task docs:site" - - # Upload the built docs so we can download and deploy them from y-scope/yscope-docs - - if: >- - contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) - && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) - && 'ubuntu-latest' == matrix.os - uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" - with: - name: "docs-html" - path: "build/docs/html" - if-no-files-found: "error" - - # Retain the artifact for a week in case there was a deployment issue - retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml deleted file mode 100644 index 17789906e8..0000000000 --- a/.github/workflows/clp-execution-image-build.yaml +++ /dev/null @@ -1,70 +0,0 @@ -name: "clp-execution-image-build" - -on: - pull_request: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - push: - paths: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/**/*" - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - filter-relevant-changes: - name: "filter-relevant-changes" - runs-on: "ubuntu-latest" - outputs: - ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Work around actions/runner-images/issues/6775" - run: "chown $(id -u):$(id -g) -R ." - shell: "bash" - - - name: "Filter relevant changes" - uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" - id: "filter" - with: - base: "main" - filters: | - ubuntu_jammy_image: - - ".github/actions/clp-execution-image-build/action.yaml" - - ".github/workflows/clp-execution-image-build.yaml" - - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" - - ubuntu-jammy-execution-image: - name: "ubuntu-jammy-execution-image" - if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" - needs: "filter-relevant-changes" - runs-on: "ubuntu-latest" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Workaround actions/runner-images/issues/6775" - shell: "bash" - run: "chown $(id -u):$(id -g) -R ." - - - uses: "./.github/actions/clp-execution-image-build" - with: - image_registry: "ghcr.io" - image_registry_username: "${{github.actor}}" - image_registry_password: "${{secrets.GITHUB_TOKEN}}" - platform_id: "ubuntu" - platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml deleted file mode 100644 index 5f800d8c23..0000000000 --- a/.github/workflows/clp-lint.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-lint" - -on: - pull_request: - push: - schedule: - # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) - - cron: "15 0 * * *" - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - lint-check: - name: "lint-check" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" - with: - python-version: "3.11" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Run lint task" - shell: "bash" - run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml deleted file mode 100644 index 0aa1533a98..0000000000 --- a/.github/workflows/clp-pr-title-checks.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: "clp-pr-title-checks" - -on: - pull_request_target: - # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the - # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: - # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the - # pull request triggered by this event. - # - Each job has `permissions` set to only those necessary. - types: ["edited", "opened", "reopened"] - branches: ["main"] - -permissions: {} - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - conventional-commits: - name: "conventional-commits" - permissions: - # For amannn/action-semantic-pull-request - pull-requests: "read" - runs-on: "ubuntu-latest" - steps: - - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" - env: - GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml deleted file mode 100644 index e605db9005..0000000000 --- a/.github/workflows/clp-s-generated-code-checks.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: "clp-s-generated-code-checks" - -on: - pull_request: - push: - workflow_dispatch: - -concurrency: - group: "${{github.workflow}}-${{github.ref}}" - # Cancel in-progress jobs for efficiency - cancel-in-progress: true - -jobs: - antlr-code-committed: - name: "antlr-code-committed" - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - runs-on: "${{matrix.os}}" - steps: - - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" - with: - submodules: "recursive" - - - name: "Install task" - shell: "bash" - run: "npm install -g @go-task/cli" - - - if: "matrix.os == 'macos-latest'" - name: "Install coreutils (for md5sum)" - run: "brew install coreutils" - - - name: "Check if the generated parsers are the latest" - shell: "bash" - run: - | - git status --porcelain \ - components/core/src/clp_s/search/kql/generated \ - components/core/src/clp_s/search/sql/generated \ - | grep . > /dev/null \ - && exit 1 \ - || exit 0 From e2434cb2c1f85931aaca05a18c6d6f336cd3129d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:21:18 -0400 Subject: [PATCH 21/42] Revert "Experiment with extra clang-tidy flags" This reverts commit cf2c37b97314433e4167b94b45b619002ed31ee4. --- taskfiles/lint.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index aef2da5d32..b07c4cad31 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -177,8 +177,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "-Drsize_t=size_t" - - "-Derrno_t=int" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" @@ -766,8 +764,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "-Drsize_t=size_t" - - "-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 39961b4327a0767b6a328abddcda5591fc4afaf6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:21:45 -0400 Subject: [PATCH 22/42] Add more environment flags --- .github/workflows/clp-core-build-macos.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 654fe0b930..0337e8e0d5 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,6 +79,9 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "CPPFLAGS=-I$LLVM_PREFIX/include" + echo "LD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 5e9d8c6221fb79140cabf5dbdafdff348fbfb637 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:41:09 -0400 Subject: [PATCH 23/42] Experiment with environment variables --- .github/workflows/clp-core-build-macos.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 0337e8e0d5..caf09366af 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -80,8 +80,10 @@ jobs: echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" echo "LDFLAGS=-L$LLVM_PREFIX/lib" + echo "CFLAGS=-I$LLVM_PREFIX/include" echo "CPPFLAGS=-I$LLVM_PREFIX/include" - echo "LD_LIBRARY_PATH=$LLVM_PREFIX/lib" + echo "LIBRARY_PATH=$LLVM_PREFIX/lib" + echo "DYLD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 9478ae8b3382e70eef9ed75b8b55b3c78fb17ced Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 15:57:44 -0400 Subject: [PATCH 24/42] Add more experiment with environment variables AR and RANLIB --- .github/workflows/clp-core-build-macos.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index caf09366af..fccc4df963 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -79,10 +79,10 @@ jobs: echo "LLVM_PREFIX=$LLVM_PREFIX" echo "CC=$LLVM_PREFIX/bin/clang" echo "CXX=$LLVM_PREFIX/bin/clang++" + echo "AR=$LLVM_PREFIX/bin/llvm-ar" + echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" echo "LDFLAGS=-L$LLVM_PREFIX/lib" - echo "CFLAGS=-I$LLVM_PREFIX/include" echo "CPPFLAGS=-I$LLVM_PREFIX/include" - echo "LIBRARY_PATH=$LLVM_PREFIX/lib" echo "DYLD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" From fca48575bf703671bd4362765200749d72068c2b Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 16:16:34 -0400 Subject: [PATCH 25/42] Remove environment flags for flags --- .github/workflows/clp-core-build-macos.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index fccc4df963..7c8c44f977 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -81,9 +81,6 @@ jobs: echo "CXX=$LLVM_PREFIX/bin/clang++" echo "AR=$LLVM_PREFIX/bin/llvm-ar" echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - echo "LDFLAGS=-L$LLVM_PREFIX/lib" - echo "CPPFLAGS=-I$LLVM_PREFIX/include" - echo "DYLD_LIBRARY_PATH=$LLVM_PREFIX/lib" } >> "$GITHUB_ENV" - run: "./tools/scripts/deps-download/init.sh" From 67d2363a06d32e1b57edc21a24fba05350d06d72 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 17:33:02 -0400 Subject: [PATCH 26/42] Experiment with clang-tidy argument --- taskfiles/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index b07c4cad31..42fadf0461 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,6 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 2d82296956b13488566ec9336a214ea09757e3c3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 23:43:46 -0400 Subject: [PATCH 27/42] Try add llvm to clang-tidy sysroot --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 42fadf0461..836a4b3419 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" + - "--extra-arg=-isysroot$LLVM_PREFIX/lib" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 260b69d8ea5c88c33d907de1f1b07d506e81e76f Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 23:58:25 -0400 Subject: [PATCH 28/42] Revert "Try add llvm to clang-tidy sysroot" This reverts commit 2d82296956b13488566ec9336a214ea09757e3c3. --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 836a4b3419..42fadf0461 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-isysroot$LLVM_PREFIX/lib" + - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From c91dc84196208537e084c5eb9de74dce26a6721c Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Fri, 27 Jun 2025 23:59:36 -0400 Subject: [PATCH 29/42] Not use llvm toolchain in clp --- .../core/cmake/Toolchains/llvm-clang-16-toolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake index 583e044652..b1c61c1b5e 100644 --- a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake +++ b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake @@ -17,5 +17,5 @@ endif() set(CMAKE_C_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang") set(CMAKE_CXX_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang++") -set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") -set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") +#set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") +#set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") From 693e2d03d5f69862c519a49ac49c9979b7376ff8 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 12:58:34 -0400 Subject: [PATCH 30/42] Revert "Not use llvm toolchain in clp" This reverts commit c91dc84196208537e084c5eb9de74dce26a6721c. --- .../core/cmake/Toolchains/llvm-clang-16-toolchain.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake index b1c61c1b5e..583e044652 100644 --- a/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake +++ b/components/core/cmake/Toolchains/llvm-clang-16-toolchain.cmake @@ -17,5 +17,5 @@ endif() set(CMAKE_C_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang") set(CMAKE_CXX_COMPILER "${LLVM_TOOLCHAIN_PREFIX}/bin/clang++") -#set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") -#set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") +set(CMAKE_AR "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ar") +set(CMAKE_RANLIB "${LLVM_TOOLCHAIN_PREFIX}/bin/llvm-ranlib") From 234a1f31fff3d6a235dae89d46f48fb9d3bfd1b6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 13:01:24 -0400 Subject: [PATCH 31/42] Experiment with extra args --- taskfiles/lint.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 42fadf0461..2bfbff3ef6 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,8 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-isysroot$(xcrun --show-sdk-path)" + - "--extra-arg=-Drsize=size_t" + - "--extra-arg=-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 99cdaccded6502dd40e33e793ffb0e8f46e284a7 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 13:33:43 -0400 Subject: [PATCH 32/42] Remove errno_t=int from clang-tidy flags --- taskfiles/lint.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 2bfbff3ef6..bec7e6e3dd 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -765,7 +765,6 @@ tasks: --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - "--extra-arg=-Drsize=size_t" - - "--extra-arg=-Derrno_t=int" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From 0f1034aec1efde6c754f8ab754fd59b7b0ccafd6 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 14:03:20 -0400 Subject: [PATCH 33/42] Bug fix --- taskfiles/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index bec7e6e3dd..9c4a867dc8 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -764,7 +764,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-Drsize=size_t" + - "--extra-arg=-Drsize_t=size_t" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From db7cd48d006e59ed6ac92cf09997f73e56095446 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 15:00:47 -0400 Subject: [PATCH 34/42] Revert "Temporarily remove other GH workflows" This reverts commit fa5dfc2d2f6d4d99106b0c3f2cda968f33e57a12. --- .github/workflows/clp-core-build.yaml | 376 ++++++++++++++++++ .github/workflows/clp-docs.yaml | 57 +++ .../workflows/clp-execution-image-build.yaml | 70 ++++ .github/workflows/clp-lint.yaml | 42 ++ .github/workflows/clp-pr-title-checks.yaml | 31 ++ .../clp-s-generated-code-checks.yaml | 42 ++ 6 files changed, 618 insertions(+) create mode 100644 .github/workflows/clp-core-build.yaml create mode 100644 .github/workflows/clp-docs.yaml create mode 100644 .github/workflows/clp-execution-image-build.yaml create mode 100644 .github/workflows/clp-lint.yaml create mode 100644 .github/workflows/clp-pr-title-checks.yaml create mode 100644 .github/workflows/clp-s-generated-code-checks.yaml diff --git a/.github/workflows/clp-core-build.yaml b/.github/workflows/clp-core-build.yaml new file mode 100644 index 0000000000..ad0ea8fbf2 --- /dev/null +++ b/.github/workflows/clp-core-build.yaml @@ -0,0 +1,376 @@ +name: "clp-core-build" + +on: + pull_request: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + push: + paths: + - ".github/actions/clp-core-build-containers/action.yaml" + - ".github/actions/run-on-image/action.yaml" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/**" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + - "!components/core/tools/scripts/lib_install/macos/**" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +env: + BINARIES_ARTIFACT_NAME_PREFIX: "clp-core-binaries-" + DEPS_IMAGE_NAME_PREFIX: "clp-core-dependencies-x86-" + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency. Exclude the `main` branch to allow uninterrupted + # publishing of container images. + cancel-in-progress: "${{github.ref != 'refs/heads/main'}}" + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + centos_stream_9_image_changed: "${{steps.filter.outputs.centos_stream_9_image}}" + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + clp_changed: "${{steps.filter.outputs.clp}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + # Consider changes between the current commit and `main` + # NOTE: If a pull request changes one of the images, then we need to (1) build the image + # (based on commits in the PR) and then (2) build CLP using the changed image. If a pull + # request doesn't change an image, then we don't need to rebuild the image; instead we can + # use the published image which is based on `main`. So when determining what files have + # changed, we need to consider the delta between the current commit and `main` (rather + # than the current and previous commits) in order to detect if we need to rebuild the + # image (since it would be different from the published image). + base: "main" + filters: | + centos_stream_9_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-centos-stream-9/**" + - "components/core/tools/scripts/lib_install/centos-stream-9/**" + ubuntu_jammy_image: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - "components/core/tools/scripts/lib_install/*.sh" + - "components/core/tools/docker-images/clp-env-base-ubuntu-jammy/**" + - "components/core/tools/scripts/lib_install/ubuntu-jammy/**" + clp: + - ".github/actions/**" + - ".github/workflows/clp-core-build.yaml" + - ".gitmodules" + - "components/core/cmake/**" + - "components/core/CMakeLists.txt" + - "components/core/src/**" + - "components/core/tests/**" + - "components/core/tools/scripts/utils/build-and-run-unit-tests.py" + - "taskfile.yaml" + - "taskfiles/**" + - "tools/scripts/deps-download/**" + + centos-stream-9-deps-image: + name: "centos-stream-9-deps-image" + if: "needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + ubuntu-jammy-deps-image: + name: "ubuntu-jammy-deps-image" + if: "needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'true'" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/clp-core-build-containers" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + docker_context: "components/core" + docker_file: "components/core/tools/docker-images/clp-env-base-${{env.OS_NAME}}\ + /Dockerfile" + push_deps_image: >- + ${{github.event_name != 'pull_request' && github.ref == 'refs/heads/main'}} + token: "${{secrets.GITHUB_TOKEN}}" + + centos-stream-9-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "centos-stream-9-deps-image" + - "filter-relevant-changes" + strategy: + matrix: + use_shared_libs: [true, false] + name: "centos-stream-9-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "centos-stream-9" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.centos_stream_9_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + ubuntu-jammy-binaries: + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + strategy: + matrix: + include: + - use_shared_libs: true + upload_binaries: false + - use_shared_libs: false + upload_binaries: true + env: + OS_NAME: "ubuntu-jammy" + name: "ubuntu-jammy-${{matrix.use_shared_libs == true && 'dynamic' || 'static'}}-linked-bins" + continue-on-error: true + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "./.github/actions/run-on-image" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) task deps:core + && python3 /mnt/repo/components/core/tools/scripts/utils/build-and-run-unit-tests.py + ${{matrix.use_shared_libs == true && '--use-shared-libs' || ''}} + --source-dir /mnt/repo/components/core + --build-dir /mnt/repo/components/core/build + --num-jobs $(getconf _NPROCESSORS_ONLN) + + - if: "matrix.upload_binaries == true" + id: "copy_binaries" + run: |- + output_dir="/tmp/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + echo "output_dir=${output_dir}" >> "$GITHUB_OUTPUT" + + mkdir -p "${output_dir}" + cd "$GITHUB_WORKSPACE/components/core/build" + tar cfvv "${output_dir}/clp.tar" clg clp clp-s glt make-dictionaries-readable + shell: "bash" + + - if: "matrix.upload_binaries == true" + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{steps.copy_binaries.outputs.output_dir}}" + retention-days: 1 + + ubuntu-jammy-binaries-image: + name: "ubuntu-jammy-binaries-image" + # Run if the ancestor jobs were successful/skipped and building clp was successful. + if: "!cancelled() && !failure() && needs.ubuntu-jammy-binaries.result == 'success'" + needs: "ubuntu-jammy-binaries" + runs-on: "ubuntu-latest" + env: + OS_NAME: "ubuntu-jammy" + TMP_OUTPUT_DIR: "/tmp" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - uses: "actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e" + with: + name: "${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + path: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + + - name: "Untar binaries" + working-directory: >- + ${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}} + run: |- + tar xf clp.tar + rm clp.tar + + - uses: "docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772" + with: + registry: "ghcr.io" + username: "${{github.actor}}" + password: "${{secrets.GITHUB_TOKEN}}" + + - name: "Sanitize the repo's name" + id: "sanitize_repo_name" + run: |- + # Docker doesn't support repository names with uppercase characters, so we convert to + # lowercase here. + lowercase_repo=$(echo '${{github.repository}}' | tr '[:upper:]' '[:lower:]') + echo "repository=${lowercase_repo}" >> "$GITHUB_OUTPUT" + shell: "bash" + + - id: "core_image_meta" + uses: "docker/metadata-action@902fa8ec7d6ecbf8d84d538b9b233a880e428804" + with: + images: >- + ghcr.io/${{steps.sanitize_repo_name.outputs.repository}}/clp-core-x86-${{env.OS_NAME}} + tags: "type=raw,value=${{github.ref_name}}" + + # Only publish the image if this workflow was triggered by a push to `main`. + # NOTE: We run the rest of the job to test that the binaries were uploaded correctly. + - if: "github.event_name == 'push' && github.ref == 'refs/heads/main'" + uses: "docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4" + with: + context: "${{env.TMP_OUTPUT_DIR}}/${{env.BINARIES_ARTIFACT_NAME_PREFIX}}${{env.OS_NAME}}" + file: "components/core/tools/docker-images/clp-core-${{env.OS_NAME}}/Dockerfile" + push: true + tags: "${{steps.core_image_meta.outputs.tags}}" + labels: "${{steps.core_image_meta.outputs.labels}}" + + ubuntu-jammy-lint: + name: "ubuntu-jammy-lint" + # Run if the ancestor jobs succeeded OR they were skipped and clp was changed. + if: >- + success() + || (!cancelled() && !failure() && needs.filter-relevant-changes.outputs.clp_changed == 'true') + needs: + - "filter-relevant-changes" + - "ubuntu-jammy-deps-image" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + # Fetch history so that the `clang-tidy-diff` task can compare against the main branch. + fetch-depth: 0 + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + # NOTE: We don't use the cache for scheduled runs so that they run lint:check-cpp-static-full + # on all files. + - if: "'schedule' != github.event_name" + name: "Restore lint:check-cpp-static-full cache" + id: "cache-restore-lint-check-cpp-static-full" + uses: "actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + + # NOTE: We use a per-OS cache since different OSes may trigger different clang-tidy + # violations. + key: "main-branch-ubuntu-jammy-lint:check-cpp-static-full" + + - uses: "./.github/actions/run-on-image" + env: + OS_NAME: "ubuntu-jammy" + with: + image_name: "${{env.DEPS_IMAGE_NAME_PREFIX}}${{env.OS_NAME}}" + use_published_image: >- + ${{needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed == 'false' + || (github.event_name != 'pull_request' && github.ref == 'refs/heads/main')}} + # TODO: When enough files are passing clang-tidy, switch to a full pass on schedule only. + # run_command: >- + # task lint:check-cpp-${{(github.event_name == 'schedule') && 'full' || 'diff'}} + run_command: >- + CLP_CORE_MAX_PARALLELISM_PER_BUILD_TASK=$(getconf _NPROCESSORS_ONLN) + task lint:check-cpp-full + + # Cache the source file checksums and the generated files (logs) for the + # lint:check-cpp-static-full task, but only if it runs successfully on the main branch. + # NOTE: If we don't cache the generated files, the task will re-run to generate them. + - if: "'pull_request' != github.event_name && 'refs/heads/main' == github.ref" + name: "Update lint:check-cpp-static-full cache" + uses: "actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684" + with: + path: | + .task/checksum/lint-check-cpp-static-full + .task/checksum/utils-cpp-lint-clang-tidy-* + build/lint-clang-tidy + key: "${{steps.cache-restore-lint-check-cpp-static-full.outputs.cache-primary-key}}" diff --git a/.github/workflows/clp-docs.yaml b/.github/workflows/clp-docs.yaml new file mode 100644 index 0000000000..82fc4a0e90 --- /dev/null +++ b/.github/workflows/clp-docs.yaml @@ -0,0 +1,57 @@ +name: "clp-docs" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + build: + name: "build" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + lfs: "true" + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.10" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Build docs" + shell: "bash" + run: "task docs:site" + + # Upload the built docs so we can download and deploy them from y-scope/yscope-docs + - if: >- + contains(fromJSON('["push", "workflow_dispatch"]'), github.event_name) + && ('refs/heads/main' == github.ref || startsWith(github.ref, 'refs/tags/v')) + && 'ubuntu-latest' == matrix.os + uses: "actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02" + with: + name: "docs-html" + path: "build/docs/html" + if-no-files-found: "error" + + # Retain the artifact for a week in case there was a deployment issue + retention-days: 7 diff --git a/.github/workflows/clp-execution-image-build.yaml b/.github/workflows/clp-execution-image-build.yaml new file mode 100644 index 0000000000..17789906e8 --- /dev/null +++ b/.github/workflows/clp-execution-image-build.yaml @@ -0,0 +1,70 @@ +name: "clp-execution-image-build" + +on: + pull_request: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + push: + paths: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/**/*" + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + filter-relevant-changes: + name: "filter-relevant-changes" + runs-on: "ubuntu-latest" + outputs: + ubuntu_jammy_image_changed: "${{steps.filter.outputs.ubuntu_jammy_image}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Work around actions/runner-images/issues/6775" + run: "chown $(id -u):$(id -g) -R ." + shell: "bash" + + - name: "Filter relevant changes" + uses: "dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36" + id: "filter" + with: + base: "main" + filters: | + ubuntu_jammy_image: + - ".github/actions/clp-execution-image-build/action.yaml" + - ".github/workflows/clp-execution-image-build.yaml" + - "tools/docker-images/clp-execution-base-ubuntu-jammy/**/*" + + ubuntu-jammy-execution-image: + name: "ubuntu-jammy-execution-image" + if: "'true' == needs.filter-relevant-changes.outputs.ubuntu_jammy_image_changed" + needs: "filter-relevant-changes" + runs-on: "ubuntu-latest" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Workaround actions/runner-images/issues/6775" + shell: "bash" + run: "chown $(id -u):$(id -g) -R ." + + - uses: "./.github/actions/clp-execution-image-build" + with: + image_registry: "ghcr.io" + image_registry_username: "${{github.actor}}" + image_registry_password: "${{secrets.GITHUB_TOKEN}}" + platform_id: "ubuntu" + platform_version_id: "jammy" diff --git a/.github/workflows/clp-lint.yaml b/.github/workflows/clp-lint.yaml new file mode 100644 index 0000000000..5f800d8c23 --- /dev/null +++ b/.github/workflows/clp-lint.yaml @@ -0,0 +1,42 @@ +name: "clp-lint" + +on: + pull_request: + push: + schedule: + # Run daily at 00:15 UTC (the 15 is to avoid periods of high load) + - cron: "15 0 * * *" + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + lint-check: + name: "lint-check" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - uses: "actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38" + with: + python-version: "3.11" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Run lint task" + shell: "bash" + run: "task lint:check-no-cpp" diff --git a/.github/workflows/clp-pr-title-checks.yaml b/.github/workflows/clp-pr-title-checks.yaml new file mode 100644 index 0000000000..0aa1533a98 --- /dev/null +++ b/.github/workflows/clp-pr-title-checks.yaml @@ -0,0 +1,31 @@ +name: "clp-pr-title-checks" + +on: + pull_request_target: + # NOTE: Workflows triggered by this event give the workflow access to secrets and grant the + # `GITHUB_TOKEN` read/write repository access by default. So we need to ensure: + # - This workflow doesn't inadvertently check out, build, or execute untrusted code from the + # pull request triggered by this event. + # - Each job has `permissions` set to only those necessary. + types: ["edited", "opened", "reopened"] + branches: ["main"] + +permissions: {} + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + conventional-commits: + name: "conventional-commits" + permissions: + # For amannn/action-semantic-pull-request + pull-requests: "read" + runs-on: "ubuntu-latest" + steps: + - uses: "amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017" + env: + GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}" diff --git a/.github/workflows/clp-s-generated-code-checks.yaml b/.github/workflows/clp-s-generated-code-checks.yaml new file mode 100644 index 0000000000..e605db9005 --- /dev/null +++ b/.github/workflows/clp-s-generated-code-checks.yaml @@ -0,0 +1,42 @@ +name: "clp-s-generated-code-checks" + +on: + pull_request: + push: + workflow_dispatch: + +concurrency: + group: "${{github.workflow}}-${{github.ref}}" + # Cancel in-progress jobs for efficiency + cancel-in-progress: true + +jobs: + antlr-code-committed: + name: "antlr-code-committed" + strategy: + matrix: + os: ["macos-latest", "ubuntu-latest"] + runs-on: "${{matrix.os}}" + steps: + - uses: "actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683" + with: + submodules: "recursive" + + - name: "Install task" + shell: "bash" + run: "npm install -g @go-task/cli" + + - if: "matrix.os == 'macos-latest'" + name: "Install coreutils (for md5sum)" + run: "brew install coreutils" + + - name: "Check if the generated parsers are the latest" + shell: "bash" + run: + | + git status --porcelain \ + components/core/src/clp_s/search/kql/generated \ + components/core/src/clp_s/search/sql/generated \ + | grep . > /dev/null \ + && exit 1 \ + || exit 0 From 9e2c951104bc70d7e782c32e78a70eef864699e3 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Sat, 28 Jun 2025 16:04:37 -0400 Subject: [PATCH 35/42] Add rsize clang-tidy flag in all clang-tidy tasks --- taskfiles/lint.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index 9c4a867dc8..0af185443e 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -177,6 +177,7 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" + - "--extra-arg=-Drsize_t=size_t" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" From ea5813a799ffcd4c051aefa1e1532dfb40a2d422 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 10:11:50 -0400 Subject: [PATCH 36/42] Remove unnecessary if guard in cmake --- components/core/CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/components/core/CMakeLists.txt b/components/core/CMakeLists.txt index 230c15f5e4..be4c883b8c 100644 --- a/components/core/CMakeLists.txt +++ b/components/core/CMakeLists.txt @@ -284,9 +284,7 @@ endif() # Add ystdlib find_package(ystdlib REQUIRED) -if (ystdlib_FOUND) - message(STATUS "Found ystdlib ${ystdlib_VERSION}") -endif() +message(STATUS "Found ystdlib ${ystdlib_VERSION}") # Find and setup ZStd Library if(CLP_NEED_ZSTD) From 453a9c77eb98a2f98fdf19a0a54c8cf345a5ff38 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Mon, 7 Jul 2025 10:16:54 -0400 Subject: [PATCH 37/42] Improve comments in install script --- .../lib_install/ubuntu-jammy/install-packages-from-source.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index 734c2015ba..d30bb45cdc 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -9,9 +9,9 @@ set -u script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir=$script_dir/.. -# NOTE: cmake must be installed first since the remaining packages depend on it to build +# NOTE: cmake must be at least version 3.23 "$lib_install_scripts_dir"/install-cmake.sh 3.23.5 -# NOTE: boost must be installed second since the remaining packages depend on it +# NOTE: boost must be installed since the remaining packages depend on it "$lib_install_scripts_dir"/install-boost.sh 1.87.0 "$lib_install_scripts_dir"/libarchive.sh 3.5.1 From 18ddf6c48bedf32f08b80310373810fbcf953c9d Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 9 Jul 2025 13:34:21 -0400 Subject: [PATCH 38/42] Remove fix for macOS --- .github/workflows/clp-core-build-macos.yaml | 12 ------------ taskfiles/lint.yaml | 2 -- 2 files changed, 14 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 7c8c44f977..51eb6985d0 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -71,18 +71,6 @@ jobs: - name: "Install dependencies" run: "./components/core/tools/scripts/lib_install/macos/install-all.sh" - - name: "Set environment to use Homebrew's llvm" - run: | - LLVM_PREFIX=$(brew --prefix llvm@16) - echo "$LLVM_PREFIX/bin" >> "$GITHUB_PATH" - { - echo "LLVM_PREFIX=$LLVM_PREFIX" - echo "CC=$LLVM_PREFIX/bin/clang" - echo "CXX=$LLVM_PREFIX/bin/clang++" - echo "AR=$LLVM_PREFIX/bin/llvm-ar" - echo "RANLIB=$LLVM_PREFIX/bin/llvm-ranlib" - } >> "$GITHUB_ENV" - - run: "./tools/scripts/deps-download/init.sh" shell: "bash" diff --git a/taskfiles/lint.yaml b/taskfiles/lint.yaml index b9eb7b4d7e..149a1ba5f8 100644 --- a/taskfiles/lint.yaml +++ b/taskfiles/lint.yaml @@ -159,7 +159,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-Drsize_t=size_t" OUTPUT_DIR: "{{.G_LINT_CLANG_TIDY_DIR}}" VENV_DIR: "{{.G_LINT_VENV_DIR}}" @@ -747,7 +746,6 @@ tasks: - >- --line-filter "[{{.G_CLANG_TIDY_LINE_FILTER_IGNORE_MSGPACK_1098}}]" - "-p '{{.G_CORE_COMPONENT_BUILD_DIR}}'" - - "--extra-arg=-Drsize_t=size_t" INCLUDE_PATTERNS: - "{{.G_CORE_COMPONENT_DIR}}/src/*" - "{{.G_CORE_COMPONENT_DIR}}/tests/*" From be467e89390d3dd0f0f412448afd443d5129b432 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Wed, 9 Jul 2025 13:48:04 -0400 Subject: [PATCH 39/42] Disable ci on macOS 13 and 14 --- .github/workflows/clp-core-build-macos.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clp-core-build-macos.yaml b/.github/workflows/clp-core-build-macos.yaml index 51eb6985d0..dd82bca50a 100644 --- a/.github/workflows/clp-core-build-macos.yaml +++ b/.github/workflows/clp-core-build-macos.yaml @@ -45,8 +45,11 @@ jobs: strategy: matrix: os: - - "macos-13" - - "macos-14" + # NOTE: We temporarily disable macOS 13 and 14 since older versions of `AppleClang` do + # not support C++ 20 required by `ystdlib`. + # See https://github.com/y-scope/clp/issues/1090 for more details. + # - "macos-13" + # - "macos-14" - "macos-15" use_shared_libs: - true From c2501df1ff8dc6e1b991d7f399baa112907f5906 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 10 Jul 2025 10:08:15 -0400 Subject: [PATCH 40/42] Improve sequence and comment in install script Co-authored-by: davidlion --- .../lib_install/ubuntu-jammy/install-packages-from-source.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh index d30bb45cdc..5750b0005c 100755 --- a/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh +++ b/components/core/tools/scripts/lib_install/ubuntu-jammy/install-packages-from-source.sh @@ -9,11 +9,10 @@ set -u script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" lib_install_scripts_dir=$script_dir/.. -# NOTE: cmake must be at least version 3.23 -"$lib_install_scripts_dir"/install-cmake.sh 3.23.5 -# NOTE: boost must be installed since the remaining packages depend on it +# NOTE: boost must be installed first since other packages depend on it "$lib_install_scripts_dir"/install-boost.sh 1.87.0 +"$lib_install_scripts_dir"/install-cmake.sh 3.23.5 "$lib_install_scripts_dir"/libarchive.sh 3.5.1 "$lib_install_scripts_dir"/liblzma.sh 5.4.6 "$lib_install_scripts_dir"/lz4.sh 1.8.2 From dcb3e2840c7170229f0e80de3f9a361f953a6ceb Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 10 Jul 2025 10:16:32 -0400 Subject: [PATCH 41/42] Remove PIC flag --- taskfiles/deps/main.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 7c696f8326..37e812cf98 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -379,7 +379,6 @@ tasks: - task: "utils:install-remote-cmake-lib" vars: CMAKE_GEN_ARGS: - - "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" - "-Dystdlib_BUILD_TESTING=OFF" - "-DCMAKE_POLICY_DEFAULT_CMP0144=NEW" LIB_NAME: "ystdlib" From eed0d817b19818ebe217d108c02152428af76e18 Mon Sep 17 00:00:00 2001 From: sitaowang1998 Date: Thu, 10 Jul 2025 10:26:48 -0400 Subject: [PATCH 42/42] Add default flags to ystdlib build --- taskfiles/deps/main.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/taskfiles/deps/main.yaml b/taskfiles/deps/main.yaml index 56a6bd1506..e34afe1818 100644 --- a/taskfiles/deps/main.yaml +++ b/taskfiles/deps/main.yaml @@ -380,6 +380,8 @@ tasks: vars: CMAKE_GEN_ARGS: - "-Dystdlib_BUILD_TESTING=OFF" + - "-DCMAKE_BUILD_TYPE=Release" + - "-DCMAKE_INSTALL_MESSAGE=LAZY" - "-DCMAKE_POLICY_DEFAULT_CMP0144=NEW" LIB_NAME: "ystdlib" WORK_DIR: "{{.G_DEPS_DIR}}/ystdlib"