From a0a36cb89279d9e99efa3df68db508463415c0cc Mon Sep 17 00:00:00 2001 From: Z1 cciauto Date: Mon, 11 May 2026 05:54:17 -0700 Subject: [PATCH 1/6] [Infra][Multi-arch] Manifest log collector mechanism --- .../multi_arch_build_portable_linux.yml | 28 +++++++++++++++++++ ...ti_arch_build_portable_linux_artifacts.yml | 16 ++++++++++- .github/workflows/teams_notifier.yml | 4 +++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index 6d3da2c15f3de..34d019f706a27 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -285,3 +285,31 @@ jobs: permissions: contents: read id-token: write + + # ========================================================================== + # STAGE: Manifest commit log (generic) + # ========================================================================== + manifest_summary: + needs: [compiler-runtime, foundation, math-libs. comm-libs] + if: ${{ !cancelled() && !failure() }} + runs-on: ubuntu-latest + steps: + - name: Download all stage submodule commit + uses: actions/download-artifact@v4 + with: + pattern: submodule-commit-* + merge-multiple: true + + - name: Merge all submodule commit + run: | + jq -s 'add | unique_by(.path)' submodule-commit-*.json \ + > manifest.json + + echo "=== ALL SUBMODULES ===" + jq -r '.[] | "\(.stage) | \(.path) → \(.commit[0:12])"' \ + manifest.json + + - uses: actions/upload-artifact@v4 + with: + name: submodule-commit-all + path: manifest.json diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index de41d8748b6c5..96470f68de74b 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -221,7 +221,21 @@ jobs: - name: Fetch sources timeout-minutes: 30 - run: ./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1 + run: | + commit_log=$(./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1) + # Save the repo and commit into json + echo "$commit_logs" | grep "^Submodule path" | \ + grep "checked out" | sed "s/Submodule path '\([^']*\)': checked out '\([^']*\)'/\1 \2/" | \ + jq -Rn --arg stage "${{ inputs.stage }}" \ + '[inputs | split(" ") | {path: .[0], commit: .[1], stage: $stage}]' \ + > submodule-commit-${{ inputs.stage }}.json + + - name: Upload submodule commit file + uses: actions/upload-artifact@v4 + with: + name: submodule-commit-${{ inputs.stage }} + path: submodule-commiy-${{ inputs.stage }}.json + retention-days: 1 - name: TheRock and llvm SHA if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.test_type != 'full') }} diff --git a/.github/workflows/teams_notifier.yml b/.github/workflows/teams_notifier.yml index 62d9b1ea4e501..39673e9e3c42d 100644 --- a/.github/workflows/teams_notifier.yml +++ b/.github/workflows/teams_notifier.yml @@ -18,6 +18,10 @@ jobs: with: sparse-checkout: .github + - uses: actions/download-artifact@v4 + with: + name: submodule-info-all + - name: Calculate URLs and Fetch Manifest id: extract_urls env: From bff24aef6057fb8a6da956a830f7422e6a7a2760 Mon Sep 17 00:00:00 2001 From: Z1 cciauto Date: Mon, 11 May 2026 05:57:52 -0700 Subject: [PATCH 2/6] Update the dependency list --- .github/workflows/multi_arch_build_portable_linux.yml | 2 +- .github/workflows/multi_arch_build_portable_linux_artifacts.yml | 2 +- .github/workflows/teams_notifier.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index 34d019f706a27..b6f1d9b84bea1 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -290,7 +290,7 @@ jobs: # STAGE: Manifest commit log (generic) # ========================================================================== manifest_summary: - needs: [compiler-runtime, foundation, math-libs. comm-libs] + needs: [compiler-runtime, foundation, math-libs, comm-libs] if: ${{ !cancelled() && !failure() }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 96470f68de74b..5e4a44e35c4e7 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -234,7 +234,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: submodule-commit-${{ inputs.stage }} - path: submodule-commiy-${{ inputs.stage }}.json + path: submodule-commit-${{ inputs.stage }}.json retention-days: 1 - name: TheRock and llvm SHA diff --git a/.github/workflows/teams_notifier.yml b/.github/workflows/teams_notifier.yml index 39673e9e3c42d..5babfd5f91f25 100644 --- a/.github/workflows/teams_notifier.yml +++ b/.github/workflows/teams_notifier.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/download-artifact@v4 with: - name: submodule-info-all + name: submodule-commit-all - name: Calculate URLs and Fetch Manifest id: extract_urls From 2e7344a091d1e85f24923eec0553bfa4f0f370de Mon Sep 17 00:00:00 2001 From: Z1 cciauto Date: Mon, 11 May 2026 06:38:26 -0700 Subject: [PATCH 3/6] Update to replace jq with sed --- .github/workflows/multi_arch_build_portable_linux.yml | 8 ++++---- .../multi_arch_build_portable_linux_artifacts.yml | 8 +++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index b6f1d9b84bea1..a09b1e9780fb6 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -302,14 +302,14 @@ jobs: - name: Merge all submodule commit run: | - jq -s 'add | unique_by(.path)' submodule-commit-*.json \ - > manifest.json + jq -s 'add | unique_by(.path)' submodule-commit-*.txt \ + > manifest.txt echo "=== ALL SUBMODULES ===" jq -r '.[] | "\(.stage) | \(.path) → \(.commit[0:12])"' \ - manifest.json + manifest.txt - uses: actions/upload-artifact@v4 with: name: submodule-commit-all - path: manifest.json + path: manifest.txt diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 5e4a44e35c4e7..907cca2cb91de 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -225,16 +225,14 @@ jobs: commit_log=$(./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1) # Save the repo and commit into json echo "$commit_logs" | grep "^Submodule path" | \ - grep "checked out" | sed "s/Submodule path '\([^']*\)': checked out '\([^']*\)'/\1 \2/" | \ - jq -Rn --arg stage "${{ inputs.stage }}" \ - '[inputs | split(" ") | {path: .[0], commit: .[1], stage: $stage}]' \ - > submodule-commit-${{ inputs.stage }}.json + grep "checked out" | sed "s/Submodule path '\([^']*\)': checked out '\([^']*\)'/\1 \2/" \ + > submodule-commit-${{ inputs.stage }}.txt - name: Upload submodule commit file uses: actions/upload-artifact@v4 with: name: submodule-commit-${{ inputs.stage }} - path: submodule-commit-${{ inputs.stage }}.json + path: submodule-commit-${{ inputs.stage }}.txt retention-days: 1 - name: TheRock and llvm SHA From 9cb0119ca394b79c1a9b5d43b7d6a108194f9ab5 Mon Sep 17 00:00:00 2001 From: Z1 cciauto Date: Mon, 11 May 2026 08:25:53 -0700 Subject: [PATCH 4/6] Update the file name to have stage_name --- .../workflows/multi_arch_build_portable_linux.yml | 12 +++++++----- .../multi_arch_build_portable_linux_artifacts.yml | 6 +++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index a09b1e9780fb6..3b661a13cead7 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -302,12 +302,14 @@ jobs: - name: Merge all submodule commit run: | - jq -s 'add | unique_by(.path)' submodule-commit-*.txt \ - > manifest.txt + for f in submodule-commit-*.txt; do + echo "--- $f ---" + cat "$f" + done - echo "=== ALL SUBMODULES ===" - jq -r '.[] | "\(.stage) | \(.path) → \(.commit[0:12])"' \ - manifest.txt + cat submodule-commit-*.txt | sort -u > manifest.txt + + cat manifest.txt - uses: actions/upload-artifact@v4 with: diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 907cca2cb91de..2a1b03c51cb37 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -226,13 +226,13 @@ jobs: # Save the repo and commit into json echo "$commit_logs" | grep "^Submodule path" | \ grep "checked out" | sed "s/Submodule path '\([^']*\)': checked out '\([^']*\)'/\1 \2/" \ - > submodule-commit-${{ inputs.stage }}.txt + > submodule-commit-${{ inputs.stage_name }}.txt - name: Upload submodule commit file uses: actions/upload-artifact@v4 with: - name: submodule-commit-${{ inputs.stage }} - path: submodule-commit-${{ inputs.stage }}.txt + name: submodule-commit-${{ inputs.stage_name }} + path: submodule-commit-${{ inputs.stage_name }}.txt retention-days: 1 - name: TheRock and llvm SHA From bf26863a52502043d73b4e775281b677c4ba2e30 Mon Sep 17 00:00:00 2001 From: Z1 cciauto Date: Mon, 11 May 2026 09:59:58 -0700 Subject: [PATCH 5/6] Enable overwrite option for upload --- .../workflows/multi_arch_build_portable_linux_artifacts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index 2a1b03c51cb37..f0a8ef35427eb 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -223,8 +223,7 @@ jobs: timeout-minutes: 30 run: | commit_log=$(./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1) - # Save the repo and commit into json - echo "$commit_logs" | grep "^Submodule path" | \ + echo "$commit_log" | grep "^Submodule path" | \ grep "checked out" | sed "s/Submodule path '\([^']*\)': checked out '\([^']*\)'/\1 \2/" \ > submodule-commit-${{ inputs.stage_name }}.txt @@ -233,6 +232,7 @@ jobs: with: name: submodule-commit-${{ inputs.stage_name }} path: submodule-commit-${{ inputs.stage_name }}.txt + overwrite: true retention-days: 1 - name: TheRock and llvm SHA From 7d807a29cdb402583baf0c280f924ecca2c5f88e Mon Sep 17 00:00:00 2001 From: Z1 cciauto Date: Wed, 13 May 2026 10:36:02 -0700 Subject: [PATCH 6/6] Parsering the manifest.txt --- .github/workflows/build_metadata_extractor.py | 58 ++++++++++--------- .github/workflows/teams_notifier.yml | 6 +- 2 files changed, 35 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build_metadata_extractor.py b/.github/workflows/build_metadata_extractor.py index 3df3a2d39d2bc..c18b638da3741 100644 --- a/.github/workflows/build_metadata_extractor.py +++ b/.github/workflows/build_metadata_extractor.py @@ -41,12 +41,13 @@ class BuildMetadataExtractor: - def __init__(self, org_name, project_name, run_id, github_token, artifacts_url, build_logs_url, output_file): + def __init__(self, org_name, project_name, run_id, github_token, manifest_path, artifacts_url, build_logs_url, output_file): """Initialize the extractor with organization, project info, and additional inputs.""" self.org_name = org_name self.project_name = project_name self.run_id = run_id self.github_token = github_token + self.manifest_path = manifest_path self.artifacts_url = artifacts_url self.build_logs_url = build_logs_url self.output_file = output_file @@ -58,24 +59,30 @@ def read_manifest_file(self): if not os.path.exists(self.manifest_path): print(f"Manifest file '{self.manifest_path}' does not exist.") return None + submodules = {} with open(self.manifest_path, 'r') as f: - return json.load(f) + for component in f: + component = component.strip() + if not component: + continue + component_list = component.split(None, 1) # this will split on component name and sha + if len(component_list) == 2: + submodule_name = component_list[0] + pin_sha = component_list[1] + submodules[submodule_name] = pin_sha + return submodules def extract_submodule_table(self, manifest_data): """Generates a table detailing submodules and URLs.""" - the_rock_commit = manifest_data.get('the_rock_commit', '') - commit_url = f"{self.base_url}/commit/{the_rock_commit}" + #the_rock_commit = manifest_data.get('the_rock_commit', '') + #commit_url = f"{self.base_url}/commit/{the_rock_commit}" table = '| Submodule | URL |\n|-----------|-----|\n' - - for submodule in manifest_data.get('submodules', []): - submodule_name = submodule['submodule_name'] + for submodule_name,pin_sha in manifest_data.items(): if submodule_name == 'rccl': # Filter out specific submodules - continue - pin_sha = submodule.get('pin_sha') - if pin_sha: - submodule_url = submodule['submodule_url'] - commit_url = f"{submodule_url.replace('.git', '')}/commit/{pin_sha}" - table += f'| {submodule_name} | ({commit_url}) |\\n' + continue + submodule_url = f"{self.base_url}/{submodule_name}" + commit_url = f"{submodule_url}/commit/{pin_sha}" + table += f'| {submodule_name} | ({commit_url}) |\n' return table @@ -112,10 +119,10 @@ def list_failures(self): failure_table += f'| {job_name} | ({job_url}) |\\n' return failure_table - def save_results_to_file(self, manifest_artifacts_table, failure_table): + def save_results_to_file(self,submodule_table, manifest_artifacts_table, failure_table): """Saves the results to a file in structured JSON format.""" results = { - #"submodule_table": submodule_table, + "submodule_table": submodule_table, "manifest_artifacts_table": manifest_artifacts_table, "failure_table": failure_table or "No failures found" } @@ -129,7 +136,7 @@ def save_results_to_file(self, manifest_artifacts_table, failure_table): PROJECT_NAME = os.getenv("PROJECT_NAME", "llvm-project") RUN_ID = os.getenv("RUN_ID", "") GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") - #MANIFEST_FILE = os.getenv("MANIFEST_FILE", "manifest.json") + MANIFEST_FILE = os.getenv("MANIFEST_FILE", "manifest.txt") #ROCK_MANIFEST_URL = os.getenv("ROCK_MANIFEST_URL") ARTIFACTS_URL = os.getenv("ARTIFACTS_URL") BUILD_LOGS_URL = os.getenv("BUILD_LOGS_URL") @@ -142,18 +149,17 @@ def save_results_to_file(self, manifest_artifacts_table, failure_table): #) extractor = BuildMetadataExtractor( - ORG_NAME, PROJECT_NAME, RUN_ID, GITHUB_TOKEN, ARTIFACTS_URL, BUILD_LOGS_URL, OUTPUT_FILE + ORG_NAME, PROJECT_NAME, RUN_ID, GITHUB_TOKEN, MANIFEST_FILE, ARTIFACTS_URL, BUILD_LOGS_URL, OUTPUT_FILE ) # Process the manifest file - #manifest_data = extractor.read_manifest_file() - #if manifest_data: - #submodule_table = extractor.extract_submodule_table(manifest_data) - manifest_artifacts_table = extractor.generate_manifest_artifact_logs_table() - failure_table = extractor.list_failures() - - # Save results to an output file - #extractor.save_results_to_file(submodule_table, manifest_artifacts_table, failure_table) - extractor.save_results_to_file(manifest_artifacts_table, failure_table) + manifest_data = extractor.read_manifest_file() + if manifest_data: + submodule_table = extractor.extract_submodule_table(manifest_data) + manifest_artifacts_table = extractor.generate_manifest_artifact_logs_table() + failure_table = extractor.list_failures() + + # Save results to an output file + extractor.save_results_to_file(submodule_table, manifest_artifacts_table, failure_table) diff --git a/.github/workflows/teams_notifier.yml b/.github/workflows/teams_notifier.yml index 5babfd5f91f25..08fa74c3003a5 100644 --- a/.github/workflows/teams_notifier.yml +++ b/.github/workflows/teams_notifier.yml @@ -45,7 +45,7 @@ jobs: curl -H "Authorization: token $GITHUB_TOKEN" -L "$LOG_URL" -o logs.txt # Parse URLs from the logs using grep - THE_ROCK_MANIFEST_URL=$(grep -oP '\[TheRock Manifest\]\(\K[^\)]+' "logs.txt" || echo "") + #THE_ROCK_MANIFEST_URL=$(grep -oP '\[TheRock Manifest\]\(\K[^\)]+' "logs.txt" || echo "") #ARTIFACTS_URL=$(grep -oP '\[Artifacts\]\(\K[^\)]+' "logs.txt" || echo "") #BUILD_LOGS_URL=$(grep -oP '\[Build Logs\]\(\K[^\)]+' "logs.txt" || echo "") ARTIFACTS_URL="https://therock-ci-artifacts-external.s3.amazonaws.com/ROCm-llvm-project/$RUN_ID-linux/index.html" @@ -69,7 +69,7 @@ jobs: echo "ORG_NAME=ROCm" >> $GITHUB_ENV echo "PROJECT_NAME=llvm-project" >> $GITHUB_ENV echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV - echo "MANIFEST_FILE=manifest.json" >> $GITHUB_ENV + echo "MANIFEST_FILE=manifest.txt" >> $GITHUB_ENV echo "OUTPUT_FILE=results.json" >> $GITHUB_ENV - name: Execute Python script for Notification Content @@ -78,7 +78,7 @@ jobs: ORG_NAME: ${{ env.ORG_NAME }} PROJECT_NAME: ${{ env.PROJECT_NAME }} RUN_ID: ${{ env.RUN_ID }} - #MANIFEST_FILE: ${{ env.MANIFEST_FILE }} + MANIFEST_FILE: ${{ env.MANIFEST_FILE }} #ROCK_MANIFEST_URL: ${{ env.THE_ROCK_MANIFEST_URL }} ARTIFACTS_URL: ${{ env.ARTIFACTS_URL }} BUILD_LOGS_URL: ${{ env.BUILD_LOGS_URL }}