diff --git a/.github/workflows/build_metadata_extractor.py b/.github/workflows/build_metadata_extractor.py index 3df3a2d39d2bc..c18b638da3741 100644 --- a/.github/workflows/build_metadata_extractor.py +++ b/.github/workflows/build_metadata_extractor.py @@ -41,12 +41,13 @@ class BuildMetadataExtractor: - def __init__(self, org_name, project_name, run_id, github_token, artifacts_url, build_logs_url, output_file): + def __init__(self, org_name, project_name, run_id, github_token, manifest_path, artifacts_url, build_logs_url, output_file): """Initialize the extractor with organization, project info, and additional inputs.""" self.org_name = org_name self.project_name = project_name self.run_id = run_id self.github_token = github_token + self.manifest_path = manifest_path self.artifacts_url = artifacts_url self.build_logs_url = build_logs_url self.output_file = output_file @@ -58,24 +59,30 @@ def read_manifest_file(self): if not os.path.exists(self.manifest_path): print(f"Manifest file '{self.manifest_path}' does not exist.") return None + submodules = {} with open(self.manifest_path, 'r') as f: - return json.load(f) + for component in f: + component = component.strip() + if not component: + continue + component_list = component.split(None, 1) # this will split on component name and sha + if len(component_list) == 2: + submodule_name = component_list[0] + pin_sha = component_list[1] + submodules[submodule_name] = pin_sha + return submodules def extract_submodule_table(self, manifest_data): """Generates a table detailing submodules and URLs.""" - the_rock_commit = manifest_data.get('the_rock_commit', '') - commit_url = f"{self.base_url}/commit/{the_rock_commit}" + #the_rock_commit = manifest_data.get('the_rock_commit', '') + #commit_url = f"{self.base_url}/commit/{the_rock_commit}" table = '| Submodule | URL |\n|-----------|-----|\n' - - for submodule in manifest_data.get('submodules', []): - submodule_name = submodule['submodule_name'] + for submodule_name,pin_sha in manifest_data.items(): if submodule_name == 'rccl': # Filter out specific submodules - continue - pin_sha = submodule.get('pin_sha') - if pin_sha: - submodule_url = submodule['submodule_url'] - commit_url = f"{submodule_url.replace('.git', '')}/commit/{pin_sha}" - table += f'| {submodule_name} | ({commit_url}) |\\n' + continue + submodule_url = f"{self.base_url}/{submodule_name}" + commit_url = f"{submodule_url}/commit/{pin_sha}" + table += f'| {submodule_name} | ({commit_url}) |\n' return table @@ -112,10 +119,10 @@ def list_failures(self): failure_table += f'| {job_name} | ({job_url}) |\\n' return failure_table - def save_results_to_file(self, manifest_artifacts_table, failure_table): + def save_results_to_file(self,submodule_table, manifest_artifacts_table, failure_table): """Saves the results to a file in structured JSON format.""" results = { - #"submodule_table": submodule_table, + "submodule_table": submodule_table, "manifest_artifacts_table": manifest_artifacts_table, "failure_table": failure_table or "No failures found" } @@ -129,7 +136,7 @@ def save_results_to_file(self, manifest_artifacts_table, failure_table): PROJECT_NAME = os.getenv("PROJECT_NAME", "llvm-project") RUN_ID = os.getenv("RUN_ID", "") GITHUB_TOKEN = os.getenv("GITHUB_TOKEN") - #MANIFEST_FILE = os.getenv("MANIFEST_FILE", "manifest.json") + MANIFEST_FILE = os.getenv("MANIFEST_FILE", "manifest.txt") #ROCK_MANIFEST_URL = os.getenv("ROCK_MANIFEST_URL") ARTIFACTS_URL = os.getenv("ARTIFACTS_URL") BUILD_LOGS_URL = os.getenv("BUILD_LOGS_URL") @@ -142,18 +149,17 @@ def save_results_to_file(self, manifest_artifacts_table, failure_table): #) extractor = BuildMetadataExtractor( - ORG_NAME, PROJECT_NAME, RUN_ID, GITHUB_TOKEN, ARTIFACTS_URL, BUILD_LOGS_URL, OUTPUT_FILE + ORG_NAME, PROJECT_NAME, RUN_ID, GITHUB_TOKEN, MANIFEST_FILE, ARTIFACTS_URL, BUILD_LOGS_URL, OUTPUT_FILE ) # Process the manifest file - #manifest_data = extractor.read_manifest_file() - #if manifest_data: - #submodule_table = extractor.extract_submodule_table(manifest_data) - manifest_artifacts_table = extractor.generate_manifest_artifact_logs_table() - failure_table = extractor.list_failures() - - # Save results to an output file - #extractor.save_results_to_file(submodule_table, manifest_artifacts_table, failure_table) - extractor.save_results_to_file(manifest_artifacts_table, failure_table) + manifest_data = extractor.read_manifest_file() + if manifest_data: + submodule_table = extractor.extract_submodule_table(manifest_data) + manifest_artifacts_table = extractor.generate_manifest_artifact_logs_table() + failure_table = extractor.list_failures() + + # Save results to an output file + extractor.save_results_to_file(submodule_table, manifest_artifacts_table, failure_table) diff --git a/.github/workflows/multi_arch_build_portable_linux.yml b/.github/workflows/multi_arch_build_portable_linux.yml index 6d3da2c15f3de..3b661a13cead7 100644 --- a/.github/workflows/multi_arch_build_portable_linux.yml +++ b/.github/workflows/multi_arch_build_portable_linux.yml @@ -285,3 +285,33 @@ jobs: permissions: contents: read id-token: write + + # ========================================================================== + # STAGE: Manifest commit log (generic) + # ========================================================================== + manifest_summary: + needs: [compiler-runtime, foundation, math-libs, comm-libs] + if: ${{ !cancelled() && !failure() }} + runs-on: ubuntu-latest + steps: + - name: Download all stage submodule commit + uses: actions/download-artifact@v4 + with: + pattern: submodule-commit-* + merge-multiple: true + + - name: Merge all submodule commit + run: | + for f in submodule-commit-*.txt; do + echo "--- $f ---" + cat "$f" + done + + cat submodule-commit-*.txt | sort -u > manifest.txt + + cat manifest.txt + + - uses: actions/upload-artifact@v4 + with: + name: submodule-commit-all + path: manifest.txt diff --git a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml index de41d8748b6c5..f0a8ef35427eb 100644 --- a/.github/workflows/multi_arch_build_portable_linux_artifacts.yml +++ b/.github/workflows/multi_arch_build_portable_linux_artifacts.yml @@ -221,7 +221,19 @@ jobs: - name: Fetch sources timeout-minutes: 30 - run: ./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1 + run: | + commit_log=$(./build_tools/fetch_sources.py --stage ${STAGE_NAME} --jobs 12 --depth 1) + echo "$commit_log" | grep "^Submodule path" | \ + grep "checked out" | sed "s/Submodule path '\([^']*\)': checked out '\([^']*\)'/\1 \2/" \ + > submodule-commit-${{ inputs.stage_name }}.txt + + - name: Upload submodule commit file + uses: actions/upload-artifact@v4 + with: + name: submodule-commit-${{ inputs.stage_name }} + path: submodule-commit-${{ inputs.stage_name }}.txt + overwrite: true + retention-days: 1 - name: TheRock and llvm SHA if: ${{ github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && inputs.test_type != 'full') }} diff --git a/.github/workflows/teams_notifier.yml b/.github/workflows/teams_notifier.yml index 62d9b1ea4e501..08fa74c3003a5 100644 --- a/.github/workflows/teams_notifier.yml +++ b/.github/workflows/teams_notifier.yml @@ -18,6 +18,10 @@ jobs: with: sparse-checkout: .github + - uses: actions/download-artifact@v4 + with: + name: submodule-commit-all + - name: Calculate URLs and Fetch Manifest id: extract_urls env: @@ -41,7 +45,7 @@ jobs: curl -H "Authorization: token $GITHUB_TOKEN" -L "$LOG_URL" -o logs.txt # Parse URLs from the logs using grep - THE_ROCK_MANIFEST_URL=$(grep -oP '\[TheRock Manifest\]\(\K[^\)]+' "logs.txt" || echo "") + #THE_ROCK_MANIFEST_URL=$(grep -oP '\[TheRock Manifest\]\(\K[^\)]+' "logs.txt" || echo "") #ARTIFACTS_URL=$(grep -oP '\[Artifacts\]\(\K[^\)]+' "logs.txt" || echo "") #BUILD_LOGS_URL=$(grep -oP '\[Build Logs\]\(\K[^\)]+' "logs.txt" || echo "") ARTIFACTS_URL="https://therock-ci-artifacts-external.s3.amazonaws.com/ROCm-llvm-project/$RUN_ID-linux/index.html" @@ -65,7 +69,7 @@ jobs: echo "ORG_NAME=ROCm" >> $GITHUB_ENV echo "PROJECT_NAME=llvm-project" >> $GITHUB_ENV echo "RUN_ID=${{ github.run_id }}" >> $GITHUB_ENV - echo "MANIFEST_FILE=manifest.json" >> $GITHUB_ENV + echo "MANIFEST_FILE=manifest.txt" >> $GITHUB_ENV echo "OUTPUT_FILE=results.json" >> $GITHUB_ENV - name: Execute Python script for Notification Content @@ -74,7 +78,7 @@ jobs: ORG_NAME: ${{ env.ORG_NAME }} PROJECT_NAME: ${{ env.PROJECT_NAME }} RUN_ID: ${{ env.RUN_ID }} - #MANIFEST_FILE: ${{ env.MANIFEST_FILE }} + MANIFEST_FILE: ${{ env.MANIFEST_FILE }} #ROCK_MANIFEST_URL: ${{ env.THE_ROCK_MANIFEST_URL }} ARTIFACTS_URL: ${{ env.ARTIFACTS_URL }} BUILD_LOGS_URL: ${{ env.BUILD_LOGS_URL }}