ABI Mapper Improvements

.gitattributes

@@ -1 +1,2 @@
 *.zig text=auto eol=lf
+*.abi text=auto eol=lf

.github/workflows/build.yml

@@ -3,6 +3,7 @@ name: Build
 env:
   ZIG_VERSION: "0.15.2"
   ZIG_GLOBAL_CACHE_DIR: ${{ github.workspace }}/.zig-global-cache
+  ZIG_LOCAL_CACHE_DIR: ${{ github.workspace }}/.zig-cache
 
 on:
   push:
@@ -34,12 +35,18 @@ jobs:
         with:
           version: ${{ env.ZIG_VERSION }}
 
+      - name: Compute Zig package cache key
+        id: zig-pkg-cache-key
+        shell: bash
+        run: |
+          python scripts/zig_package_cache_key.py >> "$GITHUB_OUTPUT"
+
       - name: Cache Zig package cache
         uses: actions/cache@v4
         with:
           # Zig stores downloaded packages in the p subdirectory of the global cache.
           path: ${{ env.ZIG_GLOBAL_CACHE_DIR }}/p
-          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ hashFiles('**/build.zig.zon') }}
+          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ steps.zig-pkg-cache-key.outputs.key }}
           restore-keys: |
             zig-pkg-cache-${{ env.ZIG_VERSION }}-
           enableCrossOsArchive: true
@@ -48,6 +55,16 @@ jobs:
         run: |
           zig build --global-cache-dir "$ZIG_GLOBAL_CACHE_DIR" ${{ matrix.platform }}
 
+      - name: Upload .zig-cache on failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() && (matrix.os == 'windows-latest' || matrix.os == 'macos-latest') }}
+        with:
+          name: zig-cache-build-${{ matrix.os }}-${{ matrix.platform }}
+          path: ${{ env.ZIG_LOCAL_CACHE_DIR }}
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 7
+
       - name: Upload disk image
         uses: actions/upload-artifact@v4
         if: ${{ matrix.os == 'ubuntu-latest' }}
@@ -73,20 +90,42 @@ jobs:
         with:
           version: ${{ env.ZIG_VERSION }}
 
+      - name: Compute Zig package cache key
+        id: zig-pkg-cache-key
+        shell: bash
+        run: |
+          python scripts/zig_package_cache_key.py >> "$GITHUB_OUTPUT"
+
       - name: Cache Zig package cache
         uses: actions/cache@v4
         with:
           # Zig stores downloaded packages in the p subdirectory of the global cache.
           path: ${{ env.ZIG_GLOBAL_CACHE_DIR }}/p
-          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ hashFiles('**/build.zig.zon') }}
+          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ steps.zig-pkg-cache-key.outputs.key }}
           restore-keys: |
             zig-pkg-cache-${{ env.ZIG_VERSION }}-
           enableCrossOsArchive: true
 
+      - name: Install Linux GUI dependencies
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libgtk-3-dev
+
       - name: Build Tools
         run: |
           zig build --global-cache-dir "$ZIG_GLOBAL_CACHE_DIR" tools
 
+      - name: Upload .zig-cache on failure
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() && (matrix.os == 'windows-latest' || matrix.os == 'macos-latest') }}
+        with:
+          name: zig-cache-tools-${{ matrix.os }}
+          path: ${{ env.ZIG_LOCAL_CACHE_DIR }}
+          include-hidden-files: true
+          if-no-files-found: warn
+          retention-days: 7
+
       - name: Upload tools
         uses: actions/upload-artifact@v4
         with:

.github/workflows/pages.yml

@@ -38,12 +38,18 @@ jobs:
         with:
           version: ${{ env.ZIG_VERSION }}
 
+      - name: Compute Zig package cache key
+        id: zig-pkg-cache-key
+        shell: bash
+        run: |
+          python scripts/zig_package_cache_key.py >> "$GITHUB_OUTPUT"
+
       - name: Cache Zig package cache
         uses: actions/cache@v4
         with:
           # Zig stores downloaded packages in the p subdirectory of the global cache.
           path: ${{ env.ZIG_GLOBAL_CACHE_DIR }}/p
-          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ hashFiles('**/build.zig.zon') }}
+          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ steps.zig-pkg-cache-key.outputs.key }}
           restore-keys: |
             zig-pkg-cache-${{ env.ZIG_VERSION }}-
           enableCrossOsArchive: true

.github/workflows/smoketest.yml

@@ -44,20 +44,26 @@ jobs:
         with:
           version: ${{ env.ZIG_VERSION }}
 
+      - name: Compute Zig package cache key
+        id: zig-pkg-cache-key
+        shell: bash
+        run: |
+          python scripts/zig_package_cache_key.py >> "$GITHUB_OUTPUT"
+
       - name: Cache Zig package cache
         uses: actions/cache@v4
         with:
           # Zig stores downloaded packages in the p subdirectory of the global cache.
           path: ${{ env.ZIG_GLOBAL_CACHE_DIR }}/p
-          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ hashFiles('**/build.zig.zon') }}
+          key: zig-pkg-cache-${{ env.ZIG_VERSION }}-${{ steps.zig-pkg-cache-key.outputs.key }}
           restore-keys: |
             zig-pkg-cache-${{ env.ZIG_VERSION }}-
           enableCrossOsArchive: true
 
       - name: Install QEMU
         uses: awalsh128/cache-apt-pkgs-action@latest
         with:
-          packages: qemu-system
+          packages: qemu-system libgtk-3-dev
           version: 1.1
 
       - name: Build ${{ matrix.platform.kernel }}

build.zig

@@ -54,6 +54,10 @@ const installed_tools: []const ToolDep = &.{
         .dependency = "emulator",
         .artifacts = &.{ "emulator-web", "emulator" },
     },
+    .{
+        .dependency = "abi_mapper",
+        .artifacts = &.{"abi-parser"},
+    },
     // .{
     //     .dependency = "agp_tester",
     //     .artifacts = &.{"agp-tester"},

build.zig.zon

@@ -40,6 +40,9 @@
         .agp_tester = .{
             .path = "src/tools/agp-tester",
         },
+        .abi_mapper = .{
+            .path = "src/tools/abi-mapper",
+        },
         // .wikitool = .{
         //     .path = "src/tools/wikitool",
         // },

scripts/zig_package_cache_key.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+
+import hashlib
+import pathlib
+import subprocess
+import sys
+
+
+digest = hashlib.sha256()
+entries: set[str] = set()
+
+git_files = subprocess.run(
+    ["git", "ls-files", "-z"],
+    check=True,
+    capture_output=True,
+).stdout.split(b"\0")
+
+manifest_paths = sorted(
+    pathlib.Path(raw.decode("utf-8"))
+    for raw in git_files
+    if raw and (raw == b"build.zig.zon" or raw.endswith(b"/build.zig.zon"))
+)
+
+for path in manifest_paths:
+    print(path.as_posix(), file=sys.stderr)
+
+for path in manifest_paths:
+    for line in path.read_text().splitlines():
+        stripped = line.strip()
+        if stripped.startswith("//"):
+            continue
+
+        for key in (".url", ".hash"):
+            prefix = f"{key} ="
+            if not stripped.startswith(prefix):
+                continue
+
+            _, value = stripped.split("=", 1)
+            value = value.strip()
+
+            assert value.startswith('"'), f"{path}: malformed {key} entry: {line!r}"
+            assert value.endswith('",'), f"{path}: malformed {key} entry: {line!r}"
+
+            entries.add(f"{key}={value}")
+
+for entry in sorted(entries):
+    digest.update(f"{entry}\n".encode("utf-8"))
+
+print(f"key={digest.hexdigest()}")

src/abi/build.zig

@@ -88,9 +88,26 @@ pub fn build(b: *std.Build) void {
 
     const abi_tests_run = b.addRunArtifact(abi_tests_exe);
     test_step.dependOn(&abi_tests_run.step);
+
+    const escaping_tests_json = abi_mapper.get_json_dump(null, b.path("tests/escaping.abi"));
+
+    for (std.enums.values(ConversionMode)) |mode| {
+        const escaping_tests_zig = convert_abi_file(b, render_zig_exe, escaping_tests_json, null, mode);
+
+        const escaping_tests_exe = b.addTest(.{
+            .root_module = b.createModule(.{
+                .root_source_file = escaping_tests_zig,
+                .target = b.graph.host,
+                .optimize = .Debug,
+            }),
+        });
+        const escaping_tests_run = b.addRunArtifact(escaping_tests_exe);
+        test_step.dependOn(&escaping_tests_run.step);
+    }
 }
 
-pub fn convert_abi_file(b: *std.Build, render: *std.Build.Step.Compile, input: std.Build.LazyPath, patch: ?std.Build.LazyPath, mode: enum { kernel, definition }) std.Build.LazyPath {
+const ConversionMode = enum { kernel, definition };
+pub fn convert_abi_file(b: *std.Build, render: *std.Build.Step.Compile, input: std.Build.LazyPath, patch: ?std.Build.LazyPath, mode: ConversionMode) std.Build.LazyPath {
     const generate_core_abi = b.addRunArtifact(render);
     generate_core_abi.addArg(@tagName(mode));
     generate_core_abi.addFileArg(input);