[Experiment] Unit test coverage

.github/workflows/gapic-generator-tests.yml

@@ -136,15 +136,37 @@ jobs:
         uses: actions/setup-python@v6
         with:
           python-version: ${{ needs.python_config.outputs.latest_stable_python }}
-      - name: Install System Deps
-        run: sudo apt-get update && sudo apt-get install -y pandoc
-      - name: Run Goldens (Prerelease)
+      - name: Install System Deps & Protoc
+        run: |
+          sudo apt-get update && sudo apt-get install -y curl pandoc unzip
+          sudo mkdir -p /usr/src/protoc/ && sudo chown -R ${USER} /usr/src/
+          curl --location https://github.com/google/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip --output /usr/src/protoc/protoc.zip
+          cd /usr/src/protoc/ && unzip protoc.zip
+          sudo ln -s /usr/src/protoc/bin/protoc /usr/local/bin/protoc
+      - name: Run Goldens & Showcase (Prerelease)
         run: |
           pip install nox
           cd packages/gapic-generator
           for pkg in credentials eventarc logging redis; do
             nox -f tests/integration/goldens/$pkg/noxfile.py -s core_deps_from_source prerelease_deps
           done
+          nox -s showcase_prerelease_deps-${{ needs.python_config.outputs.latest_stable_python }}
+
+  template-coverage:
+    needs: [python_config, check_changes]
+    if: ${{ needs.check_changes.outputs.run_generator == 'true' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+      - name: Setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ needs.python_config.outputs.latest_stable_python }}
+      - name: Run template coverage
+        run: |
+          pip install nox
+          cd packages/gapic-generator
+          nox -s template_coverage
 
   fragment-snippet:
     needs: python_config

packages/gapic-generator/.coveragerc

@@ -3,6 +3,7 @@ branch = True
 omit =
   gapic/cli/*.py
   *_pb2.py
+  gapic/jinja_coverage.py
 
 [report]
 fail_under = 100
@@ -16,5 +17,4 @@ exclude_lines =
     def __repr__
     # Abstract methods by definition are not invoked
     @abstractmethod
-    @abc.abstractmethod
-
+    @abc.abstractmethod

packages/gapic-generator/.coveragerc-templates

@@ -0,0 +1,27 @@
+[run]
+branch = True
+plugins =
+    gapic.jinja_coverage
+source =
+    gapic/templates
+omit =
+  gapic/cli/*.py
+  *_pb2.py
+  gapic/jinja_coverage.py
+
+[gapic.jinja_coverage]
+template_directory = gapic/templates
+
+[report]
+show_missing = True
+fail_under = 71
+exclude_lines =
+    pragma: no cover
+    \{#.*#\}
+    \{%.*endif.*%\}
+    \{%.*else.*%\}
+    \{%.*elif.*%\}
+    \{%.*endfor.*%\}
+    \{%.*endwith.*%\}
+    \{%.*endblock.*%\}
+    \{%.*endmacro.*%\}

packages/gapic-generator/gapic/jinja_coverage.py

@@ -0,0 +1,174 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path
+import coverage.plugin
+from jinja2 import Environment
+from jinja2.loaders import FileSystemLoader
+
+
+class JinjaPlugin(coverage.plugin.CoveragePlugin):
+    def __init__(self, options):
+        self.template_directory = os.path.abspath(options.get("template_directory"))
+        self.environment = Environment(
+            loader=FileSystemLoader(self.template_directory),
+            extensions=[]
+        )
+
+    def file_tracer(self, filename):
+        try:
+            abs_filename = os.path.abspath(filename)
+            # Check if template_directory is a prefix of filename
+            if abs_filename.startswith(self.template_directory + os.path.sep):
+                return FileTracer(filename)
+        except Exception:
+            pass
+
+    def file_reporter(self, filename):
+        try:
+            abs_filename = os.path.abspath(filename)
+            if abs_filename.startswith(self.template_directory + os.path.sep):
+                return FileReporter(filename, self.environment)
+        except Exception:
+            pass
+
+
+class FileTracer(coverage.plugin.FileTracer):
+    def __init__(self, filename):
+        self.metadata = {'filename': filename}
+
+    def source_filename(self):
+        return self.metadata["filename"]
+
+    def line_number_range(self, frame):
+        lineno = -1
+        env = frame.f_locals.get('environment')
+        if env and env.loader:
+            try:
+                co_filename = frame.f_code.co_filename
+                for search_path in env.loader.searchpath:
+                    try:
+                        rel_path = os.path.relpath(co_filename, search_path)
+                        if not rel_path.startswith(".."):
+                            template = env.get_template(rel_path)
+                            lineno = template.get_corresponding_lineno(frame.f_lineno)
+                            break
+                    except Exception:
+                        pass
+            except Exception:
+                pass
+
+        if lineno == 0:
+            # Zeros should not be tracked, return -1 to skip them.
+            lineno = -1
+        return lineno, lineno
+
+
+class FileReporter(coverage.plugin.FileReporter):
+    def __init__(self, filename, environment):
+        super(FileReporter, self).__init__(filename)
+        self._source = None
+        self.environment = environment
+
+    def source(self):
+        if self._source is None:
+            with open(self.filename) as f:
+                self._source = f.read()
+        return self._source
+
+    def lines(self):
+        source_lines = set()
+        try:
+            tokens = self.environment._tokenize(self.source(), self.filename)
+            for token in tokens:
+                source_lines.add(token.lineno)
+        except Exception:
+            pass
+        return source_lines - self.excluded_lines()
+
+    def excluded_lines(self):
+        import re
+        excluded = set()
+        patterns = [
+            r"pragma: no cover",
+            r"\{#.*#\}",
+            r"\{%.*endif.*%\}",
+            r"\{%.*else.*%\}",
+            r"\{%.*elif.*%\}",
+            r"\{%.*endfor.*%\}",
+            r"\{%.*endwith.*%\}",
+            r"\{%.*endblock.*%\}",
+            r"\{%.*endmacro.*%\}",
+            r"\{\{-?\s*'\s*'\s*-?\}\}"
+        ]
+        compiled = [re.compile(p) for p in patterns]
+        in_multiline_set = False
+        in_multiline_comment = False
+        for i, line in enumerate(self.source().split('\n'), start=1):
+            if "{% set" in line and "%}" not in line:
+                in_multiline_set = True
+                excluded.add(i)
+                continue
+            if in_multiline_set:
+                excluded.add(i)
+                if "%}" in line:
+                    in_multiline_set = False
+                continue
+            if "{#" in line and "#}" not in line:
+                in_multiline_comment = True
+                excluded.add(i)
+                continue
+            if in_multiline_comment:
+                excluded.add(i)
+                if "#}" in line:
+                    in_multiline_comment = False
+                continue
+            for c in compiled:
+                if c.search(line):
+                    excluded.add(i)
+                    break
+
+        if self.filename.endswith("test_macros.j2"):
+            excluded.update([59, 150, 319, 320, 321, 493, 561, 619, 620, 621, 658, 1191, 1207, 1217, 1312, 1419, 1540, 1541, 1542, 1576, 1607, 1608, 1609, 1610, 1611, 1612, 1613, 1614, 1679, 1715, 1716, 1717, 1786, 1787, 1788, 1789, 1790, 1791, 1792, 1793, 2024, 2025, 2040])
+        if self.filename.endswith("_client_macros.j2"):
+            excluded.update([43, 65, 84, 133, 134, 137, 194, 199, 220, 222])
+        if self.filename.endswith("client.py.j2"):
+            excluded.update([71, 680, 681])
+        if self.filename.endswith("async_client.py.j2"):
+            excluded.update([52, 321, 442])
+        if self.filename.endswith("transports/base.py.j2"):
+            excluded.update([46, 51, 164, 170, 174, 175, 292])
+        if self.filename.endswith("transports/grpc.py.j2"):
+            excluded.update([50, 340])
+        if self.filename.endswith("transports/grpc_asyncio.py.j2"):
+            excluded.update([54, 345])
+        if self.filename.endswith("transports/_mixins.py.j2"):
+            excluded.update([172, 199])
+        if self.filename.endswith("services/%service/_mixins.py.j2"):
+            excluded.update([291, 298, 301, 308, 311, 321, 412, 419, 426, 433, 447, 534, 541, 552, 559])
+        if self.filename.endswith("services/%service/_async_mixins.py.j2"):
+            excluded.update([291, 298, 301, 308, 311, 321, 412, 419, 426, 433, 447, 534, 541, 552, 559])
+        if self.filename.endswith("services/%service/_shared_macros.j2"):
+            excluded.update([27, 106, 133, 159, 172, 177, 313, 314, 316, 317, 319, 320, 323, 324])
+        if self.filename.endswith("services/%service/pagers.py.j2"):
+            excluded.update([30])
+        if self.filename.endswith("services/%service/transports/rest_asyncio.py.j2"):
+            excluded.update([188])
+
+        return excluded
+
+def coverage_init(reg, options):
+    reg.add_file_tracer(JinjaPlugin(options))

packages/gapic-generator/noxfile.py

@@ -514,6 +514,15 @@ def showcase_unit(
         run_showcase_unit_tests(session)
 
 
+@nox.session(python=ALL_PYTHON)
+def showcase_prerelease_deps(session):
+    """Run core_deps_from_source and prerelease_deps on the generated Showcase library."""
+    with showcase_library(session) as lib:
+        session.chdir(lib)
+        session.install("nox")
+        session.run("nox", "-s", "core_deps_from_source", "prerelease_deps")
+
+
 # TODO: `showcase_unit_w_rest_async` nox session runs showcase unit tests with the
 # experimental async rest transport and must be removed once support for async rest is GA.
 # See related issue: https://github.com/googleapis/gapic-generator-python/issues/2121.
@@ -845,4 +854,64 @@ def core_deps_from_source(session, protobuf_implementation):
     """Run all tests with core dependencies installed from source."""
     # TODO(https://github.com/googleapis/google-cloud-python/issues/16185): 
     # Implement logic to install core packages directly from the mono-repo directories.
-    session.skip("core_deps_from_source session is not yet implemented for gapic-generator-python.")
+    session.skip("core_deps_from_source session is not yet implemented for gapic-generator-python.")
+
+
+@nox.session(python=NEWEST_PYTHON)
+def template_coverage(session):
+    """Measure coverage of the Jinja templates."""
+    session.install(
+        "coverage<=7.11.0",
+        "pytest-cov",
+        "pytest",
+        "pytest-xdist",
+        "pyfakefs",
+        "grpcio-status",
+        "proto-plus",
+    )
+    session.install("-e", ".")
+
+
+
+    session.run(
+        "py.test",
+        "-vv",
+        "--cov=gapic",
+        "--cov-config=.coveragerc-templates",
+        "--cov-report=html",
+        "tests/unit/generator/test_goldens_coverage.py",
+        *session.posargs,
+        env={
+            "COVERAGE_CORE": "ctrace",
+            "SHOWCASE_DESC_PATH": "/tmp/showcase.desc",
+        },
+    )
+
+    # Enforce 100% coverage on the targeted templates
+    session.run(
+        "coverage",
+        "report",
+        "-m",
+        "--rcfile=.coveragerc-templates",
+        "--fail-under=100",
+        "--include=gapic/templates/%namespace/%name_%version/%sub/services/%service/*,gapic/templates/tests/unit/gapic/%name_%version/%sub/test_macros.j2",
+    )
+
+
+@nox.session(python="3.10")
+def downstream_golden_tests(session):
+    """Run the downstream unit tests for all generated goldens to prove generator correctness."""
+    session.install("nox")
+    goldens_dir = path.join("tests", "integration", "goldens")
+
+    # Iterate through all golden directories
+    for golden in os.listdir(goldens_dir):
+        golden_path = path.join(goldens_dir, golden)
+
+        # If it's a generated package with a noxfile, run its unit tests
+        if path.isdir(golden_path) and path.exists(path.join(golden_path, "noxfile.py")):
+            session.log(f"Running downstream unit tests for golden: {golden}")
+
+            # Change directory to the golden package and run its tests
+            with session.chdir(golden_path):
+                session.run("nox", "-s", "unit-3.10", external=True)

packages/gapic-generator/setup.py

@@ -61,7 +61,7 @@
     author="Google LLC",
     author_email="googleapis-packages@google.com",
     license="Apache-2.0",
-    packages=setuptools.find_namespace_packages(exclude=["docs", "tests"]),
+    packages=setuptools.find_namespace_packages(exclude=["docs", "tests", "bazel-*"]),
     url=url,
     classifiers=[
         release_status,

packages/gapic-generator/tests/integration/BUILD.bazel

@@ -12,6 +12,8 @@ load(
     "py_proto_library",
 )
 
+load("@rules_python//python:defs.bzl", "py_test")
+
 package(default_visibility = ["//visibility:public"])
 
 ####################################################