From 7dad05a06d63279fc778def262be77b010008c6e Mon Sep 17 00:00:00 2001 From: swethasukumarr Date: Tue, 16 Jun 2026 23:40:00 -0400 Subject: [PATCH 1/2] RDKEMW-17673 : Add info coverage gate to CI --- .github/scripts/compare_coverage.py | 309 ++++++++ .github/scripts/compare_coverage_test.py | 907 +++++++++++++++++++++++ .github/workflows/ci.yml | 192 +++++ 3 files changed, 1408 insertions(+) create mode 100644 .github/scripts/compare_coverage.py create mode 100644 .github/scripts/compare_coverage_test.py diff --git a/.github/scripts/compare_coverage.py b/.github/scripts/compare_coverage.py new file mode 100644 index 0000000..a112cfc --- /dev/null +++ b/.github/scripts/compare_coverage.py @@ -0,0 +1,309 @@ +#!/usr/bin/env python3 +# If not stated otherwise in this file or this component's LICENSE file the +# following copyright and licenses apply: +# +# Copyright 2026 RDK Management +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Coverage comparison script for firebolt-cpp-client. + +Reads unit test (L0) and component test (L1) coverage results, compares overall +line coverage against the stored baseline from the build-metadata branch, and +prints a summary. +""" + +import argparse +import datetime +import json +import os +import sys +from typing import Optional + + +# Minimum threshold +THRESHOLD = 75.0 + +_GREEN = "\033[32m" +_RED = "\033[31m" +_RESET = "\033[0m" + +_SEP_WIDTH = 64 +_OVERALL_WIDTH = 80 +_SEP = "\u2500" * _SEP_WIDTH +_HEADER = "\u2500\u2500 Coverage Gate Report " + "\u2500" * (_SEP_WIDTH - 24) + + +def _colored(token: str, ok: bool) -> str: + return f"{_GREEN if ok else _RED}{token}{_RESET}" + + +def _fmt_timestamp(ts: str) -> str: + """Convert '2026-05-28T12:00:00Z' -> '2026-05-28 12:00 UTC'.""" + try: + dt = datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") + return dt.strftime("%Y-%m-%d %H:%M UTC") + except (ValueError, TypeError): + return ts + + +def _delta_str(current: float, baseline: float) -> str: + delta = current - baseline + sign = "+" if delta >= 0 else "" + return f"{sign}{delta:.2f}%" + + +def _join_names(names: list) -> str: + return names[0] if len(names) == 1 else " and ".join(names) + + +def _suite_analysis(current: Optional[float], baseline: Optional[float]): + """Analyse one test suite. + + Returns (ok, result_str, delta_disp, warn_reason): + ok - True when no advisory issues found. + result_str - Coloured [PASS]/[WARN] token + detail for the table. + delta_disp - String for the Delta column ("N/A" when skipped). + warn_reason - Reason phrase for the summary line; None when ok. + """ + if current is None: + reason = "coverage data missing" + return False, f"{_colored('[WARN]', False)} {reason}", "N/A", reason + + threshold_ok = current >= THRESHOLD + + if baseline is None: + # No baseline stored — threshold check only. + regression_ok = True + detail = "" if threshold_ok else "below threshold" + delta_disp = "N/A" + elif baseline == 0.0: + # A zero baseline is unreliable — skip regression check. + regression_ok = True + base_note = "baseline unreliable (0%) \u00b7 delta skipped" + detail = f"below threshold \u00b7 {base_note}" if not threshold_ok else base_note + delta_disp = "N/A" + else: + regression_ok = current >= baseline + delta_disp = _delta_str(current, baseline) + if threshold_ok and regression_ok: + detail = "" + elif not threshold_ok and not regression_ok: + detail = "below threshold \u00b7 dropped from baseline" + elif not threshold_ok: + detail = "below threshold \u00b7 no baseline regression" + else: + detail = "above threshold but dropped from baseline" + + overall_ok = threshold_ok and regression_ok + token = _colored("[PASS]", True) if overall_ok else _colored("[WARN]", False) + result_str = f"{token} {detail}" if detail else token + warn_reason = detail if not overall_ok else None + return overall_ok, result_str, delta_disp, warn_reason + + +def _build_summary(warn_suites: list) -> str: + """Build a compact summary from WARN suite (name, reason) pairs.""" + if not warn_suites: + return "" + groups: dict = {} + for name, reason in warn_suites: + groups.setdefault(reason, []).append(name) + parts = [f"{_join_names(names)} {reason}" for reason, names in groups.items()] + return ". ".join(parts) + + +# lcov parsing +def parse_lcov_coverage(path: str) -> Optional[float]: + """Return overall line coverage % from an lcov .info file, or None. + + An lcov .info file contains per-source-file records separated by + ``end_of_record``. Each record may include: + LF: — total instrumented lines in that file + LH: — lines executed at least once + + We aggregate across all records to produce a single project-wide %. + Returns None when the file is absent, empty, or contains no line data. + """ + if not path or not os.path.isfile(path): + return None + + total_found = 0 + total_hit = 0 + + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for raw in fh: + line = raw.strip() + if line.startswith("LF:"): + try: + total_found += int(line[3:]) + except ValueError: + pass + elif line.startswith("LH:"): + try: + total_hit += int(line[3:]) + except ValueError: + pass + except OSError as exc: + print(f" WARNING: Could not read {path}: {exc}", file=sys.stderr) + return None + + if total_found == 0: + return None + + return round((total_hit / total_found) * 100.0, 2) + + + +# Baseline loading +def load_baseline(path: str) -> dict: + """Load baseline JSON; return an empty dict on any error.""" + if not path or not os.path.isfile(path): + return {} + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + return data + print( + f" WARNING: Baseline {path} is not a JSON object (got {type(data).__name__}) — ignoring", + file=sys.stderr, + ) + except (OSError, json.JSONDecodeError, ValueError) as exc: + print(f" WARNING: Could not parse baseline {path}: {exc}", file=sys.stderr) + return {} + + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Compare L0/L1 coverage against the develop baseline.\n" + "Exits 1 when coverage fails threshold or regresses from baseline." + ) + ) + parser.add_argument("--baseline", required=True, metavar="PATH", + help="Path to coverage-baseline.json.") + parser.add_argument("--l0", required=False, metavar="PATH", + help="Path to the L0 lcov filtered_coverage.info file.") + parser.add_argument("--l1", required=False, metavar="PATH", + help="Path to the L1 lcov filtered_coverage.info file.") + parser.add_argument("--output-json", required=False, metavar="PATH", + help="Write {L0, L1, commit, timestamp} JSON here for baseline update.") + parser.add_argument("--commit", required=False, default="", + help="Commit SHA to embed in --output-json.") + parser.add_argument("--timestamp", required=False, default="", + help="ISO 8601 timestamp to embed in --output-json.") + args = parser.parse_args() + + baseline = load_baseline(args.baseline) + + def _coerce_pct(value: object) -> Optional[float]: + """Coerce a baseline percentage value to float, or None if invalid.""" + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + baseline_l0: Optional[float] = _coerce_pct(baseline.get("L0")) + baseline_l1: Optional[float] = _coerce_pct(baseline.get("L1")) + + l0_coverage = parse_lcov_coverage(args.l0) if args.l0 else None + l1_coverage = parse_lcov_coverage(args.l1) if args.l1 else None + + # ------------------------------------------------------------------ + # Optional: write extracted numbers for baseline update. + # Skipped (with a warning) when either suite lacks valid coverage data. + # ------------------------------------------------------------------ + if args.output_json: + if l0_coverage is not None and l1_coverage is not None: + payload = { + "L0": l0_coverage, + "L1": l1_coverage, + "commit": args.commit or "", + "timestamp": args.timestamp or "", + } + try: + with open(args.output_json, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + fh.write("\n") + except OSError as exc: + print(f" WARNING: Could not write {args.output_json}: {exc}", file=sys.stderr) + else: + print( + f" WARNING: --output-json skipped: coverage data incomplete " + f"(L0={l0_coverage}, L1={l1_coverage})", + file=sys.stderr, + ) + + l0_ok, l0_result, l0_delta, l0_reason = _suite_analysis(l0_coverage, baseline_l0) + l1_ok, l1_result, l1_delta, l1_reason = _suite_analysis(l1_coverage, baseline_l1) + + all_ok = l0_ok and l1_ok + status_token = _colored("[PASS]", True) if all_ok else _colored("[WARN]", False) + + + # Output report + print() + print(_HEADER) + if baseline: + commit = baseline.get("commit", "unknown") + ts = _fmt_timestamp(baseline.get("timestamp", "")) + print(f" Baseline {commit} ({ts})") + else: + print(" Baseline N/A (first-time setup \u2014 regression check skipped)") + print(f" Threshold {THRESHOLD}% | Status {status_token} (informational \u2014 PRs are not blocked)") + print(_SEP) + + # Coverage table + print(f" {'Suite':<7}{'Current':<9}{'Baseline':<10}{'Delta':<10}Result") + for name, current, base, result, delta_disp in [ + ("L0", l0_coverage, baseline_l0, l0_result, l0_delta), + ("L1", l1_coverage, baseline_l1, l1_result, l1_delta), + ]: + cur_str = f"{current:.2f}%" if current is not None else "N/A" + base_str = f"{base:.2f}%" if base is not None else "N/A" + print(f" {name:<7}{cur_str:<9}{base_str:<10}{delta_disp:<10}{result}") + + print(_SEP) + + # Summary + overall bar + warn_suites = [(n, r) for n, r in [("L0", l0_reason), ("L1", l1_reason)] if r] + summary = _build_summary(warn_suites) + if summary: + print(f" {summary}") + + # Notify when one or both suites had no coverage data (artifact absent). + # Gate logic is unchanged — SKIP is treated as passing by design. + skipped = [n for n, cov in [("L0", l0_coverage), ("L1", l1_coverage)] if cov is None] + if skipped: + print(f" NOTE: {_join_names(skipped)} coverage data absent \u2014 artifact missing or unreadable.") + + # " OVERALL: [PASS/WARN] " = 1 + 9 + 6 + 1 = 17 visible chars + # left + " OVERALL: " + token(6) + " " + right == _OVERALL_WIDTH + _mid = len(" OVERALL: ") + 6 + len(" ") # 17 + left = "\u2500" * ((_OVERALL_WIDTH - _mid) // 2) # 31 + right = "\u2500" * (_OVERALL_WIDTH - _mid - len(left)) # 32 + print(f"{left} OVERALL: {status_token} {right}") + print() + + # Informational only — always exit 0 so PRs are never blocked. + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/compare_coverage_test.py b/.github/scripts/compare_coverage_test.py new file mode 100644 index 0000000..41e0362 --- /dev/null +++ b/.github/scripts/compare_coverage_test.py @@ -0,0 +1,907 @@ +#!/usr/bin/env python3 +# If not stated otherwise in this file or this component's LICENSE file the +# following copyright and licenses apply: +# +# Copyright 2026 RDK Management +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Tests for compare_coverage.py + +Covers: + - Unit tests for parse_lcov_coverage(), load_baseline(), _suite_analysis() + - Integration tests (subprocess) simulating all gate scenarios listed in the + Coverage Gate implementation spec. + +Workflow-level scenarios (L0 job fails / L1 job fails / both fail) are handled +by GitHub Actions' implicit success() dependency check on the coverage-gate job +and cannot be tested at the Python script level; they are documented inline. +""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +# --------------------------------------------------------------------------- +# Import the module under test +# --------------------------------------------------------------------------- +SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPTS_DIR) + +import compare_coverage # noqa: E402 (after sys.path manipulation) + +THRESHOLD = compare_coverage.THRESHOLD # 75.0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_lcov(lines_found: int, lines_hit: int) -> str: + """Minimal valid lcov .info content with the given LF/LH counts.""" + return ( + "SF:src/fake.cpp\n" + f"LF:{lines_found}\n" + f"LH:{lines_hit}\n" + "end_of_record\n" + ) + + +def _write_lcov(tmp_dir: str, name: str, lines_found: int, lines_hit: int) -> str: + """Write an lcov file and return its absolute path.""" + path = os.path.join(tmp_dir, name) + with open(path, "w") as fh: + fh.write(_make_lcov(lines_found, lines_hit)) + return path + + +def _write_baseline(tmp_dir: str, data: dict, name: str = "baseline.json") -> str: + """Serialise *data* to JSON and return the path.""" + path = os.path.join(tmp_dir, name) + with open(path, "w") as fh: + json.dump(data, fh) + return path + + +def _run_script(*args: str) -> subprocess.CompletedProcess: + """Invoke compare_coverage.py as a subprocess and return the result.""" + cmd = [sys.executable, os.path.join(SCRIPTS_DIR, "compare_coverage.py"), *args] + return subprocess.run(cmd, capture_output=True, text=True) + + +# =========================================================================== +# Unit tests — parse_lcov_coverage() +# =========================================================================== + +class TestParseLcovCoverage(unittest.TestCase): + """Tests for the lcov .info parser.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, name: str, content: str) -> str: + path = os.path.join(self.tmp, name) + with open(path, "w") as fh: + fh.write(content) + return path + + # --- Missing / empty inputs ------------------------------------------------- + + def test_none_path_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage(None)) + + def test_empty_path_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage("")) + + def test_nonexistent_file_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage("/no/such/file.info")) + + def test_empty_file_returns_none(self): + p = self._write("empty.info", "") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_no_lf_data_returns_none(self): + p = self._write("no_lf.info", "SF:foo.cpp\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_lf_zero_returns_none(self): + p = self._write("zero_lf.info", "SF:foo.cpp\nLF:0\nLH:0\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + # --- Basic coverage values -------------------------------------------------- + + def test_100_percent(self): + p = _write_lcov(self.tmp, "full.info", 100, 100) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 100.0) + + def test_75_percent_exact(self): + p = _write_lcov(self.tmp, "seventy_five.info", 100, 75) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 75.0) + + def test_zero_percent(self): + p = _write_lcov(self.tmp, "zero_pct.info", 100, 0) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 0.0) + + def test_partial_coverage(self): + # 150 / 200 = 75.0 % + p = _write_lcov(self.tmp, "partial.info", 200, 150) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 75.0) + + # --- Multi-record aggregation ----------------------------------------------- + + def test_aggregates_multiple_records(self): + # 80 + 60 = 140 hit out of 200 → 70.0 % + content = ( + "SF:a.cpp\nLF:100\nLH:80\nend_of_record\n" + "SF:b.cpp\nLF:100\nLH:60\nend_of_record\n" + ) + p = self._write("multi.info", content) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 70.0) + + # --- Malformed data --------------------------------------------------------- + + def test_malformed_lf_ignored_gracefully(self): + # LF with a non-numeric value; total_found stays 0 → None + p = self._write("bad_lf.info", "SF:a.cpp\nLF:abc\nLH:50\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_malformed_lh_ignored_gracefully(self): + # LH with garbage value; LF is valid, so LF=100, LH=0 → 0.0 % + p = self._write("bad_lh.info", "SF:a.cpp\nLF:100\nLH:xyz\nend_of_record\n") + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 0.0) + + def test_entirely_non_lcov_content(self): + p = self._write("corrupt.info", "THIS IS NOT A VALID LCOV FILE\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + # --- Rounding --------------------------------------------------------------- + + def test_rounds_to_two_decimal_places(self): + # 1/3 ≈ 33.33 % + p = _write_lcov(self.tmp, "third.info", 3, 1) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 33.33) + + +# =========================================================================== +# Unit tests — load_baseline() +# =========================================================================== + +class TestLoadBaseline(unittest.TestCase): + """Tests for the JSON baseline loader.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write_json(self, name: str, content: str) -> str: + path = os.path.join(self.tmp, name) + with open(path, "w") as fh: + fh.write(content) + return path + + # --- Missing / empty inputs ------------------------------------------------- + + def test_none_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline(None), {}) + + def test_empty_path_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline(""), {}) + + def test_nonexistent_file_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline("/no/such/file.json"), {}) + + # --- Valid JSON ------------------------------------------------------------- + + def test_valid_baseline_with_l0_and_l1(self): + p = _write_baseline(self.tmp, {"L0": 80.0, "L1": 85.0}) + self.assertEqual(compare_coverage.load_baseline(p), {"L0": 80.0, "L1": 85.0}) + + def test_valid_empty_json_object(self): + p = _write_baseline(self.tmp, {}) + self.assertEqual(compare_coverage.load_baseline(p), {}) + + def test_valid_baseline_extra_keys_preserved(self): + data = {"L0": 80.0, "L1": 85.0, "commit": "abc123", "timestamp": "2026-01-01"} + p = _write_baseline(self.tmp, data) + self.assertEqual(compare_coverage.load_baseline(p), data) + + # --- Invalid JSON ----------------------------------------------------------- + + def test_malformed_json_returns_empty_dict(self): + p = self._write_json("bad.json", "{not valid json}") + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + + def test_truncated_json_returns_empty_dict(self): + p = self._write_json("truncated.json", '{"L0": 80') + self.assertEqual(compare_coverage.load_baseline(p), {}) + + def test_empty_file_returns_empty_dict(self): + p = self._write_json("empty.json", "") + self.assertEqual(compare_coverage.load_baseline(p), {}) + + # --- Non-dict JSON ---------------------------------------------------------- + + def test_json_array_returns_empty_dict(self): + import io, contextlib + p = self._write_json("list.json", "[1, 2, 3]") + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_string_returns_empty_dict(self): + import io, contextlib + p = self._write_json("str.json", '"just a string"') + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_number_returns_empty_dict(self): + import io, contextlib + p = self._write_json("num.json", "42") + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_null_returns_empty_dict(self): + p = self._write_json("null.json", "null") + # null is parsed as None, which is not a dict — Warning emitted, empty dict returned + self.assertEqual(compare_coverage.load_baseline(p), {}) + + +# =========================================================================== +# Unit tests — _suite_analysis() +# =========================================================================== + +class TestSuiteAnalysis(unittest.TestCase): + """ + Tests for the core gate analysis function. + + Gate passes (ok=True) when BOTH: + 1. current >= THRESHOLD (75.0) + 2. current >= baseline (regression check) + + SKIP when current is None. + Regression check disabled when baseline is None or 0.0. + """ + + # --- Scenario 1: exceeds both threshold AND baseline → PASS ---------------- + + def test_s1_exceeds_threshold_and_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(80.0, 77.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 2: meets threshold exactly (75%) AND beats baseline → PASS --- + + def test_s2_meets_threshold_exactly_beats_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 70.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 3: meets baseline exactly, exceeds threshold → PASS ---------- + + def test_s3_meets_baseline_exactly_exceeds_threshold(self): + ok, _, _, reason = compare_coverage._suite_analysis(80.0, 80.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 4: meets BOTH exactly (75.0 == threshold == baseline) → PASS - + + def test_s4_meets_both_exactly_at_threshold(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 75.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 5: exceeds threshold but BELOW baseline → FAIL (regression) -- + + def test_s5_above_threshold_below_baseline(self): + ok, result, _, reason = compare_coverage._suite_analysis(76.0, 80.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("dropped from baseline", reason) + + # --- Scenario 6: below threshold but meets/exceeds baseline → FAIL ---------- + + def test_s6_below_threshold_meets_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 70.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertNotIn("dropped from baseline", reason) # regression check passed + + def test_s6b_below_threshold_equals_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 74.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertNotIn("dropped from baseline", reason) # regression check passed + + # --- Scenario 7: fails BOTH conditions → FAIL -------------------------------- + + def test_s7_fails_both_conditions(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, 80.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertIn("dropped from baseline", reason) + + # --- Scenario 8: no baseline → threshold-only check ------------------------- + + def test_no_baseline_above_threshold_passes(self): + ok, _, delta, reason = compare_coverage._suite_analysis(80.0, None) + self.assertTrue(ok) + self.assertIsNone(reason) + self.assertEqual(delta, "N/A") + + def test_no_baseline_below_threshold_fails(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, None) + self.assertFalse(ok) + self.assertIn("below threshold", reason) + + def test_no_baseline_at_threshold_exactly_passes(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, None) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- SKIP case: no current coverage ----------------------------------------- + + def test_skip_when_current_is_none(self): + ok, result, delta, reason = compare_coverage._suite_analysis(None, 80.0) + self.assertFalse(ok, "Missing coverage data should be treated as WARN") + self.assertIn("coverage data missing", result) + self.assertEqual(delta, "N/A") + self.assertIsNotNone(reason) + self.assertIn("coverage data missing", reason) + + def test_skip_when_both_none(self): + ok, result, delta, reason = compare_coverage._suite_analysis(None, None) + self.assertFalse(ok) + self.assertIn("coverage data missing", result) + + # --- Zero baseline: regression check disabled -------------------------------- + + def test_zero_baseline_above_threshold_passes(self): + ok, _, delta, reason = compare_coverage._suite_analysis(80.0, 0.0) + self.assertTrue(ok) + self.assertIsNone(reason) + self.assertEqual(delta, "N/A", "Delta must be N/A for zero baseline") + + def test_zero_baseline_below_threshold_fails(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, 0.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + + # --- Delta string correctness ------------------------------------------------ + + def test_delta_positive(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, 77.0) + self.assertEqual(delta, "+3.00%") + + def test_delta_negative(self): + _, _, delta, _ = compare_coverage._suite_analysis(76.0, 80.0) + self.assertEqual(delta, "-4.00%") + + def test_delta_zero(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, 80.0) + self.assertEqual(delta, "+0.00%") + + def test_delta_na_when_no_baseline(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, None) + self.assertEqual(delta, "N/A") + + # --- Boundary: one tick below threshold (74.99 is impossible from lcov, + # but 74.0 covers the just-below case) ---------------------------------- + + def test_just_below_threshold_fails(self): + # 74 / 100 = 74.0 % + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 70.0) + self.assertFalse(ok) + + def test_just_at_threshold_passes(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 70.0) + self.assertTrue(ok) + + +# =========================================================================== +# Integration tests — main() via subprocess +# =========================================================================== + +class TestMainIntegration(unittest.TestCase): + """ + End-to-end simulation of every gate scenario. + + Each test invokes the script as a subprocess (exactly as GitHub Actions + would) and asserts on exit code and stdout/stderr content. + """ + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + # --- Helpers ---------------------------------------------------------------- + + def _lcov(self, name: str, lf: int, lh: int) -> str: + return _write_lcov(self.tmp, name, lf, lh) + + def _baseline(self, data: dict, name: str = "baseline.json") -> str: + return _write_baseline(self.tmp, data, name) + + def _run(self, *args: str) -> subprocess.CompletedProcess: + return _run_script(*args) + + # =========================================================================== + # SCENARIO 1 — Coverage exceeds both threshold AND baseline + # Expected: Gate PASSES (exit 0), baseline updates + # =========================================================================== + + def test_s1_exceeds_threshold_and_baseline(self): + bl = self._baseline({"L0": 77.0, "L1": 78.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % + l1 = self._lcov("l1.info", 100, 82) # 82 % + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[PASS]", r.stdout) + + # =========================================================================== + # SCENARIO 2 — Coverage meets threshold exactly (75%) and meets baseline + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s2_meets_threshold_exactly_meets_baseline(self): + bl = self._baseline({"L0": 70.0, "L1": 70.0}) + l0 = self._lcov("l0.info", 100, 75) # 75.0 % + l1 = self._lcov("l1.info", 100, 75) # 75.0 % + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[PASS]", r.stdout) + + # =========================================================================== + # SCENARIO 3 — Meets baseline exactly but exceeds threshold + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s3_meets_baseline_exactly_exceeds_threshold(self): + bl = self._baseline({"L0": 80.0, "L1": 80.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % == baseline + l1 = self._lcov("l1.info", 100, 80) # 80 % == baseline + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 4 — Meets BOTH exactly (current == threshold == baseline == 75 %) + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s4_meets_both_exactly(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 75) + l1 = self._lcov("l1.info", 100, 75) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 5 — Exceeds threshold but falls BELOW baseline (regression) + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s5_above_threshold_below_baseline(self): + bl = self._baseline({"L0": 85.0, "L1": 85.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % < 85 % baseline + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + self.assertIn("dropped from baseline", r.stdout) + + # =========================================================================== + # SCENARIO 6 — Falls BELOW threshold but meets/exceeds baseline + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s6_below_threshold_meets_baseline(self): + bl = self._baseline({"L0": 70.0, "L1": 70.0}) + l0 = self._lcov("l0.info", 100, 74) # 74 % < 75 % threshold + l1 = self._lcov("l1.info", 100, 74) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + self.assertIn("below threshold", r.stdout) + + # =========================================================================== + # SCENARIO 7 — Fails BOTH conditions (below threshold AND below baseline) + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s7_fails_both_threshold_and_baseline(self): + bl = self._baseline({"L0": 85.0, "L1": 85.0}) + l0 = self._lcov("l0.info", 100, 70) # 70 % < threshold AND < baseline + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("below threshold", r.stdout) + self.assertIn("dropped from baseline", r.stdout) + + # =========================================================================== + # SCENARIO 8a — Baseline file is MISSING + # Expected: Graceful fallback; threshold-only check; no crash + # =========================================================================== + + def test_s8_baseline_missing_coverage_above_threshold(self): + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run( + "--baseline", "/nonexistent/coverage-baseline.json", + "--l0", l0, "--l1", l1, + ) + # No baseline → regression skipped → threshold pass → exit 0 + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s8_baseline_missing_coverage_below_threshold(self): + l0 = self._lcov("l0.info", 100, 70) # 70 % < 75 % + l1 = self._lcov("l1.info", 100, 70) + r = self._run( + "--baseline", "/nonexistent/coverage-baseline.json", + "--l0", l0, "--l1", l1, + ) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 9 — Baseline file contains invalid / malformed JSON + # Expected: Warning emitted, treated as empty baseline, gate continues + # =========================================================================== + + def test_s9_malformed_json_above_threshold(self): + path = os.path.join(self.tmp, "malformed.json") + with open(path, "w") as fh: + fh.write("{this is not json}") + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + # Warning must appear in stderr + self.assertIn("WARNING", r.stderr) + # Fallback to empty baseline → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s9_malformed_json_below_threshold(self): + path = os.path.join(self.tmp, "malformed2.json") + with open(path, "w") as fh: + fh.write("{bad json") + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + self.assertIn("WARNING", r.stderr) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0) + self.assertIn("[WARN]", r.stdout) + + def test_s9_empty_json_file(self): + path = os.path.join(self.tmp, "empty.json") + with open(path, "w") as fh: + fh.write("") + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + # Empty file → empty dict baseline → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s9_non_dict_json(self): + path = os.path.join(self.tmp, "list_json.json") + with open(path, "w") as fh: + json.dump([1, 2, 3], fh) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + # Non-dict JSON → WARNING emitted, empty dict → threshold-only → pass + self.assertIn("WARNING", r.stderr) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 10 — unit_tests job fails + # Coverage Gate does NOT trigger (workflow-level behaviour). + # + # GitHub Actions: coverage_gate has `needs: [unit_tests, component_tests]` + # with no custom `if:`. The implicit success() check means coverage_gate + # is SKIPPED whenever unit_tests fails. This cannot be unit-tested here; + # it is enforced by the workflow graph. + # =========================================================================== + + def test_s10_l0_artifacts_absent_l1_passes(self): + """ + Simulates the artifact-level effect: L0 .info absent (download step + with continue-on-error:true produced no file), L1 coverage present + and passing. Script-level: L0 is WARN (data missing), L1 is PASS. + """ + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l1 = self._lcov("l1.info", 100, 80) # L0 omitted intentionally + r = self._run("--baseline", bl, "--l1", l1) + # L0 WARN (missing) → overall WARN, but exit 0 (informational) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("coverage data missing", r.stdout) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 11 — L1 job fails (symmetric to scenario 10) + # =========================================================================== + + def test_s11_l1_artifacts_absent_l0_passes(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) # L1 omitted intentionally + r = self._run("--baseline", bl, "--l0", l0) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("coverage data missing", r.stdout) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 12 — Both L0 AND L1 jobs fail + # Coverage Gate does NOT trigger (workflow-level). At script level, both + # .info files are absent → both SKIP → exit 0 (harmless; gate is already + # blocked at the workflow graph layer before the script is ever called). + # =========================================================================== + + def test_s12_both_artifacts_absent(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + # Neither --l0 nor --l1 provided + r = self._run("--baseline", bl) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + # Both rows + summary line show coverage data missing, OVERALL WARN + self.assertGreaterEqual(r.stdout.count("coverage data missing"), 2) + self.assertIn("[WARN]", r.stdout) + self.assertIn("NOTE:", r.stdout) + + # =========================================================================== + # SCENARIO 13 — Coverage Gate step itself throws an unexpected error + # Expected: non-zero exit; error is visible; baseline NOT updated + # (Simulated by passing a completely invalid path for --baseline that + # causes the argument parser or file logic to surface an error.) + # =========================================================================== + + def test_s13_missing_required_baseline_arg(self): + """Invoking the script without --baseline must fail (argparse error).""" + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--l0", l0, "--l1", l1) + # argparse exits with code 2 on missing required argument + self.assertNotEqual(r.returncode, 0) + self.assertTrue(len(r.stderr) > 0, "Error must appear on stderr") + + # =========================================================================== + # Partial-failure cases: one suite fails, other passes + # =========================================================================== + + def test_only_l0_fails_gate_warns(self): + """L0 below threshold, L1 passes → overall [WARN] but exit 0.""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 70) # 70 % ✗ + l1 = self._lcov("l1.info", 100, 80) # 80 % ✓ + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + # L1 row still shows PASS + self.assertIn("[PASS]", r.stdout) + self.assertIn("[WARN]", r.stdout) + + def test_only_l1_fails_gate_warns(self): + """L1 below threshold, L0 passes → overall [WARN] but exit 0.""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % ✓ + l1 = self._lcov("l1.info", 100, 70) # 70 % ✗ + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + def test_only_l0_regresses_gate_warns(self): + """L0 regresses below baseline (still above threshold), L1 passes → [WARN] exit 0.""" + bl = self._baseline({"L0": 85.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % < 85 % baseline ✗ + l1 = self._lcov("l1.info", 100, 80) # 80 % >= 75 % baseline ✓ + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # First-time setup: empty baseline {} → threshold-only + # =========================================================================== + + def test_first_time_setup_empty_baseline_passes(self): + bl = self._baseline({}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_first_time_setup_empty_baseline_below_threshold_warns(self): + bl = self._baseline({}) + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # Output format validation + # =========================================================================== + + def test_overall_pass_token_in_output(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertIn("OVERALL:", r.stdout) + self.assertIn("[PASS]", r.stdout) + + def test_overall_warn_token_in_output(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertIn("OVERALL:", r.stdout) + self.assertIn("[WARN]", r.stdout) + # Informational only — always exit 0 + self.assertEqual(r.returncode, 0) + + # =========================================================================== + # --output-json baseline extraction + # =========================================================================== + + def test_output_json_written_when_both_pass(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline.json") + self._run( + "--baseline", bl, + "--l0", l0, "--l1", l1, + "--output-json", out, + "--commit", "abc123", + "--timestamp", "2026-01-01T00:00:00Z", + ) + self.assertTrue(os.path.isfile(out), "output-json must be written") + with open(out) as fh: + data = json.load(fh) + self.assertEqual(data["L0"], 80.0) + self.assertEqual(data["L1"], 82.0) + self.assertEqual(data["commit"], "abc123") + self.assertEqual(data["timestamp"], "2026-01-01T00:00:00Z") + + def test_output_json_written_even_when_gate_fails(self): + """ + --output-json is written as long as coverage data is available, + regardless of gate outcome. The update-baseline step checks + `if [ ! -s new-baseline.json ]` separately. + """ + bl = self._baseline({"L0": 90.0, "L1": 90.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % < 90 % baseline → WARN + l1 = self._lcov("l1.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-fail.json") + r = self._run( + "--baseline", bl, + "--l0", l0, "--l1", l1, + "--output-json", out, + ) + # Informational only — always exit 0 regardless of gate outcome + self.assertEqual(r.returncode, 0) + self.assertTrue(os.path.isfile(out), "output-json written even on gate warning") + + def test_output_json_not_written_when_l0_missing(self): + """When L0 .info is absent, --output-json must NOT be written (data incomplete).""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l1 = self._lcov("l1.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline-no-l0.json") + r = self._run("--baseline", bl, "--l1", l1, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when L0 absent") + self.assertIn("WARNING", r.stderr) + + def test_output_json_not_written_when_l1_missing(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-no-l1.json") + r = self._run("--baseline", bl, "--l0", l0, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when L1 absent") + self.assertIn("WARNING", r.stderr) + + def test_output_json_not_written_when_both_missing(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + out = os.path.join(self.tmp, "new-baseline-neither.json") + r = self._run("--baseline", bl, "--output-json", out) + self.assertFalse(os.path.isfile(out)) + self.assertIn("WARNING", r.stderr) + + # =========================================================================== + # Baseline coercion: non-float L0/L1 values must not crash the script + # (Fixes comment 2/7 — baseline.get("L0") not validated as float) + # =========================================================================== + + def test_baseline_string_l0_treated_as_missing(self): + """String value for L0 in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"L0": "not-a-number", "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + # Must not crash; L0 baseline treated as absent → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_null_l1_treated_as_missing(self): + """null value for L1 in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"L0": 80.0, "L1": None}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_both_non_float_threshold_only(self): + """Both L0/L1 baseline values invalid → both threshold-only → pass if above 75%.""" + bl = self._baseline({"L0": "bad", "L1": "bad"}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_both_non_float_below_threshold_warns(self): + """Both L0/L1 baseline values invalid → threshold-only → [WARN] exit 0 if below 75%.""" + bl = self._baseline({"L0": "bad", "L1": "bad"}) + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # _fmt_timestamp: null/non-string timestamp must not crash the report + # (Fixes comment 8 — only ValueError was caught, not TypeError) + # =========================================================================== + + def test_null_timestamp_in_baseline_does_not_crash(self): + """null timestamp value in baseline JSON → TypeError handled → report still runs.""" + bl = self._baseline({"L0": 80.0, "L1": 80.0, "commit": "abc", "timestamp": None}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + # Must not crash; timestamp renders as fallback; gate passes + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("OVERALL:", r.stdout) + + def test_integer_timestamp_in_baseline_does_not_crash(self): + """Integer timestamp → TypeError in strptime → handled gracefully.""" + bl = self._baseline({"L0": 80.0, "L1": 80.0, "commit": "abc", "timestamp": 12345}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c7d124..338cccf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,6 +200,7 @@ jobs: --exclude '.*/test/.*\.cpp' \ --decisions \ --medium-threshold 50 --high-threshold 75 \ + --lcov coverage/filtered_coverage.info \ --html-details coverage/index.html \ --cobertura coverage.cobertura.xml \ " @@ -210,6 +211,12 @@ jobs: name: coverage-report path: ${{ github.workspace }}/build/coverage/ + - name: Upload unit test lcov coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-unit + path: ${{ github.workspace }}/build/coverage/filtered_coverage.info + - name: Code Coverage Summary Report uses: irongut/CodeCoverageSummary@v1.3.0 with: @@ -269,6 +276,25 @@ jobs: --app-openrpc /workspace/docs/openrpc/the-spec/firebolt-app-open-rpc.json \ --test-exe /workspace/build/test/ctApp + - name: Generate Coverage Report + run: | + docker run --rm --user "$(id -u):$(id -g)" -v ${{ github.workspace }}:/workspace ${{ needs.build_docker.outputs.image_tag }} \ + bash -c " \ + set -e \ + && cd build \ + && mkdir -p coverage \ + && gcovr -r .. \ + --exclude '.*/test/.*\.h' \ + --exclude '.*/test/.*\.cpp' \ + --lcov coverage/filtered_coverage.info \ + " + + - name: Upload component test lcov coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-component + path: ${{ github.workspace }}/build/coverage/filtered_coverage.info + api_test_app: permissions: contents: read @@ -324,3 +350,169 @@ jobs: --openrpc /workspace/docs/openrpc/the-spec/firebolt-open-rpc.json \ --app-openrpc /workspace/docs/openrpc/the-spec/firebolt-app-open-rpc.json \ --test-exe /workspace/test/api_test_app/build/api-test-app + + coverage_gate: + name: Coverage Gate + needs: [unit_tests, component_tests] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Fetch baseline from build-metadata branch + # Gracefully handle a missing build-metadata branch (first-time setup) + continue-on-error: true + run: | + set -euo pipefail + if git fetch origin build-metadata 2>/dev/null; then + if git cat-file -e FETCH_HEAD:coverage-baseline.json 2>/dev/null; then + git show FETCH_HEAD:coverage-baseline.json > coverage-baseline.json + echo "Loaded coverage-baseline.json from build-metadata branch" + else + echo "build-metadata branch exists but coverage-baseline.json not found — skipping baseline comparison" + echo "{}" > coverage-baseline.json + fi + else + echo "build-metadata branch not found — absolute threshold check only (first-time setup)" + echo "{}" > coverage-baseline.json + fi + + - name: Download unit test coverage artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: coverage-unit + path: ./unit-coverage + + - name: Download component test coverage artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: coverage-component + path: ./component-coverage + + - name: Compare coverage to baseline + run: | + python3 .github/scripts/compare_coverage.py \ + --baseline coverage-baseline.json \ + --l0 ./unit-coverage/filtered_coverage.info \ + --l1 ./component-coverage/filtered_coverage.info + + update_baseline: + name: Update Coverage Baseline + # Runs on push to develop when both test suites pass. + # Independent of coverage_gate — the gate is informational and must never + # block the baseline from reflecting the actual state of passing tests. + if: >- + github.event_name == 'push' && + github.ref == 'refs/heads/develop' && + needs.unit_tests.result == 'success' && + needs.component_tests.result == 'success' + needs: [unit_tests, component_tests] + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + # Queue concurrent runs; do not cancel in-progress — each merge deserves + # a baseline update and force-push is atomic so queuing is safe. + concurrency: + group: update-baseline-develop + cancel-in-progress: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Fetch existing baseline for comparison report + # Best-effort — if the branch or file is absent we compare against nothing. + continue-on-error: true + run: | + set -euo pipefail + if git fetch origin build-metadata 2>/dev/null; then + if git cat-file -e FETCH_HEAD:coverage-baseline.json 2>/dev/null; then + git show FETCH_HEAD:coverage-baseline.json > old-baseline.json + echo "Loaded existing baseline for comparison" + else + echo '{}' > old-baseline.json + fi + else + echo '{}' > old-baseline.json + fi + + - name: Download unit test coverage artifact + uses: actions/download-artifact@v4 + with: + name: coverage-unit + path: ./unit-coverage + + - name: Download component test coverage artifact + uses: actions/download-artifact@v4 + with: + name: coverage-component + path: ./component-coverage + + - name: Extract coverage and write new baseline + id: extract + run: | + set -euo pipefail + BL_ARG="old-baseline.json" + [ -f "$BL_ARG" ] || echo '{}' > "$BL_ARG" + + python3 .github/scripts/compare_coverage.py \ + --baseline "$BL_ARG" \ + --l0 ./unit-coverage/filtered_coverage.info \ + --l1 ./component-coverage/filtered_coverage.info \ + --output-json new-baseline.json \ + --commit "$GITHUB_SHA" \ + --timestamp "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + + if [ ! -s new-baseline.json ]; then + echo "Coverage extraction produced no output — skipping baseline update" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "New baseline to commit:" + cat new-baseline.json + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push updated baseline to build-metadata + if: steps.extract.outputs.skip == 'false' + run: | + set -euo pipefail + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + + # Check out the build-metadata branch, or create it as an orphan. + if git fetch origin build-metadata 2>/dev/null; then + git checkout -B build-metadata FETCH_HEAD + else + git checkout --orphan build-metadata + git rm -rf . --quiet 2>/dev/null || true + fi + + cp -f new-baseline.json coverage-baseline.json + + git add coverage-baseline.json + + # Only commit when there is an actual change. + if git diff --cached --quiet; then + echo "Coverage baseline unchanged — no commit needed" + else + MSG=$(printf \ + 'chore: update coverage baseline after develop merge [skip ci]\n\nCommit : %s\nRun ID : %s' \ + "$GITHUB_SHA" "$GITHUB_RUN_ID") + git commit -m "$MSG" + git push --force-with-lease origin build-metadata + echo "Pushed updated baseline to build-metadata" + fi From 8f3d2dd2b7d1eb9beab6b8d7d8d4db046d7b502b Mon Sep 17 00:00:00 2001 From: swethasukumarr Date: Tue, 23 Jun 2026 16:37:45 -0400 Subject: [PATCH 2/2] RDKEMW-17673 : Address copilot comments --- .github/scripts/compare_coverage.py | 66 +++-- .github/scripts/compare_coverage_test.py | 330 +++++++++++------------ .github/workflows/ci.yml | 11 +- 3 files changed, 203 insertions(+), 204 deletions(-) diff --git a/.github/scripts/compare_coverage.py b/.github/scripts/compare_coverage.py index a112cfc..9fb7699 100644 --- a/.github/scripts/compare_coverage.py +++ b/.github/scripts/compare_coverage.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -# If not stated otherwise in this file or this component's LICENSE file the -# following copyright and licenses apply: -# -# Copyright 2026 RDK Management +# Copyright 2026 Comcast Cable Communications Management, LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 """ Coverage comparison script for firebolt-cpp-client. -Reads unit test (L0) and component test (L1) coverage results, compares overall -line coverage against the stored baseline from the build-metadata branch, and -prints a summary. +Reads unit test and component test coverage results, compares overall line +coverage against the stored baseline from the build-metadata branch, and prints +a summary. """ import argparse @@ -190,18 +188,19 @@ def load_baseline(path: str) -> dict: def main() -> None: parser = argparse.ArgumentParser( description=( - "Compare L0/L1 coverage against the develop baseline.\n" - "Exits 1 when coverage fails threshold or regresses from baseline." + "Compare unit test and component test coverage against " + "the develop baseline. Informational only — always exits 0 and does " + "not block PRs." ) ) parser.add_argument("--baseline", required=True, metavar="PATH", help="Path to coverage-baseline.json.") - parser.add_argument("--l0", required=False, metavar="PATH", - help="Path to the L0 lcov filtered_coverage.info file.") - parser.add_argument("--l1", required=False, metavar="PATH", - help="Path to the L1 lcov filtered_coverage.info file.") + parser.add_argument("--unit", required=False, metavar="PATH", + help="Path to the unit test lcov filtered_coverage.info file.") + parser.add_argument("--component", required=False, metavar="PATH", + help="Path to the component test lcov filtered_coverage.info file.") parser.add_argument("--output-json", required=False, metavar="PATH", - help="Write {L0, L1, commit, timestamp} JSON here for baseline update.") + help="Write {Unit, Component, commit, timestamp} JSON here for baseline update.") parser.add_argument("--commit", required=False, default="", help="Commit SHA to embed in --output-json.") parser.add_argument("--timestamp", required=False, default="", @@ -219,22 +218,22 @@ def _coerce_pct(value: object) -> Optional[float]: except (TypeError, ValueError): return None - baseline_l0: Optional[float] = _coerce_pct(baseline.get("L0")) - baseline_l1: Optional[float] = _coerce_pct(baseline.get("L1")) + baseline_unit: Optional[float] = _coerce_pct(baseline.get("Unit")) + baseline_component: Optional[float] = _coerce_pct(baseline.get("Component")) - l0_coverage = parse_lcov_coverage(args.l0) if args.l0 else None - l1_coverage = parse_lcov_coverage(args.l1) if args.l1 else None + unit_coverage = parse_lcov_coverage(args.unit) if args.unit else None + component_coverage = parse_lcov_coverage(args.component) if args.component else None # ------------------------------------------------------------------ # Optional: write extracted numbers for baseline update. # Skipped (with a warning) when either suite lacks valid coverage data. # ------------------------------------------------------------------ if args.output_json: - if l0_coverage is not None and l1_coverage is not None: + if unit_coverage is not None and component_coverage is not None: payload = { - "L0": l0_coverage, - "L1": l1_coverage, - "commit": args.commit or "", + "Unit": unit_coverage, + "Component": component_coverage, + "commit": args.commit or "", "timestamp": args.timestamp or "", } try: @@ -246,17 +245,16 @@ def _coerce_pct(value: object) -> Optional[float]: else: print( f" WARNING: --output-json skipped: coverage data incomplete " - f"(L0={l0_coverage}, L1={l1_coverage})", + f"(Unit={unit_coverage}, Component={component_coverage})", file=sys.stderr, ) - l0_ok, l0_result, l0_delta, l0_reason = _suite_analysis(l0_coverage, baseline_l0) - l1_ok, l1_result, l1_delta, l1_reason = _suite_analysis(l1_coverage, baseline_l1) + unit_ok, unit_result, unit_delta, unit_reason = _suite_analysis(unit_coverage, baseline_unit) + component_ok, component_result, component_delta, component_reason = _suite_analysis(component_coverage, baseline_component) - all_ok = l0_ok and l1_ok + all_ok = unit_ok and component_ok status_token = _colored("[PASS]", True) if all_ok else _colored("[WARN]", False) - # Output report print() print(_HEADER) @@ -270,26 +268,26 @@ def _coerce_pct(value: object) -> Optional[float]: print(_SEP) # Coverage table - print(f" {'Suite':<7}{'Current':<9}{'Baseline':<10}{'Delta':<10}Result") + print(f" {'Suite':<12}{'Current':<9}{'Baseline':<10}{'Delta':<10}Result") for name, current, base, result, delta_disp in [ - ("L0", l0_coverage, baseline_l0, l0_result, l0_delta), - ("L1", l1_coverage, baseline_l1, l1_result, l1_delta), + ("Unit", unit_coverage, baseline_unit, unit_result, unit_delta), + ("Component", component_coverage, baseline_component, component_result, component_delta), ]: cur_str = f"{current:.2f}%" if current is not None else "N/A" base_str = f"{base:.2f}%" if base is not None else "N/A" - print(f" {name:<7}{cur_str:<9}{base_str:<10}{delta_disp:<10}{result}") + print(f" {name:<12}{cur_str:<9}{base_str:<10}{delta_disp:<10}{result}") print(_SEP) # Summary + overall bar - warn_suites = [(n, r) for n, r in [("L0", l0_reason), ("L1", l1_reason)] if r] + warn_suites = [(n, r) for n, r in [("Unit", unit_reason), ("Component", component_reason)] if r] summary = _build_summary(warn_suites) if summary: print(f" {summary}") # Notify when one or both suites had no coverage data (artifact absent). - # Gate logic is unchanged — SKIP is treated as passing by design. - skipped = [n for n, cov in [("L0", l0_coverage), ("L1", l1_coverage)] if cov is None] + # Missing data is reported as [WARN]; the gate remains informational. + skipped = [n for n, cov in [("Unit", unit_coverage), ("Component", component_coverage)] if cov is None] if skipped: print(f" NOTE: {_join_names(skipped)} coverage data absent \u2014 artifact missing or unreadable.") diff --git a/.github/scripts/compare_coverage_test.py b/.github/scripts/compare_coverage_test.py index 41e0362..73e956f 100644 --- a/.github/scripts/compare_coverage_test.py +++ b/.github/scripts/compare_coverage_test.py @@ -1,8 +1,5 @@ #!/usr/bin/env python3 -# If not stated otherwise in this file or this component's LICENSE file the -# following copyright and licenses apply: -# -# Copyright 2026 RDK Management +# Copyright 2026 Comcast Cable Communications Management, LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# SPDX-License-Identifier: Apache-2.0 """ Tests for compare_coverage.py @@ -24,7 +22,7 @@ - Integration tests (subprocess) simulating all gate scenarios listed in the Coverage Gate implementation spec. -Workflow-level scenarios (L0 job fails / L1 job fails / both fail) are handled +Workflow-level scenarios (unit_tests fails / component_tests fails / both fail) are handled by GitHub Actions' implicit success() dependency check on the coverage-gate job and cannot be tested at the Python script level; they are documented inline. """ @@ -213,16 +211,16 @@ def test_nonexistent_file_returns_empty_dict(self): # --- Valid JSON ------------------------------------------------------------- - def test_valid_baseline_with_l0_and_l1(self): - p = _write_baseline(self.tmp, {"L0": 80.0, "L1": 85.0}) - self.assertEqual(compare_coverage.load_baseline(p), {"L0": 80.0, "L1": 85.0}) + def test_valid_baseline_with_unit_and_component(self): + p = _write_baseline(self.tmp, {"Unit": 80.0, "Component": 85.0}) + self.assertEqual(compare_coverage.load_baseline(p), {"Unit": 80.0, "Component": 85.0}) def test_valid_empty_json_object(self): p = _write_baseline(self.tmp, {}) self.assertEqual(compare_coverage.load_baseline(p), {}) def test_valid_baseline_extra_keys_preserved(self): - data = {"L0": 80.0, "L1": 85.0, "commit": "abc123", "timestamp": "2026-01-01"} + data = {"Unit": 80.0, "Component": 85.0, "commit": "abc123", "timestamp": "2026-01-01"} p = _write_baseline(self.tmp, data) self.assertEqual(compare_coverage.load_baseline(p), data) @@ -234,7 +232,7 @@ def test_malformed_json_returns_empty_dict(self): self.assertEqual(result, {}) def test_truncated_json_returns_empty_dict(self): - p = self._write_json("truncated.json", '{"L0": 80') + p = self._write_json("truncated.json", '{"Unit": 80') self.assertEqual(compare_coverage.load_baseline(p), {}) def test_empty_file_returns_empty_dict(self): @@ -467,10 +465,10 @@ def _run(self, *args: str) -> subprocess.CompletedProcess: # =========================================================================== def test_s1_exceeds_threshold_and_baseline(self): - bl = self._baseline({"L0": 77.0, "L1": 78.0}) - l0 = self._lcov("l0.info", 100, 80) # 80 % - l1 = self._lcov("l1.info", 100, 82) # 82 % - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 77.0, "Component": 78.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % + component_cov = self._lcov("component.info", 100, 82) # 82 % + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[PASS]", r.stdout) @@ -480,10 +478,10 @@ def test_s1_exceeds_threshold_and_baseline(self): # =========================================================================== def test_s2_meets_threshold_exactly_meets_baseline(self): - bl = self._baseline({"L0": 70.0, "L1": 70.0}) - l0 = self._lcov("l0.info", 100, 75) # 75.0 % - l1 = self._lcov("l1.info", 100, 75) # 75.0 % - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 70.0, "Component": 70.0}) + unit_cov = self._lcov("unit.info", 100, 75) # 75.0 % + component_cov = self._lcov("component.info", 100, 75) # 75.0 % + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[PASS]", r.stdout) @@ -493,10 +491,10 @@ def test_s2_meets_threshold_exactly_meets_baseline(self): # =========================================================================== def test_s3_meets_baseline_exactly_exceeds_threshold(self): - bl = self._baseline({"L0": 80.0, "L1": 80.0}) - l0 = self._lcov("l0.info", 100, 80) # 80 % == baseline - l1 = self._lcov("l1.info", 100, 80) # 80 % == baseline - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 80.0, "Component": 80.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % == baseline + component_cov = self._lcov("component.info", 100, 80) # 80 % == baseline + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) # =========================================================================== @@ -505,10 +503,10 @@ def test_s3_meets_baseline_exactly_exceeds_threshold(self): # =========================================================================== def test_s4_meets_both_exactly(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 75) - l1 = self._lcov("l1.info", 100, 75) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 75) + component_cov = self._lcov("component.info", 100, 75) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) # =========================================================================== @@ -517,10 +515,10 @@ def test_s4_meets_both_exactly(self): # =========================================================================== def test_s5_above_threshold_below_baseline(self): - bl = self._baseline({"L0": 85.0, "L1": 85.0}) - l0 = self._lcov("l0.info", 100, 80) # 80 % < 85 % baseline - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 85.0, "Component": 85.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % < 85 % baseline + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[WARN]", r.stdout) self.assertIn("dropped from baseline", r.stdout) @@ -531,10 +529,10 @@ def test_s5_above_threshold_below_baseline(self): # =========================================================================== def test_s6_below_threshold_meets_baseline(self): - bl = self._baseline({"L0": 70.0, "L1": 70.0}) - l0 = self._lcov("l0.info", 100, 74) # 74 % < 75 % threshold - l1 = self._lcov("l1.info", 100, 74) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 70.0, "Component": 70.0}) + unit_cov = self._lcov("unit.info", 100, 74) # 74 % < 75 % threshold + component_cov = self._lcov("component.info", 100, 74) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[WARN]", r.stdout) self.assertIn("below threshold", r.stdout) @@ -545,10 +543,10 @@ def test_s6_below_threshold_meets_baseline(self): # =========================================================================== def test_s7_fails_both_threshold_and_baseline(self): - bl = self._baseline({"L0": 85.0, "L1": 85.0}) - l0 = self._lcov("l0.info", 100, 70) # 70 % < threshold AND < baseline - l1 = self._lcov("l1.info", 100, 70) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 85.0, "Component": 85.0}) + unit_cov = self._lcov("unit.info", 100, 70) # 70 % < threshold AND < baseline + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("below threshold", r.stdout) self.assertIn("dropped from baseline", r.stdout) @@ -559,21 +557,21 @@ def test_s7_fails_both_threshold_and_baseline(self): # =========================================================================== def test_s8_baseline_missing_coverage_above_threshold(self): - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) r = self._run( "--baseline", "/nonexistent/coverage-baseline.json", - "--l0", l0, "--l1", l1, + "--unit", unit_cov, "--component", component_cov, ) # No baseline → regression skipped → threshold pass → exit 0 self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) def test_s8_baseline_missing_coverage_below_threshold(self): - l0 = self._lcov("l0.info", 100, 70) # 70 % < 75 % - l1 = self._lcov("l1.info", 100, 70) + unit_cov = self._lcov("unit.info", 100, 70) # 70 % < 75 % + component_cov = self._lcov("component.info", 100, 70) r = self._run( "--baseline", "/nonexistent/coverage-baseline.json", - "--l0", l0, "--l1", l1, + "--unit", unit_cov, "--component", component_cov, ) # Informational only — exit 0 even below threshold; [WARN] shown self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) @@ -588,9 +586,9 @@ def test_s9_malformed_json_above_threshold(self): path = os.path.join(self.tmp, "malformed.json") with open(path, "w") as fh: fh.write("{this is not json}") - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) # Warning must appear in stderr self.assertIn("WARNING", r.stderr) # Fallback to empty baseline → threshold-only → pass @@ -600,9 +598,9 @@ def test_s9_malformed_json_below_threshold(self): path = os.path.join(self.tmp, "malformed2.json") with open(path, "w") as fh: fh.write("{bad json") - l0 = self._lcov("l0.info", 100, 70) - l1 = self._lcov("l1.info", 100, 70) - r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) self.assertIn("WARNING", r.stderr) # Informational only — exit 0 even below threshold; [WARN] shown self.assertEqual(r.returncode, 0) @@ -612,9 +610,9 @@ def test_s9_empty_json_file(self): path = os.path.join(self.tmp, "empty.json") with open(path, "w") as fh: fh.write("") - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) # Empty file → empty dict baseline → threshold-only → pass self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) @@ -622,9 +620,9 @@ def test_s9_non_dict_json(self): path = os.path.join(self.tmp, "list_json.json") with open(path, "w") as fh: json.dump([1, 2, 3], fh) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) # Non-dict JSON → WARNING emitted, empty dict → threshold-only → pass self.assertIn("WARNING", r.stderr) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) @@ -639,42 +637,42 @@ def test_s9_non_dict_json(self): # it is enforced by the workflow graph. # =========================================================================== - def test_s10_l0_artifacts_absent_l1_passes(self): + def test_s10_unit_artifacts_absent_component_passes(self): """ - Simulates the artifact-level effect: L0 .info absent (download step - with continue-on-error:true produced no file), L1 coverage present - and passing. Script-level: L0 is WARN (data missing), L1 is PASS. + Simulates the artifact-level effect: unit .info absent (download step + with continue-on-error:true produced no file), component coverage present + and passing. Script-level: Unit is WARN (data missing), Component is PASS. """ - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l1 = self._lcov("l1.info", 100, 80) # L0 omitted intentionally - r = self._run("--baseline", bl, "--l1", l1) - # L0 WARN (missing) → overall WARN, but exit 0 (informational) + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + component_cov = self._lcov("component.info", 100, 80) # Unit omitted intentionally + r = self._run("--baseline", bl, "--component", component_cov) + # Unit WARN (missing) → overall WARN, but exit 0 (informational) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("coverage data missing", r.stdout) self.assertIn("[WARN]", r.stdout) # =========================================================================== - # SCENARIO 11 — L1 job fails (symmetric to scenario 10) + # SCENARIO 11 — component_tests job fails (symmetric to scenario 10) # =========================================================================== - def test_s11_l1_artifacts_absent_l0_passes(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) # L1 omitted intentionally - r = self._run("--baseline", bl, "--l0", l0) + def test_s11_component_artifacts_absent_unit_passes(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) # Component omitted intentionally + r = self._run("--baseline", bl, "--unit", unit_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("coverage data missing", r.stdout) self.assertIn("[WARN]", r.stdout) # =========================================================================== - # SCENARIO 12 — Both L0 AND L1 jobs fail + # SCENARIO 12 — Both unit_tests AND component_tests jobs fail # Coverage Gate does NOT trigger (workflow-level). At script level, both # .info files are absent → both SKIP → exit 0 (harmless; gate is already # blocked at the workflow graph layer before the script is ever called). # =========================================================================== def test_s12_both_artifacts_absent(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - # Neither --l0 nor --l1 provided + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + # Neither --unit nor --component provided r = self._run("--baseline", bl) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) # Both rows + summary line show coverage data missing, OVERALL WARN @@ -691,9 +689,9 @@ def test_s12_both_artifacts_absent(self): def test_s13_missing_required_baseline_arg(self): """Invoking the script without --baseline must fail (argparse error).""" - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--unit", unit_cov, "--component", component_cov) # argparse exits with code 2 on missing required argument self.assertNotEqual(r.returncode, 0) self.assertTrue(len(r.stderr) > 0, "Error must appear on stderr") @@ -702,32 +700,32 @@ def test_s13_missing_required_baseline_arg(self): # Partial-failure cases: one suite fails, other passes # =========================================================================== - def test_only_l0_fails_gate_warns(self): - """L0 below threshold, L1 passes → overall [WARN] but exit 0.""" - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 70) # 70 % ✗ - l1 = self._lcov("l1.info", 100, 80) # 80 % ✓ - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + def test_only_unit_fails_gate_warns(self): + """Unit below threshold, Component passes → overall [WARN] but exit 0.""" + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 70) # 70 % ✗ + component_cov = self._lcov("component.info", 100, 80) # 80 % ✓ + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) - # L1 row still shows PASS + # Component row still shows PASS self.assertIn("[PASS]", r.stdout) self.assertIn("[WARN]", r.stdout) - def test_only_l1_fails_gate_warns(self): - """L1 below threshold, L0 passes → overall [WARN] but exit 0.""" - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) # 80 % ✓ - l1 = self._lcov("l1.info", 100, 70) # 70 % ✗ - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + def test_only_component_fails_gate_warns(self): + """Component below threshold, Unit passes → overall [WARN] but exit 0.""" + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % ✓ + component_cov = self._lcov("component.info", 100, 70) # 70 % ✗ + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[WARN]", r.stdout) - def test_only_l0_regresses_gate_warns(self): - """L0 regresses below baseline (still above threshold), L1 passes → [WARN] exit 0.""" - bl = self._baseline({"L0": 85.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) # 80 % < 85 % baseline ✗ - l1 = self._lcov("l1.info", 100, 80) # 80 % >= 75 % baseline ✓ - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + def test_only_unit_regresses_gate_warns(self): + """Unit regresses below baseline (still above threshold), Component passes → [WARN] exit 0.""" + bl = self._baseline({"Unit": 85.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % < 85 % baseline ✗ + component_cov = self._lcov("component.info", 100, 80) # 80 % >= 75 % baseline ✓ + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[WARN]", r.stdout) @@ -737,16 +735,16 @@ def test_only_l0_regresses_gate_warns(self): def test_first_time_setup_empty_baseline_passes(self): bl = self._baseline({}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) def test_first_time_setup_empty_baseline_below_threshold_warns(self): bl = self._baseline({}) - l0 = self._lcov("l0.info", 100, 70) - l1 = self._lcov("l1.info", 100, 70) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) # Informational only — exit 0 even below threshold; [WARN] shown self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[WARN]", r.stdout) @@ -756,18 +754,18 @@ def test_first_time_setup_empty_baseline_below_threshold_warns(self): # =========================================================================== def test_overall_pass_token_in_output(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertIn("OVERALL:", r.stdout) self.assertIn("[PASS]", r.stdout) def test_overall_warn_token_in_output(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 70) - l1 = self._lcov("l1.info", 100, 70) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertIn("OVERALL:", r.stdout) self.assertIn("[WARN]", r.stdout) # Informational only — always exit 0 @@ -778,13 +776,13 @@ def test_overall_warn_token_in_output(self): # =========================================================================== def test_output_json_written_when_both_pass(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 82) + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 82) out = os.path.join(self.tmp, "new-baseline.json") self._run( "--baseline", bl, - "--l0", l0, "--l1", l1, + "--unit", unit_cov, "--component", component_cov, "--output-json", out, "--commit", "abc123", "--timestamp", "2026-01-01T00:00:00Z", @@ -792,8 +790,8 @@ def test_output_json_written_when_both_pass(self): self.assertTrue(os.path.isfile(out), "output-json must be written") with open(out) as fh: data = json.load(fh) - self.assertEqual(data["L0"], 80.0) - self.assertEqual(data["L1"], 82.0) + self.assertEqual(data["Unit"], 80.0) + self.assertEqual(data["Component"], 82.0) self.assertEqual(data["commit"], "abc123") self.assertEqual(data["timestamp"], "2026-01-01T00:00:00Z") @@ -803,103 +801,103 @@ def test_output_json_written_even_when_gate_fails(self): regardless of gate outcome. The update-baseline step checks `if [ ! -s new-baseline.json ]` separately. """ - bl = self._baseline({"L0": 90.0, "L1": 90.0}) - l0 = self._lcov("l0.info", 100, 80) # 80 % < 90 % baseline → WARN - l1 = self._lcov("l1.info", 100, 80) + bl = self._baseline({"Unit": 90.0, "Component": 90.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % < 90 % baseline → WARN + component_cov = self._lcov("component.info", 100, 80) out = os.path.join(self.tmp, "new-baseline-fail.json") r = self._run( "--baseline", bl, - "--l0", l0, "--l1", l1, + "--unit", unit_cov, "--component", component_cov, "--output-json", out, ) # Informational only — always exit 0 regardless of gate outcome self.assertEqual(r.returncode, 0) self.assertTrue(os.path.isfile(out), "output-json written even on gate warning") - def test_output_json_not_written_when_l0_missing(self): - """When L0 .info is absent, --output-json must NOT be written (data incomplete).""" - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l1 = self._lcov("l1.info", 100, 82) - out = os.path.join(self.tmp, "new-baseline-no-l0.json") - r = self._run("--baseline", bl, "--l1", l1, "--output-json", out) - self.assertFalse(os.path.isfile(out), "output-json must NOT be written when L0 absent") + def test_output_json_not_written_when_unit_missing(self): + """When unit .info is absent, --output-json must NOT be written (data incomplete).""" + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + component_cov = self._lcov("component.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline-no-unit.json") + r = self._run("--baseline", bl, "--component", component_cov, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when unit absent") self.assertIn("WARNING", r.stderr) - def test_output_json_not_written_when_l1_missing(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) - out = os.path.join(self.tmp, "new-baseline-no-l1.json") - r = self._run("--baseline", bl, "--l0", l0, "--output-json", out) - self.assertFalse(os.path.isfile(out), "output-json must NOT be written when L1 absent") + def test_output_json_not_written_when_component_missing(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-no-component.json") + r = self._run("--baseline", bl, "--unit", unit_cov, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when component absent") self.assertIn("WARNING", r.stderr) def test_output_json_not_written_when_both_missing(self): - bl = self._baseline({"L0": 75.0, "L1": 75.0}) + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) out = os.path.join(self.tmp, "new-baseline-neither.json") r = self._run("--baseline", bl, "--output-json", out) self.assertFalse(os.path.isfile(out)) self.assertIn("WARNING", r.stderr) # =========================================================================== - # Baseline coercion: non-float L0/L1 values must not crash the script - # (Fixes comment 2/7 — baseline.get("L0") not validated as float) + # Baseline coercion: non-float Unit/Component values must not crash the script # =========================================================================== - def test_baseline_string_l0_treated_as_missing(self): - """String value for L0 in baseline JSON → coerced to None → threshold-only.""" - bl = self._baseline({"L0": "not-a-number", "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) - # Must not crash; L0 baseline treated as absent → threshold-only → pass + + def test_baseline_string_unit_treated_as_missing(self): + """String value for Unit in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"Unit": "not-a-number", "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + # Must not crash; Unit baseline treated as absent → threshold-only → pass self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) - def test_baseline_null_l1_treated_as_missing(self): - """null value for L1 in baseline JSON → coerced to None → threshold-only.""" - bl = self._baseline({"L0": 80.0, "L1": None}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + def test_baseline_null_component_treated_as_missing(self): + """null value for Component in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"Unit": 80.0, "Component": None}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) def test_baseline_both_non_float_threshold_only(self): - """Both L0/L1 baseline values invalid → both threshold-only → pass if above 75%.""" - bl = self._baseline({"L0": "bad", "L1": "bad"}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + """Both Unit/Component baseline values invalid → both threshold-only → pass if above 75%.""" + bl = self._baseline({"Unit": "bad", "Component": "bad"}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) def test_baseline_both_non_float_below_threshold_warns(self): - """Both L0/L1 baseline values invalid → threshold-only → [WARN] exit 0 if below 75%.""" - bl = self._baseline({"L0": "bad", "L1": "bad"}) - l0 = self._lcov("l0.info", 100, 70) - l1 = self._lcov("l1.info", 100, 70) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + """Both Unit/Component baseline values invalid → threshold-only → [WARN] exit 0 if below 75%.""" + bl = self._baseline({"Unit": "bad", "Component": "bad"}) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("[WARN]", r.stdout) # =========================================================================== # _fmt_timestamp: null/non-string timestamp must not crash the report - # (Fixes comment 8 — only ValueError was caught, not TypeError) # =========================================================================== + def test_null_timestamp_in_baseline_does_not_crash(self): """null timestamp value in baseline JSON → TypeError handled → report still runs.""" - bl = self._baseline({"L0": 80.0, "L1": 80.0, "commit": "abc", "timestamp": None}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 80.0, "Component": 80.0, "commit": "abc", "timestamp": None}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) # Must not crash; timestamp renders as fallback; gate passes self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("OVERALL:", r.stdout) def test_integer_timestamp_in_baseline_does_not_crash(self): """Integer timestamp → TypeError in strptime → handled gracefully.""" - bl = self._baseline({"L0": 80.0, "L1": 80.0, "commit": "abc", "timestamp": 12345}) - l0 = self._lcov("l0.info", 100, 80) - l1 = self._lcov("l1.info", 100, 80) - r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + bl = self._baseline({"Unit": 80.0, "Component": 80.0, "commit": "abc", "timestamp": 12345}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 338cccf..9fdcc1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -366,6 +366,9 @@ jobs: with: python-version: '3.x' + - name: Run coverage script tests + run: python3 .github/scripts/compare_coverage_test.py + - name: Fetch baseline from build-metadata branch # Gracefully handle a missing build-metadata branch (first-time setup) continue-on-error: true @@ -402,8 +405,8 @@ jobs: run: | python3 .github/scripts/compare_coverage.py \ --baseline coverage-baseline.json \ - --l0 ./unit-coverage/filtered_coverage.info \ - --l1 ./component-coverage/filtered_coverage.info + --unit ./unit-coverage/filtered_coverage.info \ + --component ./component-coverage/filtered_coverage.info update_baseline: name: Update Coverage Baseline @@ -471,8 +474,8 @@ jobs: python3 .github/scripts/compare_coverage.py \ --baseline "$BL_ARG" \ - --l0 ./unit-coverage/filtered_coverage.info \ - --l1 ./component-coverage/filtered_coverage.info \ + --unit ./unit-coverage/filtered_coverage.info \ + --component ./component-coverage/filtered_coverage.info \ --output-json new-baseline.json \ --commit "$GITHUB_SHA" \ --timestamp "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"