From 97689bcf93e35578f729de81bd73e5188daac8c1 Mon Sep 17 00:00:00 2001 From: swethasukumarr Date: Tue, 23 Jun 2026 13:55:57 -0400 Subject: [PATCH 1/2] RDKEMW-17673 : Add info coverage gate to CI --- .github/scripts/compare_coverage.py | 313 ++++++++ .github/scripts/compare_coverage_test.py | 930 +++++++++++++++++++++++ .github/workflows/L1-tests.yml | 8 +- .github/workflows/main-workflow.yml | 165 ++++ 4 files changed, 1414 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/compare_coverage.py create mode 100644 .github/scripts/compare_coverage_test.py create mode 100644 .github/workflows/main-workflow.yml diff --git a/.github/scripts/compare_coverage.py b/.github/scripts/compare_coverage.py new file mode 100644 index 00000000..1cd620c4 --- /dev/null +++ b/.github/scripts/compare_coverage.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +# If not stated otherwise in this file or this component's LICENSE file the +# following copyright and licenses apply: +# +# Copyright 2026 RDK Management +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Coverage comparison script for Dobby. + +Reads L1 (and optionally L0) lcov coverage results, compares overall line +coverage against the stored baseline from the build-metadata branch, and +prints a summary. Both --l0 and --l1 are optional; only suites explicitly +passed are evaluated. +""" + +import argparse +import datetime +import json +import os +import sys +from typing import Optional + + +# Minimum threshold +THRESHOLD = 75.0 + +_GREEN = "\033[32m" +_RED = "\033[31m" +_RESET = "\033[0m" + +_SEP_WIDTH = 64 +_OVERALL_WIDTH = 80 +_SEP = "\u2500" * _SEP_WIDTH +_HEADER = "\u2500\u2500 Coverage Gate Report " + "\u2500" * (_SEP_WIDTH - 24) + + +def _colored(token: str, ok: bool) -> str: + return f"{_GREEN if ok else _RED}{token}{_RESET}" + + +def _fmt_timestamp(ts: str) -> str: + """Convert '2026-05-28T12:00:00Z' -> '2026-05-28 12:00 UTC'.""" + try: + dt = datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") + return dt.strftime("%Y-%m-%d %H:%M UTC") + except (ValueError, TypeError): + return ts + + +def _delta_str(current: float, baseline: float) -> str: + delta = current - baseline + sign = "+" if delta >= 0 else "" + return f"{sign}{delta:.2f}%" + + +def _join_names(names: list) -> str: + return names[0] if len(names) == 1 else " and ".join(names) + + +def _suite_analysis(current: Optional[float], baseline: Optional[float]): + """Analyse one test suite. + + Returns (ok, result_str, delta_disp, warn_reason): + ok - True when no advisory issues found. + result_str - Coloured [PASS]/[WARN] token + detail for the table. + delta_disp - String for the Delta column ("N/A" when skipped). + warn_reason - Reason phrase for the summary line; None when ok. + """ + if current is None: + reason = "coverage data missing" + return False, f"{_colored('[WARN]', False)} {reason}", "N/A", reason + + threshold_ok = current >= THRESHOLD + + if baseline is None: + # No baseline stored — threshold check only. + regression_ok = True + detail = "" if threshold_ok else "below threshold" + delta_disp = "N/A" + elif baseline == 0.0: + # A zero baseline is unreliable — skip regression check. + regression_ok = True + base_note = "baseline unreliable (0%) \u00b7 delta skipped" + detail = f"below threshold \u00b7 {base_note}" if not threshold_ok else base_note + delta_disp = "N/A" + else: + regression_ok = current >= baseline + delta_disp = _delta_str(current, baseline) + if threshold_ok and regression_ok: + detail = "" + elif not threshold_ok and not regression_ok: + detail = "below threshold \u00b7 dropped from baseline" + elif not threshold_ok: + detail = "below threshold \u00b7 no baseline regression" + else: + detail = "above threshold but dropped from baseline" + + overall_ok = threshold_ok and regression_ok + token = _colored("[PASS]", True) if overall_ok else _colored("[WARN]", False) + result_str = f"{token} {detail}" if detail else token + warn_reason = detail if not overall_ok else None + return overall_ok, result_str, delta_disp, warn_reason + + +def _build_summary(warn_suites: list) -> str: + """Build a compact summary from WARN suite (name, reason) pairs.""" + if not warn_suites: + return "" + groups: dict = {} + for name, reason in warn_suites: + groups.setdefault(reason, []).append(name) + parts = [f"{_join_names(names)} {reason}" for reason, names in groups.items()] + return ". ".join(parts) + + +# lcov parsing +def parse_lcov_coverage(path: str) -> Optional[float]: + """Return overall line coverage % from an lcov .info file, or None. + + An lcov .info file contains per-source-file records separated by + ``end_of_record``. Each record may include: + LF: — total instrumented lines in that file + LH: — lines executed at least once + + We aggregate across all records to produce a single project-wide %. + Returns None when the file is absent, empty, or contains no line data. + """ + if not path or not os.path.isfile(path): + return None + + total_found = 0 + total_hit = 0 + + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for raw in fh: + line = raw.strip() + if line.startswith("LF:"): + try: + total_found += int(line[3:]) + except ValueError: + pass + elif line.startswith("LH:"): + try: + total_hit += int(line[3:]) + except ValueError: + pass + except OSError as exc: + print(f" WARNING: Could not read {path}: {exc}", file=sys.stderr) + return None + + if total_found == 0: + return None + + return round((total_hit / total_found) * 100.0, 2) + + +def _coerce_pct(value: object) -> Optional[float]: + """Coerce a baseline percentage value to float, or None if invalid.""" + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +# Baseline loading +def load_baseline(path: str) -> dict: + """Load baseline JSON; return an empty dict on any error.""" + if not path or not os.path.isfile(path): + return {} + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + return data + print( + f" WARNING: Baseline {path} is not a JSON object (got {type(data).__name__}) — ignoring", + file=sys.stderr, + ) + except (OSError, json.JSONDecodeError, ValueError) as exc: + print(f" WARNING: Could not parse baseline {path}: {exc}", file=sys.stderr) + return {} + + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Compare L0/L1 coverage against the develop baseline.\n" + "Exits 1 when coverage fails threshold or regresses from baseline." + ) + ) + parser.add_argument("--baseline", required=True, metavar="PATH", + help="Path to coverage-baseline.json.") + parser.add_argument("--l0", required=False, metavar="PATH", + help="Path to the L0 lcov filtered_coverage.info file.") + parser.add_argument("--l1", required=False, metavar="PATH", + help="Path to the L1 lcov filtered_coverage.info file.") + parser.add_argument("--output-json", required=False, metavar="PATH", + help="Write {L0, L1, commit, timestamp} JSON here for baseline update.") + parser.add_argument("--commit", required=False, default="", + help="Commit SHA to embed in --output-json.") + parser.add_argument("--timestamp", required=False, default="", + help="ISO 8601 timestamp to embed in --output-json.") + args = parser.parse_args() + + baseline = load_baseline(args.baseline) + + baseline_l0: Optional[float] = _coerce_pct(baseline.get("L0")) + baseline_l1: Optional[float] = _coerce_pct(baseline.get("L1")) + + l0_coverage = parse_lcov_coverage(args.l0) if args.l0 else None + l1_coverage = parse_lcov_coverage(args.l1) if args.l1 else None + + # ------------------------------------------------------------------ + # Optional: write extracted numbers for baseline update. + # Skipped when a *requested* suite (explicitly passed as an arg) produced + # no data — indicates a broken artifact. Suites not passed as args are + # simply omitted from the payload (repos may track L1 only or L0 only). + # ------------------------------------------------------------------ + if args.output_json: + l0_failed = args.l0 is not None and l0_coverage is None + l1_failed = args.l1 is not None and l1_coverage is None + has_data = l0_coverage is not None or l1_coverage is not None + if not l0_failed and not l1_failed and has_data: + payload: dict = {"commit": args.commit or "", "timestamp": args.timestamp or ""} + if l0_coverage is not None: + payload["L0"] = l0_coverage + if l1_coverage is not None: + payload["L1"] = l1_coverage + try: + with open(args.output_json, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + fh.write("\n") + except OSError as exc: + print(f" WARNING: Could not write {args.output_json}: {exc}", file=sys.stderr) + else: + print( + f" WARNING: --output-json skipped: coverage data incomplete " + f"(L0={l0_coverage}, L1={l1_coverage})", + file=sys.stderr, + ) + + l0_ok, l0_result, l0_delta, l0_reason = _suite_analysis(l0_coverage, baseline_l0) + l1_ok, l1_result, l1_delta, l1_reason = _suite_analysis(l1_coverage, baseline_l1) + + all_ok = l0_ok and l1_ok + status_token = _colored("[PASS]", True) if all_ok else _colored("[WARN]", False) + + # Output report + print() + print(_HEADER) + if baseline: + commit = baseline.get("commit", "unknown") + ts = _fmt_timestamp(baseline.get("timestamp", "")) + print(f" Baseline {commit} ({ts})") + else: + print(" Baseline N/A (first-time setup \u2014 regression check skipped)") + print(f" Threshold {THRESHOLD}% | Status {status_token} (informational \u2014 PRs are not blocked)") + print(_SEP) + + # Coverage table + print(f" {'Suite':<7}{'Current':<9}{'Baseline':<10}{'Delta':<10}Result") + for name, current, base, result, delta_disp in [ + ("L0", l0_coverage, baseline_l0, l0_result, l0_delta), + ("L1", l1_coverage, baseline_l1, l1_result, l1_delta), + ]: + cur_str = f"{current:.2f}%" if current is not None else "N/A" + base_str = f"{base:.2f}%" if base is not None else "N/A" + print(f" {name:<7}{cur_str:<9}{base_str:<10}{delta_disp:<10}{result}") + + print(_SEP) + + # Summary + overall bar + warn_suites = [(n, r) for n, r in [("L0", l0_reason), ("L1", l1_reason)] if r] + summary = _build_summary(warn_suites) + if summary: + print(f" {summary}") + + # Notify when one or both suites had no coverage data (artifact absent). + # Gate logic is unchanged — SKIP is treated as passing by design. + skipped = [n for n, cov in [("L0", l0_coverage), ("L1", l1_coverage)] if cov is None] + if skipped: + print(f" NOTE: {_join_names(skipped)} coverage data absent \u2014 artifact missing or unreadable.") + + # " OVERALL: [PASS/WARN] " = 1 + 9 + 6 + 1 = 17 visible chars + # left + " OVERALL: " + token(6) + " " + right == _OVERALL_WIDTH + _mid = len(" OVERALL: ") + 6 + len(" ") # 17 + left = "\u2500" * ((_OVERALL_WIDTH - _mid) // 2) # 31 + right = "\u2500" * (_OVERALL_WIDTH - _mid - len(left)) # 32 + print(f"{left} OVERALL: {status_token} {right}") + print() + + # Informational only — always exit 0 so PRs are never blocked. + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/compare_coverage_test.py b/.github/scripts/compare_coverage_test.py new file mode 100644 index 00000000..2ccda338 --- /dev/null +++ b/.github/scripts/compare_coverage_test.py @@ -0,0 +1,930 @@ +#!/usr/bin/env python3 +# If not stated otherwise in this file or this component's LICENSE file the +# following copyright and licenses apply: +# +# Copyright 2026 RDK Management +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +""" +Tests for compare_coverage.py + +Covers: + - Unit tests for parse_lcov_coverage(), load_baseline(), _suite_analysis() + - Integration tests (subprocess) simulating all gate scenarios listed in the + Coverage Gate implementation spec. + +Workflow-level scenarios (L0 job fails / L1 job fails / both fail) are handled +by GitHub Actions' implicit success() dependency check on the coverage-gate job +and cannot be tested at the Python script level; they are documented inline. +""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +# --------------------------------------------------------------------------- +# Import the module under test +# --------------------------------------------------------------------------- +SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPTS_DIR) + +import compare_coverage # noqa: E402 (after sys.path manipulation) + +THRESHOLD = compare_coverage.THRESHOLD # 75.0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_lcov(lines_found: int, lines_hit: int) -> str: + """Minimal valid lcov .info content with the given LF/LH counts.""" + return ( + "SF:src/fake.cpp\n" + f"LF:{lines_found}\n" + f"LH:{lines_hit}\n" + "end_of_record\n" + ) + + +def _write_lcov(tmp_dir: str, name: str, lines_found: int, lines_hit: int) -> str: + """Write an lcov file and return its absolute path.""" + path = os.path.join(tmp_dir, name) + with open(path, "w") as fh: + fh.write(_make_lcov(lines_found, lines_hit)) + return path + + +def _write_baseline(tmp_dir: str, data: dict, name: str = "baseline.json") -> str: + """Serialise *data* to JSON and return the path.""" + path = os.path.join(tmp_dir, name) + with open(path, "w") as fh: + json.dump(data, fh) + return path + + +def _run_script(*args: str) -> subprocess.CompletedProcess: + """Invoke compare_coverage.py as a subprocess and return the result.""" + cmd = [sys.executable, os.path.join(SCRIPTS_DIR, "compare_coverage.py"), *args] + return subprocess.run(cmd, capture_output=True, text=True) + + +# =========================================================================== +# Unit tests — parse_lcov_coverage() +# =========================================================================== + +class TestParseLcovCoverage(unittest.TestCase): + """Tests for the lcov .info parser.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, name: str, content: str) -> str: + path = os.path.join(self.tmp, name) + with open(path, "w") as fh: + fh.write(content) + return path + + # --- Missing / empty inputs ------------------------------------------------- + + def test_none_path_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage(None)) + + def test_empty_path_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage("")) + + def test_nonexistent_file_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage("/no/such/file.info")) + + def test_empty_file_returns_none(self): + p = self._write("empty.info", "") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_no_lf_data_returns_none(self): + p = self._write("no_lf.info", "SF:foo.cpp\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_lf_zero_returns_none(self): + p = self._write("zero_lf.info", "SF:foo.cpp\nLF:0\nLH:0\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + # --- Basic coverage values -------------------------------------------------- + + def test_100_percent(self): + p = _write_lcov(self.tmp, "full.info", 100, 100) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 100.0) + + def test_75_percent_exact(self): + p = _write_lcov(self.tmp, "seventy_five.info", 100, 75) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 75.0) + + def test_zero_percent(self): + p = _write_lcov(self.tmp, "zero_pct.info", 100, 0) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 0.0) + + def test_partial_coverage(self): + # 150 / 200 = 75.0 % + p = _write_lcov(self.tmp, "partial.info", 200, 150) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 75.0) + + # --- Multi-record aggregation ----------------------------------------------- + + def test_aggregates_multiple_records(self): + # 80 + 60 = 140 hit out of 200 → 70.0 % + content = ( + "SF:a.cpp\nLF:100\nLH:80\nend_of_record\n" + "SF:b.cpp\nLF:100\nLH:60\nend_of_record\n" + ) + p = self._write("multi.info", content) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 70.0) + + # --- Malformed data --------------------------------------------------------- + + def test_malformed_lf_ignored_gracefully(self): + # LF with a non-numeric value; total_found stays 0 → None + p = self._write("bad_lf.info", "SF:a.cpp\nLF:abc\nLH:50\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_malformed_lh_ignored_gracefully(self): + # LH with garbage value; LF is valid, so LF=100, LH=0 → 0.0 % + p = self._write("bad_lh.info", "SF:a.cpp\nLF:100\nLH:xyz\nend_of_record\n") + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 0.0) + + def test_entirely_non_lcov_content(self): + p = self._write("corrupt.info", "THIS IS NOT A VALID LCOV FILE\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + # --- Rounding --------------------------------------------------------------- + + def test_rounds_to_two_decimal_places(self): + # 1/3 ≈ 33.33 % + p = _write_lcov(self.tmp, "third.info", 3, 1) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 33.33) + + +# =========================================================================== +# Unit tests — load_baseline() +# =========================================================================== + +class TestLoadBaseline(unittest.TestCase): + """Tests for the JSON baseline loader.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write_json(self, name: str, content: str) -> str: + path = os.path.join(self.tmp, name) + with open(path, "w") as fh: + fh.write(content) + return path + + # --- Missing / empty inputs ------------------------------------------------- + + def test_none_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline(None), {}) + + def test_empty_path_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline(""), {}) + + def test_nonexistent_file_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline("/no/such/file.json"), {}) + + # --- Valid JSON ------------------------------------------------------------- + + def test_valid_baseline_with_l0_and_l1(self): + p = _write_baseline(self.tmp, {"L0": 80.0, "L1": 85.0}) + self.assertEqual(compare_coverage.load_baseline(p), {"L0": 80.0, "L1": 85.0}) + + def test_valid_empty_json_object(self): + p = _write_baseline(self.tmp, {}) + self.assertEqual(compare_coverage.load_baseline(p), {}) + + def test_valid_baseline_extra_keys_preserved(self): + data = {"L0": 80.0, "L1": 85.0, "commit": "abc123", "timestamp": "2026-01-01"} + p = _write_baseline(self.tmp, data) + self.assertEqual(compare_coverage.load_baseline(p), data) + + # --- Invalid JSON ----------------------------------------------------------- + + def test_malformed_json_returns_empty_dict(self): + p = self._write_json("bad.json", "{not valid json}") + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + + def test_truncated_json_returns_empty_dict(self): + p = self._write_json("truncated.json", '{"L0": 80') + self.assertEqual(compare_coverage.load_baseline(p), {}) + + def test_empty_file_returns_empty_dict(self): + p = self._write_json("empty.json", "") + self.assertEqual(compare_coverage.load_baseline(p), {}) + + # --- Non-dict JSON ---------------------------------------------------------- + + def test_json_array_returns_empty_dict(self): + import io, contextlib + p = self._write_json("list.json", "[1, 2, 3]") + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_string_returns_empty_dict(self): + import io, contextlib + p = self._write_json("str.json", '"just a string"') + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_number_returns_empty_dict(self): + import io, contextlib + p = self._write_json("num.json", "42") + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_null_returns_empty_dict(self): + p = self._write_json("null.json", "null") + # null is parsed as None, which is not a dict — Warning emitted, empty dict returned + self.assertEqual(compare_coverage.load_baseline(p), {}) + + +# =========================================================================== +# Unit tests — _suite_analysis() +# =========================================================================== + +class TestSuiteAnalysis(unittest.TestCase): + """ + Tests for the core gate analysis function. + + Gate passes (ok=True) when BOTH: + 1. current >= THRESHOLD (75.0) + 2. current >= baseline (regression check) + + SKIP when current is None. + Regression check disabled when baseline is None or 0.0. + """ + + # --- Scenario 1: exceeds both threshold AND baseline → PASS ---------------- + + def test_s1_exceeds_threshold_and_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(80.0, 77.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 2: meets threshold exactly (75%) AND beats baseline → PASS --- + + def test_s2_meets_threshold_exactly_beats_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 70.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 3: meets baseline exactly, exceeds threshold → PASS ---------- + + def test_s3_meets_baseline_exactly_exceeds_threshold(self): + ok, _, _, reason = compare_coverage._suite_analysis(80.0, 80.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 4: meets BOTH exactly (75.0 == threshold == baseline) → PASS - + + def test_s4_meets_both_exactly_at_threshold(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 75.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 5: exceeds threshold but BELOW baseline → FAIL (regression) -- + + def test_s5_above_threshold_below_baseline(self): + ok, result, _, reason = compare_coverage._suite_analysis(76.0, 80.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("dropped from baseline", reason) + + # --- Scenario 6: below threshold but meets/exceeds baseline → FAIL ---------- + + def test_s6_below_threshold_meets_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 70.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertNotIn("dropped from baseline", reason) # regression check passed + + def test_s6b_below_threshold_equals_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 74.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertNotIn("dropped from baseline", reason) # regression check passed + + # --- Scenario 7: fails BOTH conditions → FAIL -------------------------------- + + def test_s7_fails_both_conditions(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, 80.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertIn("dropped from baseline", reason) + + # --- Scenario 8: no baseline → threshold-only check ------------------------- + + def test_no_baseline_above_threshold_passes(self): + ok, _, delta, reason = compare_coverage._suite_analysis(80.0, None) + self.assertTrue(ok) + self.assertIsNone(reason) + self.assertEqual(delta, "N/A") + + def test_no_baseline_below_threshold_fails(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, None) + self.assertFalse(ok) + self.assertIn("below threshold", reason) + + def test_no_baseline_at_threshold_exactly_passes(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, None) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- SKIP case: no current coverage ----------------------------------------- + + def test_skip_when_current_is_none(self): + ok, result, delta, reason = compare_coverage._suite_analysis(None, 80.0) + self.assertFalse(ok, "Missing coverage data should be treated as WARN") + self.assertIn("coverage data missing", result) + self.assertEqual(delta, "N/A") + self.assertIsNotNone(reason) + self.assertIn("coverage data missing", reason) + + def test_skip_when_both_none(self): + ok, result, delta, reason = compare_coverage._suite_analysis(None, None) + self.assertFalse(ok) + self.assertIn("coverage data missing", result) + + # --- Zero baseline: regression check disabled -------------------------------- + + def test_zero_baseline_above_threshold_passes(self): + ok, _, delta, reason = compare_coverage._suite_analysis(80.0, 0.0) + self.assertTrue(ok) + self.assertIsNone(reason) + self.assertEqual(delta, "N/A", "Delta must be N/A for zero baseline") + + def test_zero_baseline_below_threshold_fails(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, 0.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + + # --- Delta string correctness ------------------------------------------------ + + def test_delta_positive(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, 77.0) + self.assertEqual(delta, "+3.00%") + + def test_delta_negative(self): + _, _, delta, _ = compare_coverage._suite_analysis(76.0, 80.0) + self.assertEqual(delta, "-4.00%") + + def test_delta_zero(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, 80.0) + self.assertEqual(delta, "+0.00%") + + def test_delta_na_when_no_baseline(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, None) + self.assertEqual(delta, "N/A") + + # --- Boundary: one tick below threshold (74.99 is impossible from lcov, + # but 74.0 covers the just-below case) ---------------------------------- + + def test_just_below_threshold_fails(self): + # 74 / 100 = 74.0 % + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 70.0) + self.assertFalse(ok) + + def test_just_at_threshold_passes(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 70.0) + self.assertTrue(ok) + + +# =========================================================================== +# Integration tests — main() via subprocess +# =========================================================================== + +class TestMainIntegration(unittest.TestCase): + """ + End-to-end simulation of every gate scenario. + + Each test invokes the script as a subprocess (exactly as GitHub Actions + would) and asserts on exit code and stdout/stderr content. + """ + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + # --- Helpers ---------------------------------------------------------------- + + def _lcov(self, name: str, lf: int, lh: int) -> str: + return _write_lcov(self.tmp, name, lf, lh) + + def _baseline(self, data: dict, name: str = "baseline.json") -> str: + return _write_baseline(self.tmp, data, name) + + def _run(self, *args: str) -> subprocess.CompletedProcess: + return _run_script(*args) + + # =========================================================================== + # SCENARIO 1 — Coverage exceeds both threshold AND baseline + # Expected: Gate PASSES (exit 0), baseline updates + # =========================================================================== + + def test_s1_exceeds_threshold_and_baseline(self): + bl = self._baseline({"L0": 77.0, "L1": 78.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % + l1 = self._lcov("l1.info", 100, 82) # 82 % + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[PASS]", r.stdout) + + # =========================================================================== + # SCENARIO 2 — Coverage meets threshold exactly (75%) and meets baseline + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s2_meets_threshold_exactly_meets_baseline(self): + bl = self._baseline({"L0": 70.0, "L1": 70.0}) + l0 = self._lcov("l0.info", 100, 75) # 75.0 % + l1 = self._lcov("l1.info", 100, 75) # 75.0 % + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[PASS]", r.stdout) + + # =========================================================================== + # SCENARIO 3 — Meets baseline exactly but exceeds threshold + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s3_meets_baseline_exactly_exceeds_threshold(self): + bl = self._baseline({"L0": 80.0, "L1": 80.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % == baseline + l1 = self._lcov("l1.info", 100, 80) # 80 % == baseline + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 4 — Meets BOTH exactly (current == threshold == baseline == 75 %) + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s4_meets_both_exactly(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 75) + l1 = self._lcov("l1.info", 100, 75) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 5 — Exceeds threshold but falls BELOW baseline (regression) + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s5_above_threshold_below_baseline(self): + bl = self._baseline({"L0": 85.0, "L1": 85.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % < 85 % baseline + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + self.assertIn("dropped from baseline", r.stdout) + + # =========================================================================== + # SCENARIO 6 — Falls BELOW threshold but meets/exceeds baseline + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s6_below_threshold_meets_baseline(self): + bl = self._baseline({"L0": 70.0, "L1": 70.0}) + l0 = self._lcov("l0.info", 100, 74) # 74 % < 75 % threshold + l1 = self._lcov("l1.info", 100, 74) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + self.assertIn("below threshold", r.stdout) + + # =========================================================================== + # SCENARIO 7 — Fails BOTH conditions (below threshold AND below baseline) + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s7_fails_both_threshold_and_baseline(self): + bl = self._baseline({"L0": 85.0, "L1": 85.0}) + l0 = self._lcov("l0.info", 100, 70) # 70 % < threshold AND < baseline + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("below threshold", r.stdout) + self.assertIn("dropped from baseline", r.stdout) + + # =========================================================================== + # SCENARIO 8a — Baseline file is MISSING + # Expected: Graceful fallback; threshold-only check; no crash + # =========================================================================== + + def test_s8_baseline_missing_coverage_above_threshold(self): + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run( + "--baseline", "/nonexistent/coverage-baseline.json", + "--l0", l0, "--l1", l1, + ) + # No baseline → regression skipped → threshold pass → exit 0 + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s8_baseline_missing_coverage_below_threshold(self): + l0 = self._lcov("l0.info", 100, 70) # 70 % < 75 % + l1 = self._lcov("l1.info", 100, 70) + r = self._run( + "--baseline", "/nonexistent/coverage-baseline.json", + "--l0", l0, "--l1", l1, + ) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 9 — Baseline file contains invalid / malformed JSON + # Expected: Warning emitted, treated as empty baseline, gate continues + # =========================================================================== + + def test_s9_malformed_json_above_threshold(self): + path = os.path.join(self.tmp, "malformed.json") + with open(path, "w") as fh: + fh.write("{this is not json}") + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + # Warning must appear in stderr + self.assertIn("WARNING", r.stderr) + # Fallback to empty baseline → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s9_malformed_json_below_threshold(self): + path = os.path.join(self.tmp, "malformed2.json") + with open(path, "w") as fh: + fh.write("{bad json") + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + self.assertIn("WARNING", r.stderr) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0) + self.assertIn("[WARN]", r.stdout) + + def test_s9_empty_json_file(self): + path = os.path.join(self.tmp, "empty.json") + with open(path, "w") as fh: + fh.write("") + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + # Empty file → empty dict baseline → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s9_non_dict_json(self): + path = os.path.join(self.tmp, "list_json.json") + with open(path, "w") as fh: + json.dump([1, 2, 3], fh) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", path, "--l0", l0, "--l1", l1) + # Non-dict JSON → WARNING emitted, empty dict → threshold-only → pass + self.assertIn("WARNING", r.stderr) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 10 — L0 job fails + # Coverage Gate does NOT trigger (workflow-level behaviour). + # + # GitHub Actions: coverage-gate has `needs: [trigger-L0, trigger-L1]` + # with no custom `if:`. The implicit success() check means coverage-gate + # is SKIPPED whenever trigger-L0 fails. update-baseline then sees + # needs.coverage-gate.result == 'skipped' (not 'success') and is also + # skipped. This cannot be unit-tested here; it is enforced by the + # workflow graph. + # =========================================================================== + + def test_s10_l0_artifacts_absent_l1_passes(self): + """ + Simulates the artifact-level effect: L0 .info absent (download step + with continue-on-error:true produced no file), L1 coverage present + and passing. Script-level: L0 is WARN (data missing), L1 is PASS. + """ + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l1 = self._lcov("l1.info", 100, 80) # L0 omitted intentionally + r = self._run("--baseline", bl, "--l1", l1) + # L0 WARN (missing) → overall WARN, but exit 0 (informational) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("coverage data missing", r.stdout) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 11 — L1 job fails (symmetric to scenario 10) + # =========================================================================== + + def test_s11_l1_artifacts_absent_l0_passes(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) # L1 omitted intentionally + r = self._run("--baseline", bl, "--l0", l0) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("coverage data missing", r.stdout) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 12 — Both L0 AND L1 jobs fail + # Coverage Gate does NOT trigger (workflow-level). At script level, both + # .info files are absent → both SKIP → exit 0 (harmless; gate is already + # blocked at the workflow graph layer before the script is ever called). + # =========================================================================== + + def test_s12_both_artifacts_absent(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + # Neither --l0 nor --l1 provided + r = self._run("--baseline", bl) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + # Both rows + summary line show coverage data missing, OVERALL WARN + self.assertGreaterEqual(r.stdout.count("coverage data missing"), 2) + self.assertIn("[WARN]", r.stdout) + self.assertIn("NOTE:", r.stdout) + + # =========================================================================== + # SCENARIO 13 — Coverage Gate step itself throws an unexpected error + # Expected: non-zero exit; error is visible; baseline NOT updated + # (Simulated by passing a completely invalid path for --baseline that + # causes the argument parser or file logic to surface an error.) + # =========================================================================== + + def test_s13_missing_required_baseline_arg(self): + """Invoking the script without --baseline must fail (argparse error).""" + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--l0", l0, "--l1", l1) + # argparse exits with code 2 on missing required argument + self.assertNotEqual(r.returncode, 0) + self.assertTrue(len(r.stderr) > 0, "Error must appear on stderr") + + # =========================================================================== + # Partial-failure cases: one suite fails, other passes + # =========================================================================== + + def test_only_l0_fails_gate_warns(self): + """L0 below threshold, L1 passes → overall [WARN] but exit 0.""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 70) # 70 % ✗ + l1 = self._lcov("l1.info", 100, 80) # 80 % ✓ + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + # L1 row still shows PASS + self.assertIn("[PASS]", r.stdout) + self.assertIn("[WARN]", r.stdout) + + def test_only_l1_fails_gate_warns(self): + """L1 below threshold, L0 passes → overall [WARN] but exit 0.""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % ✓ + l1 = self._lcov("l1.info", 100, 70) # 70 % ✗ + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + def test_only_l0_regresses_gate_warns(self): + """L0 regresses below baseline (still above threshold), L1 passes → [WARN] exit 0.""" + bl = self._baseline({"L0": 85.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % < 85 % baseline ✗ + l1 = self._lcov("l1.info", 100, 80) # 80 % >= 75 % baseline ✓ + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # First-time setup: empty baseline {} → threshold-only + # =========================================================================== + + def test_first_time_setup_empty_baseline_passes(self): + bl = self._baseline({}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_first_time_setup_empty_baseline_below_threshold_warns(self): + bl = self._baseline({}) + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # Output format validation + # =========================================================================== + + def test_overall_pass_token_in_output(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertIn("OVERALL:", r.stdout) + self.assertIn("[PASS]", r.stdout) + + def test_overall_warn_token_in_output(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertIn("OVERALL:", r.stdout) + self.assertIn("[WARN]", r.stdout) + # Informational only — always exit 0 + self.assertEqual(r.returncode, 0) + + # =========================================================================== + # --output-json baseline extraction + # =========================================================================== + + def test_output_json_written_when_both_pass(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline.json") + self._run( + "--baseline", bl, + "--l0", l0, "--l1", l1, + "--output-json", out, + "--commit", "abc123", + "--timestamp", "2026-01-01T00:00:00Z", + ) + self.assertTrue(os.path.isfile(out), "output-json must be written") + with open(out) as fh: + data = json.load(fh) + self.assertEqual(data["L0"], 80.0) + self.assertEqual(data["L1"], 82.0) + self.assertEqual(data["commit"], "abc123") + self.assertEqual(data["timestamp"], "2026-01-01T00:00:00Z") + + def test_output_json_written_even_when_gate_fails(self): + """ + --output-json is written as long as coverage data is available, + regardless of gate outcome. The update-baseline step checks + `if [ ! -s new-baseline.json ]` separately. + """ + bl = self._baseline({"L0": 90.0, "L1": 90.0}) + l0 = self._lcov("l0.info", 100, 80) # 80 % < 90 % baseline → WARN + l1 = self._lcov("l1.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-fail.json") + r = self._run( + "--baseline", bl, + "--l0", l0, "--l1", l1, + "--output-json", out, + ) + # Informational only — always exit 0 regardless of gate outcome + self.assertEqual(r.returncode, 0) + self.assertTrue(os.path.isfile(out), "output-json written even on gate warning") + + def test_output_json_not_written_when_l0_missing(self): + """When --l0 is explicitly requested but its .info file is absent, --output-json must NOT be written.""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l1 = self._lcov("l1.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline-no-l0.json") + r = self._run("--baseline", bl, "--l0", "/nonexistent/l0.info", "--l1", l1, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when requested L0 data is absent") + self.assertIn("WARNING", r.stderr) + + def test_output_json_not_written_when_l1_missing(self): + """When --l1 is explicitly requested but its .info file is absent, --output-json must NOT be written.""" + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-no-l1.json") + r = self._run("--baseline", bl, "--l0", l0, "--l1", "/nonexistent/l1.info", "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when requested L1 data is absent") + self.assertIn("WARNING", r.stderr) + + def test_output_json_not_written_when_both_missing(self): + bl = self._baseline({"L0": 75.0, "L1": 75.0}) + out = os.path.join(self.tmp, "new-baseline-neither.json") + r = self._run("--baseline", bl, "--output-json", out) + self.assertFalse(os.path.isfile(out)) + self.assertIn("WARNING", r.stderr) + + def test_output_json_written_with_l1_only(self): + """When only --l1 is provided (no --l0), --output-json is written with just the L1 key. + This is the standard case for repos that have no L0 tests (e.g. Dobby).""" + bl = self._baseline({"L1": 75.0}) + l1 = self._lcov("l1.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-l1-only.json") + self._run( + "--baseline", bl, + "--l1", l1, + "--output-json", out, + "--commit", "abc123", + "--timestamp", "2026-06-22T00:00:00Z", + ) + self.assertTrue(os.path.isfile(out), "output-json must be written for L1-only repos") + with open(out) as fh: + data = json.load(fh) + self.assertEqual(data["L1"], 80.0) + self.assertNotIn("L0", data, "L0 key must not appear when --l0 was not passed") + self.assertEqual(data["commit"], "abc123") + + # =========================================================================== + # Baseline coercion: non-float L0/L1 values must not crash the script + # (Fixes comment 2/7 — baseline.get("L0") not validated as float) + # =========================================================================== + + def test_baseline_string_l0_treated_as_missing(self): + """String value for L0 in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"L0": "not-a-number", "L1": 75.0}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + # Must not crash; L0 baseline treated as absent → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_null_l1_treated_as_missing(self): + """null value for L1 in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"L0": 80.0, "L1": None}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_both_non_float_threshold_only(self): + """Both L0/L1 baseline values invalid → both threshold-only → pass if above 75%.""" + bl = self._baseline({"L0": "bad", "L1": "bad"}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_both_non_float_below_threshold_warns(self): + """Both L0/L1 baseline values invalid → threshold-only → [WARN] exit 0 if below 75%.""" + bl = self._baseline({"L0": "bad", "L1": "bad"}) + l0 = self._lcov("l0.info", 100, 70) + l1 = self._lcov("l1.info", 100, 70) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # _fmt_timestamp: null/non-string timestamp must not crash the report + # (Fixes comment 8 — only ValueError was caught, not TypeError) + # =========================================================================== + + def test_null_timestamp_in_baseline_does_not_crash(self): + """null timestamp value in baseline JSON → TypeError handled → report still runs.""" + bl = self._baseline({"L0": 80.0, "L1": 80.0, "commit": "abc", "timestamp": None}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + # Must not crash; timestamp renders as fallback; gate passes + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("OVERALL:", r.stdout) + + def test_integer_timestamp_in_baseline_does_not_crash(self): + """Integer timestamp → TypeError in strptime → handled gracefully.""" + bl = self._baseline({"L0": 80.0, "L1": 80.0, "commit": "abc", "timestamp": 12345}) + l0 = self._lcov("l0.info", 100, 80) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", l1) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/L1-tests.yml b/.github/workflows/L1-tests.yml index fbf1d252..7f2e6bb6 100755 --- a/.github/workflows/L1-tests.yml +++ b/.github/workflows/L1-tests.yml @@ -1,5 +1,8 @@ name: DobbyL1Test -on: [push, pull_request] +on: + push: + pull_request: + workflow_call: jobs: build: @@ -114,11 +117,12 @@ jobs: if: ${{ !env.ACT && matrix.coverage == 'with-coverage' && matrix.extra_flags == 'RUN_TESTS' && matrix.build_type == 'Debug' }} uses: actions/upload-artifact@v4 with: - name: artifacts + name: artifacts-L1-dobby path: | DobbyL1TestResults.json DobbyUtilsL1TestResults.json DobbyManagerL1TestResults.json DobbySpecConfigL1TestResults.json + filtered_coverage.info coverage if-no-files-found: warn diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml new file mode 100644 index 00000000..70fb4053 --- /dev/null +++ b/.github/workflows/main-workflow.yml @@ -0,0 +1,165 @@ +permissions: + contents: read +name: main-workflow + +on: + push: + branches: [ main, develop, 'sprint/**', 'release/**' ] + pull_request: + branches: [ main, develop, 'sprint/**', 'release/**' ] + +jobs: + trigger-L1: + uses: ./.github/workflows/L1-tests.yml + + coverage-gate: + name: Coverage Gate + needs: [trigger-L1] + runs-on: ubuntu-latest + # Transient runner/action failures must not block + continue-on-error: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Fetch baseline from build-metadata branch + # Gracefully handle a missing build-metadata branch (first-time setup) + continue-on-error: true + run: | + set -euo pipefail + if git fetch origin build-metadata 2>/dev/null; then + if git cat-file -e FETCH_HEAD:coverage-baseline.json 2>/dev/null; then + git show FETCH_HEAD:coverage-baseline.json > coverage-baseline.json + echo "Loaded coverage-baseline.json from build-metadata branch" + else + echo "build-metadata branch exists but coverage-baseline.json not found — skipping baseline comparison" + echo "{}" > coverage-baseline.json + fi + else + echo "build-metadata branch not found — absolute threshold check only (first-time setup)" + echo "{}" > coverage-baseline.json + fi + + - name: Download L1 coverage artifacts + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: artifacts-L1-dobby + path: ./l1-coverage + + - name: Compare coverage to baseline + run: | + python3 .github/scripts/compare_coverage.py \ + --baseline coverage-baseline.json \ + --l1 ./l1-coverage/filtered_coverage.info + + update-baseline: + name: Update Coverage Baseline + # Runs ONLY on a direct push to develop AND only when the L1 test suite + # passes. The coverage gate is informational only (always exits 0), so + # needs.coverage-gate.result == 'success' guards against infrastructure + # failures in the gate job itself, not coverage quality. + if: >- + github.event_name == 'push' && + github.ref == 'refs/heads/develop' && + needs.trigger-L1.result == 'success' && + needs.coverage-gate.result == 'success' + needs: [trigger-L1, coverage-gate] + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + # Queue concurrent runs; do not cancel in-progress — each merge deserves a baseline update and force-push is atomic so queuing is safe. + concurrency: + group: update-baseline-develop + cancel-in-progress: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Fetch existing baseline for comparison report + # Best-effort — if the branch or file is absent we compare against nothing. + continue-on-error: true + run: | + set -euo pipefail + if git fetch origin build-metadata 2>/dev/null; then + if git cat-file -e FETCH_HEAD:coverage-baseline.json 2>/dev/null; then + git show FETCH_HEAD:coverage-baseline.json > old-baseline.json + echo "Loaded existing baseline for comparison" + else + echo '{}' > old-baseline.json + fi + else + echo '{}' > old-baseline.json + fi + + - name: Download L1 coverage artifacts + uses: actions/download-artifact@v4 + with: + name: artifacts-L1-dobby + path: ./l1-coverage + + - name: Extract coverage and write new baseline + id: extract + run: | + set -euo pipefail + # Pass the old baseline so the report shows a before/after delta. + BL_ARG="old-baseline.json" + [ -f "$BL_ARG" ] || echo '{}' > "$BL_ARG" + + python3 .github/scripts/compare_coverage.py \ + --baseline "$BL_ARG" \ + --l1 ./l1-coverage/filtered_coverage.info \ + --output-json new-baseline.json \ + --commit "$GITHUB_SHA" \ + --timestamp "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + + if [ ! -s new-baseline.json ]; then + echo "Coverage extraction produced no output — skipping baseline update" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "New baseline to commit:" + cat new-baseline.json + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push updated baseline to build-metadata + if: steps.extract.outputs.skip == 'false' + run: | + set -euo pipefail + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + + # Check out the build-metadata branch, or create it as an orphan. + if git fetch origin build-metadata 2>/dev/null; then + git checkout -B build-metadata FETCH_HEAD + else + git checkout --orphan build-metadata + git rm -rf . --quiet 2>/dev/null || true + fi + + cp -f new-baseline.json coverage-baseline.json + + git add coverage-baseline.json + + # Only commit when there is an actual change. + if git diff --cached --quiet; then + echo "Coverage baseline unchanged — no commit needed" + else + MSG=$(printf \ + 'chore: update coverage baseline after develop merge [skip ci]\n\nCommit : %s\nRun ID : %s' \ + "$GITHUB_SHA" "$GITHUB_RUN_ID") + git commit -m "$MSG" + git push --force-with-lease origin build-metadata + echo "Pushed updated baseline to build-metadata" + fi From 36d61ce4e19fc3b5ed67d0ad4fa4d86bf4b7ecd9 Mon Sep 17 00:00:00 2001 From: swethasukumarr Date: Tue, 23 Jun 2026 13:55:57 -0400 Subject: [PATCH 2/2] RDKEMW-17673 : Address copilot comments --- .github/scripts/compare_coverage.py | 33 +++++++++-------- .github/scripts/compare_coverage_test.py | 45 +++++++++++------------- .github/workflows/L1-tests.yml | 2 ++ .github/workflows/main-workflow.yml | 3 ++ NOTICE | 3 ++ 5 files changed, 47 insertions(+), 39 deletions(-) diff --git a/.github/scripts/compare_coverage.py b/.github/scripts/compare_coverage.py index 1cd620c4..49044cd2 100644 --- a/.github/scripts/compare_coverage.py +++ b/.github/scripts/compare_coverage.py @@ -200,8 +200,8 @@ def load_baseline(path: str) -> dict: def main() -> None: parser = argparse.ArgumentParser( description=( - "Compare L0/L1 coverage against the develop baseline.\n" - "Exits 1 when coverage fails threshold or regresses from baseline." + "Compare L1 (and optionally L0) coverage against the develop baseline.\n" + "Always exits 0; prints [WARN] when coverage is below threshold or regresses from baseline." ) ) parser.add_argument("--baseline", required=True, metavar="PATH", @@ -255,10 +255,18 @@ def main() -> None: file=sys.stderr, ) - l0_ok, l0_result, l0_delta, l0_reason = _suite_analysis(l0_coverage, baseline_l0) - l1_ok, l1_result, l1_delta, l1_reason = _suite_analysis(l1_coverage, baseline_l1) - - all_ok = l0_ok and l1_ok + # Analyse only suites that were explicitly requested via --l0 / --l1. + # Non-requested suites are omitted entirely from the table and overall status. + _candidates = [("L0", l0_coverage, baseline_l0, args.l0), + ("L1", l1_coverage, baseline_l1, args.l1)] + suite_results = [] + for _name, _cov, _base, _arg in _candidates: + if not _arg: + continue + _ok, _result, _delta, _reason = _suite_analysis(_cov, _base) + suite_results.append((_name, _cov, _base, _result, _delta, _ok, _reason)) + + all_ok = all(ok for _, _, _, _, _, ok, _ in suite_results) if suite_results else True status_token = _colored("[PASS]", True) if all_ok else _colored("[WARN]", False) # Output report @@ -275,10 +283,7 @@ def main() -> None: # Coverage table print(f" {'Suite':<7}{'Current':<9}{'Baseline':<10}{'Delta':<10}Result") - for name, current, base, result, delta_disp in [ - ("L0", l0_coverage, baseline_l0, l0_result, l0_delta), - ("L1", l1_coverage, baseline_l1, l1_result, l1_delta), - ]: + for name, current, base, result, delta_disp, ok, reason in suite_results: cur_str = f"{current:.2f}%" if current is not None else "N/A" base_str = f"{base:.2f}%" if base is not None else "N/A" print(f" {name:<7}{cur_str:<9}{base_str:<10}{delta_disp:<10}{result}") @@ -286,14 +291,14 @@ def main() -> None: print(_SEP) # Summary + overall bar - warn_suites = [(n, r) for n, r in [("L0", l0_reason), ("L1", l1_reason)] if r] + warn_suites = [(name, reason) for name, _, _, _, _, ok, reason in suite_results if reason] summary = _build_summary(warn_suites) if summary: print(f" {summary}") - # Notify when one or both suites had no coverage data (artifact absent). - # Gate logic is unchanged — SKIP is treated as passing by design. - skipped = [n for n, cov in [("L0", l0_coverage), ("L1", l1_coverage)] if cov is None] + # Notify only when a requested suite had no coverage data (broken artifact). + # Gate logic is unchanged — missing data is treated as WARN by design. + skipped = [name for name, cov, _, _, _, _, _ in suite_results if cov is None] if skipped: print(f" NOTE: {_join_names(skipped)} coverage data absent \u2014 artifact missing or unreadable.") diff --git a/.github/scripts/compare_coverage_test.py b/.github/scripts/compare_coverage_test.py index 2ccda338..5fade93a 100644 --- a/.github/scripts/compare_coverage_test.py +++ b/.github/scripts/compare_coverage_test.py @@ -630,56 +630,51 @@ def test_s9_non_dict_json(self): self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) # =========================================================================== - # SCENARIO 10 — L0 job fails - # Coverage Gate does NOT trigger (workflow-level behaviour). - # - # GitHub Actions: coverage-gate has `needs: [trigger-L0, trigger-L1]` - # with no custom `if:`. The implicit success() check means coverage-gate - # is SKIPPED whenever trigger-L0 fails. update-baseline then sees - # needs.coverage-gate.result == 'skipped' (not 'success') and is also - # skipped. This cannot be unit-tested here; it is enforced by the - # workflow graph. + # SCENARIO 10 — Requested suite artifact missing (requested but no data) + # Script-level: suite is reported as WARN (data missing), exit 0. + # Workflow-level: coverage-gate depends on trigger-L1 via needs:; + # if trigger-L1 fails, coverage-gate is skipped automatically. # =========================================================================== def test_s10_l0_artifacts_absent_l1_passes(self): """ - Simulates the artifact-level effect: L0 .info absent (download step - with continue-on-error:true produced no file), L1 coverage present - and passing. Script-level: L0 is WARN (data missing), L1 is PASS. + --l0 is explicitly requested but the artifact file is absent. + L1 coverage is present and passing. + Script-level: L0 is WARN (data missing), L1 is PASS. """ bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l1 = self._lcov("l1.info", 100, 80) # L0 omitted intentionally - r = self._run("--baseline", bl, "--l1", l1) - # L0 WARN (missing) → overall WARN, but exit 0 (informational) + l1 = self._lcov("l1.info", 100, 80) + r = self._run("--baseline", bl, "--l0", "/nonexistent/l0.info", "--l1", l1) + # L0 requested but missing → overall WARN, but exit 0 (informational) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("coverage data missing", r.stdout) self.assertIn("[WARN]", r.stdout) # =========================================================================== - # SCENARIO 11 — L1 job fails (symmetric to scenario 10) + # SCENARIO 11 — L1 artifact missing (symmetric to scenario 10) # =========================================================================== def test_s11_l1_artifacts_absent_l0_passes(self): bl = self._baseline({"L0": 75.0, "L1": 75.0}) - l0 = self._lcov("l0.info", 100, 80) # L1 omitted intentionally - r = self._run("--baseline", bl, "--l0", l0) + l0 = self._lcov("l0.info", 100, 80) + r = self._run("--baseline", bl, "--l0", l0, "--l1", "/nonexistent/l1.info") self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) self.assertIn("coverage data missing", r.stdout) self.assertIn("[WARN]", r.stdout) # =========================================================================== - # SCENARIO 12 — Both L0 AND L1 jobs fail - # Coverage Gate does NOT trigger (workflow-level). At script level, both - # .info files are absent → both SKIP → exit 0 (harmless; gate is already - # blocked at the workflow graph layer before the script is ever called). + # SCENARIO 12 — Both artifacts missing (both explicitly requested but absent) # =========================================================================== def test_s12_both_artifacts_absent(self): bl = self._baseline({"L0": 75.0, "L1": 75.0}) - # Neither --l0 nor --l1 provided - r = self._run("--baseline", bl) + r = self._run( + "--baseline", bl, + "--l0", "/nonexistent/l0.info", + "--l1", "/nonexistent/l1.info", + ) self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) - # Both rows + summary line show coverage data missing, OVERALL WARN + # Both rows show coverage data missing, OVERALL WARN self.assertGreaterEqual(r.stdout.count("coverage data missing"), 2) self.assertIn("[WARN]", r.stdout) self.assertIn("NOTE:", r.stdout) diff --git a/.github/workflows/L1-tests.yml b/.github/workflows/L1-tests.yml index 7f2e6bb6..e877e6a5 100755 --- a/.github/workflows/L1-tests.yml +++ b/.github/workflows/L1-tests.yml @@ -1,7 +1,9 @@ name: DobbyL1Test on: push: + branches-ignore: [ main, develop, 'sprint/**', 'release/**' ] pull_request: + branches-ignore: [ main, develop, 'sprint/**', 'release/**' ] workflow_call: jobs: diff --git a/.github/workflows/main-workflow.yml b/.github/workflows/main-workflow.yml index 70fb4053..89f8cbab 100644 --- a/.github/workflows/main-workflow.yml +++ b/.github/workflows/main-workflow.yml @@ -16,6 +16,9 @@ jobs: name: Coverage Gate needs: [trigger-L1] runs-on: ubuntu-latest + permissions: + contents: read + actions: read # Transient runner/action failures must not block continue-on-error: true steps: diff --git a/NOTICE b/NOTICE index 2cd07df1..3959705f 100644 --- a/NOTICE +++ b/NOTICE @@ -17,3 +17,6 @@ Used test results from RFC 1321 Copyright 2023 Synamedia Licensed under the Apache License, Version 2.0 +Copyright 2026 RDK Management +Licensed under the Apache License, Version 2.0 +