diff --git a/.github/scripts/compare_coverage.py b/.github/scripts/compare_coverage.py new file mode 100644 index 0000000..9fb7699 --- /dev/null +++ b/.github/scripts/compare_coverage.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 +# Copyright 2026 Comcast Cable Communications Management, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +""" +Coverage comparison script for firebolt-cpp-client. + +Reads unit test and component test coverage results, compares overall line +coverage against the stored baseline from the build-metadata branch, and prints +a summary. +""" + +import argparse +import datetime +import json +import os +import sys +from typing import Optional + + +# Minimum threshold +THRESHOLD = 75.0 + +_GREEN = "\033[32m" +_RED = "\033[31m" +_RESET = "\033[0m" + +_SEP_WIDTH = 64 +_OVERALL_WIDTH = 80 +_SEP = "\u2500" * _SEP_WIDTH +_HEADER = "\u2500\u2500 Coverage Gate Report " + "\u2500" * (_SEP_WIDTH - 24) + + +def _colored(token: str, ok: bool) -> str: + return f"{_GREEN if ok else _RED}{token}{_RESET}" + + +def _fmt_timestamp(ts: str) -> str: + """Convert '2026-05-28T12:00:00Z' -> '2026-05-28 12:00 UTC'.""" + try: + dt = datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ") + return dt.strftime("%Y-%m-%d %H:%M UTC") + except (ValueError, TypeError): + return ts + + +def _delta_str(current: float, baseline: float) -> str: + delta = current - baseline + sign = "+" if delta >= 0 else "" + return f"{sign}{delta:.2f}%" + + +def _join_names(names: list) -> str: + return names[0] if len(names) == 1 else " and ".join(names) + + +def _suite_analysis(current: Optional[float], baseline: Optional[float]): + """Analyse one test suite. + + Returns (ok, result_str, delta_disp, warn_reason): + ok - True when no advisory issues found. + result_str - Coloured [PASS]/[WARN] token + detail for the table. + delta_disp - String for the Delta column ("N/A" when skipped). + warn_reason - Reason phrase for the summary line; None when ok. + """ + if current is None: + reason = "coverage data missing" + return False, f"{_colored('[WARN]', False)} {reason}", "N/A", reason + + threshold_ok = current >= THRESHOLD + + if baseline is None: + # No baseline stored — threshold check only. + regression_ok = True + detail = "" if threshold_ok else "below threshold" + delta_disp = "N/A" + elif baseline == 0.0: + # A zero baseline is unreliable — skip regression check. + regression_ok = True + base_note = "baseline unreliable (0%) \u00b7 delta skipped" + detail = f"below threshold \u00b7 {base_note}" if not threshold_ok else base_note + delta_disp = "N/A" + else: + regression_ok = current >= baseline + delta_disp = _delta_str(current, baseline) + if threshold_ok and regression_ok: + detail = "" + elif not threshold_ok and not regression_ok: + detail = "below threshold \u00b7 dropped from baseline" + elif not threshold_ok: + detail = "below threshold \u00b7 no baseline regression" + else: + detail = "above threshold but dropped from baseline" + + overall_ok = threshold_ok and regression_ok + token = _colored("[PASS]", True) if overall_ok else _colored("[WARN]", False) + result_str = f"{token} {detail}" if detail else token + warn_reason = detail if not overall_ok else None + return overall_ok, result_str, delta_disp, warn_reason + + +def _build_summary(warn_suites: list) -> str: + """Build a compact summary from WARN suite (name, reason) pairs.""" + if not warn_suites: + return "" + groups: dict = {} + for name, reason in warn_suites: + groups.setdefault(reason, []).append(name) + parts = [f"{_join_names(names)} {reason}" for reason, names in groups.items()] + return ". ".join(parts) + + +# lcov parsing +def parse_lcov_coverage(path: str) -> Optional[float]: + """Return overall line coverage % from an lcov .info file, or None. + + An lcov .info file contains per-source-file records separated by + ``end_of_record``. Each record may include: + LF: — total instrumented lines in that file + LH: — lines executed at least once + + We aggregate across all records to produce a single project-wide %. + Returns None when the file is absent, empty, or contains no line data. + """ + if not path or not os.path.isfile(path): + return None + + total_found = 0 + total_hit = 0 + + try: + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for raw in fh: + line = raw.strip() + if line.startswith("LF:"): + try: + total_found += int(line[3:]) + except ValueError: + pass + elif line.startswith("LH:"): + try: + total_hit += int(line[3:]) + except ValueError: + pass + except OSError as exc: + print(f" WARNING: Could not read {path}: {exc}", file=sys.stderr) + return None + + if total_found == 0: + return None + + return round((total_hit / total_found) * 100.0, 2) + + + +# Baseline loading +def load_baseline(path: str) -> dict: + """Load baseline JSON; return an empty dict on any error.""" + if not path or not os.path.isfile(path): + return {} + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + if isinstance(data, dict): + return data + print( + f" WARNING: Baseline {path} is not a JSON object (got {type(data).__name__}) — ignoring", + file=sys.stderr, + ) + except (OSError, json.JSONDecodeError, ValueError) as exc: + print(f" WARNING: Could not parse baseline {path}: {exc}", file=sys.stderr) + return {} + + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Compare unit test and component test coverage against " + "the develop baseline. Informational only — always exits 0 and does " + "not block PRs." + ) + ) + parser.add_argument("--baseline", required=True, metavar="PATH", + help="Path to coverage-baseline.json.") + parser.add_argument("--unit", required=False, metavar="PATH", + help="Path to the unit test lcov filtered_coverage.info file.") + parser.add_argument("--component", required=False, metavar="PATH", + help="Path to the component test lcov filtered_coverage.info file.") + parser.add_argument("--output-json", required=False, metavar="PATH", + help="Write {Unit, Component, commit, timestamp} JSON here for baseline update.") + parser.add_argument("--commit", required=False, default="", + help="Commit SHA to embed in --output-json.") + parser.add_argument("--timestamp", required=False, default="", + help="ISO 8601 timestamp to embed in --output-json.") + args = parser.parse_args() + + baseline = load_baseline(args.baseline) + + def _coerce_pct(value: object) -> Optional[float]: + """Coerce a baseline percentage value to float, or None if invalid.""" + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + baseline_unit: Optional[float] = _coerce_pct(baseline.get("Unit")) + baseline_component: Optional[float] = _coerce_pct(baseline.get("Component")) + + unit_coverage = parse_lcov_coverage(args.unit) if args.unit else None + component_coverage = parse_lcov_coverage(args.component) if args.component else None + + # ------------------------------------------------------------------ + # Optional: write extracted numbers for baseline update. + # Skipped (with a warning) when either suite lacks valid coverage data. + # ------------------------------------------------------------------ + if args.output_json: + if unit_coverage is not None and component_coverage is not None: + payload = { + "Unit": unit_coverage, + "Component": component_coverage, + "commit": args.commit or "", + "timestamp": args.timestamp or "", + } + try: + with open(args.output_json, "w", encoding="utf-8") as fh: + json.dump(payload, fh, indent=2) + fh.write("\n") + except OSError as exc: + print(f" WARNING: Could not write {args.output_json}: {exc}", file=sys.stderr) + else: + print( + f" WARNING: --output-json skipped: coverage data incomplete " + f"(Unit={unit_coverage}, Component={component_coverage})", + file=sys.stderr, + ) + + unit_ok, unit_result, unit_delta, unit_reason = _suite_analysis(unit_coverage, baseline_unit) + component_ok, component_result, component_delta, component_reason = _suite_analysis(component_coverage, baseline_component) + + all_ok = unit_ok and component_ok + status_token = _colored("[PASS]", True) if all_ok else _colored("[WARN]", False) + + # Output report + print() + print(_HEADER) + if baseline: + commit = baseline.get("commit", "unknown") + ts = _fmt_timestamp(baseline.get("timestamp", "")) + print(f" Baseline {commit} ({ts})") + else: + print(" Baseline N/A (first-time setup \u2014 regression check skipped)") + print(f" Threshold {THRESHOLD}% | Status {status_token} (informational \u2014 PRs are not blocked)") + print(_SEP) + + # Coverage table + print(f" {'Suite':<12}{'Current':<9}{'Baseline':<10}{'Delta':<10}Result") + for name, current, base, result, delta_disp in [ + ("Unit", unit_coverage, baseline_unit, unit_result, unit_delta), + ("Component", component_coverage, baseline_component, component_result, component_delta), + ]: + cur_str = f"{current:.2f}%" if current is not None else "N/A" + base_str = f"{base:.2f}%" if base is not None else "N/A" + print(f" {name:<12}{cur_str:<9}{base_str:<10}{delta_disp:<10}{result}") + + print(_SEP) + + # Summary + overall bar + warn_suites = [(n, r) for n, r in [("Unit", unit_reason), ("Component", component_reason)] if r] + summary = _build_summary(warn_suites) + if summary: + print(f" {summary}") + + # Notify when one or both suites had no coverage data (artifact absent). + # Missing data is reported as [WARN]; the gate remains informational. + skipped = [n for n, cov in [("Unit", unit_coverage), ("Component", component_coverage)] if cov is None] + if skipped: + print(f" NOTE: {_join_names(skipped)} coverage data absent \u2014 artifact missing or unreadable.") + + # " OVERALL: [PASS/WARN] " = 1 + 9 + 6 + 1 = 17 visible chars + # left + " OVERALL: " + token(6) + " " + right == _OVERALL_WIDTH + _mid = len(" OVERALL: ") + 6 + len(" ") # 17 + left = "\u2500" * ((_OVERALL_WIDTH - _mid) // 2) # 31 + right = "\u2500" * (_OVERALL_WIDTH - _mid - len(left)) # 32 + print(f"{left} OVERALL: {status_token} {right}") + print() + + # Informational only — always exit 0 so PRs are never blocked. + sys.exit(0) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/compare_coverage_test.py b/.github/scripts/compare_coverage_test.py new file mode 100644 index 0000000..73e956f --- /dev/null +++ b/.github/scripts/compare_coverage_test.py @@ -0,0 +1,905 @@ +#!/usr/bin/env python3 +# Copyright 2026 Comcast Cable Communications Management, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +""" +Tests for compare_coverage.py + +Covers: + - Unit tests for parse_lcov_coverage(), load_baseline(), _suite_analysis() + - Integration tests (subprocess) simulating all gate scenarios listed in the + Coverage Gate implementation spec. + +Workflow-level scenarios (unit_tests fails / component_tests fails / both fail) are handled +by GitHub Actions' implicit success() dependency check on the coverage-gate job +and cannot be tested at the Python script level; they are documented inline. +""" + +import json +import os +import subprocess +import sys +import tempfile +import unittest + +# --------------------------------------------------------------------------- +# Import the module under test +# --------------------------------------------------------------------------- +SCRIPTS_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, SCRIPTS_DIR) + +import compare_coverage # noqa: E402 (after sys.path manipulation) + +THRESHOLD = compare_coverage.THRESHOLD # 75.0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_lcov(lines_found: int, lines_hit: int) -> str: + """Minimal valid lcov .info content with the given LF/LH counts.""" + return ( + "SF:src/fake.cpp\n" + f"LF:{lines_found}\n" + f"LH:{lines_hit}\n" + "end_of_record\n" + ) + + +def _write_lcov(tmp_dir: str, name: str, lines_found: int, lines_hit: int) -> str: + """Write an lcov file and return its absolute path.""" + path = os.path.join(tmp_dir, name) + with open(path, "w") as fh: + fh.write(_make_lcov(lines_found, lines_hit)) + return path + + +def _write_baseline(tmp_dir: str, data: dict, name: str = "baseline.json") -> str: + """Serialise *data* to JSON and return the path.""" + path = os.path.join(tmp_dir, name) + with open(path, "w") as fh: + json.dump(data, fh) + return path + + +def _run_script(*args: str) -> subprocess.CompletedProcess: + """Invoke compare_coverage.py as a subprocess and return the result.""" + cmd = [sys.executable, os.path.join(SCRIPTS_DIR, "compare_coverage.py"), *args] + return subprocess.run(cmd, capture_output=True, text=True) + + +# =========================================================================== +# Unit tests — parse_lcov_coverage() +# =========================================================================== + +class TestParseLcovCoverage(unittest.TestCase): + """Tests for the lcov .info parser.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write(self, name: str, content: str) -> str: + path = os.path.join(self.tmp, name) + with open(path, "w") as fh: + fh.write(content) + return path + + # --- Missing / empty inputs ------------------------------------------------- + + def test_none_path_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage(None)) + + def test_empty_path_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage("")) + + def test_nonexistent_file_returns_none(self): + self.assertIsNone(compare_coverage.parse_lcov_coverage("/no/such/file.info")) + + def test_empty_file_returns_none(self): + p = self._write("empty.info", "") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_no_lf_data_returns_none(self): + p = self._write("no_lf.info", "SF:foo.cpp\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_lf_zero_returns_none(self): + p = self._write("zero_lf.info", "SF:foo.cpp\nLF:0\nLH:0\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + # --- Basic coverage values -------------------------------------------------- + + def test_100_percent(self): + p = _write_lcov(self.tmp, "full.info", 100, 100) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 100.0) + + def test_75_percent_exact(self): + p = _write_lcov(self.tmp, "seventy_five.info", 100, 75) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 75.0) + + def test_zero_percent(self): + p = _write_lcov(self.tmp, "zero_pct.info", 100, 0) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 0.0) + + def test_partial_coverage(self): + # 150 / 200 = 75.0 % + p = _write_lcov(self.tmp, "partial.info", 200, 150) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 75.0) + + # --- Multi-record aggregation ----------------------------------------------- + + def test_aggregates_multiple_records(self): + # 80 + 60 = 140 hit out of 200 → 70.0 % + content = ( + "SF:a.cpp\nLF:100\nLH:80\nend_of_record\n" + "SF:b.cpp\nLF:100\nLH:60\nend_of_record\n" + ) + p = self._write("multi.info", content) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 70.0) + + # --- Malformed data --------------------------------------------------------- + + def test_malformed_lf_ignored_gracefully(self): + # LF with a non-numeric value; total_found stays 0 → None + p = self._write("bad_lf.info", "SF:a.cpp\nLF:abc\nLH:50\nend_of_record\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + def test_malformed_lh_ignored_gracefully(self): + # LH with garbage value; LF is valid, so LF=100, LH=0 → 0.0 % + p = self._write("bad_lh.info", "SF:a.cpp\nLF:100\nLH:xyz\nend_of_record\n") + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 0.0) + + def test_entirely_non_lcov_content(self): + p = self._write("corrupt.info", "THIS IS NOT A VALID LCOV FILE\n") + self.assertIsNone(compare_coverage.parse_lcov_coverage(p)) + + # --- Rounding --------------------------------------------------------------- + + def test_rounds_to_two_decimal_places(self): + # 1/3 ≈ 33.33 % + p = _write_lcov(self.tmp, "third.info", 3, 1) + self.assertEqual(compare_coverage.parse_lcov_coverage(p), 33.33) + + +# =========================================================================== +# Unit tests — load_baseline() +# =========================================================================== + +class TestLoadBaseline(unittest.TestCase): + """Tests for the JSON baseline loader.""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + def _write_json(self, name: str, content: str) -> str: + path = os.path.join(self.tmp, name) + with open(path, "w") as fh: + fh.write(content) + return path + + # --- Missing / empty inputs ------------------------------------------------- + + def test_none_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline(None), {}) + + def test_empty_path_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline(""), {}) + + def test_nonexistent_file_returns_empty_dict(self): + self.assertEqual(compare_coverage.load_baseline("/no/such/file.json"), {}) + + # --- Valid JSON ------------------------------------------------------------- + + def test_valid_baseline_with_unit_and_component(self): + p = _write_baseline(self.tmp, {"Unit": 80.0, "Component": 85.0}) + self.assertEqual(compare_coverage.load_baseline(p), {"Unit": 80.0, "Component": 85.0}) + + def test_valid_empty_json_object(self): + p = _write_baseline(self.tmp, {}) + self.assertEqual(compare_coverage.load_baseline(p), {}) + + def test_valid_baseline_extra_keys_preserved(self): + data = {"Unit": 80.0, "Component": 85.0, "commit": "abc123", "timestamp": "2026-01-01"} + p = _write_baseline(self.tmp, data) + self.assertEqual(compare_coverage.load_baseline(p), data) + + # --- Invalid JSON ----------------------------------------------------------- + + def test_malformed_json_returns_empty_dict(self): + p = self._write_json("bad.json", "{not valid json}") + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + + def test_truncated_json_returns_empty_dict(self): + p = self._write_json("truncated.json", '{"Unit": 80') + self.assertEqual(compare_coverage.load_baseline(p), {}) + + def test_empty_file_returns_empty_dict(self): + p = self._write_json("empty.json", "") + self.assertEqual(compare_coverage.load_baseline(p), {}) + + # --- Non-dict JSON ---------------------------------------------------------- + + def test_json_array_returns_empty_dict(self): + import io, contextlib + p = self._write_json("list.json", "[1, 2, 3]") + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_string_returns_empty_dict(self): + import io, contextlib + p = self._write_json("str.json", '"just a string"') + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_number_returns_empty_dict(self): + import io, contextlib + p = self._write_json("num.json", "42") + buf = io.StringIO() + with contextlib.redirect_stderr(buf): + result = compare_coverage.load_baseline(p) + self.assertEqual(result, {}) + self.assertIn("WARNING", buf.getvalue()) + + def test_json_null_returns_empty_dict(self): + p = self._write_json("null.json", "null") + # null is parsed as None, which is not a dict — Warning emitted, empty dict returned + self.assertEqual(compare_coverage.load_baseline(p), {}) + + +# =========================================================================== +# Unit tests — _suite_analysis() +# =========================================================================== + +class TestSuiteAnalysis(unittest.TestCase): + """ + Tests for the core gate analysis function. + + Gate passes (ok=True) when BOTH: + 1. current >= THRESHOLD (75.0) + 2. current >= baseline (regression check) + + SKIP when current is None. + Regression check disabled when baseline is None or 0.0. + """ + + # --- Scenario 1: exceeds both threshold AND baseline → PASS ---------------- + + def test_s1_exceeds_threshold_and_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(80.0, 77.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 2: meets threshold exactly (75%) AND beats baseline → PASS --- + + def test_s2_meets_threshold_exactly_beats_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 70.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 3: meets baseline exactly, exceeds threshold → PASS ---------- + + def test_s3_meets_baseline_exactly_exceeds_threshold(self): + ok, _, _, reason = compare_coverage._suite_analysis(80.0, 80.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 4: meets BOTH exactly (75.0 == threshold == baseline) → PASS - + + def test_s4_meets_both_exactly_at_threshold(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 75.0) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- Scenario 5: exceeds threshold but BELOW baseline → FAIL (regression) -- + + def test_s5_above_threshold_below_baseline(self): + ok, result, _, reason = compare_coverage._suite_analysis(76.0, 80.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("dropped from baseline", reason) + + # --- Scenario 6: below threshold but meets/exceeds baseline → FAIL ---------- + + def test_s6_below_threshold_meets_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 70.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertNotIn("dropped from baseline", reason) # regression check passed + + def test_s6b_below_threshold_equals_baseline(self): + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 74.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertNotIn("dropped from baseline", reason) # regression check passed + + # --- Scenario 7: fails BOTH conditions → FAIL -------------------------------- + + def test_s7_fails_both_conditions(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, 80.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + self.assertIn("dropped from baseline", reason) + + # --- Scenario 8: no baseline → threshold-only check ------------------------- + + def test_no_baseline_above_threshold_passes(self): + ok, _, delta, reason = compare_coverage._suite_analysis(80.0, None) + self.assertTrue(ok) + self.assertIsNone(reason) + self.assertEqual(delta, "N/A") + + def test_no_baseline_below_threshold_fails(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, None) + self.assertFalse(ok) + self.assertIn("below threshold", reason) + + def test_no_baseline_at_threshold_exactly_passes(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, None) + self.assertTrue(ok) + self.assertIsNone(reason) + + # --- SKIP case: no current coverage ----------------------------------------- + + def test_skip_when_current_is_none(self): + ok, result, delta, reason = compare_coverage._suite_analysis(None, 80.0) + self.assertFalse(ok, "Missing coverage data should be treated as WARN") + self.assertIn("coverage data missing", result) + self.assertEqual(delta, "N/A") + self.assertIsNotNone(reason) + self.assertIn("coverage data missing", reason) + + def test_skip_when_both_none(self): + ok, result, delta, reason = compare_coverage._suite_analysis(None, None) + self.assertFalse(ok) + self.assertIn("coverage data missing", result) + + # --- Zero baseline: regression check disabled -------------------------------- + + def test_zero_baseline_above_threshold_passes(self): + ok, _, delta, reason = compare_coverage._suite_analysis(80.0, 0.0) + self.assertTrue(ok) + self.assertIsNone(reason) + self.assertEqual(delta, "N/A", "Delta must be N/A for zero baseline") + + def test_zero_baseline_below_threshold_fails(self): + ok, _, _, reason = compare_coverage._suite_analysis(70.0, 0.0) + self.assertFalse(ok) + self.assertIsNotNone(reason) + self.assertIn("below threshold", reason) + + # --- Delta string correctness ------------------------------------------------ + + def test_delta_positive(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, 77.0) + self.assertEqual(delta, "+3.00%") + + def test_delta_negative(self): + _, _, delta, _ = compare_coverage._suite_analysis(76.0, 80.0) + self.assertEqual(delta, "-4.00%") + + def test_delta_zero(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, 80.0) + self.assertEqual(delta, "+0.00%") + + def test_delta_na_when_no_baseline(self): + _, _, delta, _ = compare_coverage._suite_analysis(80.0, None) + self.assertEqual(delta, "N/A") + + # --- Boundary: one tick below threshold (74.99 is impossible from lcov, + # but 74.0 covers the just-below case) ---------------------------------- + + def test_just_below_threshold_fails(self): + # 74 / 100 = 74.0 % + ok, _, _, reason = compare_coverage._suite_analysis(74.0, 70.0) + self.assertFalse(ok) + + def test_just_at_threshold_passes(self): + ok, _, _, reason = compare_coverage._suite_analysis(75.0, 70.0) + self.assertTrue(ok) + + +# =========================================================================== +# Integration tests — main() via subprocess +# =========================================================================== + +class TestMainIntegration(unittest.TestCase): + """ + End-to-end simulation of every gate scenario. + + Each test invokes the script as a subprocess (exactly as GitHub Actions + would) and asserts on exit code and stdout/stderr content. + """ + + def setUp(self): + self.tmp = tempfile.mkdtemp() + + def tearDown(self): + import shutil + shutil.rmtree(self.tmp, ignore_errors=True) + + # --- Helpers ---------------------------------------------------------------- + + def _lcov(self, name: str, lf: int, lh: int) -> str: + return _write_lcov(self.tmp, name, lf, lh) + + def _baseline(self, data: dict, name: str = "baseline.json") -> str: + return _write_baseline(self.tmp, data, name) + + def _run(self, *args: str) -> subprocess.CompletedProcess: + return _run_script(*args) + + # =========================================================================== + # SCENARIO 1 — Coverage exceeds both threshold AND baseline + # Expected: Gate PASSES (exit 0), baseline updates + # =========================================================================== + + def test_s1_exceeds_threshold_and_baseline(self): + bl = self._baseline({"Unit": 77.0, "Component": 78.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % + component_cov = self._lcov("component.info", 100, 82) # 82 % + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[PASS]", r.stdout) + + # =========================================================================== + # SCENARIO 2 — Coverage meets threshold exactly (75%) and meets baseline + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s2_meets_threshold_exactly_meets_baseline(self): + bl = self._baseline({"Unit": 70.0, "Component": 70.0}) + unit_cov = self._lcov("unit.info", 100, 75) # 75.0 % + component_cov = self._lcov("component.info", 100, 75) # 75.0 % + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[PASS]", r.stdout) + + # =========================================================================== + # SCENARIO 3 — Meets baseline exactly but exceeds threshold + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s3_meets_baseline_exactly_exceeds_threshold(self): + bl = self._baseline({"Unit": 80.0, "Component": 80.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % == baseline + component_cov = self._lcov("component.info", 100, 80) # 80 % == baseline + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 4 — Meets BOTH exactly (current == threshold == baseline == 75 %) + # Expected: Gate PASSES (exit 0) + # =========================================================================== + + def test_s4_meets_both_exactly(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 75) + component_cov = self._lcov("component.info", 100, 75) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 5 — Exceeds threshold but falls BELOW baseline (regression) + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s5_above_threshold_below_baseline(self): + bl = self._baseline({"Unit": 85.0, "Component": 85.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % < 85 % baseline + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + self.assertIn("dropped from baseline", r.stdout) + + # =========================================================================== + # SCENARIO 6 — Falls BELOW threshold but meets/exceeds baseline + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s6_below_threshold_meets_baseline(self): + bl = self._baseline({"Unit": 70.0, "Component": 70.0}) + unit_cov = self._lcov("unit.info", 100, 74) # 74 % < 75 % threshold + component_cov = self._lcov("component.info", 100, 74) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + self.assertIn("below threshold", r.stdout) + + # =========================================================================== + # SCENARIO 7 — Fails BOTH conditions (below threshold AND below baseline) + # Expected: Gate WARNS (exit 0 — informational only), [WARN] shown + # =========================================================================== + + def test_s7_fails_both_threshold_and_baseline(self): + bl = self._baseline({"Unit": 85.0, "Component": 85.0}) + unit_cov = self._lcov("unit.info", 100, 70) # 70 % < threshold AND < baseline + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("below threshold", r.stdout) + self.assertIn("dropped from baseline", r.stdout) + + # =========================================================================== + # SCENARIO 8a — Baseline file is MISSING + # Expected: Graceful fallback; threshold-only check; no crash + # =========================================================================== + + def test_s8_baseline_missing_coverage_above_threshold(self): + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run( + "--baseline", "/nonexistent/coverage-baseline.json", + "--unit", unit_cov, "--component", component_cov, + ) + # No baseline → regression skipped → threshold pass → exit 0 + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s8_baseline_missing_coverage_below_threshold(self): + unit_cov = self._lcov("unit.info", 100, 70) # 70 % < 75 % + component_cov = self._lcov("component.info", 100, 70) + r = self._run( + "--baseline", "/nonexistent/coverage-baseline.json", + "--unit", unit_cov, "--component", component_cov, + ) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 9 — Baseline file contains invalid / malformed JSON + # Expected: Warning emitted, treated as empty baseline, gate continues + # =========================================================================== + + def test_s9_malformed_json_above_threshold(self): + path = os.path.join(self.tmp, "malformed.json") + with open(path, "w") as fh: + fh.write("{this is not json}") + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) + # Warning must appear in stderr + self.assertIn("WARNING", r.stderr) + # Fallback to empty baseline → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s9_malformed_json_below_threshold(self): + path = os.path.join(self.tmp, "malformed2.json") + with open(path, "w") as fh: + fh.write("{bad json") + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) + self.assertIn("WARNING", r.stderr) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0) + self.assertIn("[WARN]", r.stdout) + + def test_s9_empty_json_file(self): + path = os.path.join(self.tmp, "empty.json") + with open(path, "w") as fh: + fh.write("") + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) + # Empty file → empty dict baseline → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_s9_non_dict_json(self): + path = os.path.join(self.tmp, "list_json.json") + with open(path, "w") as fh: + json.dump([1, 2, 3], fh) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", path, "--unit", unit_cov, "--component", component_cov) + # Non-dict JSON → WARNING emitted, empty dict → threshold-only → pass + self.assertIn("WARNING", r.stderr) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + # =========================================================================== + # SCENARIO 10 — unit_tests job fails + # Coverage Gate does NOT trigger (workflow-level behaviour). + # + # GitHub Actions: coverage_gate has `needs: [unit_tests, component_tests]` + # with no custom `if:`. The implicit success() check means coverage_gate + # is SKIPPED whenever unit_tests fails. This cannot be unit-tested here; + # it is enforced by the workflow graph. + # =========================================================================== + + def test_s10_unit_artifacts_absent_component_passes(self): + """ + Simulates the artifact-level effect: unit .info absent (download step + with continue-on-error:true produced no file), component coverage present + and passing. Script-level: Unit is WARN (data missing), Component is PASS. + """ + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + component_cov = self._lcov("component.info", 100, 80) # Unit omitted intentionally + r = self._run("--baseline", bl, "--component", component_cov) + # Unit WARN (missing) → overall WARN, but exit 0 (informational) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("coverage data missing", r.stdout) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 11 — component_tests job fails (symmetric to scenario 10) + # =========================================================================== + + def test_s11_component_artifacts_absent_unit_passes(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) # Component omitted intentionally + r = self._run("--baseline", bl, "--unit", unit_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("coverage data missing", r.stdout) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # SCENARIO 12 — Both unit_tests AND component_tests jobs fail + # Coverage Gate does NOT trigger (workflow-level). At script level, both + # .info files are absent → both SKIP → exit 0 (harmless; gate is already + # blocked at the workflow graph layer before the script is ever called). + # =========================================================================== + + def test_s12_both_artifacts_absent(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + # Neither --unit nor --component provided + r = self._run("--baseline", bl) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + # Both rows + summary line show coverage data missing, OVERALL WARN + self.assertGreaterEqual(r.stdout.count("coverage data missing"), 2) + self.assertIn("[WARN]", r.stdout) + self.assertIn("NOTE:", r.stdout) + + # =========================================================================== + # SCENARIO 13 — Coverage Gate step itself throws an unexpected error + # Expected: non-zero exit; error is visible; baseline NOT updated + # (Simulated by passing a completely invalid path for --baseline that + # causes the argument parser or file logic to surface an error.) + # =========================================================================== + + def test_s13_missing_required_baseline_arg(self): + """Invoking the script without --baseline must fail (argparse error).""" + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--unit", unit_cov, "--component", component_cov) + # argparse exits with code 2 on missing required argument + self.assertNotEqual(r.returncode, 0) + self.assertTrue(len(r.stderr) > 0, "Error must appear on stderr") + + # =========================================================================== + # Partial-failure cases: one suite fails, other passes + # =========================================================================== + + def test_only_unit_fails_gate_warns(self): + """Unit below threshold, Component passes → overall [WARN] but exit 0.""" + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 70) # 70 % ✗ + component_cov = self._lcov("component.info", 100, 80) # 80 % ✓ + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + # Component row still shows PASS + self.assertIn("[PASS]", r.stdout) + self.assertIn("[WARN]", r.stdout) + + def test_only_component_fails_gate_warns(self): + """Component below threshold, Unit passes → overall [WARN] but exit 0.""" + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % ✓ + component_cov = self._lcov("component.info", 100, 70) # 70 % ✗ + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + def test_only_unit_regresses_gate_warns(self): + """Unit regresses below baseline (still above threshold), Component passes → [WARN] exit 0.""" + bl = self._baseline({"Unit": 85.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % < 85 % baseline ✗ + component_cov = self._lcov("component.info", 100, 80) # 80 % >= 75 % baseline ✓ + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # First-time setup: empty baseline {} → threshold-only + # =========================================================================== + + def test_first_time_setup_empty_baseline_passes(self): + bl = self._baseline({}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_first_time_setup_empty_baseline_below_threshold_warns(self): + bl = self._baseline({}) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + # Informational only — exit 0 even below threshold; [WARN] shown + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # Output format validation + # =========================================================================== + + def test_overall_pass_token_in_output(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertIn("OVERALL:", r.stdout) + self.assertIn("[PASS]", r.stdout) + + def test_overall_warn_token_in_output(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertIn("OVERALL:", r.stdout) + self.assertIn("[WARN]", r.stdout) + # Informational only — always exit 0 + self.assertEqual(r.returncode, 0) + + # =========================================================================== + # --output-json baseline extraction + # =========================================================================== + + def test_output_json_written_when_both_pass(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline.json") + self._run( + "--baseline", bl, + "--unit", unit_cov, "--component", component_cov, + "--output-json", out, + "--commit", "abc123", + "--timestamp", "2026-01-01T00:00:00Z", + ) + self.assertTrue(os.path.isfile(out), "output-json must be written") + with open(out) as fh: + data = json.load(fh) + self.assertEqual(data["Unit"], 80.0) + self.assertEqual(data["Component"], 82.0) + self.assertEqual(data["commit"], "abc123") + self.assertEqual(data["timestamp"], "2026-01-01T00:00:00Z") + + def test_output_json_written_even_when_gate_fails(self): + """ + --output-json is written as long as coverage data is available, + regardless of gate outcome. The update-baseline step checks + `if [ ! -s new-baseline.json ]` separately. + """ + bl = self._baseline({"Unit": 90.0, "Component": 90.0}) + unit_cov = self._lcov("unit.info", 100, 80) # 80 % < 90 % baseline → WARN + component_cov = self._lcov("component.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-fail.json") + r = self._run( + "--baseline", bl, + "--unit", unit_cov, "--component", component_cov, + "--output-json", out, + ) + # Informational only — always exit 0 regardless of gate outcome + self.assertEqual(r.returncode, 0) + self.assertTrue(os.path.isfile(out), "output-json written even on gate warning") + + def test_output_json_not_written_when_unit_missing(self): + """When unit .info is absent, --output-json must NOT be written (data incomplete).""" + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + component_cov = self._lcov("component.info", 100, 82) + out = os.path.join(self.tmp, "new-baseline-no-unit.json") + r = self._run("--baseline", bl, "--component", component_cov, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when unit absent") + self.assertIn("WARNING", r.stderr) + + def test_output_json_not_written_when_component_missing(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + out = os.path.join(self.tmp, "new-baseline-no-component.json") + r = self._run("--baseline", bl, "--unit", unit_cov, "--output-json", out) + self.assertFalse(os.path.isfile(out), "output-json must NOT be written when component absent") + self.assertIn("WARNING", r.stderr) + + def test_output_json_not_written_when_both_missing(self): + bl = self._baseline({"Unit": 75.0, "Component": 75.0}) + out = os.path.join(self.tmp, "new-baseline-neither.json") + r = self._run("--baseline", bl, "--output-json", out) + self.assertFalse(os.path.isfile(out)) + self.assertIn("WARNING", r.stderr) + + # =========================================================================== + # Baseline coercion: non-float Unit/Component values must not crash the script + # =========================================================================== + + + def test_baseline_string_unit_treated_as_missing(self): + """String value for Unit in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"Unit": "not-a-number", "Component": 75.0}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + # Must not crash; Unit baseline treated as absent → threshold-only → pass + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_null_component_treated_as_missing(self): + """null value for Component in baseline JSON → coerced to None → threshold-only.""" + bl = self._baseline({"Unit": 80.0, "Component": None}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_both_non_float_threshold_only(self): + """Both Unit/Component baseline values invalid → both threshold-only → pass if above 75%.""" + bl = self._baseline({"Unit": "bad", "Component": "bad"}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + def test_baseline_both_non_float_below_threshold_warns(self): + """Both Unit/Component baseline values invalid → threshold-only → [WARN] exit 0 if below 75%.""" + bl = self._baseline({"Unit": "bad", "Component": "bad"}) + unit_cov = self._lcov("unit.info", 100, 70) + component_cov = self._lcov("component.info", 100, 70) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("[WARN]", r.stdout) + + # =========================================================================== + # _fmt_timestamp: null/non-string timestamp must not crash the report + # =========================================================================== + + + def test_null_timestamp_in_baseline_does_not_crash(self): + """null timestamp value in baseline JSON → TypeError handled → report still runs.""" + bl = self._baseline({"Unit": 80.0, "Component": 80.0, "commit": "abc", "timestamp": None}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + # Must not crash; timestamp renders as fallback; gate passes + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + self.assertIn("OVERALL:", r.stdout) + + def test_integer_timestamp_in_baseline_does_not_crash(self): + """Integer timestamp → TypeError in strptime → handled gracefully.""" + bl = self._baseline({"Unit": 80.0, "Component": 80.0, "commit": "abc", "timestamp": 12345}) + unit_cov = self._lcov("unit.info", 100, 80) + component_cov = self._lcov("component.info", 100, 80) + r = self._run("--baseline", bl, "--unit", unit_cov, "--component", component_cov) + self.assertEqual(r.returncode, 0, msg=r.stdout + r.stderr) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c7d124..9fdcc1d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -200,6 +200,7 @@ jobs: --exclude '.*/test/.*\.cpp' \ --decisions \ --medium-threshold 50 --high-threshold 75 \ + --lcov coverage/filtered_coverage.info \ --html-details coverage/index.html \ --cobertura coverage.cobertura.xml \ " @@ -210,6 +211,12 @@ jobs: name: coverage-report path: ${{ github.workspace }}/build/coverage/ + - name: Upload unit test lcov coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-unit + path: ${{ github.workspace }}/build/coverage/filtered_coverage.info + - name: Code Coverage Summary Report uses: irongut/CodeCoverageSummary@v1.3.0 with: @@ -269,6 +276,25 @@ jobs: --app-openrpc /workspace/docs/openrpc/the-spec/firebolt-app-open-rpc.json \ --test-exe /workspace/build/test/ctApp + - name: Generate Coverage Report + run: | + docker run --rm --user "$(id -u):$(id -g)" -v ${{ github.workspace }}:/workspace ${{ needs.build_docker.outputs.image_tag }} \ + bash -c " \ + set -e \ + && cd build \ + && mkdir -p coverage \ + && gcovr -r .. \ + --exclude '.*/test/.*\.h' \ + --exclude '.*/test/.*\.cpp' \ + --lcov coverage/filtered_coverage.info \ + " + + - name: Upload component test lcov coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-component + path: ${{ github.workspace }}/build/coverage/filtered_coverage.info + api_test_app: permissions: contents: read @@ -324,3 +350,172 @@ jobs: --openrpc /workspace/docs/openrpc/the-spec/firebolt-open-rpc.json \ --app-openrpc /workspace/docs/openrpc/the-spec/firebolt-app-open-rpc.json \ --test-exe /workspace/test/api_test_app/build/api-test-app + + coverage_gate: + name: Coverage Gate + needs: [unit_tests, component_tests] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Run coverage script tests + run: python3 .github/scripts/compare_coverage_test.py + + - name: Fetch baseline from build-metadata branch + # Gracefully handle a missing build-metadata branch (first-time setup) + continue-on-error: true + run: | + set -euo pipefail + if git fetch origin build-metadata 2>/dev/null; then + if git cat-file -e FETCH_HEAD:coverage-baseline.json 2>/dev/null; then + git show FETCH_HEAD:coverage-baseline.json > coverage-baseline.json + echo "Loaded coverage-baseline.json from build-metadata branch" + else + echo "build-metadata branch exists but coverage-baseline.json not found — skipping baseline comparison" + echo "{}" > coverage-baseline.json + fi + else + echo "build-metadata branch not found — absolute threshold check only (first-time setup)" + echo "{}" > coverage-baseline.json + fi + + - name: Download unit test coverage artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: coverage-unit + path: ./unit-coverage + + - name: Download component test coverage artifact + continue-on-error: true + uses: actions/download-artifact@v4 + with: + name: coverage-component + path: ./component-coverage + + - name: Compare coverage to baseline + run: | + python3 .github/scripts/compare_coverage.py \ + --baseline coverage-baseline.json \ + --unit ./unit-coverage/filtered_coverage.info \ + --component ./component-coverage/filtered_coverage.info + + update_baseline: + name: Update Coverage Baseline + # Runs on push to develop when both test suites pass. + # Independent of coverage_gate — the gate is informational and must never + # block the baseline from reflecting the actual state of passing tests. + if: >- + github.event_name == 'push' && + github.ref == 'refs/heads/develop' && + needs.unit_tests.result == 'success' && + needs.component_tests.result == 'success' + needs: [unit_tests, component_tests] + runs-on: ubuntu-latest + permissions: + contents: write + actions: read + # Queue concurrent runs; do not cancel in-progress — each merge deserves + # a baseline update and force-push is atomic so queuing is safe. + concurrency: + group: update-baseline-develop + cancel-in-progress: false + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Fetch existing baseline for comparison report + # Best-effort — if the branch or file is absent we compare against nothing. + continue-on-error: true + run: | + set -euo pipefail + if git fetch origin build-metadata 2>/dev/null; then + if git cat-file -e FETCH_HEAD:coverage-baseline.json 2>/dev/null; then + git show FETCH_HEAD:coverage-baseline.json > old-baseline.json + echo "Loaded existing baseline for comparison" + else + echo '{}' > old-baseline.json + fi + else + echo '{}' > old-baseline.json + fi + + - name: Download unit test coverage artifact + uses: actions/download-artifact@v4 + with: + name: coverage-unit + path: ./unit-coverage + + - name: Download component test coverage artifact + uses: actions/download-artifact@v4 + with: + name: coverage-component + path: ./component-coverage + + - name: Extract coverage and write new baseline + id: extract + run: | + set -euo pipefail + BL_ARG="old-baseline.json" + [ -f "$BL_ARG" ] || echo '{}' > "$BL_ARG" + + python3 .github/scripts/compare_coverage.py \ + --baseline "$BL_ARG" \ + --unit ./unit-coverage/filtered_coverage.info \ + --component ./component-coverage/filtered_coverage.info \ + --output-json new-baseline.json \ + --commit "$GITHUB_SHA" \ + --timestamp "$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + + if [ ! -s new-baseline.json ]; then + echo "Coverage extraction produced no output — skipping baseline update" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "New baseline to commit:" + cat new-baseline.json + echo "skip=false" >> "$GITHUB_OUTPUT" + fi + + - name: Commit and push updated baseline to build-metadata + if: steps.extract.outputs.skip == 'false' + run: | + set -euo pipefail + git config user.email "github-actions[bot]@users.noreply.github.com" + git config user.name "github-actions[bot]" + + # Check out the build-metadata branch, or create it as an orphan. + if git fetch origin build-metadata 2>/dev/null; then + git checkout -B build-metadata FETCH_HEAD + else + git checkout --orphan build-metadata + git rm -rf . --quiet 2>/dev/null || true + fi + + cp -f new-baseline.json coverage-baseline.json + + git add coverage-baseline.json + + # Only commit when there is an actual change. + if git diff --cached --quiet; then + echo "Coverage baseline unchanged — no commit needed" + else + MSG=$(printf \ + 'chore: update coverage baseline after develop merge [skip ci]\n\nCommit : %s\nRun ID : %s' \ + "$GITHUB_SHA" "$GITHUB_RUN_ID") + git commit -m "$MSG" + git push --force-with-lease origin build-metadata + echo "Pushed updated baseline to build-metadata" + fi