diff --git a/integration/robot/expected/robot-examples.yaml b/integration/robot/expected/robot-examples.yaml index ade1a8ad..53acd320 100644 --- a/integration/robot/expected/robot-examples.yaml +++ b/integration/robot/expected/robot-examples.yaml @@ -9,7 +9,7 @@ run: total: 4 results: - title: Test With Failing Step - signature: 302::steps::test_with_failing_step + signature: 302::tests::steps::test_with_failing_step testops_ids: - 302 status: invalid @@ -18,6 +18,7 @@ results: relations: suite: data: + - title: Tests - title: Steps steps: - data: @@ -39,7 +40,7 @@ results: execution: status: failed - title: Test With Multiple Steps - signature: 301::steps::test_with_multiple_steps + signature: 301::tests::steps::test_with_multiple_steps testops_ids: - 301 status: passed @@ -48,6 +49,7 @@ results: relations: suite: data: + - title: Tests - title: Steps steps: - data: @@ -78,7 +80,7 @@ results: execution: status: passed - title: Test With Single Tag - signature: 901::steps::test_with_single_tag + signature: 901::tests::steps::test_with_single_tag testops_ids: - 901 status: passed @@ -88,9 +90,10 @@ results: relations: suite: data: + - title: Tests - title: Steps - title: Test With Multiple Tags - signature: 902::steps::test_with_multiple_tags + signature: 902::tests::steps::test_with_multiple_tags testops_ids: - 902 status: passed @@ -100,4 +103,5 @@ results: relations: suite: data: + - title: Tests - title: Steps diff --git a/qase-robotframework/changelog.md b/qase-robotframework/changelog.md index 30283bdb..39ebfb74 100644 --- a/qase-robotframework/changelog.md +++ b/qase-robotframework/changelog.md @@ -1,3 +1,9 @@ +# qase-robotframework 6.0.0 + +## Breaking changes + +- Robot Framework suite hierarchy is now preserved when reporting to Qase. Previously the reporter registered only the deepest (leaf) suite, so a test under `Tests > Account > Login` ended up in a flat `Login` suite in Qase. The reporter now sends the full nested path. Existing projects whose Qase suites were created against the old flattened behaviour may see test cases land in a new nested suite tree on the next run, which can result in duplicates or breakage of any manual reorganisation. Review your project's suite layout before upgrading. ([#486](https://github.com/qase-tms/qase-python/issues/486)) + # qase-robotframework 5.1.0 ## What's new diff --git a/qase-robotframework/docs/UPGRADE.md b/qase-robotframework/docs/UPGRADE.md index d971f59f..bc6ff12a 100644 --- a/qase-robotframework/docs/UPGRADE.md +++ b/qase-robotframework/docs/UPGRADE.md @@ -1,5 +1,23 @@ # Upgrade guides +## From 5.x to 6.x + +### Suite hierarchy is preserved + +Prior to v6, the reporter registered each test under only the deepest (leaf) suite of its Robot Framework hierarchy. A test located in `Tests > Account > Login` ended up in a flat `Login` suite in Qase, with `Tests` and `Account` discarded. + +Starting with v6, the reporter sends the full nested suite path, so the same test now lands in `Tests / Account / Login` in Qase. + +Impact: + +- If you are starting from a clean Qase project, no action is required. +- If your Qase project already has cases created against the old flattened layout (or you reorganised the tree manually), the next run will create cases in a new nested location, which can result in duplicates. + +To avoid duplicates, either: + +- accept the new hierarchy and remove or archive the old flat suites in Qase, or +- pin annotated tests to the existing cases by setting the Qase ID on each `*** Test Cases ***` entry (see [usage docs](usage.md)). + ## From 2.x to 3.x ### Execution diff --git a/qase-robotframework/pyproject.toml b/qase-robotframework/pyproject.toml index 62737eb2..08d25e3b 100644 --- a/qase-robotframework/pyproject.toml +++ b/qase-robotframework/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "qase-robotframework" -version = "5.1.0" +version = "6.0.0" description = "Qase Robot Framework Plugin" readme = "README.md" authors = [{name = "Qase Team", email = "support@qase.io"}] diff --git a/qase-robotframework/src/qase/robotframework/listener.py b/qase-robotframework/src/qase/robotframework/listener.py index 59aa66c0..8490636d 100644 --- a/qase-robotframework/src/qase/robotframework/listener.py +++ b/qase-robotframework/src/qase/robotframework/listener.py @@ -70,7 +70,12 @@ def start_suite(self, suite, result): selector = Filter(*execution_plan) suite.visit(selector) - self.tests.update(self.__extract_tests_with_suites(suite)) + # Robot Framework invokes start_suite for every suite in the hierarchy + # (root → leaves). Extract tests only on the root call so each test is + # registered once with its full suite path; otherwise subsequent calls + # for child suites would overwrite the entry with a shorter hierarchy. + if getattr(suite, "parent", None) is None: + self.tests.update(self.__extract_tests_with_suites(suite)) def start_test(self, test, result): self.logger.log_debug(f"Starting test '{test.name}'") diff --git a/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py b/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py index 4ad8d857..fc2aa3d8 100644 --- a/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py +++ b/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py @@ -83,3 +83,143 @@ def test_single_failed_child_sets_passed(self): Listener._set_step_status_based_on_children(step, [child1]) step.execution.set_status.assert_called_once_with("passed") + + +def _make_test(name, lineno): + test = MagicMock(spec=["name", "lineno"]) + test.name = name + test.lineno = lineno + return test + + +def _make_suite(name, parent=None, suites=None, tests=None): + """Build a Robot Framework-like Suite mock with parent/suites/tests.""" + suite = MagicMock(spec=["name", "parent", "suites", "tests"]) + suite.name = name + suite.parent = parent + suite.suites = suites or [] + suite.tests = tests or [] + return suite + + +def _bare_listener(): + """Create a Listener instance without running __init__.""" + listener = Listener.__new__(Listener) + listener.tests = {} + return listener + + +class TestExtractTestsWithSuites: + """Tests for Listener.__extract_tests_with_suites tree walk.""" + + def _extract(self, listener, suite): + return listener._Listener__extract_tests_with_suites(suite) + + def test_flat_suite_with_tests(self): + listener = _bare_listener() + suite = _make_suite("Root", tests=[_make_test("test_one", 1)]) + + result = self._extract(listener, suite) + + assert result == {"test_one:1": ["Root"]} + + def test_nested_hierarchy_builds_full_path(self): + listener = _bare_listener() + login = _make_suite("Login", tests=[_make_test("test_login", 5)]) + account = _make_suite("Account", suites=[login]) + login.parent = account + root = _make_suite("Tests", suites=[account]) + account.parent = root + + result = self._extract(listener, root) + + assert result == {"test_login:5": ["Tests", "Account", "Login"]} + + def test_tests_at_multiple_levels(self): + listener = _bare_listener() + leaf = _make_suite("Leaf", tests=[_make_test("leaf_test", 10)]) + middle = _make_suite("Middle", suites=[leaf], + tests=[_make_test("mid_test", 20)]) + leaf.parent = middle + root = _make_suite("Root", suites=[middle], + tests=[_make_test("root_test", 30)]) + middle.parent = root + + result = self._extract(listener, root) + + assert result == { + "leaf_test:10": ["Root", "Middle", "Leaf"], + "mid_test:20": ["Root", "Middle"], + "root_test:30": ["Root"], + } + + +class TestStartSuiteHierarchy: + """Regression tests for issue #486: nested suite hierarchy is preserved.""" + + def _build_tree(self): + """Tests > Account > Login, with a single test in Login.""" + login = _make_suite("Login", tests=[_make_test("test_login", 5)]) + account = _make_suite("Account", suites=[login]) + login.parent = account + root = _make_suite("Tests", parent=None, suites=[account]) + account.parent = root + return root, account, login + + def _bare_listener_with_reporter(self): + listener = _bare_listener() + listener.reporter = MagicMock() + listener.reporter.get_execution_plan.return_value = None + listener.pabot_index = None + listener.last_level_flag = None + return listener + + def test_root_call_registers_full_hierarchy(self): + listener = self._bare_listener_with_reporter() + root, _, _ = self._build_tree() + + with patch("qase.robotframework.listener.get_pool_id", + return_value=None), \ + patch("qase.robotframework.listener.get_last_level_flag", + return_value=None): + listener.start_suite(root, MagicMock()) + + assert listener.tests == { + "test_login:5": ["Tests", "Account", "Login"] + } + + def test_child_calls_do_not_overwrite_with_shorter_path(self): + """Robot Framework invokes start_suite for every suite in the + hierarchy. Earlier the leaf call overwrote the full path with just + the leaf name — that's the bug reported in #486. + """ + listener = self._bare_listener_with_reporter() + root, account, login = self._build_tree() + + with patch("qase.robotframework.listener.get_pool_id", + return_value=None), \ + patch("qase.robotframework.listener.get_last_level_flag", + return_value=None): + listener.start_suite(root, MagicMock()) + listener.start_suite(account, MagicMock()) + listener.start_suite(login, MagicMock()) + + assert listener.tests == { + "test_login:5": ["Tests", "Account", "Login"] + } + + def test_non_root_call_without_prior_root_skips_extraction(self): + """If start_suite is invoked only for a sub-suite (e.g. when parent + suites are filtered out), nothing is registered. This is acceptable + because Robot Framework always emits start_suite for the run root. + """ + listener = self._bare_listener_with_reporter() + _, account, _ = self._build_tree() + + with patch("qase.robotframework.listener.get_pool_id", + return_value=None), \ + patch("qase.robotframework.listener.get_last_level_flag", + return_value=None): + listener.start_suite(account, MagicMock()) + + assert listener.tests == {}