From 4d595c0b71efb3cff95c8127187eb8c81f8a5f05 Mon Sep 17 00:00:00 2001 From: Dmitrii Gridnev Date: Thu, 14 May 2026 09:28:36 +0300 Subject: [PATCH 1/3] fix: preserve Robot Framework suite hierarchy when registering tests Robot Framework invokes start_suite for every suite in the run, from root down to the leaves. The previous implementation walked the full sub-tree on each invocation with parent_suites reset to empty, so the last (leaf) call overwrote each test's hierarchy with just the leaf suite name. As a result, Qase received only the deepest folder. Extract tests only on the root start_suite call (suite.parent is None) so each test keeps its full path from the run root. Closes #486. --- .../src/qase/robotframework/listener.py | 7 +- .../test_listener.py | 140 ++++++++++++++++++ 2 files changed, 146 insertions(+), 1 deletion(-) diff --git a/qase-robotframework/src/qase/robotframework/listener.py b/qase-robotframework/src/qase/robotframework/listener.py index 59aa66c0..8490636d 100644 --- a/qase-robotframework/src/qase/robotframework/listener.py +++ b/qase-robotframework/src/qase/robotframework/listener.py @@ -70,7 +70,12 @@ def start_suite(self, suite, result): selector = Filter(*execution_plan) suite.visit(selector) - self.tests.update(self.__extract_tests_with_suites(suite)) + # Robot Framework invokes start_suite for every suite in the hierarchy + # (root → leaves). Extract tests only on the root call so each test is + # registered once with its full suite path; otherwise subsequent calls + # for child suites would overwrite the entry with a shorter hierarchy. + if getattr(suite, "parent", None) is None: + self.tests.update(self.__extract_tests_with_suites(suite)) def start_test(self, test, result): self.logger.log_debug(f"Starting test '{test.name}'") diff --git a/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py b/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py index 4ad8d857..fc2aa3d8 100644 --- a/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py +++ b/qase-robotframework/tests/tests_qaseio_robotframework/test_listener.py @@ -83,3 +83,143 @@ def test_single_failed_child_sets_passed(self): Listener._set_step_status_based_on_children(step, [child1]) step.execution.set_status.assert_called_once_with("passed") + + +def _make_test(name, lineno): + test = MagicMock(spec=["name", "lineno"]) + test.name = name + test.lineno = lineno + return test + + +def _make_suite(name, parent=None, suites=None, tests=None): + """Build a Robot Framework-like Suite mock with parent/suites/tests.""" + suite = MagicMock(spec=["name", "parent", "suites", "tests"]) + suite.name = name + suite.parent = parent + suite.suites = suites or [] + suite.tests = tests or [] + return suite + + +def _bare_listener(): + """Create a Listener instance without running __init__.""" + listener = Listener.__new__(Listener) + listener.tests = {} + return listener + + +class TestExtractTestsWithSuites: + """Tests for Listener.__extract_tests_with_suites tree walk.""" + + def _extract(self, listener, suite): + return listener._Listener__extract_tests_with_suites(suite) + + def test_flat_suite_with_tests(self): + listener = _bare_listener() + suite = _make_suite("Root", tests=[_make_test("test_one", 1)]) + + result = self._extract(listener, suite) + + assert result == {"test_one:1": ["Root"]} + + def test_nested_hierarchy_builds_full_path(self): + listener = _bare_listener() + login = _make_suite("Login", tests=[_make_test("test_login", 5)]) + account = _make_suite("Account", suites=[login]) + login.parent = account + root = _make_suite("Tests", suites=[account]) + account.parent = root + + result = self._extract(listener, root) + + assert result == {"test_login:5": ["Tests", "Account", "Login"]} + + def test_tests_at_multiple_levels(self): + listener = _bare_listener() + leaf = _make_suite("Leaf", tests=[_make_test("leaf_test", 10)]) + middle = _make_suite("Middle", suites=[leaf], + tests=[_make_test("mid_test", 20)]) + leaf.parent = middle + root = _make_suite("Root", suites=[middle], + tests=[_make_test("root_test", 30)]) + middle.parent = root + + result = self._extract(listener, root) + + assert result == { + "leaf_test:10": ["Root", "Middle", "Leaf"], + "mid_test:20": ["Root", "Middle"], + "root_test:30": ["Root"], + } + + +class TestStartSuiteHierarchy: + """Regression tests for issue #486: nested suite hierarchy is preserved.""" + + def _build_tree(self): + """Tests > Account > Login, with a single test in Login.""" + login = _make_suite("Login", tests=[_make_test("test_login", 5)]) + account = _make_suite("Account", suites=[login]) + login.parent = account + root = _make_suite("Tests", parent=None, suites=[account]) + account.parent = root + return root, account, login + + def _bare_listener_with_reporter(self): + listener = _bare_listener() + listener.reporter = MagicMock() + listener.reporter.get_execution_plan.return_value = None + listener.pabot_index = None + listener.last_level_flag = None + return listener + + def test_root_call_registers_full_hierarchy(self): + listener = self._bare_listener_with_reporter() + root, _, _ = self._build_tree() + + with patch("qase.robotframework.listener.get_pool_id", + return_value=None), \ + patch("qase.robotframework.listener.get_last_level_flag", + return_value=None): + listener.start_suite(root, MagicMock()) + + assert listener.tests == { + "test_login:5": ["Tests", "Account", "Login"] + } + + def test_child_calls_do_not_overwrite_with_shorter_path(self): + """Robot Framework invokes start_suite for every suite in the + hierarchy. Earlier the leaf call overwrote the full path with just + the leaf name — that's the bug reported in #486. + """ + listener = self._bare_listener_with_reporter() + root, account, login = self._build_tree() + + with patch("qase.robotframework.listener.get_pool_id", + return_value=None), \ + patch("qase.robotframework.listener.get_last_level_flag", + return_value=None): + listener.start_suite(root, MagicMock()) + listener.start_suite(account, MagicMock()) + listener.start_suite(login, MagicMock()) + + assert listener.tests == { + "test_login:5": ["Tests", "Account", "Login"] + } + + def test_non_root_call_without_prior_root_skips_extraction(self): + """If start_suite is invoked only for a sub-suite (e.g. when parent + suites are filtered out), nothing is registered. This is acceptable + because Robot Framework always emits start_suite for the run root. + """ + listener = self._bare_listener_with_reporter() + _, account, _ = self._build_tree() + + with patch("qase.robotframework.listener.get_pool_id", + return_value=None), \ + patch("qase.robotframework.listener.get_last_level_flag", + return_value=None): + listener.start_suite(account, MagicMock()) + + assert listener.tests == {} From d9b030af62d4826e5129cd927a94cb6d408cbd2a Mon Sep 17 00:00:00 2001 From: Dmitrii Gridnev Date: Thu, 14 May 2026 09:31:29 +0300 Subject: [PATCH 2/3] chore: bump qase-robotframework to 6.0.0 for suite hierarchy fix Restoring the full Robot Framework suite hierarchy changes the location where reported test cases land in Qase, which can break existing projects that were built against the old flattened layout. Bump the major version and document the migration path. --- qase-robotframework/changelog.md | 6 ++++++ qase-robotframework/docs/UPGRADE.md | 18 ++++++++++++++++++ qase-robotframework/pyproject.toml | 2 +- 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/qase-robotframework/changelog.md b/qase-robotframework/changelog.md index 30283bdb..39ebfb74 100644 --- a/qase-robotframework/changelog.md +++ b/qase-robotframework/changelog.md @@ -1,3 +1,9 @@ +# qase-robotframework 6.0.0 + +## Breaking changes + +- Robot Framework suite hierarchy is now preserved when reporting to Qase. Previously the reporter registered only the deepest (leaf) suite, so a test under `Tests > Account > Login` ended up in a flat `Login` suite in Qase. The reporter now sends the full nested path. Existing projects whose Qase suites were created against the old flattened behaviour may see test cases land in a new nested suite tree on the next run, which can result in duplicates or breakage of any manual reorganisation. Review your project's suite layout before upgrading. ([#486](https://github.com/qase-tms/qase-python/issues/486)) + # qase-robotframework 5.1.0 ## What's new diff --git a/qase-robotframework/docs/UPGRADE.md b/qase-robotframework/docs/UPGRADE.md index d971f59f..bc6ff12a 100644 --- a/qase-robotframework/docs/UPGRADE.md +++ b/qase-robotframework/docs/UPGRADE.md @@ -1,5 +1,23 @@ # Upgrade guides +## From 5.x to 6.x + +### Suite hierarchy is preserved + +Prior to v6, the reporter registered each test under only the deepest (leaf) suite of its Robot Framework hierarchy. A test located in `Tests > Account > Login` ended up in a flat `Login` suite in Qase, with `Tests` and `Account` discarded. + +Starting with v6, the reporter sends the full nested suite path, so the same test now lands in `Tests / Account / Login` in Qase. + +Impact: + +- If you are starting from a clean Qase project, no action is required. +- If your Qase project already has cases created against the old flattened layout (or you reorganised the tree manually), the next run will create cases in a new nested location, which can result in duplicates. + +To avoid duplicates, either: + +- accept the new hierarchy and remove or archive the old flat suites in Qase, or +- pin annotated tests to the existing cases by setting the Qase ID on each `*** Test Cases ***` entry (see [usage docs](usage.md)). + ## From 2.x to 3.x ### Execution diff --git a/qase-robotframework/pyproject.toml b/qase-robotframework/pyproject.toml index 62737eb2..08d25e3b 100644 --- a/qase-robotframework/pyproject.toml +++ b/qase-robotframework/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "qase-robotframework" -version = "5.1.0" +version = "6.0.0" description = "Qase Robot Framework Plugin" readme = "README.md" authors = [{name = "Qase Team", email = "support@qase.io"}] From 104213a8156eaa496d4fccab0bfde1b30473df8a Mon Sep 17 00:00:00 2001 From: Dmitrii Gridnev Date: Thu, 14 May 2026 10:28:11 +0300 Subject: [PATCH 3/3] test: align robotframework integration expected with full suite hierarchy The previous expected file was generated under the bug where only the leaf suite was reported. Now that the listener preserves the full hierarchy from the run root, the validator receives the directory-level "Tests" suite as data[0] and the file-level "Steps" suite as data[1], matching the layout used by the pytest and tavern expected files. Signatures move from "::steps::" to "::tests::steps::" because they include every suite in order. --- integration/robot/expected/robot-examples.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/integration/robot/expected/robot-examples.yaml b/integration/robot/expected/robot-examples.yaml index ade1a8ad..53acd320 100644 --- a/integration/robot/expected/robot-examples.yaml +++ b/integration/robot/expected/robot-examples.yaml @@ -9,7 +9,7 @@ run: total: 4 results: - title: Test With Failing Step - signature: 302::steps::test_with_failing_step + signature: 302::tests::steps::test_with_failing_step testops_ids: - 302 status: invalid @@ -18,6 +18,7 @@ results: relations: suite: data: + - title: Tests - title: Steps steps: - data: @@ -39,7 +40,7 @@ results: execution: status: failed - title: Test With Multiple Steps - signature: 301::steps::test_with_multiple_steps + signature: 301::tests::steps::test_with_multiple_steps testops_ids: - 301 status: passed @@ -48,6 +49,7 @@ results: relations: suite: data: + - title: Tests - title: Steps steps: - data: @@ -78,7 +80,7 @@ results: execution: status: passed - title: Test With Single Tag - signature: 901::steps::test_with_single_tag + signature: 901::tests::steps::test_with_single_tag testops_ids: - 901 status: passed @@ -88,9 +90,10 @@ results: relations: suite: data: + - title: Tests - title: Steps - title: Test With Multiple Tags - signature: 902::steps::test_with_multiple_tags + signature: 902::tests::steps::test_with_multiple_tags testops_ids: - 902 status: passed @@ -100,4 +103,5 @@ results: relations: suite: data: + - title: Tests - title: Steps