Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions cmake/_mutiny_discovery.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ macro(add_test_to_script TEST_NAME TEST_LOCATION SELECT_ARG)
endmacro()

execute_process(
COMMAND ${EMULATOR} "${EXECUTABLE}" -ll
COMMAND ${EMULATOR} "${EXECUTABLE}" --list-locations
OUTPUT_VARIABLE discovered_tests
RESULT_VARIABLE result
ERROR_VARIABLE error
Expand All @@ -75,15 +75,15 @@ string(CONCAT LL_LINE_REGEX
)
string(REGEX MATCHALL "[^\n]+\n" discovered_test_lines "${discovered_tests}")

# Query group declaration locations (-lgl).
# Query group declaration locations (--list-group-locations).
execute_process(
COMMAND ${EMULATOR} "${EXECUTABLE}" -lgl
COMMAND ${EMULATOR} "${EXECUTABLE}" --list-group-locations
OUTPUT_VARIABLE group_locations_output
RESULT_VARIABLE lgl_result
ERROR_VARIABLE _lgl_error
)
if(NOT lgl_result EQUAL 0)
message(FATAL_ERROR "Error running ${EXECUTABLE} -lgl:\n${_lgl_error}")
message(FATAL_ERROR "Error running ${EXECUTABLE} --list-group-locations:\n${_lgl_error}")
endif()
string(CONCAT LGL_LINE_REGEX
"^([^.]*)" # group name
Expand All @@ -106,13 +106,13 @@ endforeach()
if(TESTS_DETAILED)
# Identify groups containing ordered tests.
execute_process(
COMMAND ${EMULATOR} "${EXECUTABLE}" -lo
COMMAND ${EMULATOR} "${EXECUTABLE}" --list-ordered-locations
OUTPUT_VARIABLE ordered_locations_output
RESULT_VARIABLE lo_result
ERROR_VARIABLE _lo_error
)
if(NOT lo_result EQUAL 0)
message(FATAL_ERROR "Error running ${EXECUTABLE} -lo:\n${_lo_error}")
message(FATAL_ERROR "Error running ${EXECUTABLE} --list-ordered-locations:\n${_lo_error}")
endif()
set(ordered_groups)
string(REGEX MATCHALL "[^\n]+\n" ordered_lines "${ordered_locations_output}")
Expand All @@ -138,11 +138,11 @@ if(TESTS_DETAILED)
set(group_location
"${_group_loc_${group_name}_file}:${_group_loc_${group_name}_line}"
)
add_test_to_script("${group_name}" "${group_location}" -sg)
add_test_to_script("${group_name}" "${group_location}" --exact-group)
endif()
else()
set(test_name "${group_name}.${CMAKE_MATCH_2}")
add_test_to_script("${test_name}" "${test_location}" -st)
add_test_to_script("${test_name}" "${test_location}" --exact-test)
endif()
endforeach()
else()
Expand All @@ -155,7 +155,7 @@ else()
set(test_location
"${_group_loc_${test_name}_file}:${_group_loc_${test_name}_line}"
)
add_test_to_script("${test_name}" "${test_location}" -sg)
add_test_to_script("${test_name}" "${test_location}" --exact-group)
endforeach()
endif()

Expand Down
93 changes: 55 additions & 38 deletions docs/command-line-reference.rst
Original file line number Diff line number Diff line change
Expand Up @@ -13,43 +13,47 @@ plugin-provided flags).
Query Flags (do not run tests)
-------------------------------

.. option:: -h
.. option:: -h, --help

Print help screen and exit.

.. option:: -lg
.. option:: --version

Print the mutiny version string and exit.

.. option:: --list-groups

Print all group names, space-separated.

.. option:: -ln
.. option:: --list-tests

Print all test names as ``group.name``, space-separated.

.. option:: -ll
.. option:: --list-locations

Print test locations as ``group.name.file.line``.

.. option:: -lo
.. option:: --list-ordered-locations

Print ordered test locations as ``group.name.file.line``.

.. option:: -lgl
.. option:: --list-group-locations

Print group source locations as ``group.file.line``. Each entry points to
the :c:macro:`TEST_GROUP` declaration rather than an individual test.

Output Flags
------------

.. option:: -c
.. option:: -c, --color

Colorize output: green for pass, red for fail.

.. option:: -v
.. option:: -v, --verbose, --verbose=1

Verbose: print each test name as it runs.

.. option:: -vv
.. option:: -vv, --verbose=2

Very verbose: print internal framework state.

Expand Down Expand Up @@ -83,62 +87,74 @@ Multiple filter flags can be combined; all must match for a test to run.
Include filters (substring match by default)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. option:: -g <group>
.. option:: -g <group>, --group <group>

Run tests whose group name *contains* ``<group>``.
Run tests whose group name *contains* ``<group>``. Also accepts
``--group=<group>``.

.. option:: -n <name>
.. option:: -n <name>, --name <name>

Run tests whose test name *contains* ``<name>``.
Run tests whose test name *contains* ``<name>``. Also accepts
``--name=<name>``.

.. option:: -t <group>.<name>
.. option:: -t <group>.<name>, --test <group>.<name>

Run tests whose group contains ``<group>`` **and** name contains ``<name>``.
Also accepts ``--test=<group>.<name>``.

Include filters (exact match)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. option:: -sg <group>
.. option:: --exact-group <group>

Run tests whose group *exactly equals* ``<group>``.
Run tests whose group *exactly equals* ``<group>``. Also accepts
``--exact-group=<group>``.

.. option:: -sn <name>
.. option:: --exact-name <name>

Run tests whose test name *exactly equals* ``<name>``.
Run tests whose test name *exactly equals* ``<name>``. Also accepts
``--exact-name=<name>``.

.. option:: -st <group>.<name>
.. option:: --exact-test <group>.<name>

Run tests where both group and name exactly equal the given values.
Run tests where both group and name exactly equal the given values. Also
accepts ``--exact-test=<group>.<name>``.

Exclude filters (substring match)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. option:: -xg <group>
.. option:: --exclude-group <group>

Exclude tests whose group contains ``<group>``.
Exclude tests whose group contains ``<group>``. Also accepts
``--exclude-group=<group>``.

.. option:: -xn <name>
.. option:: --exclude-name <name>

Exclude tests whose name contains ``<name>``.
Exclude tests whose name contains ``<name>``. Also accepts
``--exclude-name=<name>``.

.. option:: -xt <group>.<name>
.. option:: --exclude-test <group>.<name>

Exclude tests whose group and name both contain the respective substrings.
Also accepts ``--exclude-test=<group>.<name>``.

Exclude filters (exact match)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

.. option:: -xsg <group>
.. option:: --exclude-exact-group <group>

Exclude tests whose group exactly equals ``<group>``.
Exclude tests whose group exactly equals ``<group>``. Also accepts
``--exclude-exact-group=<group>``.

.. option:: -xsn <name>
.. option:: --exclude-exact-name <name>

Exclude tests whose name exactly equals ``<name>``.
Exclude tests whose name exactly equals ``<name>``. Also accepts
``--exclude-exact-name=<name>``.

.. option:: -xst <group>.<name>
.. option:: --exclude-exact-test <group>.<name>

Exclude tests where both group and name exactly equal the given values.
Also accepts ``--exclude-exact-test=<group>.<name>``.

Copy-paste filter
~~~~~~~~~~~~~~~~~
Expand All @@ -156,29 +172,30 @@ This performs an exact match on both group and name.
Execution Flags
---------------

.. option:: -b
.. option:: -b, --reverse

Run tests in reverse registration order.

.. option:: -s [seed]
.. option:: -s [seed], --shuffle [seed]

Shuffle test execution order randomly. Optional ``seed`` (integer > 0)
makes the order reproducible; if omitted, seed is time-based and
printed to console.
printed to console. Also accepts ``--shuffle=<seed>``.

.. option:: -r[#]
.. option:: -r[#], --repeat [#]

Repeat the full test suite ``#`` times. If ``#`` is omitted, repeats
twice. Useful for detecting order-dependent failures.
twice. Useful for detecting order-dependent failures. Also accepts
``--repeat=<#>``.

.. option:: -rs
.. option:: --run-skipped

Run skipped tests (:c:macro:`SKIPPED_TEST`) as if they were normal tests.

.. option:: -f
.. option:: -f, --crash-on-fail

Crash on first failure instead of continuing (useful with a debugger).

.. option:: -e
.. option:: -e, --no-rethrow

Do not rethrow unexpected exceptions as failures (catch and continue).
8 changes: 4 additions & 4 deletions docs/test-macros.rst
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@ Skipping and Expected Failures
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

:c:macro:`SKIPPED_TEST` marks a test as skipped. It is registered but skipped during
normal runs. Use :option:`-rs` to run skipped tests anyway.
normal runs. Use :option:`--run-skipped` to run skipped tests anyway.

.. code-block:: cpp

SKIPPED_TEST(MyGroup, NotImplementedYet)
{
// will not run unless -rs is passed
// will not run unless --run-skipped is passed
CHECK(false);
}

Expand Down Expand Up @@ -117,12 +117,12 @@ tests by default.
Listing Ordered Tests
~~~~~~~~~~~~~~~~~~~~~

Pass :option:`-lo` to print each ordered test's location as
Pass :option:`--list-ordered-locations` to print each ordered test's location as
``group.name.file.line``:

.. code-block:: console

$ ./my_tests -lo
$ ./my_tests --list-ordered-locations
OrderedDemo.Init.examples/tests/OrderedTest.test.cpp.53
OrderedDemo.FirstEvent.examples/tests/OrderedTest.test.cpp.62
OrderedDemo.SecondEvent.examples/tests/OrderedTest.test.cpp.68
Expand Down
2 changes: 1 addition & 1 deletion examples/tests/FEDemo.test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
/*
* To see a demonstration of tests failing as a result of
* IEEE754ExceptionsPlugin picking up floating point errors, run the test
* executable with the -rs option.
* executable with the --run-skipped option.
*
*/

Expand Down
Loading