diff --git a/.github/workflows/verify_and_publish.yml b/.github/workflows/verify_and_publish.yml index 4c20b5ae..db727ca2 100644 --- a/.github/workflows/verify_and_publish.yml +++ b/.github/workflows/verify_and_publish.yml @@ -55,7 +55,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] - python-version: ['3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10', '3.11', '3.12', '3.13'] steps: - name: Grab source @@ -159,7 +159,7 @@ jobs: coverage run \ $(which cijoe) --monitor -l \ --config cijoe-example-${{ matrix.usage_example }}/cijoe-config.toml \ - cijoe-example-${{ matrix.usage_example }}/cijoe-workflow.yaml + cijoe-example-${{ matrix.usage_example }}/cijoe-task.yaml - name: Coverage report run: | @@ -292,11 +292,11 @@ jobs: matrix: cli: - name: "integrity-check" - cmd: "$(which cijoe) example/cijoe-workflow.yaml -c example/cijoe-config.toml --integrity-check" + cmd: "$(which cijoe) example/cijoe-task.yaml -c example/cijoe-config.toml --integrity-check" - name: "script-target" cmd: "$(which cijoe) core.example_script_default -c example/cijoe-config.toml" - - name: "workflow-steps" - cmd: "$(which cijoe) example/cijoe-workflow.yaml -c example/cijoe-config.toml script_with_args" + - name: "task-steps" + cmd: "$(which cijoe) example/cijoe-task.yaml -c example/cijoe-config.toml script_with_args" - name: "emit-example" cmd: "$(which cijoe) --example core.default" - name: "archive" @@ -358,7 +358,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5.2.0 with: - python-version: '3.9' + python-version: '3.10' - name: Python dependencies and their versions run: | diff --git a/docs/emit.py b/docs/emit.py index 80bb6222..1ce7c038 100755 --- a/docs/emit.py +++ b/docs/emit.py @@ -53,7 +53,7 @@ {{ "=" * pkg_name_len }} These are the scripts provided in the package, they are listed by the **full** -name that you can use to refer to them in a workflow. +name that you can use to refer to them in a task. Scripts ------- diff --git a/docs/source/050_usage_help.out b/docs/source/050_usage_help.out index e2011a13..a6e52d16 100644 --- a/docs/source/050_usage_help.out +++ b/docs/source/050_usage_help.out @@ -8,10 +8,10 @@ options: -h, --help show this help message and exit run: - Options for running a workflow script. + Options for running a task script. - target A cijoe workflow or script to run. - step Given a workflow, the steps of the workflow it + target A cijoe task or script to run. + step Given a task, the steps of the task it should run. If none are given, all steps are run. --config CONFIG, -c CONFIG @@ -25,22 +25,22 @@ run: '-ll' for debug. (default: None) --monitor, -m Dump command output to stdout (default: False) --no-report, -n Skip the producing, and opening, a report at the - end of the workflow-run (default: False) + end of the task-run (default: False) --skip-report, -s Skip the report opening at the end of the - workflow-run (default: True) - --tag TAG, -t TAG Tags to identify a workflow-run. This will be + task-run (default: True) + --tag TAG, -t TAG Tags to identify a task-run. This will be prefixed while storing in archive (default: None) utilities: - Workflow, and workflow-related utilities + Task, and task-related utilities --archive, -a Move the output at '-o / --output' to 'cijoe- archive/YYYY-MM-DD_HH:MM:SS (default: False) --produce-report, -p Produce report, and open it, for output at '-o / --output' and exit. (default: None) --integrity-check, -i - Check integrity of workflow given as positional + Check integrity of task given as positional argument and exit. (default: False) --resources, -r List collected resources and exit. (default: False) diff --git a/docs/source/200_quickstart.cmd b/docs/source/200_quickstart.cmd index c59b6a76..6ae28b64 100644 --- a/docs/source/200_quickstart.cmd +++ b/docs/source/200_quickstart.cmd @@ -4,10 +4,10 @@ pipx install cijoe # Print a list of bundled usage examples cijoe --example -# Produce example script, config, and workflow +# Produce example script, config, and task cijoe --example core.default -# Execute the workflow -cijoe cijoe-example-core.default/cijoe-workflow.yaml \ +# Execute the task +cijoe cijoe-example-core.default/cijoe-task.yaml \ --config cijoe-example-core.default/cijoe-config.toml diff --git a/docs/source/300_lint.cmd b/docs/source/300_lint.cmd index 6f4a10b5..c5f552e2 100644 --- a/docs/source/300_lint.cmd +++ b/docs/source/300_lint.cmd @@ -1,4 +1,4 @@ -# Check format of workflow and verify existance of the scripts used +# Check format of task and verify existance of the scripts used cijoe --integrity-check \ --config cijoe-example-core.default/cijoe-config.toml \ - cijoe-example-core.default/cijoe-workflow.yaml + cijoe-example-core.default/cijoe-task.yaml diff --git a/docs/source/420_usage_task_all.cmd b/docs/source/420_usage_task_all.cmd new file mode 100644 index 00000000..ddd5bb98 --- /dev/null +++ b/docs/source/420_usage_task_all.cmd @@ -0,0 +1,3 @@ +# Run the task +cijoe cijoe-example-core.default/cijoe-task.yaml \ + --config cijoe-example-core.default/cijoe-config.toml diff --git a/docs/source/420_usage_workflow_all.out b/docs/source/420_usage_task_all.out similarity index 100% rename from docs/source/420_usage_workflow_all.out rename to docs/source/420_usage_task_all.out diff --git a/docs/source/420_usage_workflow_all.cmd b/docs/source/420_usage_workflow_all.cmd deleted file mode 100644 index a718488c..00000000 --- a/docs/source/420_usage_workflow_all.cmd +++ /dev/null @@ -1,3 +0,0 @@ -# Run the workflow -cijoe cijoe-example-core.default/cijoe-workflow.yaml \ - --config cijoe-example-core.default/cijoe-config.toml diff --git a/docs/source/450_usage_task_subset.cmd b/docs/source/450_usage_task_subset.cmd new file mode 100644 index 00000000..bd7c5c2e --- /dev/null +++ b/docs/source/450_usage_task_subset.cmd @@ -0,0 +1,4 @@ +# Run a subset of the steps in the task +cijoe cijoe-example-core.default/cijoe-task.yaml \ + --config cijoe-example-core.default/cijoe-config.toml \ + inline_commands diff --git a/docs/source/450_usage_workflow_subset.out b/docs/source/450_usage_task_subset.out similarity index 100% rename from docs/source/450_usage_workflow_subset.out rename to docs/source/450_usage_task_subset.out diff --git a/docs/source/450_usage_workflow_subset.cmd b/docs/source/450_usage_workflow_subset.cmd deleted file mode 100644 index 19fdef5a..00000000 --- a/docs/source/450_usage_workflow_subset.cmd +++ /dev/null @@ -1,4 +0,0 @@ -# Run a subset of the steps in the workflow -cijoe cijoe-example-core.default/cijoe-workflow.yaml \ - --config cijoe-example-core.default/cijoe-config.toml \ - inline_commands diff --git a/docs/source/configs/index.rst b/docs/source/configs/index.rst index e5fc8cb7..3b866520 100644 --- a/docs/source/configs/index.rst +++ b/docs/source/configs/index.rst @@ -16,12 +16,12 @@ provided, the reason being that with **cijoe**, the core interface of configured to do so, ``run()`` executes commands on a **target** machine that is not the **initiator**. -Often, tasks performed by **cijoe** can be, if not destructive, modifying -the **target** system in one way of the other. The default is for the -**initator** and the **target** to be the same system, and therefore, to avoid -accidents like invoking a destructive workflow and forgetting to supply a +Often, tasks performed by **cijoe** can be, if not destructive, modifying +the **target** system in one way of the other. The default is for the +**initator** and the **target** to be the same system, and therefore, to avoid +accidents like invoking a destructive task and forgetting to supply a configuration file, providing a configuration file in the command line tool -is required. +is required. **cijoe** specific keys ======================= @@ -32,7 +32,7 @@ you should avoid using for other things than the intended purpose: * SSH Transport Configuration * Shell Configuration -* Workflow Configuration +* Task Configuration **cijoe** abides by the convention that configuration values are grouped under a relevant key. Thus, the those for **cijoe** itself are all stored under the @@ -46,7 +46,7 @@ a relevant key. Thus, the those for **cijoe** itself are all stored under the [cijoe.run] ... - [cijoe.workflow] + [cijoe.task] ... @@ -154,27 +154,32 @@ In the table below, you can see the keys associated with different shells in - ``csh`` -.. _sec-resources-configs-workflow: +.. _sec-resources-configs-task: -Workflow Configuration ----------------------- +Task Configuration +------------------ -When a workflow is processed, each step is executed in sequence. It can be -advantageous to *fail fast*, meaning the workflow stops processing further steps +When a task is processed, each step is executed in sequence. It can be +advantageous to *fail fast*, meaning the task stops processing further steps once a failure occurs in any step. This is possible via the **cijoe** configuration option: -``cijoe.workflow.fail_fast`` which you can set in your +``cijoe.task.fail_fast`` which you can set in your :ref:`configuration-file `: .. literalinclude:: ../../../src/cijoe/core/configs/example_config_default.toml -By default **cijoe** functionality, then this can also be controlled environment -variable: +As with any **cijoe** configuration option, this can also be controlled via an +environment variable: .. code-block:: bash - CIJOE_WORKFLOW_FAIL_FAST=true + CIJOE_TASK_FAIL_FAST=true + +.. note:: + The legacy ``[cijoe.workflow]`` section, including + ``cijoe.workflow.fail_fast`` (and ``CIJOE_WORKFLOW_FAIL_FAST``), is still + honored when ``[cijoe.task]`` is not set, but emits a deprecation warning. .. _sec-resources-configs-evar-override: diff --git a/docs/source/index.rst b/docs/source/index.rst index b820c209..5b941217 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -3,28 +3,28 @@ ================================================== **cijoe** is a tool designed to collect and formalize the common elements used -during systems development, enabling an **automated** and **reproducible** -workflow. +during systems development, enabling **automated** and **reproducible** +execution. When using **cijoe**, the first step involves creating :ref:`scripts -` for automating repetitive tasks. As these :ref:`scripts +` to automate repetitive work. As these :ref:`scripts ` are developed, all input values are stored in and passed through :ref:`configuration files ` and environment variables. -By keeping input values separate from the script itself, tasks can be easily +By keeping input values separate from the script itself, scripts can be easily replicated across different environments, ensuring flexibility and consistency in execution. As your script collection grows, **cijoe** allows you to organize them -into :ref:`workflows `. -A :ref:`workflow ` consolidates the +into :ref:`tasks `. +A :ref:`task ` consolidates the :ref:`script ` execution sequence, documents their combined purpose, and provides usage instructions. This clarity makes it easier -for others to understand and consistently execute the workflow in their own +for others to understand and consistently execute the task in their own environments. After execution, **cijoe** generates a report that includes command output, -script documentation, auxiliary collected artifacts, and a workflow summary. +script documentation, auxiliary collected artifacts, and a task summary. This report facilitates collaboration by providing results transparently, making them easy to **review** and **reproduce**. @@ -43,10 +43,10 @@ errors occur. Options for customizing the behavior include: - ``-n`` / ``--no-report``: Disable the generation and display of the HTML report. - ``-l`` / ``--log-level``: Increase verbosity by setting a higher log level. -- ``-m`` / ``--monitor``: Enable real-time monitoring of the workflow. +- ``-m`` / ``--monitor``: Enable real-time monitoring of the task. For a detailed description of usage see ``cijoe --help`` and refer to the rest -of the documentation, and good place to continue is with :ref:`sec-usage`. +of the documentation; a good place to continue is with :ref:`sec-usage`. .. _sec-introduction: @@ -73,13 +73,13 @@ Unlike configuration management tools, **cijoe** is a minimal, open-ended scripting tool that emphasizes maintainability, reusability, and built-in reporting for sharing results, including command output and artifacts. -**cijoe** is designed to execute commands, scripts, or workflows +**cijoe** is designed to execute commands, scripts, or tasks within continuous integration (CI) environments such as :github:`GitHub <>`, :gitlab:`GitLab <>`, :travis:`Travis CI <>`, and :jenkins:`Jenkins <>`. It also allows for seamless execution of the same scripts on local systems, enabling developers to switch between CI providers while maintaining the ability -to run automated tasks locally. +to run the same automation locally. .. figure:: _static/cijoe-networked.drawio.png :alt: Development Environment @@ -128,10 +128,10 @@ Key Features - **Realtime Output**: - - Whether **cijoe** is executing scripts of workflows, then it **can** + - Whether **cijoe** is executing scripts or tasks, then it **can** provide you with realtime command output, directly in your console. - When running in cloud environments e.g. GitHub Actions, GitLab, Azure, - Travis, etc. then it is very convient to immediately observe execution + Travis, etc. then it is very convenient to immediately observe execution progress - **Postprocessing**: @@ -142,7 +142,7 @@ Key Features - **Reporting**: - - All data from **runs**, script and workflow executions, is collected in a + - All data from **runs**, script and task executions, is collected in a **HTML** report. Conveniently viewable even when running on remote systems In summary, **cijoe** aims to be a simple yet powerful tool that integrates @@ -151,7 +151,7 @@ without adding complexity. Once you have ensured that the system prerequisites (:ref:`sec-prerequisites`) are met, proceed to the :ref:`sec-usage` section to run an example -script and workflow. For documentation on how to create your own scripts, +script and task. For documentation on how to create your own scripts, see :ref:`sec-resources`. Finally, refer to :ref:`sec-packages` for descriptions of existing script collections and related packages. @@ -162,7 +162,7 @@ of existing script collections and related packages. usage/index.rst scripts/index.rst - workflows/index.rst + tasks/index.rst configs/index.rst testrunner/index.rst resources/index.rst diff --git a/docs/source/packages/core/index.rst b/docs/source/packages/core/index.rst index be178cdc..1964d1c3 100644 --- a/docs/source/packages/core/index.rst +++ b/docs/source/packages/core/index.rst @@ -5,7 +5,7 @@ core ==== These are the scripts provided in the package, they are listed by the **full** -name that you can use to refer to them in a workflow. +name that you can use to refer to them in a task. Scripts ------- diff --git a/docs/source/packages/linux/index.rst b/docs/source/packages/linux/index.rst index 3199853d..6ce6b50f 100644 --- a/docs/source/packages/linux/index.rst +++ b/docs/source/packages/linux/index.rst @@ -5,7 +5,7 @@ linux ===== These are the scripts provided in the package, they are listed by the **full** -name that you can use to refer to them in a workflow. +name that you can use to refer to them in a task. Scripts ------- diff --git a/docs/source/packages/qemu/index.rst b/docs/source/packages/qemu/index.rst index 5756a5ce..88151074 100644 --- a/docs/source/packages/qemu/index.rst +++ b/docs/source/packages/qemu/index.rst @@ -5,7 +5,7 @@ qemu ==== These are the scripts provided in the package, they are listed by the **full** -name that you can use to refer to them in a workflow. +name that you can use to refer to them in a task. Scripts ------- diff --git a/docs/source/packages/system_imaging/index.rst b/docs/source/packages/system_imaging/index.rst index b5cbfc4d..3a4fbd7d 100644 --- a/docs/source/packages/system_imaging/index.rst +++ b/docs/source/packages/system_imaging/index.rst @@ -5,7 +5,7 @@ system_imaging ============== These are the scripts provided in the package, they are listed by the **full** -name that you can use to refer to them in a workflow. +name that you can use to refer to them in a task. Scripts ------- diff --git a/docs/source/resources/index.rst b/docs/source/resources/index.rst index e5c333c5..669e078e 100644 --- a/docs/source/resources/index.rst +++ b/docs/source/resources/index.rst @@ -5,7 +5,7 @@ =========== In **cijoe** the most essential **resources** are :ref:`sec-resources-scripts`, -:ref:`sec-resources-workflows`, and :ref:`sec-resources-configs`. +:ref:`sec-resources-tasks`, and :ref:`sec-resources-configs`. In addition to these are :ref:`Auxiliary files `, and :ref:`sec-resources-templates`. diff --git a/docs/source/workflows/index.rst b/docs/source/tasks/index.rst similarity index 57% rename from docs/source/workflows/index.rst rename to docs/source/tasks/index.rst index 690c90a7..4f8fdba7 100644 --- a/docs/source/workflows/index.rst +++ b/docs/source/tasks/index.rst @@ -1,41 +1,41 @@ -.. _sec-resources-workflows: +.. _sec-resources-tasks: -=========== - Workflows -=========== +======= + Tasks +======= -Workflows enable the organized execution of commands and scripts. After +Tasks enable the organized execution of commands and scripts. After execution, a report is generated, containing the status and embedded -documentation of the workflow and scripts in a self-contained format. To run the -workflow produced by ``cijoe --example``, use the following command: +documentation of the task and scripts in a self-contained format. To run the +task produced by ``cijoe --example``, use the following command: -.. literalinclude:: ../420_usage_workflow_all.cmd +.. literalinclude:: ../420_usage_task_all.cmd :language: bash The command will execute **all** the -:ref:`steps ` in the workflow. To run a subset of +:ref:`steps ` in the task. To run a subset of steps, you can specify the step name(s) as arguments to the ``cijoe`` tool, similar to how targets are specified in a :make:`Makefile <>`: -.. literalinclude:: ../450_usage_workflow_subset.cmd +.. literalinclude:: ../450_usage_task_subset.cmd :language: bash -With the above, only the step named **builtin_script** will be executed. This +With the above, only the step named **inline_commands** will be executed. This becomes even more useful when utilizing **cijoe** bash completions. -There are a couple of workflow-specific options. See -the :ref:`sec-resources-configs` section for reference. +There are a couple of task-specific options. See the +:ref:`sec-resources-configs-task` section for reference. -.. _sec-resources-workflows-content: +.. _sec-resources-tasks-content: Content Overview ================ -Let's take a look at what the workflow file produced by ``cijoe --example core.default`` +Let's take a look at what the task file produced by ``cijoe --example core.default`` looks like: -.. literalinclude:: ../../../src/cijoe/core/workflows/example_workflow_default.yaml +.. literalinclude:: ../../../src/cijoe/core/tasks/example_task_default.yaml :language: yaml At a first glance, then it might feel a bit similar to GitHub Actions workflow, @@ -50,20 +50,20 @@ but dramatically simpler since: * Minimal amount of "magic" keys - - ``doc``: Describe what the workflow does using multi-line plain-text + - ``doc``: Describe what the task does using multi-line plain-text - ``steps``: Ordered list of scripts, to inline-commands, to run -Descriptions of the content is provided in the following subsections. +Descriptions of the content are provided in the following subsections. -.. _sec-resources-workflows-steps: +.. _sec-resources-tasks-steps: Steps ===== Although **cijoe** aims to be simple, with minimal "magic" and a low learning -curve, there is some **yaml-magic** involved in the workflow steps. A step can -take one of two forms: either as :ref:`sec-resources-workflows-inline-commands` -or as :ref:`sec-resources-workflows-step-scripts`. +curve, there is some **yaml-magic** involved in the task steps. A step can +take one of two forms: either as :ref:`sec-resources-tasks-inline-commands` +or as :ref:`sec-resources-tasks-step-scripts`. Both forms require that a step **must** have a **name**. This allows subsets of steps to be executed via the ``cijoe`` command-line tool. When naming steps, @@ -77,19 +77,19 @@ follow these conventions: In short, use the typical lowercase identifier convention. -.. _sec-resources-workflows-inline-commands: +.. _sec-resources-tasks-inline-commands: Inline Commands --------------- A step with **inline commands** take the form: -.. literalinclude:: ../../../src/cijoe/core/workflows/example_workflow_default.yaml +.. literalinclude:: ../../../src/cijoe/core/tasks/example_task_default.yaml :language: yaml :lines: 24-27 Each line in a multi-line string is executed. It is implemented as a call to -``cijoe.run(command)``. Thus, the above notation for **inline commands** turn +``cijoe.run(command)``. Thus, the above notation for **inline commands** turns into execution of functions in the **cijoe** Python module: .. code-block:: python @@ -99,19 +99,20 @@ into execution of functions in the **cijoe** Python module: .. note:: This is implemented in **cijoe** as "syntactic-sugar" for - running the built-in script :ref:`core.cmdrunner `. - Thus, have a look at :ref:`sec-resources-workflows-step-scripts` to see what + running the built-in script :ref:`core.cmdrunner `. + Thus, have a look at :ref:`sec-resources-tasks-step-scripts` to see what this **unfolds** as. -.. _sec-resources-workflows-step-scripts: +.. _sec-resources-tasks-step-scripts: Steps with Scripts ------------------ -When a step runs a script, then you give it a **name** and you tell it +When a step runs a script, you give it a **name** and tell it which script to +run, like so: -.. literalinclude:: ../../../src/cijoe/core/workflows/example_workflow_default.yaml +.. literalinclude:: ../../../src/cijoe/core/tasks/example_task_default.yaml :language: yaml :lines: 29-34 @@ -128,12 +129,12 @@ with ``main(args, cijoe)`` in the ``args`` argument. -.. _sec-resources-workflows-linting: +.. _sec-resources-tasks-linting: Linting ------- -When you write a workflow yourself it can be nice to check whether it is valid +When you write a task yourself it can be nice to check whether it is valid without running it. You can do so by running: .. literalinclude:: ../300_lint.cmd diff --git a/docs/source/testrunner/index.rst b/docs/source/testrunner/index.rst index 1a8bb118..f2579990 100644 --- a/docs/source/testrunner/index.rst +++ b/docs/source/testrunner/index.rst @@ -21,12 +21,12 @@ The test runner can be used in two main ways: 1. Directly via the :pytest:`pytest <>` command-line: ``pytest --config cijoe-config.toml`` -2. Via a **cijoe** workflow, with a step using the :ref:`core.testrunner +2. Via a **cijoe** task, with a step using the :ref:`core.testrunner `, executed through the ``cijoe`` command-line tool. While the first method may be more familiar and require no further explanation, the test runner was specifically designed to be used within a **cijoe** -workflow and command-line interface. +task and command-line interface. The intent of using :pytest:`pytest <>` in this context is based on the assumption that, since **cijoe** is a Python project and the :ref:`scripts @@ -43,7 +43,7 @@ essential differences. Usage ===== -In a :ref:`workflow ` the :ref:`core.testrunner +In a :ref:`task ` the :ref:`core.testrunner ` is inserted as a step with arguments like below: .. code-block:: yaml @@ -66,5 +66,5 @@ This will result in the following invocation on the initiator: The key difference between invoking the ``pytest`` command-line tool directly and using the **cijoe** script :ref:`core.testrunner ` -in the **cijoe** workflow is that the latter integrates the **pytest** report into +in the **cijoe** task is that the latter integrates the **pytest** report into **cijoe**, producing a cohesive and standalone report. \ No newline at end of file diff --git a/docs/source/usage/index.rst b/docs/source/usage/index.rst index 4fad8d95..9317dd8e 100644 --- a/docs/source/usage/index.rst +++ b/docs/source/usage/index.rst @@ -11,13 +11,13 @@ then **cijoe** also requires input to function, these are: - An individual :ref:`script ` to execute **or** a collection of :ref:`scripts `, ordered, and documented - in a :ref:`workflow ` + in a :ref:`task ` - A :ref:`configuration file `, providing all the values that your :ref:`script(s) ` need For guidance on creating these files, refer to the :ref:`sec-resources` section. For the rest of the :ref:`sec-usage` section, and subsections, we will use the -example script, workflow, and configuration file generated by running: +example script, task, and configuration file generated by running: .. literalinclude:: ../200_quickstart.cmd :lines: 4-5 @@ -36,7 +36,7 @@ manner: - Producing example resources for the :ref:`Linux package ` The following sections describe the use of :ref:`sec-resources-scripts`, -:ref:`sec-resources-workflows`, the remainder of the current section +:ref:`sec-resources-tasks`, the remainder of the current section provides subsections with information provided for reference on all :ref:`sec-usage-cli`, :ref:`sec-usage-evars`, and behaverial information on :ref:`sec-usage-sp`. @@ -61,9 +61,9 @@ Which yields the following output: Search Paths ============ -The :ref:`sec-usage-cli` for the positional target argument, and for config-files -(``--c / --config``) by default search for files named ``cijoe-workflow.yaml`` -and ``cijoe-config.toml``, respectfully. These files are searched for, in order, +The :ref:`sec-usage-cli` for the positional target argument, and for config-files +(``--c / --config``) by default search for files named ``cijoe-task.yaml`` +and ``cijoe-config.toml``, respectfully. These files are searched for, in order, in the following locations: ``$PWD`` @@ -101,10 +101,14 @@ CIJOE_DEFAULT_CONFIG When set, the value will be used as the default for the command-line ``-c/--config`` argument. -CIJOE_DEFAULT_WORKFLOW +CIJOE_DEFAULT_TASK When set, the value will be used as the default for the positional command-line argument. +CIJOE_DEFAULT_WORKFLOW + Deprecated alias for ``CIJOE_DEFAULT_TASK``. Honored when + ``CIJOE_DEFAULT_TASK`` is unset; emits a deprecation warning. + .. _sec-usage-docker: @@ -175,11 +179,11 @@ as an artifact. $(which cijoe) --example core.default mv ./cijoe-example-core.default ./example - - name: Execute workflow + - name: Execute task run: | $(which cijoe) --monitor -l \ --config ./example/cijoe-config.toml \ - ./example/cijoe-workflow.yaml + ./example/cijoe-task.yaml - name: Upload report if: always() diff --git a/pyproject.toml b/pyproject.toml index e21858b8..9b8e1f43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dynamic = ["version"] description = "A loosely coupled approach to systems development and testing" readme = { file = "README.rst", content-type = "text/x-rst" } license = { file = "LICENSE" } -requires-python = ">=3.9" +requires-python = ">=3.10" keywords = ["systems", "development", "testing"] authors = [ { name = "Simon A. F. Lund", email = "os@safl.dk" } diff --git a/src/cijoe/cli/cli.py b/src/cijoe/cli/cli.py index a3c15ab6..9ffa2471 100644 --- a/src/cijoe/cli/cli.py +++ b/src/cijoe/cli/cli.py @@ -16,7 +16,7 @@ from cijoe.core.command import Cijoe, default_output_path from cijoe.core.resources import ( Config, - Workflow, + Task, dict_from_tomlfile, dict_from_yamlfile, dict_substitute, @@ -25,7 +25,8 @@ ) DEFAULT_CONFIG_FILENAME = "cijoe-config.toml" -DEFAULT_WORKFLOW_FILENAME = "cijoe-workflow.yaml" +DEFAULT_TASK_FILENAME = "cijoe-task.yaml" +LEGACY_TASK_FILENAME = "cijoe-workflow.yaml" DEFAULT_SCRIPT_FILENAME = "cijoe-script.py" SEARCH_PATHS = [ Path.cwd(), @@ -48,6 +49,21 @@ def __call__(self, parser, namespace, values, option_string=None): super().__call__(parser, namespace, values, option_string) +def _resolve_default_task_filename() -> str: + """Pick the default task filename, honoring deprecated env var.""" + + if "CIJOE_DEFAULT_TASK" in os.environ: + return os.environ["CIJOE_DEFAULT_TASK"] + + if "CIJOE_DEFAULT_WORKFLOW" in os.environ: + log.warning( + "CIJOE_DEFAULT_WORKFLOW is deprecated; use CIJOE_DEFAULT_TASK instead" + ) + return os.environ["CIJOE_DEFAULT_WORKFLOW"] + + return DEFAULT_TASK_FILENAME + + def search_for_file(path: Path) -> Optional[Path]: """ Search for a file named path.name. In case the given 'path' does not @@ -106,17 +122,17 @@ def log_errors(errors): def cli_integrity_check(args): - """Lint a workflow""" + """Lint a task""" log.info("cli: lint") - log.info(f"workflow: '{args.workflow}'") + log.info(f"task: '{args.task}'") log.info(f"config: '{args.config}'") - workflow_dict = dict_from_yamlfile(args.workflow.resolve()) + task_dict = dict_from_yamlfile(args.task.resolve()) - steps = workflow_dict.get("steps", None) + steps = task_dict.get("steps", None) if steps is None: - log.error("No steps in workflow, nothing to do.") + log.error("No steps in task, nothing to do.") return errno.EINVAL for nr, step in enumerate(steps, 1): @@ -131,8 +147,8 @@ def cli_integrity_check(args): ) return errno.EINVAL - errors = Workflow.dict_normalize(workflow_dict) # Normalize it - errors += Workflow.dict_lint(args, workflow_dict) # Check the yaml-file + errors = Task.dict_normalize(task_dict) # Normalize it + errors += Task.dict_lint(args, task_dict) # Check the yaml-file with tempfile.NamedTemporaryFile() as config_file: create_combined_toml(args.config, Path(config_file.name)) @@ -143,7 +159,7 @@ def cli_integrity_check(args): log.error(f"failed: Config.from_path({args.config})") return errno.EINVAL - errors += dict_substitute(workflow_dict, config.options) + errors += dict_substitute(task_dict, config.options) if errors: log_errors(errors) @@ -170,13 +186,27 @@ def cli_resources(args): return 0 +def _existing_state_path(output: Path) -> Optional[Path]: + """Return the path to task.state, falling back to legacy workflow.state.""" + + candidate = output / Task.STATE_FILENAME + if candidate.exists(): + return candidate + + legacy = output / "workflow.state" + if legacy.exists(): + return legacy + + return None + + def cli_archive(args): """Move 'output' directory into archive""" if args.output.exists(): - state_path = args.output / "workflow.state" + state_path = _existing_state_path(args.output) tag = "" - if state_path.exists(): + if state_path is not None: state = dict_from_yamlfile(state_path) tag = "" if state.get("tag", None) is None else "-".join(state["tag"]) + "-" t = str(time.strftime("%Y-%m-%d_%H:%M:%S")) @@ -187,13 +217,13 @@ def cli_archive(args): def cli_produce_report(args): - """Produce workflow-report""" + """Produce task-report""" config_path = args.output / "config.orig" - state_path = args.output / "workflow.state" + state_path = _existing_state_path(args.output) - if not state_path.exists(): - log.error("no workflow.state, nothing to produce a report for") + if state_path is None: + log.error(f"no {Task.STATE_FILENAME}, nothing to produce a report for") return errno.EINVAL if not config_path.exists(): @@ -219,22 +249,29 @@ def cli_produce_report(args): return reporter.func(args, cijoe) +def _example_id_from_task_name(task_name: str) -> Optional[str]: + """Strip the example prefix from a task resource name; None if no match.""" + + _, tail = task_name.split(".", 1) + for prefix in ("example_task_", "example_workflow_"): + if tail.startswith(prefix): + pkg_name = task_name.split(".", 1)[0] + return f"{pkg_name}.{tail[len(prefix):]}" + return None + + def cli_example(args): - """Create example config.toml and workflow.yaml""" + """Create example config.toml and task.yaml""" log.info("cli: examples") resources = get_resources() # Print examples when called like "cijoe --example" if args.example == "list_examples_and_exit": - for workflow_name, workflow in sorted(resources.get("workflows").items()): - if "example" not in workflow_name: - continue - - pkg_name, tail = workflow_name.split(".", 1) - example_name = tail.replace("example_workflow_", "") - - print(f"{pkg_name}.{example_name}") + for task_name in sorted(resources.get("tasks", {})): + example_id = _example_id_from_task_name(task_name) + if example_id is not None: + print(example_id) return 0 pkg_name, *tail = args.example.split(".") @@ -243,13 +280,14 @@ def cli_example(args): return errno.EINVAL # Emit examples - for workflow_name, workflow in sorted(resources.get("workflows").items()): - if not workflow_name.startswith(pkg_name): # Not the requested package + for task_name, task in sorted(resources.get("tasks", {}).items()): + if not task_name.startswith(pkg_name): # Not the requested package continue - if "example" not in workflow_name: # Not an example workflow + + cur_example_id = _example_id_from_task_name(task_name) + if cur_example_id is None: # Not an example task continue - cur_example_id = workflow_name.replace("example_workflow_", "") cur_pkg_name, cur_example_name = cur_example_id.split(".", 1) cur_example_dir = Path.cwd() / f"cijoe-example-{cur_example_id}" @@ -260,16 +298,24 @@ def cli_example(args): cur_example_dir.mkdir() - for section, default_filename in [ - ("config", DEFAULT_CONFIG_FILENAME), - ("workflow", DEFAULT_WORKFLOW_FILENAME), - ("script", DEFAULT_SCRIPT_FILENAME), + for section, default_filename, resource_category, prefixes in [ + ("config", DEFAULT_CONFIG_FILENAME, "configs", ["example_config_"]), + ( + "task", + DEFAULT_TASK_FILENAME, + "tasks", + ["example_task_", "example_workflow_"], + ), + ("script", DEFAULT_SCRIPT_FILENAME, "scripts", ["example_script_"]), ]: - label = f"{pkg_name}.example_{section}_{cur_example_name}" + resource = None + for prefix in prefixes: + label = f"{pkg_name}.{prefix}{cur_example_name}" + log.info(f"{args.example}, label({label})") + resource = resources.get(resource_category, {}).get(label, None) + if resource is not None: + break - log.info(f"{args.example}, label({label})") - - resource = resources.get(f"{section}s", {}).get(label, None) if resource is None: if section == "script": # Providing an example script is optional continue @@ -295,23 +341,37 @@ def cli_version(args): return 0 -def cli_workflow(args): - """Process workflow""" +def _get_fail_fast(cijoe) -> bool: + """Read cijoe.task.fail_fast, falling back to deprecated cijoe.workflow.fail_fast.""" + + if cijoe.getconf("cijoe.task.fail_fast") is not None: + return bool(cijoe.getconf("cijoe.task.fail_fast", False)) + + legacy = cijoe.getconf("cijoe.workflow.fail_fast") + if legacy is not None: + log.warning("[cijoe.workflow] is deprecated; use [cijoe.task] in your config") + return bool(legacy) + + return False + + +def cli_task(args): + """Process task""" log.info("cli: run") - log.info(f"workflow: {args.workflow}") + log.info(f"task: {args.task}") log.info(f"configs: {args.config}") log.info(f"output: {args.output}") cli_archive(args) - state_path = args.output / "workflow.state" + state_path = args.output / Task.STATE_FILENAME if state_path.exists(): log.error(f"aborting; output({args.output}) directory already exists") return errno.EPERM os.makedirs(args.output) - shutil.copyfile(args.workflow, args.output / "workflow.orig") + shutil.copyfile(args.task, args.output / "task.orig") if len(args.config) > 1: for i, c in enumerate(args.config): shutil.copyfile(c, args.output / f"config{i}.orig") @@ -326,21 +386,21 @@ def cli_workflow(args): log.error(f"failed: Config({args.config}).load()") return errno.EINVAL - workflow = Workflow(args.workflow) + task = Task(args.task) - errors = workflow.load(args, config) + errors = task.load(args, config) if errors: log_errors(errors) - log.error("workflow.load(): see errors above or run 'cijoe -i'") + log.error("task.load(): see errors above or run 'cijoe -i'") return errno.EINVAL - workflow.state["tag"] = args.tag - step_names = [step["name"] for step in workflow.state["steps"]] + task.state["tag"] = args.tag + step_names = [step["name"] for step in task.state["steps"]] for step_name in args.step: if step_name in step_names: continue - log.error(f"step({step_name}) not in workflow") + log.error(f"step({step_name}) not in task") return errno.EINVAL resources = get_resources() @@ -348,7 +408,7 @@ def cli_workflow(args): # pre-load scripts and augment state with step-descriptions. # TODO: for some reason when mod.__doc__ is None, then the docstring from a previous # mod trickles in. This should be fixed... - for step in workflow.state["steps"]: + for step in task.state["steps"]: script_ident = step["uses"] resources["scripts"][script_ident].load() @@ -356,17 +416,17 @@ def cli_workflow(args): if resources["scripts"][script_ident].mod.__doc__: step["description"] = str(resources["scripts"][script_ident].mod.__doc__) - workflow.state["status"]["started"] = time.time() + task.state["status"]["started"] = time.time() cijoe = Cijoe(config, args.output, args.monitor) - fail_fast = cijoe.getconf("cijoe.workflow.fail_fast", False) + fail_fast = _get_fail_fast(cijoe) - for step in workflow.state["steps"]: + for step in task.state["steps"]: log.info(f"step({step['name']}) - begin") begin = time.time() step["status"]["started"] = begin - workflow.state_dump(args.output / Workflow.STATE_FILENAME) + task.state_dump(args.output / Task.STATE_FILENAME) cijoe.set_output_ident(step["id"]) os.makedirs(os.path.join(cijoe.output_path, step["id"]), exist_ok=True) @@ -409,10 +469,10 @@ def cli_workflow(args): delattr(args, k) for key in ["failed", "passed", "skipped"]: - workflow.state["status"][key] += step["status"][key] + task.state["status"][key] += step["status"][key] step["status"]["elapsed"] = time.time() - begin - workflow.state["status"]["elapsed"] += step["status"]["elapsed"] + task.state["status"]["elapsed"] += step["status"]["elapsed"] for text, status in step["status"].items(): if text != "elapsed" and status: @@ -422,16 +482,16 @@ def cli_workflow(args): log.error(f"exiting, fail_fast({fail_fast})") break - workflow.state_dump(args.output / Workflow.STATE_FILENAME) + task.state_dump(args.output / Task.STATE_FILENAME) - err = errno.EIO if workflow.state["status"]["failed"] else 0 + err = errno.EIO if task.state["status"]["failed"] else 0 if err: log.error("one or more steps failed") return err -def create_adhoc_workflow(args): +def create_adhoc_task(args): target = args.script_name if target.endswith(".py"): path = Path(target) @@ -439,19 +499,19 @@ def create_adhoc_workflow(args): resources = get_resources() - template_path = resources["templates"]["core.example-tmp-workflow.yaml"].path + template_path = resources["templates"]["core.example-tmp-task.yaml"].path jinja_env = jinja2.Environment( autoescape=True, loader=jinja2.FileSystemLoader(template_path.parent) ) template = jinja_env.get_template(template_path.name) - with tempfile.NamedTemporaryFile() as workflow: - setattr(args, "workflow", Path(workflow.name)) + with tempfile.NamedTemporaryFile() as task_file: + setattr(args, "task", Path(task_file.name)) setattr(args, "step", []) content = template.render(steps=[target]) - workflow.write(bytes(content, "utf-8")) - workflow.seek(0) + task_file.write(bytes(content, "utf-8")) + task_file.seek(0) sys.exit(main(args)) @@ -464,23 +524,24 @@ def parse_args(): parent_parser = argparse.ArgumentParser(add_help=False) run_group = parent_parser.add_argument_group( - "run", "Options for running a workflow script." + "run", "Options for running a task script." ) run_group.add_argument( "target", nargs="?", default=None, - help="A cijoe workflow or script to run.", + help="A cijoe task or script to run.", ) run_group.add_argument( "step", nargs="*", default=[], - help="Given a workflow, the steps of the workflow it should run. If none are given, all steps are run.", + help="Given a task, the steps of the task it should run. If none are given, all steps are run.", ) run_group.add_argument( "--workflow", "-w", + dest="workflow_deprecated", default=None, help=argparse.SUPPRESS, ) @@ -517,25 +578,25 @@ def parse_args(): "--no-report", "-n", action="store_true", - help="Skip the producing, and opening, a report at the end of the workflow-run", + help="Skip the producing, and opening, a report at the end of the task-run", ) run_group.add_argument( "--skip-report", "-s", action="store_false", - help="Skip the report opening at the end of the workflow-run", + help="Skip the report opening at the end of the task-run", ) run_group.add_argument( "--tag", "-t", type=str, action="append", - help="Tags to identify a workflow-run." + help="Tags to identify a task-run." " This will be prefixed while storing in archive", ) utils_group = parent_parser.add_argument_group( - "utilities", "Workflow, and workflow-related utilities" + "utilities", "Task, and task-related utilities" ) utils_group.add_argument( "--archive", @@ -554,7 +615,7 @@ def parse_args(): "--integrity-check", "-i", action="store_true", - help="Check integrity of workflow given as positional argument and exit.", + help="Check integrity of task given as positional argument and exit.", ) utils_group.add_argument( "--resources", @@ -595,27 +656,25 @@ def parse_args(): # If the --workflow argument is used, and a positional argument is given, the # parser will interpret it as a `target`, but we assume it to be a step # identifier. - if args.workflow: + if args.workflow_deprecated: log.warning( - "The -w / --workflow argument is deprecated" - "please specify the workflow as a positional argument instead." + "The -w / --workflow argument is deprecated; " + "specify the task as a positional argument instead." ) args = parser.parse_intermixed_args() if args.target: args.step = [args.target] + args.step - args.workflow = Path(args.workflow) + args.task = Path(args.workflow_deprecated) - # If the target ends with .yaml, we run the workflow at the given path, and + # If the target ends with .yaml, we run the task at the given path, and # assume that the remaining positional args are step identifiers of the - # given workflow. + # given task. # note: the `parse_intermixed_args` allow for multiple groups of positional - # arguments, i.e. both the workflow path and step identifiers. + # arguments, i.e. both the task path and step identifiers. elif not args.target or args.target.endswith(".yaml"): args = parser.parse_intermixed_args() - workflow = args.target or os.environ.get( - "CIJOE_DEFAULT_WORKFLOW", DEFAULT_WORKFLOW_FILENAME - ) - setattr(args, "workflow", Path(workflow)) + task = args.target or _resolve_default_task_filename() + setattr(args, "task", Path(task)) # Else, we assume the target is either a cijoe script identifier or path else: @@ -664,6 +723,9 @@ def parse_args(): force=True, ) + if hasattr(args, "workflow_deprecated"): + delattr(args, "workflow_deprecated") + return 0, args @@ -677,7 +739,7 @@ def main(args=None): return err if getattr(args, "script_name", None): # Running stand-alone script - create_adhoc_workflow(args) + create_adhoc_task(args) if args.resources: return cli_resources(args) @@ -694,7 +756,7 @@ def main(args=None): if args.archive: return cli_archive(args) - if not getattr(args, "workflow", None): + if not getattr(args, "task", None): log.error("No target given; exiting") return errno.EINVAL @@ -703,11 +765,19 @@ def main(args=None): Path(os.environ.get("CIJOE_DEFAULT_CONFIG", DEFAULT_CONFIG_FILENAME)) ] - path = search_for_file(args.workflow) + path = search_for_file(args.task) + if path is None and args.task.name == DEFAULT_TASK_FILENAME: + legacy_path = search_for_file(Path(LEGACY_TASK_FILENAME)) + if legacy_path is not None: + log.warning( + f"Using legacy '{LEGACY_TASK_FILENAME}'; " + f"rename it to '{DEFAULT_TASK_FILENAME}'" + ) + path = legacy_path if path is None: - log.error(f"workflow({args.workflow}) does not exist; exiting") + log.error(f"task({args.task}) does not exist; exiting") return errno.EINVAL - args.workflow = path + args.task = path tmp = args.config args.config = [] @@ -726,7 +796,7 @@ def main(args=None): # that here. args.output = args.output.resolve() - err = cli_workflow(args) + err = cli_task(args) if not args.no_report: report_err = cli_produce_report(args) diff --git a/src/cijoe/core/configs/example_config_default.toml b/src/cijoe/core/configs/example_config_default.toml index c235cf34..60626f0d 100644 --- a/src/cijoe/core/configs/example_config_default.toml +++ b/src/cijoe/core/configs/example_config_default.toml @@ -1,5 +1,5 @@ -[cijoe.workflow] -# Change this to true to stop processing additional workflow steps when a step fails +[cijoe.task] +# Change this to true to stop processing additional task steps when a step fails fail_fast=false [example] diff --git a/src/cijoe/core/configs/example_config_get_put.toml b/src/cijoe/core/configs/example_config_get_put.toml index a1bd8fbf..355f630e 100644 --- a/src/cijoe/core/configs/example_config_get_put.toml +++ b/src/cijoe/core/configs/example_config_get_put.toml @@ -8,7 +8,7 @@ password = "root" hostname = "localhost" port = 4200 -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by: the qemu.*.py scripts @@ -38,7 +38,7 @@ system_args.raw = """\ # INITIALIZE.DISKIMAGE: Options for the "qemu.guest_initialize" script # -# You can uncomment it in your config here, or provide as argument as a workflow step-argument +# You can uncomment it in your config here, or provide as argument as a task step-argument # #initialize.diskimage = "debian-12-aarch64" diff --git a/src/cijoe/core/configs/example_config_testrunner.toml b/src/cijoe/core/configs/example_config_testrunner.toml index 0e02ec3f..782c2d31 100644 --- a/src/cijoe/core/configs/example_config_testrunner.toml +++ b/src/cijoe/core/configs/example_config_testrunner.toml @@ -8,7 +8,7 @@ password = "root" hostname = "localhost" port = 4200 -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by core.example_script_testrunner @@ -33,7 +33,7 @@ path = "{{ local.env.HOME }}/guests/generic-bios-kvm-x86_64" system_label = "x86_64" # Name of the system_image to use; see "system_imaging.images" -# Uncomment here, or set as workflow-argument when using "qemu.guest_initialize" +# Uncomment here, or set as task-argument when using "qemu.guest_initialize" #system_image_name = "debian-12-x86_64" # Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. diff --git a/src/cijoe/core/processing.py b/src/cijoe/core/processing.py index 1c874e6e..7c5913d1 100644 --- a/src/cijoe/core/processing.py +++ b/src/cijoe/core/processing.py @@ -1,5 +1,5 @@ """ - Helper functions for processing the output produced by a workflow + Helper functions for processing the output produced by a task """ import json @@ -8,7 +8,7 @@ from typing import Any, Dict, Union from cijoe.core.misc import ENCODING, sanitize_ident -from cijoe.core.resources import dict_from_yamlfile +from cijoe.core.resources import Task, dict_from_yamlfile def cmd_number_from_path(path): @@ -175,16 +175,18 @@ def artifacts_in_path(path: Path): return sorted(artifacts) -def process_workflow_output(args, cijoe): - workflow_state = dict_from_yamlfile(args.output / "workflow.state") - workflow_state["config"] = cijoe.config.options - workflow_state["artifacts"] = artifacts_in_path(args.output) +def process_task_output(args, cijoe): + state_path = args.output / Task.STATE_FILENAME + if not state_path.exists(): + legacy = args.output / "workflow.state" + if legacy.exists(): + state_path = legacy - # workflow_state["artifacts"] = artifacts_in_path( - # args.output, args.output / "artifacts" - # ) + task_state = dict_from_yamlfile(state_path) + task_state["config"] = cijoe.config.options + task_state["artifacts"] = artifacts_in_path(args.output) - for step in workflow_state["steps"]: + for step in task_state["steps"]: if "extras" not in step: step["extras"] = {} @@ -195,11 +197,6 @@ def process_workflow_output(args, cijoe): if not step_path.exists(): continue - # artifacts = artifacts_in_path(args.output, step_path / "artifacts") - # artifacts = artifacts_in_path(step_path / "artifacts") - # if artifacts: - # step["extras"]["artifacts"] = artifacts - runlog = runlog_from_path(step_path) if runlog: step["extras"]["runlog"] = runlog @@ -208,4 +205,8 @@ def process_workflow_output(args, cijoe): if testreport: step["extras"]["testreport"] = testreport - return workflow_state + return task_state + + +# Deprecated alias for backwards compatibility; prefer process_task_output. +process_workflow_output = process_task_output diff --git a/src/cijoe/core/resources.py b/src/cijoe/core/resources.py index 5ebfb9e0..dabdb6c3 100644 --- a/src/cijoe/core/resources.py +++ b/src/cijoe/core/resources.py @@ -8,14 +8,14 @@ * cijoe.core.command (Cijoe) * cijoe.core.transport (Transport, Local, SSH) * cijoe.core.misc (As the name suggests; various helper-functions) - * cijoe.cli (Command-Line Tool and utilization of the above for workflow execution) + * cijoe.cli (Command-Line Tool and utilization of the above for task execution) Everything else, literally everything, is implemented as dynamically collectable and loadable resources. That is, cijoe - configuration-files/scripts/workflows/templates, and auxiliary files. + configuration-files/scripts/tasks/templates, and auxiliary files. The base-representation of these resources are the cijoe.core.resources.Resource - class, with content-specific subclasses (Config, Script, and Workflow). + class, with content-specific subclasses (Config, Script, and Task). These resources are collected from installed and locally available Packages, as well as for path by the cijoe.core.resources.Collector. @@ -320,9 +320,9 @@ def load(self): return ["Missing script_entry() function in loaded module"] -class Workflow(Resource): +class Task(Resource): SUFFIX = ".yaml" - STATE_FILENAME = "workflow.state" + STATE_FILENAME = "task.state" STATE: Dict[str, Any] = { "doc": "", "tag": "", @@ -344,14 +344,14 @@ def __init__(self, path, pkg=None): self.config = None def state_dump(self, path): - """Dump the current workflow-state to yaml-file""" + """Dump the current task-state to yaml-file""" with path.open("w+") as state_file: yaml.dump(self.state, state_file) @staticmethod def dict_normalize(topic: dict): - """Normalize the workflow-dict, transformation of the 'run' shorthand""" + """Normalize the task-dict, transformation of the 'run' shorthand""" errors = [] @@ -399,7 +399,7 @@ def dict_normalize(topic: dict): @staticmethod def dict_lint(args: Namespace, topic: dict): - """Returns a list of integrity-errors for the given workflow-dict(topic)""" + """Returns a list of integrity-errors for the given task-dict(topic)""" resources = get_resources() @@ -473,7 +473,7 @@ def dict_lint(args: Namespace, topic: dict): def load(self, args: Namespace, config: Config, extra_steps: list = []): """ - Load the workflow-yamlfile, normalize it, lint it, substitute, then construct + Load the task-yamlfile, normalize it, lint it, substitute, then construct the object properties """ @@ -482,26 +482,26 @@ def load(self, args: Namespace, config: Config, extra_steps: list = []): if self.state: return errors - workflow_dict = dict_from_yamlfile(self.path) - workflow_dict["steps"] += extra_steps + task_dict = dict_from_yamlfile(self.path) + task_dict["steps"] += extra_steps - errors += Workflow.dict_normalize(workflow_dict) + errors += Task.dict_normalize(task_dict) if errors: return errors - errors += Workflow.dict_lint(args, workflow_dict) + errors += Task.dict_lint(args, task_dict) if errors: return errors - errors += dict_substitute(workflow_dict, default_context(config)) + errors += dict_substitute(task_dict, default_context(config)) if errors: return errors - state = Workflow.STATE.copy() - state["doc"] = workflow_dict.get("doc") - state["config"] = workflow_dict.get("config", {}) + state = Task.STATE.copy() + state["doc"] = task_dict.get("doc") + state["config"] = task_dict.get("config", {}) state["steps"] = list(state["steps"]) - for nr, step in enumerate(workflow_dict["steps"], 1): + for nr, step in enumerate(task_dict["steps"], 1): step["nr"] = nr step["status"] = { "skipped": 0, @@ -525,10 +525,11 @@ class Collector(object): RESOURCES = [ ("configs", Config.SUFFIX), ("templates", ".jinja2"), - ("workflows", Workflow.SUFFIX), + ("tasks", Task.SUFFIX), ("scripts", Script.SUFFIX), ("auxiliary", ".*"), ] + LEGACY_CATEGORIES = {"workflows": "tasks"} IGNORE = ["__init__.py", "__pycache__", "setup.py"] def __new__(cls): @@ -540,7 +541,7 @@ def __new__(cls): def __process_candidate(self, candidate: Path, category: str, pkg): """Inserts the given candidate""" - resource: Union[Script, Config, Workflow, Resource] + resource: Union[Script, Config, Task, Resource] if category == "scripts": resource = Script(candidate, pkg) @@ -550,8 +551,8 @@ def __process_candidate(self, candidate: Path, category: str, pkg): category = "auxiliary" elif category == "configs": resource = Config(candidate, pkg) - elif category == "workflows": - resource = Workflow(candidate, pkg) + elif category == "tasks": + resource = Task(candidate, pkg) else: resource = Resource(candidate, pkg) @@ -591,16 +592,24 @@ def collect_from_packages(self, path=None, prefix=None): if prefix is None: prefix = "" + canonical_categories = {cat for cat, _ in Collector.RESOURCES} + for pkg in pkgutil.walk_packages(path, prefix): comp = pkg.name.split(".")[1:] # drop the 'cijoe.' prefix - if not ( - pkg.ispkg - and any(cat in comp for cat, _ in Collector.RESOURCES) - and len(comp) == 2 - ): # skip non-resource packages + if not (pkg.ispkg and len(comp) == 2): continue _, category = comp + if category in Collector.LEGACY_CATEGORIES: + log.warning( + f"package({pkg.name}): " + f"'{category}/' subdir is deprecated; " + f"rename to '{Collector.LEGACY_CATEGORIES[category]}/'" + ) + category = Collector.LEGACY_CATEGORIES[category] + elif category not in canonical_categories: + continue + for candidate in importlib_files(f"{pkg.name}").iterdir(): if candidate.name in Collector.IGNORE: continue @@ -637,3 +646,7 @@ def get_resources(paths=[]): collector.collect(paths) return collector.resources + + +# Deprecated alias for backwards compatibility; prefer Task. +Workflow = Task diff --git a/src/cijoe/core/scripts/example_script_default.py b/src/cijoe/core/scripts/example_script_default.py index bcda9c0c..898e7557 100644 --- a/src/cijoe/core/scripts/example_script_default.py +++ b/src/cijoe/core/scripts/example_script_default.py @@ -6,8 +6,8 @@ number of times and allows parameterization of the message content. The purpose of this script is to demonstrate how to run commands and supply input to the -script using a configuration file, environment variables, command-line arguments and -workflow step arguments. +script using a configuration file, environment variables, command-line arguments and +task step arguments. An example of using the core infrastructure of cijoe: @@ -17,7 +17,7 @@ - Output processing state.output() Input is given to scripts via configuration-files, environment variables and from -workflow-step-arguments, this is demonstrated as the first thing in the script. +task-step-arguments, this is demonstrated as the first thing in the script. cijoe also has primitives for transferring data: @@ -55,7 +55,7 @@ def main(args: Namespace, cijoe: Cijoe): # Grab message from the configuration-file message = cijoe.getconf("example.message", "Hello World!") - # When executed via workflow, grab the step-argument + # When executed via task, grab the step-argument repeat = args.repeat if repeat < 1: log.error(f"Invalid step-argument: repeat({repeat}) < 1") diff --git a/src/cijoe/core/scripts/example_script_testrunner.py b/src/cijoe/core/scripts/example_script_testrunner.py index c75b22fd..1504dcc3 100644 --- a/src/cijoe/core/scripts/example_script_testrunner.py +++ b/src/cijoe/core/scripts/example_script_testrunner.py @@ -30,7 +30,7 @@ def test_true(cijoe: Cijoe): def main(args: Namespace, cijoe: Cijoe): """ - This main function is not run as part of the example workflow in the + This main function is not run as part of the example task in the core.testrunner example, but must be here in order for it to be elicited as script when running `cijoe --example core.testrunner`. """ diff --git a/src/cijoe/core/scripts/reporter.py b/src/cijoe/core/scripts/reporter.py index 2a0209fc..44ed757a 100644 --- a/src/cijoe/core/scripts/reporter.py +++ b/src/cijoe/core/scripts/reporter.py @@ -2,7 +2,7 @@ Report generator ================ -Generates a HTML report in the workflow output directory. +Generates a HTML report in the task output directory. Retargtable: false ------------------ @@ -19,7 +19,7 @@ import jinja2 import yaml -from cijoe.core.processing import process_workflow_output +from cijoe.core.processing import process_task_output from cijoe.core.resources import get_resources @@ -60,19 +60,19 @@ def timestamp_to_txt(value): def main(args, cijoe): - """Produce a HTML report of the 'workflow.state' file in 'args.output'""" + """Produce a HTML report of the 'task.state' file in 'args.output'""" report_open = args.report_open resources = get_resources() - template_path = resources["templates"]["core.report-workflow.html"].path + template_path = resources["templates"]["core.report-task.html"].path report_path = args.output / "report.html" log.info(f"template: {template_path}") log.info(f"report: {report_path}") - workflow_state = process_workflow_output(args, cijoe) + task_state = process_task_output(args, cijoe) jinja_env = jinja2.Environment( autoescape=True, loader=jinja2.FileSystemLoader(template_path.parent) @@ -83,7 +83,7 @@ def main(args, cijoe): template = jinja_env.get_template(template_path.name) with (report_path).open("w") as report: - report.write(template.render(workflow_state)) + report.write(template.render(task_state)) if report_open: webbrowser.open("file://%s" % report_path.resolve()) diff --git a/src/cijoe/core/scripts/wait_for_transport.py b/src/cijoe/core/scripts/wait_for_transport.py index 69c723b6..8df115eb 100644 --- a/src/cijoe/core/scripts/wait_for_transport.py +++ b/src/cijoe/core/scripts/wait_for_transport.py @@ -3,7 +3,7 @@ Wait for Transport State (Up or Down) ===================================== -This script is useful in CIJOE workflows where execution should block until a +This script is useful in CIJOE tasks where execution should block until a transport becomes available (e.g., after a reboot) or unavailable (e.g., during shutdown). @@ -23,7 +23,7 @@ Example Use Case ---------------- -A typical workflow might look like:: +A typical task might look like:: - name: reboot run: shutdown -r now diff --git a/src/cijoe/core/tasks/__init__.py b/src/cijoe/core/tasks/__init__.py new file mode 100644 index 00000000..cdcae04a --- /dev/null +++ b/src/cijoe/core/tasks/__init__.py @@ -0,0 +1,3 @@ +""" + This package is a container for task files +""" diff --git a/src/cijoe/core/workflows/example_workflow_default.yaml b/src/cijoe/core/tasks/example_task_default.yaml similarity index 93% rename from src/cijoe/core/workflows/example_workflow_default.yaml rename to src/cijoe/core/tasks/example_task_default.yaml index c0e4ad88..7377ffa7 100644 --- a/src/cijoe/core/workflows/example_workflow_default.yaml +++ b/src/cijoe/core/tasks/example_task_default.yaml @@ -1,6 +1,6 @@ --- doc: | - This is a workflow file, it serves as an example on how to run commands and + This is a task file, it serves as an example on how to run commands and scripts, the structure intentionally mimics that of GitHUB actions, however, the keys you see here are all there is. diff --git a/src/cijoe/core/workflows/example_workflow_get_put.yaml b/src/cijoe/core/tasks/example_task_get_put.yaml similarity index 91% rename from src/cijoe/core/workflows/example_workflow_get_put.yaml rename to src/cijoe/core/tasks/example_task_get_put.yaml index b8c6511f..fd67cd65 100644 --- a/src/cijoe/core/workflows/example_workflow_get_put.yaml +++ b/src/cijoe/core/tasks/example_task_get_put.yaml @@ -1,6 +1,6 @@ --- doc: | - This workflow file is an example of how to use core.put and core.get. + This task file is an example of how to use core.put and core.get. steps: - name: guest_initialize diff --git a/src/cijoe/core/workflows/example_workflow_testrunner.yaml b/src/cijoe/core/tasks/example_task_testrunner.yaml similarity index 91% rename from src/cijoe/core/workflows/example_workflow_testrunner.yaml rename to src/cijoe/core/tasks/example_task_testrunner.yaml index 567924b9..b563dc52 100644 --- a/src/cijoe/core/workflows/example_workflow_testrunner.yaml +++ b/src/cijoe/core/tasks/example_task_testrunner.yaml @@ -1,9 +1,9 @@ --- doc: | - This workflow file is an example of how to use the CIJOE testrunner. + This task file is an example of how to use the CIJOE testrunner. The testrunner script takes three optional arguments. The purpose of this - workflow is to run the testrunner with different combinations of these + task is to run the testrunner with different combinations of these arguments. steps: diff --git a/src/cijoe/core/templates/example-tmp-workflow.yaml.jinja2 b/src/cijoe/core/templates/example-tmp-task.yaml.jinja2 similarity index 100% rename from src/cijoe/core/templates/example-tmp-workflow.yaml.jinja2 rename to src/cijoe/core/templates/example-tmp-task.yaml.jinja2 diff --git a/src/cijoe/core/templates/report-workflow.html.jinja2 b/src/cijoe/core/templates/report-task.html.jinja2 similarity index 97% rename from src/cijoe/core/templates/report-workflow.html.jinja2 rename to src/cijoe/core/templates/report-task.html.jinja2 index 37e07ddd..a502e195 100644 --- a/src/cijoe/core/templates/report-workflow.html.jinja2 +++ b/src/cijoe/core/templates/report-task.html.jinja2 @@ -3,7 +3,7 @@ -CIJOE: Workflow State Report +CIJOE: Task State Report @@ -114,14 +114,14 @@ function selectFilter(event) { -

Workflow Report

+

Task Report

- This is a HTML-ification of Workflow state (workflow.state) and - the associated files in the workflow output directory. - Below is the doc section of the workflow (.workflow file). + This is a HTML-ification of Task state (task.state) and + the associated files in the task output directory. + Below is the doc section of the task. 

{{ doc }}
@@ -157,7 +157,7 @@ function selectFilter(event) {
- The workflow started on {{ status.get("started", 0.0) | timestamp_to_txt }}, status on the steps + The task started on {{ status.get("started", 0.0) | timestamp_to_txt }}, status on the steps is provided below.
@@ -180,7 +180,7 @@ function selectFilter(event) {

- If a workflow step (script / run) or test produced any artifacts, then they are + If a task step (script / run) or test produced any artifacts, then they are listed here.

@@ -207,7 +207,7 @@ function selectFilter(event) {

- This is the doc section of the workflow (.workflow file). + This is the doc section of the task. 

{{ doc }}
diff --git a/src/cijoe/core/workflows/__init__.py b/src/cijoe/core/workflows/__init__.py deleted file mode 100644 index 3452024c..00000000 --- a/src/cijoe/core/workflows/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -""" - This package is a container for workflow files -""" diff --git a/src/cijoe/linux/configs/example_config_null_blk.toml b/src/cijoe/linux/configs/example_config_null_blk.toml index d17ea543..fb5b076e 100644 --- a/src/cijoe/linux/configs/example_config_null_blk.toml +++ b/src/cijoe/linux/configs/example_config_null_blk.toml @@ -8,7 +8,7 @@ password = "root" hostname = "localhost" port = 4200 -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by: the qemu.*.py scripts @@ -29,7 +29,7 @@ path = "{{ local.env.HOME }}/guests/generic-bios-kvm-x86_64" system_label = "x86_64" # Name of the system_image to use; see "system_imaging.images" -# Uncomment here, or set as workflow-argument when using "qemu.guest_initialize" +# Uncomment here, or set as task-argument when using "qemu.guest_initialize" #system_image_name = "debian-12-x86_64" # Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. @@ -72,7 +72,7 @@ system_args.raw = """\ # INITIALIZE.DISKIMAGE: Options for the "qemu.guest_initialize" script # -# You can uncomment it in your config here, or provide as argument as a workflow step-argument +# You can uncomment it in your config here, or provide as argument as a task step-argument # #initialize.diskimage = "debian-12-aarch64" diff --git a/src/cijoe/linux/workflows/__init__.py b/src/cijoe/linux/tasks/__init__.py similarity index 100% rename from src/cijoe/linux/workflows/__init__.py rename to src/cijoe/linux/tasks/__init__.py diff --git a/src/cijoe/linux/workflows/example_workflow_build_kdebs.yaml b/src/cijoe/linux/tasks/example_task_build_kdebs.yaml similarity index 80% rename from src/cijoe/linux/workflows/example_workflow_build_kdebs.yaml rename to src/cijoe/linux/tasks/example_task_build_kdebs.yaml index 388ca61d..63723453 100644 --- a/src/cijoe/linux/workflows/example_workflow_build_kdebs.yaml +++ b/src/cijoe/linux/tasks/example_task_build_kdebs.yaml @@ -1,6 +1,6 @@ --- doc: | - This workflow builds Linux kernel as .deb installable packages + This task builds Linux kernel as .deb installable packages NOTE, if you switch 'run_local' to 'False', then you have to collect the kdebs yourself. diff --git a/src/cijoe/linux/workflows/example_workflow_null_blk.yaml b/src/cijoe/linux/tasks/example_task_null_blk.yaml similarity index 100% rename from src/cijoe/linux/workflows/example_workflow_null_blk.yaml rename to src/cijoe/linux/tasks/example_task_null_blk.yaml diff --git a/src/cijoe/qemu/configs/example_config_build.toml b/src/cijoe/qemu/configs/example_config_build.toml index 4753764b..bb44f50f 100644 --- a/src/cijoe/qemu/configs/example_config_build.toml +++ b/src/cijoe/qemu/configs/example_config_build.toml @@ -4,7 +4,7 @@ # configuration here. # -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by: qemu.build.py diff --git a/src/cijoe/qemu/configs/example_config_guest_aarch64.toml b/src/cijoe/qemu/configs/example_config_guest_aarch64.toml index 743ac3ca..b50d1056 100644 --- a/src/cijoe/qemu/configs/example_config_guest_aarch64.toml +++ b/src/cijoe/qemu/configs/example_config_guest_aarch64.toml @@ -8,7 +8,7 @@ password = "root" hostname = "localhost" port = 4200 -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by: the qemu.*.py scripts @@ -29,7 +29,7 @@ path = "{{ local.env.HOME }}/guests/generic-bios-kvm-x86_64" system_label = "x86_64" # Name of the system_image to use; see "system_imaging.images" -# Uncomment here, or set as workflow-argument when using "qemu.guest_initialize" +# Uncomment here, or set as task-argument when using "qemu.guest_initialize" #system_image_name = "debian-12-x86_64" # Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. @@ -72,7 +72,7 @@ system_args.raw = """\ # INITIALIZE.DISKIMAGE: Options for the "qemu.guest_initialize" script # -# You can uncomment it in your config here, or provide as argument as a workflow step-argument +# You can uncomment it in your config here, or provide as argument as a task step-argument # #initialize.diskimage = "debian-12-aarch64" diff --git a/src/cijoe/qemu/configs/example_config_guest_x86_64.toml b/src/cijoe/qemu/configs/example_config_guest_x86_64.toml index 743ac3ca..b50d1056 100644 --- a/src/cijoe/qemu/configs/example_config_guest_x86_64.toml +++ b/src/cijoe/qemu/configs/example_config_guest_x86_64.toml @@ -8,7 +8,7 @@ password = "root" hostname = "localhost" port = 4200 -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by: the qemu.*.py scripts @@ -29,7 +29,7 @@ path = "{{ local.env.HOME }}/guests/generic-bios-kvm-x86_64" system_label = "x86_64" # Name of the system_image to use; see "system_imaging.images" -# Uncomment here, or set as workflow-argument when using "qemu.guest_initialize" +# Uncomment here, or set as task-argument when using "qemu.guest_initialize" #system_image_name = "debian-12-x86_64" # Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. @@ -72,7 +72,7 @@ system_args.raw = """\ # INITIALIZE.DISKIMAGE: Options for the "qemu.guest_initialize" script # -# You can uncomment it in your config here, or provide as argument as a workflow step-argument +# You can uncomment it in your config here, or provide as argument as a task step-argument # #initialize.diskimage = "debian-12-aarch64" diff --git a/src/cijoe/qemu/workflows/__init__.py b/src/cijoe/qemu/tasks/__init__.py similarity index 100% rename from src/cijoe/qemu/workflows/__init__.py rename to src/cijoe/qemu/tasks/__init__.py diff --git a/src/cijoe/qemu/workflows/example_workflow_build.yaml b/src/cijoe/qemu/tasks/example_task_build.yaml similarity index 87% rename from src/cijoe/qemu/workflows/example_workflow_build.yaml rename to src/cijoe/qemu/tasks/example_task_build.yaml index 3f3bba35..13c920c4 100644 --- a/src/cijoe/qemu/workflows/example_workflow_build.yaml +++ b/src/cijoe/qemu/tasks/example_task_build.yaml @@ -1,6 +1,6 @@ --- doc: | - This workflow demonstrates how to build qemu from source + This task demonstrates how to build qemu from source * Build qemu from source (x86_64 and aarch64) * Install qemu diff --git a/src/cijoe/qemu/workflows/example_workflow_guest_aarch64.yaml b/src/cijoe/qemu/tasks/example_task_guest_aarch64.yaml similarity index 100% rename from src/cijoe/qemu/workflows/example_workflow_guest_aarch64.yaml rename to src/cijoe/qemu/tasks/example_task_guest_aarch64.yaml diff --git a/src/cijoe/qemu/workflows/example_workflow_guest_x86_64.yaml b/src/cijoe/qemu/tasks/example_task_guest_x86_64.yaml similarity index 100% rename from src/cijoe/qemu/workflows/example_workflow_guest_x86_64.yaml rename to src/cijoe/qemu/tasks/example_task_guest_x86_64.yaml diff --git a/src/cijoe/system_imaging/configs/example_config_debian.toml b/src/cijoe/system_imaging/configs/example_config_debian.toml index eb6826ad..a8a0a502 100644 --- a/src/cijoe/system_imaging/configs/example_config_debian.toml +++ b/src/cijoe/system_imaging/configs/example_config_debian.toml @@ -8,7 +8,7 @@ password = "root" hostname = "localhost" port = 4200 -[cijoe.workflow] +[cijoe.task] fail_fast=true # Used by: the qemu.*.py scripts @@ -29,7 +29,7 @@ path = "{{ local.env.HOME }}/guests/generic-bios-kvm-x86_64" system_label = "x86_64" # Name of the system_image to use; see "system_imaging.images" -# Uncomment here, or set as workflow-argument when using "qemu.guest_initialize" +# Uncomment here, or set as task-argument when using "qemu.guest_initialize" #system_image_name = "debian-12-x86_64" # Keyword arguments: joined onto the form: "-cpu host -smp 4 -m 4" etc. @@ -72,7 +72,7 @@ system_args.raw = """\ # INITIALIZE.DISKIMAGE: Options for the "qemu.guest_initialize" script # -# You can uncomment it in your config here, or provide as argument as a workflow step-argument +# You can uncomment it in your config here, or provide as argument as a task step-argument # #initialize.diskimage = "debian-12-aarch64" diff --git a/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py b/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py index 73baa768..dd52b41d 100644 --- a/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py +++ b/src/cijoe/system_imaging/scripts/diskimage_from_cloudimage.py @@ -8,7 +8,7 @@ * ``system_imaging.images`` You can reduce this by providing a case-insensitive fnmatch pattern as input to the -script via workflow, such as these:: +script via task, such as these:: # This will build all images with: diff --git a/src/cijoe/system_imaging/scripts/dockerimage_from_diskimage.py b/src/cijoe/system_imaging/scripts/dockerimage_from_diskimage.py index f5a647a1..14a5fec1 100644 --- a/src/cijoe/system_imaging/scripts/dockerimage_from_diskimage.py +++ b/src/cijoe/system_imaging/scripts/dockerimage_from_diskimage.py @@ -8,7 +8,7 @@ * ``system_imaging.images`` You can reduce this by providing a case-incensitive fnmatch pattern as input to the -script via workflow, such as these:: +script via task, such as these:: # This will build all images with: diff --git a/src/cijoe/system_imaging/workflows/__init__.py b/src/cijoe/system_imaging/tasks/__init__.py similarity index 100% rename from src/cijoe/system_imaging/workflows/__init__.py rename to src/cijoe/system_imaging/tasks/__init__.py diff --git a/src/cijoe/system_imaging/workflows/example_workflow_debian.yaml b/src/cijoe/system_imaging/tasks/example_task_debian.yaml similarity index 94% rename from src/cijoe/system_imaging/workflows/example_workflow_debian.yaml rename to src/cijoe/system_imaging/tasks/example_task_debian.yaml index 3c7621cb..fbb0c2a8 100644 --- a/src/cijoe/system_imaging/workflows/example_workflow_debian.yaml +++ b/src/cijoe/system_imaging/tasks/example_task_debian.yaml @@ -3,7 +3,7 @@ doc: | Build disk and docker images ============================ - Here is what the workflow consumes and produces: + Here is what the task consumes and produces: * cloud image ==> disk image (.qcow2) ==> docker image diff --git a/tests/core/test_argparse.py b/tests/core/test_argparse.py index e360f31b..339c4766 100644 --- a/tests/core/test_argparse.py +++ b/tests/core/test_argparse.py @@ -4,7 +4,7 @@ from argparse import Namespace from pathlib import Path -from cijoe.cli.cli import DEFAULT_CONFIG_FILENAME, DEFAULT_WORKFLOW_FILENAME, parse_args +from cijoe.cli.cli import DEFAULT_CONFIG_FILENAME, DEFAULT_TASK_FILENAME, parse_args TEMPLATE_SCRIPT = """def main(args, cijoe): return 0 @@ -60,25 +60,25 @@ def test_run_group(): assert args.version -def test_target_workflow(): - workflow = "workflow.yaml" +def test_target_task(): + task = "task.yaml" - test_args = ["cijoe", workflow] + test_args = ["cijoe", task] sys.argv = test_args err, args = parse_args() assert not err - assert args.workflow.name == workflow + assert args.task.name == task -def test_target_workflow_steps(): - workflow = "workflow.yaml" +def test_target_task_steps(): + task = "task.yaml" steps = ["step1", "step2", "step3"] - test_args = ["cijoe", workflow, *steps] + test_args = ["cijoe", task, *steps] sys.argv = test_args err, args = parse_args() assert not err - assert args.workflow.name == workflow + assert args.task.name == task assert len(args.step) == len(steps) assert all(a == b for a, b in zip(args.step, steps)) @@ -170,7 +170,7 @@ def test_workflow_argument(): sys.argv = test_args err, args = parse_args() assert not err - assert args.workflow.name == workflow + assert args.task.name == workflow def test_workflow_argument_steps(): @@ -185,21 +185,21 @@ def test_workflow_argument_steps(): sys.argv = test_args err, args = parse_args() assert not err - assert args.workflow.name == workflow + assert args.task.name == workflow assert len(args.step) == len(steps) assert all(a == b for a, b in zip(args.step, steps)) def test_mixed_order(): config = "config.toml" - workflow = "workflow.yaml" + task = "task.yaml" steps = ["step1", "step2", "step3"] - test_args = ["cijoe", "-c", config, "--monitor", workflow, *steps, "-l"] + test_args = ["cijoe", "-c", config, "--monitor", task, *steps, "-l"] sys.argv = test_args err, args = parse_args() assert not err - assert args.workflow.name == workflow + assert args.task.name == task assert args.config[0].name == config assert len(args.step) == len(steps) assert all(a == b for a, b in zip(args.step, steps)) @@ -210,6 +210,4 @@ def test_defaults(): sys.argv = test_args err, args = parse_args() assert not err - assert args.workflow.name == os.environ.get( - "CIJOE_DEFAULT_WORKFLOW", DEFAULT_WORKFLOW_FILENAME - ) + assert args.task.name == os.environ.get("CIJOE_DEFAULT_TASK", DEFAULT_TASK_FILENAME) diff --git a/tests/core/test_workflow.py b/tests/core/test_task.py similarity index 73% rename from tests/core/test_workflow.py rename to tests/core/test_task.py index 83c4c322..2f0fdebd 100644 --- a/tests/core/test_workflow.py +++ b/tests/core/test_task.py @@ -9,7 +9,7 @@ from cijoe.core.processing import runlog_from_path from cijoe.core.resources import get_resources -WORKFLOW_SKELETON = { +TASK_SKELETON = { "doc": "Some description", "steps": [ {"name": "foo", "uses": "core.example_script_default"}, @@ -17,7 +17,7 @@ } -def test_workflow_load(): +def test_task_load(): resources = get_resources() config = resources["configs"]["core.example_config_default"] @@ -26,27 +26,27 @@ def test_workflow_load(): errors = config.load() assert not errors - workflow = resources["workflows"]["core.example_workflow_default"] - assert workflow + task = resources["tasks"]["core.example_task_default"] + assert task - errors = workflow.load(Namespace(), config, []) + errors = task.load(Namespace(), config, []) assert not errors -def test_workflow_lint_valid_workflow(tmp_path): +def test_task_lint_valid_task(tmp_path): config_path = (tmp_path / "test-config-empty.toml").resolve() config_path.write_text("") - data = copy.deepcopy(WORKFLOW_SKELETON) + data = copy.deepcopy(TASK_SKELETON) - workflow_file = (tmp_path / "workflow.yaml").resolve() - workflow_file.write_text(yaml.dump(data)) + task_file = (tmp_path / "task.yaml").resolve() + task_file.write_text(yaml.dump(data)) result = subprocess.run( [ "cijoe", - str(workflow_file), + str(task_file), "--integrity-check", "--config", str(config_path), @@ -56,23 +56,23 @@ def test_workflow_lint_valid_workflow(tmp_path): assert result.returncode == 0 -def test_workflow_lint_invalid_step_name(tmp_path): +def test_task_lint_invalid_step_name(tmp_path): config_path = (tmp_path / "test-config-empty.toml").resolve() config_path.write_text("") - data = copy.deepcopy(WORKFLOW_SKELETON) + data = copy.deepcopy(TASK_SKELETON) data.get("steps", []).append( {"name": "cannot have spaces", "with": "core.example_script_default"} ) - workflow_file = (tmp_path / "workflow.yaml").resolve() - workflow_file.write_text(yaml.dump(data)) + task_file = (tmp_path / "task.yaml").resolve() + task_file.write_text(yaml.dump(data)) result = subprocess.run( [ "cijoe", - str(workflow_file), + str(task_file), "--integrity-check", "--config", str(config_path), @@ -82,12 +82,12 @@ def test_workflow_lint_invalid_step_name(tmp_path): assert result.returncode != 0 -def test_workflow_report_command_ordering(tmp_path): +def test_task_report_command_ordering(tmp_path): config_path = (tmp_path / "test-config-empty.toml").resolve() config_path.write_text("") - data = copy.deepcopy(WORKFLOW_SKELETON) + data = copy.deepcopy(TASK_SKELETON) data["steps"].append( { "name": "many_commands", @@ -97,13 +97,13 @@ def test_workflow_report_command_ordering(tmp_path): ) output_path = (tmp_path / "output").resolve() - workflow_file = (tmp_path / "workflow.yaml").resolve() - workflow_file.write_text(yaml.dump(data)) + task_file = (tmp_path / "task.yaml").resolve() + task_file.write_text(yaml.dump(data)) result = subprocess.run( [ "cijoe", - str(workflow_file), + str(task_file), "--output", str(output_path), "--config", @@ -120,11 +120,11 @@ def test_workflow_report_command_ordering(tmp_path): assert count == val -def test_workflow_run(tmp_path): +def test_task_run(tmp_path): config_path = (tmp_path / "test-config-empty.toml").resolve() config_path.write_text("") - data = copy.deepcopy(WORKFLOW_SKELETON) + data = copy.deepcopy(TASK_SKELETON) data["steps"].append( { "name": "cmdrunner", @@ -136,13 +136,13 @@ def test_workflow_run(tmp_path): ) output_path = (tmp_path / "output").resolve() - workflow_file = (tmp_path / "workflow.yaml").resolve() - workflow_file.write_text(yaml.dump(data)) + task_file = (tmp_path / "task.yaml").resolve() + task_file.write_text(yaml.dump(data)) result = subprocess.run( [ "cijoe", - str(workflow_file), + str(task_file), "--output", str(output_path), "--config", @@ -163,11 +163,11 @@ def test_workflow_run(tmp_path): assert "world" in v["output"] -def test_workflow_run_multiline(tmp_path): +def test_task_run_multiline(tmp_path): config_path = (tmp_path / "test-config-empty.toml").resolve() config_path.write_text("") - data = copy.deepcopy(WORKFLOW_SKELETON) + data = copy.deepcopy(TASK_SKELETON) data["steps"].append( { "name": "cmdrunner", @@ -179,13 +179,13 @@ def test_workflow_run_multiline(tmp_path): ) output_path = (tmp_path / "output").resolve() - workflow_file = (tmp_path / "workflow.yaml").resolve() - workflow_file.write_text(yaml.dump(data)) + task_file = (tmp_path / "task.yaml").resolve() + task_file.write_text(yaml.dump(data)) result = subprocess.run( [ "cijoe", - str(workflow_file), + str(task_file), "--output", str(output_path), "--config",