diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 37a7c4983..98a6415cd 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -1,8 +1,6 @@ name: Checks - on: workflow_call: - jobs: Version-Check: name: Version @@ -10,44 +8,38 @@ jobs: permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Check Version(s) - run: poetry run -- nox -s version:check - + - name: SCM Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Check Version(s) + run: poetry run -- nox -s version:check Documentation: name: Docs - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Build Documentation - run: | - poetry run -- nox -s docs:build - - - name: Link Check - run: | - poetry run -- nox -s links:check - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Build Documentation + run: | + poetry run -- nox -s docs:build + - name: Link Check + run: | + poetry run -- nox -s links:check Changelog: name: Changelog Update Check runs-on: "ubuntu-24.04" @@ -55,143 +47,143 @@ jobs: contents: read if: ${{ github.ref != 'refs/heads/main' && github.ref != 'refs/heads/master' }} steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run changelog update check - run: poetry run -- nox -s changelog:updated - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Run changelog update check + run: poetry run -- nox -s changelog:updated Lint: name: Linting (Python-${{ matrix.python-versions }}) - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run lint - run: poetry run -- nox -s lint:code - - - name: Upload Artifacts - uses: actions/upload-artifact@v6 - with: - name: lint-python${{ matrix.python-versions }} - path: | - .lint.txt - .lint.json - include-hidden-files: true - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run lint + run: poetry run -- nox -s lint:code + - name: Upload Artifacts + uses: actions/upload-artifact@v6 + with: + name: lint-python${{ matrix.python-versions }} + path: | + .lint.txt + .lint.json + include-hidden-files: true Type-Check: name: Type Checking (Python-${{ matrix.python-versions }}) - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run type-check - run: poetry run -- nox -s lint:typing - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run type-check + run: poetry run -- nox -s lint:typing Security: name: Security Checks (Python-${{ matrix.python-versions }}) - needs: [ Version-Check ] + needs: + - Version-Check runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run security linter - run: poetry run -- nox -s lint:security - - - name: Upload Artifacts - uses: actions/upload-artifact@v6 - with: - name: security-python${{ matrix.python-versions }} - path: .security.json - include-hidden-files: true - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run security linter + run: poetry run -- nox -s lint:security + - name: Upload Artifacts + uses: actions/upload-artifact@v6 + with: + name: security-python${{ matrix.python-versions }} + path: .security.json + include-hidden-files: true Format: name: Format Check runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run format check - run: poetry run -- nox -s format:check - - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Run format check + run: poetry run -- nox -s format:check Build-Packages: name: Build Package Check - needs: [ Documentation, Lint, Type-Check, Security, Format ] + needs: + - Documentation + - Lint + - Type-Check + - Security + - Format runs-on: "ubuntu-24.04" permissions: contents: read steps: - - name: SCM Checkout - uses: actions/checkout@v6 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: "3.10" - poetry-version: "2.3.0" - - - name: Run Distribution Check - run: poetry run -- nox -s package:check - + - name: SCM Checkout + uses: actions/checkout@v6 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: "3.10" + poetry-version: "2.3.0" + - name: Run Distribution Check + run: poetry run -- nox -s package:check Lint-Imports: name: Lint Imports runs-on: ubuntu-24.04 @@ -200,46 +192,46 @@ jobs: steps: - name: SCM Checkout uses: actions/checkout@v6 - - name: Setup Python & Poetry Environment uses: ./.github/actions/python-environment with: python-version: "3.10" poetry-version: "2.3.0" - - name: Run import linter run: poetry run -- nox -s lint:import - Tests: name: Unit-Tests (Python-${{ matrix.python-versions }}) - needs: [ Build-Packages, Lint-Imports ] + needs: + - Build-Packages + - Lint-Imports runs-on: "ubuntu-24.04" permissions: contents: read strategy: fail-fast: false matrix: - python-versions: ["3.10", "3.11", "3.12", "3.13", "3.14"] - + python-versions: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" steps: - - name: SCM Checkout - uses: actions/checkout@v6 - # The PTB has unit tests which require the fetch-depth to be 0. - with: - fetch-depth: 0 - - - name: Setup Python & Poetry Environment - uses: ./.github/actions/python-environment - with: - python-version: ${{ matrix.python-versions }} - poetry-version: "2.3.0" - - - name: Run Tests and Collect Coverage - run: poetry run -- nox -s test:unit -- --coverage - - - name: Upload Artifacts - uses: actions/upload-artifact@v6 - with: - name: coverage-python${{ matrix.python-versions }}-fast - path: .coverage - include-hidden-files: true + - name: SCM Checkout + uses: actions/checkout@v6 + # The PTB has unit tests which require the fetch-depth to be 0. + with: + fetch-depth: 0 + - name: Setup Python & Poetry Environment + uses: ./.github/actions/python-environment + with: + python-version: ${{ matrix.python-versions }} + poetry-version: "2.3.0" + - name: Run Tests and Collect Coverage + run: poetry run -- nox -s test:unit -- --coverage + - name: Upload Artifacts + uses: actions/upload-artifact@v6 + with: + name: coverage-python${{ matrix.python-versions }}-fast + path: .coverage + include-hidden-files: true diff --git a/exasol/toolbox/tools/template.py b/exasol/toolbox/tools/template.py index 14829ea1f..73fa4b857 100644 --- a/exasol/toolbox/tools/template.py +++ b/exasol/toolbox/tools/template.py @@ -1,5 +1,6 @@ import difflib import io +import re from collections.abc import Mapping from contextlib import ExitStack from inspect import cleandoc @@ -15,6 +16,7 @@ from rich.columns import Columns from rich.console import Console from rich.syntax import Syntax +from yaml.resolver import Resolver from noxconfig import PROJECT_CONFIG @@ -70,6 +72,51 @@ def show_templates( ) # type: ignore +# yaml uses a shorthand to identify "on" and "off" tags. +# for GitHub workflows, we do NOT want "on" replaced with "True". +for character in ["O", "o"]: + Resolver.yaml_implicit_resolvers[character] = [ + x + for x in Resolver.yaml_implicit_resolvers[character] + if x[0] != "tag:yaml.org,2002:bool" + ] + + +class GitHubDumper(yaml.SafeDumper): + pass + + +def empty_representer(dumper, data): + """ + Leave empty fields without 'null' + + on: + workflow_call: + """ + return dumper.represent_scalar("tag:yaml.org,2002:null", "") + + +# Regex for common strings that lose quotes: +# 1. Version numbers (e.g., 2.3.0, 3.10) +# 2. OS/image names (e.g., ubuntu-24.04) +# 3. Numeric strings that look like octals or floats (e.g., 045, 1.2) +QUOTE_REGEX = re.compile(r"^(\d+\.\d+(\.\d+)?|[a-zA-Z]+-\d+\.\d+|0\d+)$") + + +def str_presenter(dumper, data): + # Use literal style '|' for strings with newlines + if "\n" in data: + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style="|") + if QUOTE_REGEX.match(data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + return dumper.represent_scalar("tag:yaml.org,2002:str", data) + + +# Register it to the dumper +GitHubDumper.add_representer(str, str_presenter) +GitHubDumper.add_representer(type(None), empty_representer) + + def _render_template( src: str | Path, stack: ExitStack, @@ -80,10 +127,15 @@ def _render_template( template = jinja_env.from_string(input_file.read()) rendered_string = template.render(PROJECT_CONFIG.github_template_dict) - # validate that the rendered content is a valid YAML. This is not - # written out as by default it does not give GitHub-safe output. - yaml.safe_load(rendered_string) - return cleandoc(rendered_string) + "\n" + # this line also checks that the rendered content is a valid YAML + data = yaml.safe_load(rendered_string) + output = yaml.dump( + data, + Dumper=GitHubDumper, + sort_keys=False, # if True, then re-orders the jobs alphabetically + ) + + return cleandoc(output) + "\n" def diff_template(template: str, dest: Path, pkg: str, template_type: str) -> None: