V0.6.15 #128
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| env: | |
| ACT: "" | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| jobs: | |
| # ---------- fast checks: lint, type, unit ---------- | |
| checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| if: ${{ env.ACT != 'true' }} | |
| uses: actions/checkout@v4 | |
| # uv + Python 3.12 | |
| - name: Setup uv (and Python) | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Diagnose Working Directory | |
| run: | | |
| echo "PWD=$(pwd)" | |
| ls -la | |
| test -f pyproject.toml || (echo "pyproject.toml fehlt!" && exit 1) | |
| # Install Dev extras (ruff, mypy, pytest, …) | |
| - name: Sync deps (dev) | |
| run: uv sync --extra dev --frozen | |
| - name: Show tool versions | |
| run: | | |
| uv run python -V | |
| uv run ruff --version | |
| uv run mypy --version | |
| uv run pytest --version | |
| - name: "Debug: commit & tree" | |
| run: | | |
| echo "PWD=$(pwd)" | |
| git rev-parse --short HEAD | |
| git log -1 --stat | |
| - name: Ruff (lint) | |
| run: uv run ruff check src tests --no-cache --output-format=github | |
| - name: Mypy (type check) | |
| run: uv run mypy src --config-file=pyproject.toml | |
| - name: Unit tests (fast) | |
| env: | |
| PYTHONWARNINGS: default | |
| run: uv run pytest -q tests -m "unit and not (postgres or databricks_spark or bigquery or snowflake)" --maxfail=1 | |
| # ---------- Engine-specific unit slices (require optional extras) ---------- | |
| unit-matrix: | |
| runs-on: ubuntu-latest | |
| needs: checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: postgres | |
| extra: postgres | |
| marker: "unit and postgres" | |
| - name: databricks_spark | |
| extra: spark | |
| marker: "unit and databricks_spark" | |
| java: true | |
| - name: bigquery | |
| extra: bigquery_bf | |
| marker: "unit and bigquery" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup uv (and Python) | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Sync deps (dev + extra) | |
| run: uv sync --extra dev --extra ${{ matrix.extra }} --frozen | |
| - name: Setup Java for Spark | |
| if: matrix.java == true | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| - name: Run unit tests for engine | |
| env: | |
| PYTHONWARNINGS: default | |
| run: uv run pytest -q tests -m "${{ matrix.marker }}" --maxfail=1 | |
| # ---------- Examples: Integration Tests ---------- | |
| examples-matrix: | |
| runs-on: ubuntu-latest | |
| needs: checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # ---------- DuckDB examples ---------- | |
| - engine: duckdb | |
| extra: "" | |
| example: api_demo | |
| env_file: examples/api_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: basic_demo | |
| env_file: examples/basic_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: cache_demo | |
| env_file: examples/cache_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: ci_demo | |
| env_file: examples/ci_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: dq_demo | |
| env_file: examples/dq_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: hooks_demo | |
| env_file: examples/hooks_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: incremental_demo | |
| env_file: examples/incremental_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: macros_demo | |
| env_file: examples/macros_demo/.env.dev_duckdb | |
| - engine: duckdb | |
| extra: "" | |
| example: materializations_demo | |
| env_file: examples/materializations_demo/.env.dev_duckdb | |
| # ---------- Postgres examples ---------- | |
| - engine: postgres | |
| extra: "postgres" | |
| example: api_demo | |
| env_file: examples/api_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: basic_demo | |
| env_file: examples/basic_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: ci_demo | |
| env_file: examples/ci_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: cache_demo | |
| env_file: examples/cache_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: dq_demo | |
| env_file: examples/dq_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: hooks_demo | |
| env_file: examples/hooks_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: incremental_demo | |
| env_file: examples/incremental_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: macros_demo | |
| env_file: examples/macros_demo/.env.dev_postgres | |
| - engine: postgres | |
| extra: "postgres" | |
| example: materializations_demo | |
| env_file: examples/materializations_demo/.env.dev_postgres | |
| # ---------- Spark examples ---------- | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: api_demo | |
| java: true | |
| env_file: examples/api_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: basic_demo | |
| java: true | |
| env_file: examples/basic_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: cache_demo | |
| java: true | |
| env_file: examples/cache_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: ci_demo | |
| java: true | |
| env_file: examples/ci_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: dq_demo | |
| java: true | |
| env_file: examples/dq_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: hooks_demo | |
| java: true | |
| env_file: examples/hooks_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: incremental_demo | |
| java: true | |
| env_file: examples/incremental_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: macros_demo | |
| java: true | |
| env_file: examples/macros_demo/.env.dev_databricks | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: materializations_demo | |
| java: true | |
| env_file: examples/materializations_demo/.env.dev_databricks | |
| services: | |
| postgres: | |
| image: postgres:16 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: fastflowtransform | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup uv (and Python) | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Sync deps (dev + extra) | |
| run: | | |
| extras="--extra dev" | |
| if [ -n "${{ matrix.extra }}" ]; then | |
| extras="$extras --extra ${{ matrix.extra }}" | |
| fi | |
| uv sync $extras --frozen | |
| - name: Setup Java for Spark | |
| if: matrix.engine == 'databricks_spark' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| - name: Run example/integration tests for engine | |
| run: | | |
| echo "Running integration tests for engine=${{ matrix.engine }} example=${{ matrix.example }}" | |
| set -a | |
| unset FF_PG_DSN FF_PG_SCHEMA | |
| if [ -f "${{ matrix.env_file }}" ]; then | |
| source "${{ matrix.env_file }}" | |
| fi | |
| if [ "${{ matrix.engine }}" = "postgres" ]; then | |
| export FF_PG_SCHEMA="${{ matrix.example }}" | |
| export FF_PG_DSN="${FF_PG_DSN:-postgresql+psycopg://postgres:postgres@localhost:5432/fastflowtransform}" | |
| fi | |
| set +a | |
| uv run pytest -m "integration and ${{ matrix.engine }}" \ | |
| -vv --show-capture=all --log-cli-level=INFO \ | |
| -k "${{ matrix.example }} and ${{ matrix.engine }}" \ | |
| --maxfail=1 -q tests/integration/examples/test_examples_matrix.py |