Added Biqquery support + added bigquery to examples + fixed issues + … #78
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| env: | |
| ACT: "" | |
| on: | |
| push: | |
| branches: [ main ] | |
| pull_request: | |
| jobs: | |
| # ---------- fast checks: lint, type, unit ---------- | |
| checks: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| if: ${{ env.ACT != 'true' }} | |
| uses: actions/checkout@v4 | |
| # uv + Python 3.12 | |
| - name: Setup uv (and Python) | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Diagnose Working Directory | |
| run: | | |
| echo "PWD=$(pwd)" | |
| ls -la | |
| test -f pyproject.toml || (echo "pyproject.toml fehlt!" && exit 1) | |
| # Install Dev extras (ruff, mypy, pytest, …) | |
| - name: Sync deps (dev) | |
| run: uv sync --extra dev --frozen | |
| - name: Show tool versions | |
| run: | | |
| uv run python -V | |
| uv run ruff --version | |
| uv run mypy --version | |
| uv run pytest --version | |
| - name: "Debug: commit & tree" | |
| run: | | |
| echo "PWD=$(pwd)" | |
| git rev-parse --short HEAD | |
| git log -1 --stat | |
| - name: "Debug: ensure pyproject present" | |
| run: | | |
| test -f pyproject.toml || (echo "pyproject.toml fehlt!" && exit 1) | |
| ls -la | |
| - name: "Debug: Ruff version & scope" | |
| run: | | |
| uv run ruff --version | |
| echo "---- Ruff will lint these files (no cache) ----" | |
| uv run ruff check src tests --no-cache --show-files | |
| - name: Ruff (lint) | |
| run: uv run ruff check src tests --no-cache --output-format=github | |
| - name: Mypy (type check) | |
| run: uv run mypy src --config-file=pyproject.toml | |
| - name: Unit tests (fast) | |
| env: | |
| PYTHONWARNINGS: default | |
| run: uv run pytest -q tests -m "unit and not (postgres or databricks_spark or bigquery or snowflake)" --maxfail=1 | |
| # ---------- Engine-specific unit slices (require optional extras) ---------- | |
| unit-matrix: | |
| runs-on: ubuntu-latest | |
| needs: checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: postgres | |
| extra: postgres | |
| marker: "unit and postgres" | |
| - name: databricks_spark | |
| extra: spark | |
| marker: "unit and databricks_spark" | |
| java: true | |
| - name: bigquery | |
| extra: bigquery_bf | |
| marker: "unit and bigquery" | |
| - name: snowflake | |
| extra: snowflake | |
| marker: "unit and snowflake" | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup uv (and Python) | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Sync deps (dev + extra) | |
| run: uv sync --extra dev --extra ${{ matrix.extra }} --frozen | |
| - name: Setup Java for Spark | |
| if: matrix.java == true | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| - name: Run unit tests for engine | |
| env: | |
| PYTHONWARNINGS: default | |
| run: uv run pytest -q tests -m "${{ matrix.marker }}" --maxfail=1 | |
| # ---------- Examples: Integration Tests ---------- | |
| examples-matrix: | |
| runs-on: ubuntu-latest | |
| needs: checks | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| # DuckDB examples | |
| - engine: duckdb | |
| extra: "" | |
| example: api_demo | |
| - engine: duckdb | |
| extra: "" | |
| example: basic_demo | |
| - engine: duckdb | |
| extra: "" | |
| example: cache_demo | |
| - engine: duckdb | |
| extra: "" | |
| example: dq_demo | |
| - engine: duckdb | |
| extra: "" | |
| example: incremental_demo | |
| - engine: duckdb | |
| extra: "" | |
| example: macros_demo | |
| - engine: duckdb | |
| extra: "" | |
| example: materializations_demo | |
| # Postgres examples | |
| - engine: postgres | |
| extra: "postgres" | |
| example: api_demo | |
| - engine: postgres | |
| extra: "postgres" | |
| example: basic_demo | |
| - engine: postgres | |
| extra: "postgres" | |
| example: cache_demo | |
| - engine: postgres | |
| extra: "postgres" | |
| example: dq_demo | |
| - engine: postgres | |
| extra: "postgres" | |
| example: incremental_demo | |
| - engine: postgres | |
| extra: "postgres" | |
| example: macros_demo | |
| - engine: postgres | |
| extra: "postgres" | |
| example: materializations_demo | |
| # Spark examples | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: api_demo | |
| java: true | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: basic_demo | |
| java: true | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: cache_demo | |
| java: true | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: dq_demo | |
| java: true | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: incremental_demo | |
| java: true | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: macros_demo | |
| java: true | |
| - engine: databricks_spark | |
| extra: "spark" | |
| example: materializations_demo | |
| java: true | |
| services: | |
| postgres: | |
| image: postgres:16 | |
| env: | |
| POSTGRES_USER: postgres | |
| POSTGRES_PASSWORD: postgres | |
| POSTGRES_DB: fastflowtransform | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U postgres" | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Setup uv (and Python) | |
| uses: astral-sh/setup-uv@v5 | |
| with: | |
| python-version: "3.12" | |
| enable-cache: true | |
| - name: Sync deps (dev + extra) | |
| run: | | |
| extras="--extra dev" | |
| if [ -n "${{ matrix.extra }}" ]; then | |
| extras="$extras --extra ${{ matrix.extra }}" | |
| fi | |
| uv sync $extras --frozen | |
| - name: Setup Java for Spark | |
| if: matrix.engine == 'databricks_spark' | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| - name: Run example/integration tests for engine | |
| env: | |
| FF_PG_DSN: postgresql+psycopg://postgres:postgres@localhost:5432/fastflowtransform | |
| FF_PG_SCHEMA: ci_examples | |
| run: | | |
| echo "Running integration tests for engine=${{ matrix.engine }} example=${{ matrix.example }}" | |
| uv run pytest -m "integration and ${{ matrix.engine }}" \ | |
| -vv --show-capture=all --log-cli-level=INFO \ | |
| -k "${{ matrix.example }} and ${{ matrix.engine }}" \ | |
| --maxfail=1 -q tests/integration/examples/test_examples_matrix.py |