DFM-4544: Migrate to SDLC standards (UV + sdlc@3 + codeartifact@3)

.circleci/config.yml

@@ -1,179 +1,146 @@
 version: 2.1
+
 orbs:
-  codeartifact: visualfabriq/codeartifact@1.3.21
-  coverage-reporter: codacy/coverage-reporter@14.1.0
+  sdlc: visualfabriq/sdlc@3
+  codeartifact: visualfabriq/codeartifact@3
 
-jobs:
-  test:
-    executor: codeartifact/default
-    parameters:
-      with-coverage:
-        description: Coverage reporting flag
-        type: boolean
-        default: false
-      runtime:
-        description: Python runtime version
-        type: string
-        default: "3.11"
-      resource-class:
-        description: Resource class (medium for x86, arm.medium for ARM64)
-        type: string
-        default: "medium"
-    resource_class: <<parameters.resource-class>>
-    docker:
-      - image: cimg/python:<<parameters.runtime>> # Ubuntu 22.04 with glibc 2.35
+python_executor: &python_executor
+  name: sdlc/python
+  python-version: "3.11"
 
-    working_directory: ~/parqueryd
-    steps:
-      - checkout
+python_executor_small: &python_executor_small
+  name: sdlc/python
+  python-version: "3.11"
+  resource-class: small
 
+commands:
+  export-uv-auth:
+    description: "Export authenticated package index for uv (bridges codeartifact/init pip auth to uv)"
+    steps:
       - run:
-          name: Install uv
+          name: Export authenticated URL for uv
           command: |
-            curl -LsSf https://astral.sh/uv/install.sh | sh
-            echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> $BASH_ENV
-            source $BASH_ENV
+            CA_URL="${PIP_INDEX_URL:-$(pip config get global.index-url 2>/dev/null)}"
 
-      - restore_cache:
-          keys:
-            - << parameters.resource-class >>-<< parameters.runtime >>-v2-uv-{{ checksum "setup.py" }}
-            - << parameters.resource-class >>-<< parameters.runtime >>-v2-uv
+            if [ -z "$CA_URL" ]; then
+              echo "No authenticated package index found"
+              exit 1
+            fi
 
-      - run:
-          name: Create virtualenv with uv
-          command: uv venv --clear ~/venv
+            echo "export UV_INDEX_URL=$CA_URL" >> $BASH_ENV
+            echo "export UV_INDEX_PRIVATE_URL=$CA_URL" >> $BASH_ENV
 
+jobs:
+  test:
+    executor:
+      <<: *python_executor
+    steps:
+      - checkout
       - codeartifact/init:
           tool: pip
-
+      - export-uv-auth
       - run:
-          name: Configure env var pip index url
-          command: echo "export PIP_INDEX_URL=$(pip config get global.index-url)" >> $BASH_ENV
-
-      - run:
-          name: Install dependencies with uv
-          command: |
-            source ~/venv/bin/activate
-            uv pip install --index-url $PIP_INDEX_URL .[test]
-
-      # Save the cached dependencies
-      - save_cache:
-          key: << parameters.resource-class >>-<< parameters.runtime >>-v2-uv-{{ checksum "setup.py" }}
-          paths:
-            - ~/venv
-            - ~/.cache/uv
-            - ~/.cargo/bin/uv
-
-      # Save git revision info
-      - run:
-          name: save GIT revision info
+          name: Set up test environment
           command: |
-            git rev-parse HEAD > git_hash.txt
-
-      - run:
-          name: set variables
-          command: |
-            # Enable debug mode
-            export LOG_LEVEL=debug
-            # Set test data directory to a writable location
-            echo "export PARQUERYD_DATA_DIR=/tmp/parquet/" >> $BASH_ENV
             mkdir -p /tmp/parquet/
-
-      - when:
-          condition:
-            not: << parameters.with-coverage >>
-          steps:
-            - run:
-                name: Run tests
-                command: ~/venv/bin/python -m pytest tests --ignore=tests/test_move_parquet.py --ignore=tests/test_simple_rpc.py
-      - when:
-          condition: << parameters.with-coverage >>
-          steps:
-            - run:
-                name: Run tests with coverage reporting
-                command: |
-                  ~/venv/bin/python -m coverage run -m \
-                    pytest tests --ignore=tests/test_move_parquet.py --ignore=tests/test_simple_rpc.py --junitxml=test-results/pytest/pytest-report.xml
-                  ~/venv/bin/python -m coverage xml -o cobertura.xml
-            - store_test_results:
-                path: test-results
-            - store_artifacts:
-                path: test-results
-            - coverage-reporter/send_report
-
-  build-and-push:
-    executor: codeartifact/default
-    parameters:
-      is-pre-release:
-        description: if true the version tag will contain the branch
-        type: boolean
-        default: false
+            echo "export PARQUERYD_DATA_DIR=/tmp/parquet/" >> $BASH_ENV
+            echo "export LOG_LEVEL=debug" >> $BASH_ENV
+      - sdlc/run-tests-with-coverage:
+          uv-sync-group: test
+          parallel: false
+          diff-coverage-threshold: 80
+          extra-pytest-args: "--ignore=tests/test_move_parquet.py --ignore=tests/test_simple_rpc.py"
+
+  test-arm64:
+    executor:
+      name: sdlc/python
+      python-version: "3.11"
+      resource-class: arm.medium
     steps:
       - checkout
-
+      - codeartifact/init:
+          tool: pip
+      - export-uv-auth
       - run:
-          name: Install uv
+          name: Set up test environment
           command: |
-            curl -LsSf https://astral.sh/uv/install.sh | sh
-            echo 'export PATH="$HOME/.cargo/bin:$PATH"' >> $BASH_ENV
-            source $BASH_ENV
-
+            mkdir -p /tmp/parquet/
+            echo "export PARQUERYD_DATA_DIR=/tmp/parquet/" >> $BASH_ENV
+            echo "export LOG_LEVEL=debug" >> $BASH_ENV
+      - sdlc/run-tests-with-coverage:
+          uv-sync-group: test
+          parallel: false
+          diff-coverage-threshold: 80
+          extra-pytest-args: "--ignore=tests/test_move_parquet.py --ignore=tests/test_simple_rpc.py"
+
+  publish-dev:
+    executor:
+      <<: *python_executor_small
+    steps:
+      - checkout
       - codeartifact/init:
           tool: pip
-
+      - export-uv-auth
+      - codeartifact/pre_release_version:
+          package: parqueryd
+          output-var-name: RELEASE_VERSION
+          suffix: .dev
+      - codeartifact/stamp-version:
+          version-file: pyproject.toml
       - run:
-          name: Configure env var pip index url
-          command: echo "export PIP_INDEX_URL=$(pip config get global.index-url)" >> $BASH_ENV
+          name: Build
+          command: uv build
+      - codeartifact/push:
+          tool: twine
 
-      - when:
-          condition: << parameters.is-pre-release >>
-          steps:
-            - codeartifact/pre_release_version:
-                package: parqueryd
+  publish-package:
+    executor:
+      <<: *python_executor_small
+    steps:
+      - checkout
+      - codeartifact/init:
+          tool: pip
+      - export-uv-auth
+      - codeartifact/timestamp_version:
+          version-file: pyproject.toml
+          output-var-name: RELEASE_VERSION
+      - codeartifact/stamp-version:
+          version-file: pyproject.toml
       - run:
           name: Build
-          command: |
-            uv pip install --system --index-url $PIP_INDEX_URL build
-            python -m build
+          command: uv build
       - codeartifact/push:
           tool: twine
+      - codeartifact/create-git-release:
+          version: $RELEASE_VERSION
+          release-assets: "dist/*.tar.gz dist/*.whl"
 
 workflows:
-  version: 2.1
-  build-and-test:
+  parqueryd:
     jobs:
       - test:
-          context: codeartifact-dev
-          name: py311-x86
-          runtime: "3.11"
-          resource-class: "medium"
-          with-coverage: true
-
-      - test:
-          context: codeartifact-dev
-          name: py311-arm64
-          runtime: "3.11"
-          resource-class: "arm.medium"
-          with-coverage: false
-
-      - build-and-push:
-          context: codeartifact-dev
-          requires:
-            - py311-x86
-            - py311-arm64
+          context: [sdlc, codeartifact-dev]
+      - test-arm64:
+          context: [sdlc, codeartifact-dev]
+
+      # Feature branches: opt-in dev publish
+      - approve-publish-dev:
+          type: approval
+          requires: [test, test-arm64]
+          filters:
+            branches:
+              ignore: main
+      - publish-dev:
+          context: [sdlc, codeartifact-dev]
+          requires: [approve-publish-dev]
           filters:
             branches:
-              only:
-                - master
-                - main
+              ignore: main
 
-      - build-and-push:
-          context: codeartifact-dev
-          requires:
-            - py311-x86
-            - py311-arm64
-          is-pre-release: true
+      # Main: every merge is a full release
+      - publish-package:
+          context: [sdlc, codeartifact-dev]
+          requires: [test, test-arm64]
           filters:
             branches:
-              only:
-                - uat
+              only: main

.coveragerc

@@ -0,0 +1,12 @@
+[run]
+relative_files = True
+branch = True
+source =
+    parqueryd
+
+[report]
+show_missing = True
+exclude_lines =
+    pragma: no cover
+    if __name__ == .__main__
+    ^\s*pass\s*$

.pre-commit-config.yaml

@@ -0,0 +1,51 @@
+default_install_hook_types: [pre-commit, pre-push]
+fail_fast: true
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-toml
+      - id: check-json
+      - id: check-added-large-files
+        args: ['--maxkb=500']
+      - id: check-merge-conflict
+      - id: debug-statements
+      - id: detect-private-key
+      - id: no-commit-to-branch
+        args: ['--branch', 'main', '--branch', 'master']
+      - id: requirements-txt-fixer
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.10
+    hooks:
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
+  - repo: https://github.com/PyCQA/bandit
+    rev: '1.9.4'
+    hooks:
+      - id: bandit
+        args: ['-c', 'pyproject.toml']
+        additional_dependencies: ['bandit[toml]']
+
+  - repo: local
+    hooks:
+      - id: validate-branch-name
+        name: validate branch name (JIRA ID)
+        entry: scripts/validate-branch-name.sh
+        language: script
+        always_run: true
+        pass_filenames: false
+        stages: [pre-push]
+
+      - id: uv-lock
+        name: update lockfile (internal packages)
+        entry: bash -c 'uv lock && git diff --quiet uv.lock || { echo ""; echo "uv.lock updated — run: git add uv.lock"; exit 1; }'
+        language: system
+        files: ^pyproject\.toml$
+        pass_filenames: false

Makefile

@@ -0,0 +1,29 @@
+.DEFAULT_GOAL := help
+
+.PHONY: help lock setup-project test test-coverage build
+
+help:  ## Show this help message
+	@echo "parqueryd — A distribution framework for parquery"
+	@echo ""
+	@echo "Available targets:"
+	@awk 'BEGIN {FS = ":.*?## "} /^[a-zA-Z_-]+:.*?## / {printf "  \033[36m%-16s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)
+
+lock:  ## Update uv.lock against CodeArtifact (requires aws sso login)
+	@echo "Updating uv.lock..."
+	$(eval CODEARTIFACT_TOKEN := $(shell aws codeartifact get-authorization-token --domain visualfabriq --query authorizationToken --output text))
+	UV_EXTRA_INDEX_URL="https://aws:$(CODEARTIFACT_TOKEN)@visualfabriq-103613169272.d.codeartifact.eu-west-1.amazonaws.com/pypi/private/simple/" uv lock
+	@echo "Lockfile updated. Commit uv.lock."
+
+setup-project:  ## Sync dependencies (test group) using the committed lockfile
+	uv sync --group test --frozen
+
+test:  ## Run the pytest test suite
+	uv run pytest tests --ignore=tests/test_move_parquet.py --ignore=tests/test_simple_rpc.py
+
+test-coverage:  ## Run tests with coverage reports (XML + HTML)
+	uv run coverage run -m pytest tests --ignore=tests/test_move_parquet.py --ignore=tests/test_simple_rpc.py --junitxml=test-results/junit.xml
+	uv run coverage xml -o coverage.xml
+	uv run coverage html -d coverage-html
+
+build:  ## Build wheel and sdist with uv
+	uv build

parqueryd/__init__.py

@@ -1,10 +1,9 @@
 from __future__ import annotations
 
 import logging
-import os
+from importlib.metadata import version as _pkg_version
 
-pre_release_version: str = os.getenv("PRE_RELEASE_VERSION", "")
-__version__: str = pre_release_version if pre_release_version else "2.0.1"
+__version__: str = _pkg_version("parqueryd")
 
 # initalize logger
 logger: logging.Logger = logging.getLogger("parqueryd")