diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000..9b2d941 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,530 @@ +name: python-tests + +on: + push: + branches: [main, py_wheel*] + release: + types: [published] + +jobs: + newsynth: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-15-intel + - macos-latest + - windows-latest + - ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Restore newsynth cache + id: newsynth-cache + uses: actions/cache@v4 + with: + path: | + externals/bin + externals/newsynth-license + key: newsynth-${{ matrix.os }}-v1 + + - name: Setup Haskell (GHC/Cabal) + if: steps.newsynth-cache.outputs.cache-hit != 'true' + uses: haskell-actions/setup@v2 + with: + ghc-version: "9.6.6" + cabal-version: "3.12.1.0" + + - name: Build newsynth binaries from Hackage + if: steps.newsynth-cache.outputs.cache-hit != 'true' + run: | + cabal update + cabal install newsynth --installdir "$PWD/externals/bin" --install-method=copy --overwrite-policy=always + if [ "$RUNNER_OS" != "Windows" ]; then + chmod +x "$PWD/externals/bin/gridsynth" + fi + ls -la "$PWD/externals/bin" + shell: bash + + - name: Download newsynth LICENSE files + if: steps.newsynth-cache.outputs.cache-hit != 'true' + run: | + rm -rf "$PWD/externals/newsynth-src" + rm -rf "$PWD/externals/newsynth-license" + mkdir -p "$PWD/externals/newsynth-src" + mkdir -p "$PWD/externals/newsynth-license" + cabal get newsynth --destdir "$PWD/externals/newsynth-src" --pristine + newsynth_src_dir=$(find "$PWD/externals/newsynth-src" -maxdepth 1 -type d -name "newsynth-*" | head -n 1) + if [ -n "$newsynth_src_dir" ]; then + find "$newsynth_src_dir" -maxdepth 2 -type f \ + \( -iname "LICENSE*" -o -iname "COPYING*" -o -iname "README*" -o -iname "GPL*" \) \ + -exec cp {} "$PWD/externals/newsynth-license/" \; + fi + echo "---" + ls -la "$PWD/externals/newsynth-src" + echo "---" + ls -la "$PWD/externals/newsynth-license" + shell: bash + + wheel: + needs: [newsynth] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-15-intel + - macos-latest + - windows-latest + - ubuntu-latest + build-target: + - cp310-cp310 + - cp311-cp311 + - cp312-abi3 + include: + - build-target: cp310-cp310 + python-version: "3.10" + - build-target: cp311-cp311 + python-version: "3.11" + - build-target: cp312-abi3 + python-version: "3.12" + continue-on-error: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: pyproject.toml + + - name: Setup MSVC (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Shorten build paths (Windows) + if: runner.os == 'Windows' + run: | + New-Item -ItemType Directory -Force -Path C:\b | Out-Null + New-Item -ItemType Directory -Force -Path C:\t | Out-Null + Add-Content -Path $env:GITHUB_ENV -Value "SKBUILD_BUILD_DIR=C:\b" + Add-Content -Path $env:GITHUB_ENV -Value "TMP=C:\t" + Add-Content -Path $env:GITHUB_ENV -Value "TEMP=C:\t" + shell: pwsh + + - name: Configure vcpkg toolchain + if: runner.os != 'Linux' + run: | + mkdir -p "${{ github.workspace }}/.vcpkg" + echo "VCPKG_ROOT=${{ github.workspace }}/.vcpkg" >> "$GITHUB_ENV" + git clone https://github.com/microsoft/vcpkg.git "${{ github.workspace }}/.vcpkg" + ls "${{ github.workspace }}/.vcpkg" + + # - name: Install libyaml-cpp (Linux) + # if: runner.os == 'Linux' + # run: | + # sudo apt-get update + # sudo apt-get install -y build-essential gcc g++ libyaml-cpp0.8 libyaml-cpp-dev + + # - name: Set compiler env (Linux) + # if: runner.os == 'Linux' + # run: | + # echo "CC=gcc" >> "$GITHUB_ENV" + # echo "CXX=g++" >> "$GITHUB_ENV" + + - name: Install libyaml-cpp (macOS) + if: runner.os == 'macOS' + run: | + brew install yaml-cpp + + - name: Configure Homebrew lib path (macOS) + if: runner.os == 'macOS' + run: | + brew_prefix=$(brew --prefix yaml-cpp) + if [ -n "$brew_prefix" ]; then + echo "DYLD_LIBRARY_PATH=$brew_prefix/lib:$DYLD_LIBRARY_PATH" >> "$GITHUB_ENV" + fi + + - name: Set compiler env (macOS) + if: runner.os == 'macOS' + run: | + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + + - name: Restore newsynth cache + uses: actions/cache@v4 + with: + path: | + externals/bin + externals/newsynth-license + key: newsynth-${{ matrix.os }}-v1 + fail-on-cache-miss: true + + - name: Fix newsynth permissions (Unix) + if: runner.os != 'Windows' + run: chmod +x "$PWD/externals/bin/gridsynth" + shell: bash + + - name: Build wheel with cibuildwheel (Linux) + if: runner.os == 'Linux' + uses: pypa/cibuildwheel@v2.23.0 + env: + CIBW_BUILD: ${{ matrix.build-target == 'cp310-cp310' && 'cp310-manylinux_x86_64' || matrix.build-target == 'cp311-cp311' && 'cp311-manylinux_x86_64' || 'cp312-manylinux_x86_64' }} + CIBW_SKIP: "*-musllinux_*" + CIBW_MANYLINUX_X86_64_IMAGE: manylinux_2_28 + CIBW_CONFIG_SETTINGS: ${{ matrix.build-target == 'cp312-abi3' && 'wheel.py-api=cp312' || 'wheel.py-api=' }} + CIBW_BEFORE_ALL_LINUX: | + dnf install -y git gcc gcc-c++ make zip unzip tar curl perl + git clone https://github.com/microsoft/vcpkg.git /project/.vcpkg + /project/.vcpkg/bootstrap-vcpkg.sh -disableMetrics + CIBW_ENVIRONMENT_LINUX: >- + VCPKG_ROOT=/project/.vcpkg + SKBUILD_CMAKE_DEFINE="CMAKE_TOOLCHAIN_FILE=/project/.vcpkg/scripts/buildsystems/vcpkg.cmake;QRET_BUILD_PYTHON=ON" + with: + package-dir: . + output-dir: dist + + - name: Build wheel (non-Linux) + if: runner.os != 'Linux' + env: + SKBUILD_CMAKE_DEFINE: "CMAKE_TOOLCHAIN_FILE=${{env.VCPKG_ROOT}}/scripts/buildsystems/vcpkg.cmake;QRET_BUILD_PYTHON=ON" + run: | + python -m pip install --upgrade pip build pytest coverage wheel numpy + python -m build --wheel -Cwheel.py-api=${{ matrix.build-target == 'cp312-abi3' && 'cp312' || '' }} + shell: bash + + - name: Embed newsynth binary and LICENSE files into wheel + run: | + wheel_file=$(ls dist/*.whl) + python -m pip install --upgrade wheel + python -m wheel unpack "$wheel_file" -d dist/unpacked + wheel_dir=$(ls -d dist/unpacked/*) + pkg_dir="$wheel_dir/pyqret/externals/newsynth" + mkdir -p "$pkg_dir" + + # Copy built binaries into pyqret/externals/newsynth. + for file in "$PWD"/externals/bin/*; do + cp "$file" "$pkg_dir/" + done + + # Copy downloaded license files. + find "$PWD/externals/newsynth-license" -mindepth 1 -maxdepth 1 -type f -exec cp {} "$pkg_dir/" \; + + # Remove unexpected top-level lib64 directory from the wheel payload. + rm -rf "$wheel_dir/lib64" + + rm -f "$wheel_file" + python -m wheel pack "$wheel_dir" -d dist + ls -la "$pkg_dir" + shell: bash + + - name: Validate wheel filename and extension tags + run: | + wheel_file=$(ls dist/*.whl) + wheel_name=$(basename "$wheel_file") + echo "Built wheel: $wheel_name" + + expected_tag="${{ matrix.build-target }}-" + + if [[ "$wheel_name" != *"$expected_tag"* ]]; then + echo "Unexpected wheel ABI tag for ${{ matrix.build-target }}." + echo "Actual filename: $wheel_name" + echo "Expected filename to contain: $expected_tag" + exit 1 + fi + + check_dir="dist/wheel-check" + rm -rf "$check_dir" + mkdir -p "$check_dir" + python -m zipfile -e "$wheel_file" "$check_dir" + + target_tag="${{ matrix.build-target }}" + py_tag="${target_tag%%-*}" + abi_tag="${target_tag#*-}" + py_major_minor="${py_tag#cp}" + py_major="${py_major_minor:0:1}" + py_minor="${py_major_minor:1}" + cpython_tag="cpython-${py_major}${py_minor}" + found_ext=0 + + while IFS= read -r ext_file; do + found_ext=1 + ext_name=$(basename "$ext_file") + echo "Found extension stub: $ext_name" + if [[ "$ext_name" == *"$abi_tag"* || "$ext_name" == *"$py_tag"* || "$ext_name" == *"$cpython_tag"* ]]; then + continue + fi + + if [[ "${{ runner.os }}" == "Windows" && "$ext_name" == *.pyd ]]; then + echo "Accepting untagged Windows extension stub: $ext_name" + continue + fi + + echo "Extension stub tag mismatch for ${{ matrix.build-target }}: $ext_name" + echo "Expected extension filename to contain one of: $abi_tag, $py_tag, $cpython_tag" + exit 1 + done < <(find "$check_dir" -type f \( -name "*.so" -o -name "*.dylib" -o -name "*.lib" -o -name "*.pyd" \)) + + if [ "$found_ext" -eq 0 ]; then + echo "No extension stub (*.so/*.dylib/*.lib/*.pyd) found in wheel." + exit 1 + fi + shell: bash + + - name: Upload Python artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: pyqret-wheel-${{ matrix.os }}-${{ matrix.build-target }} + path: | + dist/*.whl + + sdist: + needs: [newsynth] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "pip" + cache-dependency-path: pyproject.toml + + - name: Configure vcpkg toolchain + run: | + mkdir -p "${{ github.workspace }}/.vcpkg" + echo "VCPKG_ROOT=${{ github.workspace }}/.vcpkg" >> "$GITHUB_ENV" + git clone https://github.com/microsoft/vcpkg.git "${{ github.workspace }}/.vcpkg" + ls "${{ github.workspace }}/.vcpkg" + + - name: Install libyaml-cpp (Linux) + run: | + sudo apt-get update + sudo apt-get install -y build-essential gcc g++ libyaml-cpp0.8 libyaml-cpp-dev + + - name: Set compiler env (Linux) + run: | + echo "CC=gcc" >> "$GITHUB_ENV" + echo "CXX=g++" >> "$GITHUB_ENV" + + - name: Restore newsynth cache + uses: actions/cache@v4 + with: + path: | + externals/bin + externals/newsynth-license + key: newsynth-ubuntu-latest-v1 + fail-on-cache-miss: true + + - name: Set newsynth permissions + run: | + chmod +x "$PWD/externals/bin/gridsynth" + ls -la "$PWD/externals/bin" + shell: bash + + - name: Build sdist + run: | + python -m pip install --upgrade pip build pytest coverage numpy + python -m build --sdist + shell: bash + + - name: Upload sdist artifact + uses: actions/upload-artifact@v4 + with: + name: pyqret-sdist + path: | + dist/*.tar.gz + + test-wheel: + needs: [wheel] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-15-intel + - macos-latest + - windows-latest + - ubuntu-latest + python-version: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: pyproject.toml + + - name: Download wheel artifact + uses: actions/download-artifact@v4 + with: + name: pyqret-wheel-${{ matrix.os }}-${{ matrix.python-version == '3.10' && 'cp310-cp310' || matrix.python-version == '3.11' && 'cp311-cp311' || 'cp312-abi3' }} + path: dist + + - name: Install wheel + run: | + python -m pip install --upgrade pip pytest coverage numpy + python -m pip install dist/*.whl + shell: bash + + - name: test gridsynth bundle + run: python -c "import pyqret; import os; import subprocess; subprocess.run([os.environ['GRIDSYNTH_PATH'], '--help'], check=True)" + + - name: Run tests + run: python -m pytest quration-core/python/tests + shell: bash + + test-sdist: + needs: [sdist] + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - macos-15-intel + - macos-latest + - windows-latest + - ubuntu-latest + python-version: + - "3.10" + #- "3.11" + #- "3.12" + #- "3.13" + - "3.14" + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + cache-dependency-path: pyproject.toml + + - name: Setup MSVC (Windows) + if: runner.os == 'Windows' + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Shorten build paths (Windows) + if: runner.os == 'Windows' + run: | + New-Item -ItemType Directory -Force -Path C:\b | Out-Null + New-Item -ItemType Directory -Force -Path C:\t | Out-Null + Add-Content -Path $env:GITHUB_ENV -Value "SKBUILD_BUILD_DIR=C:\b" + Add-Content -Path $env:GITHUB_ENV -Value "TMP=C:\t" + Add-Content -Path $env:GITHUB_ENV -Value "TEMP=C:\t" + shell: pwsh + + - name: Configure vcpkg toolchain + run: | + mkdir -p "${{ github.workspace }}/.vcpkg" + echo "VCPKG_ROOT=${{ github.workspace }}/.vcpkg" >> "$GITHUB_ENV" + git clone https://github.com/microsoft/vcpkg.git "${{ github.workspace }}/.vcpkg" + ls "${{ github.workspace }}/.vcpkg" + + - name: Install libyaml-cpp (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y build-essential gcc g++ libyaml-cpp0.8 libyaml-cpp-dev + + - name: Set compiler env (Linux) + if: runner.os == 'Linux' + run: | + echo "CC=gcc" >> "$GITHUB_ENV" + echo "CXX=g++" >> "$GITHUB_ENV" + + - name: Install libyaml-cpp (macOS) + if: runner.os == 'macOS' + run: | + brew install yaml-cpp + + - name: Configure Homebrew lib path (macOS) + if: runner.os == 'macOS' + run: | + brew_prefix=$(brew --prefix yaml-cpp) + if [ -n "$brew_prefix" ]; then + echo "DYLD_LIBRARY_PATH=$brew_prefix/lib:$DYLD_LIBRARY_PATH" >> "$GITHUB_ENV" + fi + + - name: Set compiler env (macOS) + if: runner.os == 'macOS' + run: | + echo "CC=clang" >> "$GITHUB_ENV" + echo "CXX=clang++" >> "$GITHUB_ENV" + + - name: Download sdist artifact + uses: actions/download-artifact@v4 + with: + name: pyqret-sdist + path: dist + + - name: Install sdist + env: + SKBUILD_CMAKE_DEFINE: "CMAKE_TOOLCHAIN_FILE=${{env.VCPKG_ROOT}}/scripts/buildsystems/vcpkg.cmake;QRET_BUILD_PYTHON=ON" + run: | + python -m pip install --upgrade pip pytest coverage numpy + python -m pip install -vvv dist/*.tar.gz + shell: bash + + - name: Run tests + run: python -m pytest quration-core/python/tests + shell: bash + + merge: + needs: [test-wheel, test-sdist] + runs-on: ubuntu-latest + steps: + - name: Merge Artifacts + uses: actions/upload-artifact/merge@v4 + with: + name: pyqret + pattern: pyqret-* + + release: + if: github.event_name == 'release' + needs: [merge] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Download merged artifact + uses: actions/download-artifact@v4 + with: + name: pyqret + path: dist + + - name: Upload artifact to release + uses: softprops/action-gh-release@v2 + with: + files: dist/* diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..72ed58e --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +graft include +graft lib +graft lib64 diff --git a/pyproject.toml b/pyproject.toml index 4162c73..5da1dfd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "pyqret" description = "Python wrapper of QRET" -readme = "quration-core/python/README.md" +readme = "README.md" required-python = ">=3.10" # authors = @@ -69,6 +69,8 @@ wheel.py-api = "cp312" # Include Python package from the non-root location. wheel.packages = ["quration-core/python/pyqret"] +wheel.exclude = ["include/**", "lib/**", "lib64/**"] +sdist.include = ["externals/**", "quration-core/src/**"] [tool.scikit-build.cmake.define] # nanobind requires libraries to be position-independent in order to link correctly diff --git a/quration-core/python/pyqret/__init__.py b/quration-core/python/pyqret/__init__.py index 8e53219..f785f74 100644 --- a/quration-core/python/pyqret/__init__.py +++ b/quration-core/python/pyqret/__init__.py @@ -6,6 +6,14 @@ from typing_extensions import Self +import importlib.util + +import os + +import platform + +from pathlib import Path + from . import _qret_impl as _M if TYPE_CHECKING: @@ -281,3 +289,43 @@ def __next__(self): # noqa:ANN204 ret = self._convert_to() ret._impl = next(self._iter) return ret + + +_ENV_PATH = "GRIDSYNTH_PATH" + + +def _qret_package_root() -> Path | None: + spec = importlib.util.find_spec("pyqret") + if spec is None: + return None + if spec.origin: + return Path(spec.origin).resolve().parent + return None + + +def _set_gridsynth_env_from_package() -> None: + if _ENV_PATH in os.environ: + return + + package_root = _qret_package_root() + if package_root is None: + return + + bin_dir = package_root / "externals" / "newsynth" + if not bin_dir.is_dir(): + return + + candidates = [ + bin_dir / "gridsynth", + bin_dir / "gridsynth.exe", + ] + if platform.system().lower() == "windows": + candidates.reverse() + + for path in candidates: + if path.is_file() and os.access(path, os.X_OK): + os.environ[_ENV_PATH] = str(path.resolve()) + break + + +_set_gridsynth_env_from_package()