rattler-build, python version_independent, cuda 12/13 variants

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: build
+name: ci
 on:
   release: {types: [published]}
   push:
@@ -8,128 +8,76 @@ on:
     branches: [master]
 jobs:
   test-cuda:
-    defaults: {run: {shell: 'bash -el {0}'}}
     runs-on: [self-hosted, python, cuda]
-    strategy:
-      matrix:
-        python-version: [3.12] # penultimate supported
+    strategy: {matrix: {cuda-version: [12]}}
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with: {fetch-depth: 0, submodules: recursive}
-    - id: reqs
-      name: set requirements
-      run: |
-        envname="${GITHUB_REPOSITORY##*/}-${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}"
-        echo "envname=$envname" >> $GITHUB_OUTPUT
-    - uses: conda-incubator/setup-miniconda@v3
+    - uses: prefix-dev/rattler-build-action@v0.2.38 # https://github.com/prefix-dev/rattler-build/pull/2484
       with:
-        python-version: ${{ matrix.python-version }}
-        activate-environment: ${{ steps.reqs.outputs.envname }}
-        run-post: false
-    - id: build
-      name: build
-      run: |
-        conda activate "${{ steps.reqs.outputs.envname }}"
-        which cmake || conda install -yq cmake
-        cmake -S . -B ./build_proj -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=ON -DCMAKE_INSTALL_PREFIX=./install
-        cmake --build ./build_proj --target install
-        pip install ./src/Python
-    - name: test
-      run: |
-        conda activate "${{ steps.reqs.outputs.envname }}"
-        PYTHONPATH=./src/Python python -m unittest discover -v -s ./test
-    - if: always()
-      name: Post Run conda-incubator/setup-miniconda@v3
-      shell: bash
-      run: |
-        sed -i '/${{ steps.reqs.outputs.envname }}/d' ~/.profile
-        source ~/.profile
-        conda env remove -n "${{ steps.reqs.outputs.envname }}"
+        recipe-path: recipe.yaml
+        build-args: --experimental --variant cuda_version_major=${{ matrix.cuda-version }}
   test:
-    defaults: {run: {shell: 'bash -el {0}'}}
-    runs-on: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }}
     strategy:
       matrix:
-        python-version: ['3.10', 3.13] # min & max supported
-        os: [ubuntu-latest, windows-latest]
+        python-version: ['3.10', '3.x'] # min & max supported
+        os: [ubuntu, windows]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with: {fetch-depth: 0, submodules: recursive}
-    - uses: conda-incubator/setup-miniconda@v3
+    - uses: actions/setup-python@v6
       with:
         python-version: ${{ matrix.python-version }}
-        conda-remove-defaults: "true"
     - name: build
       run: |
-        cmake -S . -B ./build_proj -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=OFF -DCMAKE_INSTALL_PREFIX=./install
-        cmake --build ./build_proj --target install
+        cmake -S . -B ./output -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=OFF -DCMAKE_INSTALL_PREFIX=src/Python/ccpi/filters
+        cmake --build ./output --target install
         pip install ./src/Python
     - name: test
-      run: PYTHONPATH=./src/Python python -m unittest discover -v -s ./test
+      run: python -m unittest discover -v -s ./test
+      env: {PYTHONPATH: ./src/Python}
   conda-build:
-    defaults: {run: {shell: 'bash -el {0}'}}
-    runs-on: ${{ matrix.os }}-${{ matrix.os == 'windows' && '2022' || 'latest' }}
+    runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }}
     strategy:
       matrix:
-        python-version: ['3.10', 3.11, 3.12] # parallelise conda_build_config.yaml
         os: [ubuntu, windows]
-        include:
-        - {python-version: 3.13, os: ubuntu}
+        cuda-version: [None, 12, 13]
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
       with: {fetch-depth: 0, submodules: recursive}
-    - if: matrix.os == 'windows'
+    - if: matrix.os == 'windows' && matrix.cuda-version == 'None'
       uses: ilammy/msvc-dev-cmd@v1
-    - uses: conda-incubator/setup-miniconda@v3
+    - uses: prefix-dev/rattler-build-action@v0.2.38 # https://github.com/prefix-dev/rattler-build/pull/2484
       with:
-        python-version: ${{ matrix.python-version }}
-        mamba-version: "*"
-        channels: conda-forge
-        conda-remove-defaults: "true"
-    - name: conda build
-      run: |
-        if test ${{ matrix.python-version }} = 3.13; then
-          BUILD_DEP=conda-build
-          BUILD_CMD="conda build"
-        else
-          BUILD_DEP=boa
-          BUILD_CMD="conda mambabuild"
-        fi
-        conda install $BUILD_DEP conda-verify
-        $BUILD_CMD -c conda-forge --override-channels --output-folder dist recipe --python=${{ matrix.python-version }} --no-test
-        $BUILD_CMD -c conda-forge --override-channels --test dist/*/ccpi-regulariser-*-cpu_*
-    - uses: actions/upload-artifact@v4
-      with:
-        name: ccpi-regulariser-py${{ matrix.python-version}}-${{ matrix.os }}
-        path: dist/*/ccpi-regulariser-*
+        recipe-path: recipe.yaml
+        build-args: --experimental --variant cuda_version_major=${{ matrix.cuda-version }} ${{ matrix.cuda-version == 'None' && ' ' || '--no-test' }}
+        artifact-name: conda-${{ matrix.os }}-${{ matrix.cuda-version }}
   conda-upload:
     if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')
-    defaults: {run: {shell: 'bash -el {0}', working-directory: dist}}
-    runs-on: ubuntu-latest
+    defaults: {run: {working-directory: output}}
+    runs-on: ubuntu-slim
     needs: conda-build
     steps:
-    - uses: actions/checkout@v4
-    - uses: conda-incubator/setup-miniconda@v3
-      with:
-        mamba-version: "*"
-        channels: conda-forge
-        conda-remove-defaults: "true"
-    - uses: actions/download-artifact@v4
-      with: {pattern: ccpi-regulariser-*, path: dist, merge-multiple: true}
-    - run: mamba install anaconda-client
-    - name: anaconda upload -c ccpi
+    - uses: actions/checkout@v6
+    - uses: prefix-dev/rattler-build-action@v0.2.37
+      with: {setup-only: true}
+    - uses: actions/download-artifact@v8
+      with: {pattern: conda-*, path: output, merge-multiple: true}
+    - name: conda upload -c ccpi
       run: >
-        anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force
-        --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} */ccpi-regulariser-*
+        rattler-build upload anaconda */*.conda -f
+        -o ccpi -a ${{ secrets.CCPI_CONDA_TOKEN }}
+        -c ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }}
     - if: startsWith(github.ref, 'refs/tags')
       name: conda upload -c tomography.stfc.ac.uk/conda
       run: |
         echo '${{ secrets.STFC_SSH_KEY }}' > ./key
         chmod 600 ./key
-        rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -R */ccpi-regulariser-* '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}'
+        rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -R */*.conda '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}'
         ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \
           'bash -lic "conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}"'
   pass:
     needs: [test-cuda, test, conda-build]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-slim
     steps: [{run: echo success}]

.gitignore

@@ -1,6 +1,4 @@
-/build_proj/
-/install/
-/dist/
+/output/
 __pycache__/
 /src/Python/build/
 /src/Python/*.egg*/

Installation.md

@@ -29,8 +29,8 @@ Here an example of build on Linux (see also `run.sh` for additional info):
 ```sh
 git clone https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit
 cd CCPi-Regularisation-Toolkit
-cmake -S . -B ./build_proj -DBUILD_MATLAB_WRAPPER=ON -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=ON -DCMAKE_INSTALL_PREFIX=./install
-cmake --build ./build_proj --target install
+cmake -S . -B ./output -DBUILD_MATLAB_WRAPPER=ON -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=ON -DCMAKE_INSTALL_PREFIX=src/Python/ccpi/filters
+cmake --build ./output --target install
 pip install ./src/Python
 ```
 
@@ -43,11 +43,11 @@ Python binaries are distributed via the [`ccpi`](https://anaconda.org/ccpi/ccpi-
 - `conda install -c ccpi -c conda-forge ccpi-regulariser=*=cpu*` (CPU-only)
 - `conda install -c ccpi -c conda-forge ccpi-regulariser=*=cuda*` (CUDA)
 
-#### Python (conda-build)
+#### Python (rattler-build)
 
 ```sh
-conda build recipe/ --output-folder dist
-conda install ccpi-regulariser=*=cuda* -c dist -c conda-forge
+rattler-build build --experimental
+conda install ccpi-regulariser=*=cuda* -c output -c conda-forge
 cd demos/
 python demo_cpu_regularisers.py # to run CPU demo
 python demo_gpu_regularisers.py # to run GPU demo

recipe.yaml

@@ -0,0 +1,81 @@
+schema_version: 1
+
+context:
+  # https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#loading-data-from-other-files
+  project: ${{ load_from_file("src/Python/pyproject.toml").project }} # ratter-build --experimental
+  python_min: "3.10"
+  git_number: ${{ git.latest_tag_distance(".") | int }} # ratter-build>=0.65 --experimental
+  build_number: ${{ git_number + (0 if cuda_version_major == "None" else 200) }}
+
+package:
+  name: ${{ project.name }}
+  version: ${{ project.version }}
+
+source:
+  path: .
+
+build:
+  number: ${{ build_number }}
+  string: ${{ "cpu" if cuda_version_major == "None" else ["cuda", cuda_version_major] | join("") }}_h${{ hash }}_${{ build_number }}
+  script:
+   content:
+   # TODO: use scikit-build-core & setuptools_scm instead
+   - if: win
+     then:
+     - cmake -S . %CMAKE_ARGS% -DCMAKE_INSTALL_PREFIX=%SRC_DIR%\src\Python\ccpi\filters\ %CMAKE_EXTRA_ARGS%
+     - cmake --build . --target install --config RelWithDebInfo
+     - pip install -vv %SRC_DIR%\src\Python --prefix=%PREFIX%
+     else:
+     - cmake -S . $CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=$SRC_DIR/src/Python/ccpi/filters/ $CMAKE_EXTRA_ARGS
+     - cmake --build . --target install
+     - pip install -vv $SRC_DIR/src/Python --prefix=$PREFIX
+   env:
+     CMAKE_EXTRA_ARGS: -DBUILD_PYTHON_WRAPPERS=ON -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CUDA=${{ "OFF" if cuda_version_major == "None" else "ON" }}
+  python:
+    version_independent: true
+
+requirements:
+  build:
+    - cmake
+    - ninja
+    - ${{ compiler('cxx') }}
+    - if: cuda_version_major != "None"
+      then:
+      - cuda-version ${{ cuda_version_major }}.*
+      - cuda-nvcc
+  host:
+    - python ${{ python_min }}.*
+    - if: linux
+      then: libgomp
+    - if: osx
+      then: llvm-openmp
+    - setuptools
+    - pip
+    - if: cuda_version_major != "None"
+      then:
+      - cuda-version ${{ cuda_version_major }}.*
+      - cuda-cccl
+      - if: linux
+        then: cuda-cudart-dev_linux-64
+      - if: win
+        then: cuda-cudart-dev_win-64
+  run:
+    - python >=${{ python_min }}
+    - if: cuda_version_major != "None"
+      then: cuda-version ${{ cuda_version_major }}.*
+    - numpy
+  ignore_run_exports:
+    by_name:
+      - cuda-cudart  # due to CUDA::cudart_static
+
+tests:
+  - files:
+      source:
+        - ./test/
+    script:
+      - python -m unittest discover -v -s test
+
+about:
+  license: Apache-2.0
+  summary: The set of CPU/GPU optimised regularisation modules for iterative image reconstruction and other image processing tasks
+  homepage: https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit