diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 17b516d0..db22f942 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: build +name: ci on: release: {types: [published]} push: @@ -8,128 +8,76 @@ on: branches: [master] jobs: test-cuda: - defaults: {run: {shell: 'bash -el {0}'}} runs-on: [self-hosted, python, cuda] - strategy: - matrix: - python-version: [3.12] # penultimate supported + strategy: {matrix: {cuda-version: [12]}} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: {fetch-depth: 0, submodules: recursive} - - id: reqs - name: set requirements - run: | - envname="${GITHUB_REPOSITORY##*/}-${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}" - echo "envname=$envname" >> $GITHUB_OUTPUT - - uses: conda-incubator/setup-miniconda@v3 + - uses: prefix-dev/rattler-build-action@v0.2.38 # https://github.com/prefix-dev/rattler-build/pull/2484 with: - python-version: ${{ matrix.python-version }} - activate-environment: ${{ steps.reqs.outputs.envname }} - run-post: false - - id: build - name: build - run: | - conda activate "${{ steps.reqs.outputs.envname }}" - which cmake || conda install -yq cmake - cmake -S . -B ./build_proj -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=ON -DCMAKE_INSTALL_PREFIX=./install - cmake --build ./build_proj --target install - pip install ./src/Python - - name: test - run: | - conda activate "${{ steps.reqs.outputs.envname }}" - PYTHONPATH=./src/Python python -m unittest discover -v -s ./test - - if: always() - name: Post Run conda-incubator/setup-miniconda@v3 - shell: bash - run: | - sed -i '/${{ steps.reqs.outputs.envname }}/d' ~/.profile - source ~/.profile - conda env remove -n "${{ steps.reqs.outputs.envname }}" + recipe-path: recipe.yaml + build-args: --experimental --variant cuda_version_major=${{ matrix.cuda-version }} test: - defaults: {run: {shell: 'bash -el {0}'}} - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }} strategy: matrix: - python-version: ['3.10', 3.13] # min & max supported - os: [ubuntu-latest, windows-latest] + python-version: ['3.10', '3.x'] # min & max supported + os: [ubuntu, windows] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: {fetch-depth: 0, submodules: recursive} - - uses: conda-incubator/setup-miniconda@v3 + - uses: actions/setup-python@v6 with: python-version: ${{ matrix.python-version }} - conda-remove-defaults: "true" - name: build run: | - cmake -S . -B ./build_proj -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=OFF -DCMAKE_INSTALL_PREFIX=./install - cmake --build ./build_proj --target install + cmake -S . -B ./output -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=OFF -DCMAKE_INSTALL_PREFIX=src/Python/ccpi/filters + cmake --build ./output --target install pip install ./src/Python - name: test - run: PYTHONPATH=./src/Python python -m unittest discover -v -s ./test + run: python -m unittest discover -v -s ./test + env: {PYTHONPATH: ./src/Python} conda-build: - defaults: {run: {shell: 'bash -el {0}'}} - runs-on: ${{ matrix.os }}-${{ matrix.os == 'windows' && '2022' || 'latest' }} + runs-on: ${{ matrix.os }}-${{ matrix.os == 'ubuntu' && '22.04' || 'latest' }} strategy: matrix: - python-version: ['3.10', 3.11, 3.12] # parallelise conda_build_config.yaml os: [ubuntu, windows] - include: - - {python-version: 3.13, os: ubuntu} + cuda-version: [None, 12, 13] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: {fetch-depth: 0, submodules: recursive} - - if: matrix.os == 'windows' + - if: matrix.os == 'windows' && matrix.cuda-version == 'None' uses: ilammy/msvc-dev-cmd@v1 - - uses: conda-incubator/setup-miniconda@v3 + - uses: prefix-dev/rattler-build-action@v0.2.38 # https://github.com/prefix-dev/rattler-build/pull/2484 with: - python-version: ${{ matrix.python-version }} - mamba-version: "*" - channels: conda-forge - conda-remove-defaults: "true" - - name: conda build - run: | - if test ${{ matrix.python-version }} = 3.13; then - BUILD_DEP=conda-build - BUILD_CMD="conda build" - else - BUILD_DEP=boa - BUILD_CMD="conda mambabuild" - fi - conda install $BUILD_DEP conda-verify - $BUILD_CMD -c conda-forge --override-channels --output-folder dist recipe --python=${{ matrix.python-version }} --no-test - $BUILD_CMD -c conda-forge --override-channels --test dist/*/ccpi-regulariser-*-cpu_* - - uses: actions/upload-artifact@v4 - with: - name: ccpi-regulariser-py${{ matrix.python-version}}-${{ matrix.os }} - path: dist/*/ccpi-regulariser-* + recipe-path: recipe.yaml + build-args: --experimental --variant cuda_version_major=${{ matrix.cuda-version }} ${{ matrix.cuda-version == 'None' && ' ' || '--no-test' }} + artifact-name: conda-${{ matrix.os }}-${{ matrix.cuda-version }} conda-upload: if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags') - defaults: {run: {shell: 'bash -el {0}', working-directory: dist}} - runs-on: ubuntu-latest + defaults: {run: {working-directory: output}} + runs-on: ubuntu-slim needs: conda-build steps: - - uses: actions/checkout@v4 - - uses: conda-incubator/setup-miniconda@v3 - with: - mamba-version: "*" - channels: conda-forge - conda-remove-defaults: "true" - - uses: actions/download-artifact@v4 - with: {pattern: ccpi-regulariser-*, path: dist, merge-multiple: true} - - run: mamba install anaconda-client - - name: anaconda upload -c ccpi + - uses: actions/checkout@v6 + - uses: prefix-dev/rattler-build-action@v0.2.37 + with: {setup-only: true} + - uses: actions/download-artifact@v8 + with: {pattern: conda-*, path: output, merge-multiple: true} + - name: conda upload -c ccpi run: > - anaconda -v -t ${{ secrets.CCPI_CONDA_TOKEN }} upload --force - --label ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} */ccpi-regulariser-* + rattler-build upload anaconda */*.conda -f + -o ccpi -a ${{ secrets.CCPI_CONDA_TOKEN }} + -c ${{ startsWith(github.ref, 'refs/tags') && 'main' || 'dev' }} - if: startsWith(github.ref, 'refs/tags') name: conda upload -c tomography.stfc.ac.uk/conda run: | echo '${{ secrets.STFC_SSH_KEY }}' > ./key chmod 600 ./key - rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -R */ccpi-regulariser-* '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}' + rsync -e 'ssh -o StrictHostKeyChecking=no -i ./key' -R */*.conda '${{ secrets.STFC_SSH_HOST }}:${{ secrets.STFC_SSH_CONDA_DIR }}' ssh -o StrictHostKeyChecking=no -i ./key ${{ secrets.STFC_SSH_HOST }} \ 'bash -lic "conda index --bz2 --zst --run-exports --channeldata --rss -n ccpi ${{ secrets.STFC_SSH_CONDA_DIR }}"' pass: needs: [test-cuda, test, conda-build] - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: [{run: echo success}] diff --git a/.gitignore b/.gitignore index 43a57613..a9bdea7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,4 @@ -/build_proj/ -/install/ -/dist/ +/output/ __pycache__/ /src/Python/build/ /src/Python/*.egg*/ diff --git a/Installation.md b/Installation.md index e017cfa7..eb8fec55 100644 --- a/Installation.md +++ b/Installation.md @@ -29,8 +29,8 @@ Here an example of build on Linux (see also `run.sh` for additional info): ```sh git clone https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit cd CCPi-Regularisation-Toolkit -cmake -S . -B ./build_proj -DBUILD_MATLAB_WRAPPER=ON -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=ON -DCMAKE_INSTALL_PREFIX=./install -cmake --build ./build_proj --target install +cmake -S . -B ./output -DBUILD_MATLAB_WRAPPER=ON -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=ON -DCMAKE_INSTALL_PREFIX=src/Python/ccpi/filters +cmake --build ./output --target install pip install ./src/Python ``` @@ -43,11 +43,11 @@ Python binaries are distributed via the [`ccpi`](https://anaconda.org/ccpi/ccpi- - `conda install -c ccpi -c conda-forge ccpi-regulariser=*=cpu*` (CPU-only) - `conda install -c ccpi -c conda-forge ccpi-regulariser=*=cuda*` (CUDA) -#### Python (conda-build) +#### Python (rattler-build) ```sh -conda build recipe/ --output-folder dist -conda install ccpi-regulariser=*=cuda* -c dist -c conda-forge +rattler-build build --experimental +conda install ccpi-regulariser=*=cuda* -c output -c conda-forge cd demos/ python demo_cpu_regularisers.py # to run CPU demo python demo_gpu_regularisers.py # to run GPU demo diff --git a/recipe.yaml b/recipe.yaml new file mode 100644 index 00000000..5b4e5c73 --- /dev/null +++ b/recipe.yaml @@ -0,0 +1,81 @@ +schema_version: 1 + +context: + # https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#loading-data-from-other-files + project: ${{ load_from_file("src/Python/pyproject.toml").project }} # ratter-build --experimental + python_min: "3.10" + git_number: ${{ git.latest_tag_distance(".") | int }} # ratter-build>=0.65 --experimental + build_number: ${{ git_number + (0 if cuda_version_major == "None" else 200) }} + +package: + name: ${{ project.name }} + version: ${{ project.version }} + +source: + path: . + +build: + number: ${{ build_number }} + string: ${{ "cpu" if cuda_version_major == "None" else ["cuda", cuda_version_major] | join("") }}_h${{ hash }}_${{ build_number }} + script: + content: + # TODO: use scikit-build-core & setuptools_scm instead + - if: win + then: + - cmake -S . %CMAKE_ARGS% -DCMAKE_INSTALL_PREFIX=%SRC_DIR%\src\Python\ccpi\filters\ %CMAKE_EXTRA_ARGS% + - cmake --build . --target install --config RelWithDebInfo + - pip install -vv %SRC_DIR%\src\Python --prefix=%PREFIX% + else: + - cmake -S . $CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=$SRC_DIR/src/Python/ccpi/filters/ $CMAKE_EXTRA_ARGS + - cmake --build . --target install + - pip install -vv $SRC_DIR/src/Python --prefix=$PREFIX + env: + CMAKE_EXTRA_ARGS: -DBUILD_PYTHON_WRAPPERS=ON -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DBUILD_CUDA=${{ "OFF" if cuda_version_major == "None" else "ON" }} + python: + version_independent: true + +requirements: + build: + - cmake + - ninja + - ${{ compiler('cxx') }} + - if: cuda_version_major != "None" + then: + - cuda-version ${{ cuda_version_major }}.* + - cuda-nvcc + host: + - python ${{ python_min }}.* + - if: linux + then: libgomp + - if: osx + then: llvm-openmp + - setuptools + - pip + - if: cuda_version_major != "None" + then: + - cuda-version ${{ cuda_version_major }}.* + - cuda-cccl + - if: linux + then: cuda-cudart-dev_linux-64 + - if: win + then: cuda-cudart-dev_win-64 + run: + - python >=${{ python_min }} + - if: cuda_version_major != "None" + then: cuda-version ${{ cuda_version_major }}.* + - numpy + ignore_run_exports: + by_name: + - cuda-cudart # due to CUDA::cudart_static + +tests: + - files: + source: + - ./test/ + script: + - python -m unittest discover -v -s test + +about: + license: Apache-2.0 + summary: The set of CPU/GPU optimised regularisation modules for iterative image reconstruction and other image processing tasks + homepage: https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit diff --git a/recipe/bld.bat b/recipe/bld.bat deleted file mode 100644 index 83751b0c..00000000 --- a/recipe/bld.bat +++ /dev/null @@ -1,15 +0,0 @@ -mkdir "%SRC_DIR%\test" -ROBOCOPY /E "%RECIPE_DIR%\..\test" "%SRC_DIR%\test" -if exist "%SRC_DIR%\build_proj" ( - rd /s /q "%SRC_DIR%\build_proj" -) - -:: -G "Visual Studio 16 2019" specifies the the generator -:: -T v142 specifies the toolset -:: -DCUDA_TOOLKIT_ROOT_DIR="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v11.8" to the cmake command to specify the CUDA toolkit version - -cmake -S "%SRC_DIR%" -B "%SRC_DIR%\build_proj" -G Ninja -DBUILD_PYTHON_WRAPPERS=ON -DBUILD_CUDA=%BUILD_CUDA% -DCMAKE_BUILD_TYPE="RelWithDebInfo" -DCMAKE_INSTALL_PREFIX="%SRC_DIR%\install" %CMAKE_ARGS% -cmake --build "%SRC_DIR%\build_proj" --target install --config RelWithDebInfo -%PYTHON% -m pip install -vv "%SRC_DIR%\src\Python" - -if errorlevel 1 exit 1 diff --git a/recipe/build.sh b/recipe/build.sh deleted file mode 100755 index 59d53dc7..00000000 --- a/recipe/build.sh +++ /dev/null @@ -1,9 +0,0 @@ -set -xe -cp -rv "$RECIPE_DIR/../test" "$SRC_DIR/" -if test -d "$SRC_DIR/build_proj"; then - rm -rf "$SRC_DIR/build_proj" -fi - -cmake -S "$SRC_DIR" -B "$SRC_DIR/build_proj" -G Ninja -DBUILD_PYTHON_WRAPPER=ON -DBUILD_CUDA=${BUILD_CUDA:-OFF} -DCMAKE_BUILD_TYPE="RelWithDebInfo" -DCMAKE_INSTALL_PREFIX="$SRC_DIR/install" ${CMAKE_ARGS} -cmake --build "$SRC_DIR/build_proj" --target install -$PYTHON -m pip install -vv "$SRC_DIR/src/Python" diff --git a/recipe/conda_build_config.yaml b/recipe/conda_build_config.yaml deleted file mode 100644 index bd6b93c8..00000000 --- a/recipe/conda_build_config.yaml +++ /dev/null @@ -1,8 +0,0 @@ -python: - - 3.10 - - 3.11 - - 3.12 - - 3.13 -cuda_compiler_version: - - None - - 12.9 diff --git a/recipe/meta.yaml b/recipe/meta.yaml deleted file mode 100644 index 1fb6e61d..00000000 --- a/recipe/meta.yaml +++ /dev/null @@ -1,63 +0,0 @@ -# https://docs.conda.io/projects/conda-build/en/stable/resources/define-metadata.html#loading-data-from-other-files -# TODO: use scikit-build-core & setuptools_scm instead -{% set project = load_file_data('../src/Python/pyproject.toml', from_recipe_dir=True).get('project', {}) %} -{% set build_number = environ.get('GIT_DESCRIBE_NUMBER', 0) | int %} -{% if cuda_compiler_version != "None" %} -{% set build_number = build_number + 200 %} -{% endif %} - -package: - name: {{ project.get('name') }} - version: {{ project.get('version') }} - -build: - preserve_egg_dir: False - number: {{ build_number }} - run_exports: - - {{ pin_subpackage(project.get('name'), max_pin='x.x') }} - - {{ project.get('name') }} =*=cuda* # [cuda_compiler_version != "None"] - - {{ project.get('name') }} =*=cpu* # [cuda_compiler_version == "None"] - ignore_run_exports: - - cudatoolkit # [(cuda_compiler_version or "None").startswith("11")] - #ignore_run_exports_from: - # - {{ compiler('cuda') }} # [cuda_compiler_version != "None"] - script_env: - - BUILD_CUDA=ON # [cuda_compiler_version != "None"] - - BUILD_CUDA=OFF # [cuda_compiler_version == "None"] - string: cuda{{ cuda_compiler_version | replace('.', '') }}_py{{ py }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} # [cuda_compiler_version != "None"] - string: cpu_py{{ py }}h{{ PKG_HASH }}_{{ PKG_BUILDNUM }} # [cuda_compiler_version == "None"] - -source: - path: .. - -test: - source_files: - - ./test/ - commands: - - python -m unittest discover -v -s test -requirements: - build: - - cmake - - ninja - - {{ compiler('cxx') }} - #- {{ compiler('cuda') }} # [cuda_compiler_version != "None"] - - cuda-compiler ~={{ cuda_compiler_version }} # [cuda_compiler_version != "None"] - host: - - python - - llvm-openmp - - setuptools - - pip - - cuda-cccl # [cuda_compiler_version != "None"] - - cuda-version {{ cuda_compiler_version }} # [cuda_compiler_version != "None"] - run: - - {{ pin_compatible('python', min_pin='x.x', max_pin='x') }} - {% for dep in project.get("dependencies") %} - - {{ dep.lower() }} - {% endfor %} - - __cuda # [cuda_compiler_version != "None"] - - {{ pin_compatible('cuda-version', min_pin='x', max_pin='x') }} # [cuda_compiler_version != "None"] - -about: - home: https://github.com/TomographicImaging/CCPi-Regularisation-Toolkit - license: Apache-2.0 - summary: 'The set of CPU/GPU optimised regularisation modules for iterative image reconstruction and other image processing tasks' diff --git a/src/Core/CMakeLists.txt b/src/Core/CMakeLists.txt index e49d63c8..7e6f8de3 100644 --- a/src/Core/CMakeLists.txt +++ b/src/Core/CMakeLists.txt @@ -44,19 +44,11 @@ include_directories(cilreg PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/regularisers_CPU/) message(STATUS "Installing in ${CMAKE_INSTALL_PREFIX}") -if (UNIX) - install(TARGETS cilreg - LIBRARY DESTINATION . - CONFIGURATIONS ${CMAKE_BUILD_TYPE} - ) -elseif(WIN32) - install(TARGETS cilreg - RUNTIME DESTINATION bin - ARCHIVE DESTINATION . - CONFIGURATIONS ${CMAKE_BUILD_TYPE} - ) -endif() - +install(TARGETS cilreg + LIBRARY DESTINATION . + RUNTIME DESTINATION . + ARCHIVE DESTINATION . + CONFIGURATIONS ${CMAKE_BUILD_TYPE}) # GPU Regularisers if (BUILD_CUDA) @@ -66,23 +58,7 @@ if (BUILD_CUDA) set(CMAKE_CUDA_ARCHITECTURES "all") endif() - # CCCL (for thrust), vis. https://github.com/NVIDIA/cccl/tree/main/examples/basic#downloading-cpm - set(CPM_DOWNLOAD_VERSION 0.34.0) - if(CPM_SOURCE_CACHE) - set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") - elseif(DEFINED ENV{CPM_SOURCE_CACHE}) - set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake") - else() - set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake") - endif() - if(NOT (EXISTS ${CPM_DOWNLOAD_LOCATION})) - message(STATUS "Downloading CPM.cmake to ${CPM_DOWNLOAD_LOCATION}") - file(DOWNLOAD - https://github.com/TheLartians/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake - ${CPM_DOWNLOAD_LOCATION}) - endif() - include(${CPM_DOWNLOAD_LOCATION}) - CPMAddPackage(NAME CCCL GITHUB_REPOSITORY nvidia/cccl GIT_TAG main) + find_package(CCCL CONFIG REQUIRED HINTS ${CUDAToolkit_INCLUDE_DIRS} ${CUDAToolkit_LIBRARY_DIR}/cmake) if (UNIX) set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler -fPIC -shared -D_FORCE_INLINES") @@ -107,7 +83,7 @@ if (BUILD_CUDA) message(STATUS "Installing in ${CMAKE_INSTALL_PREFIX}") install(TARGETS cilregcuda LIBRARY DESTINATION . - RUNTIME DESTINATION bin + RUNTIME DESTINATION . ARCHIVE DESTINATION . CONFIGURATIONS ${CMAKE_BUILD_TYPE}) endif() diff --git a/variants.yaml b/variants.yaml new file mode 100644 index 00000000..8e0767a9 --- /dev/null +++ b/variants.yaml @@ -0,0 +1,4 @@ +cuda_version_major: +- None +- 12 +- 13