From 6dbf872257af5dd6e0afd46ef88caa783741d0c0 Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 12:21:26 +0200 Subject: [PATCH 1/9] add mi300 pipeline --- .ci/README.md | 30 ++- .ci/build.sh | 52 +++++ .ci/cscs_beverin_pipeline.yml | 52 +++++ .ci/cscs_default_pipeline.yml | 41 +++- .ci/include/cscs/00-variables.yml | 9 +- .ci/include/cscs/01-build-templates.yml | 17 ++ .ci/include/cscs/01-test-templates.yml | 53 ----- .ci/include/cscs/02-test-templates.yml | 20 ++ .../__pycache__/package.cpython-36.pyc | Bin 0 -> 1085 bytes .ci/spack_packages/lemonio/package.py | 29 +++ .../quda/__pycache__/package.cpython-36.pyc | Bin 0 -> 6557 bytes .ci/spack_packages/quda/package.py | 203 ++++++++++++++++++ .../quda/quda-hipfft-public-to-private.patch | 13 ++ .../tmlqcd/__pycache__/package.cpython-36.pyc | Bin 0 -> 4290 bytes .../tmlqcd/package.py | 39 +++- .../tmlqcd/beverin-mi300/compilers.yaml | 4 + .../tmlqcd/beverin-mi300/config.yaml | 10 + .../tmlqcd/beverin-mi300/environments.yaml | 29 +++ .../tmlqcd/beverin-mi300/modules.yaml | 23 ++ .../tmlqcd/beverin-mi300/post-install | 8 + .../tmlqcd/beverin-mi300/repo/packages | 1 + .../tmlqcd/beverin-mi300/repo/repo.yaml | 2 + .../tmlqcd/daint-gh200/config.yaml | 2 +- .../tmlqcd/daint-gh200/environments.yaml | 8 +- .../tmlqcd/daint-gh200/repo/packages | 1 + .../repo/packages/lemonio/package.py | 35 --- .../tmlqcd/daint-gh200/repo/repo.yaml | 1 - 27 files changed, 561 insertions(+), 121 deletions(-) create mode 100755 .ci/build.sh create mode 100644 .ci/cscs_beverin_pipeline.yml create mode 100644 .ci/include/cscs/01-build-templates.yml delete mode 100644 .ci/include/cscs/01-test-templates.yml create mode 100644 .ci/include/cscs/02-test-templates.yml create mode 100644 .ci/spack_packages/lemonio/__pycache__/package.cpython-36.pyc create mode 100644 .ci/spack_packages/lemonio/package.py create mode 100644 .ci/spack_packages/quda/__pycache__/package.cpython-36.pyc create mode 100644 .ci/spack_packages/quda/package.py create mode 100644 .ci/spack_packages/quda/quda-hipfft-public-to-private.patch create mode 100644 .ci/spack_packages/tmlqcd/__pycache__/package.cpython-36.pyc rename .ci/{uenv-recipes/tmlqcd/daint-gh200/repo/packages => spack_packages}/tmlqcd/package.py (72%) mode change 100644 => 100755 create mode 100644 .ci/uenv-recipes/tmlqcd/beverin-mi300/compilers.yaml create mode 100644 .ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml create mode 100755 .ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml create mode 100644 .ci/uenv-recipes/tmlqcd/beverin-mi300/modules.yaml create mode 100755 .ci/uenv-recipes/tmlqcd/beverin-mi300/post-install create mode 120000 .ci/uenv-recipes/tmlqcd/beverin-mi300/repo/packages create mode 100644 .ci/uenv-recipes/tmlqcd/beverin-mi300/repo/repo.yaml create mode 120000 .ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages delete mode 100644 .ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/lemonio/package.py diff --git a/.ci/README.md b/.ci/README.md index 057d45127..42b6b59fd 100644 --- a/.ci/README.md +++ b/.ci/README.md @@ -5,24 +5,39 @@ This document describes the external pipeline executed through CSCS. The pipeline can be triggered by commenting on a pull request with ``` -cscs-ci run default # runs the default pipeline +cscs-ci run default # runs the default pipeline (on GH200 nodes @ CSCS) +cscs-ci run beverin # runs the beverin pipeline (on MI300A nodes @ CSCS) ``` An automatic trigger on all merge-requests is currently disabled. -This pipeline has 2 stages: `build` and `test`. +This pipeline has 3 stages: `prepare`, `build` and `test`. -The `build` stage builds a uenv image that includes all necessary compilers, MPI libraries and other dependecies to build QUDA and tmLQCD against QUDA. In this stage, QUDA is built correctly for the GH200 machine at CSCS with all required build flags for production runs. The uenv recipe can be found [here](uenv-recipes/tmlqcd/daint-gh200). +## `prepare` stage -In the `test` stage, the aforementioned uenv image is loaded, tmLQCD is built and linked against the QUDA library that is inside the image. Finally a minimal HMC is executed and checked against some reference data. +The `prepare` stage builds an uenv image that includes all necessary compilers, MPI libraries and other dependecies to build QUDA and tmLQCD against QUDA. The uenv recipe can be found [here for GH200](uenv-recipes/tmlqcd/daint-gh200) and [here for MI300A](uenv-recipes/tmlqcd/beverin-mi300). -## Force recompilation of quda +## `build` stage + +In the `build` stage, the aforementioned uenv image is loaded, tmLQCD and QUDA are built using their spack packages using the dependencies from the base image. This stage exposes an artifact with tmLQCD/QUDA binaries. For tmLQCD, the current branch is compiled. For QUDA the following environment variables are respected: + + * `QUDA_GIT_REPO`: the git repository URL to use as source (defaults to `https://github.com/lattice/quda.git`) + * `QUDA_GIT_BRANCH`: the git branch to compile (defaults to `develop`) + * `QUDA_GIT_COMMIT`: the git commit to compile (defaults to the current head commit of `QUDA_GIT_BRANCH`) + +Then QUDA is cloned and compiled, completely bypassing the spack compile cache. + +## `test` stage + +In the `test` stage, the aforementioned uenv image is loaded, tmLQCD and QUDA are unpacked from the artifact. Finally a minimal HMC is executed and checked against some reference data. + +## Force recompilation of base image in `prepare` stage Remove the build cache: ```bash -/capstor/scratch/cscs/${USER}/uenv-cache/user-environment/build_cache/linux-sles15-neoverse_v2/gcc-13.2.0/quda-* -/capstor/scratch/cscs/${USER}/uenv-cache/user-environment/build_cache/linux-sles15-neoverse_v2-gcc-13.2.0-quda* +/capstor/scratch/cscs/${USER}/uenv-cache/user-environment/build_cache/linux-sles15-neoverse_v2/gcc-13.2.0/tmlqcd-* +/capstor/scratch/cscs/${USER}/uenv-cache/user-environment/build_cache/linux-sles15-neoverse_v2-gcc-13.2.0-tmlqcd* ``` Or increment the the version counter tag in [.ci/include/cscs/00-variables.yml](include/cscs/00-variables.yml): @@ -46,3 +61,4 @@ and commit. * [CSCS Uenv Writing Documentation](https://eth-cscs.github.io/alps-uenv/) * [CSCS Status Page](https://status.cscs.ch/) * [CSCS Spack Base Containers](https://github.com/orgs/eth-cscs/packages/container/package/docker-ci-ext%2Fspack-base-containers%2Fspack-build) +* [Sirius CI/CD](https://github.com/electronic-structure/SIRIUS/tree/develop/ci) where this one is based upon diff --git a/.ci/build.sh b/.ci/build.sh new file mode 100755 index 000000000..d9dc14ba2 --- /dev/null +++ b/.ci/build.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -xeuo pipefail + +export SPACK_SYSTEM_CONFIG_PATH="/user-environment/config" +export SPACK_PYTHON="$(which python3.6)" # must be <=3.12, system python is 3.6 +export CICD_SRC_DIR="$PWD" +export QUDA_SRC_DIR="$PWD/deps/src/quda" + +# QUDA git, branch and commit +export QUDA_GIT_REPO="${QUDA_GIT_REPO:=https://github.com/lattice/quda.git}" +export QUDA_GIT_BRANCH="${QUDA_GIT_BRANCH:=develop}" +export QUDA_GIT_COMMIT="${QUDA_GIT_COMMIT:=$(git ls-remote ${QUDA_GIT_REPO} refs/heads/${QUDA_GIT_BRANCH} | awk '{print $1}')}" + +# obtain QUDA +git clone -b "${QUDA_GIT_BRANCH}" "${QUDA_GIT_REPO}" "${QUDA_SRC_DIR}" +git -C "${QUDA_SRC_DIR}" checkout "${QUDA_GIT_COMMIT}" + +# make sure we keep the stage direcorty +spack config --scope=spack add config:build_stage:/dev/shm/spack-stage +# we might need to install dependencies too, e.g. nlcglib in case of API changes +spack config --scope=spack add config:install_tree:root:/dev/shm/spack-stage + +spack env create -d ./spack-env + +# add local repository with current tmlqcd recipe +spack -e ./spack-env repo add "${REPO}" + +spack -e ./spack-env config add "packages:all:variants:[${VARIANTS}]" + +spack -e ./spack-env add "${SPEC}" + +# for tmlqcd use local src instead of fetch git +spack -e ./spack-env develop -p "${CICD_SRC_DIR}" tmlqcd@cicd + +# for quda use local src instead of fetch git, to be able to tests against +# differnt repo, branch, commit and also to support that quda branch develop is +# a moving target +spack -e ./spack-env develop -p "${QUDA_SRC_DIR}" quda@cicd + +# display spack.yaml +cat ./spack-env/spack.yaml + +spack -e ./spack-env concretize +spack -e ./spack-env install + +# the tar pipe below expects a relative path +builddir_tmlqcd=$(spack -e ./spack-env location -b tmlqcd) +builddir_quda=$(spack -e ./spack-env location -b quda) + +# create a symlink to spack build directory (keep in artifacts) +tar -cf builddir.tar $builddir_tmlqcd $builddir_quda diff --git a/.ci/cscs_beverin_pipeline.yml b/.ci/cscs_beverin_pipeline.yml new file mode 100644 index 000000000..aa980c2c8 --- /dev/null +++ b/.ci/cscs_beverin_pipeline.yml @@ -0,0 +1,52 @@ +include: + - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' + - local: '/.ci/include/cscs/00-variables.yml' + - local: '/.ci/include/cscs/01-build-templates.yml' + - local: '/.ci/include/cscs/02-test-templates.yml' + + +stages: + - prepare + - build + - test + + +build-base/uenv/beverin-mi300: + stage: prepare + extends: [.uenv-builder-beverin-mi300, .beverin-mi300-secrets] + variables: + UENV_RECIPE: .ci/uenv-recipes/tmlqcd/beverin-mi300 + SLURM_TIMELIMIT: "08:00:00" + + +build-tmlqcd/uenv/beverin-mi300: + extends: [.uenv-runner-beverin-mi300, .build/base, .beverin-mi300-secrets] + needs: [build-base/uenv/beverin-mi300] + variables: + SPEC: "tmlqcd@cicd +lemon +quda ^quda@cicd +qdp +multigrid +twisted_clover +twisted_mass" + REPO: "./.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/" + VARIANTS: "amdgpu_target=gfx942,amdgpu_target_sram_ecc=gfx942,+rocm,+mpi" + SLURM_TIMELIMIT: "01:00:00" + + +test/beverin-mi300: + extends: [.uenv-runner-beverin-mi300, .test/base, .beverin-mi300-secrets] + needs: [build-tmlqcd/uenv/beverin-mi300] + variables: + REFPATH: "doc/sample-output/hmc-quda-cscs" + QUDA_ENABLE_TUNING: 0 # disable tuning + QUDA_ENABLE_P2P: 0 # disable P2P + SLURM_JOB_NUM_NODES: 2 + SLURM_NTASKS: 8 + SLURM_TIMELIMIT: "01:00:00" + script: + - hmc_tm -f doc/sample-input/sample-hmc-quda-cscs-beverin.input + - | + if test "${SLURM_PROCID}" -eq "0"; then + echo "Check the results on SLURM_PROCID=${SLURM_PROCID} ..." + numdiff -r 1.2e-6 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data + for i in $(seq 0 2 18); do + f=onlinemeas.$(printf %06d $i); + numdiff -r 5e-4 ${f} ${REFPATH}/${f}; + done + fi diff --git a/.ci/cscs_default_pipeline.yml b/.ci/cscs_default_pipeline.yml index 0a059aa40..c5a336df6 100644 --- a/.ci/cscs_default_pipeline.yml +++ b/.ci/cscs_default_pipeline.yml @@ -1,25 +1,52 @@ include: - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' - local: '/.ci/include/cscs/00-variables.yml' - - local: '/.ci/include/cscs/01-test-templates.yml' + - local: '/.ci/include/cscs/01-build-templates.yml' + - local: '/.ci/include/cscs/02-test-templates.yml' + stages: + - prepare - build - test -build-quda/uenv/daint-gh200: - stage: build + +build-base/uenv/daint-gh200: + stage: prepare extends: .uenv-builder-daint-gh200 variables: UENV_RECIPE: .ci/uenv-recipes/tmlqcd/daint-gh200 + SLURM_TIMELIMIT: "04:00:00" + + +build-tmlqcd/uenv/daint-gh200: + extends: [.uenv-runner-daint-gh200, .build/base] + needs: [build-base/uenv/daint-gh200] + variables: + SPEC: "tmlqcd@cicd +lemon +quda ^quda@cicd +qdp +multigrid +twisted_clover +twisted_mass" + REPO: "./.ci/uenv-recipes/tmlqcd/daint-gh200/repo/" + VARIANTS: "cuda_arch=90,+cuda,+mpi" + SLURM_TIMELIMIT: "01:00:00" + test/daint-gh200: - extends: .test/hmc + extends: [.uenv-runner-daint-gh200, .test/base] + needs: [build-tmlqcd/uenv/daint-gh200] variables: - INPUT_FILE: "doc/sample-input/sample-hmc-quda-cscs.input" REFPATH: "doc/sample-output/hmc-quda-cscs" QUDA_ENABLE_TUNING: 0 # disable tuning - QUDA_ENABLE_GDR: 1 # enable GPU-Direct RDMA + QUDA_ENABLE_GDR: 0 # enable GPU-Direct RDMA SLURM_JOB_NUM_NODES: 2 SLURM_NTASKS: 8 - SLURM_TIMELIMIT: "00:30:00" + SLURM_TIMELIMIT: "01:00:00" + script: + - hmc_tm -f doc/sample-input/sample-hmc-quda-cscs.input + - | + if test "${SLURM_PROCID}" -eq "0"; then + echo "Check the results on SLURM_PROCID=${SLURM_PROCID} ..." + numdiff -r 1.2e-6 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data + for i in $(seq 0 2 18); do + f=onlinemeas.$(printf %06d $i); + numdiff -r 5e-4 ${f} ${REFPATH}/${f}; + done + fi diff --git a/.ci/include/cscs/00-variables.yml b/.ci/include/cscs/00-variables.yml index 382960fac..bdafc1ae0 100644 --- a/.ci/include/cscs/00-variables.yml +++ b/.ci/include/cscs/00-variables.yml @@ -9,5 +9,10 @@ variables: UENV_NAME: tmlqcd UENV_VERSION: experimental - UENV_TAG: v0.0.7 - UENV_VERSION: v1 + UENV_TAG: v0.0.8 + +# These are the firecrest id and secret for the beverin pipeline +.beverin-mi300-secrets: + variables: + F7T_CLIENT_ID: $F7T_TDS_CONSUMER_KEY + F7T_CLIENT_SECRET: $F7T_TDS_CONSUMER_SECRET diff --git a/.ci/include/cscs/01-build-templates.yml b/.ci/include/cscs/01-build-templates.yml new file mode 100644 index 000000000..6c4a87174 --- /dev/null +++ b/.ci/include/cscs/01-build-templates.yml @@ -0,0 +1,17 @@ +include: + - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' + + +.build/base: + stage: build + image: ${UENV_NAME}/${UENV_VERSION}:${UENV_TAG} + artifacts: + paths: + - builddir.tar + variables: + SLURM_TIMELIMIT: "01:00:00" + script: + - git clone --filter=tree:0 $(jq -r .spack.repo /user-environment/meta/configure.json) /dev/shm/spack-clone + - git -C /dev/shm/spack-clone checkout $(jq -r .spack.commit /user-environment/meta/configure.json) + - source /dev/shm/spack-clone/share/spack/setup-env.sh + - bwrap --dev-bind / / --tmpfs ~ -- ./.ci/build.sh diff --git a/.ci/include/cscs/01-test-templates.yml b/.ci/include/cscs/01-test-templates.yml deleted file mode 100644 index 16b187ab5..000000000 --- a/.ci/include/cscs/01-test-templates.yml +++ /dev/null @@ -1,53 +0,0 @@ -include: - - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' - - -.test/base: - stage: test - extends: .uenv-runner-daint-gh200 - image: ${UENV_NAME}/${UENV_VERSION}:${UENV_TAG} - variables: - WITH_UENV_VIEW: "default" -# CFLAGS: "-O3 -fopenmp -mtune=neoverse-v2 -mcpu=neoverse-v2" -# CXXFLAGS: "-O3 -fopenmp -mtune=neoverse-v2 -mcpu=neoverse-v2" -# LDFLAGS: "-fopenmp" - before_script: - - | - if test "${SLURM_PROCID}" -eq "0"; then - export CC="$(which mpicc)" - export CXX="$(which mpicxx)" - mkdir -p build_dir - cd build_dir - cmake -DCMAKE_PREFIX_PATH="/user-environment/env/default" \ - -DTM_USE_MPI=ON \ - -DTM_USE_CUDA=ON \ - -DCMAKE_C_CFLAGS="-O3 -mtune=neoverse-v2 -mcpu=neoverse-v2" \ - -DCMAKE_CXX_FLAGS="-O3 -mtune=neoverse-v2 -mcpu=neoverse-v2" \ - -DCMAKE_CUDA_ARCHITECTURES=90a \ - -DTM_USE_OMP=ON \ - -DTM_USE_QUDA=ON \ - -DTM_USE_LEMON=ON \ - -DTM_USE_GAUGE_COPY=ON \ - -DTM_USE_HALFSPINOR=ON \ - -DCMAKE_INSTALL_PREFIX=../install_dir .. - make - make install - cd .. - touch preparation-done-${CI_JOB_ID} - fi - - while test ! -f preparation-done-${CI_JOB_ID}; do sleep 5; done - - -.test/hmc: - extends: .test/base - script: - - ./install_dir/bin/hmc_tm -f "${INPUT_FILE}" - - | - if test "${SLURM_PROCID}" -eq "0"; then - echo "Check the results on SLURM_PROCID=${SLURM_PROCID} ..." - numdiff -r 1.2e-6 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data - for i in $(seq 0 2 18); do - f=onlinemeas.$(printf %06d $i); - numdiff -r 5e-4 ${f} ${REFPATH}/${f}; - done - fi diff --git a/.ci/include/cscs/02-test-templates.yml b/.ci/include/cscs/02-test-templates.yml new file mode 100644 index 000000000..d70a8a92c --- /dev/null +++ b/.ci/include/cscs/02-test-templates.yml @@ -0,0 +1,20 @@ +include: + - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml' + + +.test/base: + stage: test + image: ${UENV_NAME}/${UENV_VERSION}:${UENV_TAG} + variables: + WITH_UENV_VIEW: "default" + before_script: + - | + if test "${SLURM_LOCALID}" -eq "0"; then + tar xf ./builddir.tar -C / + touch preparation-done-${CI_JOB_ID} + fi + - while test ! -f preparation-done-${CI_JOB_ID}; do sleep 5; done + - bindir=$(echo /dev/shm/spack-stage/*/spack-stage-tmlqcd-cicd-*/spack-build-*/src/bin) + - libdir=$(dirname $(echo /dev/shm/spack-stage/*/spack-stage-quda-cicd-*/spack-build-*/lib/libquda.so)) + - export PATH=:${bindir}:$PATH + - export LD_LIBRARY_PATH=:${libdir}:$LD_LIBRARY_PATH diff --git a/.ci/spack_packages/lemonio/__pycache__/package.cpython-36.pyc b/.ci/spack_packages/lemonio/__pycache__/package.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5482e0babe4501307d8cd4423c88bfb473b4be43 GIT binary patch literal 1085 zcmah|&2AGh5VrRxyG;{XK}GE6rC#o*e)EsB)+stf+~fWY;=b^%-8A5>52zRXf>Tfr8~hZ6&vJS$ zdXDKF9053??3_-7*_enzC|Z|_S?|B!Z6s5$2$W)uQitQznlVjLLTstZ>(^84R5aDPP1==cdh34_|^?d+k6`$zHfq|$jd^Oi|50)N5`Z6XecMs z`jfCSI-!bYnaHAd$#Im)v7%}oos}vY(z=#RMDGvY#EZw%x~}wol1yYhosDBw=833t zmSpzm#gqRo$FOO!eRwkL@5O_DsrnhMg)*(2YS2Jq3vMmg)L>@(@l0kMY+gODgmKx` zl?fo68az@rucYw{S$w1nuUK_JTYLahhae{C3;v8DaXk8lpfsKq*_rXEnrO8J>y7x$ zl9?7tClgiXv`8)(XC)U&oev@3BxW+1iQ=-a1e2A}V9&rM8H-DyWYN#%-e8a@QI$!x zthrt?A#V+FH8=NIS)9qqObKGIe`G_nO@Oe67dxw}0GI1dqu!6jCI?y(AVmXLy&1zXRniD;TvgSG_D{2U> zB)NQ+MtAK{@N`kSdzV|>JUV5GZ1PFF&5n3=T@&HI(lg^O;Xgi9mh6BQW6u2q)EzhFq1_z*Ny))7C#q+{}v{GNq`^1<2K=i1SFb5hkhkfv?N8cVoFS;c+8Aj2_?bcxS6z4 zN{YdvneGTX0!pCdj{-_!2`6ysB%w^9R9jd~{|s7%MKKV+T$F;@(w_DdH#PmK*2lqA zAKRE{-jD!)rZhqt=TNlOC|MlUsfzLMeE=T9<8H?U0Sk(V1Vuuk5<`*_M=>RV;!2Vl z)0dPKPNNh`pGcyBrw|x~rhlUFG@60`EQjY%2AxCmKfy48a~>_A3yd>^-a{ABB?ix; z%O@i6=g|A;19XMa=FsH{_`?bK>ID1|z#pSenD#kDF_~i-bPamGjy`3)4EhXxj&4NQ zO~Ag0utmVWjIdjP-9}lI`za1KWl$dQrHESqY&pVK09%c)HNe&*Yy+^(2)hH=-3a>% zuzL~qHDLE6>;Yg8Q8DU&3-D6JEdy4Gux-G0B5W71YJ`0Q*tZc@18gtC>VP#OtO?ko z2x|e>j<613`w{jSe0mUJPk{H`gr^kX{*H6cq2Hf~f4adu_&xdu;Hc;a?okbO0oOzP zKtSk`&=Q`|`M-%UkBK?d10RI(F^3+YegqF9_#85zj~t7fLmoj-`PbxMi+^qUo#<;G z4UvPMu~?f&1i2CHp-}`Mfep{m3l>B3=olHukFY-aNA!|o8T1OhM*oEV75y_EBl;Kg zhaUyyJVLNY{}{v{!RqrXkM_$&*>Gh|HjOUP$g$iT64}87K9*tkWnDA%k*Rsc&~|0R zmI3q(9m|hOWqD}JJI#GJ=YKNrJjcD8&vS0BZ+L@IH)jm<&!C&U|6nY@8T!x~%Z3sf z(Fev6&J)~o^Pbjs^GAhTA-9zCG?MH4{-?j8fcB`w2G~(j+y*77547RXRpu;DGfWfr zMwWqze_^lMksB>}t6k2nWJ{(taxtWzX?=k8rx>Z-;~<6b5jKYokdxpPTOarrnKjwf zwL2TxeqpoQTV26uwWE9y!AttanueD**74d}x4Vf@Z@IU=s%dL$g(ZDu19IV%1Ea%) zU=$?50hZzU38q$g*j??dudL~J?yM|ttSuFmHa3^AwsZ$=;?&7Qg@&&udVx^Qjcc`hSTeLS!dKW4L$1(vkoziG!N$- z&4b}c5AQ;g`11j_gM@9^hnhd5Tac+AuH*`vcY{QCWSEGGc*hP7L_LVZqvQ)>)-7-EpIKrDl7gT@q9tLeN}3)%p#@ z@`OQjQX<8j>UL{_Ic0&*jXr^NG0zo5lptjvxdRJZeulH^kK4O@l|7hM9l|L{($Rw_ z$Ab;g$jeFH93H{ix*AS;h{~K^AaKn51zWax~HyT(A-DtE?MX^xXcHP_aF5%|>#Ut$xM{64(G=MnTQ6ap?Hd zw@#yQ*6OPk8MvwmldnjHbwAq~U`QS`6nZif%mQC_4IjI*H_$vt44w?h)GYtj71+Qrk(mbY#~B51Tp;+_$e^*B zp^a3!dw&h92K(2VCWciERw6@({RR#ek%WAtnIjCF2Hh{nCfzg^Gz;~ekqWy!Ok}9y z^S4!;kqXuLQ?J2q@@tykmwC1to8`(lG@rUw2J@Mpy& zb^OaTD8k64beXoGaU7&tm0G3Pt^|p4<#DA(KU@1d!PIV}-3h^TyI8Mws!C;YCkc{l z#I7#FSm`eT&u8$I;Bo&$5`>q+p>QZ7@kAu6NO~nQTtcxAgu~b?3B?cNVU_h#Dhnqd z^(tn6%-f`(O$()?HdCxk47dpd<<5!3pxKuqhvr_!I0VHHhvw0F4nYaTAt-?ulth=H z1fp7!v;^YN74#v8uA+}P1jP=AWOR)~P~>n3iX0BzKsPz`1zP0Lm*^IUpuFKw4&^zt zgbEy5Mk^dzMQa>dM;jd4M0YrJ7k$N{d+2Kp-A4~N1SJfIwor*fWmMtNHrnCPE~;|q z8w&NWzlw9Oe|sYSStS4SGJcq1{;i?C!;mMTI-LQSC5ayEBZunR9!VT9O;146)Ej+#_p2O* z#?Kjheh1|gr$9;ZeVrc`lsL^A!Bn*Fe-Y(q1whOTP{s=`eQ0I>Quu!U6_oTZ#lHxD zl^zSC@C=H3&?hM|*HY_$11OMkOMz$*x@_3%mml(QYA0^KPvBT!=VAZRhY06As|Kg3 zd`>s=BWxdK3FK4IGv7Q)q)-qO>mY~#dhIvT(^k%{hs}PRmoa0}n?$Q1K zg+uQDftpI~jtl;ES~ca^Ng0w}-_6}KhfpE8_j8dr-vgA`ZnYWAoylCvd$inAxAv>G zvRbUw)J~<{X>Tj@jNB=r9CAD1HF7DSlYGZr5qnw{$lxmH~ zl@_@{8(e*huI@q>tZ(hNOS|OeTRIzi#cEwWfRT|eD05+=eXZE66-yPeNEshbFzV&X zj@mh>wmX%w%6tFv&EC%l?G@W?a*K+D{w%W*wL8U~ol2`xCbuc;gNZKQ(4M9I%kSVb z#W~8I^vFTA)^5~Ep3>8-73|66gHsf?YI0=)tv1Mq6b=1dt#>M|Z5X=RXjbZUGOki) zI8CR_Jt!Z@N0bwK?UeJV+$0~rs|l?f$S3b=0(AqCDdQru_lzzm8OXJFsiXA*xlTD@ zxa~oFRCij{GWnDeXPGL#(-OLiDKm^jhXo#hTgGwAtzzlhgJP@PCUk{!b^Gm#TB+_- z>ZCx4lS`q!TZDP5Rkz4R$~qq|kdUeZxjlJGRNfEgU~HXQe_SbnRJ-$wc$=(J1#e31 zwRYP~f-D33^xVd$nU6``ZR}MzX{X#OHJaZs5?MdBcf2Ci_Wt&E^+^yXSTlq9(q8e~ zids@j4G4`IET0W%5-f1?lP3`)$Q1X=R7tql+rhb0neAF}r|n>baDSHvqodk=a6D83L|5dCv+FsJ97C@^kRG{|2wL7?-F!;!;|gmS&~dm?X}| z=syjP5C0jFx(?{@&WQ8yoc)KCF<$~nNlZ)Q?^qV3q-n;RtV~yseCh5HOEko3d)V&Y{9~i#Hf>~ zpOw#ly9LBIp@;JShb*m&ND^d&Ii?qG(%8{~4e4Do6{PQl;dGxK|J`YLr4#dXQlxb3 Fe*ya2Z5RLm literal 0 HcmV?d00001 diff --git a/.ci/spack_packages/quda/package.py b/.ci/spack_packages/quda/package.py new file mode 100644 index 000000000..ad34b3fdb --- /dev/null +++ b/.ci/spack_packages/quda/package.py @@ -0,0 +1,203 @@ +# Copyright Spack Project Developers. See COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack_repo.builtin.build_systems.cmake import CMakePackage, generator +from spack_repo.builtin.build_systems.cuda import CudaPackage +from spack_repo.builtin.build_systems.rocm import ROCmPackage + +from spack.package import * + + +class Quda(CMakePackage, CudaPackage, ROCmPackage): + """QUDA is a library for performing calculations in lattice QCD on GPUs.""" + + homepage = "https://lattice.github.io/quda/" + url = "https://github.com/lattice/quda/archive/refs/tags/v1.1.0.tar.gz" + git = "https://github.com/lattice/quda.git" + + tags = ["hep", "lattice"] + + maintainers("chaoos", "mtaillefumier") + + license("MIT OR BSD-3-Clause", checked_by="chaoos") + + version("develop", branch="develop") + + # git describe --tags --match 'v*' 18bf43ed40c75ae276e55bb8ddf2f64aa5510c37 + version( + "1.1.0-4597-g18bf43ed4", preferred=True, commit="18bf43ed40c75ae276e55bb8ddf2f64aa5510c37" + ) + + version("1.1.0", sha256="b4f635c993275010780ea09d8e593e0713a6ca1af1db6cc86c64518714fcc745") + + # don't expose ${ROCM_PATH}/include/hipfft explicitly, spack finds it alone + patch("quda-hipfft-public-to-private.patch", when="@:") + + # build dependencies + generator("ninja") + depends_on("cmake@3.18:", type="build") + depends_on("ninja", type="build") + depends_on("c", type="build") + depends_on("cxx", type="build") + depends_on("fortran", type="build", when="+tifr") + depends_on("fortran", type="build", when="+bqcd") + + variant("shared", default=True, description="Build shared libraries") + variant( + "backwards", default=False, description="Enable stacktrace generation using backwards-cpp" + ) + variant("mpi", default=False, description="Enable MPI support") + variant("qmp", default=False, description="Enable QMP") + variant("qio", default=False, description="Enable QIO", when="+qmp") + variant("openqcd", default=False, description="Enable openQCD interface") + variant("milc", default=False, description="Enable MILC interface") + variant("qdp", default=False, description="Enable QDP interface") + variant("bqcd", default=False, description="Enable BQCD interface") + variant("cps", default=False, description="Enable CPS interface") + variant("qdpjit", default=False, description="Enable QDPJIT interface") + variant("tifr", default=False, description="Enable TIFR interface") + variant("multigrid", default=False, description="Enable multigrid") + variant("nvshmem", default=False, description="Enable NVSHMEM", when="+cuda") + variant("openmp", default=False, description="Enable openmp support") + variant("clover", default=False, description="Build clover Dirac operators") + variant( + "clover_hasenbusch", default=False, description="Build clover Hasenbusch twist operators" + ) + variant("domain_wall", default=False, description="Build domain wall Dirac operators") + variant("laplace", default=False, description="Build laplace operator") + variant( + "ndeg_twisted_clover", + default=False, + description="Build non-degenerate twisted clover Dirac operators", + ) + variant( + "ndeg_twisted_mass", + default=False, + description="Build non-degenerate twisted mass Dirac operators", + ) + variant("staggered", default=False, description="Build staggered Dirac operators") + variant("twisted_clover", default=False, description="Build twisted clover Dirac operators") + variant("twisted_mass", default=False, description="Build twisted mass Dirac operators") + variant("wilson", default=True, description="Build Wilson Dirac operators") + variant("usqcd", default=False, description="Download and build usqcd", when="+qmp") + variant("eigen", default=True, description="Enable eigen support") + + with when("+multigrid"): + variant( + "mg_mrhs_list", + default="16", + multi=True, + description="The list of multi-rhs sizes that get compiled", + ) + variant( + "mg_nvec_list", + default="6,24,32", + multi=True, + description="The list of null space vector sizes that get compiled", + ) + + # dependencies + depends_on("mpi", when="+mpi") + depends_on("cuda", when="+cuda") + depends_on("nvshmem", when="+nvshmem") + depends_on("gdrcopy", when="+nvshmem") + + with when("+rocm"): + depends_on("hip") + depends_on("hipblas") + depends_on("hipfft") + depends_on("hiprand") + depends_on("hipcub") + + conflicts("+qmp +mpi", msg="Specifying both QMP and MPI might result in undefined behavior") + conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive") + conflicts("~cuda ~rocm", msg="Either CUDA or ROCm support is required") + conflicts("cuda_arch=none", when="+cuda", msg="Please indicate a cuda_arch value") + conflicts("amdgpu_target=none", when="+rocm", msg="Please indicate a amdgpu_target value") + conflicts( + "+nvshmem", when="~mpi ~qmp", msg="NVSHMEM requires either +mpi or +qmp to be enabled" + ) + + # CMAKE_BUILD_TYPE + variant( + "build_type", + default="STRICT", + description="The build type to build", + values=("STRICT", "RELEASE", "DEVEL", "DEBUG", "HOSTDEBUG", "SANITIZE"), + ) + + def cmake_args(self): + if self.spec.satisfies("+cuda"): + target = "CUDA" + cuda_archs = self.spec.variants["cuda_arch"].value + arch = " ".join(f"sm_{i}" for i in cuda_archs) + elif self.spec.satisfies("+rocm"): + target = "HIP" + arch = self.spec.variants["amdgpu_target"].value + + args = [ + self.define("QUDA_BUILD_ALL_TESTS", False), + self.define("QUDA_TARGET_TYPE", target), + self.define("QUDA_GPU_ARCH", arch), + self.define("QUDA_PRECISION", 14), + self.define("QUDA_RECONSTRUCT", 7), + self.define("QUDA_DOWNLOAD_USQCD", "usqcd"), + self.define("QUDA_DIRAC_DEFAULT_OFF", True), + self.define_from_variant("QUDA_DIRAC_CLOVER", "clover"), + self.define_from_variant("QUDA_DIRAC_CLOVER_HASENBUSCH", "clover_hasenbusch"), + self.define_from_variant("QUDA_DIRAC_DOMAIN_WALL", "domain_wall"), + self.define_from_variant("QUDA_DIRAC_LAPLACE", "laplace"), + self.define_from_variant("QUDA_DIRAC_NDEG_TWISTED_CLOVER", "ndeg_twisted_clover"), + self.define_from_variant("QUDA_DIRAC_NDEG_TWISTED_MASS", "ndeg_twisted_mass"), + self.define_from_variant("QUDA_DIRAC_STAGGERED", "staggered"), + self.define_from_variant("QUDA_DIRAC_TWISTED_CLOVER", "twisted_clover"), + self.define_from_variant("QUDA_DIRAC_TWISTED_MASS", "twisted_mass"), + self.define_from_variant("QUDA_DIRAC_WILSON", "wilson"), + self.define_from_variant("QUDA_MPI", "mpi"), + self.define_from_variant("QUDA_QMP", "qmp"), + self.define_from_variant("QUDA_QIO", "qio"), + self.define_from_variant("QUDA_INTERFACE_OPENQCD", "openqcd"), + self.define_from_variant("QUDA_INTERFACE_MILC", "milc"), + self.define_from_variant("QUDA_INTERFACE_QDP", "qdp"), + self.define_from_variant("QUDA_INTERFACE_BQCD", "bqcd"), + self.define_from_variant("QUDA_INTERFACE_CPS", "cps"), + self.define_from_variant("QUDA_INTERFACE_QDPJIT", "qdpjit"), + self.define_from_variant("QUDA_INTERFACE_TIFR", "tifr"), + self.define_from_variant("QUDA_MULTIGRID", "multigrid"), + self.define_from_variant("QUDA_NVSHMEM", "nvshmem"), + self.define_from_variant("QUDA_OPENMP", "openmp"), + self.define_from_variant("QUDA_BACKWARDS", "backwards"), + self.define_from_variant("QUDA_USE_EIGEN", "eigen"), + self.define_from_variant("QUDA_BUILD_SHAREDLIB", "shared"), + self.define_from_variant("BUILD_SHARED_LIBS", "shared"), + ] + if self.spec.satisfies("+multigrid"): + args.append( + self.define( + "QUDA_MULTIGRID_NVEC_LIST", ",".join(self.spec.variants["mg_nvec_list"].value) + ) + ) + args.append( + self.define( + "QUDA_MULTIGRID_MRHS_LIST", ",".join(self.spec.variants["mg_mrhs_list"].value) + ) + ) + + if self.spec.satisfies("+nvshmem"): + args.append(self.define("QUDA_NVSHMEM_HOME", self.spec["nvshmem"].prefix)) + args.append(self.define("QUDA_GDRCOPY_HOME", self.spec["gdrcopy"].prefix)) + + if self.spec.satisfies("+cuda"): + args.append(self.define("QUDA_GPU_ARCH_SUFFIX", "real")) # real or virtual + elif self.spec.satisfies("+rocm"): + args.append(self.define("CMAKE_C_COMPILER", self.spec["hip"].hipcc)) + args.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc)) + # args.append(self.define("ROCM_PATH", self.spec["hip"].prefix)) + + # required when building on a machine with no AMD GPU present + args.append(self.define("AMDGPU_TARGETS", arch)) + + # suppress _GLIBCXX17_DEPRECATED warnings when compiling c++17 + args.append(self.define("CMAKE_CXX_FLAGS", "-Wno-deprecated-declarations")) + return args diff --git a/.ci/spack_packages/quda/quda-hipfft-public-to-private.patch b/.ci/spack_packages/quda/quda-hipfft-public-to-private.patch new file mode 100644 index 000000000..127944195 --- /dev/null +++ b/.ci/spack_packages/quda/quda-hipfft-public-to-private.patch @@ -0,0 +1,13 @@ +diff --git a/lib/targets/hip/target_hip.cmake b/lib/targets/hip/target_hip.cmake +index 6bdca1127..b22a469a1 100644 +--- a/lib/targets/hip/target_hip.cmake ++++ b/lib/targets/hip/target_hip.cmake +@@ -132,7 +132,7 @@ set_source_files_properties( ${QUDA_CU_OBJS} PROPERTIES LANGUAGE HIP) + target_link_libraries(quda PUBLIC hip::hiprand roc::rocrand hip::hipcub roc::rocprim_hip) + target_link_libraries(quda PUBLIC roc::hipblas roc::rocblas) + +-target_include_directories(quda PUBLIC ${ROCM_PATH}/hipfft/include) ++target_include_directories(quda PRIVATE ${ROCM_PATH}/hipfft/include) + target_link_libraries(quda PUBLIC hip::hipfft) + + add_subdirectory(targets/hip) diff --git a/.ci/spack_packages/tmlqcd/__pycache__/package.cpython-36.pyc b/.ci/spack_packages/tmlqcd/__pycache__/package.cpython-36.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae94de8acfe128b02623f1c1db2e6ebcf57aa6b5 GIT binary patch literal 4290 zcma(V*>c;+5f^V#mStJK*Vy&3;c`jpu$@Y3?U)uNSy@rHVi`G5AZAEH!sP*|1F=)7 z(!RQq-^j=0C6Du(r@ZDdPw5#BA^E8C5`{*0Pxo|BPft%v3zbUoUl0H3{P(IP{ad>5 z%L4u!-pB`lBqY1iF8eC3>?wvKGhB65Pct+xV`Mn3xmhn~> zR3n2lBa3q=b0*6Y&Z8{<7ElhJ{Cik6E+F=d@FFUr5-Ow0dlg2CXaZeAlPPr>#->v0 z3XDys)K#EnQtBE|*U=4h^S#FBxdrs?3-leJze=U=A|3r6JwOkI^tUkji*&RK)ORUW z10TLmsX9=Nlxl+HESgKLj_CQ6UI6Jwsq|y?1pNU$1+SjHhuy{{^cx90?TNB@o_^zk(lp(@5gl-vapv@HDLfZnL zKs#sh-{!%?GTIe0>|N0M9r!Zxq>NsHFC2d*q5UmsTiRz@uaPn0qJ7B4Kho^3=f1H~ za`%N5VcmAEDAIcY(Yr&(MULOsW6#}MY&VkmK^%wCuba)|7)` z{$)tImOW@jK`%bG2yP5OFKLZ+`c6C;b{lr!HF50O&DdLmNt+h22hI_065NZLvDJ?N zbL)@mbM+^U*dmR7lGHvkg%?|n>*C(fb1)HeHNZqNxw5%dUuewM15$Ub7!z9gK$d1b zD*`qgqHLE~zCAE1{QQ~GFbo3{(~OI~z+X-NldFQcyuABLcYIhv&$6*G5n>WKup&P;!D8}DA$HEznIpF4x}#lAwS1)a ztzjSQJqNoeO0*itRaH`f>Om#|Q0;htedCfJ_+2-!4%mVF72t zQ(78WZqFPAE_tvAhs4s`4zX+@G?lw@i)8_(28V0%VpYjfc2Cy#~KT$*vl;`|J+ zLZZ0(J`Q}o;~W8@$e4g2n+QVeN07P^l^;=gj>_{?UZC;`RZdQ+dUBd1u|M>>nCL-I zKbPibf~8dmp5Q3L5yTjSKok?FJB%@Ea1n7zbSKh7i@^3^*GXWfz?r;@MC8^%MPUHHUyhY=Jku&sJg zKgrs47ouorU8tKz*0n+?60{H=_U*v$IenVrwZjRLY>i_{$#svsx;W}(Zn`F@OpPai z!(eTcF}3)Zoo-}8*@{xac3J#t;^QOSH^*|H<8j?%$BhEN_6CJD4@#HQCk{&T+!SnO zQs5X*uo23VY&|Km;#0Rm=lKF89wVw50=<+(rWu%F;2Hzh0aSBDXZY6)e8a$92JSI% zpMeJes+VZqG=0m%rb&yY=>=%$GQ4b>Zy?HIBxjl^uz{T)1Rf4qN2CskOI4`y5LCy8 z%*C|iSx^h%g(^sM5EGWofLO$_{Fr3G9xWmag%m|5j1<_efN~u1r1*zKeqeqv@V{hTeI24VAb*Kp0hAJ3zt_IG+7FO zzp03z8{sL*gCSO2qrU>k%bKFenySfpB`=q?vN9o$A4MKLdF33N5IhCs$8wYMByh{A z6pX2PwJgIUU|wbqr+Owwz=o+BT{~npoLVSUC>0A`hqnF znAkE1vY!SoyE+!(joQF{D;+>njg&KmV=B@FmVu^o#DkVoU%uUGV&3k{cxRL zezCW@);4!uwzikr=Gy9u9da9DY?OC9=HAYdxwh2VSSMx1DhgI-bCq0TjFMomTZDXd zk;6(I)!1oIwO+15tHE!Uu^MYN$mNSOZ*(@vBx8)N@4W0Rb;upYx-5*XwDwk(%*Bn( z*W~6!{>#?d^3LY!`o=c71$=RhQFY*ibaJuN`Vo{s4V+(F+NM*2v%jD6lH9mXUM!); z#HdXI<>j^3%1)B6_eSN^m>iG5Bz(-68gp16&GlAiX@{!NZy3|#0T4}xQgM)A-0jx( z>!kAM*>_F2#(~#}0%PWa;_CY1+FpCfY_D!t3-k)yeo&Xp9+VSP6eOxeA+~AO3Ry*> z*^vm%Kn!eP!wm+F49$kn4Lc`P1v50~_;BSQ3@1;Eu-k(^i1Z`!0zycZ7)T2kc?u*g zj=EVUj~$@<-a$kGa*w;`6n%Pe31=i)^3 bKa6g`XRMS)>~hFw%23szf-8A-`cMA{=ttPn literal 0 HcmV?d00001 diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/tmlqcd/package.py b/.ci/spack_packages/tmlqcd/package.py old mode 100644 new mode 100755 similarity index 72% rename from .ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/tmlqcd/package.py rename to .ci/spack_packages/tmlqcd/package.py index 6cb679f4f..5ed85c341 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/tmlqcd/package.py +++ b/.ci/spack_packages/tmlqcd/package.py @@ -15,12 +15,18 @@ class Tmlqcd(CMakePackage, CudaPackage, ROCmPackage): homepage = "https://www.itkp.uni-bonn.de/~urbach/software.html" url = "https://github.com/etmc/tmLQCD/archive/refs/tags/rel-5-1-6.tar.gz" - git = "https://github.com/etmc/tmLQCD.git" + + # todo: change this back to etmc as soon as cmake PR is merged + git = "https://github.com/mtaillefumier/tmLQCD.git" license("GPL-3.0-or-later") maintainers("mtaillefumier") version("master", branch="master") + # todo: remove this version as soon as + # https://github.com/etmc/tmLQCD/pull/664 is merged + version("cmake_support", branch="cmake_support") + variant("lemon", default=False, description="Enable the lemon backend") variant("mpi", default=True, description="Enable mpi support") variant("DDalphaAMG", default=False, description="Enable DAlphaAMG support") @@ -72,7 +78,7 @@ class Tmlqcd(CMakePackage, CudaPackage, ROCmPackage): # conflicts conflicts("+cuda", when="cuda_arch=none") conflicts("+rocm", when="amdgpu_target=none") -conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive") + conflicts("+cuda +rocm", msg="CUDA and ROCm support are mutually exclusive") # hard dependencies depends_on("c-lime") @@ -82,11 +88,13 @@ class Tmlqcd(CMakePackage, CudaPackage, ROCmPackage): # dependencies depends_on("mpi", when="+mpi") - depends_on("lemon-io", when="+lemon") + depends_on("lemonio", when="+lemon") + + depends_on("llvm-openmp", when="+rocm+openmp") with when("+quda"): depends_on( - "quda+twisted_mass+twisted_clover+clover+ndeg_twisted_clover+ndeg_twisted_mass+wilson+qdp+staggered+usqcd+multigrid" + "quda+shared+twisted_mass+twisted_clover+clover+ndeg_twisted_clover+ndeg_twisted_mass+wilson+qdp+multigrid" ) depends_on("quda+mpi", when="+mpi") @@ -98,19 +106,34 @@ class Tmlqcd(CMakePackage, CudaPackage, ROCmPackage): class CMakeBuilder(cmake.CMakeBuilder): def cmake_args(self): - spec = self.spec args = [ - self.define_from_variant("DBUILD_SHARED_LIBS", "shared"), + self.define_from_variant("BUILD_SHARED_LIBS", "shared"), self.define_from_variant("TM_USE_LEMON", "lemon"), self.define_from_variant("TM_USE_MPI", "mpi"), self.define_from_variant("TM_USE_QUDA", "quda"), self.define_from_variant("TM_USE_CUDA", "cuda"), - self.define_from_variant("TM_USE_HIP", "cuda"), + self.define_from_variant("TM_USE_HIP", "rocm"), self.define_from_variant("TM_USE_FFTW", "fftw"), - self.define_from_variant("TM_FIXEDVOLUME", "fixed_volume"), self.define_from_variant("TM_USE_OMP", "openmp"), self.define_from_variant("TM_USE_SHMEM", "shmem"), self.define_from_variant("TM_USE_GAUGE_COPY", "gauge_copy"), self.define_from_variant("TM_USE_HALFSPINOR", "half_spinor"), ] + + # Use hipcc is case of a ROCm build + if "+rocm" in self.spec: + hip = self.spec["hip"] + args.append(self.define("CMAKE_C_COMPILER", hip.hipcc)) + args.append(self.define("CMAKE_CXX_COMPILER", hip.hipcc)) + + # help hipcc find openmp + if "+openmp" in self.spec: + omp = self.spec["llvm-openmp"] + args.append(self.define("OpenMP_C_FLAGS", "-fopenmp")) + args.append(self.define("OpenMP_CXX_FLAGS", "-fopenmp")) + args.append(self.define("OpenMP_C_LIB_NAMES", "omp")) + args.append(self.define("OpenMP_CXX_LIB_NAMES", "omp")) + args.append(self.define("OpenMP_omp_LIBRARY", "{0}/libomp.so".format(omp.prefix.lib))) + args.append(self.define("OpenMP_CXX_INCLUDE_DIR", omp.prefix.include)) + return args diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/compilers.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/compilers.yaml new file mode 100644 index 000000000..38a8faa0b --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/compilers.yaml @@ -0,0 +1,4 @@ +gcc: + version: "13" +llvm-amdgpu: + version: "6.3.3" \ No newline at end of file diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml new file mode 100644 index 000000000..b4b3c6495 --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml @@ -0,0 +1,10 @@ +name: tmlqcd +store: /user-environment +spack: + repo: https://github.com/spack/spack.git + commit: releases/v1.1 + packages: + repo: https://github.com/spack/spack-packages.git + commit: 5f20b9190596e0b875141e8cee03f0d3847ad65c +description: "tmLQCD dependencies for CSCS CIon GH200." +version: 2 diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml new file mode 100755 index 000000000..1b861a3cb --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml @@ -0,0 +1,29 @@ +gcc-env: + compiler: [gcc, llvm-amdgpu] + network: + mpi: cray-mpich@8.1.32+rocm + specs: [ 'libfabric@2.3 +rocm' ] + unify: when_possible + duplicates: + strategy: full + specs: + # add GPU-specific packages here + - hip@6.3.3 ^mesa@23.3.6 + - llvm-amdgpu + # tmlqcd and quda are not required, since we build their newest commits in the + # build stage. Although, we want all their dependencies in the base uenv. + - tmlqcd@cmake_support +lemon +quda ^quda@develop +qdp +multigrid +twisted_clover +twisted_mass + - numdiff + variants: + - +mpi + - +rocm + - amdgpu_target=gfx942 + - amdgpu_target_sram_ecc=gfx942 + views: + default: + link: roots + exclude: ["llvm"] + uenv: + add_compilers: true + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/modules.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/modules.yaml new file mode 100644 index 000000000..623307b09 --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/modules.yaml @@ -0,0 +1,23 @@ +modules: + # Paths to check when creating modules for all module sets + prefix_inspections: + bin: + - PATH + lib: + - LD_LIBRARY_PATH + lib64: + - LD_LIBRARY_PATH + + default: + arch_folder: false + # Where to install modules + roots: + tcl: /user-environment/modules + tcl: + all: + autoload: none + hash_length: 0 + exclude_implicits: true + exclude: ['%gcc@7.5.0', 'gcc %gcc@7.5.0'] + projections: + all: '{name}/{version}' diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/post-install b/.ci/uenv-recipes/tmlqcd/beverin-mi300/post-install new file mode 100755 index 000000000..a5e932cc5 --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/post-install @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e + +# remove offending environment variables +jq '.views["default"].env.values.scalar["HIPCC_LINK_FLAGS_APPEND"]|="" | .views["default"].env.values.scalar["HIPCC_COMPILE_FLAGS_APPEND"]|="" ' /user-environment/meta/env.json > /tmp/env.json +# copy file back to destination +cp /tmp/env.json /user-environment/meta/env.json \ No newline at end of file diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/packages b/.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/packages new file mode 120000 index 000000000..1229fc196 --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/packages @@ -0,0 +1 @@ +../../../../spack_packages \ No newline at end of file diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/repo.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/repo.yaml new file mode 100644 index 000000000..f08fa46a4 --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/repo/repo.yaml @@ -0,0 +1,2 @@ +repo: + namespace: apps diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml b/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml index b15e4e7ad..f6b76048e 100644 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml +++ b/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml @@ -7,5 +7,5 @@ spack: repo: https://github.com/spack/spack-packages.git #commit: modules: true -description: "tmLQCD is a freely available software suite providing a set of tools to be used in lattice QCD simulations." +description: "tmLQCD dependencies for CSCS CI on MI300A." version: 2 diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml b/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml index fd4c1568f..cafba6582 100644 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml +++ b/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml @@ -5,14 +5,8 @@ gcc-env: specs: ['libfabric@2.4.0+cuda'] unify: true specs: - - python@3.12 + - tmlqcd@cmake_support +lemon +quda ^quda@develop +qdp +multigrid +twisted_clover +twisted_mass - numdiff - - quda@develop +qdp +multigrid +twisted_clover +twisted_mass - - lemonio - - c-lime - - openblas - - cmake@3.31 - - cuda variants: - +mpi - +cuda diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages b/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages new file mode 120000 index 000000000..39e779607 --- /dev/null +++ b/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages @@ -0,0 +1 @@ +../../../../spack_packages/ \ No newline at end of file diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/lemonio/package.py b/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/lemonio/package.py deleted file mode 100644 index ff3367b26..000000000 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/packages/lemonio/package.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright Spack Project Developers. See COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack_repo.builtin.build_systems import cmake -from spack_repo.builtin.build_systems.cmake import CMakePackage, generator - - -from spack.package import * - - -class Lemonio(CMakePackage): - """LEMON: Lightweight Parallel I/O library for Lattice QCD.""" - - homepage = "https://github.com/etmc/lemon" - git = "https://github.com/etmc/lemon.git" - license("GPL-3.0-or-later") - - version("master", branch="master") - variant("shared", default=False, description="Build shared library") - depends_on("libtool", type="build", when="@master build_system=cmake") - depends_on("cmake@3.28:", type="build", when="@master build_system=cmake") - - depends_on("mpi") - - generator("ninja") - - -class CMakeBuilder(cmake.CMakeBuilder): - def cmake_args(self): - spec = self.spec - args = [ - self.define_from_variant("DBUILD_SHARED_LIBS", "shared"), - ] - return args diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/repo.yaml b/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/repo.yaml index 7070c57de..f08fa46a4 100644 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/repo.yaml +++ b/.ci/uenv-recipes/tmlqcd/daint-gh200/repo/repo.yaml @@ -1,3 +1,2 @@ repo: namespace: apps - api: v2.2 \ No newline at end of file From ec794a4175753e61a3e84d86321da30bbefae75f Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 12:22:59 +0200 Subject: [PATCH 2/9] ncreases solver precision --- .../hmc-quda-cscs/onlinemeas.000000 | 32 +++++------ .../hmc-quda-cscs/onlinemeas.000002 | 36 ++++++------ .../hmc-quda-cscs/onlinemeas.000004 | 42 +++++++------- .../hmc-quda-cscs/onlinemeas.000006 | 52 ++++++++--------- .../hmc-quda-cscs/onlinemeas.000008 | 42 +++++++------- .../hmc-quda-cscs/onlinemeas.000010 | 50 ++++++++--------- .../hmc-quda-cscs/onlinemeas.000012 | 44 +++++++-------- .../hmc-quda-cscs/onlinemeas.000014 | 42 +++++++------- .../hmc-quda-cscs/onlinemeas.000016 | 52 ++++++++--------- .../hmc-quda-cscs/onlinemeas.000018 | 56 +++++++++---------- doc/sample-output/hmc-quda-cscs/output.data | 40 ++++++------- 11 files changed, 244 insertions(+), 244 deletions(-) diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000000 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000000 index 10f608d9a..479908edd 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000000 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000000 @@ -13,13 +13,13 @@ 1 1 12 5.492661e-06 4.990176e-06 1 1 13 2.510807e-06 2.168667e-06 1 1 14 1.174275e-06 9.503277e-07 -1 1 15 5.504661e-07 4.274134e-07 +1 1 15 5.504660e-07 4.274133e-07 1 1 16 2.640490e-07 1.930577e-07 1 1 17 1.282607e-07 8.958758e-08 1 1 18 6.237032e-08 4.164187e-08 1 1 19 3.097786e-08 1.964061e-08 1 1 20 1.535375e-08 9.472014e-09 -1 1 21 7.619405e-09 4.712859e-09 +1 1 21 7.619404e-09 4.712859e-09 1 1 22 3.881572e-09 2.447911e-09 1 1 23 2.114137e-09 1.551426e-09 1 1 24 1.456717e-09 0.000000e+00 @@ -40,14 +40,14 @@ 2 1 14 7.891094e-07 -6.201568e-07 2 1 15 3.792138e-07 -2.811320e-07 2 1 16 1.828621e-07 -1.285597e-07 -2 1 17 8.968731e-08 -6.069332e-08 +2 1 17 8.968730e-08 -6.069332e-08 2 1 18 4.405744e-08 -2.858593e-08 2 1 19 2.211476e-08 -1.346975e-08 2 1 20 1.095733e-08 -6.528353e-09 2 1 21 5.382921e-09 -3.131290e-09 2 1 22 2.605178e-09 -1.407126e-09 -2 1 23 1.151035e-09 -4.352199e-10 -2 1 24 2.888924e-10 0.000000e+00 +2 1 23 1.151035e-09 -4.352200e-10 +2 1 24 2.888922e-10 0.000000e+00 6 1 0 3.539273e-01 0.000000e+00 6 1 1 2.987785e-02 -3.509622e-02 6 1 2 4.925621e-03 -6.237448e-03 @@ -61,15 +61,15 @@ 6 1 10 1.101650e-06 -8.365754e-07 6 1 11 5.076687e-07 -4.084558e-07 6 1 12 2.266688e-07 -1.750887e-07 -6 1 13 1.145216e-07 -7.272627e-08 -6 1 14 5.196569e-08 -3.460753e-08 +6 1 13 1.145216e-07 -7.272626e-08 +6 1 14 5.196569e-08 -3.460752e-08 6 1 15 2.547258e-08 -1.678207e-08 -6 1 16 1.297871e-08 -7.218226e-09 -6 1 17 6.754910e-09 -3.551694e-09 -6 1 18 3.366834e-09 -1.654749e-09 -6 1 19 1.769402e-09 -6.968345e-10 -6 1 20 9.566535e-10 -2.521800e-10 -6 1 21 5.374748e-10 -4.924400e-11 -6 1 22 3.326744e-10 5.379628e-11 -6 1 23 2.284239e-10 1.084086e-10 -6 1 24 1.607744e-10 0.000000e+00 +6 1 16 1.297871e-08 -7.218224e-09 +6 1 17 6.754911e-09 -3.551693e-09 +6 1 18 3.366835e-09 -1.654748e-09 +6 1 19 1.769403e-09 -6.968342e-10 +6 1 20 9.566543e-10 -2.521799e-10 +6 1 21 5.374754e-10 -4.924395e-11 +6 1 22 3.326749e-10 5.379632e-11 +6 1 23 2.284242e-10 1.084087e-10 +6 1 24 1.607746e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000002 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000002 index ab43eb0d0..3a99cbfe9 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000002 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000002 @@ -21,7 +21,7 @@ 1 1 20 2.926687e-08 9.556092e-09 1 1 21 1.473178e-08 4.625843e-09 1 1 22 7.447244e-09 2.524952e-09 -1 1 23 3.884716e-09 1.908626e-09 +1 1 23 3.884717e-09 1.908627e-09 1 1 24 2.294109e-09 0.000000e+00 2 1 0 -7.103018e-02 0.000000e+00 2 1 1 1.943510e+00 -1.928281e+00 @@ -40,13 +40,13 @@ 2 1 14 1.447009e-06 -6.753512e-07 2 1 15 7.087101e-07 -2.995620e-07 2 1 16 3.499277e-07 -1.354359e-07 -2 1 17 1.725383e-07 -6.172770e-08 +2 1 17 1.725383e-07 -6.172771e-08 2 1 18 8.596483e-08 -2.838038e-08 2 1 19 4.309581e-08 -1.330236e-08 -2 1 20 2.135009e-08 -6.178562e-09 +2 1 20 2.135009e-08 -6.178563e-09 2 1 21 1.072099e-08 -2.774220e-09 2 1 22 5.349358e-09 -1.010126e-09 -2 1 23 2.591416e-09 7.157582e-11 +2 1 23 2.591416e-09 7.157581e-11 2 1 24 1.084680e-09 0.000000e+00 6 1 0 8.146874e-02 0.000000e+00 6 1 1 3.071372e-02 -3.100596e-02 @@ -58,18 +58,18 @@ 6 1 7 1.449492e-05 -1.229745e-05 6 1 8 6.429511e-06 -4.768202e-06 6 1 9 2.818257e-06 -2.051921e-06 -6 1 10 1.311223e-06 -8.773974e-07 +6 1 10 1.311223e-06 -8.773973e-07 6 1 11 6.279936e-07 -3.184070e-07 -6 1 12 3.138525e-07 -1.429467e-07 -6 1 13 1.453985e-07 -6.260423e-08 -6 1 14 7.378119e-08 -2.334188e-08 -6 1 15 3.531083e-08 -1.149394e-08 -6 1 16 1.751972e-08 -4.929553e-09 -6 1 17 9.396155e-09 -1.848625e-09 -6 1 18 4.573290e-09 -6.508186e-10 -6 1 19 2.469320e-09 -7.657671e-11 -6 1 20 1.533960e-09 1.913244e-10 -6 1 21 9.474612e-10 3.066090e-10 -6 1 22 6.733063e-10 3.678591e-10 -6 1 23 5.246354e-10 4.128507e-10 -6 1 24 4.634131e-10 0.000000e+00 +6 1 12 3.138525e-07 -1.429466e-07 +6 1 13 1.453985e-07 -6.260422e-08 +6 1 14 7.378118e-08 -2.334187e-08 +6 1 15 3.531083e-08 -1.149393e-08 +6 1 16 1.751972e-08 -4.929550e-09 +6 1 17 9.396153e-09 -1.848623e-09 +6 1 18 4.573290e-09 -6.508173e-10 +6 1 19 2.469320e-09 -7.657590e-11 +6 1 20 1.533960e-09 1.913249e-10 +6 1 21 9.474618e-10 3.066094e-10 +6 1 22 6.733068e-10 3.678594e-10 +6 1 23 5.246358e-10 4.128511e-10 +6 1 24 4.634135e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000004 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000004 index f0ab253a5..49c8d30f9 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000004 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000004 @@ -18,11 +18,11 @@ 1 1 17 8.568119e-08 6.388395e-08 1 1 18 4.068229e-08 3.012431e-08 1 1 19 1.952869e-08 1.432809e-08 -1 1 20 9.420793e-09 6.936895e-09 +1 1 20 9.420794e-09 6.936896e-09 1 1 21 4.607251e-09 3.403440e-09 -1 1 22 2.302054e-09 1.745053e-09 +1 1 22 2.302054e-09 1.745054e-09 1 1 23 1.271915e-09 1.059947e-09 -1 1 24 9.182412e-10 0.000000e+00 +1 1 24 9.182417e-10 0.000000e+00 2 1 0 2.791085e-01 0.000000e+00 2 1 1 1.975256e+00 -1.947535e+00 2 1 2 2.637947e-01 -2.575891e-01 @@ -46,8 +46,8 @@ 2 1 20 6.527384e-09 -4.851827e-09 2 1 21 3.115434e-09 -2.339897e-09 2 1 22 1.450097e-09 -1.102457e-09 -2 1 23 5.839700e-10 -4.248999e-10 -2 1 24 5.777233e-11 0.000000e+00 +2 1 23 5.839701e-10 -4.249000e-10 +2 1 24 5.777230e-11 0.000000e+00 6 1 0 1.986460e-01 0.000000e+00 6 1 1 3.603496e-02 -3.279518e-02 6 1 2 5.093838e-03 -4.454695e-03 @@ -55,21 +55,21 @@ 6 1 4 2.507136e-04 -2.279460e-04 6 1 5 7.760255e-05 -5.988181e-05 6 1 6 2.552021e-05 -2.163111e-05 -6 1 7 8.505172e-06 -7.235137e-06 -6 1 8 3.506106e-06 -2.861361e-06 +6 1 7 8.505172e-06 -7.235136e-06 +6 1 8 3.506106e-06 -2.861360e-06 6 1 9 1.408006e-06 -1.446651e-06 -6 1 10 6.014566e-07 -5.493341e-07 -6 1 11 2.806749e-07 -2.195035e-07 +6 1 10 6.014566e-07 -5.493340e-07 +6 1 11 2.806748e-07 -2.195035e-07 6 1 12 1.138857e-07 -1.037624e-07 -6 1 13 4.946001e-08 -4.466437e-08 -6 1 14 2.163646e-08 -2.040975e-08 -6 1 15 9.663609e-09 -1.001374e-08 -6 1 16 4.935596e-09 -4.745932e-09 -6 1 17 2.587359e-09 -2.136077e-09 -6 1 18 1.345688e-09 -9.244505e-10 -6 1 19 6.666852e-10 -4.206024e-10 -6 1 20 3.625817e-10 -1.316187e-10 -6 1 21 2.363958e-10 -6.796692e-12 -6 1 22 1.689267e-10 5.409926e-11 -6 1 23 1.343482e-10 9.088591e-11 -6 1 24 1.100460e-10 0.000000e+00 +6 1 13 4.946000e-08 -4.466436e-08 +6 1 14 2.163645e-08 -2.040974e-08 +6 1 15 9.663606e-09 -1.001373e-08 +6 1 16 4.935594e-09 -4.745929e-09 +6 1 17 2.587358e-09 -2.136075e-09 +6 1 18 1.345687e-09 -9.244494e-10 +6 1 19 6.666851e-10 -4.206017e-10 +6 1 20 3.625817e-10 -1.316183e-10 +6 1 21 2.363959e-10 -6.796472e-12 +6 1 22 1.689268e-10 5.409941e-11 +6 1 23 1.343483e-10 9.088601e-11 +6 1 24 1.100461e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000006 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000006 index a71db420c..6e482e97f 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000006 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000006 @@ -12,17 +12,17 @@ 1 1 11 1.003966e-05 8.860619e-06 1 1 12 4.264168e-06 3.756218e-06 1 1 13 1.881049e-06 1.630955e-06 -1 1 14 8.362330e-07 7.096463e-07 +1 1 14 8.362329e-07 7.096463e-07 1 1 15 3.749180e-07 3.118861e-07 1 1 16 1.716453e-07 1.405501e-07 -1 1 17 7.942954e-08 6.441725e-08 -1 1 18 3.750754e-08 2.931846e-08 +1 1 17 7.942954e-08 6.441726e-08 +1 1 18 3.750754e-08 2.931847e-08 1 1 19 1.761604e-08 1.349287e-08 1 1 20 8.413554e-09 6.307100e-09 -1 1 21 4.129837e-09 3.030550e-09 +1 1 21 4.129837e-09 3.030551e-09 1 1 22 2.061332e-09 1.532140e-09 -1 1 23 1.135427e-09 9.177563e-10 -1 1 24 8.086135e-10 0.000000e+00 +1 1 23 1.135428e-09 9.177569e-10 +1 1 24 8.086140e-10 0.000000e+00 2 1 0 1.443813e-01 0.000000e+00 2 1 1 1.980432e+00 -1.955471e+00 2 1 2 2.688567e-01 -2.652889e-01 @@ -40,14 +40,14 @@ 2 1 14 5.480899e-07 -4.568376e-07 2 1 15 2.489206e-07 -2.030843e-07 2 1 16 1.153991e-07 -9.253461e-08 -2 1 17 5.375226e-08 -4.302536e-08 +2 1 17 5.375226e-08 -4.302537e-08 2 1 18 2.549536e-08 -1.978656e-08 -2 1 19 1.211562e-08 -9.187132e-09 +2 1 19 1.211562e-08 -9.187133e-09 2 1 20 5.797424e-09 -4.319513e-09 2 1 21 2.828983e-09 -2.029331e-09 -2 1 22 1.325369e-09 -9.138667e-10 -2 1 23 5.719788e-10 -3.172625e-10 -2 1 24 9.793438e-11 0.000000e+00 +2 1 22 1.325369e-09 -9.138670e-10 +2 1 23 5.719788e-10 -3.172627e-10 +2 1 24 9.793431e-11 0.000000e+00 6 1 0 7.540843e-02 0.000000e+00 6 1 1 3.188156e-02 -3.933759e-02 6 1 2 6.301936e-03 -5.250555e-03 @@ -57,19 +57,19 @@ 6 1 6 3.097306e-05 -2.316780e-05 6 1 7 1.151376e-05 -9.523560e-06 6 1 8 3.971446e-06 -3.396248e-06 -6 1 9 1.837776e-06 -1.185310e-06 +6 1 9 1.837776e-06 -1.185309e-06 6 1 10 8.012363e-07 -5.070785e-07 -6 1 11 3.656580e-07 -2.173167e-07 -6 1 12 1.530370e-07 -9.173622e-08 -6 1 13 6.774223e-08 -3.741388e-08 -6 1 14 3.348654e-08 -1.779523e-08 -6 1 15 1.459596e-08 -8.265521e-09 -6 1 16 6.816263e-09 -3.397408e-09 -6 1 17 3.597149e-09 -1.667427e-09 -6 1 18 1.816675e-09 -6.323436e-10 -6 1 19 9.655677e-10 -2.682715e-10 -6 1 20 5.579683e-10 -2.653917e-11 -6 1 21 3.689427e-10 8.701888e-11 -6 1 22 2.715191e-10 1.359774e-10 -6 1 23 2.226442e-10 1.691270e-10 -6 1 24 1.956150e-10 0.000000e+00 +6 1 11 3.656579e-07 -2.173167e-07 +6 1 12 1.530370e-07 -9.173620e-08 +6 1 13 6.774222e-08 -3.741387e-08 +6 1 14 3.348653e-08 -1.779522e-08 +6 1 15 1.459596e-08 -8.265515e-09 +6 1 16 6.816262e-09 -3.397404e-09 +6 1 17 3.597148e-09 -1.667424e-09 +6 1 18 1.816675e-09 -6.323421e-10 +6 1 19 9.655680e-10 -2.682704e-10 +6 1 20 5.579687e-10 -2.653855e-11 +6 1 21 3.689431e-10 8.701927e-11 +6 1 22 2.715194e-10 1.359777e-10 +6 1 23 2.226445e-10 1.691273e-10 +6 1 24 1.956152e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000008 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000008 index 89c891a26..b5edf210e 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000008 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000008 @@ -19,10 +19,10 @@ 1 1 18 4.417130e-08 4.925120e-08 1 1 19 2.102784e-08 2.318244e-08 1 1 20 1.002160e-08 1.112148e-08 -1 1 21 4.832782e-09 5.393734e-09 -1 1 22 2.419419e-09 2.727151e-09 -1 1 23 1.422427e-09 1.524056e-09 -1 1 24 1.154672e-09 0.000000e+00 +1 1 21 4.832782e-09 5.393735e-09 +1 1 22 2.419420e-09 2.727152e-09 +1 1 23 1.422428e-09 1.524057e-09 +1 1 24 1.154673e-09 0.000000e+00 2 1 0 6.998337e-02 0.000000e+00 2 1 1 1.964645e+00 -1.980538e+00 2 1 2 2.629194e-01 -2.728318e-01 @@ -43,11 +43,11 @@ 2 1 17 6.332855e-08 -7.219078e-08 2 1 18 3.009555e-08 -3.396179e-08 2 1 19 1.441408e-08 -1.610918e-08 -2 1 20 6.830602e-09 -7.765465e-09 -2 1 21 3.194213e-09 -3.738973e-09 +2 1 20 6.830602e-09 -7.765466e-09 +2 1 21 3.194213e-09 -3.738974e-09 2 1 22 1.442648e-09 -1.806013e-09 -2 1 23 5.284978e-10 -7.584535e-10 -2 1 24 -1.000620e-10 0.000000e+00 +2 1 23 5.284977e-10 -7.584538e-10 +2 1 24 -1.000622e-10 0.000000e+00 6 1 0 3.437692e-02 0.000000e+00 6 1 1 2.916716e-02 -3.756462e-02 6 1 2 5.273848e-03 -6.595382e-03 @@ -58,18 +58,18 @@ 6 1 7 9.428820e-06 -1.230345e-05 6 1 8 3.894521e-06 -4.900977e-06 6 1 9 1.757259e-06 -2.047862e-06 -6 1 10 8.514413e-07 -9.191270e-07 +6 1 10 8.514413e-07 -9.191269e-07 6 1 11 3.402850e-07 -4.291271e-07 6 1 12 1.517852e-07 -1.796169e-07 -6 1 13 7.036894e-08 -7.980512e-08 -6 1 14 3.544763e-08 -3.742752e-08 -6 1 15 1.653378e-08 -1.721321e-08 -6 1 16 8.082139e-09 -8.117061e-09 -6 1 17 4.000928e-09 -3.848533e-09 -6 1 18 2.112419e-09 -1.656637e-09 -6 1 19 1.223668e-09 -5.663624e-10 -6 1 20 7.498577e-10 -9.922179e-11 -6 1 21 5.508828e-10 1.136108e-10 -6 1 22 4.339090e-10 2.343745e-10 -6 1 23 3.769135e-10 2.946851e-10 -6 1 24 3.350230e-10 0.000000e+00 +6 1 13 7.036893e-08 -7.980510e-08 +6 1 14 3.544762e-08 -3.742750e-08 +6 1 15 1.653378e-08 -1.721320e-08 +6 1 16 8.082137e-09 -8.117056e-09 +6 1 17 4.000928e-09 -3.848529e-09 +6 1 18 2.112419e-09 -1.656635e-09 +6 1 19 1.223668e-09 -5.663612e-10 +6 1 20 7.498578e-10 -9.922107e-11 +6 1 21 5.508830e-10 1.136112e-10 +6 1 22 4.339091e-10 2.343747e-10 +6 1 23 3.769137e-10 2.946852e-10 +6 1 24 3.350232e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000010 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000010 index 27cc053a4..fbec54093 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000010 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000010 @@ -18,10 +18,10 @@ 1 1 17 5.480045e-08 1.397835e-07 1 1 18 2.543284e-08 6.737551e-08 1 1 19 1.183873e-08 3.267542e-08 -1 1 20 5.623177e-09 1.589759e-08 -1 1 21 2.739800e-09 7.773447e-09 -1 1 22 1.467770e-09 3.894322e-09 -1 1 23 1.038581e-09 2.004864e-09 +1 1 20 5.623178e-09 1.589760e-08 +1 1 21 2.739801e-09 7.773448e-09 +1 1 22 1.467771e-09 3.894323e-09 +1 1 23 1.038582e-09 2.004864e-09 1 1 24 1.190665e-09 0.000000e+00 2 1 0 -7.443101e-02 0.000000e+00 2 1 1 1.981456e+00 -1.973297e+00 @@ -42,34 +42,34 @@ 2 1 16 7.931581e-08 -2.013744e-07 2 1 17 3.702204e-08 -9.747260e-08 2 1 18 1.724736e-08 -4.716621e-08 -2 1 19 8.042751e-09 -2.302897e-08 +2 1 19 8.042752e-09 -2.302897e-08 2 1 20 3.818724e-09 -1.116915e-08 -2 1 21 1.792533e-09 -5.442671e-09 -2 1 22 7.596244e-10 -2.636097e-09 -2 1 23 1.532169e-10 -1.190922e-09 -2 1 24 -3.957785e-10 0.000000e+00 +2 1 21 1.792534e-09 -5.442672e-09 +2 1 22 7.596246e-10 -2.636097e-09 +2 1 23 1.532170e-10 -1.190922e-09 +2 1 24 -3.957786e-10 0.000000e+00 6 1 0 -3.734513e-02 0.000000e+00 6 1 1 3.382087e-02 -3.994453e-02 6 1 2 5.678437e-03 -5.309477e-03 6 1 3 9.612662e-04 -1.218598e-03 6 1 4 2.573361e-04 -2.851272e-04 -6 1 5 7.726855e-05 -8.453868e-05 +6 1 5 7.726855e-05 -8.453867e-05 6 1 6 2.345080e-05 -2.814289e-05 -6 1 7 6.848480e-06 -1.196483e-05 +6 1 7 6.848479e-06 -1.196483e-05 6 1 8 2.821777e-06 -4.652664e-06 6 1 9 1.299362e-06 -2.318196e-06 6 1 10 5.019566e-07 -1.062183e-06 -6 1 11 2.161586e-07 -4.585456e-07 -6 1 12 1.081199e-07 -2.140401e-07 -6 1 13 4.605144e-08 -1.067713e-07 -6 1 14 1.935861e-08 -5.053659e-08 -6 1 15 9.487326e-09 -2.478269e-08 -6 1 16 4.313067e-09 -1.232044e-08 -6 1 17 1.992949e-09 -5.896019e-09 -6 1 18 8.206148e-10 -2.922179e-09 -6 1 19 2.988785e-10 -1.570016e-09 -6 1 20 1.212380e-11 -8.799754e-10 -6 1 21 -1.201183e-10 -5.531506e-10 -6 1 22 -1.902858e-10 -4.001744e-10 -6 1 23 -2.289539e-10 -3.075422e-10 -6 1 24 -2.689842e-10 0.000000e+00 +6 1 11 2.161586e-07 -4.585455e-07 +6 1 12 1.081199e-07 -2.140400e-07 +6 1 13 4.605143e-08 -1.067713e-07 +6 1 14 1.935860e-08 -5.053658e-08 +6 1 15 9.487320e-09 -2.478268e-08 +6 1 16 4.313063e-09 -1.232043e-08 +6 1 17 1.992947e-09 -5.896017e-09 +6 1 18 8.206135e-10 -2.922178e-09 +6 1 19 2.988778e-10 -1.570015e-09 +6 1 20 1.212340e-11 -8.799755e-10 +6 1 21 -1.201186e-10 -5.531508e-10 +6 1 22 -1.902860e-10 -4.001746e-10 +6 1 23 -2.289541e-10 -3.075424e-10 +6 1 24 -2.689844e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000012 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000012 index f5a7d450a..6ba4d9e87 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000012 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000012 @@ -17,12 +17,12 @@ 1 1 16 1.224749e-07 7.142267e-08 1 1 17 5.575999e-08 2.981231e-08 1 1 18 2.552405e-08 1.235040e-08 -1 1 19 1.183355e-08 5.162969e-09 -1 1 20 5.580784e-09 2.196218e-09 -1 1 21 2.631965e-09 9.680013e-10 -1 1 22 1.267939e-09 4.706905e-10 -1 1 23 6.292653e-10 3.110587e-10 -1 1 24 3.624892e-10 0.000000e+00 +1 1 19 1.183355e-08 5.162970e-09 +1 1 20 5.580785e-09 2.196218e-09 +1 1 21 2.631965e-09 9.680015e-10 +1 1 22 1.267939e-09 4.706907e-10 +1 1 23 6.292657e-10 3.110589e-10 +1 1 24 3.624895e-10 0.000000e+00 2 1 0 2.227810e-01 0.000000e+00 2 1 1 2.004400e+00 -1.968699e+00 2 1 2 2.683424e-01 -2.698962e-01 @@ -41,13 +41,13 @@ 2 1 15 1.776590e-07 -1.011208e-07 2 1 16 8.162318e-08 -4.245626e-08 2 1 17 3.756884e-08 -1.780450e-08 -2 1 18 1.728172e-08 -7.436234e-09 +2 1 18 1.728172e-08 -7.436235e-09 2 1 19 8.108120e-09 -3.136631e-09 2 1 20 3.837685e-09 -1.335628e-09 2 1 21 1.804845e-09 -5.598238e-10 -2 1 22 8.608680e-10 -2.048715e-10 -2 1 23 3.943598e-10 -1.161601e-11 -2 1 24 1.534580e-10 0.000000e+00 +2 1 22 8.608682e-10 -2.048714e-10 +2 1 23 3.943600e-10 -1.161591e-11 +2 1 24 1.534582e-10 0.000000e+00 6 1 0 -4.607480e-02 0.000000e+00 6 1 1 2.859650e-02 -4.086807e-02 6 1 2 5.732395e-03 -5.159777e-03 @@ -58,18 +58,18 @@ 6 1 7 9.093599e-06 -9.994772e-06 6 1 8 3.895470e-06 -3.045631e-06 6 1 9 1.606808e-06 -9.814479e-07 -6 1 10 6.559298e-07 -4.248833e-07 +6 1 10 6.559297e-07 -4.248833e-07 6 1 11 2.668211e-07 -1.740468e-07 6 1 12 1.270907e-07 -7.550573e-08 -6 1 13 5.235510e-08 -2.810196e-08 -6 1 14 2.435970e-08 -9.778956e-09 +6 1 13 5.235508e-08 -2.810196e-08 +6 1 14 2.435969e-08 -9.778955e-09 6 1 15 1.172182e-08 -4.844785e-09 -6 1 16 5.209711e-09 -1.653986e-09 -6 1 17 2.572303e-09 -6.524137e-10 -6 1 18 1.192195e-09 -2.253211e-10 -6 1 19 5.705387e-10 -8.864369e-11 -6 1 20 3.074660e-10 -9.891358e-12 -6 1 21 1.670375e-10 2.307083e-11 -6 1 22 1.003302e-10 3.642122e-11 -6 1 23 6.844699e-11 4.430867e-11 -6 1 24 5.460662e-11 0.000000e+00 +6 1 16 5.209708e-09 -1.653985e-09 +6 1 17 2.572302e-09 -6.524135e-10 +6 1 18 1.192194e-09 -2.253209e-10 +6 1 19 5.705384e-10 -8.864358e-11 +6 1 20 3.074659e-10 -9.891271e-12 +6 1 21 1.670376e-10 2.307094e-11 +6 1 22 1.003303e-10 3.642133e-11 +6 1 23 6.844713e-11 4.430879e-11 +6 1 24 5.460677e-11 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000014 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000014 index 5885d538d..ca5720926 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000014 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000014 @@ -18,11 +18,11 @@ 1 1 17 9.317581e-08 7.498321e-08 1 1 18 4.396164e-08 3.540867e-08 1 1 19 2.079624e-08 1.675954e-08 -1 1 20 9.887567e-09 8.028301e-09 -1 1 21 4.751653e-09 3.958087e-09 -1 1 22 2.341886e-09 1.993940e-09 -1 1 23 1.295038e-09 1.160237e-09 -1 1 24 9.622983e-10 0.000000e+00 +1 1 20 9.887567e-09 8.028302e-09 +1 1 21 4.751653e-09 3.958088e-09 +1 1 22 2.341887e-09 1.993941e-09 +1 1 23 1.295039e-09 1.160237e-09 +1 1 24 9.622988e-10 0.000000e+00 2 1 0 -1.113927e-02 0.000000e+00 2 1 1 2.027433e+00 -2.007558e+00 2 1 2 2.792690e-01 -2.759283e-01 @@ -40,14 +40,14 @@ 2 1 14 6.408779e-07 -5.063199e-07 2 1 15 2.907816e-07 -2.342977e-07 2 1 16 1.342175e-07 -1.091885e-07 -2 1 17 6.377705e-08 -5.145848e-08 +2 1 17 6.377704e-08 -5.145848e-08 2 1 18 3.025085e-08 -2.463078e-08 -2 1 19 1.438256e-08 -1.168876e-08 -2 1 20 6.792513e-09 -5.621675e-09 +2 1 19 1.438256e-08 -1.168877e-08 +2 1 20 6.792513e-09 -5.621676e-09 2 1 21 3.216379e-09 -2.717888e-09 2 1 22 1.480601e-09 -1.259061e-09 -2 1 23 5.878047e-10 -4.812858e-10 -2 1 24 2.971374e-11 0.000000e+00 +2 1 23 5.878045e-10 -4.812861e-10 +2 1 24 2.971356e-11 0.000000e+00 6 1 0 2.941174e-01 0.000000e+00 6 1 1 2.452445e-02 -3.505923e-02 6 1 2 5.129573e-03 -5.124339e-03 @@ -61,15 +61,15 @@ 6 1 10 6.843463e-07 -5.851427e-07 6 1 11 3.406545e-07 -2.445193e-07 6 1 12 1.379457e-07 -1.076819e-07 -6 1 13 7.005768e-08 -4.482594e-08 -6 1 14 3.053413e-08 -2.161933e-08 -6 1 15 1.305396e-08 -9.647185e-09 -6 1 16 6.349365e-09 -4.573920e-09 -6 1 17 2.953388e-09 -2.154212e-09 -6 1 18 1.563122e-09 -9.970633e-10 -6 1 19 6.861313e-10 -4.751218e-10 -6 1 20 3.742511e-10 -1.916018e-10 -6 1 21 2.103993e-10 -6.902335e-11 -6 1 22 1.218020e-10 -2.245122e-12 -6 1 23 7.920598e-11 2.723197e-11 +6 1 13 7.005768e-08 -4.482593e-08 +6 1 14 3.053413e-08 -2.161932e-08 +6 1 15 1.305396e-08 -9.647181e-09 +6 1 16 6.349365e-09 -4.573919e-09 +6 1 17 2.953388e-09 -2.154211e-09 +6 1 18 1.563122e-09 -9.970632e-10 +6 1 19 6.861316e-10 -4.751220e-10 +6 1 20 3.742514e-10 -1.916021e-10 +6 1 21 2.103995e-10 -6.902356e-11 +6 1 22 1.218022e-10 -2.245287e-12 +6 1 23 7.920607e-11 2.723188e-11 6 1 24 4.957548e-11 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000016 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000016 index 2b85f7575..735e1209a 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000016 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000016 @@ -15,14 +15,14 @@ 1 1 14 1.348218e-06 7.085645e-07 1 1 15 6.185843e-07 3.034401e-07 1 1 16 2.883066e-07 1.324012e-07 -1 1 17 1.339907e-07 5.708070e-08 -1 1 18 6.325561e-08 2.534272e-08 +1 1 17 1.339907e-07 5.708071e-08 +1 1 18 6.325560e-08 2.534272e-08 1 1 19 2.967819e-08 1.137509e-08 -1 1 20 1.426970e-08 5.194407e-09 -1 1 21 6.848167e-09 2.467781e-09 -1 1 22 3.306704e-09 1.289519e-09 -1 1 23 1.699400e-09 8.843615e-10 -1 1 24 1.001308e-09 0.000000e+00 +1 1 20 1.426970e-08 5.194408e-09 +1 1 21 6.848168e-09 2.467782e-09 +1 1 22 3.306705e-09 1.289519e-09 +1 1 23 1.699401e-09 8.843622e-10 +1 1 24 1.001309e-09 0.000000e+00 2 1 0 5.106437e-02 0.000000e+00 2 1 1 2.034021e+00 -2.043046e+00 2 1 2 2.810013e-01 -2.795533e-01 @@ -41,13 +41,13 @@ 2 1 15 4.179563e-07 -1.910400e-07 2 1 16 1.958454e-07 -8.370417e-08 2 1 17 9.196787e-08 -3.659937e-08 -2 1 18 4.375557e-08 -1.639871e-08 -2 1 19 2.052777e-08 -7.394699e-09 -2 1 20 9.933357e-09 -3.387176e-09 -2 1 21 4.733291e-09 -1.537310e-09 -2 1 22 2.237188e-09 -6.276473e-10 +2 1 18 4.375557e-08 -1.639872e-08 +2 1 19 2.052777e-08 -7.394700e-09 +2 1 20 9.933357e-09 -3.387177e-09 +2 1 21 4.733292e-09 -1.537310e-09 +2 1 22 2.237189e-09 -6.276475e-10 2 1 23 1.032030e-09 -1.056509e-10 -2 1 24 3.599125e-10 0.000000e+00 +2 1 24 3.599126e-10 0.000000e+00 6 1 0 -1.548986e-01 0.000000e+00 6 1 1 3.490088e-02 -3.490058e-02 6 1 2 6.452706e-03 -6.074149e-03 @@ -59,17 +59,17 @@ 6 1 8 5.042372e-06 -4.130278e-06 6 1 9 2.623904e-06 -1.944126e-06 6 1 10 1.126464e-06 -6.761730e-07 -6 1 11 4.903197e-07 -2.718746e-07 +6 1 11 4.903196e-07 -2.718746e-07 6 1 12 2.120057e-07 -1.160378e-07 -6 1 13 9.867097e-08 -4.432289e-08 -6 1 14 4.879353e-08 -2.179089e-08 -6 1 15 2.131044e-08 -8.928534e-09 -6 1 16 1.035324e-08 -3.980027e-09 -6 1 17 4.804036e-09 -1.371807e-09 -6 1 18 2.575925e-09 -5.081930e-10 -6 1 19 1.415083e-09 3.350693e-13 -6 1 20 8.848134e-10 1.886890e-10 -6 1 21 6.021059e-10 2.838427e-10 -6 1 22 4.843230e-10 3.360630e-10 -6 1 23 4.191977e-10 3.599491e-10 -6 1 24 3.811458e-10 0.000000e+00 +6 1 13 9.867096e-08 -4.432288e-08 +6 1 14 4.879352e-08 -2.179088e-08 +6 1 15 2.131044e-08 -8.928527e-09 +6 1 16 1.035324e-08 -3.980023e-09 +6 1 17 4.804036e-09 -1.371804e-09 +6 1 18 2.575925e-09 -5.081909e-10 +6 1 19 1.415083e-09 3.364389e-13 +6 1 20 8.848139e-10 1.886899e-10 +6 1 21 6.021064e-10 2.838433e-10 +6 1 22 4.843235e-10 3.360635e-10 +6 1 23 4.191981e-10 3.599495e-10 +6 1 24 3.811462e-10 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/onlinemeas.000018 b/doc/sample-output/hmc-quda-cscs/onlinemeas.000018 index 7fec988b6..5aa38195d 100644 --- a/doc/sample-output/hmc-quda-cscs/onlinemeas.000018 +++ b/doc/sample-output/hmc-quda-cscs/onlinemeas.000018 @@ -18,11 +18,11 @@ 1 1 17 4.466300e-08 3.928509e-08 1 1 18 2.029572e-08 1.796401e-08 1 1 19 9.276464e-09 8.307450e-09 -1 1 20 4.253179e-09 3.877159e-09 -1 1 21 1.986807e-09 1.836270e-09 -1 1 22 9.588521e-10 8.927816e-10 -1 1 23 5.200707e-10 4.911553e-10 -1 1 24 3.930194e-10 0.000000e+00 +1 1 20 4.253180e-09 3.877160e-09 +1 1 21 1.986808e-09 1.836270e-09 +1 1 22 9.588524e-10 8.927819e-10 +1 1 23 5.200710e-10 4.911557e-10 +1 1 24 3.930197e-10 0.000000e+00 2 1 0 2.923335e-01 0.000000e+00 2 1 1 2.075320e+00 -2.047021e+00 2 1 2 2.882222e-01 -2.832152e-01 @@ -36,18 +36,18 @@ 2 1 10 1.085987e-05 -9.043525e-06 2 1 11 4.413293e-06 -3.692645e-06 2 1 12 1.830618e-06 -1.558337e-06 -2 1 13 7.720092e-07 -6.603431e-07 +2 1 13 7.720091e-07 -6.603431e-07 2 1 14 3.280744e-07 -2.876250e-07 2 1 15 1.452749e-07 -1.271635e-07 2 1 16 6.475539e-08 -5.757872e-08 2 1 17 2.908135e-08 -2.595773e-08 2 1 18 1.333627e-08 -1.205876e-08 -2 1 19 6.184378e-09 -5.620864e-09 -2 1 20 2.823173e-09 -2.652221e-09 -2 1 21 1.297236e-09 -1.247362e-09 -2 1 22 5.738562e-10 -5.732724e-10 -2 1 23 2.098344e-10 -2.317509e-10 -2 1 24 -1.424961e-11 0.000000e+00 +2 1 19 6.184379e-09 -5.620864e-09 +2 1 20 2.823174e-09 -2.652221e-09 +2 1 21 1.297237e-09 -1.247362e-09 +2 1 22 5.738563e-10 -5.732726e-10 +2 1 23 2.098344e-10 -2.317510e-10 +2 1 24 -1.424964e-11 0.000000e+00 6 1 0 7.875815e-02 0.000000e+00 6 1 1 3.844783e-02 -3.422169e-02 6 1 2 6.448041e-03 -5.712296e-03 @@ -56,20 +56,20 @@ 6 1 5 7.439429e-05 -5.843208e-05 6 1 6 2.660783e-05 -1.891933e-05 6 1 7 8.967594e-06 -7.264131e-06 -6 1 8 3.449015e-06 -2.689642e-06 +6 1 8 3.449014e-06 -2.689642e-06 6 1 9 1.538211e-06 -9.576270e-07 -6 1 10 5.705715e-07 -3.918583e-07 -6 1 11 2.278308e-07 -1.414196e-07 -6 1 12 9.978928e-08 -5.952328e-08 -6 1 13 3.998643e-08 -3.188154e-08 -6 1 14 1.925299e-08 -1.368451e-08 -6 1 15 8.628437e-09 -6.392641e-09 -6 1 16 3.238905e-09 -2.750349e-09 -6 1 17 1.453179e-09 -1.272178e-09 -6 1 18 7.410422e-10 -5.806722e-10 -6 1 19 3.619470e-10 -2.772995e-10 -6 1 20 1.592007e-10 -1.328018e-10 -6 1 21 6.720605e-11 -6.318018e-11 -6 1 22 2.886177e-11 -3.139596e-11 -6 1 23 8.682894e-12 -1.410129e-11 -6 1 24 -3.211380e-12 0.000000e+00 +6 1 10 5.705714e-07 -3.918583e-07 +6 1 11 2.278307e-07 -1.414195e-07 +6 1 12 9.978927e-08 -5.952326e-08 +6 1 13 3.998642e-08 -3.188153e-08 +6 1 14 1.925299e-08 -1.368450e-08 +6 1 15 8.628434e-09 -6.392639e-09 +6 1 16 3.238903e-09 -2.750348e-09 +6 1 17 1.453177e-09 -1.272177e-09 +6 1 18 7.410413e-10 -5.806718e-10 +6 1 19 3.619465e-10 -2.772993e-10 +6 1 20 1.592004e-10 -1.328017e-10 +6 1 21 6.720591e-11 -6.318014e-11 +6 1 22 2.886169e-11 -3.139597e-11 +6 1 23 8.682839e-12 -1.410132e-11 +6 1 24 -3.211418e-12 0.000000e+00 diff --git a/doc/sample-output/hmc-quda-cscs/output.data b/doc/sample-output/hmc-quda-cscs/output.data index a85381d5a..79bb89284 100644 --- a/doc/sample-output/hmc-quda-cscs/output.data +++ b/doc/sample-output/hmc-quda-cscs/output.data @@ -1,20 +1,20 @@ -00000000 0.125364873560 -0.608284764632 1.837277e+00 51 378 223 1260 198 765 6099 0 197 231 742 1362 50 138 896 918 1 5.915427e+02 5.344868e-02 -00000001 0.128408045279 -0.587905187218 1.800213e+00 51 378 223 1269 197 765 6107 0 197 230 742 1366 50 139 902 944 1 1.180847e+02 5.338762e-02 -00000002 0.131376873453 -0.572429344116 1.772568e+00 51 378 223 1242 197 765 6173 0 199 231 742 1360 50 138 903 933 1 1.179065e+02 5.331320e-02 -00000003 0.134343770965 -0.584832404420 1.794690e+00 51 378 223 1269 199 769 6124 0 197 230 749 1375 50 138 901 919 1 1.176003e+02 5.325953e-02 -00000004 0.137317403447 -0.585643463943 1.796146e+00 51 378 222 1260 200 774 6147 0 197 230 745 1373 50 138 906 917 1 1.178213e+02 5.324785e-02 -00000005 0.140199325404 -0.568929989910 1.766376e+00 51 375 223 1258 198 769 6138 0 197 230 742 1373 50 139 899 933 1 1.176954e+02 5.323685e-02 -00000006 0.143058265009 -0.561928249197 1.754051e+00 51 378 223 1252 196 765 6128 0 199 231 743 1371 50 138 905 906 1 1.177954e+02 5.324760e-02 -00000007 0.145909667142 -0.552890848281 1.738271e+00 51 376 222 1233 198 765 6115 0 198 235 739 1363 52 139 901 940 1 1.180213e+02 5.331030e-02 -00000008 0.148740825610 -0.555667092791 1.743103e+00 51 369 221 1225 197 765 6092 0 198 235 743 1366 52 140 900 925 1 1.176481e+02 5.335337e-02 -00000009 0.151521112100 -0.525475466100 1.691263e+00 51 369 221 1261 197 765 6086 0 197 231 739 1361 52 141 892 941 1 1.178343e+02 5.342947e-02 -00000010 0.154303033049 -0.528955127898 1.697158e+00 51 369 221 1244 196 765 6078 0 197 231 739 1362 50 139 888 950 1 1.173622e+02 5.351707e-02 -00000011 0.157024587923 -0.516312118503 1.675836e+00 51 369 221 1253 196 765 6075 0 196 232 732 1362 50 138 893 921 1 1.175283e+02 5.363134e-02 -00000012 0.159753167297 -0.540551629267 1.716954e+00 51 369 221 1259 196 765 6044 0 196 230 736 1359 50 138 882 955 1 1.174025e+02 5.377451e-02 -00000013 0.162417581776 -0.522848537803 1.686826e+00 50 369 220 1243 196 764 6052 0 196 230 735 1354 50 138 890 926 1 1.176419e+02 5.391239e-02 -00000014 0.165052736983 -0.511975627975 1.668584e+00 50 369 218 1227 194 756 6000 0 195 230 727 1347 50 138 882 908 1 1.172012e+02 5.407580e-02 -00000015 0.167681842451 -0.501714109589 1.651550e+00 50 369 218 1237 194 755 5971 0 194 230 723 1337 50 138 878 918 1 1.170374e+02 5.424336e-02 -00000016 0.170294616387 -0.497908631107 1.645277e+00 50 369 218 1224 195 756 5947 0 194 230 725 1345 50 138 883 908 1 1.170327e+02 5.442650e-02 -00000017 0.172879304900 -0.497918661073 1.645293e+00 50 369 219 1209 195 760 6020 0 194 232 727 1342 50 138 884 895 1 1.174718e+02 5.464446e-02 -00000018 0.175409611123 -0.485990868910 1.625785e+00 50 369 220 1215 196 762 6013 0 195 230 735 1360 51 138 892 909 1 1.173493e+02 5.481913e-02 -00000019 0.177895449276 -0.502492832748 1.652836e+00 51 369 221 1234 196 765 6043 0 196 230 733 1364 50 138 893 910 1 1.169952e+02 5.502989e-02 +00000000 0.125364873560 -0.608296194026 1.837298e+00 53 441 233 1480 211 962 6662 0 207 254 797 1786 54 151 957 1269 1 1.183341e+02 5.344868e-02 +00000001 0.128408045279 -0.587907814595 1.800218e+00 53 441 233 1484 210 962 6672 0 206 255 798 1786 53 150 949 1272 1 7.287926e+01 5.338762e-02 +00000002 0.131376873453 -0.572439337702 1.772586e+00 53 439 232 1484 211 964 6720 0 208 260 796 1786 54 153 950 1276 1 7.541747e+01 5.331320e-02 +00000003 0.134343770965 -0.584835973568 1.794697e+00 53 439 233 1485 212 974 6678 0 207 258 804 1804 52 153 960 1272 1 7.310211e+01 5.325953e-02 +00000004 0.137317403447 -0.585653491697 1.796164e+00 53 440 232 1479 213 989 6713 0 206 255 801 1799 53 153 960 1270 1 7.396474e+01 5.324785e-02 +00000005 0.140199325404 -0.568942085607 1.766397e+00 53 441 233 1484 211 973 6700 0 206 255 799 1774 52 153 952 1263 1 7.379433e+01 5.323685e-02 +00000006 0.143058265009 -0.561929027463 1.754053e+00 53 438 233 1485 210 963 6674 0 207 255 805 1795 52 152 959 1264 1 7.345601e+01 5.324760e-02 +00000007 0.145909667142 -0.552882006537 1.738255e+00 53 438 232 1478 210 960 6669 0 206 260 797 1783 54 154 948 1276 1 7.290285e+01 5.331030e-02 +00000008 0.148740825610 -0.555670880683 1.743110e+00 53 433 231 1472 211 962 6627 0 207 260 796 1781 54 156 958 1271 1 7.423367e+01 5.335337e-02 +00000009 0.151521112101 -0.525481108285 1.691272e+00 53 432 232 1475 211 964 6635 0 207 260 794 1784 54 156 941 1267 1 7.258511e+01 5.342947e-02 +00000010 0.154303033049 -0.528965757141 1.697176e+00 53 432 231 1473 210 960 6629 0 206 256 794 1784 53 153 939 1263 1 7.382434e+01 5.351707e-02 +00000011 0.157024587923 -0.516314011533 1.675839e+00 53 432 232 1476 208 954 6627 0 205 250 791 1762 52 150 947 1254 1 7.505262e+01 5.363134e-02 +00000012 0.159753167297 -0.540551629412 1.716954e+00 53 432 232 1474 209 954 6601 0 206 252 791 1777 52 150 938 1240 1 7.374676e+01 5.377451e-02 +00000013 0.162417581776 -0.522851749847 1.686831e+00 53 432 230 1465 208 954 6598 0 206 250 790 1767 53 150 944 1256 1 7.180762e+01 5.391239e-02 +00000014 0.165052736983 -0.511977740389 1.668588e+00 52 432 229 1461 207 949 6533 0 206 250 782 1755 52 150 932 1229 1 7.283259e+01 5.407580e-02 +00000015 0.167681842451 -0.501711128629 1.651545e+00 52 432 228 1458 208 945 6506 0 204 250 780 1742 52 153 924 1225 1 7.208989e+01 5.424336e-02 +00000016 0.170294616387 -0.497907179029 1.645274e+00 52 432 228 1458 208 950 6477 0 204 255 782 1746 52 150 938 1236 1 7.246499e+01 5.442650e-02 +00000017 0.172879304900 -0.497921135451 1.645297e+00 53 432 229 1454 208 954 6578 0 204 255 784 1750 54 153 947 1221 1 7.361947e+01 5.464446e-02 +00000018 0.175409611123 -0.485992272967 1.625787e+00 53 432 230 1462 209 954 6544 0 205 255 789 1777 54 153 949 1235 1 7.350811e+01 5.481913e-02 +00000019 0.177895449276 -0.502494765824 1.652840e+00 53 432 231 1468 209 961 6576 0 206 255 789 1774 54 153 955 1242 1 7.508589e+01 5.502989e-02 From b7ce92431c70148a5a74cc44f066e6c53459cb2b Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 12:28:31 +0200 Subject: [PATCH 3/9] remove __pycache__ --- .../lemonio/__pycache__/package.cpython-36.pyc | Bin 1085 -> 0 bytes .../quda/__pycache__/package.cpython-36.pyc | Bin 6557 -> 0 bytes .../tmlqcd/__pycache__/package.cpython-36.pyc | Bin 4290 -> 0 bytes 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .ci/spack_packages/lemonio/__pycache__/package.cpython-36.pyc delete mode 100644 .ci/spack_packages/quda/__pycache__/package.cpython-36.pyc delete mode 100644 .ci/spack_packages/tmlqcd/__pycache__/package.cpython-36.pyc diff --git a/.ci/spack_packages/lemonio/__pycache__/package.cpython-36.pyc b/.ci/spack_packages/lemonio/__pycache__/package.cpython-36.pyc deleted file mode 100644 index 5482e0babe4501307d8cd4423c88bfb473b4be43..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1085 zcmah|&2AGh5VrRxyG;{XK}GE6rC#o*e)EsB)+stf+~fWY;=b^%-8A5>52zRXf>Tfr8~hZ6&vJS$ zdXDKF9053??3_-7*_enzC|Z|_S?|B!Z6s5$2$W)uQitQznlVjLLTstZ>(^84R5aDPP1==cdh34_|^?d+k6`$zHfq|$jd^Oi|50)N5`Z6XecMs z`jfCSI-!bYnaHAd$#Im)v7%}oos}vY(z=#RMDGvY#EZw%x~}wol1yYhosDBw=833t zmSpzm#gqRo$FOO!eRwkL@5O_DsrnhMg)*(2YS2Jq3vMmg)L>@(@l0kMY+gODgmKx` zl?fo68az@rucYw{S$w1nuUK_JTYLahhae{C3;v8DaXk8lpfsKq*_rXEnrO8J>y7x$ zl9?7tClgiXv`8)(XC)U&oev@3BxW+1iQ=-a1e2A}V9&rM8H-DyWYN#%-e8a@QI$!x zthrt?A#V+FH8=NIS)9qqObKGIe`G_nO@Oe67dxw}0GI1dqu!6jCI?y(AVmXLy&1zXRniD;TvgSG_D{2U> zB)NQ+MtAK{@N`kSdzV|>JUV5GZ1PFF&5n3=T@&HI(lg^O;Xgi9mh6BQW6u2q)EzhFq1_z*Ny))7C#q+{}v{GNq`^1<2K=i1SFb5hkhkfv?N8cVoFS;c+8Aj2_?bcxS6z4 zN{YdvneGTX0!pCdj{-_!2`6ysB%w^9R9jd~{|s7%MKKV+T$F;@(w_DdH#PmK*2lqA zAKRE{-jD!)rZhqt=TNlOC|MlUsfzLMeE=T9<8H?U0Sk(V1Vuuk5<`*_M=>RV;!2Vl z)0dPKPNNh`pGcyBrw|x~rhlUFG@60`EQjY%2AxCmKfy48a~>_A3yd>^-a{ABB?ix; z%O@i6=g|A;19XMa=FsH{_`?bK>ID1|z#pSenD#kDF_~i-bPamGjy`3)4EhXxj&4NQ zO~Ag0utmVWjIdjP-9}lI`za1KWl$dQrHESqY&pVK09%c)HNe&*Yy+^(2)hH=-3a>% zuzL~qHDLE6>;Yg8Q8DU&3-D6JEdy4Gux-G0B5W71YJ`0Q*tZc@18gtC>VP#OtO?ko z2x|e>j<613`w{jSe0mUJPk{H`gr^kX{*H6cq2Hf~f4adu_&xdu;Hc;a?okbO0oOzP zKtSk`&=Q`|`M-%UkBK?d10RI(F^3+YegqF9_#85zj~t7fLmoj-`PbxMi+^qUo#<;G z4UvPMu~?f&1i2CHp-}`Mfep{m3l>B3=olHukFY-aNA!|o8T1OhM*oEV75y_EBl;Kg zhaUyyJVLNY{}{v{!RqrXkM_$&*>Gh|HjOUP$g$iT64}87K9*tkWnDA%k*Rsc&~|0R zmI3q(9m|hOWqD}JJI#GJ=YKNrJjcD8&vS0BZ+L@IH)jm<&!C&U|6nY@8T!x~%Z3sf z(Fev6&J)~o^Pbjs^GAhTA-9zCG?MH4{-?j8fcB`w2G~(j+y*77547RXRpu;DGfWfr zMwWqze_^lMksB>}t6k2nWJ{(taxtWzX?=k8rx>Z-;~<6b5jKYokdxpPTOarrnKjwf zwL2TxeqpoQTV26uwWE9y!AttanueD**74d}x4Vf@Z@IU=s%dL$g(ZDu19IV%1Ea%) zU=$?50hZzU38q$g*j??dudL~J?yM|ttSuFmHa3^AwsZ$=;?&7Qg@&&udVx^Qjcc`hSTeLS!dKW4L$1(vkoziG!N$- z&4b}c5AQ;g`11j_gM@9^hnhd5Tac+AuH*`vcY{QCWSEGGc*hP7L_LVZqvQ)>)-7-EpIKrDl7gT@q9tLeN}3)%p#@ z@`OQjQX<8j>UL{_Ic0&*jXr^NG0zo5lptjvxdRJZeulH^kK4O@l|7hM9l|L{($Rw_ z$Ab;g$jeFH93H{ix*AS;h{~K^AaKn51zWax~HyT(A-DtE?MX^xXcHP_aF5%|>#Ut$xM{64(G=MnTQ6ap?Hd zw@#yQ*6OPk8MvwmldnjHbwAq~U`QS`6nZif%mQC_4IjI*H_$vt44w?h)GYtj71+Qrk(mbY#~B51Tp;+_$e^*B zp^a3!dw&h92K(2VCWciERw6@({RR#ek%WAtnIjCF2Hh{nCfzg^Gz;~ekqWy!Ok}9y z^S4!;kqXuLQ?J2q@@tykmwC1to8`(lG@rUw2J@Mpy& zb^OaTD8k64beXoGaU7&tm0G3Pt^|p4<#DA(KU@1d!PIV}-3h^TyI8Mws!C;YCkc{l z#I7#FSm`eT&u8$I;Bo&$5`>q+p>QZ7@kAu6NO~nQTtcxAgu~b?3B?cNVU_h#Dhnqd z^(tn6%-f`(O$()?HdCxk47dpd<<5!3pxKuqhvr_!I0VHHhvw0F4nYaTAt-?ulth=H z1fp7!v;^YN74#v8uA+}P1jP=AWOR)~P~>n3iX0BzKsPz`1zP0Lm*^IUpuFKw4&^zt zgbEy5Mk^dzMQa>dM;jd4M0YrJ7k$N{d+2Kp-A4~N1SJfIwor*fWmMtNHrnCPE~;|q z8w&NWzlw9Oe|sYSStS4SGJcq1{;i?C!;mMTI-LQSC5ayEBZunR9!VT9O;146)Ej+#_p2O* z#?Kjheh1|gr$9;ZeVrc`lsL^A!Bn*Fe-Y(q1whOTP{s=`eQ0I>Quu!U6_oTZ#lHxD zl^zSC@C=H3&?hM|*HY_$11OMkOMz$*x@_3%mml(QYA0^KPvBT!=VAZRhY06As|Kg3 zd`>s=BWxdK3FK4IGv7Q)q)-qO>mY~#dhIvT(^k%{hs}PRmoa0}n?$Q1K zg+uQDftpI~jtl;ES~ca^Ng0w}-_6}KhfpE8_j8dr-vgA`ZnYWAoylCvd$inAxAv>G zvRbUw)J~<{X>Tj@jNB=r9CAD1HF7DSlYGZr5qnw{$lxmH~ zl@_@{8(e*huI@q>tZ(hNOS|OeTRIzi#cEwWfRT|eD05+=eXZE66-yPeNEshbFzV&X zj@mh>wmX%w%6tFv&EC%l?G@W?a*K+D{w%W*wL8U~ol2`xCbuc;gNZKQ(4M9I%kSVb z#W~8I^vFTA)^5~Ep3>8-73|66gHsf?YI0=)tv1Mq6b=1dt#>M|Z5X=RXjbZUGOki) zI8CR_Jt!Z@N0bwK?UeJV+$0~rs|l?f$S3b=0(AqCDdQru_lzzm8OXJFsiXA*xlTD@ zxa~oFRCij{GWnDeXPGL#(-OLiDKm^jhXo#hTgGwAtzzlhgJP@PCUk{!b^Gm#TB+_- z>ZCx4lS`q!TZDP5Rkz4R$~qq|kdUeZxjlJGRNfEgU~HXQe_SbnRJ-$wc$=(J1#e31 zwRYP~f-D33^xVd$nU6``ZR}MzX{X#OHJaZs5?MdBcf2Ci_Wt&E^+^yXSTlq9(q8e~ zids@j4G4`IET0W%5-f1?lP3`)$Q1X=R7tql+rhb0neAF}r|n>baDSHvqodk=a6D83L|5dCv+FsJ97C@^kRG{|2wL7?-F!;!;|gmS&~dm?X}| z=syjP5C0jFx(?{@&WQ8yoc)KCF<$~nNlZ)Q?^qV3q-n;RtV~yseCh5HOEko3d)V&Y{9~i#Hf>~ zpOw#ly9LBIp@;JShb*m&ND^d&Ii?qG(%8{~4e4Do6{PQl;dGxK|J`YLr4#dXQlxb3 Fe*ya2Z5RLm diff --git a/.ci/spack_packages/tmlqcd/__pycache__/package.cpython-36.pyc b/.ci/spack_packages/tmlqcd/__pycache__/package.cpython-36.pyc deleted file mode 100644 index ae94de8acfe128b02623f1c1db2e6ebcf57aa6b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4290 zcma(V*>c;+5f^V#mStJK*Vy&3;c`jpu$@Y3?U)uNSy@rHVi`G5AZAEH!sP*|1F=)7 z(!RQq-^j=0C6Du(r@ZDdPw5#BA^E8C5`{*0Pxo|BPft%v3zbUoUl0H3{P(IP{ad>5 z%L4u!-pB`lBqY1iF8eC3>?wvKGhB65Pct+xV`Mn3xmhn~> zR3n2lBa3q=b0*6Y&Z8{<7ElhJ{Cik6E+F=d@FFUr5-Ow0dlg2CXaZeAlPPr>#->v0 z3XDys)K#EnQtBE|*U=4h^S#FBxdrs?3-leJze=U=A|3r6JwOkI^tUkji*&RK)ORUW z10TLmsX9=Nlxl+HESgKLj_CQ6UI6Jwsq|y?1pNU$1+SjHhuy{{^cx90?TNB@o_^zk(lp(@5gl-vapv@HDLfZnL zKs#sh-{!%?GTIe0>|N0M9r!Zxq>NsHFC2d*q5UmsTiRz@uaPn0qJ7B4Kho^3=f1H~ za`%N5VcmAEDAIcY(Yr&(MULOsW6#}MY&VkmK^%wCuba)|7)` z{$)tImOW@jK`%bG2yP5OFKLZ+`c6C;b{lr!HF50O&DdLmNt+h22hI_065NZLvDJ?N zbL)@mbM+^U*dmR7lGHvkg%?|n>*C(fb1)HeHNZqNxw5%dUuewM15$Ub7!z9gK$d1b zD*`qgqHLE~zCAE1{QQ~GFbo3{(~OI~z+X-NldFQcyuABLcYIhv&$6*G5n>WKup&P;!D8}DA$HEznIpF4x}#lAwS1)a ztzjSQJqNoeO0*itRaH`f>Om#|Q0;htedCfJ_+2-!4%mVF72t zQ(78WZqFPAE_tvAhs4s`4zX+@G?lw@i)8_(28V0%VpYjfc2Cy#~KT$*vl;`|J+ zLZZ0(J`Q}o;~W8@$e4g2n+QVeN07P^l^;=gj>_{?UZC;`RZdQ+dUBd1u|M>>nCL-I zKbPibf~8dmp5Q3L5yTjSKok?FJB%@Ea1n7zbSKh7i@^3^*GXWfz?r;@MC8^%MPUHHUyhY=Jku&sJg zKgrs47ouorU8tKz*0n+?60{H=_U*v$IenVrwZjRLY>i_{$#svsx;W}(Zn`F@OpPai z!(eTcF}3)Zoo-}8*@{xac3J#t;^QOSH^*|H<8j?%$BhEN_6CJD4@#HQCk{&T+!SnO zQs5X*uo23VY&|Km;#0Rm=lKF89wVw50=<+(rWu%F;2Hzh0aSBDXZY6)e8a$92JSI% zpMeJes+VZqG=0m%rb&yY=>=%$GQ4b>Zy?HIBxjl^uz{T)1Rf4qN2CskOI4`y5LCy8 z%*C|iSx^h%g(^sM5EGWofLO$_{Fr3G9xWmag%m|5j1<_efN~u1r1*zKeqeqv@V{hTeI24VAb*Kp0hAJ3zt_IG+7FO zzp03z8{sL*gCSO2qrU>k%bKFenySfpB`=q?vN9o$A4MKLdF33N5IhCs$8wYMByh{A z6pX2PwJgIUU|wbqr+Owwz=o+BT{~npoLVSUC>0A`hqnF znAkE1vY!SoyE+!(joQF{D;+>njg&KmV=B@FmVu^o#DkVoU%uUGV&3k{cxRL zezCW@);4!uwzikr=Gy9u9da9DY?OC9=HAYdxwh2VSSMx1DhgI-bCq0TjFMomTZDXd zk;6(I)!1oIwO+15tHE!Uu^MYN$mNSOZ*(@vBx8)N@4W0Rb;upYx-5*XwDwk(%*Bn( z*W~6!{>#?d^3LY!`o=c71$=RhQFY*ibaJuN`Vo{s4V+(F+NM*2v%jD6lH9mXUM!); z#HdXI<>j^3%1)B6_eSN^m>iG5Bz(-68gp16&GlAiX@{!NZy3|#0T4}xQgM)A-0jx( z>!kAM*>_F2#(~#}0%PWa;_CY1+FpCfY_D!t3-k)yeo&Xp9+VSP6eOxeA+~AO3Ry*> z*^vm%Kn!eP!wm+F49$kn4Lc`P1v50~_;BSQ3@1;Eu-k(^i1Z`!0zycZ7)T2kc?u*g zj=EVUj~$@<-a$kGa*w;`6n%Pe31=i)^3 bKa6g`XRMS)>~hFw%23szf-8A-`cMA{=ttPn From 8223a56e94e1c675139deb728eb9a2527d6b103f Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 12:34:50 +0200 Subject: [PATCH 4/9] use tmlqcd@master --- .ci/spack_packages/tmlqcd/package.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.ci/spack_packages/tmlqcd/package.py b/.ci/spack_packages/tmlqcd/package.py index 5ed85c341..ac318ab36 100755 --- a/.ci/spack_packages/tmlqcd/package.py +++ b/.ci/spack_packages/tmlqcd/package.py @@ -16,17 +16,12 @@ class Tmlqcd(CMakePackage, CudaPackage, ROCmPackage): homepage = "https://www.itkp.uni-bonn.de/~urbach/software.html" url = "https://github.com/etmc/tmLQCD/archive/refs/tags/rel-5-1-6.tar.gz" - # todo: change this back to etmc as soon as cmake PR is merged - git = "https://github.com/mtaillefumier/tmLQCD.git" + git = "https://github.com/etmc/tmLQCD.git" license("GPL-3.0-or-later") - maintainers("mtaillefumier") + maintainers("mtaillefumier", "chaoos") version("master", branch="master") - # todo: remove this version as soon as - # https://github.com/etmc/tmLQCD/pull/664 is merged - version("cmake_support", branch="cmake_support") - variant("lemon", default=False, description="Enable the lemon backend") variant("mpi", default=True, description="Enable mpi support") variant("DDalphaAMG", default=False, description="Enable DAlphaAMG support") From a12cc81431b01d6aab14eec160af1aa9372575b7 Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 12:35:25 +0200 Subject: [PATCH 5/9] use tmlqcd@master now --- .ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml | 2 +- .ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml index 1b861a3cb..f492d6b66 100755 --- a/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/environments.yaml @@ -12,7 +12,7 @@ gcc-env: - llvm-amdgpu # tmlqcd and quda are not required, since we build their newest commits in the # build stage. Although, we want all their dependencies in the base uenv. - - tmlqcd@cmake_support +lemon +quda ^quda@develop +qdp +multigrid +twisted_clover +twisted_mass + - tmlqcd@master +lemon +quda ^quda@develop +qdp +multigrid +twisted_clover +twisted_mass - numdiff variants: - +mpi diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml b/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml index cafba6582..d4de305e8 100644 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml +++ b/.ci/uenv-recipes/tmlqcd/daint-gh200/environments.yaml @@ -5,7 +5,7 @@ gcc-env: specs: ['libfabric@2.4.0+cuda'] unify: true specs: - - tmlqcd@cmake_support +lemon +quda ^quda@develop +qdp +multigrid +twisted_clover +twisted_mass + - tmlqcd@master +lemon +quda ^quda@develop +qdp +multigrid +twisted_clover +twisted_mass - numdiff variants: - +mpi From 057efd66f2e4efa76810a23a6a3aff45652a2f89 Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 12:39:38 +0200 Subject: [PATCH 6/9] description string fixed --- .ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml | 2 +- .ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml b/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml index b4b3c6495..b127f81b6 100644 --- a/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml +++ b/.ci/uenv-recipes/tmlqcd/beverin-mi300/config.yaml @@ -6,5 +6,5 @@ spack: packages: repo: https://github.com/spack/spack-packages.git commit: 5f20b9190596e0b875141e8cee03f0d3847ad65c -description: "tmLQCD dependencies for CSCS CIon GH200." +description: "tmLQCD dependencies for CSCS CIon MI300A." version: 2 diff --git a/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml b/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml index f6b76048e..6a54a230d 100644 --- a/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml +++ b/.ci/uenv-recipes/tmlqcd/daint-gh200/config.yaml @@ -7,5 +7,5 @@ spack: repo: https://github.com/spack/spack-packages.git #commit: modules: true -description: "tmLQCD dependencies for CSCS CI on MI300A." +description: "tmLQCD dependencies for CSCS CI on GH200." version: 2 From 5cd7205e44e8576641c1f9f2dbb5de6196da5c3b Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Tue, 7 Apr 2026 13:08:23 +0200 Subject: [PATCH 7/9] increase solver precision --- .../sample-hmc-quda-cscs-beverin.input | 274 ++++++++++++++++++ doc/sample-input/sample-hmc-quda-cscs.input | 34 +-- 2 files changed, 291 insertions(+), 17 deletions(-) create mode 100755 doc/sample-input/sample-hmc-quda-cscs-beverin.input diff --git a/doc/sample-input/sample-hmc-quda-cscs-beverin.input b/doc/sample-input/sample-hmc-quda-cscs-beverin.input new file mode 100755 index 000000000..0f0c92694 --- /dev/null +++ b/doc/sample-input/sample-hmc-quda-cscs-beverin.input @@ -0,0 +1,274 @@ +L=24 +T=48 + +NrXProcs = 2 +NrYProcs = 2 +NrZprocs = 1 + +ompnumthreads=32 + +BarrierMonomialsConverge = yes + +Measurements = 20 + +thermalisationsweeps = 0 + +seed=146555 + +Startcondition = hot +InitialStoreCounter = 0 + +2KappaMu = 0.0023801411000 +2KappaMuBar = 0.03875149727400 +2KappaEpsBar = 0.04103923289600 +CSW = 1.7112 +kappa = 0.1400083 +NSave = 10 +ThetaT = 1.0 +UseEvenOdd = yes +userelativeprecision=yes + +ReversibilityCheck = no +ReversibilityCheckIntervall = 10 + +DebugLevel = 2 + +ReproduceRandomNumbers = yes +RanluxdLevel = 2 + +BeginMeasurement CORRELATORS + Frequency = 2 +EndMeasurement + +BeginExternalInverter QUDA + Pipeline = 0 + gcrNkrylov = 24 + MGNumberOfLevels = 3 + MGNumberOfVectors = 24, 32 + MGSetupSolver = cg + MGSetup2KappaMu = 0.0023801411000 + MGVerbosity = silent, silent, silent + MGSetupSolverTolerance = 5e-7, 5e-7 + MGSetupMaxSolverIterations = 1500, 1500 + MGCoarseSolverType = gcr, gcr, cagcr + MGSmootherType = cagcr, cagcr, cagcr + MGBlockSizesX = 3,2 + MGBlockSizesY = 3,2 + MGBlockSizesZ = 4,3 + MGBlockSizesT = 3,2 + MGResetSetupMDUThreshold = 1.0 + MGRefreshSetupMDUThreshold = 0.0149 + MGRefreshSetupMaxSolverIterations = 40, 40 + + MGCoarseMuFactor = 1.4, 2.4, 110.0 + MGCoarseMaxSolverIterations = 45, 40, 10 + MGCoarseSolverTolerance = 0.1, 0.35, 0.45 + MGSmootherPostIterations = 2, 3, 6 + MGSmootherPreIterations = 2, 0, 1 + MGSmootherTolerance = 0.1, 0.1, 0.2 + MGOverUnderRelaxationFactor = 0.90, 0.85, 1.00 +EndExternalInverter + +BeginMonomial GAUGE + Type = Iwasaki + beta = 1.745 + Timescale = 0 + UseExternalLibrary = quda +EndMonomial + +BeginMonomial CLOVERDET + Timescale = 1 + kappa = 0.1400083 + 2KappaMu = 0.0023801411000 + CSW = 1.7112 + rho = 0.45 + MaxSolverIterations = 5000 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 + Name = cloverdetlight + solver= cg + UseExternalInverter = quda + UseSloppyPrecision = half + UseExternalLibrary = quda +EndMonomial + +BeginMonomial CLOVERDETRATIO + Timescale = 1 + kappa = 0.1400083 + 2KappaMu = 0.0023801411000 + rho = 0.030 + rho2 = 0.45 + CSW = 1.7112 + MaxSolverIterations = 1000 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 + Name = cloverdetratio1light + + solver= cg + UseExternalInverter = quda + UseSloppyPrecision = half + UseExternalLibrary = quda +EndMonomial + +BeginMonomial CLOVERDETRATIO + Timescale = 2 + kappa = 0.1400083 + 2KappaMu = 0.0023801411000 + rho = 0.0030 + # rho2 = 0.045 + rho2 = 0.030 + CSW = 1.7112 + MaxSolverIterations = 1000 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 + Name = cloverdetratio2light + solver = mg + UseExternalInverter = quda + UseSloppyPrecision = single + + HB_solver = cg + HB_usesloppyprecision = half + HB_UseExternalInverter = quda + HB_MaxSolverIterations = 3000 + UseExternalLibrary = quda +EndMonomial + + +BeginMonomial CLOVERDETRATIO + Timescale = 3 + kappa = 0.1400083 + 2KappaMu = 0.0023801411000 + rho = 0.0 + rho2 = 0.0030 + CSW = 1.7112 + MaxSolverIterations = 1000 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 + Name = cloverdetratio3light + solver = mg + UseExternalInverter = quda + UseSloppyPrecision = single + UseExternalLibrary = quda +EndMonomial + + +BeginMonomial NDCLOVERRAT + Timescale = 1 + kappa = 0.1400083 + CSW = 1.7112 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 + StildeMin = 0.0000376 + StildeMax = 4.7 + Name = ndcloverrat1 + DegreeOfRational = 10 + Cmin = 0 + Cmax = 3 + ComputeEVFreq = 0 + 2KappaMuBar = 0.03875149727400 + 2KappaEpsBar = 0.04103923289600 + AddTrLog = yes + solver= cgmmsnd + UseExternalInverter = quda + UseSloppyPrecision = single + RefinementPrecision = half + MaxSolverIterations = 5000 +EndMonomial + +BeginMonomial NDCLOVERRAT + Timescale = 2 + kappa = 0.1400083 + CSW = 1.7112 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 + StildeMin = 0.0000376 + StildeMax = 4.7 + Name = ndcloverrat2 + DegreeOfRational = 10 + Cmin = 4 + Cmax = 6 + ComputeEVFreq = 0 + 2KappaMuBar = 0.03875149727400 + 2KappaEpsBar = 0.04103923289600 + AddTrLog = no + solver= cgmmsnd + UseExternalInverter = quda + UseSloppyPrecision = single + RefinementPrecision = half + MaxSolverIterations = 5000 +EndMonomial + +BeginMonomial NDCLOVERRAT + Timescale = 3 + kappa = 0.1400083 + CSW = 1.7112 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 + StildeMin = 0.0000376 + StildeMax = 4.7 + Name = ndcloverrat3 + DegreeOfRational = 10 + Cmin = 7 + Cmax = 9 + ComputeEVFreq = 0 + 2KappaMuBar = 0.03875149727400 + 2KappaEpsBar = 0.04103923289600 + AddTrLog = no + solver= cgmmsnd + UseExternalInverter = quda + UseSloppyPrecision = single + RefinementPrecision = half + MaxSolverIterations = 5000 +EndMonomial + +BeginMonomial NDCLOVERRATCOR + Timescale = 1 + kappa = 0.1400083 + CSW = 1.7112 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 + StildeMin = 0.0000376 + StildeMax = 4.7 + Name = ndcloverratcor + DegreeOfRational = 10 + ComputeEVFreq = 0 + #UseExternalEigsolver = quda + 2KappaMuBar = 0.03875149727400 + 2KappaEpsBar = 0.04103923289600 + solver= cgmmsnd + UseExternalInverter = quda + UseSloppyPrecision = single + RefinementPrecision = half + MaxSolverIterations = 15000 +EndMonomial + +BeginIntegrator + Type0 = 2MN + Type1 = 2MN + Type2 = 2MN + Type3 = 2MN + IntegrationSteps0 = 1 + IntegrationSteps1 = 1 + IntegrationSteps2 = 1 + IntegrationSteps3 = 1 + tau = 0.03 + Lambda0 = 0.19318332750 + Lambda1 = 0.194 + Lambda2 = 0.196 + Lambda2 = 0.198 + NumberOfTimescales = 4 + MonitorForces = no +EndIntegrator + +BeginOperator CLOVER + CSW = 1.7112 + kappa = 0.1400083 + 2KappaMu = 0.0023801411000 + SolverPrecision = 1e-24 + useevenodd = yes + useexternalinverter = quda + usesloppyprecision = single + solver = mg + MaxSolverIterations = 500 +EndOperator + diff --git a/doc/sample-input/sample-hmc-quda-cscs.input b/doc/sample-input/sample-hmc-quda-cscs.input index a7bfc522a..104abb93c 100644 --- a/doc/sample-input/sample-hmc-quda-cscs.input +++ b/doc/sample-input/sample-hmc-quda-cscs.input @@ -83,8 +83,8 @@ BeginMonomial CLOVERDET CSW = 1.7112 rho = 0.45 MaxSolverIterations = 5000 - AcceptancePrecision = 1.e-23 - ForcePrecision = 1.e-19 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 Name = cloverdetlight solver= cg UseExternalInverter = quda @@ -100,8 +100,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.45 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-23 - ForcePrecision = 1.e-19 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 Name = cloverdetratio1light solver= cg @@ -119,8 +119,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.030 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-23 - ForcePrecision = 1.e-20 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 Name = cloverdetratio2light solver = mg UseExternalInverter = quda @@ -142,8 +142,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.0030 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-23 - ForcePrecision = 1.e-20 + AcceptancePrecision = 1.e-24 + ForcePrecision = 1.e-22 Name = cloverdetratio3light solver = mg UseExternalInverter = quda @@ -156,8 +156,8 @@ BeginMonomial NDCLOVERRAT Timescale = 1 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-23 - ForcePrecision = 1e-19 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat1 @@ -179,8 +179,8 @@ BeginMonomial NDCLOVERRAT Timescale = 2 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-23 - ForcePrecision = 1e-19 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat2 @@ -202,8 +202,8 @@ BeginMonomial NDCLOVERRAT Timescale = 3 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-23 - ForcePrecision = 1e-19 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat3 @@ -225,8 +225,8 @@ BeginMonomial NDCLOVERRATCOR Timescale = 1 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-23 - ForcePrecision = 1e-19 + AcceptancePrecision = 1e-24 + ForcePrecision = 1e-22 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverratcor @@ -264,7 +264,7 @@ BeginOperator CLOVER CSW = 1.7112 kappa = 0.1400083 2KappaMu = 0.0023801411000 - SolverPrecision = 1e-20 + SolverPrecision = 1e-24 useevenodd = yes useexternalinverter = quda usesloppyprecision = single From 18adc7423d2b1cbbf9694779ed43bd739136f2fd Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Wed, 8 Apr 2026 10:21:27 +0200 Subject: [PATCH 8/9] increase solver precision (again) --- .../sample-hmc-quda-cscs-beverin.input | 34 +++++++++---------- doc/sample-input/sample-hmc-quda-cscs.input | 34 +++++++++---------- 2 files changed, 34 insertions(+), 34 deletions(-) diff --git a/doc/sample-input/sample-hmc-quda-cscs-beverin.input b/doc/sample-input/sample-hmc-quda-cscs-beverin.input index 0f0c92694..b248acf56 100755 --- a/doc/sample-input/sample-hmc-quda-cscs-beverin.input +++ b/doc/sample-input/sample-hmc-quda-cscs-beverin.input @@ -83,8 +83,8 @@ BeginMonomial CLOVERDET CSW = 1.7112 rho = 0.45 MaxSolverIterations = 5000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetlight solver= cg UseExternalInverter = quda @@ -100,8 +100,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.45 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetratio1light solver= cg @@ -119,8 +119,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.030 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetratio2light solver = mg UseExternalInverter = quda @@ -142,8 +142,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.0030 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetratio3light solver = mg UseExternalInverter = quda @@ -156,8 +156,8 @@ BeginMonomial NDCLOVERRAT Timescale = 1 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat1 @@ -179,8 +179,8 @@ BeginMonomial NDCLOVERRAT Timescale = 2 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat2 @@ -202,8 +202,8 @@ BeginMonomial NDCLOVERRAT Timescale = 3 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat3 @@ -225,8 +225,8 @@ BeginMonomial NDCLOVERRATCOR Timescale = 1 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverratcor @@ -264,7 +264,7 @@ BeginOperator CLOVER CSW = 1.7112 kappa = 0.1400083 2KappaMu = 0.0023801411000 - SolverPrecision = 1e-24 + SolverPrecision = 1e-26 useevenodd = yes useexternalinverter = quda usesloppyprecision = single diff --git a/doc/sample-input/sample-hmc-quda-cscs.input b/doc/sample-input/sample-hmc-quda-cscs.input index 104abb93c..80fddb002 100644 --- a/doc/sample-input/sample-hmc-quda-cscs.input +++ b/doc/sample-input/sample-hmc-quda-cscs.input @@ -83,8 +83,8 @@ BeginMonomial CLOVERDET CSW = 1.7112 rho = 0.45 MaxSolverIterations = 5000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetlight solver= cg UseExternalInverter = quda @@ -100,8 +100,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.45 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetratio1light solver= cg @@ -119,8 +119,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.030 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetratio2light solver = mg UseExternalInverter = quda @@ -142,8 +142,8 @@ BeginMonomial CLOVERDETRATIO rho2 = 0.0030 CSW = 1.7112 MaxSolverIterations = 1000 - AcceptancePrecision = 1.e-24 - ForcePrecision = 1.e-22 + AcceptancePrecision = 1.e-26 + ForcePrecision = 1.e-24 Name = cloverdetratio3light solver = mg UseExternalInverter = quda @@ -156,8 +156,8 @@ BeginMonomial NDCLOVERRAT Timescale = 1 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat1 @@ -179,8 +179,8 @@ BeginMonomial NDCLOVERRAT Timescale = 2 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat2 @@ -202,8 +202,8 @@ BeginMonomial NDCLOVERRAT Timescale = 3 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverrat3 @@ -225,8 +225,8 @@ BeginMonomial NDCLOVERRATCOR Timescale = 1 kappa = 0.1400083 CSW = 1.7112 - AcceptancePrecision = 1e-24 - ForcePrecision = 1e-22 + AcceptancePrecision = 1e-26 + ForcePrecision = 1e-24 StildeMin = 0.0000376 StildeMax = 4.7 Name = ndcloverratcor @@ -264,7 +264,7 @@ BeginOperator CLOVER CSW = 1.7112 kappa = 0.1400083 2KappaMu = 0.0023801411000 - SolverPrecision = 1e-24 + SolverPrecision = 1e-26 useevenodd = yes useexternalinverter = quda usesloppyprecision = single From 4473001594d7a3739423f82d07c65729e0afb590 Mon Sep 17 00:00:00 2001 From: Roman Gruber Date: Thu, 9 Apr 2026 09:38:22 +0200 Subject: [PATCH 9/9] looser check precision --- .ci/cscs_beverin_pipeline.yml | 2 +- .ci/cscs_default_pipeline.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.ci/cscs_beverin_pipeline.yml b/.ci/cscs_beverin_pipeline.yml index aa980c2c8..295519625 100644 --- a/.ci/cscs_beverin_pipeline.yml +++ b/.ci/cscs_beverin_pipeline.yml @@ -44,7 +44,7 @@ test/beverin-mi300: - | if test "${SLURM_PROCID}" -eq "0"; then echo "Check the results on SLURM_PROCID=${SLURM_PROCID} ..." - numdiff -r 1.2e-6 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data + numdiff -r 1e-5 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data for i in $(seq 0 2 18); do f=onlinemeas.$(printf %06d $i); numdiff -r 5e-4 ${f} ${REFPATH}/${f}; diff --git a/.ci/cscs_default_pipeline.yml b/.ci/cscs_default_pipeline.yml index c5a336df6..58abe6728 100644 --- a/.ci/cscs_default_pipeline.yml +++ b/.ci/cscs_default_pipeline.yml @@ -44,7 +44,7 @@ test/daint-gh200: - | if test "${SLURM_PROCID}" -eq "0"; then echo "Check the results on SLURM_PROCID=${SLURM_PROCID} ..." - numdiff -r 1.2e-6 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data + numdiff -r 1e-5 -X 1:22 -X 1:5-21 -X 2:22 -X 2:5-21 output.data ${REFPATH}/output.data for i in $(seq 0 2 18); do f=onlinemeas.$(printf %06d $i); numdiff -r 5e-4 ${f} ${REFPATH}/${f};