1.4.0 Release Candidate

.clang-format

@@ -100,6 +100,8 @@ IncludeCategories:
     Priority: 2
   - Regex: '^"utils\/.*.h"'
     Priority: 2
+  - Regex: '^"traits\/.*.h"'
+    Priority: 2
   - Regex: '^"metrics\/.*\.h"'
     Priority: 3
   - Regex: '^"framework\/.*\.h"'
@@ -108,6 +110,10 @@ IncludeCategories:
     Priority: 4
   - Regex: '^"checkpoint\/.*\.h"'
     Priority: 4
+  - Regex: '^"kernels\/.*\.hpp"'
+    Priority: 4
+  - Regex: '^"kernels\/.*\.h"'
+    Priority: 4
   - Regex: '^"output\/.*\.h"'
     Priority: 4
   - Regex: '^"archetypes\/.*\.h"'

.clang-tidy

@@ -0,0 +1,36 @@
+Checks: >
+  -*,
+  bugprone-*,
+  performance-*,
+  clang-analyzer-*,
+  clang-diagnostic-*,
+  misc-include-cleaner,
+  misc-const-correctness,
+  misc-unused-parameters,
+  misc-unused-using-decls,
+  cppcoreguidelines-pro-type-member-init,
+  cppcoreguidelines-slicing,
+  cppcoreguidelines-virtual-class-destructor,
+  cppcoreguidelines-prefer-member-initializer,
+  modernize-use-nullptr,
+  modernize-use-override,
+  modernize-use-default-member-init,
+  modernize-use-emplace,
+  modernize-loop-convert,
+  modernize-make-unique,
+  modernize-make-shared,
+  modernize-use-using,
+  readability-container-size-empty,
+  readability-redundant-*,
+  readability-simplify-boolean-expr,
+  readability-misleading-indentation,
+  readability-inconsistent-declaration-parameter-name,
+  -misc-const-correctness,
+  -bugprone-easily-swappable-parameters,
+  -bugprone-unchecked-optional-access
+WarningsAsErrors: "*"
+HeaderFilterRegex: "src/.*"
+FormatStyle: file
+SystemHeaders: false
+CheckOptions:
+  misc-include-cleaner.IgnoreHeaders: 'adios2\.h;adios2/.*;Kokkos_.*\.hpp'

.gitattributes

@@ -0,0 +1,2 @@
+*.mp4 filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

.github/workflows/pgens-tests.yml

@@ -0,0 +1,70 @@
+name: Premerge compilation on CPUs for all pgens
+
+on:
+  push:
+  pull_request:
+    types: [ready_for_review]
+
+jobs:
+  check-commit:
+    runs-on: ubuntu-24.04
+    outputs:
+      run_tests: ${{ steps.check_message.outputs.run_tests }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Check commit message
+        id: check_message
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.action }}" == "ready_for_review" ]]; then
+            echo "run_tests=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if git log -1 --pretty=%B | grep -q "RUNPGENS"; then
+            echo "run_tests=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "run_tests=false" >> "$GITHUB_OUTPUT"
+          fi
+  pgens:
+    needs: check-commit
+    if: needs.check-commit.outputs.run_tests == 'true'
+    name: PGENS (mpi=${{ matrix.mpi }})
+    runs-on: ubuntu-24.04
+    strategy:
+      fail-fast: false
+      matrix:
+        mpi: [ON, OFF]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install GCC 14 and build tools
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y gcc-14 g++-14 gfortran-14 build-essential wget libevent-dev libhwloc-dev 
+          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 50
+          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 50
+          sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-14 50
+          gcc --version
+          g++ --version
+      - name: Install CMake (3.x)
+        run: |
+          sudo apt-get install -y cmake
+          cmake --version
+      - name: Install OpenMPI 5
+        run: |
+          if [ "${{ matrix.mpi }}" = "ON" ]; then
+            sudo apt-get install -y libopenmpi-dev openmpi-bin
+            mpirun --version
+          fi
+      - name: Compile all pgens
+        run: |
+          if [ "${{ matrix.mpi }}" = "ON" ]; then
+            export CC=mpicc
+            export CXX=mpicxx
+          else
+            export CC=gcc-14
+            export CXX=g++-14
+          fi
+          ./dev/scripts/tests.sh --build build --with_pgens --flags "-D mpi=${{ matrix.mpi }} -D output=OFF"

.github/workflows/cpuarch.yml → .github/workflows/unit-tests.yml

@@ -1,7 +1,9 @@
-name: CPU Compilation/Unit Tests
+name: Premerge tests on CPUs with different configurations
 
 on:
   push:
+  pull_request:
+    types: [ready_for_review]
 
 jobs:
   check-commit:
@@ -14,15 +16,19 @@ jobs:
       - name: Check commit message
         id: check_message
         run: |
-          if git log -1 --pretty=%B | grep -q "CPUTEST"; then
+          if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.action }}" == "ready_for_review" ]]; then
+            echo "run_tests=true" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
+          if git log -1 --pretty=%B | grep -q "RUNTESTS"; then
             echo "run_tests=true" >> "$GITHUB_OUTPUT"
           else
             echo "run_tests=false" >> "$GITHUB_OUTPUT"
           fi
   tests:
     needs: check-commit
     if: needs.check-commit.outputs.run_tests == 'true'
-    name: UNIT_TESTS (precision=${{ matrix.precision }}, mpi=${{ matrix.mpi }}, output=${{ matrix.output }})
+    name: UNIT_TESTS with (precision=${{ matrix.precision }}, mpi=${{ matrix.mpi }}, output=${{ matrix.output }})
     runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
@@ -57,9 +63,11 @@ jobs:
           cmake --version
       - name: Install OpenMPI 5
         run: |
-          sudo apt-get install -y libopenmpi-dev openmpi-bin
-          mpirun --version
-      - name: Configure
+          if [ "${{ matrix.mpi }}" = "ON" ]; then
+            sudo apt-get install -y libopenmpi-dev openmpi-bin
+            mpirun --version
+          fi
+      - name: Compile and run all tests
         run: |
           if [ "${{ matrix.mpi }}" = "ON" ]; then
             export CC=mpicc
@@ -68,40 +76,4 @@ jobs:
             export CC=gcc-14
             export CXX=g++-14
           fi
-          cmake -B build -D TESTS=ON -D precision=${{ matrix.precision }} -D mpi=${{ matrix.mpi }} -D output=${{ matrix.output }}
-      - name: Compile
-        run: cmake --build build -j "$(nproc)"
-      - name: Run tests
-        run: ctest --test-dir build --output-on-failure
-  pgens:
-    needs: check-commit
-    if: needs.check-commit.outputs.run_tests == 'true'
-    name: PGENS (pgen=${{ matrix.pgen }})
-    runs-on: ubuntu-24.04
-    strategy:
-      fail-fast: false
-      matrix:
-        pgen: [streaming, turbulence, reconnection, shock, magnetosphere, accretion, wald]
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          submodules: recursive
-      - name: Install GCC 14 and build tools
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y gcc-14 g++-14 gfortran-14 build-essential wget libevent-dev libhwloc-dev
-          sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 50
-          sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 50
-          sudo update-alternatives --install /usr/bin/gfortran gfortran /usr/bin/gfortran-14 50
-          gcc --version
-          g++ --version
-      - name: Install CMake (3.x)
-        run: |
-          sudo apt-get install -y cmake
-          cmake --version
-      - name: Configure
-        run: |
-          cmake -B build -D pgen=${{ matrix.pgen }} -D output=OFF
-      - name: Compile
-        run: cmake --build build -j "$(nproc)"
+          ./dev/scripts/tests.sh --build build --with_tests --flags "-D precision=${{ matrix.precision }} -D mpi=${{ matrix.mpi }} -D output=${{ matrix.output }}"

.gitignore

@@ -24,6 +24,8 @@ temp*/
 logs/
 *.log
 *.bak
+temp*
+tmp*
 
 # Trash files
 .trash/
@@ -40,6 +42,7 @@ venv/
 *.mov
 *.mp4
 !pgens/**/*.png
+!pgens/**/*.mp4
 
 # Accidental files
 *.xc
@@ -59,3 +62,6 @@ tags
 action-token
 *.vim
 ignore-*
+tombi/
+tidy/
+.claude

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "extern/plog"]
-	path = extern/plog
-	url = https://github.com/SergiusTheBest/plog.git
 [submodule "extern/adios2"]
 	path = extern/adios2
 	url = https://github.com/ornladios/ADIOS2.git

CLAUDE.md

@@ -0,0 +1,122 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## Project Overview
+
+This is an astrophysical particle-in-cell plasma simulation code which works in arbitrary curvilinear coordinates and supports multiple simulation engines. It is built using the Kokkos performance portability library with C++20. It is parallelized with MPI, and uses the ADIOS2 library for outputting and checkpointing the simulation data.
+
+## Repository Structure
+
+```
+entity
+├── cmake                        # additional cmake files
+│   ├── adios2Config.cmake       #   default configurations for in-tree build of adios2
+│   ├── config.cmake             #   compile-time configuration options
+│   ├── defaults.cmake           #   default values for configurations
+│   ├── dependencies.cmake       #   functions to fetch and build the dependencies
+│   ├── kokkosConfig.cmake       #   default configurations for in-tree build of Kokkos
+│   ├── report.cmake             #   configuration repoting
+│   ├── styling.cmake            #   styling functions
+│   └── tests.cmake              #   root cmake for tests
+├── dev                          # developer-specific tools
+│   ├── nix                      #   nix-shells
+│   ├── runners                  #   dockerfiles for github runners on different architectures
+│   ├── scripts                  #   developer-specific scripts
+│   ├── Dockerfile.common        #   parent docker environment for development
+│   ├── Dockerfile.cuda          #   cuda docker environment
+│   ├── Dockerfile.rocm          #   rocm docker environment
+│   ├── welcome.cuda
+│   └── welcome.rocm
+├── extern                       # git submodules
+│   ├── Kokkos
+│   ├── adios2
+│   └── entity-pgens
+├── include                      # included header-only third-party libraries
+│   ├── plog
+│   └── toml11
+├── minimal                      # set of minimalist programs for testing MPI/Kokkos/adios2
+├── pgens                        # problem generators
+├── src                          # main code containing all separate submodules
+│   ├── archetypes               #   archetypes which can be used by the user in problem generators
+│   ├── engines                  #   simulation engines
+│   ├── framework                #   main structures, classes and containers
+│   ├── global                   #   global definitions and utilities
+│   ├── kernels                  #   core kernels (defined as functors)
+│   ├── metrics                  #   various metric classes
+│   ├── output                   #   functions related to output
+│   ├── CMakeLists.txt
+│   └── entity.cpp               #   main entry-point
+├── tests                        # unit tests for all submodules
+├── .clang-format                # code formatting guidelines for clang-format
+├── .clang-tidy                  # configurations for the clang-tidy
+├── .gitattributes
+├── .gitignore
+├── .gitmodules
+├── .taplo.toml                  # formatting guidelines for toml files
+├── CITATION
+├── CLAUDE.md
+├── CMakeLists.txt               # root cmake file
+├── CODE_OF_CONDUCT.md
+├── LICENSE
+├── README.md
+├── dependencies.py              # deployment scripts on various machines
+└── input.example.toml           # most complete toml file with all possible input options
+```
+
+## Testing
+
+The code is tested using the `./dev/scripts/tests.sh` script which compiles and runs all the unit tests using `ctest`:
+
+```sh
+./dev/scripts/tests.sh --build build_dir --flags "-D mpi=ON" --with_tests
+```
+
+All the unit tests are inside the `tests/` directory each within the respective subdirectory; e.g., tests for `src/kernels` are in `tests/kernels`. When testing, build the tests both with and without MPI and, ideally, with and without GPU (when available).
+
+You can also compile all the problem generators:
+
+```sh
+./dev/scripts/tests.sh --build build_dir --flags "-D mpi=ON" --with_pgens
+```
+
+## Code guidelines
+
+* Format of the code is enforced using `clang-format` and `cmake-format`. You can run the formatting on all files with `./dev/scripts/format.sh`.
+
+* Best practices are also enforced using `clang-tidy`; to generate recommendations for all the files, run `./dev/scripts/tidy.sh --build build_dir` where `build_dir` is the directory where the code was built, or for specific files: `./dev/scripts/tidy.sh --build build_dir --files "(file1|file2).cpp"` or only for the changed files: `./dev/scripts/tidy.sh --build build_dir --changed`. The recommendations will be in the `tidy/` directory.
+
+* Use `const` and `auto` declarations where possible.
+
+* For real-valued literals, use `ONE`, `ZERO`, `HALF` etc. instead of `1.0`, `0.0`, `0.5` to ensure the compiler will not need to cast. If the value is not defined as a macro, use `static_cast<real_t>(123.4)`.
+
+* Use {} in declarations to signify a null (placeholder) value for the given variable:
+  ```cpp
+  auto a { -1 }; // <- value of `a` *will* be changed later (-1 is a placeholder)
+  auto b = -1; // <- value of `b` is known at the time of declaration (but *may* change later)
+  const auto b = -1; // <- value of `b` is not expected to change later
+  ```
+
+* Each header file has to have a description at the top, consisting of the following fields:
+
+  * `@file` [required] the name of the file (as it should be included in other files)
+  * `@brief` [required] brief description of what the file contains
+  * `@implements` list of class/function/macros implementations
+    * structs/classes in this section have no prefix (templates are marked with <>)
+    * functions are marked with their return type, e.g. -> void
+    * type aliases have a prefix type
+    * enums or enum-like objects are marked with enum
+    * macros have a prefix macro
+    * all of the above are also marked with their respective namespaces (if any): namespace::
+  * `@cpp`: list of cpp files that implement the header
+  * `@namespaces`: list of namespaces defined in the file
+  * `@macros`: list of macros that the file depends on
+  * `@note` any additional notes (stack as many as necessary)
+
+* `#ifdef`/`#define` macros should be avoided. Use C++20 concept and `if constexpr ()` expressions to specialize functions and classes instead (ideally, specialize them explicitly). `#ifdef`-s are only acceptable in platform/library-specific parts of the code (e.g., `MPI_ENABLED`, `GPU_ENABLED`, `DEBUG`, etc.), or for major shortcuts.
+
+* Header files should start with `#ifndef` ... `#define` ... and end with `#endif`; do not use `#pragma` guards. The name of the macro should be the same as the name of the file in uppercase, with underscores instead of dots and slashes. For example, for `global/utils/formatting.h`, the macro should be `GLOBAL_UTILS_FORMATTING_H`.
+
+* There is no difference between `.h` and `.hpp` files as both indicate C++ header files. As a consistency convention, we use `.h` for common headers which may be included from multiple `.cpp` files (e.g., metrics), while `.hpp` are very specific headers for only a single (or a couple of) .cpp file (e.g. kernels).
+
+* Do assertions on parameters and quantities whenever possible. Outside the kernels, use `raise::Error(message, HERE)` and `raise::ErrorIf(condition, message, HERE)` to throw exceptions. Inside the kernels, use `raise::KernelError(HERE, message, **args)`. To enable compile-time errors, use `static_assert(condition, message)`. The `HERE` keyword is macro that includes the filename and line number in the error message.