Fixes

.gitignore

@@ -1,3 +1,7 @@
 build/*
 profile/*.[0-9]*
 profile/*/
+.*/*
+release*
+pmcl3d*
+!pmcl3d*h

CMakeLists.txt

@@ -1,22 +1,28 @@
 cmake_minimum_required(VERSION 3.10)
-cmake_policy(SET CMP0074 NEW)
-project(AWP_MINI VERSION 1.0 LANGUAGES C CUDA)
+project(AWP VERSION 1.0 LANGUAGES C CUDA)
+if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+  set(CMAKE_CUDA_ARCHITECTURES 90)
+endif()
+include(CMakePrintHelpers)
+#include(FindMPI.cmake)
 
 include_directories(${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
-set(GCC_COMPILE_FLAGS "-std=c99 -Wall -Werror\
+set(GCC_COMPILE_FLAGS "-std=c99 -Wall\
    -Wextra -Wmissing-prototypes -Wstrict-prototypes \
-          -Wold-style-definition -Wno-unused-parameter")
-if (DEFINED ENV{ARCH})
-        set(ARCH $ENV{ARCH})
-else()
-        set(ARCH sm_70)
-endif()
-set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -arch=${ARCH} -Xptxas=-v -lineinfo -use_fast_math")
-
-set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} ${GCC_COMPILE_FLAGS} -D${ARCH}")
-set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=gnu11 -D${ARCH}")
+   -Wno-unused-parameter")
+set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -O4 -Xcompiler -std=c++17 -use_fast_math -Xptxas=-v -g -lineinfo --allow-unsupported-compiler")
+set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} ${GCC_COMPILE_FLAGS}")
+set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -std=c++17")
+#add_compile_definitions(MPICH_SKIP_MPICXX=1)
 
+#### MPI
 find_package(MPI REQUIRED)
+if (MPI_FOUND)
+    cmake_print_variables(CMAKE_INCLUDE_PATH)
+    include_directories(${MY_INCLUDE_PATH})
+else (MPI_FOUND)
+    message(SEND_ERROR "This application cannot compile without MPI")
+endif (MPI_FOUND)
 
 
 include(CTest)

compile_ls6_impi.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+module unload intel
+module load cmake gcc impi cuda
+
+rm -r release
+mkdir -p release
+
+cd release
+export CC=$(which mpigcc)
+export CXX=$(which mpigxx)
+export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib:$LD_LIBRARY_PATH
+export CPATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include:$CPATH
+#export PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/bin:$PATH
+module list
+env | grep "PATH"
+
+cmake -DCMAKE_VERBOSE_MAKEFILE=ON ..
+make

compile_ls6_mv2.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+module unload intel impi
+module load cmake gcc mvapich2 cuda
+module list
+
+rm -r release
+mkdir -p release
+
+cd release
+#export CC=$(which mpicc)
+#export CXX=$(which mpicxx)
+export LD_LIBRARY_PATH=/opt/apps/gcc11_2/mvapich2/2.3.7/lib:$LD_LIBRARY_PATH
+export CPATH=/opt/apps/gcc11_2/mvapich2/2.3.7/include:$CPATH
+#export LD_LIBRARY_PATH=/.../mvapich2/lib:$LD_LIBRARY_PATH
+#export CPATH=/.../mvapich2/include:$CPATH
+cmake -DCMAKE_VERBOSE_MAKEFILE=ON ..
+make

compile_summit.sh

@@ -0,0 +1,9 @@
+#!/bin/tcsh
+
+rm -r release
+mkdir -p release
+module load cmake gcc cuda
+
+cd release
+cmake ..
+make

compile_vista_mv2.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+
+module unload cmake
+#module unload gcc 
+#ml reset
+#ml nvhpc-hpcx-cuda11/23.7
+#module load intel cmake impi cuda
+module unload openmpi
+module use /scratch/00494/tg457572/packages/modulefiles
+
+module load e4s
+module load mvapich
+module load tau 
+ml cuda/12.4
+
+rm -r release
+mkdir -p release
+
+cd release
+#export CXX
+
+#export MPI_HOME="/home1/07936/tg872351/mvp-pre-rc-ofi-cuda12.5"
+export MPI_HOME="/home1/07936/tg872351/mvp-pre-rc-zfp-cuda12.4"
+
+export PATH=${MPI_HOME}/bin:$PATH
+export LD_LIBRARY_PATH=${MPI_HOME}/lib:$LD_LIBRARY_PATH
+export CPATH=${MPI_HOME}/include:$CPATH
+export C_INCLUDE_PATH=${MPI_HOME}/include:$C_INCLUDE_PATH
+
+export LD_LIBRARY_PATH=/home1/apps/nvidia/Linux_aarch64/24.7/cuda/12.5/lib64:$LD_LIBRARY_PATH
+
+export CC=$(which mpicc)
+export CXX=$(which mpicxx)
+export FC=$(which mpifort)
+export MPI_C_COMPILER=$(which mpicc)
+export MPI_CXX_COMPILER=$(which mpicxx)
+export MPI_INCLUDE_PATH=${MPI_HOME}/include
+
+
+#export CXX
+#export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib:$LD_LIBRARY_PATH
+#export CPATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include:$CPATH
+#export C_INCLUDE_PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include:$C_INCLUDE_PATH
+#export PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/bin:$PATH
+
+echo -e "\n"
+echo "======== PATH=============="
+echo $PATH | tr : '\n'
+
+echo -e "\n"
+echo "======== INCLUDE =============="
+echo $INCLUDE | tr : '\n'
+
+echo -e "\n"
+echo "======== LD_LIBRARY_PATH=============="
+echo $LD_LIBRARY_PATH | tr : '\n'
+
+module list
+echo -e "mpicc = `which mpicc`"
+echo ""
+
+echo "LD_PRELOAD=$LD_PRELOAD"
+
+#export MPI_HOME=${TACC_MPI_DIR}
+
+echo "TACC_IMPI_INC=$TACC_IMPI_INC"
+echo "MPI_HOME=$MPI_HOME"
+
+
+cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_C_COMPILER=`which mpicc` -DCMAKE_CXX_COMPILER=`which mpicxx` -DMY_INCLUDE_PATH=$MPI_INCLUDE_PATH ..
+make

compile_vista_openmpi.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+
+module unload cmake
+#module unload gcc 
+#ml reset
+#ml nvhpc-hpcx-cuda11/23.7
+#module load intel cmake impi cuda
+
+rm -r release
+mkdir -p release
+
+cd release
+export CC=$(which mpicc)
+export CXX=$(which mpicxx)
+export FC=$(which mpifort)
+export MPI_C_COMPILER=$(which mpicc)
+export MPI_INCLUDE_PATH=${TACC_MPI_DIR}/include
+#export CXX
+#export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/lib:$LD_LIBRARY_PATH
+#export CPATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include:$CPATH
+#export C_INCLUDE_PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/include:$C_INCLUDE_PATH
+#export PATH=/opt/intel/compilers_and_libraries_2020.4.304/linux/mpi/intel64/bin:$PATH
+
+echo -e "\n"
+echo "======== PATH=============="
+echo $PATH | tr : '\n'
+
+echo -e "\n"
+echo "======== INCLUDE =============="
+echo $INCLUDE | tr : '\n'
+
+echo -e "\n"
+echo "======== LD_LIBRARY_PATH=============="
+echo $LD_LIBRARY_PATH | tr : '\n'
+
+module list
+echo -e "mpicc = `which mpicc`"
+echo ""
+
+echo "LD_PRELOAD=$LD_PRELOAD"
+
+export MPI_HOME=${TACC_MPI_DIR}
+
+echo "TACC_IMPI_INC=$TACC_IMPI_INC"
+echo "MPI_HOME=$MPI_HOME"
+
+
+cmake -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_C_COMPILER=`which mpicc` -DCMAKE_CXX_COMPILER=`which mpicxx` -DMY_INCLUDE_PATH=$MPI_INCLUDE_PATH ..
+make

include/awp/definitions.h

@@ -1,10 +1,6 @@
 #ifndef DEFINITIONS_H
 #define DEFINITIONS_H
 
-#define BLOCK_SIZE_X 2
-#define BLOCK_SIZE_Y 2
-#define BLOCK_SIZE_Z 4
-
 #include <mpi.h>
 
 #ifndef _prec
@@ -29,11 +25,11 @@ typedef float prec;
 #endif
 
 #ifndef ngsl
-#define ngsl 8
+#define ngsl 4
 #endif
 
 #ifndef ngsl2
-#define ngsl2 16
+#define ngsl2 8
 #endif
 
 #ifndef align
@@ -47,6 +43,7 @@ typedef float prec;
 #define STR_LEN 2048
 
 
+
 typedef struct
 {
         _prec x, y, z;

include/awp/error.h

@@ -1,8 +1,6 @@
 #ifndef ERROR_H
 #define ERROR_H
 
-int _last_error;
-
 enum error_codes {SUCCESS, 
                   ERR_FILE_OPEN = 100, 
                   ERR_FILE_READ = 101, 

include/awp/pmcl3d.h

@@ -36,7 +36,9 @@ void command(int argc, char **argv, _prec *TMAX, _prec *DH, _prec *DT,
              int *USETOPO, char *SOURCEFILE,
              int *USESOURCEFILE, char *RECVFILE, int *USERECVFILE,
              char *FORCEFILE, int *USEFORCEFILE,
-             char *SGTFILE, int *USESGTFILE);
+             char *SGTFILE, int *USESGTFILE, char *MMSFILE, int *USEMMSFILE, float *DHB, float *DHT,
+             char *ENERGYFILE, int *USEENERGYFILE, 
+             _prec *QSI, _prec *QPQSR, _prec *MAXVPVSR, _prec *VMIN, _prec *VMAX, _prec *DMIN);
 
 int read_src_ifault_2(int rank, int READ_STEP, 
     char *INSRC, char *INSRC_I2, 
@@ -72,6 +74,7 @@ void inimesh(int rank, int MEDIASTART, Grid3D d1, Grid3D mu, Grid3D lam, Grid3D
 	     Grid3D tau, Grid3D weights,Grid1D coeff,
 	     int nvar, _prec FP,  _prec FAC, _prec Q0, _prec EX, int nxt, int nyt, int nzt, int PX, int PY, int NX, int NY,
              int NZ, int *coords, MPI_Comm MCW, int IDYNA, int NVE, int SoCalQ, char *INVEL,
+            _prec qsi, _prec qpqsr, _prec maxvpvsr, _prec vmin, _prec vmax, _prec dmin,
              _prec *vse, _prec *vpe, _prec *dde);
 
 int checkmesh(int nxtl, int nytl, int nztl, int nxth, int nyth, int nzth, Grid3D varl, Grid3D varh,
@@ -109,7 +112,7 @@ void init_texture(int nxt,  int nyt,  int nzt,  Grid3D tau1,  Grid3D tau2,  Grid
 
 Grid3D Alloc3D(int nx, int ny, int nz);
 Grid3Dww Alloc3Dww(int nx, int ny, int nz); 
-Grid1D Alloc1D(int nx);    
+Grid1D Alloc1D(long nx);    
 PosInf Alloc1P(int nx);
 
 void Delloc3D(Grid3D U);

include/awp/pmcl3d_cons.h

@@ -1,7 +1,7 @@
 #ifndef DEFINITIONS_H
 #define BLOCK_SIZE_X 2
 #define BLOCK_SIZE_Y 2
-#define BLOCK_SIZE_Z 4
+#define BLOCK_SIZE_Z 32
 #endif
 // Set floating-point precision. Make sure to configure both `_prec` and
 // `_mpi_prec`.
@@ -15,12 +15,9 @@
 #endif
 #define align 32
 #define loop  1 
-// Number of ghost cells is increased from 4 to 8 for topography kernels.
-// In the future, it should be possible to keep this number at four, but modify
-// the vertical velocity exchange so that 6 points is exchanged instead of 4.
-// No modifications necessary to the other velocity components. 
-#define ngsl 8     /* number of ghost cells x loop */
-#define ngsl2 16  /* ngsl * 2 */
+// Do not change the number of ghost cells.
+#define ngsl 4     /* number of ghost cells x loop */
+#define ngsl2 8  /* ngsl * 2 */
 
 #define Both  0
 #define Left  1

include/buffers/buffer.h

@@ -214,7 +214,7 @@ void buffer_copy_to_device(buffer_t *buffer, size_t step);
  *      buffer: Buffer data structure.
  *      step: Time step to query buffer at.
  */
-void buffer_copy_to_host(buffer_t *buffer, int step);
+void buffer_copy_to_host(buffer_t *buffer, size_t step);
 
 #ifdef __cplusplus
 }

include/grid/grid_3d.h

@@ -131,6 +131,11 @@ grid3_t grid_init_metric_grid(const int3_t size, const int3_t shift,
                          const int3_t boundary2,
                          const _prec gridspacing);
 
+grid3_t grid_init_full_grid(const int3_t size, const int3_t shift,
+                         const int3_t coordinate, const int3_t boundary1,
+                         const int3_t boundary2,
+                         const _prec gridspacing);
+
 /* Initialize grid
  * 
  * Input arguments:
@@ -184,11 +189,29 @@ grid1_t grid_grid1_z(const grid3_t grid);
  *      out: Array to fill
  *      n: Array size. Must be greater than the grid size.
  *      grid: 1D grid data structure.
+ *      isxdir: Specify to `1` if the grid should be filled in the x-direction.
+ *          Adjusts for the particular internal coordinate system used by AWP, i.e., fields stored
+ *          in the (-,+,+) octant
+ *
+ * Return value:
+ *      Number of elements written.
+ */ 
+int grid_fill1(prec *out, const grid1_t grid, const int isxdir);
+/*
+ *
+ * Fill the array `out` with the grid point values in the y-direction for a given DM block
+ * ('blocknum') in one dimension.
+ *
+ * Arguments:
+ *      out: Array to fill
+ *      n: Array size. Must be greater than the grid size.
+ *      grid: 1D grid data structure.
+ *      blocknum: Block number. Must be a non-negative integer.
  *
  * Return value:
  *      Number of elements written.
  */ 
-int grid_fill1(prec *out, const grid1_t grid);
+int grid_fill_y_dm(prec *out, const grid1_t grid, const int blocknum);
 
 /*
  * Check if a query point is in bounds or not. The query point is in bounds if
@@ -204,8 +227,8 @@ int grid_fill1(prec *out, const grid1_t grid);
  */
 int grid_in_bounds1(const _prec *x, const _prec q, const grid1_t grid);
 
-int grid_in_bounds_ext1(const _prec *x, const _prec q, const grid1_t grid);
-
+int grid_in_bounds_sgt(const _prec *x, const _prec q, const grid1_t grid);
+int grid_in_bounds_moment_tensor(const _prec *x, const _prec q, const grid1_t grid);
 
 /*
  * Fill the array `out` with the grid point values in the x-direction of a grid
@@ -308,6 +331,13 @@ int grid_pow3(_prec *out, const _prec p, const grid3_t grid);
  */
 double grid_reduce3(const _prec *in, const grid3_t grid);
 
+_prec grid_overlap(const _prec h);
+_prec grid_height(const int nz, const _prec h, const int istopo);
+
+void global_to_local(_prec *zloc, int *block_index, const _prec z,
+                     const _prec h, const int *nz, const int num_grids,
+                     const int istopo);
+
 
 #ifdef __cplusplus
 }