Skip to content

Fix for GPU aware MPI #200

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
LudwigBoess wants to merge 1 commit into from
Closed

Fix for GPU aware MPI #200

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,6 @@ set(mpi
${default_mpi}
CACHE BOOL "Use MPI")

set(gpu_aware_mpi
${default_gpu_aware_mpi}
CACHE BOOL "Enable GPU-aware MPI")

# -------------------------- Compilation settings -------------------------- #
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
Expand Down Expand Up @@ -142,15 +138,6 @@ if(${mpi})
include_directories(${MPI_CXX_INCLUDE_PATH})
add_compile_options("-D MPI_ENABLED")
set(DEPENDENCIES ${DEPENDENCIES} MPI::MPI_CXX)
if(${DEVICE_ENABLED})
if(${gpu_aware_mpi})
add_compile_options("-D GPU_AWARE_MPI")
endif()
else()
set(gpu_aware_mpi
OFF
CACHE BOOL "Use explicit copy when using MPI + GPU")
endif()
endif()

# Output
Expand Down
12 changes: 0 additions & 12 deletions cmake/defaults.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,3 @@ else()
endif()

set_property(CACHE default_mpi PROPERTY TYPE BOOL)

if(DEFINED ENV{Entity_ENABLE_GPU_AWARE_MPI})
set(default_gpu_aware_mpi
$ENV{Entity_ENABLE_GPU_AWARE_MPI}
CACHE INTERNAL "Default flag for GPU-aware MPI")
else()
set(default_gpu_aware_mpi
ON
CACHE INTERNAL "Default flag for GPU-aware MPI")
endif()

set_property(CACHE default_gpu_aware_mpi PROPERTY TYPE BOOL)
15 changes: 0 additions & 15 deletions cmake/report.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,17 +111,6 @@ printchoices(
"${Green}"
MPI_REPORT
46)
if(${mpi} AND ${DEVICE_ENABLED})
printchoices(
"GPU-aware MPI"
"gpu_aware_mpi"
"${ON_OFF_VALUES}"
${gpu_aware_mpi}
OFF
"${Green}"
GPU_AWARE_MPI_REPORT
46)
endif()
printchoices(
"Debug mode"
"DEBUG"
Expand Down Expand Up @@ -193,10 +182,6 @@ string(
${MPI_REPORT}
"\n")

if(${mpi} AND ${DEVICE_ENABLED})
string(APPEND REPORT_TEXT " " ${GPU_AWARE_MPI_REPORT} "\n")
endif()

string(
APPEND
REPORT_TEXT
Expand Down
56 changes: 11 additions & 45 deletions src/framework/containers/particles_comm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,9 @@ namespace ntt {
npart_t nsend,
npart_t nrecv,
npart_t offset) {
#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
#if defined(DEVICE_ENABLED)
Kokkos::fence();
#endif
MPI_Sendrecv(send_arr.data(),
nsend,
mpi::get_type<T>(),
Expand All @@ -41,27 +43,6 @@ namespace ntt {
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
#else
const auto slice = std::make_pair(offset, offset + nrecv);

auto send_arr_h = Kokkos::create_mirror_view(send_arr);
auto recv_arr_h = Kokkos::create_mirror_view(
Kokkos::subview(recv_arr, slice));
Kokkos::deep_copy(send_arr_h, send_arr);
MPI_Sendrecv(send_arr_h.data(),
nsend,
mpi::get_type<T>(),
send_rank,
0,
recv_arr_h.data(),
nrecv,
mpi::get_type<T>(),
recv_rank,
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
Kokkos::deep_copy(Kokkos::subview(recv_arr, slice), recv_arr_h);
#endif
}

void send_recv_count(int send_rank,
Expand Down Expand Up @@ -98,39 +79,24 @@ namespace ntt {

template <typename T>
void send(array_t<T*>& send_arr, int send_rank, npart_t nsend) {
#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
MPI_Send(send_arr.data(), nsend, mpi::get_type<T>(), send_rank, 0, MPI_COMM_WORLD);
#else
auto send_arr_h = Kokkos::create_mirror_view(send_arr);
Kokkos::deep_copy(send_arr_h, send_arr);
MPI_Send(send_arr_h.data(), nsend, mpi::get_type<T>(), send_rank, 0, MPI_COMM_WORLD);
#if defined(DEVICE_ENABLED)
Kokkos::fence();
#endif
MPI_Send(send_arr.data(), nsend, mpi::get_type<T>(), send_rank, 0, MPI_COMM_WORLD);
}

template <typename T>
void recv(array_t<T*>& recv_arr, int recv_rank, npart_t nrecv, npart_t offset) {
#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
#if defined(DEVICE_ENABLED)
Kokkos::fence();
#endif
MPI_Recv(recv_arr.data() + offset,
nrecv,
mpi::get_type<T>(),
recv_rank,
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
#else
const auto slice = std::make_pair(offset, offset + nrecv);

auto recv_arr_h = Kokkos::create_mirror_view(
Kokkos::subview(recv_arr, slice));
MPI_Recv(recv_arr_h.data(),
nrecv,
mpi::get_type<T>(),
recv_rank,
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
Kokkos::deep_copy(Kokkos::subview(recv_arr, slice), recv_arr_h);
#endif
}

template <typename T>
Expand Down Expand Up @@ -232,8 +198,8 @@ namespace ntt {

// buffers to store recv data
const auto npart_recv = std::accumulate(npptag_recv_vec.begin(),
npptag_recv_vec.end(),
static_cast<npart_t>(0));
npptag_recv_vec.end(),
static_cast<npart_t>(0));
array_t<int*> recv_buff_int { "recv_buff_int", npart_recv * NINTS };
array_t<real_t*> recv_buff_real { "recv_buff_real", npart_recv * NREALS };
array_t<prtldx_t*> recv_buff_prtldx { "recv_buff_prtldx", npart_recv * NPRTLDX };
Expand Down
52 changes: 10 additions & 42 deletions src/framework/domain/comm_mpi.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,9 @@ namespace comm {
int recv_rank,
ncells_t nsend,
ncells_t nrecv) {
#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
#if defined(DEVICE_ENABLED)
Kokkos::fence();
#endif
MPI_Sendrecv(send_arr.data(),
nsend,
mpi::get_type<real_t>(),
Expand All @@ -46,63 +48,29 @@ namespace comm {
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
#else
auto send_arr_h = Kokkos::create_mirror_view(send_arr);
auto recv_arr_h = Kokkos::create_mirror_view(recv_arr);
Kokkos::deep_copy(send_arr_h, send_arr);
MPI_Sendrecv(send_arr_h.data(),
nsend,
mpi::get_type<real_t>(),
send_rank,
0,
recv_arr_h.data(),
nrecv,
mpi::get_type<real_t>(),
recv_rank,
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
Kokkos::deep_copy(recv_arr, recv_arr_h);
#endif
}

template <unsigned short D>
void send(ndarray_t<D>& send_arr, int send_rank, ncells_t nsend) {
#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
MPI_Send(send_arr.data(), nsend, mpi::get_type<real_t>(), send_rank, 0, MPI_COMM_WORLD);
#else
auto send_arr_h = Kokkos::create_mirror_view(send_arr);
Kokkos::deep_copy(send_arr_h, send_arr);
MPI_Send(send_arr_h.data(),
nsend,
mpi::get_type<real_t>(),
send_rank,
0,
MPI_COMM_WORLD);
#if defined(DEVICE_ENABLED)
Kokkos::fence();
#endif
MPI_Send(send_arr.data(), nsend, mpi::get_type<real_t>(), send_rank, 0, MPI_COMM_WORLD);

}

template <unsigned short D>
void recv(ndarray_t<D>& recv_arr, int recv_rank, ncells_t nrecv) {
#if !defined(DEVICE_ENABLED) || defined(GPU_AWARE_MPI)
#if defined(DEVICE_ENABLED)
Kokkos::fence();
#endif
MPI_Recv(recv_arr.data(),
nrecv,
mpi::get_type<real_t>(),
recv_rank,
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
#else
auto recv_arr_h = Kokkos::create_mirror_view(recv_arr);
MPI_Recv(recv_arr_h.data(),
nrecv,
mpi::get_type<real_t>(),
recv_rank,
0,
MPI_COMM_WORLD,
MPI_STATUS_IGNORE);
Kokkos::deep_copy(recv_arr, recv_arr_h);
#endif
}

template <unsigned short D>
Expand Down
13 changes: 0 additions & 13 deletions src/global/utils/reporter.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,14 +185,6 @@ namespace reporter {
AddParam(report, 4, "HIP", "%s", hip_version.c_str());
#endif
AddParam(report, 4, "MPI", "%s", mpi_version.c_str());
#if defined(MPI_ENABLED) && defined(DEVICE_ENABLED)
#if defined(GPU_AWARE_MPI)
const std::string gpu_aware_mpi = "ON";
#else
const std::string gpu_aware_mpi = "OFF";
#endif
AddParam(report, 4, "GPU-aware MPI", "%s", gpu_aware_mpi.c_str());
#endif
AddParam(report, 4, "Kokkos", "%s", kokkos_version.c_str());
AddParam(report, 4, "ADIOS2", "%s", adios2_version.c_str());
AddParam(report, 4, "Precision", "%s", precision);
Expand Down Expand Up @@ -245,11 +237,6 @@ namespace reporter {
AddParam(report, 4, "MPI_ENABLED", "%s", "OFF");
#endif

#if defined(GPU_AWARE_MPI)
AddParam(report, 4, "GPU_AWARE_MPI", "%s", "ON");
#else
AddParam(report, 4, "GPU_AWARE_MPI", "%s", "OFF");
#endif
report += "\n";
return report;
}
Expand Down
1 change: 0 additions & 1 deletion src/global/utils/reporter.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
* - DEVICE_ENABLED
* - DEBUG
* - SINGLE_PRECISION
* - GPU_AWARE_MPI
*/

#ifndef GLOBAL_UTILS_REPORTER_H
Expand Down
Loading