From 4ab852415007e3e9a1740845001ddc1760010c22 Mon Sep 17 00:00:00 2001 From: Jacob Moore Date: Fri, 8 May 2026 13:01:22 -0500 Subject: [PATCH] ENH: Adding node tally mask to mesh class --- .../src/mesh_decomp_example.cpp | 25 +++++++++ src/decomp_utilities/decomp_utils.h | 17 ++++++ src/swage/unstructured_mesh.h | 55 +++++++++---------- 3 files changed, 69 insertions(+), 28 deletions(-) diff --git a/examples/decomp_example/src/mesh_decomp_example.cpp b/examples/decomp_example/src/mesh_decomp_example.cpp index 7e2ad07c..b82cb839 100644 --- a/examples/decomp_example/src/mesh_decomp_example.cpp +++ b/examples/decomp_example/src/mesh_decomp_example.cpp @@ -154,6 +154,31 @@ int main(int argc, char** argv) { // Gauss points share the same communication plan as elements. // This test initializes gauss point fields on owned elements and exchanges them with ghost elements. + if(world_size != 1) { + // Test that the shared_tally_owned_nodes mask works correctly by counting all nodes across all ranks and verifying that the number of unique nodes is equal to the number of owned nodes. + int total_num_nodes = 0; + int total_local_nodes = 0; + int total_global_nodes = 0; + + + FOR_REDUCE_SUM(node_gid, 0, final_mesh.num_owned_nodes, total_local_nodes, { + + if(final_mesh.shared_tally_owned_nodes(node_gid)){ + total_local_nodes++; + } + + }, total_num_nodes); + MATAR_FENCE(); + + MPI_Allreduce(&total_num_nodes, &total_global_nodes, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD); + + if(rank == 0){ + std::cout<<"Total number of nodes: "<(output_mesh.num_owned_nodes, "shared_tally_owned_nodes"); + for (int local_lid = 0; local_lid < static_cast(output_mesh.num_owned_nodes); local_lid++) { + size_t gid = output_mesh.local_to_global_node_mapping.host(local_lid); + std::set tally_ranks; + tally_ranks.insert(rank); + auto gr_it = local_node_gid_to_ghosting_ranks.find(gid); + if (gr_it != local_node_gid_to_ghosting_ranks.end()) { + tally_ranks.insert(gr_it->second.begin(), gr_it->second.end()); + } + const int min_participant_rank = *tally_ranks.begin(); + output_mesh.shared_tally_owned_nodes.host(local_lid) = (rank == min_participant_rank); + } + output_mesh.shared_tally_owned_nodes.update_device(); + MPI_Barrier(MPI_COMM_WORLD); // rebuild the local element-node connectivity using the local node ids // extended_nodes_in_elem already contains extended local node IDs, so we can use them directly diff --git a/src/swage/unstructured_mesh.h b/src/swage/unstructured_mesh.h index 9563a848..83b54b4b 100644 --- a/src/swage/unstructured_mesh.h +++ b/src/swage/unstructured_mesh.h @@ -233,14 +233,14 @@ struct Mesh size_t num_dims = 3; ///< Number of spatial dimension // ---- Element Data Definitions ---- // - size_t num_elems; ///< Number of elements in the mesh - size_t num_nodes_in_elem; ///< Number of nodes in an element - size_t num_patches_in_elem; ///< Number of patches in an element - size_t num_surfs_in_elem; ///< Number of surfaces in an element - size_t num_zones_in_elem; ///< Number of zones in an element + size_t num_elems = 0; ///< Number of elements in the mesh + size_t num_nodes_in_elem = 0; ///< Number of nodes in an element + size_t num_patches_in_elem = 0; ///< Number of patches in an element + size_t num_surfs_in_elem = 0; ///< Number of surfaces in an element + size_t num_zones_in_elem = 0; ///< Number of zones in an element - size_t num_gauss_in_elem; ///< Number of Gauss points in an element - size_t num_lobatto_in_elem; ///< Number of Gauss Lobatto points in an element + size_t num_gauss_in_elem = 0; ///< Number of Gauss points in an element + size_t num_lobatto_in_elem = 0; ///< Number of Gauss Lobatto points in an element DCArrayKokkos nodes_in_elem; ///< Nodes in an element CArrayKokkos corners_in_elem; ///< Corners in an element -- this can just be a functor @@ -257,7 +257,7 @@ struct Mesh gauss_in_elem_t gauss_in_elem; ///< Gauss points in an element // ---- Node Data Definitions ---- // - size_t num_nodes; ///< Number of nodes in the mesh + size_t num_nodes = 0; ///< Number of nodes in the mesh RaggedRightArrayKokkos corners_in_node; ///< Corners connected to a node CArrayKokkos num_corners_in_node; ///< Number of corners connected to a node @@ -266,17 +266,17 @@ struct Mesh CArrayKokkos num_nodes_in_node; ///< Number of nodes connected to a node along an edge // ---- Surface Data Definitions ---- // - size_t num_surfs; ///< Number of surfaces in the mesh - size_t num_nodes_in_surf; ///< Number of nodes in a surface - size_t num_patches_in_surf; ///< Number of patches in a surface + size_t num_surfs = 0; ///< Number of surfaces in the mesh + size_t num_nodes_in_surf = 0; ///< Number of nodes in a surface + size_t num_patches_in_surf = 0; ///< Number of patches in a surface CArrayKokkos patches_in_surf; ///< Patches in a surface CArrayKokkos nodes_in_surf; ///< Nodes connected to a surface CArrayKokkos elems_in_surf; ///< Elements connected to a surface // ---- Patch Data Definitions ---- // - size_t num_patches; ///< Number of patches in the mesh - size_t num_nodes_in_patch; ///< Number of nodes in a patch + size_t num_patches = 0; ///< Number of patches in the mesh + size_t num_nodes_in_patch = 0; ///< Number of nodes in a patch // size_t num_lobatto_in_patch; ///< Number of Gauss Lobatto nodes in a patch // size_t num_gauss_in_patch; ///< Number of Gauss nodes in a patch @@ -285,19 +285,19 @@ struct Mesh CArrayKokkos surf_in_patch; ///< Surfaces connected to a patch (co-planar) // ---- Corner Data Definitions ---- // - size_t num_corners; ///< Number of corners (define) in the mesh + size_t num_corners = 0; ///< Number of corners (define) in the mesh // ---- Zone Data Definitions ---- // - size_t num_zones; ///< Number of zones in the mesh - size_t num_nodes_in_zone; ///< Number of nodes in a zone + size_t num_zones = 0; ///< Number of zones in the mesh + size_t num_nodes_in_zone = 0; ///< Number of nodes in a zone CArrayKokkos nodes_in_zone; ///< Nodes defining a zone // nodes_in_zone_t nodes_in_zone; // ---- Boundary Data Definitions ---- // - size_t num_bdy_sets; ///< Number of boundary sets - size_t num_bdy_nodes; ///< Number of boundary nodes - size_t num_bdy_patches; ///< Number of boundary patches + size_t num_bdy_sets = 0; ///< Number of boundary sets + size_t num_bdy_nodes = 0; ///< Number of boundary nodes + size_t num_bdy_patches = 0; ///< Number of boundary patches CArrayKokkos bdy_patches; ///< Boundary patches CArrayKokkos bdy_nodes; ///< Boundary nodes @@ -314,16 +314,17 @@ struct Mesh DCArrayKokkos local_to_global_elem_mapping; ///< Local to global element mapping // Element communicaiton data definitions - size_t num_owned_elems; ///< Number of owned elements on this rank - size_t num_boundary_elems; ///< Number of boundary elements on this rank (send data to neighboring MPI ranks) + size_t num_owned_elems = 0; ///< Number of owned elements on this rank + size_t num_boundary_elems = 0; ///< Number of boundary elements on this rank (send data to neighboring MPI ranks) DCArrayKokkos boundary_elem_local_ids; ///< Local IDs of boundary elements on this rank (send data to neighboring MPI ranks) - size_t num_ghost_elems; ///< Number of ghost elements on this rank (receive data from neighboring MPI ranks) + size_t num_ghost_elems = 0; ///< Number of ghost elements on this rank (receive data from neighboring MPI ranks) // Node communicaiton data definitions - size_t num_owned_nodes; ///< Number of owned nodes on this rank - size_t num_boundary_nodes; ///< Number of boundary nodes on this rank (send data to neighboring MPI ranks) - DCArrayKokkos boundary_node_local_ids; ///< Local IDs of boundary nodes on this rank (send data to neighboring MPI ranks) - size_t num_ghost_nodes; ///< Number of ghost nodes on this rank (receive data from neighboring MPI ranks) + size_t num_owned_nodes = 0; ///< Number of owned nodes on this rank + size_t num_boundary_nodes = 0; ///< Number of boundary nodes on this rank (send data to neighboring MPI ranks) + DCArrayKokkos shared_tally_owned_nodes; ///< Owned-node mask: true where this rank is the min MPI rank among ranks that own the global node (domain tally contributor); length num_owned_nodes + // DCArrayKokkos boundary_node_local_ids; ///< Local IDs of boundary nodes on this rank (send data to neighboring MPI ranks) + size_t num_ghost_nodes = 0; ///< Number of ghost nodes on this rank (receive data from neighboring MPI ranks) // initialization methods void initialize_nodes(const size_t num_nodes_inp) @@ -362,8 +363,6 @@ struct Mesh const size_t num_dims_inp, const size_t Pn_order) { - - elem_kind = mesh_init::arbitrary_tensor_element; num_dims = num_dims_inp;