Move kokkos IS-view storage onto per-PCAIR handle

Makefile

@@ -161,7 +161,8 @@ export TEST_TARGETS = ex12f \
 		  matrandom_check_reset \
 		  ex12f_gmres_poly \
 		  mat_diag \
-		  adv_dg_upwind
+		  adv_dg_upwind \
+		  ex6_two_airg
 # Include kokkos examples
 ifeq ($(PETSC_HAVE_KOKKOS),1)
 export TEST_TARGETS := $(TEST_TARGETS) adv_1dk

include/kokkos_helper.hpp

@@ -42,11 +42,13 @@ PETSC_INTERN void pmisr_existing_measure_implicit_transpose_kokkos(Mat *strength
 PETSC_INTERN void copy_diag_dom_ratio_d2h(PetscReal *diag_dom_ratio_local);
 PETSC_INTERN void delete_device_diag_dom_ratio();
 
-// Define array of shared pointers representing fine and coarse IS's 
-// on each level on the device
-extern ViewPetscIntPtr* IS_fine_views_local;
-extern ViewPetscIntPtr* IS_coarse_views_local;
-extern int max_levels;
+// Per-PCAIR IS views (fine/coarse per multigrid level) live behind an opaque
+// handle owned by the air_data on the Fortran side; see VecISCopyLocalk for
+// the definition of the storage struct and accessors.
+// Not PETSC_INTERN: that forces extern "C" linkage, which is incompatible with
+// returning a C++ Kokkos View. Callers are all C++ (.kokkos.cxx).
+PETSC_VISIBILITY_INTERNAL PetscIntKokkosView VecISCopyLocal_kokkos_get_view(void *handle, int our_level, int fine_int);
+
 extern intKokkosView cf_markers_local_d;
 extern PetscScalarKokkosView diag_dom_ratio_local_d;
 

src/AIR_Data_Type.F90

@@ -337,9 +337,16 @@ module air_data_type
       type(gmres_poly_data) :: inv_coarsest_poly_data       
 
       ! Temporary storage
-      type(petsc_vec_array), dimension(4) :: temp_vecs_fine, temp_vecs_coarse 
+      type(petsc_vec_array), dimension(4) :: temp_vecs_fine, temp_vecs_coarse
       type(petsc_vec_array), dimension(1) :: temp_vecs
 
+      ! Per-PCAIR opaque handle to the kokkos-side IS view storage. Set up by
+      ! create_VecISCopyLocal_kokkos, torn down by destroy_VecISCopyLocal_kokkos.
+      ! Stored here (instead of as a file-scope global in
+      ! src/VecISCopyLocalk.kokkos.cxx) so concurrent PCAIR instances do not
+      ! overwrite each other's per-level IS views.
+      type(c_ptr) :: kokkos_is_views_handle = c_null_ptr
+
       ! Temporary reuse
       type(air_reuse_data), allocatable, dimension(:) :: reuse
 

src/AIR_MG_Setup.F90

@@ -421,18 +421,21 @@ subroutine setup_air_pcmg(amat, pmat, air_data, pcmg_input)
                call MatCreateSubMatrixWrapper(air_data%coarse_matrix(our_level), &
                      air_data%IS_fine_index(our_level), air_data%IS_fine_index(our_level), MAT_REUSE_MATRIX, &
                      air_data%A_ff(our_level), &
-                     our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE.)
+                     our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE., &
+                     kokkos_is_views_handle = air_data%kokkos_is_views_handle)
             else
                call MatCreateSubMatrixWrapper(air_data%coarse_matrix(our_level), &
                      air_data%IS_fine_index(our_level), air_data%IS_fine_index(our_level), MAT_INITIAL_MATRIX, &
                      air_data%A_ff(our_level), &
-                     our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE.)
+                     our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE., &
+                     kokkos_is_views_handle = air_data%kokkos_is_views_handle)
             end if
          else
             call MatCreateSubMatrixWrapper(air_data%coarse_matrix(our_level), &
                   air_data%IS_fine_index(our_level), air_data%IS_fine_index(our_level), MAT_INITIAL_MATRIX, &
                   air_data%A_ff(our_level), &
-                  our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE.)
+                  our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE., &
+                  kokkos_is_views_handle = air_data%kokkos_is_views_handle)
          end if
 
          call timer_finish(TIMER_ID_AIR_EXTRACT)   

src/AIR_Operators_Setup.F90

@@ -92,12 +92,14 @@ subroutine get_submatrices_start_poly_coeff_comms(input_mat, our_level, air_data
             call MatCreateSubMatrixWrapper(air_data%reuse(our_level)%reuse_mat(MAT_A_DROP), &
                         air_data%IS_fine_index(our_level), air_data%IS_fine_index(our_level), MAT_REUSE_MATRIX, &
                         air_data%reuse(our_level)%reuse_mat(MAT_AFF_DROP), &
-                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE.)              
+                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE., &
+                        kokkos_is_views_handle = air_data%kokkos_is_views_handle)              
          else
             call MatCreateSubMatrixWrapper(air_data%reuse(our_level)%reuse_mat(MAT_A_DROP), &
                         air_data%IS_fine_index(our_level), air_data%IS_fine_index(our_level), MAT_INITIAL_MATRIX, &
                         air_data%reuse(our_level)%reuse_mat(MAT_AFF_DROP), &
-                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE.)                              
+                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .TRUE., &
+                        kokkos_is_views_handle = air_data%kokkos_is_views_handle)                              
          end if
 
          call timer_finish(TIMER_ID_AIR_EXTRACT)                             
@@ -162,12 +164,14 @@ subroutine get_submatrices_start_poly_coeff_comms(input_mat, our_level, air_data
             call MatCreateSubMatrixWrapper(input_mat, &
                   air_data%IS_coarse_index(our_level), air_data%IS_coarse_index(our_level), MAT_REUSE_MATRIX, &
                   air_data%A_cc(our_level), &
-                  our_level = our_level, is_row_fine = .FALSE., is_col_fine = .FALSE.)
+                  our_level = our_level, is_row_fine = .FALSE., is_col_fine = .FALSE., &
+                  kokkos_is_views_handle = air_data%kokkos_is_views_handle)
          else
             call MatCreateSubMatrixWrapper(input_mat, &
                   air_data%IS_coarse_index(our_level), air_data%IS_coarse_index(our_level), MAT_INITIAL_MATRIX, &
                   air_data%A_cc(our_level), &
-                  our_level = our_level, is_row_fine = .FALSE., is_col_fine = .FALSE.)
+                  our_level = our_level, is_row_fine = .FALSE., is_col_fine = .FALSE., &
+                  kokkos_is_views_handle = air_data%kokkos_is_views_handle)
          end if
 
          call timer_start(TIMER_ID_AIR_INVERSE)    
@@ -200,20 +204,24 @@ subroutine get_submatrices_start_poly_coeff_comms(input_mat, our_level, air_data
          call MatCreateSubMatrixWrapper(input_mat, &
                air_data%IS_fine_index(our_level), air_data%IS_coarse_index(our_level), MAT_REUSE_MATRIX, &
                air_data%A_fc(our_level), &
-               our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE.)
+               our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE., &
+               kokkos_is_views_handle = air_data%kokkos_is_views_handle)
          call MatCreateSubMatrixWrapper(input_mat, &
                air_data%IS_coarse_index(our_level), air_data%IS_fine_index(our_level), MAT_REUSE_MATRIX, &
                air_data%A_cf(our_level), &
-               our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE.)
+               our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE., &
+               kokkos_is_views_handle = air_data%kokkos_is_views_handle)
       else
          call MatCreateSubMatrixWrapper(input_mat, &
                air_data%IS_fine_index(our_level), air_data%IS_coarse_index(our_level), MAT_INITIAL_MATRIX, &
                air_data%A_fc(our_level), &
-               our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE.)
+               our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE., &
+               kokkos_is_views_handle = air_data%kokkos_is_views_handle)
          call MatCreateSubMatrixWrapper(input_mat, &
                air_data%IS_coarse_index(our_level), air_data%IS_fine_index(our_level), MAT_INITIAL_MATRIX, &
                air_data%A_cf(our_level), &
-               our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE.)
+               our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE., &
+               kokkos_is_views_handle = air_data%kokkos_is_views_handle)
       end if
 
       call timer_finish(TIMER_ID_AIR_EXTRACT)   
@@ -241,12 +249,14 @@ subroutine get_submatrices_start_poly_coeff_comms(input_mat, our_level, air_data
             call MatCreateSubMatrixWrapper(air_data%reuse(our_level)%reuse_mat(MAT_A_DROP), &
                         air_data%IS_coarse_index(our_level), air_data%IS_fine_index(our_level), MAT_REUSE_MATRIX, &
                         air_data%reuse(our_level)%reuse_mat(MAT_ACF_DROP), &
-                        our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE.)               
+                        our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE., &
+                        kokkos_is_views_handle = air_data%kokkos_is_views_handle)               
          else
             call MatCreateSubMatrixWrapper(air_data%reuse(our_level)%reuse_mat(MAT_A_DROP), &
                         air_data%IS_coarse_index(our_level), air_data%IS_fine_index(our_level), MAT_INITIAL_MATRIX, &
                         air_data%reuse(our_level)%reuse_mat(MAT_ACF_DROP), &
-                        our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE.)             
+                        our_level = our_level, is_row_fine = .FALSE., is_col_fine = .TRUE., &
+                        kokkos_is_views_handle = air_data%kokkos_is_views_handle)             
          end if
 
          if (.NOT. air_data%options%one_point_classical_prolong) then
@@ -256,12 +266,14 @@ subroutine get_submatrices_start_poly_coeff_comms(input_mat, our_level, air_data
                call MatCreateSubMatrixWrapper(air_data%reuse(our_level)%reuse_mat(MAT_A_DROP), &
                         air_data%IS_fine_index(our_level), air_data%IS_coarse_index(our_level), MAT_REUSE_MATRIX, &
                         air_data%reuse(our_level)%reuse_mat(MAT_AFC_DROP), &
-                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE.)
+                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE., &
+                        kokkos_is_views_handle = air_data%kokkos_is_views_handle)
             else
                call MatCreateSubMatrixWrapper(air_data%reuse(our_level)%reuse_mat(MAT_A_DROP), &
                         air_data%IS_fine_index(our_level), air_data%IS_coarse_index(our_level), MAT_INITIAL_MATRIX, &
                         air_data%reuse(our_level)%reuse_mat(MAT_AFC_DROP), &
-                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE.)
+                        our_level = our_level, is_row_fine = .TRUE., is_col_fine = .FALSE., &
+                        kokkos_is_views_handle = air_data%kokkos_is_views_handle)
             end if
          end if                  
 

src/C_PETSc_Interfaces.F90

@@ -337,47 +337,51 @@ end subroutine MatSetAllValues_cpu
 
    interface   
 
-      subroutine create_VecISCopyLocal_kokkos(max_levels_input) &
+      subroutine create_VecISCopyLocal_kokkos(max_levels_input, handle) &
          bind(c, name="create_VecISCopyLocal_kokkos")
          use iso_c_binding
          integer(c_int), value :: max_levels_input
-      end subroutine create_VecISCopyLocal_kokkos         
-
-   end interface  
-
-   interface   
-
-      subroutine destroy_VecISCopyLocal_kokkos() &
+         type(c_ptr) :: handle
+      end subroutine create_VecISCopyLocal_kokkos
+
+   end interface
+
+   interface
+
+      subroutine destroy_VecISCopyLocal_kokkos(handle) &
          bind(c, name="destroy_VecISCopyLocal_kokkos")
          use iso_c_binding
-      end subroutine destroy_VecISCopyLocal_kokkos         
-
-   end interface    
-
-   interface   
-
-      subroutine set_VecISCopyLocal_kokkos_our_level(our_level, global_row_start, index_fine, index_coarse) &
+         type(c_ptr) :: handle
+      end subroutine destroy_VecISCopyLocal_kokkos
+
+   end interface
+
+   interface
+
+      subroutine set_VecISCopyLocal_kokkos_our_level(handle, our_level, global_row_start, index_fine, index_coarse) &
          bind(c, name="set_VecISCopyLocal_kokkos_our_level")
          use iso_c_binding
+         type(c_ptr), value :: handle
          integer(c_int), value :: our_level
          integer(PFLARE_PETSCINT_C_KIND), value :: global_row_start
          integer(c_long_long) :: index_fine
          integer(c_long_long) :: index_coarse
-      end subroutine set_VecISCopyLocal_kokkos_our_level         
- 
+      end subroutine set_VecISCopyLocal_kokkos_our_level
+
    end interface
-   
-   interface   
-      
-      subroutine VecISCopyLocal_kokkos(our_level, fine_int, vfull, mode_int, vreduced) &
+
+   interface
+
+      subroutine VecISCopyLocal_kokkos(handle, our_level, fine_int, vfull, mode_int, vreduced) &
          bind(c, name="VecISCopyLocal_kokkos")
          use iso_c_binding
+         type(c_ptr), value :: handle
          integer(c_int), value :: our_level, fine_int, mode_int
          integer(c_long_long) :: vfull
          integer(c_long_long) :: vreduced
-      end subroutine VecISCopyLocal_kokkos         
- 
-   end interface    
+      end subroutine VecISCopyLocal_kokkos
+
+   end interface
 
    interface   
 
@@ -594,12 +598,14 @@ end subroutine MatAXPY_kokkos
 
       subroutine MatCreateSubMatrix_kokkos(A_array, is_row, is_col, &
                      reuse_int, B_array, &
+                     kokkos_is_views_handle, &
                      our_level, is_row_fine_int, is_col_fine_int) &
          bind(c, name="MatCreateSubMatrix_kokkos")
          use iso_c_binding
          integer(c_long_long) :: A_array
          integer(c_long_long) :: B_array
          integer(c_long_long) :: is_row, is_col
+         type(c_ptr), value :: kokkos_is_views_handle
          integer(c_int), value :: our_level, is_row_fine_int, is_col_fine_int, reuse_int
 
       end subroutine MatCreateSubMatrix_kokkos

src/FC_Smooth.F90

@@ -82,13 +82,14 @@ subroutine create_VecISCopyLocalWrapper(air_data, our_level, mat_type, input_mat
                mat_type == MATAIJKOKKOS) then
 
             ! Build in case not built yet
-            call create_VecISCopyLocal_kokkos(air_data%options%max_levels)
+            call create_VecISCopyLocal_kokkos(air_data%options%max_levels, air_data%kokkos_is_views_handle)
             call MatGetOwnershipRange(input_mat, global_row_start, global_row_end_plus_one, ierr)
 
             ! Copy the IS's over to the device
             is_fine_array = air_data%IS_fine_index(our_level)%v
             is_coarse_array = air_data%IS_coarse_index(our_level)%v
-            call set_VecISCopyLocal_kokkos_our_level(our_level, global_row_start, is_fine_array, is_coarse_array)
+            call set_VecISCopyLocal_kokkos_our_level(air_data%kokkos_is_views_handle, our_level, &
+                     global_row_start, is_fine_array, is_coarse_array)
 
          end if
 #endif
@@ -122,8 +123,8 @@ subroutine destroy_VecISCopyLocalWrapper(air_data, our_level)
          end if 
 
       else
-#if defined(PETSC_HAVE_KOKKOS) 
-         call destroy_VecISCopyLocal_kokkos()
+#if defined(PETSC_HAVE_KOKKOS)
+         call destroy_VecISCopyLocal_kokkos(air_data%kokkos_is_views_handle)
 #endif
       end if
 
@@ -181,7 +182,7 @@ subroutine VecISCopyLocalWrapper(air_data, our_level, fine, vfull, mode, vreduce
                   if (fine) fine_int = 1
                   vfull_array = vfull%v
                   vreduced_array = vreduced%v
-                  call VecISCopyLocal_kokkos(our_level, fine_int, vfull_array, &
+                  call VecISCopyLocal_kokkos(air_data%kokkos_is_views_handle, our_level, fine_int, vfull_array, &
                            mode_int, vreduced_array)
 
                   ! If debugging do a comparison between CPU and Kokkos results
@@ -245,7 +246,7 @@ subroutine VecISCopyLocalWrapper(air_data, our_level, fine, vfull, mode, vreduce
                   if (fine) fine_int = 1
                   vfull_array = vfull%v
                   vreduced_array = vreduced%v
-                  call VecISCopyLocal_kokkos(our_level, fine_int, vfull_array, &
+                  call VecISCopyLocal_kokkos(air_data%kokkos_is_views_handle, our_level, fine_int, vfull_array, &
                            mode_int, vreduced_array)
 
                   ! If debugging do a comparison between CPU and Kokkos results
@@ -293,7 +294,7 @@ subroutine VecISCopyLocalWrapper(air_data, our_level, fine, vfull, mode, vreduce
                   if (fine) fine_int = 1
                   vfull_array = vfull%v
                   vreduced_array = vreduced%v
-                  call VecISCopyLocal_kokkos(our_level, fine_int, vfull_array, &
+                  call VecISCopyLocal_kokkos(air_data%kokkos_is_views_handle, our_level, fine_int, vfull_array, &
                            mode_int, vreduced_array)
 
                   ! If debugging do a comparison between CPU and Kokkos results
@@ -357,7 +358,7 @@ subroutine VecISCopyLocalWrapper(air_data, our_level, fine, vfull, mode, vreduce
                   if (fine) fine_int = 1
                   vfull_array = vfull%v
                   vreduced_array = vreduced%v
-                  call VecISCopyLocal_kokkos(our_level, fine_int, vfull_array, &
+                  call VecISCopyLocal_kokkos(air_data%kokkos_is_views_handle, our_level, fine_int, vfull_array, &
                            mode_int, vreduced_array)
 
                   ! If debugging do a comparison between CPU and Kokkos results