Skip to content

Loci communicator refactor #335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
EdwardALuke wants to merge 7 commits into
base: dev
Choose a base branch
from
Draft

Loci communicator refactor #335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9932661
removed unused elements of the rule API.
EdwardALuke May 18, 2026
b92dea4
Phase 0 updates
EdwardALuke May 18, 2026
4f4e8df
refactoring code to use fact_db communicator instead of MPI_COMM_WORLD
EdwardALuke May 19, 2026
1f38dcd
Update to get communicator size and rank from fact_db
EdwardALuke May 21, 2026
143441e
Extended to replace use of MPI_COMM_WORLD with new API calls, support…
EdwardALuke May 21, 2026
da32dea
More complete threading of MPI communicator through distributed utili…
EdwardALuke May 22, 2026
6df53d5
Fixing a few deadlock issues when running on a sub communicator.
EdwardALuke May 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions quickTest/FVM/IOgrid.loci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,12 @@ namespace flowPsi {


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

Loci::writeContainer(file_id,"pos",pos.Rep()) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

void dump_pos(const sequence &seq,const_store<vector3d<Loci::FADd> > &pos,
Expand All @@ -77,7 +78,8 @@ namespace flowPsi {


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

store<vector3d<double> > pcopy ;
entitySet dom = pos.domain() ;
Expand All @@ -87,7 +89,7 @@ namespace flowPsi {
} ENDFORALL ;
Loci::writeContainer(file_id,"pos",pcopy.Rep()) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

void dump_pos(const sequence &seq,const_store<vector3d<Loci::FAD2d> > &pos,
Expand All @@ -108,7 +110,8 @@ namespace flowPsi {


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

store<vector3d<double> > pcopy ;
entitySet dom = pos.domain() ;
Expand All @@ -118,7 +121,7 @@ namespace flowPsi {
} ENDFORALL ;
Loci::writeContainer(file_id,"pos",pcopy.Rep()) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}


Expand Down Expand Up @@ -260,9 +263,9 @@ namespace flowPsi {

entitySet set(seq) ;
hid_t file_id = Loci::createUnorderedFile(filename.c_str(),set) ;
Loci::writeUnorderedStore(file_id,$area,set,"area") ;
Loci::writeUnorderedStore(file_id,$facecenter,set,"facecenter") ;
Loci::closeUnorderedFile(file_id) ;
Loci::writeUnorderedStore(file_id,$area,set,"area",Loci::exec_current_fact_db->get_comm()) ;
Loci::writeUnorderedStore(file_id,$facecenter,set,"facecenter",Loci::exec_current_fact_db->get_comm()) ;
Loci::closeUnorderedFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
} ;

}
Expand Down
28 changes: 16 additions & 12 deletions quickTest/FVM/IOplot.loci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,11 +279,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

Loci::writeContainer(file_id,sname,c2n.Rep()) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

scalar_node_output::scalar_node_output(const char *vname,
Expand Down Expand Up @@ -359,11 +360,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

Loci::writeContainer(file_id,sname,c2n.Rep()) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}


Expand Down Expand Up @@ -410,11 +412,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

Loci::writeContainer(file_id,sname,c2n.Rep()) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

OUTPUT_SCALAR_ALWAYS("cell2nodeD(temperature)",t) ;
Expand Down Expand Up @@ -444,11 +447,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw


hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC,
H5P_DEFAULT, H5P_DEFAULT) ;
H5P_DEFAULT, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm()) ;

Loci::writeContainer(file_id,sname,var) ;

Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

$rule singleton(OUTPUT<-plot_postfix,modelName,Pambient),conditional(do_plot),
Expand Down Expand Up @@ -483,8 +487,8 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw
// cout << "opening file " << filename << endl ;
entitySet set(seq) ;
hid_t file_id = Loci::createUnorderedFile(filename.c_str(),set) ;
Loci::writeUnorderedStore(file_id,var,set,value_name.c_str()) ;
Loci::closeUnorderedFile(file_id) ;
Loci::writeUnorderedStore(file_id,var,set,value_name.c_str(),Loci::exec_current_fact_db->get_comm()) ;
Loci::closeUnorderedFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

OUTPUT_BNDRY_SCALAR("gagePressure_f",bpg,"ci") ;
Expand Down Expand Up @@ -516,8 +520,8 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw
// cout << "opening file " << filename << endl ;
entitySet set(seq) ;
hid_t file_id = Loci::createUnorderedFile(filename.c_str(),set) ;
Loci::writeUnorderedStore(file_id,var,set,value_name.c_str()) ;
Loci::closeUnorderedFile(file_id) ;
Loci::writeUnorderedStore(file_id,var,set,value_name.c_str(),Loci::exec_current_fact_db->get_comm()) ;
Loci::closeUnorderedFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
}

OUTPUT_BNDRY_VECTOR("wall_stress",tau,"AllViscousBCs") ;
Expand Down
7 changes: 4 additions & 3 deletions quickTest/FVM/solverIC.loci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -579,15 +579,16 @@ namespace flowPsi {
has_file = 2 ;
}
}
MPI_Bcast(&has_file,1,MPI_INT,0,MPI_COMM_WORLD) ;
MPI_Bcast(&has_file,1,MPI_INT,0,Loci::exec_current_fact_db->get_comm()) ;
if(has_file == 2) {
$[Once] {
cerr << "unable to find file '" << filename << "'" << endl ;
}
Loci::Abort() ;
}
hid_t file_id = Loci::hdf5OpenFile(filename.c_str(),
H5F_ACC_RDONLY, H5P_DEFAULT);
H5F_ACC_RDONLY, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm());
entitySet read_set = entitySet(seq) ;
Loci::readContainer(file_id,"temperature",$temperature_ic.Rep(),read_set) ;
Loci::readContainer(file_id,"gagePressure",$gagePressure_ic.Rep(),read_set) ;
Expand All @@ -598,7 +599,7 @@ namespace flowPsi {
FORALL(read_set,ii) {
$gagePressure_ic[ii] += dp ;
} ENDFORALL ;
Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;
} ;


Expand Down
7 changes: 4 additions & 3 deletions quickTest/FVM/solverTimestep.loci
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,13 +114,14 @@ namespace flowPsi {
}

void ic_time_from_file::compute(const sequence &seq) {
if(Loci::GLOBAL_AND(seq==EMPTY))
if(Loci::GLOBAL_AND(seq==EMPTY, Loci::exec_current_fact_db->get_comm()))
return ;

string filename = *icfile + "flowVars_" + *modelName ;

hid_t file_id = Loci::hdf5OpenFile((filename).c_str(),
H5F_ACC_RDONLY, H5P_DEFAULT);
H5F_ACC_RDONLY, H5P_DEFAULT,
Loci::exec_current_fact_db->get_comm());
entitySet dom = ~EMPTY ;

// Set up defaults
Expand All @@ -129,7 +130,7 @@ namespace flowPsi {
// Read in restart values
Loci::readContainer(file_id,"stime",stime.Rep(),dom) ;
Loci::readContainer(file_id,"ncycle",ncycle.Rep(),dom) ;
Loci::hdf5CloseFile(file_id) ;
Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ;

if(*stime == 0)
*ncycle = 0 ;
Expand Down
Loading