diff --git a/quickTest/FVM/IOgrid.loci b/quickTest/FVM/IOgrid.loci index f57dd8eb..4a933cb1 100644 --- a/quickTest/FVM/IOgrid.loci +++ b/quickTest/FVM/IOgrid.loci @@ -52,11 +52,12 @@ namespace flowPsi { hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; Loci::writeContainer(file_id,"pos",pos.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } void dump_pos(const sequence &seq,const_store > &pos, @@ -77,7 +78,8 @@ namespace flowPsi { hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; store > pcopy ; entitySet dom = pos.domain() ; @@ -87,7 +89,7 @@ namespace flowPsi { } ENDFORALL ; Loci::writeContainer(file_id,"pos",pcopy.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } void dump_pos(const sequence &seq,const_store > &pos, @@ -108,7 +110,8 @@ namespace flowPsi { hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; store > pcopy ; entitySet dom = pos.domain() ; @@ -118,7 +121,7 @@ namespace flowPsi { } ENDFORALL ; Loci::writeContainer(file_id,"pos",pcopy.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } @@ -260,9 +263,9 @@ namespace flowPsi { entitySet set(seq) ; hid_t file_id = Loci::createUnorderedFile(filename.c_str(),set) ; - Loci::writeUnorderedStore(file_id,$area,set,"area") ; - Loci::writeUnorderedStore(file_id,$facecenter,set,"facecenter") ; - Loci::closeUnorderedFile(file_id) ; + Loci::writeUnorderedStore(file_id,$area,set,"area",Loci::exec_current_fact_db->get_comm()) ; + Loci::writeUnorderedStore(file_id,$facecenter,set,"facecenter",Loci::exec_current_fact_db->get_comm()) ; + Loci::closeUnorderedFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } ; } diff --git a/quickTest/FVM/IOplot.loci b/quickTest/FVM/IOplot.loci index 5366bc3a..665f24a1 100644 --- a/quickTest/FVM/IOplot.loci +++ b/quickTest/FVM/IOplot.loci @@ -279,11 +279,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; Loci::writeContainer(file_id,sname,c2n.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } scalar_node_output::scalar_node_output(const char *vname, @@ -359,11 +360,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; Loci::writeContainer(file_id,sname,c2n.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } @@ -410,11 +412,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; Loci::writeContainer(file_id,sname,c2n.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } OUTPUT_SCALAR_ALWAYS("cell2nodeD(temperature)",t) ; @@ -444,11 +447,12 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()) ; Loci::writeContainer(file_id,sname,var) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } $rule singleton(OUTPUT<-plot_postfix,modelName,Pambient),conditional(do_plot), @@ -483,8 +487,8 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw // cout << "opening file " << filename << endl ; entitySet set(seq) ; hid_t file_id = Loci::createUnorderedFile(filename.c_str(),set) ; - Loci::writeUnorderedStore(file_id,var,set,value_name.c_str()) ; - Loci::closeUnorderedFile(file_id) ; + Loci::writeUnorderedStore(file_id,var,set,value_name.c_str(),Loci::exec_current_fact_db->get_comm()) ; + Loci::closeUnorderedFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } OUTPUT_BNDRY_SCALAR("gagePressure_f",bpg,"ci") ; @@ -516,8 +520,8 @@ $rule pointwise(boundary::cell2nodeD(X)<-boundary_scalar_sumD(X),boundary_nodalw // cout << "opening file " << filename << endl ; entitySet set(seq) ; hid_t file_id = Loci::createUnorderedFile(filename.c_str(),set) ; - Loci::writeUnorderedStore(file_id,var,set,value_name.c_str()) ; - Loci::closeUnorderedFile(file_id) ; + Loci::writeUnorderedStore(file_id,var,set,value_name.c_str(),Loci::exec_current_fact_db->get_comm()) ; + Loci::closeUnorderedFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } OUTPUT_BNDRY_VECTOR("wall_stress",tau,"AllViscousBCs") ; diff --git a/quickTest/FVM/solverIC.loci b/quickTest/FVM/solverIC.loci index 62d44b3f..a97b4202 100644 --- a/quickTest/FVM/solverIC.loci +++ b/quickTest/FVM/solverIC.loci @@ -579,7 +579,7 @@ namespace flowPsi { has_file = 2 ; } } - MPI_Bcast(&has_file,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&has_file,1,MPI_INT,0,Loci::exec_current_fact_db->get_comm()) ; if(has_file == 2) { $[Once] { cerr << "unable to find file '" << filename << "'" << endl ; @@ -587,7 +587,8 @@ namespace flowPsi { Loci::Abort() ; } hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT); + H5F_ACC_RDONLY, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()); entitySet read_set = entitySet(seq) ; Loci::readContainer(file_id,"temperature",$temperature_ic.Rep(),read_set) ; Loci::readContainer(file_id,"gagePressure",$gagePressure_ic.Rep(),read_set) ; @@ -598,7 +599,7 @@ namespace flowPsi { FORALL(read_set,ii) { $gagePressure_ic[ii] += dp ; } ENDFORALL ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; } ; diff --git a/quickTest/FVM/solverTimestep.loci b/quickTest/FVM/solverTimestep.loci index 974d9489..c936d06f 100644 --- a/quickTest/FVM/solverTimestep.loci +++ b/quickTest/FVM/solverTimestep.loci @@ -114,13 +114,14 @@ namespace flowPsi { } void ic_time_from_file::compute(const sequence &seq) { - if(Loci::GLOBAL_AND(seq==EMPTY)) + if(Loci::GLOBAL_AND(seq==EMPTY, Loci::exec_current_fact_db->get_comm())) return ; string filename = *icfile + "flowVars_" + *modelName ; hid_t file_id = Loci::hdf5OpenFile((filename).c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT); + H5F_ACC_RDONLY, H5P_DEFAULT, + Loci::exec_current_fact_db->get_comm()); entitySet dom = ~EMPTY ; // Set up defaults @@ -129,7 +130,7 @@ namespace flowPsi { // Read in restart values Loci::readContainer(file_id,"stime",stime.Rep(),dom) ; Loci::readContainer(file_id,"ncycle",ncycle.Rep(),dom) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; if(*stime == 0) *ncycle = 0 ; diff --git a/src/Doc/communicator_evolution_plan.md b/src/Doc/communicator_evolution_plan.md new file mode 100644 index 00000000..335cb468 --- /dev/null +++ b/src/Doc/communicator_evolution_plan.md @@ -0,0 +1,341 @@ +# Communicator Evolution Plan + +## Motivation + +The Loci framework currently hardcodes `MPI_COMM_WORLD` as the +communicator for all parallel operations. This prevents building +parallel schedules over processor subsets --- a capability needed for +coupled multi-physics, sub-domain solvers, and ensemble runs. + +The framework already accepts an `MPI_Comm` parameter in many +lower-level APIs (e.g. `redistribute`, `GLOBAL_OR`, `hdf5CreateFile`), +but those parameters default to `MPI_COMM_WORLD`. The `fact_db` +class, which is the central data manager, has no communicator member at +all. The global variables `MPI_processes` and `MPI_rank` are likewise +derived from `MPI_COMM_WORLD` during `Loci::Init()`. + +The plan below describes a phased, backward-compatible evolution that: + +1. Stores a communicator in `fact_db` and derives rank/size from it. +2. Threads the communicator through APIs that currently hardcode + `MPI_COMM_WORLD`. +3. Introduces a compile-time feature flag (`LOCI_STRICT_COMM`) that + removes the `MPI_COMM_WORLD` defaults, forcing solver codes to + pass communicators explicitly. + +Each phase is designed so that **existing solver codes compile and run +without modification** until a project opts in to strict mode. + +--- + +## Phase 0 — Communicator in `fact_db` + +### 0.1 Add an `MPI_Comm` member to `fact_db` + +Add a private member and public accessors: + +```cpp +private: + MPI_Comm comm_ ; +public: + MPI_Comm get_comm() const { return comm_ ; } + void set_comm(MPI_Comm c) ; +``` + +The constructor initializes `comm_` to `MPI_COMM_WORLD`. +`set_comm` duplicates the communicator (via `MPI_Comm_dup`) so that +`fact_db` owns its own copy, and caches the corresponding rank and +size in two new private members (`comm_rank_`, `comm_size_`). + +### 0.2 Local rank/size accessors + +```cpp +int get_comm_rank() const { return comm_rank_ ; } +int get_comm_size() const { return comm_size_ ; } +``` + +These replace direct reads of the globals `Loci::MPI_rank` and +`Loci::MPI_processes` for any code that has access to a `fact_db`. + +### 0.3 Propagate the communicator inside `fact_db` + +Replace the two `MPI_Alltoall` calls in `fact_db::get_dist_alloc()` +(currently hardcoded to `MPI_COMM_WORLD`) with calls that use +`comm_`. Similarly update `fact_db::put_distribute_info()` and any +allocation helpers that reference the globals. + +### 0.4 Propagate to `distribute_info` + +Add an `MPI_Comm comm` member to the `distribute_info` struct so that +distributed metadata can carry the communicator to redistribution and +I/O routines downstream. + +--- + +## Phase 1 — Thread the communicator through internal APIs + +In this phase the communicator stored in `fact_db` is passed down to +internal subsystems. Because these APIs already accept an optional +`MPI_Comm` parameter with a default of `MPI_COMM_WORLD`, the changes +are mechanical: pass `facts.get_comm()` instead of relying on the +default. Note to accomplish this when no fact_db is present create a +get_exec_comm() function that returns the communicator from the +exec_current_fact_db unless this pointer is null, then return +MPI_COMM_WORLD. + +### 1.1 Scheduler (`scheduler.cc`) + +The scheduler holds a pointer to the active `fact_db`. All +`MPI_Allreduce` calls used for profiling and barrier-style operations +(approximately 30 call sites) should be updated to use the +communicator retrieved from the fact database. + +### 1.2 Grid reader and partitioning (`FVMGridReader.cc`, `distribute.cc`) + +The grid reader passes `MPI_COMM_WORLD` to collective operations, +partition routines (`ORBPartition`, `parSampleSort`, +`balanceDistribution`, ParMETIS wrappers), and I/O helpers. At each +call site, replace the literal `MPI_COMM_WORLD` with the communicator +obtained from the `fact_db` that is already available in context. + +For the partitioning utilities themselves, the signatures already +accept `MPI_Comm`; no signature changes are needed. + +### 1.3 Data redistribution (`distribute_container.cc`, `Map.cc`, etc.) + +Container redistribution helpers call `MPI_Alltoall` / +`MPI_Alltoallv` with `MPI_COMM_WORLD`. These functions should accept +an `MPI_Comm` parameter (defaulting to `MPI_COMM_WORLD`) and the call +sites in the framework should forward the `fact_db` communicator. + +### 1.4 Computation support (`comp_tools.cc`, `comp_reduce.cc`) + +Point-to-point communication (`MPI_Irecv`, `MPI_Send`) and packing +routines use `MPI_COMM_WORLD`. Add an `MPI_Comm` parameter to +affected routines and forward it from the execution context, which has +access to the `fact_db`. + +### 1.5 I/O subsystem (`distribute_io.cc`, `multiStoreIO.cc/.h`, +`mpi_multiStoreIO.h`) + +The core I/O functions (`hdf5CreateFile`, `hdf5OpenFile`, +`writeContainer`, `readContainer`, `readContainerRAW`, etc.) already +accept `MPI_Comm`. The overloads without a communicator argument +forward to `MPI_COMM_WORLD`. No signature changes are needed here; +the callers should simply start passing `facts.get_comm()`. + +### 1.6 Dynamic execution (`dynamic_exec.cc`) + +Dynamic rule execution uses `MPI_Send`/`MPI_Recv`/`MPI_Probe` with +`MPI_COMM_WORLD`. The executing context carries a `fact_db` pointer; +extract the communicator from it. + +### 1.7 Pack/Unpack in store representations + +The templated `MPI_Pack`/`MPI_Unpack` calls in `gstore_impl.h`, +`gparameter_impl.h`, `gmultistore_impl.h`, `gmapvec_impl.h`, and GPU +variants use `MPI_COMM_WORLD`. Since packing is communicator- +independent in MPI (the communicator is only used to look up +datatypes), these can remain as `MPI_COMM_WORLD` or be parameterized +for consistency. + +### 1.8 PETSc integration (`petsc.loci`) + +PETSc objects (`VecCreate`, `MatCreate`, `KSPCreate`, +`PetscViewerASCIIOpen`) are created with `MPI_COMM_WORLD`. These +should use the communicator from the `fact_db` available through the +Loci rule context. This change enables PETSc solvers to operate on +processor subsets transparently. + +### 1.9 Overset and adaptation modules + +`FVMOverset/interpolation.loci`, `FVMOverset/motion.loci`, +`FVMAdapt/library/write_vog.cc`, and `FVMAdapt/library/color_matrix.cc` +contain many `MPI_COMM_WORLD` references. These modules operate on +rule-provided stores and have access to the `fact_db`. Forward the +communicator at each call site. + +### 1.10 Tool programs (`FVMtools/`) + +Standalone conversion tools (`ugrid2vog.cc`, `vogtools.cc`, +`fluent2vog.cc`, `cgns2vog.cc`, etc.) use `MPI_COMM_WORLD` directly. +Because these are not library code consumed by solver developers, they +can be updated independently and do not affect the public API. + +--- + +## Phase 2 — Update global state management + +### 2.1 Context-local rank and size + +Today the globals `Loci::MPI_processes` and `Loci::MPI_rank` are set +once in `Loci::Init()` from `MPI_COMM_WORLD`. After Phase 0, code +inside the framework that has access to a `fact_db` should prefer +`facts.get_comm_rank()` and `facts.get_comm_size()`. + +To avoid breaking existing solver code that reads these globals, they +will continue to exist and remain set to the `MPI_COMM_WORLD` values. +A deprecation warning should be emitted (or documented) advising +solvers to migrate to the `fact_db` accessors. + +### 2.2 `Loci::SetDefaultComm()` post-initialization + +Since `Loci::Init()` calls `MPI_Init()` internally, a communicator +cannot be passed to it (MPI communicators are not valid before +`MPI_Init`). Instead, a separate function is provided: + +```cpp +void SetDefaultComm(MPI_Comm comm) ; +``` + +This must be called **after** `Loci::Init()`. It updates the global +`MPI_processes` and `MPI_rank` to reflect the given communicator. +Solvers that wish to run on a sub-communicator call this before +setting up the `fact_db`. + +### 2.3 `exec_current_fact_db` communicator forwarding + +The global `exec_current_fact_db` pointer is used by convenience +functions like `collect_entitySet`. These should be updated to pull +the communicator from `exec_current_fact_db->get_comm()` rather than +assuming `MPI_COMM_WORLD`. + +--- + +## Phase 3 — Feature flag for strict communicator mode + +### 3.1 Define `LOCI_STRICT_COMM` + +Introduce a compile-time macro following the pattern established by +`LOCI_COMPAT_MODE1`. When `LOCI_STRICT_COMM` is defined, the +`= MPI_COMM_WORLD` default is removed from every public API function +signature. This forces downstream solver codes to supply a +communicator explicitly, ensuring correctness when running on a +sub-communicator. + +The affected public APIs include: + +| Header | Functions / Methods | +|--------|---------------------| +| `store_rep.h` | `storeRep::redistribute()` (both overloads) | +| `distribute.h` | `GLOBAL_OR`, `GLOBAL_AND`, `GLOBAL_MAX`, `GLOBAL_MIN` | +| `distribute_io.h` | `hdf5CreateFile`, `hdf5OpenFile`, overloads of `writeContainer`, `readContainer`, etc. | +| `distribute_long.h` | `g_GLOBAL_MAX`, `g_GLOBAL_MIN`, `g_all_collect_entitySet`, etc. | +| `partition.h` | `transposePtn` | +| All concrete store/map headers | `redistribute`, `recompose`, `split_redistribute`, `expand`, `inplace_compose` | + +Implementation strategy using the preprocessor: + +```cpp +#ifdef LOCI_STRICT_COMM +#define LOCI_DEFAULT_COMM +#else +#define LOCI_DEFAULT_COMM = MPI_COMM_WORLD +#endif +``` + +Then each declaration becomes: + +```cpp +int GLOBAL_OR(int b, MPI_Comm comm LOCI_DEFAULT_COMM) ; +``` + +### 3.2 Convenience overloads under compat mode + +The existing convenience overloads that omit `MPI_Comm` (e.g. the +one-argument `hdf5OpenFile`, `all_collect_vectors(e)`) should be +wrapped in `#ifndef LOCI_STRICT_COMM` guards. Under strict mode, +only the explicit-communicator versions are available. + +### 3.3 Solver migration path + +Solver codes that wish to support sub-communicators can adopt +`LOCI_STRICT_COMM` at their own pace: + +1. Add `-DLOCI_STRICT_COMM` to their build flags. +2. Fix all resulting compilation errors by passing + `facts.get_comm()` where a communicator is now required. +3. Verify that the solver works correctly with a sub-communicator by + running on a subset of MPI ranks. + +Solver codes that do not define `LOCI_STRICT_COMM` continue to +compile and run exactly as before. + +--- + +## Phase 4 — Validation and testing + +### 4.1 Unit tests + +Write tests that create a `fact_db` on a sub-communicator +(`MPI_Comm_split`) and exercise: + +- Entity allocation (`get_distributed_alloc`). +- Store redistribution. +- Collective reductions (`GLOBAL_OR`, etc.). +- HDF5 parallel I/O. +- PETSc object creation. + +### 4.2 Integration test with an existing solver + +Run an existing Loci-based solver (e.g. CHEM) in two modes: + +- **Full communicator**: all ranks participate. Verify bit-identical + results with the pre-change code. +- **Sub-communicator**: split ranks into two groups, run independent + solver instances on each. Verify correctness. + +### 4.3 Performance regression + +Ensure that passing the communicator through the extra parameter does +not introduce measurable overhead. The communicator is a lightweight +handle; no performance impact is expected. + +--- + +## Summary of API changes + +| Change | Backward compatible? | Requires solver update? | +|--------|:--------------------:|:-----------------------:| +| `fact_db::get_comm()` / `set_comm()` | Yes | No | +| `fact_db::get_comm_rank()` / `get_comm_size()` | Yes | No | +| `Loci::SetDefaultComm(comm)` | Yes (new function) | No | +| Internal call sites forwarding communicator | Yes | No | +| `LOCI_STRICT_COMM` flag removing defaults | Opt-in | Only if flag is set | +| Deprecation of `Loci::MPI_rank` / `MPI_processes` | Yes (still available) | Recommended migration | + +--- + +## File impact estimate + +| Area | Files | Approximate call sites | +|------|------:|-----------------------:| +| `fact_db` (header + impl) | 2 | ~5 | +| Scheduler | 1 | ~30 | +| Grid reader / partitioner | 3 | ~80 | +| Data redistribution | 4 | ~30 | +| Computation / reduction | 3 | ~25 | +| I/O subsystem | 5 | ~40 | +| Dynamic execution | 1 | ~50 | +| Store/map representations (headers) | ~20 | ~60 (default parameter macro) | +| PETSc integration | 1 | ~10 | +| Overset / adaptation | 6 | ~50 | +| Tool programs | 8 | ~100 | +| Initialization | 1 | ~5 | +| **Total** | **~55** | **~485** | + +--- + +## Recommended implementation order + +1. **Phase 0** — smallest change, highest value. Enables all + downstream work and is independently useful for framework + developers. +2. **Phase 2** — global state management must align with Phase 0 + before internal subsystems are updated. +3. **Phase 1** — bulk of the work; can be done subsystem by subsystem + in any order. Each subsystem can be validated independently. +4. **Phase 3** — introduce the feature flag only after Phase 1 is + complete and tested. +5. **Phase 4** — continuous throughout, with a final comprehensive + pass after Phase 3. diff --git a/src/FVMAdapt/cellplan_io.loci b/src/FVMAdapt/cellplan_io.loci index 327418ab..16f3330a 100644 --- a/src/FVMAdapt/cellplan_io.loci +++ b/src/FVMAdapt/cellplan_io.loci @@ -130,6 +130,7 @@ class get_DBcellPlan : public pointwise_rule{ readPlan = Loci::DataXFER_DB.getItem((*planDB_par).c_str()) ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; if(dist==0) { FORALL(dom,ii) { cellPlan[ii] = readPlan[ii] ; @@ -139,7 +140,7 @@ class get_DBcellPlan : public pointwise_rule{ tmp.allocate(dom) ; Loci::storeRepP tmpRep = tmp.Rep() ; int offset = 0 ; - File2LocalOrder(tmpRep,dom,readPlan.Rep(),offset,dist,MPI_COMM_WORLD) ; + File2LocalOrder(tmpRep,dom,readPlan.Rep(),offset,dist,comm) ; FORALL(dom,ii) { cellPlan[ii].swap(tmp[ii]) ; } ENDFORALL ; @@ -175,6 +176,7 @@ class get_balanced_cellPlanDB : public pointwise_rule{ readPlan = Loci::DataXFER_DB.getItem((*planDB_par).c_str()) ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; if(dist==0) { FORALL(dom,ii) { cellPlan[ii] = readPlan[ii] ; @@ -184,7 +186,7 @@ class get_balanced_cellPlanDB : public pointwise_rule{ tmp.allocate(dom) ; Loci::storeRepP tmpRep = tmp.Rep() ; int offset = 0 ; - File2LocalOrder(tmpRep,dom,readPlan.Rep(),offset,dist,MPI_COMM_WORLD) ; + File2LocalOrder(tmpRep,dom,readPlan.Rep(),offset,dist,comm) ; FORALL(dom,ii) { cellPlan[ii].swap(tmp[ii]) ; } ENDFORALL ; @@ -296,6 +298,7 @@ class get_parentPlanDB : public pointwise_rule{ readPlan = Loci::DataXFER_DB.getItem((*parent_planDB_par).c_str()) ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; if(dist==0) { FORALL(dom,ii) { parentPlan[ii]= readPlan[ii] ; @@ -305,7 +308,7 @@ class get_parentPlanDB : public pointwise_rule{ tmp.allocate(dom) ; Loci::storeRepP tmpRep = tmp.Rep() ; int offset = 0 ; - File2LocalOrder(tmpRep,dom,readPlan.Rep(),offset,dist,MPI_COMM_WORLD) ; + File2LocalOrder(tmpRep,dom,readPlan.Rep(),offset,dist,comm) ; FORALL(dom,ii) { parentPlan[ii].swap(tmp[ii]) ; } ENDFORALL ; @@ -392,11 +395,11 @@ public: hid_t file_id; file_id = Loci::hdf5CreateFile((*outfile_par).c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, Loci::exec_current_fact_db->get_comm()) ; if(Loci::MPI_rank == 0)cout << "writing cellplan into " << *outfile_par << endl; Loci::writeContainer(file_id,"cellPlan",tmpPlan.Rep()) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, Loci::exec_current_fact_db->get_comm()) ; if(Loci::MPI_rank == 0) cout << "Finish writing cellPlan " << endl; @@ -425,6 +428,7 @@ public: entitySet dom = entitySet(seq); fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; if(dist==0) { store > pcopy ; pcopy.allocate(dom) ; @@ -437,7 +441,7 @@ public: int offset = 0 ; Loci::storeRepP vardist = Loci::Local2FileOrder(tmpPlan.Rep(),dom,offset, dist, - MPI_COMM_WORLD) ; + comm) ; vardist->shift(offset) ; Loci::DataXFER_DB.insertItem((*outDB_par).c_str(),vardist) ; } @@ -496,6 +500,7 @@ public: entitySet dom = entitySet(seq); fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; if(dist==0) { store pcopy ; pcopy.allocate(dom) ; @@ -508,7 +513,7 @@ public: int offset = 0 ; Loci::storeRepP vardist = Loci::Local2FileOrder(num_fine_cells.Rep(),dom,offset, dist, - MPI_COMM_WORLD) ; + comm) ; vardist->shift(offset) ; Loci::DataXFER_DB.insertItem((*outDB_par).c_str(),vardist) ; } diff --git a/src/FVMAdapt/library/classify_cell.cc b/src/FVMAdapt/library/classify_cell.cc index c0bd730a..72ad1494 100644 --- a/src/FVMAdapt/library/classify_cell.cc +++ b/src/FVMAdapt/library/classify_cell.cc @@ -55,7 +55,7 @@ namespace Loci{ int fkeyspace = upper.getRangeKeySpace() ; vector init_ptn = facts.get_init_ptn(fkeyspace) ; entitySet out_of_dom = face_dom - face2node.domain(); - face2node.setRep(MapRepP(face2node.Rep())->expand(out_of_dom, init_ptn)); + face2node.setRep(MapRepP(face2node.Rep())->expand(out_of_dom, init_ptn, facts.get_comm())); } entitySet hexcell; diff --git a/src/FVMAdapt/library/color_matrix.cc b/src/FVMAdapt/library/color_matrix.cc index 81d2c6a5..0ad60d25 100644 --- a/src/FVMAdapt/library/color_matrix.cc +++ b/src/FVMAdapt/library/color_matrix.cc @@ -43,7 +43,7 @@ using Loci::UNIVERSE_MIN; namespace Loci{ - std::vector all_collect_sizes(int size); + std::vector all_collect_sizes(int size, MPI_Comm comm); // Convert container from local numbering to file numbering // pass in store rep pointer: sp // entitySet to write: dom @@ -188,6 +188,7 @@ namespace Loci{ //reorder store first, from local to io entities fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; constraint my_faces, my_geom_cells; entitySet my_entities = dist->my_entities ; my_faces = Loci::exec_current_fact_db->get_variable("faces"); @@ -209,24 +210,24 @@ namespace Loci{ int offset = 0; store pos_io; - pos_io = Loci::Local2FileOrder(pos, local_nodes, offset, dist, MPI_COMM_WORLD) ; + pos_io = Loci::Local2FileOrder(pos, local_nodes, offset, dist, comm) ; entitySet file_nodes = pos_io.domain(); offset = 0; store edge_inner_nodes; - edge_inner_nodes = Loci::Local2FileOrder(inner_nodes_edge.Rep(),local_edges,offset,dist,MPI_COMM_WORLD) ; + edge_inner_nodes = Loci::Local2FileOrder(inner_nodes_edge.Rep(),local_edges,offset,dist,comm) ; entitySet file_edges = edge_inner_nodes.domain(); offset= 0; // Create container vardist that store cell_inner_nodes; - cell_inner_nodes = Loci::Local2FileOrder(inner_nodes_cell.Rep(),local_cells,offset,dist,MPI_COMM_WORLD) ; + cell_inner_nodes = Loci::Local2FileOrder(inner_nodes_cell.Rep(),local_cells,offset,dist,comm) ; entitySet file_cells = cell_inner_nodes.domain(); offset= 0; // Create container vardist that store face_inner_nodes; - face_inner_nodes = Loci::Local2FileOrder(inner_nodes_face.Rep(),local_faces,offset,dist,MPI_COMM_WORLD) ; + face_inner_nodes = Loci::Local2FileOrder(inner_nodes_face.Rep(),local_faces,offset,dist,comm) ; entitySet file_faces = face_inner_nodes.domain(); @@ -234,7 +235,7 @@ namespace Loci{ int local_pos_size = file_nodes.size(); std::vector pos_sizes(Loci::MPI_processes) ; MPI_Gather(&local_pos_size,1,MPI_INT, - &pos_sizes[0],1,MPI_INT,0,MPI_COMM_WORLD) ; + &pos_sizes[0],1,MPI_INT,0,comm) ; @@ -257,16 +258,16 @@ namespace Loci{ std::vector inner_edge_nodes_sizes(Loci::MPI_processes); MPI_Gather(&num_local_edge_nodes,1,MPI_INT, - &inner_edge_nodes_sizes[0],1,MPI_INT,0,MPI_COMM_WORLD) ; + &inner_edge_nodes_sizes[0],1,MPI_INT,0,comm) ; std::vector inner_cell_nodes_sizes(Loci::MPI_processes); MPI_Gather(&num_local_cell_nodes,1,MPI_INT, - &inner_cell_nodes_sizes[0],1,MPI_INT,0,MPI_COMM_WORLD) ; + &inner_cell_nodes_sizes[0],1,MPI_INT,0,comm) ; std::vector inner_face_nodes_sizes(Loci::MPI_processes); MPI_Gather(&num_local_face_nodes,1,MPI_INT, - &inner_face_nodes_sizes[0],1,MPI_INT,0,MPI_COMM_WORLD) ; + &inner_face_nodes_sizes[0],1,MPI_INT,0,comm) ; @@ -337,10 +338,10 @@ namespace Loci{ if(pos_sizes[i] == 0) continue ; int flag = 0 ; - MPI_Send(&flag,1,MPI_INT,i,0,MPI_COMM_WORLD) ; + MPI_Send(&flag,1,MPI_INT,i,0,comm) ; std::vector rv(pos_sizes[i]) ; MPI_Status mstat ; - MPI_Recv(&rv[0],sizeof(vect3d)*pos_sizes[i],MPI_BYTE,i,1,MPI_COMM_WORLD, + MPI_Recv(&rv[0],sizeof(vect3d)*pos_sizes[i],MPI_BYTE,i,1,comm, &mstat) ; count = pos_sizes[i] ; H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &start, &stride, &count, NULL) ; @@ -378,10 +379,10 @@ namespace Loci{ if(inner_edge_nodes_sizes[i] == 0) continue ; int flag = 0 ; - MPI_Send(&flag,1,MPI_INT,i,2,MPI_COMM_WORLD) ; + MPI_Send(&flag,1,MPI_INT,i,2,comm) ; std::vector > rv(inner_edge_nodes_sizes[i]) ; MPI_Status mstat ; - MPI_Recv(&rv[0],sizeof(vector3d )*inner_edge_nodes_sizes[i],MPI_BYTE,i,3,MPI_COMM_WORLD, + MPI_Recv(&rv[0],sizeof(vector3d )*inner_edge_nodes_sizes[i],MPI_BYTE,i,3,comm, &mstat) ; count = inner_edge_nodes_sizes[i] ; H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &start, &stride, &count, NULL) ; @@ -419,10 +420,10 @@ namespace Loci{ if(inner_cell_nodes_sizes[i] == 0) continue ; int flag = 0 ; - MPI_Send(&flag,1,MPI_INT,i,4,MPI_COMM_WORLD) ; + MPI_Send(&flag,1,MPI_INT,i,4,comm) ; std::vector > rv(inner_cell_nodes_sizes[i]) ; MPI_Status mstat ; - MPI_Recv(&rv[0],sizeof(vector3d )*inner_cell_nodes_sizes[i],MPI_BYTE,i,5,MPI_COMM_WORLD, + MPI_Recv(&rv[0],sizeof(vector3d )*inner_cell_nodes_sizes[i],MPI_BYTE,i,5,comm, &mstat) ; count = inner_cell_nodes_sizes[i] ; H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &start, &stride, &count, NULL) ; @@ -460,10 +461,10 @@ namespace Loci{ if(inner_face_nodes_sizes[i] == 0) continue ; int flag = 0 ; - MPI_Send(&flag,1,MPI_INT,i,6,MPI_COMM_WORLD) ; + MPI_Send(&flag,1,MPI_INT,i,6,comm) ; std::vector > rv(inner_face_nodes_sizes[i]) ; MPI_Status mstat ; - MPI_Recv(&rv[0],sizeof(vector3d )*inner_face_nodes_sizes[i],MPI_BYTE,i,7,MPI_COMM_WORLD, + MPI_Recv(&rv[0],sizeof(vector3d )*inner_face_nodes_sizes[i],MPI_BYTE,i,7,comm, &mstat) ; count = inner_face_nodes_sizes[i] ; H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, &start, &stride, &count, NULL) ; @@ -489,8 +490,8 @@ namespace Loci{ int flag = 0; MPI_Status mstat ; - MPI_Recv(&flag,1,MPI_INT,0,0,MPI_COMM_WORLD,&mstat) ; - MPI_Send(&v_pos[0],sizeof(vector3d )*local_pos_size,MPI_BYTE,0,1,MPI_COMM_WORLD) ; + MPI_Recv(&flag,1,MPI_INT,0,0,comm,&mstat) ; + MPI_Send(&v_pos[0],sizeof(vector3d )*local_pos_size,MPI_BYTE,0,1,comm) ; } if(num_local_edge_nodes != 0){ std::vector > v_nodes(num_local_edge_nodes); @@ -503,8 +504,8 @@ namespace Loci{ }ENDFORALL; int flag = 0; MPI_Status mstat ; - MPI_Recv(&flag,1,MPI_INT,0,2,MPI_COMM_WORLD,&mstat) ; - MPI_Send(&v_nodes[0],sizeof(vector3d )*num_local_edge_nodes,MPI_BYTE,0,3,MPI_COMM_WORLD) ; + MPI_Recv(&flag,1,MPI_INT,0,2,comm,&mstat) ; + MPI_Send(&v_nodes[0],sizeof(vector3d )*num_local_edge_nodes,MPI_BYTE,0,3,comm) ; } if(num_local_cell_nodes != 0){ @@ -518,8 +519,8 @@ namespace Loci{ }ENDFORALL; int flag = 0; MPI_Status mstat ; - MPI_Recv(&flag,1,MPI_INT,0,4,MPI_COMM_WORLD,&mstat) ; - MPI_Send(&v_nodes[0],sizeof(vector3d )*num_local_cell_nodes,MPI_BYTE,0,5,MPI_COMM_WORLD) ; + MPI_Recv(&flag,1,MPI_INT,0,4,comm,&mstat) ; + MPI_Send(&v_nodes[0],sizeof(vector3d )*num_local_cell_nodes,MPI_BYTE,0,5,comm) ; } if(num_local_face_nodes != 0){ @@ -533,8 +534,8 @@ namespace Loci{ }ENDFORALL; int flag = 0; MPI_Status mstat ; - MPI_Recv(&flag,1,MPI_INT,0,6,MPI_COMM_WORLD,&mstat) ; - MPI_Send(&v_nodes[0],sizeof(vector3d )*num_local_face_nodes,MPI_BYTE,0,7,MPI_COMM_WORLD) ; + MPI_Recv(&flag,1,MPI_INT,0,6,comm,&mstat) ; + MPI_Send(&v_nodes[0],sizeof(vector3d )*num_local_face_nodes,MPI_BYTE,0,7,comm) ; } } @@ -561,6 +562,7 @@ namespace Loci{ hid_t group_id = 0 ; //reorder store first, from local to io entities fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; constraint my_faces, my_geom_cells; entitySet my_entities = dist->my_entities ; my_faces = Loci::exec_current_fact_db->get_variable("faces"); @@ -582,24 +584,24 @@ namespace Loci{ int offset = 0; store pos_io; - pos_io = Loci::Local2FileOrder(pos, local_nodes, offset, dist, MPI_COMM_WORLD) ; + pos_io = Loci::Local2FileOrder(pos, local_nodes, offset, dist, comm) ; entitySet file_nodes = pos_io.domain(); offset = 0; store edge_inner_nodes; - edge_inner_nodes = Loci::Local2FileOrder(inner_nodes_edge.Rep(),local_edges,offset,dist,MPI_COMM_WORLD) ; + edge_inner_nodes = Loci::Local2FileOrder(inner_nodes_edge.Rep(),local_edges,offset,dist,comm) ; entitySet file_edges = edge_inner_nodes.domain(); offset= 0; // Create container vardist that store cell_inner_nodes; - cell_inner_nodes = Loci::Local2FileOrder(inner_nodes_cell.Rep(),local_cells,offset,dist,MPI_COMM_WORLD) ; + cell_inner_nodes = Loci::Local2FileOrder(inner_nodes_cell.Rep(),local_cells,offset,dist,comm) ; entitySet file_cells = cell_inner_nodes.domain(); offset= 0; // Create container vardist that store face_inner_nodes; - face_inner_nodes = Loci::Local2FileOrder(inner_nodes_face.Rep(),local_faces,offset,dist,MPI_COMM_WORLD) ; + face_inner_nodes = Loci::Local2FileOrder(inner_nodes_face.Rep(),local_faces,offset,dist,comm) ; entitySet file_faces = face_inner_nodes.domain(); @@ -607,7 +609,7 @@ namespace Loci{ int local_pos_size = file_nodes.size(); std::vector pos_sizes(Loci::MPI_processes) ; MPI_Allgather(&local_pos_size,1,MPI_INT, - &pos_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + &pos_sizes[0],1,MPI_INT,comm) ; @@ -630,16 +632,16 @@ namespace Loci{ std::vector inner_edge_nodes_sizes(Loci::MPI_processes); MPI_Allgather(&num_local_edge_nodes,1,MPI_INT, - &inner_edge_nodes_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + &inner_edge_nodes_sizes[0],1,MPI_INT,comm) ; std::vector inner_cell_nodes_sizes(Loci::MPI_processes); MPI_Allgather(&num_local_cell_nodes,1,MPI_INT, - &inner_cell_nodes_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + &inner_cell_nodes_sizes[0],1,MPI_INT,comm) ; std::vector inner_face_nodes_sizes(Loci::MPI_processes); MPI_Allgather(&num_local_face_nodes,1,MPI_INT, - &inner_face_nodes_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + &inner_face_nodes_sizes[0],1,MPI_INT,comm) ; @@ -845,11 +847,12 @@ void getDist( Loci::entitySet &faces, Loci::entitySet &cells, vector &fptn, vector &cptn, Map &cl, Map &cr, multiMap &face2node) { // Get entity distributions + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; faces = face2node.domain() ; FATAL(cl.domain() != faces) ; FATAL(cr.domain() != faces) ; - fptn = all_collect_vectors(faces,MPI_COMM_WORLD) ; + fptn = all_collect_vectors(faces,comm) ; entitySet tmp_cells = cl.image(faces)+cr.image(faces) ; for(int i = 0 ; i pl = Loci::simplePartitionVec(minC,maxC,MPI_processes) ; for(int i=0;iget_comm() ; entitySet clone_cells = cl.image(interior_faces) + cr.image(interior_faces) ; clone_cells -= geom_cells ; Loci::storeRepP cp_sp = color.Rep() ; - Loci::fill_clone(cp_sp, clone_cells, cptn) ; + Loci::fill_clone(cp_sp, clone_cells, cptn, comm) ; FORALL(interior_faces,fc) { int color_l = color[cl[fc]] ; diff --git a/src/FVMAdapt/library/write_vog.cc b/src/FVMAdapt/library/write_vog.cc index ebf49ffc..93b41e4d 100644 --- a/src/FVMAdapt/library/write_vog.cc +++ b/src/FVMAdapt/library/write_vog.cc @@ -1,3112 +1,3120 @@ -//############################################################################# -//# -//# Copyright 2008-2025, Mississippi State University -//# -//# This file is part of the Loci Framework. -//# -//# The Loci Framework is free software: you can redistribute it and/or modify -//# it under the terms of the Lesser GNU General Public License as published by -//# the Free Software Foundation, either version 3 of the License, or -//# (at your option) any later version. -//# -//# The Loci Framework is distributed in the hope that it will be useful, -//# but WITHOUT ANY WARRANTY; without even the implied warranty of -//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//# Lesser GNU General Public License for more details. -//# -//# You should have received a copy of the Lesser GNU General Public License -//# along with the Loci Framework. If not, see -//# -//############################################################################# -#include -#include -#include -#include -#include -#include -#include -#include "sciTypes.h" -#include "defines.h" -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "FVMAdapt/defines.h" -#include "FVMAdapt/dataxferDB.h" -#include "FVMAdapt/gridInterface.h" - -using std::cerr; -using std::cout; -using std::endl; -using std::string; -using std::ifstream; -using Loci::storeRepP; -using Loci::constraint; -using std::vector; -using Loci::MPI_rank; -using Loci::MPI_processes; -using std::map ; -using std::ostringstream ; -using std::istringstream ; -///---------------------------------------------------------------------- -// HACK WARNING -// -// The FVMGridReader has been changed to support 4 gig entities. This has -// an impact on how boundary faces are represented in the FVMadapt code. We -// just copied the old code into here for now, but that means that adapt -// doesn't work with large interval sets. This needs to be fixed... however -// we are reworking how the fact_db will be created so this can wait until that -// process is further along. - -// RSM MOD 20181108 -#ifdef LOCI_USE_METIS -#ifdef USE_SCOTCH -#include -#endif -#include -#include - -#ifdef USE_SCOTCH -typedef float metisreal_t ; -#else -#if REALTYPEWIDTH == 32 -typedef float metisreal_t ; -#else -typedef double metisreal_t ; -#endif -#endif - -#endif - -namespace Loci { - // get cell2parent map with cells in current global numbering - - storeRepP getC2PGlobal(fact_db &facts) { - - // Check to see if a global version has already been generated - storeRepP ptr = DataXFER_DB.getItem("c2pglobal") ; - if(ptr != 0) { - return ptr ; - } - // Now convert c2p from a file numbering of cells to the current - // global numbering - store > c2pset ; - c2pset = DataXFER_DB.getItem("c2p") ; - entitySet dom = c2pset.domain() ; - protoMap c2p(dom.size()) ; - int cnt = 0 ; - FORALL(dom,ii) { - c2p[cnt++] = c2pset[ii] ; - } ENDFORALL ; - // find the max and min numbering of the file numbering in cells in - // c2p - int mcol = c2p[0].first ; - int xcol = mcol ; - for(int i=0;i > c2pg ; - c2pg.allocate(dom) ; - FORALL(dom,ii) { - c2pg[ii] = c2p[ii] ; - } ENDFORALL ; - DataXFER_DB.insertItem("c2pglobal",c2pg.Rep()) ; - return c2pg.Rep() ; - } - // Now get the maps needed to translate from global to file - dMap g2f ; - g2f = dist->g2f.Rep() ; - Map l2g ; - l2g = dist->l2g.Rep() ; - constraint geom_cells ; - geom_cells = facts.get_fact("geom_cells") ; - dom = geom_cells.Rep()->domain() ; - dom = dom & dist->my_entities ; - - // Create protomap from cell file number to global number - protoMap f2g(dom.size()) ; - cnt = 0 ; - int mfol = std::numeric_limits::max() ; - int xfol = std::numeric_limits::lowest() ; - FORALL(dom,ii) { - int g = l2g[ii] ; - int f = g2f[g] ; - f2g[cnt++] = pair(f,g) ; - mfol = min(mfol,f) ; - xfol = max(xfol,f) ; - } ENDFORALL ; - - int mfo = mfol ; - int xfo = xfol ; - MPI_Allreduce(&mfol,&mfo,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; - MPI_Allreduce(&xfol,&xfo,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; - // Check to make sure that the c2p map is consistent with the number of - // cells in this mesh - if(xfo-mfo != xco-mco) { - if(MPI_rank == 0) { - cerr << "ERROR!: Mismatch between number of cells in c2p map!" << endl ; - cerr << " : number of file no cells =" << xfo-mfo << ", local =" << xco-mco << endl ; - } - } - // Remap file numbers to be consistent with c2p numbering - for(int i=0;i > c2pg ; - c2pg.allocate(c2pgset) ; - FORALL(c2pgset,ii) { - c2pg[ii] = v[ii] ; - } ENDFORALL ; - // Write into XFER DB so we don't have to do this multiple times - DataXFER_DB.insertItem("c2pglobal",c2pg.Rep()) ; - // return new map - return c2pg.Rep() ; - } - - namespace { - void computeWeights(vector3d *wvec, const vector3d *deltas, const real_t *areas, - int ndeltas) { - tmp_array > weights(ndeltas) ; - - real_t r[3][3] ; - r[0][0] = 0 ; - for(int i=0;i(weightx,weighty,weightz)*areas[i] ; - } - } - } - - void AMRrefinementMapping:: - setupRefinementMapping(const store > &c2pg, - const store &volw, - multiStore &gradCells, - multiStore > &deltas, - const_store > &cell_center, - const_store &vol, - fact_db &facts) { - //######################################################################## - // - // get the geom_cells of child cells and convert to global numbering - // - auto sp = facts.get_variable("geom_cells") ; - if(sp==0) { - cerr << "expecting facts to contain geom_cells to build AMR second order interpolation" << endl ; - Loci::Abort() ; - } - - // geom_cells is the local numbering, convert to global numbering - fact_db::distribute_infoP dist = facts.get_distribute_info() ; - // get the geom_cells owned by this processor - geom_cells_local = sp->domain()&dist->my_entities ; - // convert form local to global numbering - l2g.allocate(geom_cells_local) ; - FORALL(geom_cells_local,ii) { - geom_cells_global += dist->l2g[ii] ; - l2g[ii] = dist->l2g[ii] ; - } ENDFORALL ; - -#ifdef DEBUG - debugout << "geom_cells =" << geom_cells_global << endl ; -#endif - //######################################################################## - // - // Get partition of child cells by collecting each procesors geom_cells - // set - // - vector ptn = Loci::all_collect_vectors(geom_cells_global, - MPI_COMM_WORLD) ; - dataPartitionP partition_child = createPartition(ptn,MPI_COMM_WORLD) ; - - //######################################################################## - // - // Get partition of parent cells by computing first element on each - // processor of the volw container - // - int p = Loci::MPI_processes ; - int psplit = volw.domain().Min() ; - vector psplits(p) ; - // Gather splits - MPI_Allgather(&psplit,1,MPI_INT,&psplits[0],1,MPI_INT, - MPI_COMM_WORLD) ; -#ifdef DEBUG - debugout << "parent partition splits = " << endl ; - for(int i=0;i::max() ; - minval[1] = minval[0] ; - maxval[0] = std::numeric_limits::lowest() ; - maxval[1] = maxval[0] ; - entitySet domc2pg = c2pg.domain() ; - FORALL(domc2pg,ii) { - minval[0] = min(minval[0],c2pg[ii].first) ; - minval[1] = min(minval[1],c2pg[ii].second) ; - maxval[0] = max(maxval[0],c2pg[ii].first) ; - maxval[1] = max(maxval[1],c2pg[ii].second) ; - } ENDFORALL ; - - int maxvalg[2] = {maxval[0],maxval[1]} ; - int minvalg[2] = {minval[0],minval[1]} ; - MPI_Allreduce(&minval[0],&minvalg[0],2,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; - MPI_Allreduce(&maxval[0],&maxvalg[0],2,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; - - //######################################################################## - // - // Transpose cell to parent map to get parent to cell map. Add offsets - // to directly make the parent to cell maps start with zero indexing. - // - int sz = domc2pg.size() ; - vector > p2c(sz) ; - int cnt = 0 ; - int offsetsecond = -minvalg[1] ; - int offsetfirst = -minvalg[0] ; - // Transpose cell-to-parent to parent to cell, adjust so the indexing starts - // at zero - FORALL(domc2pg,ii) { - p2c[cnt].first = c2pg[ii].second + offsetsecond ; - p2c[cnt].second = c2pg[ii].first + offsetfirst ; - cnt++ ; - } ENDFORALL ; - FATAL(cnt != sz) ; - - - //######################################################################## - // - // Adjust parent and cell numbering in map to align with numbering used - // in volw and geom_cells. - // - // cstart is child starting number,csplit is describing the distribution of - // parent cells across processors. - int csplit = geom_cells_global.Min() ; - int cstart = csplit ; - - MPI_Allreduce(&csplit,&cstart,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; - -#ifdef DEBUG - debugout << "pstart=" << pstart << endl ; - debugout << "cstart=" << cstart << endl ; -#endif - // Convert parent and child numbering to current global numbering - for(size_t i=0;i > splits(p-1) ; - for(int i=0;i(psplits[i+1],std::numeric_limits::lowest()) ; - - sort(p2c.begin(),p2c.end()) ; - Loci::parSplitSort(p2c,splits,MPI_COMM_WORLD) ; - - //######################################################################## - // - // Find when a parent gives to multiple children, these will be parent - // cells that have been refined and thus we need to interpolate - // - int refinecnt = 0 ; - sz = p2c.size() ; - FATAL(sz == 0) ; - for(int i=0;i 1) - refinecnt += nsame ; - i += nsame ; - } -#ifdef DEBUG - // report total number of refined cells in interpolation - int refinecntglobal = refinecnt ; - MPI_Allreduce(&refinecnt, &refinecntglobal,1,MPI_INT,MPI_SUM, - MPI_COMM_WORLD) ; - debugout<< "refine cnt = " << refinecntglobal << endl ; -#endif - // Remove the refined cells from the parent to cell map and then make a - // child-to-parent map distributed in child - // c2p is the cell to parent map for cells that were not refined. - vector > c2p(sz-refinecnt) ; - // separate parent to cell map only for refined cells - vector > p2crefine(refinecnt) ; - - int cnt1 = 0 ; - int cnt2 = 0 ; - for(int i=0;i 1) { // If refinement group copy to p2crefine - for(int j=0;j csplits(p) ; - MPI_Allgather(&csplit,1,MPI_INT,&csplits[0],1,MPI_INT,MPI_COMM_WORLD) ; - for(int i=0;i(csplits[i+1],std::numeric_limits::lowest()) ; - sort(c2p.begin(),c2p.end()) ; - - Loci::parSplitSort(c2p,splits,MPI_COMM_WORLD) ; - - - //######################################################################## - // - // Figure out child cells that have multiple parents. These are derefined - // cells that need to be averaged during the interpolation step - // -#ifdef DEBUG - int derefinecnt = 0 ; -#endif - sz = c2p.size() ; - // WARN(sz == 0) ; - - for(int i=0;i 1) - derefinecnt += nsame ; -#endif - i += nsame ; - } -#ifdef DEBUG - int derefinecntglobal = derefinecnt ; - MPI_Allreduce(&derefinecnt, &derefinecntglobal,1,MPI_INT,MPI_SUM, - MPI_COMM_WORLD) ; - debugout<< "derefine cnt = " << derefinecntglobal << endl ; - - // Now generate communication schedules for refined cells - debugout << "refinesize=" << p2crefine.size() << endl ; -#endif - - - //######################################################################## - // - // Build data structures for refined cells to compute gradients for - // interpolation. - // - entitySet refinedParents ; - vector refinecells(p2crefine.size()) ; - for(size_t i=0;i0) ; - // Setup the rule database. - // Add all registered rules. - rule_db refine_rules; - refine_rules = refmesh_rdb ; - - // Setup the refine_fact database. - fact_db refine_facts; - - // Read in the mesh file. - Loci::storeRepP cellwt = Loci::DataXFER_DB.getItem("cellweights") ; - if(!Loci::setupFVMGridWithWeightInStore(refine_facts,meshFile, cellwt)) { - std::cerr << "unable to read grid file '" << meshFile << "'" << std::endl ; - Loci::Abort() ; - } - Loci::DataXFER_DB.deleteItem("cellweights") ; - - - blackbox > > origVolTags ; - origVolTags.Rep()->allocate(~EMPTY) ; - if(MPI_rank==0 || Loci::use_parallel_io){ - hid_t file_id = Loci::hdf5OpenFile(meshFile.c_str(), - H5F_ACC_RDONLY,H5P_DEFAULT) ; - - Loci::readVolTags(file_id,*origVolTags); - Loci::hdf5CloseFile(file_id) ; - } - refine_facts.create_fact("volTagInput",origVolTags) ; - - //Setup Loci datastructures - Loci::createLowerUpper(refine_facts) ; - Loci::createEdgesPar(refine_facts) ; - Loci::parallelClassifyCell(refine_facts); - - - //this is a dummy parameter to trick Loci scheduler - param beginWithMarker; - *beginWithMarker = true; - refine_facts.create_fact("beginWithMarker",beginWithMarker) ; - - param plan_outDB_par ; - *plan_outDB_par = "nextPlan" ; - refine_facts.create_fact("plan_outDB_par",plan_outDB_par) ; - - if(tags != 0) { - store tag_info ; - tag_info = tags ; - Loci::entitySet dom = tag_info.domain() ; - vector tagset(dom.size()) ; - int cnt = 0 ; - FORALL(dom,ii) { - tagset[cnt] = tag_info[ii] ; - cnt++ ; - } ENDFORALL ; - blackbox > inputTagsData ; - inputTagsData.set_entitySet(~EMPTY) ; - (*inputTagsData).swap(tagset) ; - refine_facts.create_fact("inputCellTagsData",inputTagsData) ; - Loci::DataXFER_DB.deleteItem("refineTag") ; - } - - - if(firstlevel){ - param parent_planfile_par ; - *parent_planfile_par = "currentPlan" ; - refine_facts.create_fact("parent_planDB_par",parent_planfile_par) ; - } - - gridDataP->boundary_ids.clear(); - Loci::readBCfromVOG(meshFile, gridDataP->boundary_ids); - - - - //parameters to identify different options - if(firstlevel){ - param planfile_par ; - *planfile_par = "currentPlan" ; - refine_facts.create_fact("planDB_par",planfile_par) ; - - param restart_tag_par; - *restart_tag_par = 1; - refine_facts.create_fact("restart_tag_par",restart_tag_par); - }else{ - param norestart_tag_par; - *norestart_tag_par = 1; - refine_facts.create_fact("norestart_tag_par",norestart_tag_par); - } - - param split_mode_par; - *split_mode_par = adaptmode; - refine_facts.create_fact("split_mode_par", split_mode_par); - - { - param cellweight_outDB_par; - *cellweight_outDB_par = "cellweights"; - refine_facts.create_fact("cellweight_outDB_par",cellweight_outDB_par); - - if(!Loci::makeQuery(refine_rules,refine_facts, - "cellplan_output,cellweight_output,cell2parent_DB,inner_nodes_cell,inner_nodes_face,inner_nodes_edge,fine_faces_cell,fine_faces,volTag_blackbox")) { - std::cerr << "adapt query failed!" << std::endl; - Loci::Abort(); - } - } - - Loci::DataXFER_DB.deleteItem("currentPlan") ; - Loci::storeRepP newPlan = Loci::DataXFER_DB.getItem("nextPlan") ; - Loci::DataXFER_DB.insertItem("currentPlan",newPlan) ; - Loci::DataXFER_DB.deleteItem("nextPlan") ; - - store inner_nodes_cell; - inner_nodes_cell = refine_facts.get_variable("inner_nodes_cell"); - store inner_nodes_face; - inner_nodes_face = refine_facts.get_variable("inner_nodes_face"); - store inner_nodes_edge; - inner_nodes_edge = refine_facts.get_variable("inner_nodes_edge"); - - - int num_nodes; - createVOGNode(gridDataP->new_pos, - inner_nodes_cell,inner_nodes_face,inner_nodes_edge, - num_nodes, - refine_facts, - gridDataP->local_nodes - ); - if(Loci::MPI_rank ==0) - cerr<< "num_nodes: " << num_nodes << " in adaptcycle" << endl; - - store fine_faces_cell; - fine_faces_cell = refine_facts.get_variable("fine_faces_cell"); - store fine_faces; - fine_faces = refine_facts.get_variable("fine_faces"); - - int num_faces = 0; - int num_cells = 0; - - createVOGFace( num_nodes, - fine_faces_cell, - fine_faces, - refine_facts, - num_faces, - num_cells, - gridDataP->new_cl, - gridDataP->new_cr, - gridDataP->new_face2node, - gridDataP->local_faces, - gridDataP->local_cells - ); - - //update volume tags - { - blackbox > > volTag_blackbox; - volTag_blackbox = refine_facts.get_variable("volTag_blackbox"); - gridDataP->volTags = *volTag_blackbox; - } - - if(Loci::MPI_rank==0){ - cerr<< "num_faces: " << num_faces << " in adaptcycle" < &gridDataP, - int &level, - rule_db &refmesh_rdb, - string casename, string weightfile, - string restartplanfile) { - string meshFile = casename+".vog"; - // Setup the rule database. - // Add all registered rules. - rule_db refine_rules; - refine_rules = refmesh_rdb ; - - // Setup the refine_fact database. - fact_db refine_facts; - - if(Loci::MPI_rank == 0) - cout <<"reading in meshfile: " - << meshFile << " for chem run" << std::endl; - - // Read in the mesh file. - if(!Loci::setupFVMGridWithWeightInFile(refine_facts,meshFile, weightfile)) { - std::cerr << "unable to read mesh file '" << meshFile << "'" << std::endl ; - Loci::Abort() ; - } - - //Setup Loci datastructures - Loci::createLowerUpper(refine_facts) ; - Loci::createEdgesPar(refine_facts) ; - Loci::parallelClassifyCell(refine_facts); - blackbox > > origVolTags ; - origVolTags.Rep()->allocate(~EMPTY) ; - //if(MPI_rank==0){ - if(MPI_rank==0 || Loci::use_parallel_io){ - hid_t file_id = Loci::hdf5OpenFile(meshFile.c_str(), - H5F_ACC_RDONLY,H5P_DEFAULT) ; - - Loci::readVolTags(file_id,*origVolTags); - Loci::hdf5CloseFile(file_id) ; - } - refine_facts.create_fact("volTagInput",origVolTags) ; - - if(Loci::MPI_rank == 0) - cout <<"reading in cellplan file: " - << restartplanfile << std::endl; - - hid_t file_id = Loci::hdf5OpenFile(restartplanfile.c_str(), - H5F_ACC_RDONLY,H5P_DEFAULT) ; - - store > cellPlan ; - param refineLevel ; - hid_t group_id = 0 ; - if(Loci::MPI_rank == 0) - group_id = H5Gopen(file_id,"refineLevel",H5P_DEFAULT) ; - - Loci::read_parameter(group_id,refineLevel.Rep(),MPI_COMM_WORLD) ; - if(Loci::MPI_rank == 0) { - H5Gclose(group_id) ; - group_id = H5Gopen(file_id,"cellPlan",H5P_DEFAULT) ; - cout << "restart refine level = " << *refineLevel << endl ; - } - level = *refineLevel ; - int offset = 0; - Loci::read_store(group_id,cellPlan.Rep(),offset,MPI_COMM_WORLD) ;//read in file numbering store, simplePartition - cellPlan.Rep()->shift(offset) ; - if(Loci::MPI_rank == 0) { - H5Gclose(group_id) ; - } - Loci::hdf5CloseFile(file_id) ; - - //finish reading currentPlan - //inserting currentPlan into DataXFER_DB - Loci::DataXFER_DB.insertItem("currentPlan",cellPlan.Rep()) ; - param balanced_planDB_par ; - *balanced_planDB_par = "currentPlan" ; - refine_facts.create_fact("balanced_planDB_par",balanced_planDB_par) ; - - if(!Loci::makeQuery(refine_rules,refine_facts, - "inner_nodes_cell,inner_nodes_face,inner_nodes_edge,fine_faces,fine_faces_cell,volTag_blackbox")) { - std::cerr << "adapt query failed!" << std::endl; - Loci::Abort(); - } - - store inner_nodes_cell; - inner_nodes_cell = refine_facts.get_variable("inner_nodes_cell"); - store inner_nodes_face; - inner_nodes_face = refine_facts.get_variable("inner_nodes_face"); - store inner_nodes_edge; - inner_nodes_edge = refine_facts.get_variable("inner_nodes_edge"); - - - gridDataP = new(refinedGridData) ; - gridDataP->boundary_ids.clear(); - Loci::readBCfromVOG(meshFile, gridDataP->boundary_ids); - - int num_nodes; - createVOGNode(gridDataP->new_pos, - inner_nodes_cell,inner_nodes_face,inner_nodes_edge, - num_nodes, - refine_facts, - gridDataP->local_nodes - ); - if(Loci::MPI_rank ==0)cerr<< "num_nodes: " << num_nodes << " before chem run" << endl; - - store fine_faces_cell; - fine_faces_cell = refine_facts.get_variable("fine_faces_cell"); - store fine_faces; - fine_faces = refine_facts.get_variable("fine_faces"); - - int num_faces = 0; - int num_cells = 0; - - createVOGFace( num_nodes, - fine_faces_cell, - fine_faces, - refine_facts, - num_faces, - num_cells, - gridDataP->new_cl, - gridDataP->new_cr, - gridDataP->new_face2node, - gridDataP->local_faces, - gridDataP->local_cells - ); - - if(Loci::MPI_rank ==0)cerr<< "num_faces: " << num_faces << " before chem run" << endl; - if(Loci::MPI_rank ==0)cerr<< "num_cells: " << num_cells << " before chem run" << endl; - - { - blackbox > > volTag_blackbox; - volTag_blackbox = refine_facts.get_variable("volTag_blackbox"); - gridDataP->volTags = *volTag_blackbox; - } - } - - extern bool load_cell_weights ; - extern string cell_weight_file ; - extern storeRepP cell_weight_store ; - bool redistribute_cell_weight(storeRepP old_store, storeRepP new_store); - - void fill_clone_proc( map &mapdata, entitySet &out_of_dom, std::vector &init_ptn) ; - void redistribute_container(const vector &ptn, - const vector &ptn_t, - entitySet new_alloc, - storeRepP inRep,storeRepP outRep) ; - extern void ORBPartition(const vector > &pnts, - vector &procid, - MPI_Comm comm) ; - void assignOwner(vector > > &scratchPad, - vector ptn, - vector &out_ptn) ; - void SFC_Partition_Mesh(const vector &local_nodes, - const vector &local_faces, - const vector &local_cells, - const store > &pos, - const Map &cl, const Map &cr, - const multiMap &face2node, - const store &boundary_tags, - const store &cell_weights, - vector &cell_ptn, - vector &face_ptn, - vector &node_ptn) ; - void RND_Partition_Mesh(const vector &local_nodes, - const vector &local_faces, - const vector &local_cells, - vector &cell_ptn, - vector &face_ptn, - vector &node_ptn) ; - Loci::storeRepP getCellPartitionWeights(entitySet localcelldom) { - Loci::storeRepP wptr = Loci::DataXFER_DB.getItem("cellPartitionWeights") ; - if(wptr == 0) - return wptr ; - store cell_weights_parent ; - cell_weights_parent = wptr ; - store > c2pset ; - Loci::storeRepP ptr = Loci::DataXFER_DB.getItem("c2p") ; - if(ptr == 0) - return ptr ; - c2pset = ptr ; - entitySet domc2p = c2pset.domain() ; - int r = Loci::MPI_rank ; - int p = Loci::MPI_processes ; - - int minParent = std::numeric_limits::max() ; - int minTarget = std::numeric_limits::max() ; - FORALL(domc2p,ii) { - minParent = min(minParent,c2pset[ii].second) ; - minTarget = min(minTarget,c2pset[ii].first) ; - } ENDFORALL ; - - int parentOffset = minParent ; - int targetOffset = minTarget ; - MPI_Allreduce(&minParent,&parentOffset,1,MPI_INT,MPI_MIN, - MPI_COMM_WORLD) ; - MPI_Allreduce(&minTarget,&targetOffset,1,MPI_INT,MPI_MIN, - MPI_COMM_WORLD) ; - int sz = domc2p.size() ; - vector > p2c(sz) ; - int cnt = 0 ; - FORALL(domc2p,ii) { - p2c[cnt].first = c2pset[ii].second-parentOffset ; - p2c[cnt].second = c2pset[ii].first-targetOffset ; - cnt++ ;; - } ENDFORALL ; - - sort(p2c.begin(),p2c.end()) ; - - // Now distribute c2p to processors in parent file ordering - entitySet cwdom = cell_weights_parent.domain() ; - int cwsz = cwdom.size() ; - vector parentsizes(p) ; - MPI_Allgather(&cwsz,1,MPI_INT,&parentsizes[0],1,MPI_INT, - MPI_COMM_WORLD) ; - vector parentoffsets(p+1,0) ; - for(int i=0;i > splits(p-1) ; - for(int i=0;i(parentoffsets[i+1],-1) ; - - Loci::parSplitSort(p2c,splits,MPI_COMM_WORLD) ; - - int psz = p2c.size() ; - - vector > cell2weights(psz) ; - entitySet::const_iterator ii = cwdom.begin() ; - cnt = parentoffsets[r] ; - int mincell = p2c[0].second ; - int cnt2 = 0 ; - for(int i=0;i(p2c[i].second, - cell_weights_parent[*ii]) ; - cnt2++ ; - i++ ; - } - cnt++ ; - ii++ ; - } - int mincell_global =mincell ; - MPI_Allreduce(&mincell,&mincell_global,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; - - int dmsz = localcelldom.size() ; - vector cellsizes(p) ; - MPI_Allgather(&dmsz,1,MPI_INT,&cellsizes[0],1,MPI_INT, - MPI_COMM_WORLD) ; - parentoffsets[0] = mincell_global ; - for(int i=0;i(parentoffsets[i+1],-1) ; - - - sort(cell2weights.begin(),cell2weights.end()) ; - Loci::parSplitSort(cell2weights,splits,MPI_COMM_WORLD) ; - - store cellweightschild ; - cellweightschild.allocate(localcelldom) ; - cnt = 0; - - FORALL(localcelldom,ii) { - cellweightschild[ii] = cell2weights[cnt].second ; - cnt++ ; - } ENDFORALL ; - return cellweightschild.Rep() ; - } - - - -#ifdef LOCI_USE_METIS - vector AdaptMetisPartitionOfCells(const vector &local_cells, - const Map &cl, const Map &cr, - const store &boundary_tags, - storeRepP cellwts) { - entitySet dom = cl.domain() & cr.domain() ; - entitySet refset = boundary_tags.domain() ; - entitySet bcfaces = cr.preimage(refset).first ; - dom -= bcfaces ; - int cnt = dom.size() ; - - vector > rawMap(cnt*2) ; - int j = 0 ; - entitySet::const_iterator ei ; - for(ei=dom.begin();ei!=dom.end();++ei) { - rawMap[j++] = pair(cl[*ei],cr[*ei]) ; - rawMap[j++] = pair(cr[*ei],cl[*ei]) ; - } - - sort(rawMap.begin(),rawMap.end()) ; - - multiMap cell2cell ; - entitySet all_cells ; - for(int i=0;i >().swap(rawMap) ; // Free up memory from rawMap - int count = 0 ; - idx_t size_map = local_cells[Loci::MPI_rank].size() ; - vector size_adj(size_map) ; - count = 0 ; - for(entitySet::const_iterator ei = local_cells[Loci::MPI_rank].begin(); ei != local_cells[Loci::MPI_rank].end(); ++ei) { - size_adj[count] = cell2cell[*ei].size() ; - ++count ; - } - - vector part(size_map) ; - vector xadj(size_map+1) ; - idx_t edgecut ; - vector vdist(Loci::MPI_processes + 1) ; - int cmin = local_cells[0].Min(); - for(int i = 0; i < Loci::MPI_processes; i++) { - cmin = min(local_cells[i].Min(), cmin); - } - - // check local_cells to be consistent with parmetis partitioning - for(int i=0;i adjncy(tot) ; - count = 0 ; - for(entitySet::const_iterator ei = local_cells[Loci::MPI_rank].begin(); ei != local_cells[Loci::MPI_rank].end(); ++ei) { - size_t sz = cell2cell[*ei].size() ; - for(size_t i = 0; i != sz; ++i) { - adjncy[count] = cell2cell[*ei][i] - cmin ; - count ++ ; - } - } - cell2cell.setRep(multiMap().Rep()) ;// Free up memory from multiMap - - vdist[0] = 0 ; - for(int i = 1; i <= Loci::MPI_processes; ++i) - vdist[i] = vdist[i-1] + local_cells[i-1].size() ; - idx_t top = vdist[Loci::MPI_processes] ; - - bool trouble = false ; - for(idx_t i=0;i= top) - trouble = true ; - if(trouble) - cerr << "adjacency list contains out of bounds reference" << endl ; - - MPI_Comm mc = MPI_COMM_WORLD ; - idx_t nparts = Loci::MPI_processes ; // number of partitions - idx_t wgtflag = 0 ; - idx_t numflag = 0 ; - idx_t options = 0 ; - - // Loci::storeRepP ptr = getCellPartitionWeights(local_cells[MPI_rank]) ; - Loci::storeRepP ptr = cellwts ; - - if(ptr != 0) { - store cell_weights ; - cell_weights = ptr ; - - // compute necessary ParMETIS data-structure - wgtflag = 2 ; // weights on the vertices only - idx_t ncon = 2 ; // number of weights per vertex - idx_t tpwgts_len = ncon*nparts ; - vector tpwgts(tpwgts_len) ; - - for(idx_t i=0;i ubvec(ncon) ; - for(idx_t i=0;i vwgt(ncon*size_map) ; - int cnt = 0 ; - for(entitySet::const_iterator - ei=local_cells[Loci::MPI_rank].begin(); - ei!=local_cells[Loci::MPI_rank].end();++ei,cnt+=ncon) { - // first weight for cell is 1 (the cell computation) - vwgt[cnt] = 1 ; - // the second weight is from the store cell_weights[*ei] - vwgt[cnt+1] = cell_weights[*ei] ; - } - - // now call the ParMETIS routine (V3) - ParMETIS_V3_PartKway(&vdist[0],&xadj[0],&adjncy[0],&vwgt[0],NULL, - &wgtflag,&numflag,&ncon,&nparts, - &tpwgts[0],&ubvec[0],&options,&edgecut,&part[0], - &mc) ; - - } else { - // idx_t ncon = 1 ; - idx_t ncon = 2 ; - idx_t tpwgts_len = ncon*nparts ; - vector tpwgts(tpwgts_len) ; - for(idx_t i=0;i ubvec(ncon) ; - for(idx_t i=0;i vwgt(ncon*size_map) ; - - int cnt = 0 ; - for(int i=0;i ptn ; - - for(int i = 0; i < Loci::MPI_processes; ++i) - ptn.push_back(EMPTY) ; - cmin = local_cells[Loci::MPI_rank].Min() ; - for(int i=0;i &local_nodes, - const vector &local_faces, - const vector &local_cells, - const store > &pos, - const Map &cl, const Map &cr, - const multiMap &face2node, - const store &boundary_tags, - vector &cell_ptn, - vector &face_ptn, - vector &node_ptn) ; - vector partitionFaces(vector cell_ptn, const Map &cl, - const Map &cr, - const store &boundary_tags) ; - - //Input: Mapping from faces to its right cells. - //Output: Entities of boundary cells. - entitySet getBoundaryCells(const MapRepP tmp_cr_rep) { - entitySet cri = tmp_cr_rep->image(tmp_cr_rep->domain()) ; - return(cri & interval(Loci::UNIVERSE_MIN,-1)) ; - } - - void copyGridStructures( entitySet nodes, entitySet faces, entitySet cells, - const store > &t_pos, - const Map &tmp_cl, const Map &tmp_cr, - const multiMap &tmp_face2node, - store > &pos, Map &cl, Map &cr, - multiMap &face2node) { - - entitySet boundary_cells = getBoundaryCells(Loci::MapRepP(tmp_cr.Rep())); - - dMap identity_map; - FORALL(nodes, ei) { - identity_map[ei] = ei; - } ENDFORALL ; - FORALL(faces, ei) { - identity_map[ei] = ei ; - } ENDFORALL ; - FORALL(cells, ei) { - identity_map[ei] = ei ; - } ENDFORALL ; - FORALL(boundary_cells, ei) { - identity_map[ei] = ei ; - } ENDFORALL ; - - pos = t_pos.Rep()->remap(identity_map); - cl = tmp_cl.Rep()->remap(identity_map); - cr = tmp_cr.Rep()->remap(identity_map); - face2node = MapRepP(tmp_face2node.Rep())->get_map(); - - } - - - vector transposePtn(const vector &ptn); - vector partitionNodes(vector face_ptn, MapRepP face2node,entitySet old_node_dom); - inline bool fieldSort(const std::pair &p1, - const std::pair &p2) { - return p1.first < p2.first ; - } - - - void create_face_info(fact_db &facts); - void create_ref(fact_db &facts) ; - void create_ghost_cells(fact_db &facts); - - std::vector simplePartitionVec(int mn, int mx, int p); - vector simplePartition(int mn, int mx, MPI_Comm comm); - vector transposeSeq(const vector sv); -} -void colorMatrix(Map &cl, Map &cr, multiMap &face2node) ; - -namespace Loci{ - - // Convert container from global numbering to file numbering - // pass in store rep pointer: sp - // entitySet to write: dom - // return offset in file numbering (each processor will allocate from zero, - // add offset to domain to get actual file numbering) - // distribution info pointer (dist) - // MPI Communicator(comm) - storeRepP Global2FileOrder(storeRepP sp, entitySet dom, int &offset, - fact_db::distribute_infoP dist, MPI_Comm comm) { - - // Now get global to file numbering - dMap g2f ; - g2f = dist->g2fv[0].Rep() ; // FIX THIS - - // Compute map from local numbering to file numbering - Map newnum ; - newnum.allocate(dom) ; - FORALL(dom,i) { - newnum[i] = g2f[i] ; - } ENDFORALL ; - - int imx = std::numeric_limits::min() ; - int imn = std::numeric_limits::max() ; - - // Find bounds in file numbering from this processor - FORALL(dom,i) { - imx = max(newnum[i],imx) ; - imn = min(newnum[i],imn) ; - } ENDFORALL ; - - // Find overall bounds - imx = GLOBAL_MAX(imx) ; - imn = GLOBAL_MIN(imn) ; - - // Get number of processors - int p = 0 ; - MPI_Comm_size(comm,&p) ; - int prank = 0 ; - MPI_Comm_rank(comm,&prank) ; - // Get partitioning of file numbers across processors - vector out_ptn = simplePartition(imn,imx,comm) ; - - // Now compute where to send data to put in file ordering - vector send_sets(p) ; - vector send_seqs(p) ; - - // Loop over processors and compute sets of entities to send - // To efficiently compute this mapping, first sort the transpose - // of the newnum map to quickly find the set of entities to send - // without searching entire newnum map for each processor - vector > file2num(dom.size()) ; - size_t cnt = 0 ; - FORALL(dom,ii) { - file2num[cnt].first = newnum[ii] ; - file2num[cnt].second = ii ; - cnt++ ; - } ENDFORALL ; - sort(file2num.begin(),file2num.end()) ; - - // Check each processor, find out which sets to send - cnt = 0 ; - for(int i=0;i recv_seqs = transposeSeq(send_seqs) ; - - - // shift by the offset - offset = out_ptn[prank].Min() ; - for(int i=0;inew_store(file_dom) ; - - // Now communicate the container - vector send_sizes(p),recv_sizes(p) ; - - for(int i=0;ipack_size(send_sets[i]) ; - - MPI_Alltoall(&send_sizes[0],1,MPI_INT, - &recv_sizes[0],1,MPI_INT, - comm) ; - - vector send_dspl(p),recv_dspl(p) ; - send_dspl[0] = 0 ; - recv_dspl[0] = 0 ; - for(int i=1;i send_store(send_sz) ; - vector recv_store(recv_sz) ; - - - for(int i=0;ipack(&send_store[send_dspl[i]],loc_pack, send_sizes[i], - send_sets[i]) ; - } - - MPI_Alltoallv(&send_store[0], &send_sizes[0], &send_dspl[0], MPI_PACKED, - &recv_store[0], &recv_sizes[0], &recv_dspl[0], MPI_PACKED, - comm) ; - - for(int i=0;iunpack(&recv_store[recv_dspl[i]],loc_pack,recv_sizes[i], - recv_seqs[i]) ; - } - return qcol_rep ; - } - - extern bool useDomainKeySpaces ; - extern void remapGrid(vector &node_ptn, - vector &face_ptn, - vector &cell_ptn, - vector &node_ptn_t, - vector &face_ptn_t, - vector &cell_ptn_t, - store > &t_pos, Map &tmp_cl, - Map &tmp_cr, multiMap &tmp_face2node, - vector &bcsurf_ptn, - store &tmp_boundary_names, - store &tmp_boundary_tags, - entitySet nodes, entitySet faces, entitySet cells, - store > &pos, Map &cl, Map &cr, - multiMap &face2node, - store &boundary_names, - store &boundary_tags, - entitySet bcsurfset, - fact_db &facts) ; - - bool inputFVMGrid(fact_db &facts, - vector& local_nodes, - vector& local_faces, - vector& local_cells, - store >& t_pos, - Map& tmp_cl, - Map& tmp_cr, - multiMap& tmp_face2node, - vector >& boundary_ids, - vector >& volTags, - storeRepP cellwts) { - double t1 = MPI_Wtime() ; - // Identify boundary tags - if(Loci::MPI_processes == 1) { - entitySet local_boundary_cells = getBoundaryCells(MapRepP(tmp_cr.Rep())); - entitySet global_boundary_cells = all_collect_entitySet(local_boundary_cells) ; - - int npnts = local_nodes[0].size(); - int nfaces = local_faces[0].size(); - int ncells = local_cells[0].size(); - - entitySet nodes = facts.get_distributed_alloc(npnts,0).first ; // FIX THIS - entitySet faces = facts.get_distributed_alloc(nfaces,0).first ; - entitySet cells = facts.get_distributed_alloc(ncells,0).first; - - store > pos ; - Map cl ; - Map cr ; - multiMap face2node ; - Loci::copyGridStructures(nodes, faces, cells, - t_pos, tmp_cl, tmp_cr, tmp_face2node, - pos, cl, cr, face2node); - store boundary_names ; - store boundary_tags ; - boundary_names.allocate(global_boundary_cells) ; - boundary_tags.allocate(global_boundary_cells) ; - cout << " boundaries identified as:" ; - - FORALL(global_boundary_cells, bc) { - char buf[512] ; - bzero(buf,512) ; - snprintf(buf,511,"BC_%d",-bc) ; - boundary_tags[bc] = string(buf) ; - boundary_names[bc] = string(buf) ; - cout << " " << boundary_names[bc] ; - } ENDFORALL ; - - for(size_t i=0;i Tag ; - *Tag = volTags[i].first ; - Tag.set_entitySet(volTags[i].second >> cells_base) ; - std::ostringstream oss ; - oss << "volumeTag(" << volTags[i].first << ")" ; - facts.create_fact(oss.str(),Tag) ; - } - - return true ; - } - - entitySet local_boundary_cells = getBoundaryCells(MapRepP(tmp_cr.Rep())); - entitySet global_boundary_cells = all_collect_entitySet(local_boundary_cells) ; - int maxc = 0 ; - entitySet dom = tmp_cr.domain()&tmp_cl.domain() ; - FORALL(dom,ii) { - maxc = max(maxc,max(tmp_cr[ii],tmp_cl[ii])) ; - } ENDFORALL ; - int maxctot = maxc ; - MPI_Allreduce(&maxc,&maxctot,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; - entitySet boundary_taglist = global_boundary_cells ; - int ref_base = maxctot+1 ; - entitySet refSet = interval(ref_base,ref_base+(boundary_taglist.size()-1)) ; - - store boundary_info ; - boundary_info.allocate(refSet) ; - map boundary_remap ; - int ct = 0 ; - FORALL(boundary_taglist,ii) { - boundary_remap[ii]=ct+ref_base ; - boundary_info[ct+ref_base]=ii ; - ct++ ; - } ENDFORALL ; - - entitySet domcr = tmp_cr.domain() ; - FORALL(domcr,fc) { - if(tmp_cr[fc] < 0) - tmp_cr[fc] = boundary_remap[tmp_cr[fc]] ; - } ENDFORALL ; - - store tmp_boundary_tags,tmp_boundary_names ; - tmp_boundary_names.allocate(refSet) ; - tmp_boundary_tags.allocate(refSet) ; - - map bcmap ; - FORALL(refSet, ii) { - int bc = boundary_info[ii] ; - char buf[128] ; - bzero(buf,128) ; - snprintf(buf,127,"BC_%d",-bc) ; - bcmap[-bc] = ii ; - tmp_boundary_tags[ii] = string(buf) ; - tmp_boundary_names[ii] = string(buf) ; - } ENDFORALL ; - - for(size_t i=0;i::const_iterator mi = bcmap.find(id) ; - if(mi != bcmap.end()) - tmp_boundary_names[mi->second] = boundary_ids[i].second ; - else - debugout << "id " << id << " for boundary surface " << boundary_ids[i].second - << " not found!" << endl ; - } - - - if(Loci::MPI_rank == 0) { - Loci::debugout << " boundaries identified as:" ; - FORALL(refSet, bc) { - debugout << " " << tmp_boundary_names[bc] ; - } ENDFORALL ; - Loci::debugout << endl ; - } - - REPORTMEM() ; - - vector cell_ptn,face_ptn,node_ptn ; - - store cell_weights ; - if(cellwts != 0) - cell_weights = cellwts ; - else { - Loci::storeRepP ptr = getCellPartitionWeights(local_cells[MPI_rank]) ; - if(ptr != 0) { - cell_weights = ptr ; - } - } - - partitionerSelector partitioner_type = partitionerMethod ; - - if(partitioner_type == GRAPH) { - int lcpp = local_cells[MPI_rank].size() ; - int mincpp = lcpp ; - MPI_Allreduce(&lcpp,&mincpp,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; - if(mincpp < metis_cpp_threshold) { - partitioner_type = SFC ; // Space filling curve partitioner - debugout << "switching from metis to space filling curve partitioner" - << " mincpp = " << mincpp << endl ; - } -#ifndef LOCI_USE_METIS - partitioner_type = SFC ; // Space filling curve partitioner if METIS - // not available -#endif - } - switch(partitioner_type) { - case ORB: - ORB_Partition_Mesh(local_nodes, local_faces, local_cells, - t_pos, tmp_cl, tmp_cr, tmp_face2node, - tmp_boundary_tags, - cell_ptn,face_ptn,node_ptn) ; - break ; - case SFC: - SFC_Partition_Mesh(local_nodes, local_faces, local_cells, - t_pos, tmp_cl, tmp_cr, tmp_face2node, - tmp_boundary_tags, - cell_weights, - cell_ptn,face_ptn,node_ptn) ; - break ; - case SIMPLE: // Simple partition - { - cell_ptn = vector(MPI_processes) ; - cell_ptn[MPI_rank] = local_cells[MPI_rank] ; - REPORTMEM() ; - face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags) ; - REPORTMEM() ; - - node_ptn = partitionNodes(face_ptn, - MapRepP(tmp_face2node.Rep()), - t_pos.domain()) ; - - } - break ; -#ifdef LOCI_USE_METIS - case GRAPH: // METIS partition - { - cell_ptn = AdaptMetisPartitionOfCells(local_cells,tmp_cl,tmp_cr,tmp_boundary_tags,cellwts) ; - // cell_ptn = newMetisPartitionOfCells(local_cells,tmp_cl,tmp_cr,tmp_boundary_tags,cellwts) ; - REPORTMEM() ; - face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags) ; - REPORTMEM() ; - - node_ptn = partitionNodes(face_ptn, - MapRepP(tmp_face2node.Rep()), - t_pos.domain()) ; - } - break ; -#endif - case RANDOM: - { - RND_Partition_Mesh(local_nodes, local_faces, local_cells, - cell_ptn,face_ptn,node_ptn) ; - } - break ; - default: // SFC partition no weight balance - { - store cell_weights ; - - SFC_Partition_Mesh(local_nodes, local_faces, local_cells, - t_pos, tmp_cl, tmp_cr, tmp_face2node, - tmp_boundary_tags, - cell_weights, - cell_ptn,face_ptn,node_ptn) ; - } - break ; - } - - vector bcsurf_ptn(MPI_processes) ; - entitySet refset = tmp_boundary_tags.domain() ; - - // round robin allocate boundary faces - // NOTE!!!! This could be improved. - int cnt = 0 ; - // int refsetsz = refset.size() ; - FORALL(refset,ii) { - if(cnt == MPI_rank) - bcsurf_ptn[cnt] += ii ; - cnt++ ; - if(cnt == MPI_processes) - cnt = 0 ; - } ENDFORALL ; - REPORTMEM() ; - - vector cell_ptn_t = transposePtn(cell_ptn) ; - vector face_ptn_t = transposePtn(face_ptn) ; - vector node_ptn_t = transposePtn(node_ptn) ; - - int newnodes = 0 ; - for(int p=0;p node_alloc(newnodes) ; - int i=0; - for(int p=0;p face_alloc(newfaces) ; - i = 0 ; - for(int p=0;p cell_alloc(newcells) ; - i = 0 ; - for(int p=0;pg2fv[0].Rep() ; // FIX THIS - - int cells_base=local_cells[0].Min() ; - for(size_t i=0;i Tag ; - *Tag = volTags[i].first ; - - // Map entitySet to new ordering - entitySet inputTag = (volTags[i].second >> cells_base) ; - // Map entitySet to new ordering - entitySet tagset ; - FORALL(cells,cc) { - if(inputTag.inSet(g2f[cc])) - tagset += cc ; - } ENDFORALL ; - Tag.set_entitySet(tagset) ; - ostringstream oss ; - oss << "volumeTag(" << volTags[i].first << ")" ; - facts.create_fact(oss.str(),Tag) ; - } - // if(collect_perf_data) - // perfAnalysis->stop_timer(read_file_timer); - double t2 = MPI_Wtime() ; - debugout << "Time to process and partition adapted mesh is " << t2-t1 - << endl ; - REPORTMEM() ; - return true ; - - } - - - bool setupFVMGridFromContainer(fact_db &facts, - vector& local_nodes, - vector& local_faces, - vector& local_cells, - store >& t_pos, - Map& tmp_cl, - Map& tmp_cr, - multiMap& tmp_face2node, - vector >& boundary_ids, - vector >& volTags, - storeRepP cellwts) { - - if(!inputFVMGrid(facts, - local_nodes, - local_faces, - local_cells, - t_pos, - tmp_cl, - tmp_cr, - tmp_face2node, - boundary_ids, - volTags,cellwts)) - return false ; - REPORTMEM() ; - - create_face_info(facts) ; - create_ref(facts) ; - create_ghost_cells(facts) ; - - return true ; - } -} - - -namespace Loci { - void createVOGNode(store &new_pos, - const store &inner_nodes, - int& num_nodes, - fact_db & facts, - vector& local_nodes) { - cerr << "AMR API has changed, this call is deprecated. Update your AMR API calls!" << endl ; - Loci::Abort() ; - } - void createVOGFace(int numNodes, - const store &fine_faces, - fact_db & facts, - int& numFaces, - int& ncells, - Map& cl, - Map& cr, - multiMap& face2node, - vector& local_faces, - vector& local_cells - ) { - cerr << "AMR API has changed, this call is deprecated. Update your AMR API calls!" << endl ; - Loci::Abort() ; - } - - //create a new store new_pos from pos and inner_nodes - //re_number the nodes in pos and inner_nodes, - //and then redistribute them across the processes - void createVOGNode(store > &new_pos, - const store &inner_nodes_cell, - const store &inner_nodes_face, - const store &inner_nodes_edge, - - int& num_nodes, - fact_db & facts,//in global numbering - vector& nodes_ptn - ){ - //get store pos - store > pos; - pos = facts.get_variable("pos"); - - if(MPI_processes == 1){ - //firsr write out numNodes - long num_original_nodes = pos.domain().size(); - long num_inner_nodes = 0; - FORALL(inner_nodes_cell.domain(), cc){ - num_inner_nodes += inner_nodes_cell[cc].size(); - }ENDFORALL; - FORALL(inner_nodes_face.domain(), cc){ - num_inner_nodes += inner_nodes_face[cc].size(); - }ENDFORALL; - FORALL(inner_nodes_edge.domain(), cc){ - num_inner_nodes += inner_nodes_edge[cc].size(); - }ENDFORALL; - - int node_base = 0; - long npnts = num_original_nodes + num_inner_nodes; - entitySet new_domain= interval(node_base, node_base+npnts-1); - new_pos.allocate(new_domain); - - entitySet::const_iterator nei = new_domain.begin(); - entitySet::const_iterator ei = pos.domain().begin(); - - for(long count = 0; count < num_original_nodes; count++, nei++, ei++){ - new_pos[*nei] = pos[*ei]; - } - - Loci::constraint faces, geom_cells; - faces = facts.get_variable("faces"); - geom_cells = facts.get_variable("geom_cells"); - entitySet local_edges = facts.get_variable("edge2node")->domain(); - - FORALL(local_edges, cc){ - for(unsigned int i = 0; i < inner_nodes_edge[cc].size(); i++, nei++){ - new_pos[*nei] = inner_nodes_edge[cc][i]; - } - }ENDFORALL; - FORALL(*geom_cells, cc){ - for(unsigned int i = 0; i < inner_nodes_cell[cc].size(); i++, nei++){ - new_pos[*nei] = inner_nodes_cell[cc][i]; - } - }ENDFORALL; - FORALL(*faces, cc){ - for(unsigned int i = 0; i < inner_nodes_face[cc].size(); i++, nei++){ - new_pos[*nei] = inner_nodes_face[cc][i]; - } - }ENDFORALL; - num_nodes = new_domain.size(); - nodes_ptn.resize(1); - nodes_ptn[0] = new_domain; - return; - } - - - - - store > pos_t; //temp container - vector temp_node_ptn; - entitySet my_temp_nodes; - { - //reorder store first, from global to io entities - fact_db::distribute_infoP dist = facts.get_distribute_info() ; - constraint my_faces, my_geom_cells, my_edges; - - my_faces = facts.get_variable("faces"); - my_geom_cells = facts.get_variable("geom_cells"); - my_edges = facts.get_variable("edges"); - - entitySet local_edges = *my_edges; - entitySet local_faces = *my_faces; - entitySet local_cells = *my_geom_cells; - entitySet local_nodes = pos.domain(); - - //before write out,create stores for pos and inner_nodes which - //are ordered across processors in the file numbering, the domain of this container - //shifted by offset is the actual file numbering. offset will be modified after function call - - int noffset, eoffset, coffset, foffset; - noffset = 0; - store > pos_io; - pos_io = Loci::Global2FileOrder(pos.Rep(), local_nodes, noffset, dist, MPI_COMM_WORLD) ; - entitySet file_nodes = pos_io.domain(); - - - eoffset = 0; - store edge_inner_nodes; - edge_inner_nodes = Loci::Global2FileOrder(inner_nodes_edge.Rep(),local_edges,eoffset,dist,MPI_COMM_WORLD) ; - entitySet file_edges = edge_inner_nodes.domain(); - - coffset= 0; - // Create container - store cell_inner_nodes; - cell_inner_nodes = Loci::Global2FileOrder(inner_nodes_cell.Rep(),local_cells,coffset,dist,MPI_COMM_WORLD) ; - entitySet file_cells = cell_inner_nodes.domain(); - - foffset= 0; - // Create container - store face_inner_nodes; - face_inner_nodes = Loci::Global2FileOrder(inner_nodes_face.Rep(),local_faces,foffset,dist,MPI_COMM_WORLD) ; - entitySet file_faces = face_inner_nodes.domain(); - - //Now allocate temp entitySet in file numbering - - - //compute the size of pos - int num_pos_nodes = file_nodes.size(); - std::vector pos_sizes(Loci::MPI_processes) ; - pos_sizes = Loci::all_collect_sizes(num_pos_nodes); - - //compute the size of inner_nodes - int num_local_edge_nodes = 0; - FORALL(file_edges, cc){ - num_local_edge_nodes += edge_inner_nodes[cc].size(); - }ENDFORALL; - - int num_local_cell_nodes = 0; - FORALL(file_cells, cc){ - num_local_cell_nodes += cell_inner_nodes[cc].size(); - }ENDFORALL; - - int num_local_face_nodes = 0; - FORALL(file_faces, cc){ - num_local_face_nodes += face_inner_nodes[cc].size(); - }ENDFORALL; - std::vector inner_edge_nodes_sizes(Loci::MPI_processes) ; - inner_edge_nodes_sizes = Loci::all_collect_sizes(num_local_edge_nodes); - - std::vector inner_cell_nodes_sizes(Loci::MPI_processes) ; - inner_cell_nodes_sizes = Loci::all_collect_sizes(num_local_cell_nodes); - - std::vector inner_face_nodes_sizes(Loci::MPI_processes) ; - inner_face_nodes_sizes = Loci::all_collect_sizes(num_local_face_nodes); - - //set up the offset - int my_id = Loci::MPI_rank; - int num_procs = Loci::MPI_processes; - noffset = 0; - for(int i = 0; i < my_id; i++){ - noffset += pos_sizes[i]; - } - - eoffset = 0; - for(int i = 0; i < num_procs; i++){ - eoffset += pos_sizes[i]; - } - for(int i = 0; i < my_id; i++){ - eoffset += inner_edge_nodes_sizes[i]; - } - - coffset = 0; - for(int i = 0; i < num_procs; i++){ - coffset += pos_sizes[i]+inner_edge_nodes_sizes[i]; - } - for(int i = 0; i < my_id; i++){ - coffset += inner_cell_nodes_sizes[i]; - } - - - foffset = 0; - for(int i = 0; i < num_procs; i++){ - foffset += pos_sizes[i]+inner_edge_nodes_sizes[i]+inner_cell_nodes_sizes[i]; - } - for(int i = 0; i < my_id; i++){ - foffset += inner_face_nodes_sizes[i]; - } - - - num_nodes = 0; - for(int i = 0; i < num_procs; i++){ - num_nodes += pos_sizes[i]+inner_edge_nodes_sizes[i]+ - inner_cell_nodes_sizes[i]+inner_face_nodes_sizes[i]; - } - - //create a new store pos_t in file numbering that contains all the nodes - entitySet my_original_nodes, my_inner_edge_nodes; - entitySet my_inner_cell_nodes, my_inner_face_nodes; - if(pos_sizes[my_id]>0) my_original_nodes = interval(noffset, noffset+pos_sizes[my_id]-1); - if(inner_edge_nodes_sizes[my_id]>0)my_inner_edge_nodes = interval(eoffset, eoffset+inner_edge_nodes_sizes[my_id]-1); - if(inner_cell_nodes_sizes[my_id]>0)my_inner_cell_nodes = interval(coffset, coffset+inner_cell_nodes_sizes[my_id]-1); - if(inner_face_nodes_sizes[my_id]>0)my_inner_face_nodes = interval(foffset, foffset+inner_face_nodes_sizes[my_id]-1); - my_temp_nodes = my_original_nodes + my_inner_edge_nodes + - my_inner_cell_nodes + my_inner_face_nodes; - pos_t.allocate(my_temp_nodes); - - //fill container pos_t will pos_io - entitySet::const_iterator ti = my_temp_nodes.begin(); - FORALL(file_nodes, ei){ - pos_t[*ti] = pos_io[ei]; - ti++; - }ENDFORALL; - //fill container pos_t will edge nodes - FORALL(file_edges, ei){ - for(unsigned int i = 0; i < edge_inner_nodes[ei].size(); i++){ - pos_t[*ti] = edge_inner_nodes[ei][i]; - ti++; - } - - }ENDFORALL; - //fill container pos_t will cell nodes - FORALL(file_cells, ei){ - for(unsigned int i = 0; i < cell_inner_nodes[ei].size(); i++){ - pos_t[*ti] = cell_inner_nodes[ei][i]; - ti++; - } - - }ENDFORALL; - //fill container pos_t will face nodes - FORALL(file_faces, ei){ - for(unsigned int i = 0; i < face_inner_nodes[ei].size(); i++){ - pos_t[*ti] = face_inner_nodes[ei][i]; - ti++; - } - }ENDFORALL; - if(ti != my_temp_nodes.end()){ - debugout<<"ERROR in createVOGNOde()" << endl; - } - temp_node_ptn = all_collect_vectors(my_temp_nodes); - } - //create new partition - nodes_ptn.resize(Loci::MPI_processes); - // create node allocation - long npnts = num_nodes ; - int node_ivl = npnts / Loci::MPI_processes; - int node_ivl_rem = npnts % Loci::MPI_processes ; - int node_accum = 0 ; - int nodes_base = 0 ; - for(int i = 0; i < Loci::MPI_processes; ++i) { - int j = Loci::MPI_processes - i - 1 ; - int node_accum_update = node_accum + node_ivl + ((j send_sets(p) ; - vector send_seqs(p) ; - - // Check each processor, find out which sets to send - for(int i=0;i recv_seqs = transposeSeq(send_seqs) ; - - storeRepP sp =pos_t.Rep(); - storeRepP qcol_rep =new_pos.Rep(); - - - // Now communicate the container - vector send_sizes(p),recv_sizes(p) ; - - for(int i=0;ipack_size(send_sets[i]) ; - - MPI_Alltoall(&send_sizes[0],1,MPI_INT, - &recv_sizes[0],1,MPI_INT, - MPI_COMM_WORLD ) ; - - vector send_dspl(p),recv_dspl(p) ; - send_dspl[0] = 0 ; - recv_dspl[0] = 0 ; - for(int i=1;i send_store(send_sz) ; - vector recv_store(recv_sz) ; - - - for(int i=0;ipack(&send_store[send_dspl[i]],loc_pack, send_sizes[i], - send_sets[i]) ; - } - - MPI_Alltoallv(&send_store[0], &send_sizes[0], &send_dspl[0], MPI_PACKED, - &recv_store[0], &recv_sizes[0], &recv_dspl[0], MPI_PACKED, - MPI_COMM_WORLD ) ; - - for(int i=0;iunpack(&recv_store[recv_dspl[i]],loc_pack,recv_sizes[i], - recv_seqs[i]) ; - } - pos_t.allocate(EMPTY) ; - } - - - //create cl, cr and face2node maps from fine_faces - //re_number the nodes in pos and inner_nodes, - //and then redistribute them across the processes - void createVOGFace(int numNodes, - const store &fine_faces_cell, - const store &fine_faces, - fact_db & facts, - int& numFaces, - int& ncells, - Map& cl, - Map& cr, - multiMap& face2node, - vector& local_faces, - vector& local_cells - ){ - - //compute numFaces - constraint my_faces, my_geom_cells; - my_faces = facts.get_variable("faces"); - my_geom_cells = facts.get_variable("geom_cells"); - entitySet dom = fine_faces.domain() & (*my_faces+*my_geom_cells); - entitySet domc = fine_faces_cell.domain() ; - int local_num_face = 0; - for(entitySet::const_iterator ei = domc.begin(); ei != domc.end(); ei++){ - local_num_face += fine_faces_cell[*ei].size(); - } - for(entitySet::const_iterator ei = dom.begin(); ei != dom.end(); ei++){ - local_num_face += fine_faces[*ei].size(); - } - numFaces = 0; - std::vector face_sizes= Loci::all_collect_sizes(local_num_face); - for(int i =0; i < MPI_processes; i++) numFaces += face_sizes[i]; - - //get face domains and allocate the maps - int face_min = numNodes; - for(int i =0; i < MPI_rank; i++) face_min += face_sizes[i]; - - int face_max = face_min + face_sizes[MPI_rank] -1; - store count; - entitySet faces = EMPTY ; - if(face_sizes[MPI_rank] > 0) - faces = interval(face_min, face_max); - - cl.allocate(faces); - cr.allocate(faces); - count.allocate(faces); - - local_faces.resize(MPI_processes); - local_faces = all_collect_vectors(faces); - //fill up the maps cl , cr and count - int cell_base = numNodes + numFaces; - int cell_max = std::numeric_limits::min(); - int cell_min = std::numeric_limits::max(); - - entitySet::const_iterator fid = faces.begin(); - for(entitySet::const_iterator ei = domc.begin(); ei != domc.end(); ei++){ - for(unsigned int i = 0; i < fine_faces_cell[*ei].size(); i++){ - cl[*fid] = fine_faces_cell[*ei][i][0] + cell_base -1;// -1 finefaces cell index start at 1 - if(fine_faces_cell[*ei][i][1]>=0) cr[*fid] = fine_faces_cell[*ei][i][1] + cell_base -1; - else cr[*fid] = fine_faces_cell[*ei][i][1]; - cell_max = max(cell_max,cl[*fid]); - cell_max = max(cell_max,cr[*fid]); - cell_min = min(cell_min,cl[*fid]); - if (cr[*fid]>=0) cell_min = min(cell_min,cr[*fid]); - count[*fid] = fine_faces_cell[*ei][i].size()-2; - fid++; - } - } - for(entitySet::const_iterator ei = dom.begin(); ei != dom.end(); ei++){ - for(unsigned int i = 0; i < fine_faces[*ei].size(); i++){ - cl[*fid] = fine_faces[*ei][i][0] + cell_base -1;// -1 finefaces cell index start at 1 - if(fine_faces[*ei][i][1]>=0) cr[*fid] = fine_faces[*ei][i][1] + cell_base -1; - else cr[*fid] = fine_faces[*ei][i][1]; - cell_max = max(cell_max,cl[*fid]); - cell_max = max(cell_max,cr[*fid]); - cell_min = min(cell_min,cl[*fid]); - if (cr[*fid]>=0) cell_min = min(cell_min,cr[*fid]); - count[*fid] = fine_faces[*ei][i].size()-2; - fid++; - } - } - - //get cells distribution - int global_max_cell = cell_max ; - MPI_Allreduce(&cell_max,&global_max_cell,1,MPI_INT,MPI_MAX, - MPI_COMM_WORLD) ; - int global_min_cell = cell_max ; - MPI_Allreduce(&cell_min,&global_min_cell,1,MPI_INT,MPI_MIN, - MPI_COMM_WORLD) ; - - - ncells = global_max_cell - global_min_cell +1; - - - local_cells.resize(MPI_processes); - int cell_ivl = ncells / MPI_processes; - int cell_ivl_rem = ncells % MPI_processes ; - int cell_accum = 0 ; - - for(int i = 0; i < Loci::MPI_processes; ++i) { - int j = MPI_processes - i - 1 ; - int cell_accum_update = cell_accum + cell_ivl + ((j &cluster_info, - vector &cluster_sizes) ; - bool readBCfromVOG(string filename, - vector > &boundary_ids); - bool readVolTags(hid_t input_fid, - vector > &volDat); - - hid_t writeVOGOpen(string filename); - void writeVOGSurf(hid_t file_id, std::vector > surface_ids); - void writeVOGTag(hid_t output_fid, vector >& volTags); - void writeVOGClose(hid_t file_id) ; - -} - - - -void colorMatrix(Map &cl, Map &cr, multiMap &face2node); -namespace Loci{ - - //copied from ditribute_io.cc - hid_t writeVOGOpen(string filename) { - if(use_parallel_io){ - hid_t file_id = 0; - hid_t acc_plist; - // open collectively by all processor in MPI_COMM_WORLD, - acc_plist = Loci::create_faccess_plist(MPI_COMM_WORLD, - Loci::PHDF5_MPI_Info, - Loci::hdf5_const::facc_type); - file_id = H5Fcreate(filename.c_str(),H5F_ACC_TRUNC,H5P_DEFAULT,acc_plist) ; - H5Pclose(acc_plist); - if(file_id == 0) { - if(MPI_rank==0) cerr << "unable to open file " << filename << endl ; - Loci::Abort() ; - } - return file_id ; - }else{ - hid_t file_id = 0 ; - if(MPI_rank==0) - file_id = H5Fcreate(filename.c_str(),H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT) ; - return file_id ; - } - } - - - //copied from FVMGridWriter.cc - void writeVOGClose(hid_t file_id) {//parallel io included - if(MPI_rank == 0 || use_parallel_io) H5Fclose(file_id) ; - } - - //similar to the one in FVMGridWriter.cc, but no modofication to surface_ids - void writeVOGSurf(hid_t file_id, std::vector > surface_ids) { - hid_t group_id = 0 ; - if(MPI_rank == 0 || use_parallel_io) { - if(surface_ids.size() != 0) { -#ifdef H5_USE_16_API - group_id = H5Gcreate(file_id,"surface_info",0) ; -#else - group_id = H5Gcreate(file_id,"surface_info", - H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; -#endif - for(size_t i=0;i > &volDat) { - using namespace Loci ; - /* Save old error handler */ - H5E_auto_t old_func = 0; - void *old_client_data = 0 ; -#ifdef H5_USE_16_API - H5Eget_auto(&old_func, &old_client_data); - H5Eset_auto(NULL, NULL); -#else - H5Eget_auto(H5E_DEFAULT,&old_func, &old_client_data); - H5Eset_auto(H5E_DEFAULT,NULL, NULL); -#endif - /* Turn off error handling */ - - vector > volTags ; -#ifdef H5_USE_16_API - hid_t cell_info = H5Gopen(input_fid,"cell_info") ; -#else - hid_t cell_info = H5Gopen(input_fid,"cell_info",H5P_DEFAULT) ; -#endif - if(cell_info > 0) { - vector vol_tag ; - vector vol_set ; - vector vol_id ; - - hsize_t num_tags = 0 ; - H5Gget_num_objs(cell_info,&num_tags) ; - for(hsize_t tg=0;tg > tmp(maxi+1) ; - volTags.swap(tmp) ; - for(size_t i=0;i - (string("Main"), - entitySet(interval(0,numCells-1)))) ; - H5Gclose(file_info) ; - } - - /* Restore previous error handler */ -#ifdef H5_USE_16_API - H5Eset_auto(old_func, old_client_data); -#else - H5Eset_auto(H5E_DEFAULT,old_func, old_client_data); -#endif - volDat.swap(volTags) ; - return true ; - } - -} -//same as the function in FVMGridWriter.cc -void writeVOGFace(hid_t file_id, Map &cl, Map &cr, multiMap &face2node) { - // Compute cell set - entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; - entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells) ; - - Map tmp_cl, tmp_cr; - multiMap tmp_face2node; - - - long long local_num_faces = face2node.domain().size() ; - - - long long num_cells = geom_cells.size() ; - long long num_faces = 0 ; - - // Reduce these variables - MPI_Allreduce(&local_num_faces,&num_faces,1,MPI_LONG_LONG_INT, - MPI_SUM,MPI_COMM_WORLD) ; - - hid_t group_id = 0 ; - if(MPI_rank == 0 || Loci::use_parallel_io) { -#ifdef H5_USE_16_API - group_id = H5Gopen(file_id,"file_info") ; -#else - group_id = H5Gopen(file_id,"file_info",H5P_DEFAULT) ; -#endif - - if(MPI_rank == 0) std::cerr<< "num_cells = " << num_cells << endl - << "num_faces = " << num_faces << endl ; - - hsize_t dims = 1 ; - hid_t dataspace_id = H5Screate_simple(1,&dims,NULL) ; - -#ifdef H5_USE_16_API - hid_t att_id = H5Acreate(group_id,"numFaces", H5T_STD_I64BE, - dataspace_id, H5P_DEFAULT) ; -#else - hid_t att_id = H5Acreate(group_id,"numFaces", H5T_STD_I64BE, - dataspace_id, H5P_DEFAULT,H5P_DEFAULT) ; -#endif - H5Awrite(att_id,H5T_NATIVE_LLONG,&num_faces) ; - H5Aclose(att_id) ; -#ifdef H5_USE_16_API - att_id = H5Acreate(group_id,"numCells", H5T_STD_I64BE, - dataspace_id, H5P_DEFAULT) ; -#else - att_id = H5Acreate(group_id,"numCells", H5T_STD_I64BE, - dataspace_id, H5P_DEFAULT,H5P_DEFAULT) ; -#endif - H5Awrite(att_id,H5T_NATIVE_LLONG,&num_cells) ; - H5Aclose(att_id) ; - H5Gclose(group_id) ; -#ifdef H5_USE_16_API - group_id = H5Gcreate(file_id,"face_info",0) ; -#else - group_id = H5Gcreate(file_id,"face_info", - H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; -#endif - } - - entitySet faces = face2node.domain() ; - vector, int> > f_ord(faces.size()) ; - int i = 0 ; - // For small number of cells, sort to keep bc groupings - if(num_cells<100000) { - FORALL(faces,fc) { - f_ord[i].first.first = cr[fc] ; - f_ord[i].first.second = cl[fc] ; - f_ord[i].second = fc ; - i++ ; - } ENDFORALL ; - std::sort(f_ord.begin(),f_ord.end()) ; - } else { - FORALL(faces,fc) { - f_ord[i].first.first = cl[fc] ; - f_ord[i].first.second = cr[fc] ; - f_ord[i].second = fc ; - i++ ; - } ENDFORALL ; - } - - i=0 ; - store count ; - count.allocate(faces) ; - FORALL(faces,fc) { - int nfc = f_ord[i].second ; - count[fc] = face2node[nfc].size() ; - i++ ; - } ENDFORALL ; - tmp_face2node.allocate(count) ; - tmp_cl.allocate(faces) ; - tmp_cr.allocate(faces) ; - i=0 ; - - int mc = (geom_cells).Min() ; - // Nodes should be adjusted to start from zero also... for the general case - FORALL(faces,fc) { - int nfc = f_ord[i].second ; - tmp_cl[fc] = cl[nfc]-mc ; - tmp_cr[fc] = cr[nfc] ; - if(tmp_cr[fc] >= 0) - tmp_cr[fc] -= mc ; - for(int j=0;j cluster_info ; - vector cluster_sizes ; - while(faces != EMPTY) { - entitySet fcluster = Loci::faceCluster(tmp_face2node,tmp_cl,tmp_cr,faces, - cluster_info,cluster_sizes) ; - faces -= fcluster ; - } - - Loci::writeUnorderedVector(group_id,"cluster_sizes",cluster_sizes) ; - Loci::writeUnorderedVector(group_id,"cluster_info",cluster_info) ; - - - if(MPI_rank == 0 || Loci::use_parallel_io) { - H5Gclose(group_id) ; - } -} - - -vector > getVOGTagFromLocal(const vector > &origVolTags, - // string outfile, - const_store &cell_offset, - const_store & num_fine_cells, - int num_original_nodes, - int num_original_faces) { - - - - vector > volTags(origVolTags.size()); - - //serial version - if(Loci::MPI_processes == 1){ - for(unsigned int i = 0; i < origVolTags.size(); i++){ - string name = origVolTags[i].first; - //transfer entitySet into vector of interval - entitySet set=origVolTags[i].second; - int sz = set.num_intervals() ; - vector vlist(sz) ; - for(int j=0;jget_distribute_info() ; - Loci::constraint geom_cells = Loci::exec_current_fact_db->get_variable("geom_cells"); - Loci::constraint my_entities; - my_entities = dist->my_entities ; - //don't know if it's necessray - entitySet local_geom_cells = (*my_entities)&(*geom_cells); - - if(Loci::MPI_processes > 1){ - - //trasnfer the store to file numbering - int offset = 0; - store file_num_fine_cells; - file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; - offset = 0; - store file_cell_offset; - file_cell_offset = Loci::Local2FileOrder(cell_offset.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; - - int coffset = 0; - std::vector local_cell_sizes; - int num_local_cells = file_cell_offset.domain().size(); - local_cell_sizes = Loci::all_collect_sizes(num_local_cells); - for(int i = 0; i < MPI_rank; i++){ - coffset += local_cell_sizes[i]; - } - - - //process 0 broadcast the entitySet to all - int buf_size = 0; - if(MPI_rank==0){ - for(unsigned int i = 0; i < origVolTags.size(); i++){ - //transfer entitySet into vector of interval - entitySet set=origVolTags[i].second; - int sz = set.num_intervals() ; - buf_size += sz; - } - } - MPI_Bcast(&buf_size,1,MPI_INT, 0, MPI_COMM_WORLD) ; - vector buffer(2*buf_size); - if(MPI_rank==0){ - int pnt = 0; - for(unsigned int i = 0; i < origVolTags.size(); i++){ - //transfer entitySet into vector of interval - entitySet set=origVolTags[i].second; - int sz = set.num_intervals() ; - for(int j=0;j=0 && domain.inSet(start)) buffer[2*i] = file_cell_offset[start]; - else buffer[2*i] = 0; - if(end >=0 && domain.inSet(end)) buffer[2*i+1] = file_cell_offset[end]+file_num_fine_cells[end]-1; - else buffer[2*i+1] = 0; - - } - //process 0 receive the values in buffer from all the other processes, add up the values from each process. - if(MPI_rank == 0){ - - for(int pi = 1; pi recv_buf(2*buf_size); - MPI_Status mstat ; - MPI_Recv(&recv_buf[0],sizeof(int)*2*buf_size,MPI_BYTE,pi,7,MPI_COMM_WORLD, - &mstat) ; - for(int i = 0; i < 2*buf_size; i++)buffer[i] += recv_buf[i]; - } - }else{ - int flag = 0; - MPI_Status mstat ; - MPI_Recv(&flag,1,MPI_INT,0,6,MPI_COMM_WORLD,&mstat) ; - MPI_Send(&buffer[0],sizeof(int)*2*buf_size,MPI_BYTE,0,7,MPI_COMM_WORLD); - } - //process 0 re-organize the data in buffer into volTags and write it out - if(MPI_rank == 0){ - int pnt = 0; - for(unsigned int i = 0; i < origVolTags.size(); i++){ - //transfer entitySet into vector of interval - string name = origVolTags[i].first; - entitySet set=origVolTags[i].second; - int sz = set.num_intervals() ; - entitySet new_set = EMPTY; - for(int j = 0; j < sz; j++){ - int start = buffer[pnt++]; - int end = buffer[pnt++]; - new_set += interval(start, end); - } - volTags[i] = make_pair(name, new_set); - std::cout << "old tag: " << name << " " << set << std::endl; - std::cout << "new tag: " << name << " " << new_set << std::endl; - } - - } - - //broadcast volTags to other processes - int nvtags = volTags.size() ; - MPI_Bcast(&nvtags,1,MPI_INT,0,MPI_COMM_WORLD) ; - if(MPI_rank != 0) volTags.clear(); - for(int i=0;i(name,set)) ; - delete[] ibuf ; - } - - } - return volTags; -} - - - - - - +//############################################################################# +//# +//# Copyright 2008-2025, Mississippi State University +//# +//# This file is part of the Loci Framework. +//# +//# The Loci Framework is free software: you can redistribute it and/or modify +//# it under the terms of the Lesser GNU General Public License as published by +//# the Free Software Foundation, either version 3 of the License, or +//# (at your option) any later version. +//# +//# The Loci Framework is distributed in the hope that it will be useful, +//# but WITHOUT ANY WARRANTY; without even the implied warranty of +//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//# Lesser GNU General Public License for more details. +//# +//# You should have received a copy of the Lesser GNU General Public License +//# along with the Loci Framework. If not, see +//# +//############################################################################# +#include +#include +#include +#include +#include +#include +#include +#include "sciTypes.h" +#include "defines.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "FVMAdapt/defines.h" +#include "FVMAdapt/dataxferDB.h" +#include "FVMAdapt/gridInterface.h" + +using std::cerr; +using std::cout; +using std::endl; +using std::string; +using std::ifstream; +using Loci::storeRepP; +using Loci::constraint; +using std::vector; +using Loci::MPI_rank; +using Loci::MPI_processes; +using std::map ; +using std::ostringstream ; +using std::istringstream ; +///---------------------------------------------------------------------- +// HACK WARNING +// +// The FVMGridReader has been changed to support 4 gig entities. This has +// an impact on how boundary faces are represented in the FVMadapt code. We +// just copied the old code into here for now, but that means that adapt +// doesn't work with large interval sets. This needs to be fixed... however +// we are reworking how the fact_db will be created so this can wait until that +// process is further along. + +// RSM MOD 20181108 +#ifdef LOCI_USE_METIS +#ifdef USE_SCOTCH +#include +#endif +#include +#include + +#ifdef USE_SCOTCH +typedef float metisreal_t ; +#else +#if REALTYPEWIDTH == 32 +typedef float metisreal_t ; +#else +typedef double metisreal_t ; +#endif +#endif + +#endif + +namespace Loci { + // get cell2parent map with cells in current global numbering + + storeRepP getC2PGlobal(fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; + + // Check to see if a global version has already been generated + storeRepP ptr = DataXFER_DB.getItem("c2pglobal") ; + if(ptr != 0) { + return ptr ; + } + // Now convert c2p from a file numbering of cells to the current + // global numbering + store > c2pset ; + c2pset = DataXFER_DB.getItem("c2p") ; + entitySet dom = c2pset.domain() ; + protoMap c2p(dom.size()) ; + int cnt = 0 ; + FORALL(dom,ii) { + c2p[cnt++] = c2pset[ii] ; + } ENDFORALL ; + // find the max and min numbering of the file numbering in cells in + // c2p + int mcol = c2p[0].first ; + int xcol = mcol ; + for(int i=0;i > c2pg ; + c2pg.allocate(dom) ; + FORALL(dom,ii) { + c2pg[ii] = c2p[ii] ; + } ENDFORALL ; + DataXFER_DB.insertItem("c2pglobal",c2pg.Rep()) ; + return c2pg.Rep() ; + } + // Now get the maps needed to translate from global to file + dMap g2f ; + g2f = dist->g2f.Rep() ; + Map l2g ; + l2g = dist->l2g.Rep() ; + constraint geom_cells ; + geom_cells = facts.get_fact("geom_cells") ; + dom = geom_cells.Rep()->domain() ; + dom = dom & dist->my_entities ; + + // Create protomap from cell file number to global number + protoMap f2g(dom.size()) ; + cnt = 0 ; + int mfol = std::numeric_limits::max() ; + int xfol = std::numeric_limits::lowest() ; + FORALL(dom,ii) { + int g = l2g[ii] ; + int f = g2f[g] ; + f2g[cnt++] = pair(f,g) ; + mfol = min(mfol,f) ; + xfol = max(xfol,f) ; + } ENDFORALL ; + + int mfo = mfol ; + int xfo = xfol ; + MPI_Allreduce(&mfol,&mfo,1,MPI_INT,MPI_MIN,comm) ; + MPI_Allreduce(&xfol,&xfo,1,MPI_INT,MPI_MAX,comm) ; + // Check to make sure that the c2p map is consistent with the number of + // cells in this mesh + if(xfo-mfo != xco-mco) { + if(MPI_rank == 0) { + cerr << "ERROR!: Mismatch between number of cells in c2p map!" << endl ; + cerr << " : number of file no cells =" << xfo-mfo << ", local =" << xco-mco << endl ; + } + } + // Remap file numbers to be consistent with c2p numbering + for(int i=0;i > c2pg ; + c2pg.allocate(c2pgset) ; + FORALL(c2pgset,ii) { + c2pg[ii] = v[ii] ; + } ENDFORALL ; + // Write into XFER DB so we don't have to do this multiple times + DataXFER_DB.insertItem("c2pglobal",c2pg.Rep()) ; + // return new map + return c2pg.Rep() ; + } + + namespace { + void computeWeights(vector3d *wvec, const vector3d *deltas, const real_t *areas, + int ndeltas) { + tmp_array > weights(ndeltas) ; + + real_t r[3][3] ; + r[0][0] = 0 ; + for(int i=0;i(weightx,weighty,weightz)*areas[i] ; + } + } + } + + void AMRrefinementMapping:: + setupRefinementMapping(const store > &c2pg, + const store &volw, + multiStore &gradCells, + multiStore > &deltas, + const_store > &cell_center, + const_store &vol, + fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; + //######################################################################## + // + // get the geom_cells of child cells and convert to global numbering + // + auto sp = facts.get_variable("geom_cells") ; + if(sp==0) { + cerr << "expecting facts to contain geom_cells to build AMR second order interpolation" << endl ; + Loci::Abort() ; + } + + // geom_cells is the local numbering, convert to global numbering + fact_db::distribute_infoP dist = facts.get_distribute_info() ; + // get the geom_cells owned by this processor + geom_cells_local = sp->domain()&dist->my_entities ; + // convert form local to global numbering + l2g.allocate(geom_cells_local) ; + FORALL(geom_cells_local,ii) { + geom_cells_global += dist->l2g[ii] ; + l2g[ii] = dist->l2g[ii] ; + } ENDFORALL ; + +#ifdef DEBUG + debugout << "geom_cells =" << geom_cells_global << endl ; +#endif + //######################################################################## + // + // Get partition of child cells by collecting each procesors geom_cells + // set + // + vector ptn = Loci::all_collect_vectors(geom_cells_global, + comm) ; + dataPartitionP partition_child = createPartition(ptn,comm) ; + + //######################################################################## + // + // Get partition of parent cells by computing first element on each + // processor of the volw container + // + int p = Loci::MPI_processes ; + int psplit = volw.domain().Min() ; + vector psplits(p) ; + // Gather splits + MPI_Allgather(&psplit,1,MPI_INT,&psplits[0],1,MPI_INT, + comm) ; +#ifdef DEBUG + debugout << "parent partition splits = " << endl ; + for(int i=0;i::max() ; + minval[1] = minval[0] ; + maxval[0] = std::numeric_limits::lowest() ; + maxval[1] = maxval[0] ; + entitySet domc2pg = c2pg.domain() ; + FORALL(domc2pg,ii) { + minval[0] = min(minval[0],c2pg[ii].first) ; + minval[1] = min(minval[1],c2pg[ii].second) ; + maxval[0] = max(maxval[0],c2pg[ii].first) ; + maxval[1] = max(maxval[1],c2pg[ii].second) ; + } ENDFORALL ; + + int maxvalg[2] = {maxval[0],maxval[1]} ; + int minvalg[2] = {minval[0],minval[1]} ; + MPI_Allreduce(&minval[0],&minvalg[0],2,MPI_INT,MPI_MIN,comm) ; + MPI_Allreduce(&maxval[0],&maxvalg[0],2,MPI_INT,MPI_MAX,comm) ; + + //######################################################################## + // + // Transpose cell to parent map to get parent to cell map. Add offsets + // to directly make the parent to cell maps start with zero indexing. + // + int sz = domc2pg.size() ; + vector > p2c(sz) ; + int cnt = 0 ; + int offsetsecond = -minvalg[1] ; + int offsetfirst = -minvalg[0] ; + // Transpose cell-to-parent to parent to cell, adjust so the indexing starts + // at zero + FORALL(domc2pg,ii) { + p2c[cnt].first = c2pg[ii].second + offsetsecond ; + p2c[cnt].second = c2pg[ii].first + offsetfirst ; + cnt++ ; + } ENDFORALL ; + FATAL(cnt != sz) ; + + + //######################################################################## + // + // Adjust parent and cell numbering in map to align with numbering used + // in volw and geom_cells. + // + // cstart is child starting number,csplit is describing the distribution of + // parent cells across processors. + int csplit = geom_cells_global.Min() ; + int cstart = csplit ; + + MPI_Allreduce(&csplit,&cstart,1,MPI_INT,MPI_MIN,comm) ; + +#ifdef DEBUG + debugout << "pstart=" << pstart << endl ; + debugout << "cstart=" << cstart << endl ; +#endif + // Convert parent and child numbering to current global numbering + for(size_t i=0;i > splits(p-1) ; + for(int i=0;i(psplits[i+1],std::numeric_limits::lowest()) ; + + sort(p2c.begin(),p2c.end()) ; + Loci::parSplitSort(p2c,splits,comm) ; + + //######################################################################## + // + // Find when a parent gives to multiple children, these will be parent + // cells that have been refined and thus we need to interpolate + // + int refinecnt = 0 ; + sz = p2c.size() ; + FATAL(sz == 0) ; + for(int i=0;i 1) + refinecnt += nsame ; + i += nsame ; + } +#ifdef DEBUG + // report total number of refined cells in interpolation + int refinecntglobal = refinecnt ; + MPI_Allreduce(&refinecnt, &refinecntglobal,1,MPI_INT,MPI_SUM, + comm) ; + debugout<< "refine cnt = " << refinecntglobal << endl ; +#endif + // Remove the refined cells from the parent to cell map and then make a + // child-to-parent map distributed in child + // c2p is the cell to parent map for cells that were not refined. + vector > c2p(sz-refinecnt) ; + // separate parent to cell map only for refined cells + vector > p2crefine(refinecnt) ; + + int cnt1 = 0 ; + int cnt2 = 0 ; + for(int i=0;i 1) { // If refinement group copy to p2crefine + for(int j=0;j csplits(p) ; + MPI_Allgather(&csplit,1,MPI_INT,&csplits[0],1,MPI_INT,comm) ; + for(int i=0;i(csplits[i+1],std::numeric_limits::lowest()) ; + sort(c2p.begin(),c2p.end()) ; + + Loci::parSplitSort(c2p,splits,comm) ; + + + //######################################################################## + // + // Figure out child cells that have multiple parents. These are derefined + // cells that need to be averaged during the interpolation step + // +#ifdef DEBUG + int derefinecnt = 0 ; +#endif + sz = c2p.size() ; + // WARN(sz == 0) ; + + for(int i=0;i 1) + derefinecnt += nsame ; +#endif + i += nsame ; + } +#ifdef DEBUG + int derefinecntglobal = derefinecnt ; + MPI_Allreduce(&derefinecnt, &derefinecntglobal,1,MPI_INT,MPI_SUM, + comm) ; + debugout<< "derefine cnt = " << derefinecntglobal << endl ; + + // Now generate communication schedules for refined cells + debugout << "refinesize=" << p2crefine.size() << endl ; +#endif + + + //######################################################################## + // + // Build data structures for refined cells to compute gradients for + // interpolation. + // + entitySet refinedParents ; + vector refinecells(p2crefine.size()) ; + for(size_t i=0;i0) ; + // Setup the rule database. + // Add all registered rules. + rule_db refine_rules; + refine_rules = refmesh_rdb ; + + // Setup the refine_fact database. + fact_db refine_facts; + + // Read in the mesh file. + Loci::storeRepP cellwt = Loci::DataXFER_DB.getItem("cellweights") ; + if(!Loci::setupFVMGridWithWeightInStore(refine_facts,meshFile, cellwt)) { + std::cerr << "unable to read grid file '" << meshFile << "'" << std::endl ; + Loci::Abort() ; + } + Loci::DataXFER_DB.deleteItem("cellweights") ; + + + blackbox > > origVolTags ; + origVolTags.Rep()->allocate(~EMPTY) ; + if(MPI_rank==0 || Loci::use_parallel_io){ + hid_t file_id = Loci::hdf5OpenFile(meshFile.c_str(), + H5F_ACC_RDONLY,H5P_DEFAULT, refine_facts.get_comm()) ; + + Loci::readVolTags(file_id,*origVolTags); + Loci::hdf5CloseFile(file_id, refine_facts.get_comm()) ; + } + refine_facts.create_fact("volTagInput",origVolTags) ; + + //Setup Loci datastructures + Loci::createLowerUpper(refine_facts) ; + Loci::createEdgesPar(refine_facts) ; + Loci::parallelClassifyCell(refine_facts); + + + //this is a dummy parameter to trick Loci scheduler + param beginWithMarker; + *beginWithMarker = true; + refine_facts.create_fact("beginWithMarker",beginWithMarker) ; + + param plan_outDB_par ; + *plan_outDB_par = "nextPlan" ; + refine_facts.create_fact("plan_outDB_par",plan_outDB_par) ; + + if(tags != 0) { + store tag_info ; + tag_info = tags ; + Loci::entitySet dom = tag_info.domain() ; + vector tagset(dom.size()) ; + int cnt = 0 ; + FORALL(dom,ii) { + tagset[cnt] = tag_info[ii] ; + cnt++ ; + } ENDFORALL ; + blackbox > inputTagsData ; + inputTagsData.set_entitySet(~EMPTY) ; + (*inputTagsData).swap(tagset) ; + refine_facts.create_fact("inputCellTagsData",inputTagsData) ; + Loci::DataXFER_DB.deleteItem("refineTag") ; + } + + + if(firstlevel){ + param parent_planfile_par ; + *parent_planfile_par = "currentPlan" ; + refine_facts.create_fact("parent_planDB_par",parent_planfile_par) ; + } + + gridDataP->boundary_ids.clear(); + Loci::readBCfromVOG(meshFile, gridDataP->boundary_ids, Loci::exec_current_fact_db->get_comm()); + + + + //parameters to identify different options + if(firstlevel){ + param planfile_par ; + *planfile_par = "currentPlan" ; + refine_facts.create_fact("planDB_par",planfile_par) ; + + param restart_tag_par; + *restart_tag_par = 1; + refine_facts.create_fact("restart_tag_par",restart_tag_par); + }else{ + param norestart_tag_par; + *norestart_tag_par = 1; + refine_facts.create_fact("norestart_tag_par",norestart_tag_par); + } + + param split_mode_par; + *split_mode_par = adaptmode; + refine_facts.create_fact("split_mode_par", split_mode_par); + + { + param cellweight_outDB_par; + *cellweight_outDB_par = "cellweights"; + refine_facts.create_fact("cellweight_outDB_par",cellweight_outDB_par); + + if(!Loci::makeQuery(refine_rules,refine_facts, + "cellplan_output,cellweight_output,cell2parent_DB,inner_nodes_cell,inner_nodes_face,inner_nodes_edge,fine_faces_cell,fine_faces,volTag_blackbox")) { + std::cerr << "adapt query failed!" << std::endl; + Loci::Abort(); + } + } + + Loci::DataXFER_DB.deleteItem("currentPlan") ; + Loci::storeRepP newPlan = Loci::DataXFER_DB.getItem("nextPlan") ; + Loci::DataXFER_DB.insertItem("currentPlan",newPlan) ; + Loci::DataXFER_DB.deleteItem("nextPlan") ; + + store inner_nodes_cell; + inner_nodes_cell = refine_facts.get_variable("inner_nodes_cell"); + store inner_nodes_face; + inner_nodes_face = refine_facts.get_variable("inner_nodes_face"); + store inner_nodes_edge; + inner_nodes_edge = refine_facts.get_variable("inner_nodes_edge"); + + + int num_nodes; + createVOGNode(gridDataP->new_pos, + inner_nodes_cell,inner_nodes_face,inner_nodes_edge, + num_nodes, + refine_facts, + gridDataP->local_nodes + ); + if(Loci::MPI_rank ==0) + cerr<< "num_nodes: " << num_nodes << " in adaptcycle" << endl; + + store fine_faces_cell; + fine_faces_cell = refine_facts.get_variable("fine_faces_cell"); + store fine_faces; + fine_faces = refine_facts.get_variable("fine_faces"); + + int num_faces = 0; + int num_cells = 0; + + createVOGFace( num_nodes, + fine_faces_cell, + fine_faces, + refine_facts, + num_faces, + num_cells, + gridDataP->new_cl, + gridDataP->new_cr, + gridDataP->new_face2node, + gridDataP->local_faces, + gridDataP->local_cells + ); + + //update volume tags + { + blackbox > > volTag_blackbox; + volTag_blackbox = refine_facts.get_variable("volTag_blackbox"); + gridDataP->volTags = *volTag_blackbox; + } + + if(Loci::MPI_rank==0){ + cerr<< "num_faces: " << num_faces << " in adaptcycle" < &gridDataP, + int &level, + rule_db &refmesh_rdb, + string casename, string weightfile, + string restartplanfile) { + string meshFile = casename+".vog"; + // Setup the rule database. + // Add all registered rules. + rule_db refine_rules; + refine_rules = refmesh_rdb ; + + // Setup the refine_fact database. + fact_db refine_facts; + MPI_Comm comm = refine_facts.get_comm() ; + + if(Loci::MPI_rank == 0) + cout <<"reading in meshfile: " + << meshFile << " for chem run" << std::endl; + + // Read in the mesh file. + if(!Loci::setupFVMGridWithWeightInFile(refine_facts,meshFile, weightfile)) { + std::cerr << "unable to read mesh file '" << meshFile << "'" << std::endl ; + Loci::Abort() ; + } + + //Setup Loci datastructures + Loci::createLowerUpper(refine_facts) ; + Loci::createEdgesPar(refine_facts) ; + Loci::parallelClassifyCell(refine_facts); + blackbox > > origVolTags ; + origVolTags.Rep()->allocate(~EMPTY) ; + //if(MPI_rank==0){ + if(MPI_rank==0 || Loci::use_parallel_io){ + hid_t file_id = Loci::hdf5OpenFile(meshFile.c_str(), + H5F_ACC_RDONLY,H5P_DEFAULT, comm) ; + + Loci::readVolTags(file_id,*origVolTags); + Loci::hdf5CloseFile(file_id, comm) ; + } + refine_facts.create_fact("volTagInput",origVolTags) ; + + if(Loci::MPI_rank == 0) + cout <<"reading in cellplan file: " + << restartplanfile << std::endl; + + hid_t file_id = Loci::hdf5OpenFile(restartplanfile.c_str(), + H5F_ACC_RDONLY,H5P_DEFAULT, comm) ; + + store > cellPlan ; + param refineLevel ; + hid_t group_id = 0 ; + if(Loci::MPI_rank == 0) + group_id = H5Gopen(file_id,"refineLevel",H5P_DEFAULT) ; + + Loci::read_parameter(group_id,refineLevel.Rep(),comm) ; + if(Loci::MPI_rank == 0) { + H5Gclose(group_id) ; + group_id = H5Gopen(file_id,"cellPlan",H5P_DEFAULT) ; + cout << "restart refine level = " << *refineLevel << endl ; + } + level = *refineLevel ; + int offset = 0; + Loci::read_store(group_id,cellPlan.Rep(),offset,comm) ;//read in file numbering store, simplePartition + cellPlan.Rep()->shift(offset) ; + if(Loci::MPI_rank == 0) { + H5Gclose(group_id) ; + } + Loci::hdf5CloseFile(file_id, comm) ; + + //finish reading currentPlan + //inserting currentPlan into DataXFER_DB + Loci::DataXFER_DB.insertItem("currentPlan",cellPlan.Rep()) ; + param balanced_planDB_par ; + *balanced_planDB_par = "currentPlan" ; + refine_facts.create_fact("balanced_planDB_par",balanced_planDB_par) ; + + if(!Loci::makeQuery(refine_rules,refine_facts, + "inner_nodes_cell,inner_nodes_face,inner_nodes_edge,fine_faces,fine_faces_cell,volTag_blackbox")) { + std::cerr << "adapt query failed!" << std::endl; + Loci::Abort(); + } + + store inner_nodes_cell; + inner_nodes_cell = refine_facts.get_variable("inner_nodes_cell"); + store inner_nodes_face; + inner_nodes_face = refine_facts.get_variable("inner_nodes_face"); + store inner_nodes_edge; + inner_nodes_edge = refine_facts.get_variable("inner_nodes_edge"); + + + gridDataP = new(refinedGridData) ; + gridDataP->boundary_ids.clear(); + Loci::readBCfromVOG(meshFile, gridDataP->boundary_ids, Loci::exec_current_fact_db->get_comm()); + + int num_nodes; + createVOGNode(gridDataP->new_pos, + inner_nodes_cell,inner_nodes_face,inner_nodes_edge, + num_nodes, + refine_facts, + gridDataP->local_nodes + ); + if(Loci::MPI_rank ==0)cerr<< "num_nodes: " << num_nodes << " before chem run" << endl; + + store fine_faces_cell; + fine_faces_cell = refine_facts.get_variable("fine_faces_cell"); + store fine_faces; + fine_faces = refine_facts.get_variable("fine_faces"); + + int num_faces = 0; + int num_cells = 0; + + createVOGFace( num_nodes, + fine_faces_cell, + fine_faces, + refine_facts, + num_faces, + num_cells, + gridDataP->new_cl, + gridDataP->new_cr, + gridDataP->new_face2node, + gridDataP->local_faces, + gridDataP->local_cells + ); + + if(Loci::MPI_rank ==0)cerr<< "num_faces: " << num_faces << " before chem run" << endl; + if(Loci::MPI_rank ==0)cerr<< "num_cells: " << num_cells << " before chem run" << endl; + + { + blackbox > > volTag_blackbox; + volTag_blackbox = refine_facts.get_variable("volTag_blackbox"); + gridDataP->volTags = *volTag_blackbox; + } + } + + extern bool load_cell_weights ; + extern string cell_weight_file ; + extern storeRepP cell_weight_store ; + bool redistribute_cell_weight(storeRepP old_store, storeRepP new_store); + + void fill_clone_proc( map &mapdata, entitySet &out_of_dom, std::vector &init_ptn) ; + void redistribute_container(const vector &ptn, + const vector &ptn_t, + entitySet new_alloc, + storeRepP inRep,storeRepP outRep) ; + extern void ORBPartition(const vector > &pnts, + vector &procid, + MPI_Comm comm) ; + void assignOwner(vector > > &scratchPad, + vector ptn, + vector &out_ptn) ; + void SFC_Partition_Mesh(const vector &local_nodes, + const vector &local_faces, + const vector &local_cells, + const store > &pos, + const Map &cl, const Map &cr, + const multiMap &face2node, + const store &boundary_tags, + const store &cell_weights, + vector &cell_ptn, + vector &face_ptn, + vector &node_ptn, + MPI_Comm comm = MPI_COMM_WORLD) ; + void RND_Partition_Mesh(const vector &local_nodes, + const vector &local_faces, + const vector &local_cells, + vector &cell_ptn, + vector &face_ptn, + vector &node_ptn, + MPI_Comm comm = MPI_COMM_WORLD) ; + Loci::storeRepP getCellPartitionWeights(entitySet localcelldom) { + MPI_Comm comm = exec_current_fact_db->get_comm() ; + Loci::storeRepP wptr = Loci::DataXFER_DB.getItem("cellPartitionWeights") ; + if(wptr == 0) + return wptr ; + store cell_weights_parent ; + cell_weights_parent = wptr ; + store > c2pset ; + Loci::storeRepP ptr = Loci::DataXFER_DB.getItem("c2p") ; + if(ptr == 0) + return ptr ; + c2pset = ptr ; + entitySet domc2p = c2pset.domain() ; + int r = Loci::MPI_rank ; + int p = Loci::MPI_processes ; + + int minParent = std::numeric_limits::max() ; + int minTarget = std::numeric_limits::max() ; + FORALL(domc2p,ii) { + minParent = min(minParent,c2pset[ii].second) ; + minTarget = min(minTarget,c2pset[ii].first) ; + } ENDFORALL ; + + int parentOffset = minParent ; + int targetOffset = minTarget ; + MPI_Allreduce(&minParent,&parentOffset,1,MPI_INT,MPI_MIN, + comm) ; + MPI_Allreduce(&minTarget,&targetOffset,1,MPI_INT,MPI_MIN, + comm) ; + int sz = domc2p.size() ; + vector > p2c(sz) ; + int cnt = 0 ; + FORALL(domc2p,ii) { + p2c[cnt].first = c2pset[ii].second-parentOffset ; + p2c[cnt].second = c2pset[ii].first-targetOffset ; + cnt++ ;; + } ENDFORALL ; + + sort(p2c.begin(),p2c.end()) ; + + // Now distribute c2p to processors in parent file ordering + entitySet cwdom = cell_weights_parent.domain() ; + int cwsz = cwdom.size() ; + vector parentsizes(p) ; + MPI_Allgather(&cwsz,1,MPI_INT,&parentsizes[0],1,MPI_INT, + comm) ; + vector parentoffsets(p+1,0) ; + for(int i=0;i > splits(p-1) ; + for(int i=0;i(parentoffsets[i+1],-1) ; + + Loci::parSplitSort(p2c,splits,comm) ; + + int psz = p2c.size() ; + + vector > cell2weights(psz) ; + entitySet::const_iterator ii = cwdom.begin() ; + cnt = parentoffsets[r] ; + int mincell = p2c[0].second ; + int cnt2 = 0 ; + for(int i=0;i(p2c[i].second, + cell_weights_parent[*ii]) ; + cnt2++ ; + i++ ; + } + cnt++ ; + ii++ ; + } + int mincell_global =mincell ; + MPI_Allreduce(&mincell,&mincell_global,1,MPI_INT,MPI_MIN,comm) ; + + int dmsz = localcelldom.size() ; + vector cellsizes(p) ; + MPI_Allgather(&dmsz,1,MPI_INT,&cellsizes[0],1,MPI_INT, + comm) ; + parentoffsets[0] = mincell_global ; + for(int i=0;i(parentoffsets[i+1],-1) ; + + + sort(cell2weights.begin(),cell2weights.end()) ; + Loci::parSplitSort(cell2weights,splits,comm) ; + + store cellweightschild ; + cellweightschild.allocate(localcelldom) ; + cnt = 0; + + FORALL(localcelldom,ii) { + cellweightschild[ii] = cell2weights[cnt].second ; + cnt++ ; + } ENDFORALL ; + return cellweightschild.Rep() ; + } + + + +#ifdef LOCI_USE_METIS + vector AdaptMetisPartitionOfCells(const vector &local_cells, + const Map &cl, const Map &cr, + const store &boundary_tags, + storeRepP cellwts) { + entitySet dom = cl.domain() & cr.domain() ; + entitySet refset = boundary_tags.domain() ; + entitySet bcfaces = cr.preimage(refset).first ; + dom -= bcfaces ; + int cnt = dom.size() ; + + vector > rawMap(cnt*2) ; + int j = 0 ; + entitySet::const_iterator ei ; + for(ei=dom.begin();ei!=dom.end();++ei) { + rawMap[j++] = pair(cl[*ei],cr[*ei]) ; + rawMap[j++] = pair(cr[*ei],cl[*ei]) ; + } + + sort(rawMap.begin(),rawMap.end()) ; + + multiMap cell2cell ; + entitySet all_cells ; + for(int i=0;i >().swap(rawMap) ; // Free up memory from rawMap + int count = 0 ; + idx_t size_map = local_cells[Loci::MPI_rank].size() ; + vector size_adj(size_map) ; + count = 0 ; + for(entitySet::const_iterator ei = local_cells[Loci::MPI_rank].begin(); ei != local_cells[Loci::MPI_rank].end(); ++ei) { + size_adj[count] = cell2cell[*ei].size() ; + ++count ; + } + + vector part(size_map) ; + vector xadj(size_map+1) ; + idx_t edgecut ; + vector vdist(Loci::MPI_processes + 1) ; + int cmin = local_cells[0].Min(); + for(int i = 0; i < Loci::MPI_processes; i++) { + cmin = min(local_cells[i].Min(), cmin); + } + + // check local_cells to be consistent with parmetis partitioning + for(int i=0;i adjncy(tot) ; + count = 0 ; + for(entitySet::const_iterator ei = local_cells[Loci::MPI_rank].begin(); ei != local_cells[Loci::MPI_rank].end(); ++ei) { + size_t sz = cell2cell[*ei].size() ; + for(size_t i = 0; i != sz; ++i) { + adjncy[count] = cell2cell[*ei][i] - cmin ; + count ++ ; + } + } + cell2cell.setRep(multiMap().Rep()) ;// Free up memory from multiMap + + vdist[0] = 0 ; + for(int i = 1; i <= Loci::MPI_processes; ++i) + vdist[i] = vdist[i-1] + local_cells[i-1].size() ; + idx_t top = vdist[Loci::MPI_processes] ; + + bool trouble = false ; + for(idx_t i=0;i= top) + trouble = true ; + if(trouble) + cerr << "adjacency list contains out of bounds reference" << endl ; + + MPI_Comm mc = exec_current_fact_db->get_comm() ; + idx_t nparts = Loci::MPI_processes ; // number of partitions + idx_t wgtflag = 0 ; + idx_t numflag = 0 ; + idx_t options = 0 ; + + // Loci::storeRepP ptr = getCellPartitionWeights(local_cells[MPI_rank]) ; + Loci::storeRepP ptr = cellwts ; + + if(ptr != 0) { + store cell_weights ; + cell_weights = ptr ; + + // compute necessary ParMETIS data-structure + wgtflag = 2 ; // weights on the vertices only + idx_t ncon = 2 ; // number of weights per vertex + idx_t tpwgts_len = ncon*nparts ; + vector tpwgts(tpwgts_len) ; + + for(idx_t i=0;i ubvec(ncon) ; + for(idx_t i=0;i vwgt(ncon*size_map) ; + int cnt = 0 ; + for(entitySet::const_iterator + ei=local_cells[Loci::MPI_rank].begin(); + ei!=local_cells[Loci::MPI_rank].end();++ei,cnt+=ncon) { + // first weight for cell is 1 (the cell computation) + vwgt[cnt] = 1 ; + // the second weight is from the store cell_weights[*ei] + vwgt[cnt+1] = cell_weights[*ei] ; + } + + // now call the ParMETIS routine (V3) + ParMETIS_V3_PartKway(&vdist[0],&xadj[0],&adjncy[0],&vwgt[0],NULL, + &wgtflag,&numflag,&ncon,&nparts, + &tpwgts[0],&ubvec[0],&options,&edgecut,&part[0], + &mc) ; + + } else { + // idx_t ncon = 1 ; + idx_t ncon = 2 ; + idx_t tpwgts_len = ncon*nparts ; + vector tpwgts(tpwgts_len) ; + for(idx_t i=0;i ubvec(ncon) ; + for(idx_t i=0;i vwgt(ncon*size_map) ; + + int cnt = 0 ; + for(int i=0;i ptn ; + + for(int i = 0; i < Loci::MPI_processes; ++i) + ptn.push_back(EMPTY) ; + cmin = local_cells[Loci::MPI_rank].Min() ; + for(int i=0;i &local_nodes, + const vector &local_faces, + const vector &local_cells, + const store > &pos, + const Map &cl, const Map &cr, + const multiMap &face2node, + const store &boundary_tags, + vector &cell_ptn, + vector &face_ptn, + vector &node_ptn, + MPI_Comm comm = MPI_COMM_WORLD) ; + vector partitionFaces(vector cell_ptn, const Map &cl, + const Map &cr, + const store &boundary_tags, + MPI_Comm comm = MPI_COMM_WORLD) ; + + //Input: Mapping from faces to its right cells. + //Output: Entities of boundary cells. + entitySet getBoundaryCells(const MapRepP tmp_cr_rep) { + entitySet cri = tmp_cr_rep->image(tmp_cr_rep->domain()) ; + return(cri & interval(Loci::UNIVERSE_MIN,-1)) ; + } + + void copyGridStructures( entitySet nodes, entitySet faces, entitySet cells, + const store > &t_pos, + const Map &tmp_cl, const Map &tmp_cr, + const multiMap &tmp_face2node, + store > &pos, Map &cl, Map &cr, + multiMap &face2node) { + + entitySet boundary_cells = getBoundaryCells(Loci::MapRepP(tmp_cr.Rep())); + + dMap identity_map; + FORALL(nodes, ei) { + identity_map[ei] = ei; + } ENDFORALL ; + FORALL(faces, ei) { + identity_map[ei] = ei ; + } ENDFORALL ; + FORALL(cells, ei) { + identity_map[ei] = ei ; + } ENDFORALL ; + FORALL(boundary_cells, ei) { + identity_map[ei] = ei ; + } ENDFORALL ; + + pos = t_pos.Rep()->remap(identity_map); + cl = tmp_cl.Rep()->remap(identity_map); + cr = tmp_cr.Rep()->remap(identity_map); + face2node = MapRepP(tmp_face2node.Rep())->get_map(); + + } + + + vector transposePtn(const vector &ptn, MPI_Comm comm = MPI_COMM_WORLD); + vector partitionNodes(vector face_ptn, MapRepP face2node,entitySet old_node_dom, MPI_Comm comm = MPI_COMM_WORLD); + inline bool fieldSort(const std::pair &p1, + const std::pair &p2) { + return p1.first < p2.first ; + } + + + void create_face_info(fact_db &facts); + void create_ref(fact_db &facts) ; + void create_ghost_cells(fact_db &facts); + + std::vector simplePartitionVec(int mn, int mx, int p); + vector simplePartition(int mn, int mx, MPI_Comm comm); + vector transposeSeq(const vector sv, MPI_Comm comm = MPI_COMM_WORLD); +} +void colorMatrix(Map &cl, Map &cr, multiMap &face2node) ; + +namespace Loci{ + + // Convert container from global numbering to file numbering + // pass in store rep pointer: sp + // entitySet to write: dom + // return offset in file numbering (each processor will allocate from zero, + // add offset to domain to get actual file numbering) + // distribution info pointer (dist) + // MPI Communicator(comm) + storeRepP Global2FileOrder(storeRepP sp, entitySet dom, int &offset, + fact_db::distribute_infoP dist, MPI_Comm comm) { + + // Now get global to file numbering + dMap g2f ; + g2f = dist->g2fv[0].Rep() ; // FIX THIS + + // Compute map from local numbering to file numbering + Map newnum ; + newnum.allocate(dom) ; + FORALL(dom,i) { + newnum[i] = g2f[i] ; + } ENDFORALL ; + + int imx = std::numeric_limits::min() ; + int imn = std::numeric_limits::max() ; + + // Find bounds in file numbering from this processor + FORALL(dom,i) { + imx = max(newnum[i],imx) ; + imn = min(newnum[i],imn) ; + } ENDFORALL ; + + // Find overall bounds + imx = GLOBAL_MAX(imx, comm) ; + imn = GLOBAL_MIN(imn, comm) ; + + // Get number of processors + int p = 0 ; + MPI_Comm_size(comm,&p) ; + int prank = 0 ; + MPI_Comm_rank(comm,&prank) ; + // Get partitioning of file numbers across processors + vector out_ptn = simplePartition(imn,imx,comm) ; + + // Now compute where to send data to put in file ordering + vector send_sets(p) ; + vector send_seqs(p) ; + + // Loop over processors and compute sets of entities to send + // To efficiently compute this mapping, first sort the transpose + // of the newnum map to quickly find the set of entities to send + // without searching entire newnum map for each processor + vector > file2num(dom.size()) ; + size_t cnt = 0 ; + FORALL(dom,ii) { + file2num[cnt].first = newnum[ii] ; + file2num[cnt].second = ii ; + cnt++ ; + } ENDFORALL ; + sort(file2num.begin(),file2num.end()) ; + + // Check each processor, find out which sets to send + cnt = 0 ; + for(int i=0;i recv_seqs = transposeSeq(send_seqs) ; + + + // shift by the offset + offset = out_ptn[prank].Min() ; + for(int i=0;inew_store(file_dom) ; + + // Now communicate the container + vector send_sizes(p),recv_sizes(p) ; + + for(int i=0;ipack_size(send_sets[i]) ; + + MPI_Alltoall(&send_sizes[0],1,MPI_INT, + &recv_sizes[0],1,MPI_INT, + comm) ; + + vector send_dspl(p),recv_dspl(p) ; + send_dspl[0] = 0 ; + recv_dspl[0] = 0 ; + for(int i=1;i send_store(send_sz) ; + vector recv_store(recv_sz) ; + + + for(int i=0;ipack(&send_store[send_dspl[i]],loc_pack, send_sizes[i], + send_sets[i]) ; + } + + MPI_Alltoallv(&send_store[0], &send_sizes[0], &send_dspl[0], MPI_PACKED, + &recv_store[0], &recv_sizes[0], &recv_dspl[0], MPI_PACKED, + comm) ; + + for(int i=0;iunpack(&recv_store[recv_dspl[i]],loc_pack,recv_sizes[i], + recv_seqs[i]) ; + } + return qcol_rep ; + } + + extern bool useDomainKeySpaces ; + extern void remapGrid(vector &node_ptn, + vector &face_ptn, + vector &cell_ptn, + vector &node_ptn_t, + vector &face_ptn_t, + vector &cell_ptn_t, + store > &t_pos, Map &tmp_cl, + Map &tmp_cr, multiMap &tmp_face2node, + vector &bcsurf_ptn, + store &tmp_boundary_names, + store &tmp_boundary_tags, + entitySet nodes, entitySet faces, entitySet cells, + store > &pos, Map &cl, Map &cr, + multiMap &face2node, + store &boundary_names, + store &boundary_tags, + entitySet bcsurfset, + fact_db &facts) ; + + bool inputFVMGrid(fact_db &facts, + vector& local_nodes, + vector& local_faces, + vector& local_cells, + store >& t_pos, + Map& tmp_cl, + Map& tmp_cr, + multiMap& tmp_face2node, + vector >& boundary_ids, + vector >& volTags, + storeRepP cellwts) { + MPI_Comm comm = facts.get_comm() ; + double t1 = MPI_Wtime() ; + // Identify boundary tags + if(Loci::MPI_processes == 1) { + entitySet local_boundary_cells = getBoundaryCells(MapRepP(tmp_cr.Rep())); + entitySet global_boundary_cells = all_collect_entitySet(local_boundary_cells,comm) ; + + int npnts = local_nodes[0].size(); + int nfaces = local_faces[0].size(); + int ncells = local_cells[0].size(); + + entitySet nodes = facts.get_distributed_alloc(npnts,0).first ; // FIX THIS + entitySet faces = facts.get_distributed_alloc(nfaces,0).first ; + entitySet cells = facts.get_distributed_alloc(ncells,0).first; + + store > pos ; + Map cl ; + Map cr ; + multiMap face2node ; + Loci::copyGridStructures(nodes, faces, cells, + t_pos, tmp_cl, tmp_cr, tmp_face2node, + pos, cl, cr, face2node); + store boundary_names ; + store boundary_tags ; + boundary_names.allocate(global_boundary_cells) ; + boundary_tags.allocate(global_boundary_cells) ; + cout << " boundaries identified as:" ; + + FORALL(global_boundary_cells, bc) { + char buf[512] ; + bzero(buf,512) ; + snprintf(buf,511,"BC_%d",-bc) ; + boundary_tags[bc] = string(buf) ; + boundary_names[bc] = string(buf) ; + cout << " " << boundary_names[bc] ; + } ENDFORALL ; + + for(size_t i=0;i Tag ; + *Tag = volTags[i].first ; + Tag.set_entitySet(volTags[i].second >> cells_base) ; + std::ostringstream oss ; + oss << "volumeTag(" << volTags[i].first << ")" ; + facts.create_fact(oss.str(),Tag) ; + } + + return true ; + } + + entitySet local_boundary_cells = getBoundaryCells(MapRepP(tmp_cr.Rep())); + entitySet global_boundary_cells = all_collect_entitySet(local_boundary_cells,comm) ; + int maxc = 0 ; + entitySet dom = tmp_cr.domain()&tmp_cl.domain() ; + FORALL(dom,ii) { + maxc = max(maxc,max(tmp_cr[ii],tmp_cl[ii])) ; + } ENDFORALL ; + int maxctot = maxc ; + MPI_Allreduce(&maxc,&maxctot,1,MPI_INT,MPI_MAX,comm) ; + entitySet boundary_taglist = global_boundary_cells ; + int ref_base = maxctot+1 ; + entitySet refSet = interval(ref_base,ref_base+(boundary_taglist.size()-1)) ; + + store boundary_info ; + boundary_info.allocate(refSet) ; + map boundary_remap ; + int ct = 0 ; + FORALL(boundary_taglist,ii) { + boundary_remap[ii]=ct+ref_base ; + boundary_info[ct+ref_base]=ii ; + ct++ ; + } ENDFORALL ; + + entitySet domcr = tmp_cr.domain() ; + FORALL(domcr,fc) { + if(tmp_cr[fc] < 0) + tmp_cr[fc] = boundary_remap[tmp_cr[fc]] ; + } ENDFORALL ; + + store tmp_boundary_tags,tmp_boundary_names ; + tmp_boundary_names.allocate(refSet) ; + tmp_boundary_tags.allocate(refSet) ; + + map bcmap ; + FORALL(refSet, ii) { + int bc = boundary_info[ii] ; + char buf[128] ; + bzero(buf,128) ; + snprintf(buf,127,"BC_%d",-bc) ; + bcmap[-bc] = ii ; + tmp_boundary_tags[ii] = string(buf) ; + tmp_boundary_names[ii] = string(buf) ; + } ENDFORALL ; + + for(size_t i=0;i::const_iterator mi = bcmap.find(id) ; + if(mi != bcmap.end()) + tmp_boundary_names[mi->second] = boundary_ids[i].second ; + else + debugout << "id " << id << " for boundary surface " << boundary_ids[i].second + << " not found!" << endl ; + } + + + if(Loci::MPI_rank == 0) { + Loci::debugout << " boundaries identified as:" ; + FORALL(refSet, bc) { + debugout << " " << tmp_boundary_names[bc] ; + } ENDFORALL ; + Loci::debugout << endl ; + } + + REPORTMEM() ; + + vector cell_ptn,face_ptn,node_ptn ; + + store cell_weights ; + if(cellwts != 0) + cell_weights = cellwts ; + else { + Loci::storeRepP ptr = getCellPartitionWeights(local_cells[MPI_rank]) ; + if(ptr != 0) { + cell_weights = ptr ; + } + } + + partitionerSelector partitioner_type = partitionerMethod ; + + if(partitioner_type == GRAPH) { + int lcpp = local_cells[MPI_rank].size() ; + int mincpp = lcpp ; + MPI_Allreduce(&lcpp,&mincpp,1,MPI_INT,MPI_MIN,comm) ; + if(mincpp < metis_cpp_threshold) { + partitioner_type = SFC ; // Space filling curve partitioner + debugout << "switching from metis to space filling curve partitioner" + << " mincpp = " << mincpp << endl ; + } +#ifndef LOCI_USE_METIS + partitioner_type = SFC ; // Space filling curve partitioner if METIS + // not available +#endif + } + switch(partitioner_type) { + case ORB: + ORB_Partition_Mesh(local_nodes, local_faces, local_cells, + t_pos, tmp_cl, tmp_cr, tmp_face2node, + tmp_boundary_tags, + cell_ptn,face_ptn,node_ptn) ; + break ; + case SFC: + SFC_Partition_Mesh(local_nodes, local_faces, local_cells, + t_pos, tmp_cl, tmp_cr, tmp_face2node, + tmp_boundary_tags, + cell_weights, + cell_ptn,face_ptn,node_ptn) ; + break ; + case SIMPLE: // Simple partition + { + cell_ptn = vector(MPI_processes) ; + cell_ptn[MPI_rank] = local_cells[MPI_rank] ; + REPORTMEM() ; + face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags) ; + REPORTMEM() ; + + node_ptn = partitionNodes(face_ptn, + MapRepP(tmp_face2node.Rep()), + t_pos.domain()) ; + + } + break ; +#ifdef LOCI_USE_METIS + case GRAPH: // METIS partition + { + cell_ptn = AdaptMetisPartitionOfCells(local_cells,tmp_cl,tmp_cr,tmp_boundary_tags,cellwts) ; + // cell_ptn = newMetisPartitionOfCells(local_cells,tmp_cl,tmp_cr,tmp_boundary_tags,cellwts) ; + REPORTMEM() ; + face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags) ; + REPORTMEM() ; + + node_ptn = partitionNodes(face_ptn, + MapRepP(tmp_face2node.Rep()), + t_pos.domain()) ; + } + break ; +#endif + case RANDOM: + { + RND_Partition_Mesh(local_nodes, local_faces, local_cells, + cell_ptn,face_ptn,node_ptn) ; + } + break ; + default: // SFC partition no weight balance + { + store cell_weights ; + + SFC_Partition_Mesh(local_nodes, local_faces, local_cells, + t_pos, tmp_cl, tmp_cr, tmp_face2node, + tmp_boundary_tags, + cell_weights, + cell_ptn,face_ptn,node_ptn) ; + } + break ; + } + + vector bcsurf_ptn(MPI_processes) ; + entitySet refset = tmp_boundary_tags.domain() ; + + // round robin allocate boundary faces + // NOTE!!!! This could be improved. + int cnt = 0 ; + // int refsetsz = refset.size() ; + FORALL(refset,ii) { + if(cnt == MPI_rank) + bcsurf_ptn[cnt] += ii ; + cnt++ ; + if(cnt == MPI_processes) + cnt = 0 ; + } ENDFORALL ; + REPORTMEM() ; + + vector cell_ptn_t = transposePtn(cell_ptn) ; + vector face_ptn_t = transposePtn(face_ptn) ; + vector node_ptn_t = transposePtn(node_ptn) ; + + int newnodes = 0 ; + for(int p=0;p node_alloc(newnodes) ; + int i=0; + for(int p=0;p face_alloc(newfaces) ; + i = 0 ; + for(int p=0;p cell_alloc(newcells) ; + i = 0 ; + for(int p=0;pg2fv[0].Rep() ; // FIX THIS + + int cells_base=local_cells[0].Min() ; + for(size_t i=0;i Tag ; + *Tag = volTags[i].first ; + + // Map entitySet to new ordering + entitySet inputTag = (volTags[i].second >> cells_base) ; + // Map entitySet to new ordering + entitySet tagset ; + FORALL(cells,cc) { + if(inputTag.inSet(g2f[cc])) + tagset += cc ; + } ENDFORALL ; + Tag.set_entitySet(tagset) ; + ostringstream oss ; + oss << "volumeTag(" << volTags[i].first << ")" ; + facts.create_fact(oss.str(),Tag) ; + } + // if(collect_perf_data) + // perfAnalysis->stop_timer(read_file_timer); + double t2 = MPI_Wtime() ; + debugout << "Time to process and partition adapted mesh is " << t2-t1 + << endl ; + REPORTMEM() ; + return true ; + + } + + + bool setupFVMGridFromContainer(fact_db &facts, + vector& local_nodes, + vector& local_faces, + vector& local_cells, + store >& t_pos, + Map& tmp_cl, + Map& tmp_cr, + multiMap& tmp_face2node, + vector >& boundary_ids, + vector >& volTags, + storeRepP cellwts) { + + if(!inputFVMGrid(facts, + local_nodes, + local_faces, + local_cells, + t_pos, + tmp_cl, + tmp_cr, + tmp_face2node, + boundary_ids, + volTags,cellwts)) + return false ; + REPORTMEM() ; + + create_face_info(facts) ; + create_ref(facts) ; + create_ghost_cells(facts) ; + + return true ; + } +} + + +namespace Loci { + void createVOGNode(store &new_pos, + const store &inner_nodes, + int& num_nodes, + fact_db & facts, + vector& local_nodes) { + cerr << "AMR API has changed, this call is deprecated. Update your AMR API calls!" << endl ; + Loci::Abort() ; + } + void createVOGFace(int numNodes, + const store &fine_faces, + fact_db & facts, + int& numFaces, + int& ncells, + Map& cl, + Map& cr, + multiMap& face2node, + vector& local_faces, + vector& local_cells + ) { + cerr << "AMR API has changed, this call is deprecated. Update your AMR API calls!" << endl ; + Loci::Abort() ; + } + + //create a new store new_pos from pos and inner_nodes + //re_number the nodes in pos and inner_nodes, + //and then redistribute them across the processes + void createVOGNode(store > &new_pos, + const store &inner_nodes_cell, + const store &inner_nodes_face, + const store &inner_nodes_edge, + + int& num_nodes, + fact_db & facts,//in global numbering + vector& nodes_ptn + ){ + MPI_Comm comm = facts.get_comm() ; + //get store pos + store > pos; + pos = facts.get_variable("pos"); + + if(MPI_processes == 1){ + //firsr write out numNodes + long num_original_nodes = pos.domain().size(); + long num_inner_nodes = 0; + FORALL(inner_nodes_cell.domain(), cc){ + num_inner_nodes += inner_nodes_cell[cc].size(); + }ENDFORALL; + FORALL(inner_nodes_face.domain(), cc){ + num_inner_nodes += inner_nodes_face[cc].size(); + }ENDFORALL; + FORALL(inner_nodes_edge.domain(), cc){ + num_inner_nodes += inner_nodes_edge[cc].size(); + }ENDFORALL; + + int node_base = 0; + long npnts = num_original_nodes + num_inner_nodes; + entitySet new_domain= interval(node_base, node_base+npnts-1); + new_pos.allocate(new_domain); + + entitySet::const_iterator nei = new_domain.begin(); + entitySet::const_iterator ei = pos.domain().begin(); + + for(long count = 0; count < num_original_nodes; count++, nei++, ei++){ + new_pos[*nei] = pos[*ei]; + } + + Loci::constraint faces, geom_cells; + faces = facts.get_variable("faces"); + geom_cells = facts.get_variable("geom_cells"); + entitySet local_edges = facts.get_variable("edge2node")->domain(); + + FORALL(local_edges, cc){ + for(unsigned int i = 0; i < inner_nodes_edge[cc].size(); i++, nei++){ + new_pos[*nei] = inner_nodes_edge[cc][i]; + } + }ENDFORALL; + FORALL(*geom_cells, cc){ + for(unsigned int i = 0; i < inner_nodes_cell[cc].size(); i++, nei++){ + new_pos[*nei] = inner_nodes_cell[cc][i]; + } + }ENDFORALL; + FORALL(*faces, cc){ + for(unsigned int i = 0; i < inner_nodes_face[cc].size(); i++, nei++){ + new_pos[*nei] = inner_nodes_face[cc][i]; + } + }ENDFORALL; + num_nodes = new_domain.size(); + nodes_ptn.resize(1); + nodes_ptn[0] = new_domain; + return; + } + + + + + store > pos_t; //temp container + vector temp_node_ptn; + entitySet my_temp_nodes; + { + //reorder store first, from global to io entities + fact_db::distribute_infoP dist = facts.get_distribute_info() ; + constraint my_faces, my_geom_cells, my_edges; + + my_faces = facts.get_variable("faces"); + my_geom_cells = facts.get_variable("geom_cells"); + my_edges = facts.get_variable("edges"); + + entitySet local_edges = *my_edges; + entitySet local_faces = *my_faces; + entitySet local_cells = *my_geom_cells; + entitySet local_nodes = pos.domain(); + + //before write out,create stores for pos and inner_nodes which + //are ordered across processors in the file numbering, the domain of this container + //shifted by offset is the actual file numbering. offset will be modified after function call + + int noffset, eoffset, coffset, foffset; + noffset = 0; + store > pos_io; + pos_io = Loci::Global2FileOrder(pos.Rep(), local_nodes, noffset, dist, comm) ; + entitySet file_nodes = pos_io.domain(); + + + eoffset = 0; + store edge_inner_nodes; + edge_inner_nodes = Loci::Global2FileOrder(inner_nodes_edge.Rep(),local_edges,eoffset,dist,comm) ; + entitySet file_edges = edge_inner_nodes.domain(); + + coffset= 0; + // Create container + store cell_inner_nodes; + cell_inner_nodes = Loci::Global2FileOrder(inner_nodes_cell.Rep(),local_cells,coffset,dist,comm) ; + entitySet file_cells = cell_inner_nodes.domain(); + + foffset= 0; + // Create container + store face_inner_nodes; + face_inner_nodes = Loci::Global2FileOrder(inner_nodes_face.Rep(),local_faces,foffset,dist,comm) ; + entitySet file_faces = face_inner_nodes.domain(); + + //Now allocate temp entitySet in file numbering + + + //compute the size of pos + int num_pos_nodes = file_nodes.size(); + std::vector pos_sizes(Loci::MPI_processes) ; + pos_sizes = Loci::all_collect_sizes(num_pos_nodes, comm); + + //compute the size of inner_nodes + int num_local_edge_nodes = 0; + FORALL(file_edges, cc){ + num_local_edge_nodes += edge_inner_nodes[cc].size(); + }ENDFORALL; + + int num_local_cell_nodes = 0; + FORALL(file_cells, cc){ + num_local_cell_nodes += cell_inner_nodes[cc].size(); + }ENDFORALL; + + int num_local_face_nodes = 0; + FORALL(file_faces, cc){ + num_local_face_nodes += face_inner_nodes[cc].size(); + }ENDFORALL; + std::vector inner_edge_nodes_sizes(Loci::MPI_processes) ; + inner_edge_nodes_sizes = Loci::all_collect_sizes(num_local_edge_nodes, comm); + + std::vector inner_cell_nodes_sizes(Loci::MPI_processes) ; + inner_cell_nodes_sizes = Loci::all_collect_sizes(num_local_cell_nodes, comm); + + std::vector inner_face_nodes_sizes(Loci::MPI_processes) ; + inner_face_nodes_sizes = Loci::all_collect_sizes(num_local_face_nodes, comm); + + //set up the offset + int my_id = Loci::MPI_rank; + int num_procs = Loci::MPI_processes; + noffset = 0; + for(int i = 0; i < my_id; i++){ + noffset += pos_sizes[i]; + } + + eoffset = 0; + for(int i = 0; i < num_procs; i++){ + eoffset += pos_sizes[i]; + } + for(int i = 0; i < my_id; i++){ + eoffset += inner_edge_nodes_sizes[i]; + } + + coffset = 0; + for(int i = 0; i < num_procs; i++){ + coffset += pos_sizes[i]+inner_edge_nodes_sizes[i]; + } + for(int i = 0; i < my_id; i++){ + coffset += inner_cell_nodes_sizes[i]; + } + + + foffset = 0; + for(int i = 0; i < num_procs; i++){ + foffset += pos_sizes[i]+inner_edge_nodes_sizes[i]+inner_cell_nodes_sizes[i]; + } + for(int i = 0; i < my_id; i++){ + foffset += inner_face_nodes_sizes[i]; + } + + + num_nodes = 0; + for(int i = 0; i < num_procs; i++){ + num_nodes += pos_sizes[i]+inner_edge_nodes_sizes[i]+ + inner_cell_nodes_sizes[i]+inner_face_nodes_sizes[i]; + } + + //create a new store pos_t in file numbering that contains all the nodes + entitySet my_original_nodes, my_inner_edge_nodes; + entitySet my_inner_cell_nodes, my_inner_face_nodes; + if(pos_sizes[my_id]>0) my_original_nodes = interval(noffset, noffset+pos_sizes[my_id]-1); + if(inner_edge_nodes_sizes[my_id]>0)my_inner_edge_nodes = interval(eoffset, eoffset+inner_edge_nodes_sizes[my_id]-1); + if(inner_cell_nodes_sizes[my_id]>0)my_inner_cell_nodes = interval(coffset, coffset+inner_cell_nodes_sizes[my_id]-1); + if(inner_face_nodes_sizes[my_id]>0)my_inner_face_nodes = interval(foffset, foffset+inner_face_nodes_sizes[my_id]-1); + my_temp_nodes = my_original_nodes + my_inner_edge_nodes + + my_inner_cell_nodes + my_inner_face_nodes; + pos_t.allocate(my_temp_nodes); + + //fill container pos_t will pos_io + entitySet::const_iterator ti = my_temp_nodes.begin(); + FORALL(file_nodes, ei){ + pos_t[*ti] = pos_io[ei]; + ti++; + }ENDFORALL; + //fill container pos_t will edge nodes + FORALL(file_edges, ei){ + for(unsigned int i = 0; i < edge_inner_nodes[ei].size(); i++){ + pos_t[*ti] = edge_inner_nodes[ei][i]; + ti++; + } + + }ENDFORALL; + //fill container pos_t will cell nodes + FORALL(file_cells, ei){ + for(unsigned int i = 0; i < cell_inner_nodes[ei].size(); i++){ + pos_t[*ti] = cell_inner_nodes[ei][i]; + ti++; + } + + }ENDFORALL; + //fill container pos_t will face nodes + FORALL(file_faces, ei){ + for(unsigned int i = 0; i < face_inner_nodes[ei].size(); i++){ + pos_t[*ti] = face_inner_nodes[ei][i]; + ti++; + } + }ENDFORALL; + if(ti != my_temp_nodes.end()){ + debugout<<"ERROR in createVOGNOde()" << endl; + } + temp_node_ptn = all_collect_vectors(my_temp_nodes, comm); + } + //create new partition + nodes_ptn.resize(Loci::MPI_processes); + // create node allocation + long npnts = num_nodes ; + int node_ivl = npnts / Loci::MPI_processes; + int node_ivl_rem = npnts % Loci::MPI_processes ; + int node_accum = 0 ; + int nodes_base = 0 ; + for(int i = 0; i < Loci::MPI_processes; ++i) { + int j = Loci::MPI_processes - i - 1 ; + int node_accum_update = node_accum + node_ivl + ((j send_sets(p) ; + vector send_seqs(p) ; + + // Check each processor, find out which sets to send + for(int i=0;i recv_seqs = transposeSeq(send_seqs) ; + + storeRepP sp =pos_t.Rep(); + storeRepP qcol_rep =new_pos.Rep(); + + + // Now communicate the container + vector send_sizes(p),recv_sizes(p) ; + + for(int i=0;ipack_size(send_sets[i]) ; + + MPI_Alltoall(&send_sizes[0],1,MPI_INT, + &recv_sizes[0],1,MPI_INT, + comm ) ; + + vector send_dspl(p),recv_dspl(p) ; + send_dspl[0] = 0 ; + recv_dspl[0] = 0 ; + for(int i=1;i send_store(send_sz) ; + vector recv_store(recv_sz) ; + + + for(int i=0;ipack(&send_store[send_dspl[i]],loc_pack, send_sizes[i], + send_sets[i]) ; + } + + MPI_Alltoallv(&send_store[0], &send_sizes[0], &send_dspl[0], MPI_PACKED, + &recv_store[0], &recv_sizes[0], &recv_dspl[0], MPI_PACKED, + comm ) ; + + for(int i=0;iunpack(&recv_store[recv_dspl[i]],loc_pack,recv_sizes[i], + recv_seqs[i]) ; + } + pos_t.allocate(EMPTY) ; + } + + + //create cl, cr and face2node maps from fine_faces + //re_number the nodes in pos and inner_nodes, + //and then redistribute them across the processes + void createVOGFace(int numNodes, + const store &fine_faces_cell, + const store &fine_faces, + fact_db & facts, + int& numFaces, + int& ncells, + Map& cl, + Map& cr, + multiMap& face2node, + vector& local_faces, + vector& local_cells + ){ + MPI_Comm comm = facts.get_comm() ; + + //compute numFaces + constraint my_faces, my_geom_cells; + my_faces = facts.get_variable("faces"); + my_geom_cells = facts.get_variable("geom_cells"); + entitySet dom = fine_faces.domain() & (*my_faces+*my_geom_cells); + entitySet domc = fine_faces_cell.domain() ; + int local_num_face = 0; + for(entitySet::const_iterator ei = domc.begin(); ei != domc.end(); ei++){ + local_num_face += fine_faces_cell[*ei].size(); + } + for(entitySet::const_iterator ei = dom.begin(); ei != dom.end(); ei++){ + local_num_face += fine_faces[*ei].size(); + } + numFaces = 0; + std::vector face_sizes= Loci::all_collect_sizes(local_num_face, comm); + for(int i =0; i < MPI_processes; i++) numFaces += face_sizes[i]; + + //get face domains and allocate the maps + int face_min = numNodes; + for(int i =0; i < MPI_rank; i++) face_min += face_sizes[i]; + + int face_max = face_min + face_sizes[MPI_rank] -1; + store count; + entitySet faces = EMPTY ; + if(face_sizes[MPI_rank] > 0) + faces = interval(face_min, face_max); + + cl.allocate(faces); + cr.allocate(faces); + count.allocate(faces); + + local_faces.resize(MPI_processes); + local_faces = all_collect_vectors(faces, comm); + //fill up the maps cl , cr and count + int cell_base = numNodes + numFaces; + int cell_max = std::numeric_limits::min(); + int cell_min = std::numeric_limits::max(); + + entitySet::const_iterator fid = faces.begin(); + for(entitySet::const_iterator ei = domc.begin(); ei != domc.end(); ei++){ + for(unsigned int i = 0; i < fine_faces_cell[*ei].size(); i++){ + cl[*fid] = fine_faces_cell[*ei][i][0] + cell_base -1;// -1 finefaces cell index start at 1 + if(fine_faces_cell[*ei][i][1]>=0) cr[*fid] = fine_faces_cell[*ei][i][1] + cell_base -1; + else cr[*fid] = fine_faces_cell[*ei][i][1]; + cell_max = max(cell_max,cl[*fid]); + cell_max = max(cell_max,cr[*fid]); + cell_min = min(cell_min,cl[*fid]); + if (cr[*fid]>=0) cell_min = min(cell_min,cr[*fid]); + count[*fid] = fine_faces_cell[*ei][i].size()-2; + fid++; + } + } + for(entitySet::const_iterator ei = dom.begin(); ei != dom.end(); ei++){ + for(unsigned int i = 0; i < fine_faces[*ei].size(); i++){ + cl[*fid] = fine_faces[*ei][i][0] + cell_base -1;// -1 finefaces cell index start at 1 + if(fine_faces[*ei][i][1]>=0) cr[*fid] = fine_faces[*ei][i][1] + cell_base -1; + else cr[*fid] = fine_faces[*ei][i][1]; + cell_max = max(cell_max,cl[*fid]); + cell_max = max(cell_max,cr[*fid]); + cell_min = min(cell_min,cl[*fid]); + if (cr[*fid]>=0) cell_min = min(cell_min,cr[*fid]); + count[*fid] = fine_faces[*ei][i].size()-2; + fid++; + } + } + + //get cells distribution + int global_max_cell = cell_max ; + MPI_Allreduce(&cell_max,&global_max_cell,1,MPI_INT,MPI_MAX, + comm) ; + int global_min_cell = cell_max ; + MPI_Allreduce(&cell_min,&global_min_cell,1,MPI_INT,MPI_MIN, + comm) ; + + + ncells = global_max_cell - global_min_cell +1; + + + local_cells.resize(MPI_processes); + int cell_ivl = ncells / MPI_processes; + int cell_ivl_rem = ncells % MPI_processes ; + int cell_accum = 0 ; + + for(int i = 0; i < Loci::MPI_processes; ++i) { + int j = MPI_processes - i - 1 ; + int cell_accum_update = cell_accum + cell_ivl + ((j &cluster_info, + vector &cluster_sizes) ; + bool readBCfromVOG(string filename, + vector > &boundary_ids, + MPI_Comm comm); + bool readVolTags(hid_t input_fid, + vector > &volDat); + + hid_t writeVOGOpen(string filename, MPI_Comm comm); + void writeVOGSurf(hid_t file_id, std::vector > surface_ids, MPI_Comm comm); + void writeVOGTag(hid_t output_fid, vector >& volTags, MPI_Comm comm); + void writeVOGClose(hid_t file_id, MPI_Comm comm) ; + +} + + + +void colorMatrix(Map &cl, Map &cr, multiMap &face2node); +namespace Loci{ + + //copied from ditribute_io.cc + hid_t writeVOGOpen(string filename, MPI_Comm comm) { + if(use_parallel_io){ + hid_t file_id = 0; + hid_t acc_plist; + // open collectively by all processor in comm, + acc_plist = Loci::create_faccess_plist(comm, + Loci::PHDF5_MPI_Info, + Loci::hdf5_const::facc_type); + file_id = H5Fcreate(filename.c_str(),H5F_ACC_TRUNC,H5P_DEFAULT,acc_plist) ; + H5Pclose(acc_plist); + if(file_id == 0) { + if(MPI_rank==0) cerr << "unable to open file " << filename << endl ; + Loci::Abort() ; + } + return file_id ; + }else{ + hid_t file_id = 0 ; + if(MPI_rank==0) + file_id = H5Fcreate(filename.c_str(),H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT) ; + return file_id ; + } + } + + + //copied from FVMGridWriter.cc + void writeVOGClose(hid_t file_id, MPI_Comm comm) {//parallel io included + if(MPI_rank == 0 || use_parallel_io) H5Fclose(file_id) ; + } + + //similar to the one in FVMGridWriter.cc, but no modofication to surface_ids + void writeVOGSurf(hid_t file_id, std::vector > surface_ids, MPI_Comm comm) { + hid_t group_id = 0 ; + if(MPI_rank == 0 || use_parallel_io) { + if(surface_ids.size() != 0) { +#ifdef H5_USE_16_API + group_id = H5Gcreate(file_id,"surface_info",0) ; +#else + group_id = H5Gcreate(file_id,"surface_info", + H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; +#endif + for(size_t i=0;i > &volDat) { + using namespace Loci ; + /* Save old error handler */ + H5E_auto_t old_func = 0; + void *old_client_data = 0 ; +#ifdef H5_USE_16_API + H5Eget_auto(&old_func, &old_client_data); + H5Eset_auto(NULL, NULL); +#else + H5Eget_auto(H5E_DEFAULT,&old_func, &old_client_data); + H5Eset_auto(H5E_DEFAULT,NULL, NULL); +#endif + /* Turn off error handling */ + + vector > volTags ; +#ifdef H5_USE_16_API + hid_t cell_info = H5Gopen(input_fid,"cell_info") ; +#else + hid_t cell_info = H5Gopen(input_fid,"cell_info",H5P_DEFAULT) ; +#endif + if(cell_info > 0) { + vector vol_tag ; + vector vol_set ; + vector vol_id ; + + hsize_t num_tags = 0 ; + H5Gget_num_objs(cell_info,&num_tags) ; + for(hsize_t tg=0;tg > tmp(maxi+1) ; + volTags.swap(tmp) ; + for(size_t i=0;i + (string("Main"), + entitySet(interval(0,numCells-1)))) ; + H5Gclose(file_info) ; + } + + /* Restore previous error handler */ +#ifdef H5_USE_16_API + H5Eset_auto(old_func, old_client_data); +#else + H5Eset_auto(H5E_DEFAULT,old_func, old_client_data); +#endif + volDat.swap(volTags) ; + return true ; + } + +} // namespace Loci + +//same as the function in FVMGridWriter.cc +void Loci::writeVOGFace(hid_t file_id, Map &cl, Map &cr, multiMap &face2node, MPI_Comm comm) { + // Compute cell set + entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; + entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; + entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells,comm) ; + + Map tmp_cl, tmp_cr; + multiMap tmp_face2node; + + + long long local_num_faces = face2node.domain().size() ; + + + long long num_cells = geom_cells.size() ; + long long num_faces = 0 ; + + // Reduce these variables + MPI_Allreduce(&local_num_faces,&num_faces,1,MPI_LONG_LONG_INT, + MPI_SUM,comm) ; + + hid_t group_id = 0 ; + if(Loci::MPI_rank == 0 || Loci::use_parallel_io) { +#ifdef H5_USE_16_API + group_id = H5Gopen(file_id,"file_info") ; +#else + group_id = H5Gopen(file_id,"file_info",H5P_DEFAULT) ; +#endif + + if(Loci::MPI_rank == 0) std::cerr<< "num_cells = " << num_cells << endl + << "num_faces = " << num_faces << endl ; + + hsize_t dims = 1 ; + hid_t dataspace_id = H5Screate_simple(1,&dims,NULL) ; + +#ifdef H5_USE_16_API + hid_t att_id = H5Acreate(group_id,"numFaces", H5T_STD_I64BE, + dataspace_id, H5P_DEFAULT) ; +#else + hid_t att_id = H5Acreate(group_id,"numFaces", H5T_STD_I64BE, + dataspace_id, H5P_DEFAULT,H5P_DEFAULT) ; +#endif + H5Awrite(att_id,H5T_NATIVE_LLONG,&num_faces) ; + H5Aclose(att_id) ; +#ifdef H5_USE_16_API + att_id = H5Acreate(group_id,"numCells", H5T_STD_I64BE, + dataspace_id, H5P_DEFAULT) ; +#else + att_id = H5Acreate(group_id,"numCells", H5T_STD_I64BE, + dataspace_id, H5P_DEFAULT,H5P_DEFAULT) ; +#endif + H5Awrite(att_id,H5T_NATIVE_LLONG,&num_cells) ; + H5Aclose(att_id) ; + H5Gclose(group_id) ; +#ifdef H5_USE_16_API + group_id = H5Gcreate(file_id,"face_info",0) ; +#else + group_id = H5Gcreate(file_id,"face_info", + H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; +#endif + } + + entitySet faces = face2node.domain() ; + vector, int> > f_ord(faces.size()) ; + int i = 0 ; + // For small number of cells, sort to keep bc groupings + if(num_cells<100000) { + FORALL(faces,fc) { + f_ord[i].first.first = cr[fc] ; + f_ord[i].first.second = cl[fc] ; + f_ord[i].second = fc ; + i++ ; + } ENDFORALL ; + std::sort(f_ord.begin(),f_ord.end()) ; + } else { + FORALL(faces,fc) { + f_ord[i].first.first = cl[fc] ; + f_ord[i].first.second = cr[fc] ; + f_ord[i].second = fc ; + i++ ; + } ENDFORALL ; + } + + i=0 ; + store count ; + count.allocate(faces) ; + FORALL(faces,fc) { + int nfc = f_ord[i].second ; + count[fc] = face2node[nfc].size() ; + i++ ; + } ENDFORALL ; + tmp_face2node.allocate(count) ; + tmp_cl.allocate(faces) ; + tmp_cr.allocate(faces) ; + i=0 ; + + int mc = (geom_cells).Min() ; + // Nodes should be adjusted to start from zero also... for the general case + FORALL(faces,fc) { + int nfc = f_ord[i].second ; + tmp_cl[fc] = cl[nfc]-mc ; + tmp_cr[fc] = cr[nfc] ; + if(tmp_cr[fc] >= 0) + tmp_cr[fc] -= mc ; + for(int j=0;j cluster_info ; + vector cluster_sizes ; + while(faces != EMPTY) { + entitySet fcluster = Loci::faceCluster(tmp_face2node,tmp_cl,tmp_cr,faces, + cluster_info,cluster_sizes) ; + faces -= fcluster ; + } + + Loci::writeUnorderedVector(group_id,"cluster_sizes",cluster_sizes, comm) ; + Loci::writeUnorderedVector(group_id,"cluster_info",cluster_info, comm) ; + + + if(Loci::MPI_rank == 0 || Loci::use_parallel_io) { + H5Gclose(group_id) ; + } +} + + +vector > getVOGTagFromLocal(const vector > &origVolTags, + // string outfile, + const_store &cell_offset, + const_store & num_fine_cells, + int num_original_nodes, + int num_original_faces) { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; + + + vector > volTags(origVolTags.size()); + + //serial version + if(Loci::MPI_processes == 1){ + for(unsigned int i = 0; i < origVolTags.size(); i++){ + string name = origVolTags[i].first; + //transfer entitySet into vector of interval + entitySet set=origVolTags[i].second; + int sz = set.num_intervals() ; + vector vlist(sz) ; + for(int j=0;jget_distribute_info() ; + Loci::constraint geom_cells = Loci::exec_current_fact_db->get_variable("geom_cells"); + Loci::constraint my_entities; + my_entities = dist->my_entities ; + //don't know if it's necessray + entitySet local_geom_cells = (*my_entities)&(*geom_cells); + + if(Loci::MPI_processes > 1){ + + //trasnfer the store to file numbering + int offset = 0; + store file_num_fine_cells; + file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,comm) ; + offset = 0; + store file_cell_offset; + file_cell_offset = Loci::Local2FileOrder(cell_offset.Rep(),local_geom_cells,offset,dist,comm) ; + + int coffset = 0; + std::vector local_cell_sizes; + int num_local_cells = file_cell_offset.domain().size(); + local_cell_sizes = Loci::all_collect_sizes(num_local_cells, comm); + for(int i = 0; i < MPI_rank; i++){ + coffset += local_cell_sizes[i]; + } + + + //process 0 broadcast the entitySet to all + int buf_size = 0; + if(MPI_rank==0){ + for(unsigned int i = 0; i < origVolTags.size(); i++){ + //transfer entitySet into vector of interval + entitySet set=origVolTags[i].second; + int sz = set.num_intervals() ; + buf_size += sz; + } + } + MPI_Bcast(&buf_size,1,MPI_INT, 0, comm) ; + vector buffer(2*buf_size); + if(MPI_rank==0){ + int pnt = 0; + for(unsigned int i = 0; i < origVolTags.size(); i++){ + //transfer entitySet into vector of interval + entitySet set=origVolTags[i].second; + int sz = set.num_intervals() ; + for(int j=0;j=0 && domain.inSet(start)) buffer[2*i] = file_cell_offset[start]; + else buffer[2*i] = 0; + if(end >=0 && domain.inSet(end)) buffer[2*i+1] = file_cell_offset[end]+file_num_fine_cells[end]-1; + else buffer[2*i+1] = 0; + + } + //process 0 receive the values in buffer from all the other processes, add up the values from each process. + if(MPI_rank == 0){ + + for(int pi = 1; pi recv_buf(2*buf_size); + MPI_Status mstat ; + MPI_Recv(&recv_buf[0],sizeof(int)*2*buf_size,MPI_BYTE,pi,7,comm, + &mstat) ; + for(int i = 0; i < 2*buf_size; i++)buffer[i] += recv_buf[i]; + } + }else{ + int flag = 0; + MPI_Status mstat ; + MPI_Recv(&flag,1,MPI_INT,0,6,comm,&mstat) ; + MPI_Send(&buffer[0],sizeof(int)*2*buf_size,MPI_BYTE,0,7,comm); + } + //process 0 re-organize the data in buffer into volTags and write it out + if(MPI_rank == 0){ + int pnt = 0; + for(unsigned int i = 0; i < origVolTags.size(); i++){ + //transfer entitySet into vector of interval + string name = origVolTags[i].first; + entitySet set=origVolTags[i].second; + int sz = set.num_intervals() ; + entitySet new_set = EMPTY; + for(int j = 0; j < sz; j++){ + int start = buffer[pnt++]; + int end = buffer[pnt++]; + new_set += interval(start, end); + } + volTags[i] = make_pair(name, new_set); + std::cout << "old tag: " << name << " " << set << std::endl; + std::cout << "new tag: " << name << " " << new_set << std::endl; + } + + } + + //broadcast volTags to other processes + int nvtags = volTags.size() ; + MPI_Bcast(&nvtags,1,MPI_INT,0,comm) ; + if(MPI_rank != 0) volTags.clear(); + for(int i=0;i(name,set)) ; + delete[] ibuf ; + } + + } + return volTags; +} diff --git a/src/FVMAdapt/read_tag.loci b/src/FVMAdapt/read_tag.loci index 17b8c1a4..89a6ca08 100644 --- a/src/FVMAdapt/read_tag.loci +++ b/src/FVMAdapt/read_tag.loci @@ -45,7 +45,7 @@ using Loci::storeRepP ; namespace Loci{ - std::vector all_collect_sizes(int size); + std::vector all_collect_sizes(int size, MPI_Comm comm); // Convert container from local numbering to file numbering // pass in store rep pointer: sp // entitySet to write: dom @@ -338,7 +338,7 @@ public: } virtual void compute(const sequence &seq) { string filename = *tagfile_par; - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; size_t p = filename.find("ref_sca"); if(p != string::npos){//read hdf5 file @@ -365,7 +365,7 @@ public: disable_threading(); } virtual void compute(const sequence &seq){ - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; vector filedata = *inputTagsData ; @@ -473,7 +473,7 @@ getNodeOffsets(store &nodeloc, const_store &numsplits, File2LocalOrder(localVar, locdom, fo_numsplits.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } return tnodes ; } @@ -499,7 +499,7 @@ public: virtual void compute(const sequence &seq){ vector filedata = *inputTagsData ; - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; // Now determine what data we need to collect from filedata fact_db::distribute_infoP dist = (Loci::exec_current_fact_db)->get_distribute_info() ; @@ -683,7 +683,7 @@ public: virtual void compute(const sequence &seq){ vector filedata = *inputTagsData ; - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; // Now determine what data we need to collect from filedata fact_db::distribute_infoP dist = (Loci::exec_current_fact_db)->get_distribute_info() ; @@ -804,7 +804,7 @@ public: } virtual void compute(const sequence &seq) { string filename = *tagfile_par; - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; size_t p = filename.find("ref_sca"); if(p != string::npos){//read hdf5 file @@ -831,7 +831,7 @@ public: disable_threading(); } virtual void compute(const sequence &seq){ - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; vector filedata = *inputTagsData ; diff --git a/src/FVMAdapt/reset_offset.loci b/src/FVMAdapt/reset_offset.loci index 0dbbe4ab..41ee142b 100644 --- a/src/FVMAdapt/reset_offset.loci +++ b/src/FVMAdapt/reset_offset.loci @@ -40,7 +40,7 @@ using Loci::storeRepP; // using std::ofstream; typedef Loci::vector3d vect3d; namespace Loci{ - std::vector all_collect_sizes(int size); + std::vector all_collect_sizes(int size, MPI_Comm comm); // Convert container from local numbering to file numbering // pass in store rep pointer: sp // entitySet to write: dom @@ -81,6 +81,7 @@ public: int num_procs = Loci::MPI_processes; int my_id = Loci::MPI_rank; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; @@ -135,11 +136,11 @@ public: //put into block to make the memory for store freeed after it's used { store edge_num_inner_nodes; - edge_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_edges,offset,dist,MPI_COMM_WORLD) ; + edge_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_edges,offset,dist,comm) ; FORALL(edge_num_inner_nodes.domain(), ee){ num_local_nodes += edge_num_inner_nodes[ee]; }ENDFORALL; - local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes); + local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ @@ -156,7 +157,7 @@ public: Loci::File2LocalOrder(localVar, local_edges, edge_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } //finish with edge nodes @@ -181,13 +182,13 @@ public: // is the actual file numbering. { store cell_num_inner_nodes; - cell_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; + cell_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_geom_cells,offset,dist,comm) ; num_local_nodes = 0; FORALL(cell_num_inner_nodes.domain(), ei){ num_local_nodes += cell_num_inner_nodes[ei]; }ENDFORALL; - local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes); + local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ noffset += local_nodes_sizes[i]; } @@ -203,7 +204,7 @@ public: Loci::File2LocalOrder(localVar, local_geom_cells, cell_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); //finish with cell nodes } //update noffset @@ -223,14 +224,14 @@ public: // is the actual file numbering. { store face_num_inner_nodes; - face_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_faces, offset,dist,MPI_COMM_WORLD) ; + face_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_faces, offset,dist,comm) ; num_local_nodes = 0; FORALL(face_num_inner_nodes.domain(), ei){ num_local_nodes += face_num_inner_nodes[ei]; }ENDFORALL; - local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes); + local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ noffset += local_nodes_sizes[i]; } @@ -246,7 +247,7 @@ public: Loci::File2LocalOrder(localVar, local_faces, face_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } } @@ -270,6 +271,7 @@ public: int num_procs = Loci::MPI_processes; int my_id = Loci::MPI_rank; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; Loci::constraint geom_cells = Loci::exec_current_fact_db->get_variable("geom_cells"); @@ -301,11 +303,11 @@ public: //put into block to make the memory for store freeed after it's used { store file_num_fine_cells; - file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; + file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,comm) ; FORALL(file_num_fine_cells.domain(), ee){ num_local_cells += file_num_fine_cells[ee]; }ENDFORALL; - local_cells_sizes = Loci::all_collect_sizes(num_local_cells); + local_cells_sizes = Loci::all_collect_sizes(num_local_cells, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ @@ -322,7 +324,7 @@ public: Loci::File2LocalOrder(localVar, local_geom_cells, cell_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } diff --git a/src/FVMAdapt/set_offset.loci b/src/FVMAdapt/set_offset.loci index 2fe62df4..79a760c2 100644 --- a/src/FVMAdapt/set_offset.loci +++ b/src/FVMAdapt/set_offset.loci @@ -42,7 +42,7 @@ using std::string; using std::ofstream; typedef Loci::vector3d vect3d; namespace Loci{ - std::vector all_collect_sizes(int size); + std::vector all_collect_sizes(int size, MPI_Comm comm); // Convert container from local numbering to file numbering // pass in store rep pointer: sp // entitySet to write: dom @@ -83,6 +83,7 @@ public: int num_procs = Loci::MPI_processes; int my_id = Loci::MPI_rank; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; @@ -137,11 +138,11 @@ public: //put into block to make the memory for store freeed after it's used { store edge_num_inner_nodes; - edge_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_edges,offset,dist,MPI_COMM_WORLD) ; + edge_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_edges,offset,dist,comm) ; FORALL(edge_num_inner_nodes.domain(), ee){ num_local_nodes += edge_num_inner_nodes[ee]; }ENDFORALL; - local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes); + local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ @@ -158,7 +159,7 @@ public: Loci::File2LocalOrder(localVar, local_edges, edge_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } //finish with edge nodes @@ -183,13 +184,13 @@ public: // is the actual file numbering. { store cell_num_inner_nodes; - cell_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; + cell_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_geom_cells,offset,dist,comm) ; num_local_nodes = 0; FORALL(cell_num_inner_nodes.domain(), ei){ num_local_nodes += cell_num_inner_nodes[ei]; }ENDFORALL; - local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes); + local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ noffset += local_nodes_sizes[i]; } @@ -205,7 +206,7 @@ public: Loci::File2LocalOrder(localVar, local_geom_cells, cell_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); //finish with cell nodes } //update noffset @@ -225,14 +226,14 @@ public: // is the actual file numbering. { store face_num_inner_nodes; - face_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_faces, offset,dist,MPI_COMM_WORLD) ; + face_num_inner_nodes = Loci::Local2FileOrder(num_inner_nodes.Rep(),local_faces, offset,dist,comm) ; num_local_nodes = 0; FORALL(face_num_inner_nodes.domain(), ei){ num_local_nodes += face_num_inner_nodes[ei]; }ENDFORALL; - local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes); + local_nodes_sizes = Loci::all_collect_sizes(num_local_nodes, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ noffset += local_nodes_sizes[i]; } @@ -248,7 +249,7 @@ public: Loci::File2LocalOrder(localVar, local_faces, face_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } } @@ -271,6 +272,7 @@ public: virtual void compute(const sequence &seq) { int num_procs = Loci::MPI_processes; int my_id = Loci::MPI_rank; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; Loci::constraint geom_cells = Loci::exec_current_fact_db->get_variable("geom_cells"); @@ -302,11 +304,11 @@ public: //put into block to make the memory for store freeed after it's used { store file_num_fine_cells; - file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; + file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,comm) ; FORALL(file_num_fine_cells.domain(), ee){ num_local_cells += file_num_fine_cells[ee]; }ENDFORALL; - local_cells_sizes = Loci::all_collect_sizes(num_local_cells); + local_cells_sizes = Loci::all_collect_sizes(num_local_cells, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ @@ -323,7 +325,7 @@ public: Loci::File2LocalOrder(localVar, local_geom_cells, cell_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } @@ -352,6 +354,7 @@ public: virtual void compute(const sequence &seq) { int num_procs = Loci::MPI_processes; int my_id = Loci::MPI_rank; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; fact_db::distribute_infoP dist = Loci::exec_current_fact_db->get_distribute_info() ; Loci::constraint geom_cells = Loci::exec_current_fact_db->get_variable("geom_cells"); @@ -383,11 +386,11 @@ public: //put into block to make the memory for store freeed after it's used { store file_num_fine_cells; - file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,MPI_COMM_WORLD) ; + file_num_fine_cells = Loci::Local2FileOrder(num_fine_cells.Rep(),local_geom_cells,offset,dist,comm) ; FORALL(file_num_fine_cells.domain(), ee){ num_local_cells += file_num_fine_cells[ee]; }ENDFORALL; - local_cells_sizes = Loci::all_collect_sizes(num_local_cells); + local_cells_sizes = Loci::all_collect_sizes(num_local_cells, Loci::exec_current_fact_db->get_comm()); for(int i = 0; i < my_id; i++){ @@ -404,7 +407,7 @@ public: Loci::File2LocalOrder(localVar, local_geom_cells, cell_file_offset.Rep(), offset, dist, - MPI_COMM_WORLD); + comm); } @@ -553,6 +556,7 @@ public: disable_threading(); } virtual void compute(const sequence &seq){ + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; std::ofstream outFile; int nprocs = Loci::MPI_processes; @@ -595,10 +599,10 @@ public: //compute buf_size on each process unsigned int buf_size = 0; MPI_Allreduce(&my_size, &buf_size, 1, MPI_INT, - MPI_MAX, MPI_COMM_WORLD); + MPI_MAX, comm); //process 0 find out size of buffer for each process - MPI_Gather(&my_size, 1, MPI_INT, size_buf, 1, MPI_INT, 0, MPI_COMM_WORLD); + MPI_Gather(&my_size, 1, MPI_INT, size_buf, 1, MPI_INT, 0, comm); int32 *buf = new int32[buf_size]; @@ -614,7 +618,7 @@ public: //process 0 recv the buf and write it out for(int i = 1; i < nprocs; i++){ MPI_Status status; - MPI_Recv(buf, buf_size, MPI_INT, i, 20, MPI_COMM_WORLD, &status); + MPI_Recv(buf, buf_size, MPI_INT, i, 20, comm, &status); int recv_size = size_buf[i]/2; for(int j = 0; j < recv_size; j++) { @@ -633,7 +637,7 @@ public: } }ENDFORALL; int send_size = ptr; - MPI_Send(buf, send_size, MPI_INT, 0, 20, MPI_COMM_WORLD); + MPI_Send(buf, send_size, MPI_INT, 0, 20, comm); } diff --git a/src/FVMAdapt/write_vog_module.loci b/src/FVMAdapt/write_vog_module.loci index 1348225d..cdc1f379 100644 --- a/src/FVMAdapt/write_vog_module.loci +++ b/src/FVMAdapt/write_vog_module.loci @@ -1,374 +1,378 @@ -//############################################################################# -//# -//# Copyright 2015-2025, Mississippi State University -//# -//# This file is part of the Loci Framework. -//# -//# The Loci Framework is free software: you can redistribute it and/or modify -//# it under the terms of the Lesser GNU General Public License as published by -//# the Free Software Foundation, either version 3 of the License, or -//# (at your option) any later version. -//# -//# The Loci Framework is distributed in the hope that it will be useful, -//# but WITHOUT ANY WARRANTY; without even the implied warranty of -//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -//# Lesser GNU General Public License for more details. -//# -//# You should have received a copy of the Lesser GNU General Public License -//# along with the Loci Framework. If not, see -//# -//############################################################################# -#include -#include -#include -#include -#include -#include -#include -#include -#include "sciTypes.h" -#include "defines.h" -#include - -using std::cerr; -using std::cout; -using std::endl; -using std::string; -using std::ifstream; -using Loci::storeRepP; -using Loci::constraint; -using std::vector; -using Loci::MPI_rank; -using Loci::MPI_processes; -namespace Loci{ - void writeVOGTag(hid_t output_fid, vector >& volTags); - hid_t hdf5OpenFile(const char *name, unsigned flags, hid_t access_id, - MPI_Comm comm) ; - -} -vector > getVOGTagFromLocal(const vector >& volTagsInput, - const_store &cell_offset, - const_store & num_fine_cells, - int num_original_nodes, - int num_original_faces); - - -class node_output_file : public blackbox_rule { - - param node_output ; - const_param outfile_par ; - const_param meshfile_par ; - const_store inner_nodes_cell; - const_store inner_nodes_face; - const_store inner_nodes_edge; - - - -public: - node_output_file(){ - name_store("node_output", node_output); - name_store("outfile_par", outfile_par); - name_store("meshfile_par", meshfile_par); - - name_store("inner_nodes_cell", inner_nodes_cell); - name_store("inner_nodes_face", inner_nodes_face); - name_store("inner_nodes_edge", inner_nodes_edge); - - input("outfile_par"); - input("meshfile_par"); - input("inner_nodes_cell"); - input("inner_nodes_face"); - input("inner_nodes_edge"); - - - output("node_output"); - disable_threading(); - } - virtual void compute(const sequence &seq) { - hid_t file_id = Loci::writeVOGOpen(*outfile_par) ; - vector > boundary_ids; - //process 0 read in boundary_ids and broadcast it to all prcesses - Loci::readBCfromVOG(*meshfile_par, boundary_ids); - Loci::writeVOGSurf(file_id,boundary_ids); - Loci::storeRepP pos = Loci::exec_current_fact_db->get_variable("pos"); - writeVOGNode(file_id, pos, - inner_nodes_cell, - inner_nodes_face, - inner_nodes_edge); - Loci::writeVOGClose(file_id) ; - } -}; -register_rule register_node_output_file; - - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-but-set-variable" -#endif - -class face_output_file : public blackbox_rule { - - param face_output ; - const_param outfile_par ; - const_store fine_faces_cell; - const_store fine_faces; - - -public: - face_output_file(){ - name_store("face_output", face_output); - name_store("outfile_par", outfile_par); - name_store("fine_faces_cell", fine_faces_cell); - name_store("fine_faces", fine_faces); - - - input("outfile_par"); - input("fine_faces_cell"); - input("fine_faces"); - - output("face_output"); - disable_threading(); - } - virtual void compute(const sequence &seq) { - //first put fine_faces into a maps - hid_t file_id = 0 ; - - long long numNodes = 0; - - - if(MPI_rank==0 || Loci::use_parallel_io){ - //read numNodes - long long tmp_numNodes; - file_id = Loci::hdf5OpenFile((*outfile_par).c_str(),H5F_ACC_RDWR,H5P_DEFAULT) ; - -#ifdef H5_USE_16_API - hid_t group_id = H5Gopen(file_id,"file_info"); -#else - hid_t group_id = H5Gopen(file_id,"file_info",H5P_DEFAULT); -#endif - - hid_t attr = H5Aopen_name(group_id,"numNodes"); - //data type and variable has to match -// hid_t ret = - H5Aread(attr, H5T_NATIVE_LLONG, &tmp_numNodes); - -// ret = - H5Aclose(attr); -// ret = - H5Gclose(group_id); - numNodes = tmp_numNodes; - } - - //problem here; MPI has no long long - if(MPI_processes > 1 && !Loci::use_parallel_io) MPI_Bcast(&numNodes, 1, MPI_LONG_LONG_INT, 0, MPI_COMM_WORLD); - - // get - //compute numFaces - int local_num_face = 0; - - fact_db::distribute_infoP dist = - Loci::exec_current_fact_db->get_distribute_info() ; - entitySet my_entities = (dist->my_entities) ; - entitySet domc = my_entities & fine_faces_cell.domain() ; - entitySet dom = my_entities & fine_faces.domain() ; - - for(auto ei = domc.begin(); ei != domc.end(); ei++){ - local_num_face += fine_faces_cell[*ei].size(); - } - for(auto ei = dom.begin(); ei != dom.end(); ei++){ - local_num_face += fine_faces[*ei].size(); - } - // for(sequence::const_iterator ei = seq.begin(); ei != seq.end(); ei++){ - // local_num_face += fine_faces[*ei].size(); - // } - - std::vector face_sizes= Loci::all_collect_sizes(local_num_face); - -// long long numFaces = 0; -// for(int i =0; i < MPI_processes; i++) numFaces += face_sizes[i]; - - long long cell_min = 1 ; //numNodes + numFaces +1; - long long face_min = std::numeric_limits::min()+1024 ; //numNodes+1; - for(int i =0; i < MPI_rank; i++) face_min += face_sizes[i]; - long long face_max = face_min + face_sizes[MPI_rank] -1; - - - Map cl, cr; - multiMap face2node; - store count; - entitySet faces = interval(face_min, face_max); - cl.allocate(faces); - cr.allocate(faces); - count.allocate(faces); - - entitySet::const_iterator fid = faces.begin(); - for(auto ei = domc.begin(); ei != domc.end(); ei++){ - for(unsigned int i = 0; i < fine_faces_cell[*ei].size(); i++){ - cl[*fid] = fine_faces_cell[*ei][i][0] + cell_min; - if(fine_faces_cell[*ei][i][1]>=0) cr[*fid] = fine_faces_cell[*ei][i][1] + cell_min; - else cr[*fid] = fine_faces_cell[*ei][i][1]; - count[*fid] = fine_faces_cell[*ei][i].size()-2; - fid++; - } - } - for(auto ei = dom.begin(); ei != dom.end(); ei++){ - for(unsigned int i = 0; i < fine_faces[*ei].size(); i++){ - cl[*fid] = fine_faces[*ei][i][0] + cell_min; - if(fine_faces[*ei][i][1]>=0) cr[*fid] = fine_faces[*ei][i][1] + cell_min; - else cr[*fid] = fine_faces[*ei][i][1]; - count[*fid] = fine_faces[*ei][i].size()-2; - fid++; - } - } - - face2node.allocate(count); - - fid = faces.begin(); - for(auto ei = domc.begin(); ei != domc.end(); ei++){ - for(unsigned int i = 0; i < fine_faces_cell[*ei].size(); i++){ - for(int j = 0; j < count[*fid]; j++){ - //vog file node index start with 0 - // face2node[*fid][j] = fine_faces_cell[*ei][i][j+2]-1; - // if(fine_faces_cell[*ei][i][j+2] < 0) cerr <<"WARNING: negative node index" << endl; - face2node[*fid][j] = fine_faces_cell[*ei][i][j+2]; - } - fid++; - } - } - for(auto ei = dom.begin(); ei != dom.end(); ei++){ - for(unsigned int i = 0; i < fine_faces[*ei].size(); i++){ - for(int j = 0; j < count[*fid]; j++){ - //vog file node index start with 0 - // face2node[*fid][j] = fine_faces[*ei][i][j+2]-1; - // if(fine_faces[*ei][i][j+2] < 0) cerr <<"WARNING: negative node index" << endl; - face2node[*fid][j] = fine_faces[*ei][i][j+2]; - } - fid++; - } - } - - - colorMatrix(cl, cr, face2node); - writeVOGFace(file_id, cl, cr, face2node); - Loci::writeVOGClose(file_id); - } -}; - -register_rule register_face_output_file; - -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC diagnostic pop -#endif - - -class volTag_output_file : public singleton_rule { - param volTag_output; - const_blackbox > > volTag_blackbox; - const_param outfile_par ; - - - - -public: - volTag_output_file(){ - name_store("volTag_output", volTag_output); - name_store("outfile_par", outfile_par); - name_store("volTag_blackbox", volTag_blackbox); - - input("outfile_par"); - input("volTag_blackbox"); - output("volTag_output"); - disable_threading(); - } - virtual void compute(const sequence &seq) { - //write the new volTags out - if(MPI_rank==0){ - hid_t out_file_id = H5Fopen((*outfile_par).c_str(),H5F_ACC_RDWR,H5P_DEFAULT) ; - std::vector > volTag = *volTag_blackbox; - Loci::writeVOGTag(out_file_id, - volTag); - H5Fclose(out_file_id); - } - } -}; -register_rule register_volTag_output_file; - -$type meshfile_par param ; -$type volTagInput blackbox > > ; - -$rule blackbox(volTagInput<-meshfile_par),prelude { - string meshfile = *$meshfile_par ; - if(MPI_rank==0 || Loci::use_parallel_io){ - hid_t file_id = Loci::hdf5OpenFile(meshfile.c_str(), - H5F_ACC_RDONLY,H5P_DEFAULT) ; - - Loci::readVolTags(file_id, - *$volTagInput); - H5Fclose(file_id); - } else { - *$volTagInput = vector >() ; - } -} ; - - - - - -class get_volTag : public singleton_rule { - - blackbox > > volTag_blackbox ; - const_blackbox > > volTagInput ; - - const_param meshfile_par ; - const_store cell_offset; - const_store num_fine_cells; - const_param num_original_nodes; - const_param num_original_faces; - - - -public: - get_volTag(){ - name_store("volTag_blackbox", volTag_blackbox); - name_store("volTagInput", volTagInput); - - name_store("balanced_cell_offset", cell_offset); - name_store("balanced_num_fine_cells", num_fine_cells); - name_store("num_original_nodes", num_original_nodes); - name_store("num_original_faces", num_original_faces); - - input("volTagInput"); - input("balanced_num_fine_cells"); - input("balanced_cell_offset"); - input("num_original_nodes"); - input("num_original_faces"); - - output("volTag_blackbox"); - disable_threading(); - } - virtual void compute(const sequence &seq) { - - vector > origVolTags = *volTagInput ; - //current volume tags - *volTag_blackbox = getVOGTagFromLocal( origVolTags, - cell_offset, - num_fine_cells, - *num_original_nodes, - *num_original_faces); - - } -}; -register_rule register_get_volTag; - - -// Code to write out diagnostics if application using older AMR API. -$type inner_nodes store ; - -$rule pointwise(inner_nodes),constraint(UNIVERSE), prelude { - $[Once] { - cerr << "AMR API has changed, program needs to be updated to support AMR!" - << endl ; - } - Loci::Abort() ; -} ; - +//############################################################################# +//# +//# Copyright 2015-2025, Mississippi State University +//# +//# This file is part of the Loci Framework. +//# +//# The Loci Framework is free software: you can redistribute it and/or modify +//# it under the terms of the Lesser GNU General Public License as published by +//# the Free Software Foundation, either version 3 of the License, or +//# (at your option) any later version. +//# +//# The Loci Framework is distributed in the hope that it will be useful, +//# but WITHOUT ANY WARRANTY; without even the implied warranty of +//# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//# Lesser GNU General Public License for more details. +//# +//# You should have received a copy of the Lesser GNU General Public License +//# along with the Loci Framework. If not, see +//# +//############################################################################# +#include +#include +#include +#include +#include +#include +#include +#include +#include "sciTypes.h" +#include "defines.h" +#include + +using std::cerr; +using std::cout; +using std::endl; +using std::string; +using std::ifstream; +using Loci::storeRepP; +using Loci::constraint; +using std::vector; +using Loci::MPI_rank; +using Loci::MPI_processes; +namespace Loci{ + void writeVOGTag(hid_t output_fid, vector >& volTags, MPI_Comm comm); + hid_t hdf5OpenFile(const char *name, unsigned flags, hid_t access_id, + MPI_Comm comm) ; + +} +vector > getVOGTagFromLocal(const vector >& volTagsInput, + const_store &cell_offset, + const_store & num_fine_cells, + int num_original_nodes, + int num_original_faces); + + +class node_output_file : public blackbox_rule { + + param node_output ; + const_param outfile_par ; + const_param meshfile_par ; + const_store inner_nodes_cell; + const_store inner_nodes_face; + const_store inner_nodes_edge; + + + +public: + node_output_file(){ + name_store("node_output", node_output); + name_store("outfile_par", outfile_par); + name_store("meshfile_par", meshfile_par); + + name_store("inner_nodes_cell", inner_nodes_cell); + name_store("inner_nodes_face", inner_nodes_face); + name_store("inner_nodes_edge", inner_nodes_edge); + + input("outfile_par"); + input("meshfile_par"); + input("inner_nodes_cell"); + input("inner_nodes_face"); + input("inner_nodes_edge"); + + + output("node_output"); + disable_threading(); + } + virtual void compute(const sequence &seq) { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; + hid_t file_id = Loci::writeVOGOpen(*outfile_par, comm) ; + vector > boundary_ids; + //process 0 read in boundary_ids and broadcast it to all prcesses + Loci::readBCfromVOG(*meshfile_par, boundary_ids, comm); + Loci::writeVOGSurf(file_id,boundary_ids,comm); + Loci::storeRepP pos = Loci::exec_current_fact_db->get_variable("pos"); + writeVOGNode(file_id, pos, + inner_nodes_cell, + inner_nodes_face, + inner_nodes_edge); + Loci::writeVOGClose(file_id,comm) ; + } +}; +register_rule register_node_output_file; + + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-but-set-variable" +#endif + +class face_output_file : public blackbox_rule { + + param face_output ; + const_param outfile_par ; + const_store fine_faces_cell; + const_store fine_faces; + + +public: + face_output_file(){ + name_store("face_output", face_output); + name_store("outfile_par", outfile_par); + name_store("fine_faces_cell", fine_faces_cell); + name_store("fine_faces", fine_faces); + + + input("outfile_par"); + input("fine_faces_cell"); + input("fine_faces"); + + output("face_output"); + disable_threading(); + } + virtual void compute(const sequence &seq) { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; + //first put fine_faces into a maps + hid_t file_id = 0 ; + + long long numNodes = 0; + + + if(MPI_rank==0 || Loci::use_parallel_io){ + //read numNodes + long long tmp_numNodes; + file_id = Loci::hdf5OpenFile((*outfile_par).c_str(),H5F_ACC_RDWR,H5P_DEFAULT, Loci::exec_current_fact_db->get_comm()) ; + +#ifdef H5_USE_16_API + hid_t group_id = H5Gopen(file_id,"file_info"); +#else + hid_t group_id = H5Gopen(file_id,"file_info",H5P_DEFAULT); +#endif + + hid_t attr = H5Aopen_name(group_id,"numNodes"); + //data type and variable has to match +// hid_t ret = + H5Aread(attr, H5T_NATIVE_LLONG, &tmp_numNodes); + +// ret = + H5Aclose(attr); +// ret = + H5Gclose(group_id); + numNodes = tmp_numNodes; + } + + //problem here; MPI has no long long + if(MPI_processes > 1 && !Loci::use_parallel_io) MPI_Bcast(&numNodes, 1, MPI_LONG_LONG_INT, 0, comm); + + // get + //compute numFaces + int local_num_face = 0; + + fact_db::distribute_infoP dist = + Loci::exec_current_fact_db->get_distribute_info() ; + entitySet my_entities = (dist->my_entities) ; + entitySet domc = my_entities & fine_faces_cell.domain() ; + entitySet dom = my_entities & fine_faces.domain() ; + + for(auto ei = domc.begin(); ei != domc.end(); ei++){ + local_num_face += fine_faces_cell[*ei].size(); + } + for(auto ei = dom.begin(); ei != dom.end(); ei++){ + local_num_face += fine_faces[*ei].size(); + } + // for(sequence::const_iterator ei = seq.begin(); ei != seq.end(); ei++){ + // local_num_face += fine_faces[*ei].size(); + // } + + std::vector face_sizes= Loci::all_collect_sizes(local_num_face, Loci::exec_current_fact_db->get_comm()); + +// long long numFaces = 0; +// for(int i =0; i < MPI_processes; i++) numFaces += face_sizes[i]; + + long long cell_min = 1 ; //numNodes + numFaces +1; + long long face_min = std::numeric_limits::min()+1024 ; //numNodes+1; + for(int i =0; i < MPI_rank; i++) face_min += face_sizes[i]; + long long face_max = face_min + face_sizes[MPI_rank] -1; + + + Map cl, cr; + multiMap face2node; + store count; + entitySet faces = interval(face_min, face_max); + cl.allocate(faces); + cr.allocate(faces); + count.allocate(faces); + + entitySet::const_iterator fid = faces.begin(); + for(auto ei = domc.begin(); ei != domc.end(); ei++){ + for(unsigned int i = 0; i < fine_faces_cell[*ei].size(); i++){ + cl[*fid] = fine_faces_cell[*ei][i][0] + cell_min; + if(fine_faces_cell[*ei][i][1]>=0) cr[*fid] = fine_faces_cell[*ei][i][1] + cell_min; + else cr[*fid] = fine_faces_cell[*ei][i][1]; + count[*fid] = fine_faces_cell[*ei][i].size()-2; + fid++; + } + } + for(auto ei = dom.begin(); ei != dom.end(); ei++){ + for(unsigned int i = 0; i < fine_faces[*ei].size(); i++){ + cl[*fid] = fine_faces[*ei][i][0] + cell_min; + if(fine_faces[*ei][i][1]>=0) cr[*fid] = fine_faces[*ei][i][1] + cell_min; + else cr[*fid] = fine_faces[*ei][i][1]; + count[*fid] = fine_faces[*ei][i].size()-2; + fid++; + } + } + + face2node.allocate(count); + + fid = faces.begin(); + for(auto ei = domc.begin(); ei != domc.end(); ei++){ + for(unsigned int i = 0; i < fine_faces_cell[*ei].size(); i++){ + for(int j = 0; j < count[*fid]; j++){ + //vog file node index start with 0 + // face2node[*fid][j] = fine_faces_cell[*ei][i][j+2]-1; + // if(fine_faces_cell[*ei][i][j+2] < 0) cerr <<"WARNING: negative node index" << endl; + face2node[*fid][j] = fine_faces_cell[*ei][i][j+2]; + } + fid++; + } + } + for(auto ei = dom.begin(); ei != dom.end(); ei++){ + for(unsigned int i = 0; i < fine_faces[*ei].size(); i++){ + for(int j = 0; j < count[*fid]; j++){ + //vog file node index start with 0 + // face2node[*fid][j] = fine_faces[*ei][i][j+2]-1; + // if(fine_faces[*ei][i][j+2] < 0) cerr <<"WARNING: negative node index" << endl; + face2node[*fid][j] = fine_faces[*ei][i][j+2]; + } + fid++; + } + } + + + colorMatrix(cl, cr, face2node); + MPI_Comm wcomm = Loci::exec_current_fact_db->get_comm() ; + writeVOGFace(file_id, cl, cr, face2node, wcomm); + Loci::writeVOGClose(file_id, wcomm); + } +}; + +register_rule register_face_output_file; + +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC diagnostic pop +#endif + + +class volTag_output_file : public singleton_rule { + param volTag_output; + const_blackbox > > volTag_blackbox; + const_param outfile_par ; + + + + +public: + volTag_output_file(){ + name_store("volTag_output", volTag_output); + name_store("outfile_par", outfile_par); + name_store("volTag_blackbox", volTag_blackbox); + + input("outfile_par"); + input("volTag_blackbox"); + output("volTag_output"); + disable_threading(); + } + virtual void compute(const sequence &seq) { + //write the new volTags out + if(MPI_rank==0){ + hid_t out_file_id = H5Fopen((*outfile_par).c_str(),H5F_ACC_RDWR,H5P_DEFAULT) ; + std::vector > volTag = *volTag_blackbox; + Loci::writeVOGTag(out_file_id, + volTag, + Loci::exec_current_fact_db->get_comm()); + H5Fclose(out_file_id); + } + } +}; +register_rule register_volTag_output_file; + +$type meshfile_par param ; +$type volTagInput blackbox > > ; + +$rule blackbox(volTagInput<-meshfile_par),prelude { + string meshfile = *$meshfile_par ; + if(MPI_rank==0 || Loci::use_parallel_io){ + hid_t file_id = Loci::hdf5OpenFile(meshfile.c_str(), + H5F_ACC_RDONLY,H5P_DEFAULT, Loci::exec_current_fact_db->get_comm()) ; + + Loci::readVolTags(file_id, + *$volTagInput); + H5Fclose(file_id); + } else { + *$volTagInput = vector >() ; + } +} ; + + + + + +class get_volTag : public singleton_rule { + + blackbox > > volTag_blackbox ; + const_blackbox > > volTagInput ; + + const_param meshfile_par ; + const_store cell_offset; + const_store num_fine_cells; + const_param num_original_nodes; + const_param num_original_faces; + + + +public: + get_volTag(){ + name_store("volTag_blackbox", volTag_blackbox); + name_store("volTagInput", volTagInput); + + name_store("balanced_cell_offset", cell_offset); + name_store("balanced_num_fine_cells", num_fine_cells); + name_store("num_original_nodes", num_original_nodes); + name_store("num_original_faces", num_original_faces); + + input("volTagInput"); + input("balanced_num_fine_cells"); + input("balanced_cell_offset"); + input("num_original_nodes"); + input("num_original_faces"); + + output("volTag_blackbox"); + disable_threading(); + } + virtual void compute(const sequence &seq) { + + vector > origVolTags = *volTagInput ; + //current volume tags + *volTag_blackbox = getVOGTagFromLocal( origVolTags, + cell_offset, + num_fine_cells, + *num_original_nodes, + *num_original_faces); + + } +}; +register_rule register_get_volTag; + + +// Code to write out diagnostics if application using older AMR API. +$type inner_nodes store ; + +$rule pointwise(inner_nodes),constraint(UNIVERSE), prelude { + $[Once] { + cerr << "AMR API has changed, program needs to be updated to support AMR!" + << endl ; + } + Loci::Abort() ; +} ; + diff --git a/src/FVMMod/petsc.loci b/src/FVMMod/petsc.loci index 789a14b2..fc8539da 100644 --- a/src/FVMMod/petsc.loci +++ b/src/FVMMod/petsc.loci @@ -276,8 +276,8 @@ namespace Loci { char const * options_prefix ) { if(created) return 0; - - LociPetscCall(VecCreate(MPI_COMM_WORLD, &v)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(VecCreate(comm, &v)); if(Loci::MPI_processes > 1) { LociPetscCall(VecSetType(v, VECMPI)); } else { @@ -311,8 +311,8 @@ namespace Loci { char const * options_prefix ) { if(created) return 0; - - LociPetscCall(VecCreate(MPI_COMM_WORLD, &v)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(VecCreate(comm, &v)); if(Loci::MPI_processes > 1) { LociPetscCall(VecSetType(v, VECMPI)); } else { @@ -373,8 +373,9 @@ namespace Loci { } int Print(char const * filename) const { + MPI_Comm comm = get_exec_comm() ; PetscViewer viewer; - LociPetscCall(PetscViewerASCIIOpen(MPI_COMM_WORLD, filename, &viewer)); + LociPetscCall(PetscViewerASCIIOpen(comm, filename, &viewer)); LociPetscCall(VecView(v, viewer)); LociPetscCall(PetscViewerDestroy(&viewer)); return 0; @@ -428,8 +429,8 @@ namespace Loci { char const * options_prefix ) { if(created) return 0; - - LociPetscCall(VecCreate(MPI_COMM_WORLD, &v)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(VecCreate(comm, &v)); if(Loci::MPI_processes > 1) { LociPetscCall(VecSetType(v, VECMPI)); } else { @@ -464,8 +465,8 @@ namespace Loci { char const * options_prefix ) { if(created) return 0; - - LociPetscCall(VecCreate(MPI_COMM_WORLD, &v)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(VecCreate(comm, &v)); if(Loci::MPI_processes > 1) { LociPetscCall(VecSetType(v, VECMPI)); } else { @@ -527,8 +528,9 @@ namespace Loci { } int Print(char const * filename) const { + MPI_Comm comm = get_exec_comm() ; PetscViewer viewer; - LociPetscCall(PetscViewerASCIIOpen(MPI_COMM_WORLD, filename, &viewer)); + LociPetscCall(PetscViewerASCIIOpen(comm, filename, &viewer)); LociPetscCall(VecView(v, viewer)); LociPetscCall(PetscViewerDestroy(&viewer)); return 0; @@ -582,8 +584,8 @@ namespace Loci { char const * system_name, char const * options_prefix ) { if(created) return 0; - - LociPetscCall(MatCreate(MPI_COMM_WORLD, &m)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(MatCreate(comm, &m)); if(Loci::MPI_processes > 1) { LociPetscCall(MatSetType(m, MATMPIAIJ)); } else { @@ -634,8 +636,8 @@ namespace Loci { char const * system_name, char const * options_prefix ) { if(created) return 0; - - LociPetscCall(MatCreate(MPI_COMM_WORLD, &m)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(MatCreate(comm, &m)); if(Loci::MPI_processes > 1) { LociPetscCall(MatSetType(m, MATMPIAIJ)); } else { @@ -713,8 +715,9 @@ namespace Loci { } int Print(char const * filename) const { + MPI_Comm comm = get_exec_comm() ; PetscViewer viewer; - LociPetscCall(PetscViewerASCIIOpen(MPI_COMM_WORLD, filename, &viewer)); + LociPetscCall(PetscViewerASCIIOpen(comm, filename, &viewer)); LociPetscCall(MatView(m, viewer)); LociPetscCall(PetscViewerDestroy(&viewer)); return 0; @@ -768,8 +771,8 @@ namespace Loci { char const * system_name, char const * options_prefix ) { if(created) return 0; - - LociPetscCall(MatCreate(MPI_COMM_WORLD, &m)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(MatCreate(comm, &m)); if(Loci::MPI_processes > 1) { LociPetscCall(MatSetType(m, MATMPIBAIJ)); } else { @@ -823,8 +826,8 @@ namespace Loci { char const * system_name, char const * options_prefix ) { if(created) return 0; - - LociPetscCall(MatCreate(MPI_COMM_WORLD, &m)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(MatCreate(comm, &m)); if(Loci::MPI_processes > 1) { LociPetscCall(MatSetType(m, MATMPIBAIJ)); } else { @@ -954,8 +957,9 @@ namespace Loci { } int Print(char const * filename) const { + MPI_Comm comm = get_exec_comm() ; PetscViewer viewer; - LociPetscCall(PetscViewerASCIIOpen(MPI_COMM_WORLD, filename, &viewer)); + LociPetscCall(PetscViewerASCIIOpen(comm, filename, &viewer)); LociPetscCall(MatView(m, viewer)); LociPetscCall(PetscViewerDestroy(&viewer)); return 0; @@ -997,8 +1001,8 @@ namespace Loci { char const * options_prefix ) { if(created) return 0; - - LociPetscCall(KSPCreate(MPI_COMM_WORLD, &ksp)); + MPI_Comm comm = get_exec_comm() ; + LociPetscCall(KSPCreate(comm, &ksp)); LociPetscCall(KSPSetTolerances( ksp, reltol, abstol, PETSC_CURRENT, maxit )); @@ -1150,12 +1154,13 @@ namespace Loci { Loci::debugout << "CUDA: device=" << int(dev_id) << ", type=" << int(dev_type) << endl ; + MPI_Comm comm = get_exec_comm() ; PetscSynchronizedPrintf( - MPI_COMM_WORLD, + comm, "MPI rank %d: device=%d, type=%d\n", Loci::MPI_rank, (int)dev_id, (int)dev_type ); - PetscSynchronizedFlush(MPI_COMM_WORLD, PETSC_STDOUT); + PetscSynchronizedFlush(comm, PETSC_STDOUT); } #endif diff --git a/src/FVMOverset/holeCutting.loci b/src/FVMOverset/holeCutting.loci index 1f41139f..bf6662ac 100644 --- a/src/FVMOverset/holeCutting.loci +++ b/src/FVMOverset/holeCutting.loci @@ -144,12 +144,6 @@ namespace Loci { //-------------------------------------------------------------------------- $type stencilSize2 param ; - $type crossComponentDist_node store ; - $type crossComponentDistGrad_node store > ; - $type crossComponentID_node store ; - - $type distanceFieldGradient store > ; - $type surfaceNodeNormal store > ; $rule unit(surfaceNodeNormal<-pos) { @@ -187,6 +181,7 @@ namespace Loci { componentID_node,componentPriorityList,holeCuttingTolerance, componentNameList,componentGeometryList,stencilSize2), option(disable_threading), prelude { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; entitySet dom = entitySet(seq) ; int nc = (*$componentNameList).size() ; float delta = realToFloat(sqrt(*$stencilSize2)) ; @@ -228,7 +223,7 @@ namespace Loci { // Exchange request bounding boxes MPI_Allgather(&bbox_set[0],sizeof(Array)*nc,MPI_BYTE, &bbox_all[0],sizeof(Array)*nc,MPI_BYTE, - MPI_COMM_WORLD) ; + comm) ; vector > send_sets ; vector component_search ; @@ -306,7 +301,7 @@ namespace Loci { } vector recv_sz(p) ; MPI_Alltoall(&send_sz[0],1,MPI_INT,&recv_sz[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; vector recv_off(p+1) ; recv_off[0] = 0 ; @@ -323,10 +318,10 @@ namespace Loci { if(i != Loci::MPI_rank) { MPI_Request tmp ; MPI_Irecv(&recv_pnts[recv_off[i]],7*recv_sz[i],MPI_FLOAT,i,10, - MPI_COMM_WORLD,&tmp) ; + comm,&tmp) ; req_queue.push_back(tmp) ; MPI_Irecv(&recv_ids[recv_off[i]],recv_sz[i],MPI_INT,i,11, - MPI_COMM_WORLD,&tmp) ; + comm,&tmp) ; req_queue.push_back(tmp) ; } } @@ -376,8 +371,8 @@ namespace Loci { cerr << "Something wrong with componentID" << endl ; } } - MPI_Send(&send_buf[0],7*sz,MPI_FLOAT,i,10,MPI_COMM_WORLD) ; - MPI_Send(&send_id[0],sz,MPI_INT,i,11,MPI_COMM_WORLD) ; + MPI_Send(&send_buf[0],7*sz,MPI_FLOAT,i,10,comm) ; + MPI_Send(&send_id[0],sz,MPI_INT,i,11,comm) ; } } } diff --git a/src/FVMOverset/interpolation.loci b/src/FVMOverset/interpolation.loci index 04a32c26..c4e9351b 100644 --- a/src/FVMOverset/interpolation.loci +++ b/src/FVMOverset/interpolation.loci @@ -54,6 +54,7 @@ namespace { void memSpace(string s) { using Loci::debugout ; debugout << s <<":" << endl; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; struct mallinfo info = mallinfo() ; int mem_alloc_local @@ -63,15 +64,15 @@ namespace { int mem_alloc_global = 0 ; int mem_req_global = 0 ; MPI_Allreduce(&mem_alloc_local,&mem_alloc_global,1,MPI_INT,MPI_SUM, - MPI_COMM_WORLD) ; + comm) ; MPI_Allreduce(&mem_req_local,&mem_req_global,1,MPI_INT,MPI_SUM, - MPI_COMM_WORLD) ; + comm) ; int mem_alloc_max = 0 ; int mem_req_max = 0 ; MPI_Allreduce(&mem_alloc_local,&mem_alloc_max,1,MPI_INT,MPI_MAX, - MPI_COMM_WORLD) ; + comm) ; MPI_Allreduce(&mem_req_local,&mem_req_max,1,MPI_INT,MPI_MAX, - MPI_COMM_WORLD) ; + comm) ; debugout << " alloc=" << ((mem_alloc_global+512)>>10) <<"Mb" << ",imb=" @@ -112,10 +113,11 @@ namespace Loci { int r = Loci::MPI_rank ; int p = Loci::MPI_processes ; int num_entities = dom.size() ; + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; // Store the distribution of entity across MPI ranks vector dist_sizes(p,0) ; // number of entities per MPI rank - MPI_Allgather(&num_entities,1,MPI_INT,&dist_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + MPI_Allgather(&num_entities,1,MPI_INT,&dist_sizes[0],1,MPI_INT,comm) ; vector offsets(p+1,0) ; // prefix sum of entity counts across ranks for(int i=1;iget_comm() ; #ifdef REPORT_TIMES Loci::stopWatch s ; @@ -330,10 +333,10 @@ namespace Loci { } } ENDFORALL ; - Loci::balanceDistribution(points,MPI_COMM_WORLD) ; + Loci::balanceDistribution(points,comm) ; // Sort points using ORB method - Loci::ORBSort(points,MPI_COMM_WORLD) ; + Loci::ORBSort(points,comm) ; cnt = points.size() ; @@ -357,7 +360,7 @@ namespace Loci { // Now collect all of these points onto all of the processors vector rcounts(p,0) ; int tsz = results.size() ; - MPI_Allgather(&tsz,1,MPI_INT,&rcounts[0],1,MPI_INT,MPI_COMM_WORLD) ; + MPI_Allgather(&tsz,1,MPI_INT,&rcounts[0],1,MPI_INT,comm) ; int rsize = rcounts[0] ; for(int i=1;i pt_id(rsize,0); coord3df pt0(0.0f,0.0f,0.0f) ; vector pts(rsize,pt0) ; @@ -403,13 +406,14 @@ namespace Loci { $rule apply(interpolateFaceStencil<-interpolatePoints(cellcenter,iblank), stencilSize2,interpolateMinStencilSize,stencilSizer,facecenter,area, ci->cellcenter,ci->iblank)[Loci::NullOp], option(disable_threading), prelude { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; MEMORY_PROFILE(compute_start) ; #ifdef REPORT_TIMES Loci::stopWatch s ; s.start() ; - MPI_Barrier(MPI_COMM_WORLD) ; + MPI_Barrier(comm) ; // Loci::debugout << "time to sync: " << s.stop() << endl ; s.start() ; #endif @@ -454,7 +458,7 @@ namespace Loci { // Compute how big the decomposition chunks should be int tsz = 0 ; - MPI_Allreduce(&sz,&tsz,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&sz,&tsz,1,MPI_INT,MPI_SUM,comm) ; // split boxes until each box has no more then split_lim points int p = Loci::MPI_processes ; @@ -511,7 +515,7 @@ namespace Loci { vector box_sp,box_tp,b_sizes,send_block_id,recv_block_id ; boundingBoxDistribution(box_sp,box_tp,b_sizes,send_block_id,recv_block_id, - boxes,MPI_COMM_WORLD) ; + boxes,comm) ; // ------------------------------------------------------------------------ @@ -519,11 +523,11 @@ namespace Loci { vector::coord_info> > loc_pbox ; communicateBoxPoints(loc_pbox,box_sp,box_tp,b_sizes, send_block_id,recv_block_id,pbox,boxes, - MPI_COMM_WORLD) ; + comm) ; vector > > loc_ndelta ; communicateBoxPoints(loc_ndelta,box_sp,box_tp,b_sizes, send_block_id,recv_block_id,ndelta,boxes, - MPI_COMM_WORLD) ; + comm) ; #ifdef REPORT_TIMES Loci::debugout << "face time recv box search points: " << s.stop() << endl ; @@ -534,7 +538,7 @@ namespace Loci { // First collect bounding box information for each block vector boxes_g ; - allGatherVec(boxes_g,boxes,MPI_COMM_WORLD) ; + allGatherVec(boxes_g,boxes,comm) ; #ifdef REPORT_TIMES @@ -548,7 +552,7 @@ namespace Loci { recieveTargetPoints(recv_targets,interp_pnts->kd, boxes_g,box_sp,box_tp,b_sizes, send_block_id,recv_block_id, - MPI_COMM_WORLD) ; + comm) ; #ifdef REPORT_TIMES Loci::debugout << "time to receive target points: " << s.stop() << endl ; @@ -834,7 +838,7 @@ namespace Loci { #endif int tot = 0 ; - MPI_Allreduce(&st_error,&tot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&st_error,&tot,1,MPI_INT,MPI_SUM,comm) ; if(tot != 0) { if(Loci::MPI_rank == 0) { cerr << "STENCIL ERROR: Degenerate interface stencil formed, aborting" @@ -843,7 +847,7 @@ namespace Loci { } } - MPI_Allreduce(&st_warnings,&tot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&st_warnings,&tot,1,MPI_INT,MPI_SUM,comm) ; if(Loci::MPI_rank == 0 && tot != 0) { cerr << "Warning: Suboptimal interface stencil in reconstruction: " << tot << endl ; @@ -858,7 +862,7 @@ namespace Loci { vector > stencil_rcv ; returnBlockData(stencil_rcv,stencil_block,2,boxes, box_sp,box_tp,b_sizes,send_block_id,recv_block_id, - MPI_COMM_WORLD) ; + comm) ; int stsz = stencil_rcv.size() ; vector stmp(stsz*4) ; @@ -920,7 +924,7 @@ namespace Loci { vector > weight_rcv ; // returnBlockData(weight_rcv,weights_block,3,boxes, returnBlockData(weight_rcv,weights_block,2,boxes,box_sp,box_tp,b_sizes, - send_block_id,recv_block_id,MPI_COMM_WORLD) ; + send_block_id,recv_block_id,comm) ; $interpolateFaceStencil->weights.swap(weight_rcv) ; @@ -935,7 +939,7 @@ namespace Loci { s.start() ; #endif #ifdef REPORT_TIMES - MPI_Barrier(MPI_COMM_WORLD) ; + MPI_Barrier(comm) ; Loci::debugout << "time to sync: " << s.stop() << endl ; s.start() ; #endif @@ -1246,13 +1250,14 @@ namespace Loci { $rule apply(interpolateCellStencil<-interpolatePoints(cellcenter,iblank), stencilSize2,cellcenter,iblank,stencilSizer,interpolateMinStencilSize) [Loci::NullOp],option(disable_threading), prelude { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; MEMORY_PROFILE(compute_start) ; MEMORY_PROFILE(buildcellstencil) ; #ifdef REPORT_TIMES Loci::stopWatch s ; s.start() ; - MPI_Barrier(MPI_COMM_WORLD) ; + MPI_Barrier(comm) ; // Loci::debugout << "time to sync: " << s.stop() << endl ; s.start() ; #endif @@ -1292,7 +1297,7 @@ namespace Loci { // Compute how big the decomposition chunks should be int tsz = 0 ; - MPI_Allreduce(&sz,&tsz,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&sz,&tsz,1,MPI_INT,MPI_SUM,comm) ; MEMORY_PROFILE(pre_split_boxes) ; // split boxes until each box has no more then split_lim points int p = Loci::MPI_processes ; @@ -1340,7 +1345,7 @@ namespace Loci { vector box_sp,box_tp,b_sizes,send_block_id,recv_block_id ; boundingBoxDistribution(box_sp,box_tp,b_sizes,send_block_id,recv_block_id, - boxes,MPI_COMM_WORLD) ; + boxes,comm) ; MEMORY_PROFILE(post_distribution) ; // ------------------------------------------------------------------------ @@ -1348,7 +1353,7 @@ namespace Loci { vector::coord_info> > loc_pbox ; communicateBoxPoints(loc_pbox,box_sp,box_tp,b_sizes, send_block_id,recv_block_id,pbox,boxes, - MPI_COMM_WORLD) ; + comm) ; #ifdef REPORT_TIMES Loci::debugout << "time recv box search points: " << s.stop() << endl ; @@ -1361,7 +1366,7 @@ namespace Loci { // First collect bounding box information for each block vector boxes_g ; - allGatherVec(boxes_g,boxes,MPI_COMM_WORLD) ; + allGatherVec(boxes_g,boxes,comm) ; #ifdef REPORT_TIMES Loci::debugout << "time allGatherVec boxes: " << s.stop() << endl ; @@ -1374,7 +1379,7 @@ namespace Loci { recieveTargetPoints(recv_targets,interp_pnts->kd, boxes_g,box_sp,box_tp,b_sizes, send_block_id,recv_block_id, - MPI_COMM_WORLD) ; + comm) ; MEMORY_PROFILE(recv_targetPnts) ; #ifdef REPORT_TIMES @@ -1500,12 +1505,12 @@ namespace Loci { } int tot = 0 ; - MPI_Allreduce(&st_error,&tot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&st_error,&tot,1,MPI_INT,MPI_SUM,comm) ; if(Loci::MPI_rank == 0 && tot != 0) { cerr << "STENCIL ERROR: Degenerate stencil formed" << endl ; //Loci::Abort() ; } - MPI_Allreduce(&st_warning,&tot,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&st_warning,&tot,1,MPI_INT,MPI_SUM,comm) ; if(Loci::MPI_rank == 0 && tot != 0) { cerr << "Warning: Suboptimal stencil in reconstruction: " << tot << endl ; @@ -1522,7 +1527,7 @@ namespace Loci { vector > stencil_rcv ; returnBlockData(stencil_rcv,stencil_block,1,boxes, box_sp,box_tp,b_sizes,send_block_id,recv_block_id, - MPI_COMM_WORLD) ; + comm) ; MEMORY_PROFILE(recv_stencil) ; int stsz = stencil_rcv.size() ; @@ -1585,7 +1590,7 @@ namespace Loci { vector > weight_rcv ; returnBlockData(weight_rcv,weights_block,1,boxes, box_sp,box_tp,b_sizes,send_block_id,recv_block_id, - MPI_COMM_WORLD) ; + comm) ; $interpolateCellStencil->weights.swap(weight_rcv) ; @@ -1598,7 +1603,7 @@ namespace Loci { #ifdef REPORT_TIMES Loci::debugout << "time in new code find stencil: " << s.stop() << endl ; s.start() ; - MPI_Barrier(MPI_COMM_WORLD) ; + MPI_Barrier(comm) ; Loci::debugout << "time to sync: " << s.stop() << endl ; s.start() ; #endif diff --git a/src/FVMOverset/motion.loci b/src/FVMOverset/motion.loci index ed761346..f00973e5 100644 --- a/src/FVMOverset/motion.loci +++ b/src/FVMOverset/motion.loci @@ -525,6 +525,7 @@ namespace Loci { } ; prescribedMotion::prescribedMotion(std::string name, options_list &ol) { + MPI_Comm comm = Loci::exec_current_fact_db->get_comm() ; // Rank 0 reads in the motion data and broadcasts to all other ranks if(Loci::MPI_rank==0) { string filename = "motion_" + name + ".dat" ; @@ -557,29 +558,29 @@ namespace Loci { Loci::Abort() ; } } - MPI_Bcast(&npnts,1,MPI_INT,0,MPI_COMM_WORLD) ; - MPI_Bcast(&t[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&x[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&y[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&z[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q0[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q1[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q2[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q3[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; + MPI_Bcast(&npnts,1,MPI_INT,0,comm) ; + MPI_Bcast(&t[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&x[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&y[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&z[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q0[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q1[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q2[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q3[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; motion.initialize(t,x,y,z,q0,q1,q2,q3) ; } else { int npnts ; - MPI_Bcast(&npnts,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&npnts,1,MPI_INT,0,comm) ; vector t(npnts),x(npnts),y(npnts),z(npnts) ; vector q0(npnts),q1(npnts),q2(npnts),q3(npnts) ; - MPI_Bcast(&t[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&x[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&y[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&z[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q0[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q1[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q2[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; - MPI_Bcast(&q3[0],npnts*sizeof(real_t),MPI_BYTE,0,MPI_COMM_WORLD) ; + MPI_Bcast(&t[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&x[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&y[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&z[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q0[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q1[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q2[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; + MPI_Bcast(&q3[0],npnts*sizeof(real_t),MPI_BYTE,0,comm) ; motion.initialize(t,x,y,z,q0,q1,q2,q3) ; } } diff --git a/src/FVMtools/ccm2vog.cc b/src/FVMtools/ccm2vog.cc index 2af22a60..02fe7c17 100644 --- a/src/FVMtools/ccm2vog.cc +++ b/src/FVMtools/ccm2vog.cc @@ -1268,7 +1268,7 @@ int main(int argc, char *argv[]) { } if (Loci::MPI_rank == 0) cout << "writing VOG file" << endl; - Loci::writeVOG(outfile, pos, cl, cr, face2node, surf_ids); + Loci::writeVOG(outfile, pos, cl, cr, face2node, surf_ids, MPI_COMM_WORLD); Loci::Finalize(); return 0; diff --git a/src/FVMtools/cfd++2vog.cc b/src/FVMtools/cfd++2vog.cc index d3fa3f8f..966c2858 100644 --- a/src/FVMtools/cfd++2vog.cc +++ b/src/FVMtools/cfd++2vog.cc @@ -1121,7 +1121,7 @@ int main(int ac, char* av[]) { surf_ids.push_back(pair(bcs[i].id,bcs[i].name)) ; string outfile = gridName ; - Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids) ; + Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids, MPI_COMM_WORLD) ; Loci::Finalize() ; return 0 ; diff --git a/src/FVMtools/cobalt2vog.cc b/src/FVMtools/cobalt2vog.cc index 76416756..ab274189 100644 --- a/src/FVMtools/cobalt2vog.cc +++ b/src/FVMtools/cobalt2vog.cc @@ -268,7 +268,7 @@ int main(int ac, char *av[]) { for(size_t i=0;i(bcs[i].id,bcs[i].name)) ; - Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids) ; + Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids, MPI_COMM_WORLD) ; Loci::Finalize() ; return 0 ; diff --git a/src/FVMtools/extract.cc b/src/FVMtools/extract.cc index d4483bd5..157690d1 100644 --- a/src/FVMtools/extract.cc +++ b/src/FVMtools/extract.cc @@ -281,14 +281,14 @@ volumePart::volumePart(string out_dir, string iteration, string casename, if(stat(iblankname.c_str(),&tmpstat)== 0) { file_id = Loci::hdf5OpenFile(iblankname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { return ; } fact_db facts ; store iblank_tmp ; readData(file_id,"iblank",iblank_tmp.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet pdom = interval(1,nnodes) ; iblank.allocate(pdom) ; entitySet dom = iblank_tmp.domain() ; @@ -470,7 +470,7 @@ volumePart::volumePart(string out_dir, string iteration, string casename, cout << ngenc-ngenc_b << " general cells iblanked" << endl ; } H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } ntets_orig = ntets ; @@ -617,7 +617,7 @@ void volumePart::getTetBlock(vector > &tets, size_t start, size_t s vector > tets_local(lsize) ; readElementTypeBlock(elg,"tetrahedra",tets_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = tetsIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -654,7 +654,7 @@ void volumePart::getTetIds(vector &tetids, size_t start, size_t size) const vector tets_local(lsize) ; readElementTypeBlock(elg,"tetrahedra_ids",tets_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = tetsIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -691,7 +691,7 @@ void volumePart::getPyrmBlock(vector > &pyrms, size_t start, size_t vector > pyrms_local(lsize) ; readElementTypeBlock(elg,"pyramid",pyrms_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = pyrmIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -728,7 +728,7 @@ void volumePart::getPyrmIds(vector &pyrmids, size_t start, size_t size) con vector pyrms_local(lsize) ; readElementTypeBlock(elg,"pyramid_ids",pyrms_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = pyrmIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -764,7 +764,7 @@ void volumePart::getPrsmBlock(vector > &prsms, size_t start, size_t vector > prsms_local(lsize) ; readElementTypeBlock(elg,"prism",prsms_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = prsmIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -800,7 +800,7 @@ void volumePart::getPrsmIds(vector &prsmids, size_t start, size_t size) con vector prsms_local(lsize) ; readElementTypeBlock(elg,"prism_ids",prsms_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = prsmIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -837,7 +837,7 @@ void volumePart::getHexBlock(vector > &hexs, size_t start, size_t s vector > hexs_local(lsize) ; readElementTypeBlock(elg,"hexahedra",hexs_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = hexsIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -873,7 +873,7 @@ void volumePart::getHexIds(vector &hexids, size_t start, size_t size) const vector hexs_local(lsize) ; readElementTypeBlock(elg,"hexahedra_ids",hexs_local,start,lsize) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = hexsIblanked & interval(start,start+lsize-1) ; if(iblank==EMPTY) { @@ -913,7 +913,7 @@ void volumePart::getGenCell(vector &genCellNfaces, vector GeneralCellNodes(nnodes) ; readElementType(elg,"GeneralCellNodes",GeneralCellNodes) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; // If no general cells iblanked, then return if(ngenc_orig == ngenc) { genCellNfaces.swap(GeneralCellNfaces) ; @@ -980,7 +980,7 @@ void volumePart::getGenIds(vector &genids) const { vector gens_local(lsize) ; readElementType(elg,"GeneralCell_ids",gens_local) ; H5Gclose(elg) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet iblank = gencIblanked & interval(0,lsize-1) ; if(iblank==EMPTY) { @@ -1144,13 +1144,13 @@ volumePartDerivedVars::volumePartDerivedVars(volumePartP part, if(stat(filename.c_str(),&tmpstat) == 0) { hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; Pambient = 0 ; if(file_id >= 0) { fact_db facts ; param Pamb ; readData(file_id,"Pambient",Pamb.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; Pambient = *Pamb ; } else { cerr << "Unable to open file " << filename << endl ; @@ -1345,17 +1345,17 @@ surfacePart::surfacePart(string name, string dir, string iteration, posFile = dir + "/pos." + iteration ; hid_t file_id = Loci::hdf5OpenFile(posFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) cerr << posFile << " fail" << endl ; if(file_id < 0) return ; nnodes = sizeElementType(file_id,"data") ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; topoFile = dir + "/" + topolink ; file_id = Loci::hdf5OpenFile(topoFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) cerr << topoFile << " fail" << endl ; if(file_id < 0) return ; @@ -1370,7 +1370,7 @@ surfacePart::surfacePart(string name, string dir, string iteration, if(ngenf > 0) genSet = interval(0,ngenf-1) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; bool has_element_data = false ; for(size_t i=0;i= 0) { int nsz = sizeElementType(file_id,"data") ; @@ -1386,28 +1386,28 @@ surfacePart::surfacePart(string name, string dir, string iteration, nodalScalarVars[varname] = svar ; found_var = true ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } if(!found_var) { svar = dir+"/" + varname+"_vec."+iteration ; file_id = Loci::hdf5OpenFile(svar.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id >= 0) { int nsz = sizeElementType(file_id,"data") ; if(nsz == nnodes) { nodalVectorVars[varname] = svar ; found_var = true ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } } if(!found_var) { svar = dir+"/" + varname+"_bsca."+iteration ; file_id = Loci::hdf5OpenFile(svar.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id >= 0) { int nsz = sizeElementType(file_id,"data") ; if(nsz == (nquads+ntrias+ngenf)) { @@ -1415,14 +1415,14 @@ surfacePart::surfacePart(string name, string dir, string iteration, found_var = true ; has_element_data = true ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } } if(!found_var) { svar = dir+"/" + varname+"_bvec."+iteration ; file_id = Loci::hdf5OpenFile(svar.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id >= 0) { int nsz = sizeElementType(file_id,"data") ; if(nsz == (nquads+ntrias+ngenf)) { @@ -1430,25 +1430,25 @@ surfacePart::surfacePart(string name, string dir, string iteration, found_var = true ; has_element_data = true ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } } } vector iblank ; string iblank_file = dir +"/iblank."+iteration ; - file_id = Loci::hdf5OpenFile(iblank_file.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT); + file_id = Loci::hdf5OpenFile(iblank_file.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT, MPI_COMM_WORLD); if(file_id >=0) { vector tmp(nquads+ntrias+ngenf) ; iblank.swap(tmp) ; readElementType(file_id,"data",iblank) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; for(size_t i=0;i 1) || has_element_data ; } if(has_element_data) { file_id = Loci::hdf5OpenFile(topoFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(nquads > 0) { vector tmp(nquads) ; readElementType(file_id,"quads_ord",tmp) ; @@ -1466,7 +1466,7 @@ surfacePart::surfacePart(string name, string dir, string iteration, readElementType(file_id,"nside_ord",tmp) ; gen_ord.swap(tmp) ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(iblank.size() > 0) { // compute iblanked set quadSet = EMPTY ; @@ -1544,7 +1544,7 @@ void surfacePart::getQuads(vector > &quads) const { if(nquads > 0) { hid_t file_id = Loci::hdf5OpenFile(topoFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; int nq = sizeElementType(file_id,"quads") ; @@ -1556,7 +1556,7 @@ void surfacePart::getQuads(vector > &quads) const { quads[cnt] = tmp[ii] ; cnt++ ; } ENDFORALL ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } } void surfacePart::getQuadsIds(vector &quads_ids) const { @@ -1576,7 +1576,7 @@ void surfacePart::getTrias(vector > &trias) const { if(ntrias > 0) { hid_t file_id = Loci::hdf5OpenFile(topoFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; int nt = sizeElementType(file_id,"triangles") ; vector > tmp(nt) ; @@ -1587,7 +1587,7 @@ void surfacePart::getTrias(vector > &trias) const { trias[cnt] = tmp[ii] ; cnt++ ; } ENDFORALL ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } } void surfacePart::getTriasIds(vector &trias_ids) const{ @@ -1606,7 +1606,7 @@ void surfacePart::getGenf(vector &numGenFnodes, vector &genNodes) cons if(ngenf > 0) { hid_t file_id = Loci::hdf5OpenFile(topoFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; int ng = sizeElementType(file_id,"nside_sizes") ; @@ -1626,7 +1626,7 @@ void surfacePart::getGenf(vector &numGenFnodes, vector &genNodes) cons for(int i=0;i > &pos) const { pos.swap(tmp) ; hid_t file_id = Loci::hdf5OpenFile(posFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; readElementType(file_id,"data",pos) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } void surfacePart::getPos(vector > &pos) const { @@ -1657,11 +1657,11 @@ void surfacePart::getPos(vector > &pos) const { pos.swap(tmp) ; hid_t file_id = Loci::hdf5OpenFile(posFile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; readElementType(file_id,"data",pos) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } void surfacePart::getNodalScalar(string varname, @@ -1674,11 +1674,11 @@ void surfacePart::getNodalScalar(string varname, string filename = mi->second ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; readElementType(file_id,"data",vals) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } void surfacePart::getNodalVector(string varname, @@ -1691,11 +1691,11 @@ void surfacePart::getNodalVector(string varname, string filename = mi->second ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; readElementType(file_id,"data",vals) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } void surfacePart::getElementScalar(string varname, @@ -1712,12 +1712,12 @@ void surfacePart::getElementScalar(string varname, string filename = mi->second ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; vector vals(quad_ord.size()+tri_ord.size()+gen_ord.size()) ; readElementType(file_id,"data",vals) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; int i=0 ; FORALL(quadSet,ii) { qvals[i] = vals[quad_ord[ii]] ; @@ -1749,12 +1749,12 @@ void surfacePart::getElementVector(string varname, string filename = mi->second ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; vector > vals(quad_ord.size()+tri_ord.size()+gen_ord.size()) ; readElementType(file_id,"data",vals) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; int i=0 ; FORALL(quadSet,ii) { qvals[i] = vals[quad_ord[ii]] ; @@ -1836,13 +1836,13 @@ surfacePartDerivedVars::surfacePartDerivedVars(surfacePartP part, if(stat(filename.c_str(),&tmpstat) == 0) { hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; Pambient = 0 ; if(file_id >= 0) { fact_db facts ; param Pamb ; readData(file_id,"Pambient",Pamb.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; Pambient = *Pamb ; } else { cerr << "unable to open " << filename << endl ; @@ -2268,7 +2268,7 @@ particlePart::particlePart(string output_dir, string iteration, string casename, } hid_t file_id = Loci::hdf5OpenFile(posfile.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) return ; numParticles = sizeElementType(file_id, "particle position") ; @@ -2321,7 +2321,7 @@ std::vector particlePart::getVectorVars() const { } void particlePart::getParticlePositions(vector > &ppos) const { hid_t file_id = Loci::hdf5OpenFile(posfile.c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT) ; + H5F_ACC_RDONLY, H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << posfile << "'!" << endl ; return ; @@ -2329,7 +2329,7 @@ void particlePart::getParticlePositions(vector > &ppos) const { size_t np = sizeElementType(file_id, "particle position") ; vector > tmp(np) ; readElementType(file_id, "particle position", tmp) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(stride_size == 1) ppos.swap(tmp) ; @@ -2345,14 +2345,14 @@ void particlePart::getParticleScalar(string varname, vector &val) const { mi = scalarVars.find(varname) ; string filename = mi->second ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT) ; + H5F_ACC_RDONLY, H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; } size_t np = sizeElementType(file_id, varname.c_str()) ; vector scalar(np) ; readElementType(file_id, varname.c_str(), scalar) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(stride_size == 1) val.swap(scalar) ; @@ -2370,7 +2370,7 @@ void particlePart::getParticleVector(string varname, mi = vectorVars.find(varname) ; string filename = mi->second ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT) ; + H5F_ACC_RDONLY, H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2378,7 +2378,7 @@ void particlePart::getParticleVector(string varname, size_t np = sizeElementType(file_id, varname.c_str()) ; vector > tmp(np) ; readElementType(file_id, varname.c_str(), tmp) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(stride_size == 1) val.swap(tmp) ; @@ -2398,7 +2398,7 @@ void getDerivedVar(vector &dval, string var_name, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2407,12 +2407,12 @@ void getDerivedVar(vector &dval, string var_name, fact_db facts ; store soundSpeed ; readData(file_id,"a",soundSpeed.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+"/v_vec." + iteration +"_" + casename ; file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2420,7 +2420,7 @@ void getDerivedVar(vector &dval, string var_name, store > u ; readData(file_id,"v",u.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet dom = u.domain() ; int c = 0 ; @@ -2433,7 +2433,7 @@ void getDerivedVar(vector &dval, string var_name, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2442,13 +2442,13 @@ void getDerivedVar(vector &dval, string var_name, fact_db facts ; store pg ; readData(file_id,"pg",pg.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+"/Pambient_par." + iteration +"_" + casename ; file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2456,7 +2456,7 @@ void getDerivedVar(vector &dval, string var_name, param Pambient ; readData(file_id,"Pambient",Pambient.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet dom = pg.domain() ; bool log = (var_name == "p") ; @@ -2473,7 +2473,7 @@ void getDerivedVar(vector &dval, string var_name, string filename = output_dir+"/v_vec." + iteration +"_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2481,7 +2481,7 @@ void getDerivedVar(vector &dval, string var_name, store > u ; readData(file_id,"v",u.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet dom = u.domain() ; int c = 0 ; @@ -2494,7 +2494,7 @@ void getDerivedVar(vector &dval, string var_name, string posname = getPosFile(output_dir,iteration,casename) ; hid_t file_id = Loci::hdf5OpenFile(posname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to get grid positions for iteration " << iteration << endl ; @@ -2505,7 +2505,7 @@ void getDerivedVar(vector &dval, string var_name, fact_db facts ; readData(file_id,"pos",pos.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet dom = pos.domain() ; int c = 0 ; if(var_name == "x") { @@ -2528,7 +2528,7 @@ void getDerivedVar(vector &dval, string var_name, string filename = output_dir+"/v_vec." + iteration +"_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -2536,7 +2536,7 @@ void getDerivedVar(vector &dval, string var_name, store > u ; readData(file_id,"v",u.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet dom = u.domain() ; int c = 0 ; @@ -2631,7 +2631,7 @@ void extractVolumeSurfaces(vector &volSurface, string varname = varlist[id] ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; @@ -2666,7 +2666,7 @@ void extractVolumeSurfaces(vector &volSurface, string varname = varlist[id] ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; diff --git a/src/FVMtools/extract_2dgv.cc b/src/FVMtools/extract_2dgv.cc index 2640cef8..0789316e 100644 --- a/src/FVMtools/extract_2dgv.cc +++ b/src/FVMtools/extract_2dgv.cc @@ -55,7 +55,7 @@ void get_2dgv(string casename, string iteration, string posname = getPosFile(output_dir,iteration,casename) ; hid_t file_id = Loci::hdf5OpenFile(posname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to get grid positions for iteration " << iteration << endl ; @@ -66,7 +66,7 @@ void get_2dgv(string casename, string iteration, fact_db facts ; readData(file_id,"pos",pos.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; int npnts = pos.domain().size() ; @@ -79,7 +79,7 @@ void get_2dgv(string casename, string iteration, if(stat(iblankname.c_str(),&tmpstat)== 0) { hid_t file_id = Loci::hdf5OpenFile(iblankname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to get iblank info for iteration " << iteration << endl ; @@ -90,7 +90,7 @@ void get_2dgv(string casename, string iteration, store iblank_tmp ; readData(file_id,"iblank",iblank_tmp.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; entitySet dom = iblank_tmp.domain() ; int cnt = 1 ; FORALL(dom,nd) { @@ -318,7 +318,7 @@ void get_2dgv(string casename, string iteration, { hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return ; @@ -334,7 +334,7 @@ void get_2dgv(string casename, string iteration, int nd = node_set[i]-1+min_val ; out << scalar[nd] << endl ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } break; case NODAL_DERIVED: @@ -352,7 +352,7 @@ void get_2dgv(string casename, string iteration, { hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; Loci::Abort() ; @@ -369,7 +369,7 @@ void get_2dgv(string casename, string iteration, int nd = node_set[i]-1+min_val ; out << norm(vec[nd]) << endl ; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } break; case NODAL_MASSFRACTION: @@ -378,7 +378,7 @@ void get_2dgv(string casename, string iteration, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; Loci::Abort() ; @@ -390,7 +390,7 @@ void get_2dgv(string casename, string iteration, readData(file_id,"mixture",mix.Rep(),EMPTY,facts) ; param species_names ; readData(file_id,"species_names",species_names.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; map smap ; std::istringstream iss(*species_names) ; diff --git a/src/FVMtools/extract_ascii.cc b/src/FVMtools/extract_ascii.cc index 017c9535..b6403c57 100644 --- a/src/FVMtools/extract_ascii.cc +++ b/src/FVMtools/extract_ascii.cc @@ -81,7 +81,7 @@ void process_ascii_nodal(string casename, string iteration, string posname = getPosFile(output_dir,iteration,casename) ; hid_t file_id = Loci::hdf5OpenFile(posname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to get grid positions for iteration " << iteration << endl ; @@ -92,11 +92,11 @@ void process_ascii_nodal(string casename, string iteration, fact_db facts ; readData(file_id,"pos",pos.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; int npnts = pos.domain().size() ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; list > values ; for(size_t i=0;i scalar ; readData(file_id,var_name,scalar.Rep(),EMPTY,facts) ; entitySet dom = scalar.domain() ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; values.push_back(vector(0)) ; vector val(npnts) ; @@ -141,7 +141,7 @@ void process_ascii_nodal(string casename, string iteration, { hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; Loci::Abort() ; @@ -152,7 +152,7 @@ void process_ascii_nodal(string casename, string iteration, store > vec ; readData(file_id,var_name,vec.Rep(),EMPTY,facts) ; entitySet dom = vec.domain() ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; values.push_back(vector(npnts)) ; int c = 0 ; FORALL(dom,nd) { @@ -176,7 +176,7 @@ void process_ascii_nodal(string casename, string iteration, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; Loci::Abort() ; @@ -188,7 +188,7 @@ void process_ascii_nodal(string casename, string iteration, readData(file_id,"mixture",mix.Rep(),EMPTY,facts) ; param species_names ; readData(file_id,"species_names",species_names.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; map smap ; std::istringstream iss(*species_names) ; @@ -375,7 +375,7 @@ void process_ascii_bndry(string casename, string iteration, const string filename(variable_filenames[i]) ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; @@ -417,7 +417,7 @@ void process_ascii_bndry(string casename, string iteration, const string filename(variable_filenames[i]) ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; diff --git a/src/FVMtools/extract_cgns.cc b/src/FVMtools/extract_cgns.cc index cf7563de..a09cffb8 100644 --- a/src/FVMtools/extract_cgns.cc +++ b/src/FVMtools/extract_cgns.cc @@ -162,7 +162,7 @@ void read_surf_info(string casename, string iteration, cout<<"boundary: " << the_surf.name << endl; } - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } bool read_pambient(string casename, string iteration, float& pambient){ @@ -170,7 +170,7 @@ bool read_pambient(string casename, string iteration, float& pambient){ hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; return false; @@ -178,7 +178,7 @@ bool read_pambient(string casename, string iteration, float& pambient){ fact_db facts ; param Pambient ; Loci::readContainer(file_id,"Pambient",Pambient.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; pambient = *Pambient; return true; } diff --git a/src/FVMtools/extract_mean.cc b/src/FVMtools/extract_mean.cc index 3d4ab0cf..13a4dd89 100644 --- a/src/FVMtools/extract_mean.cc +++ b/src/FVMtools/extract_mean.cc @@ -84,7 +84,7 @@ void process_mean(string casename, string iteration, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; @@ -94,7 +94,7 @@ void process_mean(string casename, string iteration, store scalar ; readData(file_id,var_name,scalar.Rep(),EMPTY,facts) ; entitySet dom = scalar.domain() ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(it == start_iter) { mean.allocate(dom) ; M2.allocate(dom) ; @@ -122,22 +122,22 @@ void process_mean(string casename, string iteration, string filename = output_dir+'/' + var + "Mean_sca." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"Mean" ; cout << "Writing Variables: " << sname ; fact_db facts ; Loci::writeContainer(file_id,sname,m.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Var_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Var" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,v.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; cout << endl ; } break; @@ -147,7 +147,7 @@ void process_mean(string casename, string iteration, string posname = getPosFile(output_dir,iteration,casename) ; hid_t file_id = Loci::hdf5OpenFile(posname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to get grid positions for iteration " << iteration << endl ; @@ -158,11 +158,11 @@ void process_mean(string casename, string iteration, fact_db facts ; readData(file_id,"pos",pos.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; int npnts = pos.domain().size() ; entitySet dom = pos.domain() ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; store mean ; store M2 ; @@ -181,7 +181,7 @@ void process_mean(string casename, string iteration, string iter_it = string(buf)+postfix ; vector dval(npnts) ; getDerivedVar(dval,var_name,casename,iter_it) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(it == start_iter) { mean.allocate(dom) ; M2.allocate(dom) ; @@ -210,21 +210,21 @@ void process_mean(string casename, string iteration, string filename = output_dir+'/' + var + "Mean_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"Mean" ; cout << "Writing Variables: " << sname ; Loci::writeContainer(file_id,sname,m.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Var_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Var" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,v.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; cout << endl ; } break; @@ -248,7 +248,7 @@ void process_mean(string casename, string iteration, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; @@ -258,7 +258,7 @@ void process_mean(string casename, string iteration, store > vect ; readData(file_id,var_name,vect.Rep(),EMPTY,facts) ; entitySet dom = vect.domain() ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; if(it == start_iter) { mean.allocate(dom) ; M2.allocate(dom) ; @@ -323,51 +323,51 @@ void process_mean(string casename, string iteration, string filename = output_dir+'/' + var + "Mean_vec." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"Mean" ; cout << "Writing Variables: " << sname ; fact_db facts ; Loci::writeContainer(file_id,sname,m.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Var_vec." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Var" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,v.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Cuv_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Cuv" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,cuv.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Cuw_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Cuw" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,cuw.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; Loci::writeContainer(file_id,sname,cuv.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Cvw_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Cvw" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,cvw.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; cout << endl ; } break; @@ -390,7 +390,7 @@ void process_mean(string casename, string iteration, hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; Loci::Abort() ; @@ -402,7 +402,7 @@ void process_mean(string casename, string iteration, readData(file_id,"mixture",mix.Rep(),EMPTY,facts) ; param species_names ; readData(file_id,"species_names",species_names.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; map smap ; std::istringstream iss(*species_names) ; @@ -455,22 +455,22 @@ void process_mean(string casename, string iteration, string filename = output_dir+'/' + var + "Mean_sca." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"Mean" ; cout << "Writing Variables: " << sname ; fact_db facts ; Loci::writeContainer(file_id,sname,m.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "Var_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Var" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,v.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; cout << endl ; } break ; @@ -493,7 +493,7 @@ void process_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; @@ -552,7 +552,7 @@ void process_mean(string casename, string iteration, string filename = output_dir+'/' + var + "Mean_bnd." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"Mean" ; cout << "Writing Variables: " << sname ; #ifdef H5_USE_16_API @@ -569,7 +569,7 @@ void process_mean(string casename, string iteration, filename = output_dir+'/' + var + "Var_bnd." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Var" ; cout << ", " << sname ; #ifdef H5_USE_16_API @@ -604,7 +604,7 @@ void process_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to open file '" << filename << "'!" << endl ; continue ; @@ -667,7 +667,7 @@ void process_mean(string casename, string iteration, string filename = output_dir+'/' + var + "Mean_bndvec." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"Mean" ; cout << "Writing Variables: " << sname ; #ifdef H5_USE_16_API @@ -684,7 +684,7 @@ void process_mean(string casename, string iteration, filename = output_dir+'/' + var + "Var_bndvec." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"Var" ; cout << ", " << sname ; @@ -773,9 +773,9 @@ void combine_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; readData(file_id,vname,scalar_mean.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } entitySet dom = scalar_mean.domain() ; @@ -786,9 +786,9 @@ void combine_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; readData(file_id,vname,scalar_var.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } store rentry ; @@ -798,9 +798,9 @@ void combine_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; readData(file_id,vname,rentry.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } else { rentry.allocate(dom) ; FORALL(dom,ii) { @@ -853,22 +853,22 @@ void combine_mean(string casename, string iteration, string filename = output_dir+'/' + var + "CMean_sca." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"CMean" ; cout << "Writing Variables: " << sname ; fact_db facts ; Loci::writeContainer(file_id,sname,m.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "CVar_sca." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"CVar" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,v.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; cout << endl ; } break; @@ -896,9 +896,9 @@ void combine_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; readData(file_id,vname,vect_mean.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } entitySet dom = vect_mean.domain() ; @@ -909,9 +909,9 @@ void combine_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; readData(file_id,vname,vect_var.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } store rentry ; @@ -921,9 +921,9 @@ void combine_mean(string casename, string iteration, string(buf) + postfix + "_" + casename ; hid_t file_id = Loci::hdf5OpenFile(filename.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; readData(file_id,vname,rentry.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; } else { rentry.allocate(dom) ; FORALL(dom,ii) { @@ -981,22 +981,22 @@ void combine_mean(string casename, string iteration, string filename = output_dir+'/' + var + "CMean_vec." + iteration + "_" + casename ; hid_t file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; string sname = var+"CMean" ; cout << "Writing Variables: " << sname ; fact_db facts ; Loci::writeContainer(file_id,sname,m.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; filename = output_dir+'/' + var + "CVar_vec." + iteration + "_" + casename ; file_id = Loci::hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, MPI_COMM_WORLD) ; sname = var+"CVar" ; cout << ", " << sname ; Loci::writeContainer(file_id,sname,v.Rep(),facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; cout << endl ; } break; diff --git a/src/FVMtools/extract_surf.cc b/src/FVMtools/extract_surf.cc index 81e76e94..6b26b199 100644 --- a/src/FVMtools/extract_surf.cc +++ b/src/FVMtools/extract_surf.cc @@ -61,7 +61,7 @@ void get_surf(string casename, string iteration, string posname = getPosFile(output_dir,iteration,casename) ; hid_t file_id = Loci::hdf5OpenFile(posname.c_str(), H5F_ACC_RDONLY, - H5P_DEFAULT) ; + H5P_DEFAULT, MPI_COMM_WORLD) ; if(file_id < 0) { cerr << "unable to get grid positions for iteration " << iteration << endl ; @@ -72,7 +72,7 @@ void get_surf(string casename, string iteration, fact_db facts ; Loci::readContainer(file_id,"pos",pos.Rep(),EMPTY,facts) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, MPI_COMM_WORLD) ; string gridtopo = getTopoFileName(output_dir, casename, iteration) ; diff --git a/src/FVMtools/fluent2vog.cc b/src/FVMtools/fluent2vog.cc index 28141d29..a793897a 100644 --- a/src/FVMtools/fluent2vog.cc +++ b/src/FVMtools/fluent2vog.cc @@ -1084,7 +1084,7 @@ int main(int ac, char *av[]) { for(size_t i=0;i(bcs[i].id,bcs[i].name)) ; - Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids) ; + Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids, MPI_COMM_WORLD) ; Loci::Finalize() ; return 0 ; diff --git a/src/FVMtools/foam2vog.cc b/src/FVMtools/foam2vog.cc index 841a649e..98600ee4 100644 --- a/src/FVMtools/foam2vog.cc +++ b/src/FVMtools/foam2vog.cc @@ -2095,9 +2095,9 @@ int main(int ac, char *av[]) { if(MPI_rank == 0) cerr << "writing VOG file" << endl ; if(volTags.empty()) - Loci::writeVOG(outfile,pos,cl,cr,face2node,surf_ids) ; + Loci::writeVOG(outfile,pos,cl,cr,face2node,surf_ids, MPI_COMM_WORLD) ; else - Loci::writeVOG(outfile,pos,cl,cr,face2node,surf_ids,volTags) ; + Loci::writeVOG(outfile,pos,cl,cr,face2node,surf_ids,volTags, MPI_COMM_WORLD) ; Loci::Finalize() ; return 0 ; diff --git a/src/FVMtools/make_periodic.cc b/src/FVMtools/make_periodic.cc index e48c2769..555c290f 100644 --- a/src/FVMtools/make_periodic.cc +++ b/src/FVMtools/make_periodic.cc @@ -7180,7 +7180,7 @@ namespace Loci { store &boundary_names, store &boundary_tags, vector > &volTags, - int max_alloc, string filename) ; + int max_alloc, string filename, MPI_Comm &comm) ; } /////////////////////////////////////////////////////////// // The Main // @@ -7485,7 +7485,7 @@ int main(int ac, char* av[]) { fact_db facts ; string gridfile = problem_name + string(".vog") ; vector > boundary_ids ; - if(!Loci::readBCfromVOG(gridfile,boundary_ids)) { + if(!Loci::readBCfromVOG(gridfile,boundary_ids,MPI_COMM_WORLD)) { cerr << "unable to open grid file '" << gridfile << "'" << endl ; exit(-1) ; } @@ -7526,10 +7526,11 @@ int main(int ac, char* av[]) { store boundary_names ; store boundary_tags ; + MPI_Comm comm = facts.get_comm() ; if(!Loci::readGridVOG(local_nodes,local_faces,local_cells, pos,cl,cr,face2node, boundary_names,boundary_tags,volTags, - max_alloc,gridfile)) { + max_alloc,gridfile,comm)) { if(Loci::MPI_rank == 0) { cerr << endl @@ -8421,7 +8422,7 @@ int main(int ac, char* av[]) { //get boundary names vector > surf_ids = boundary_ids ; - Loci::writeVOG(output_name, newPos, newCl, newCr, newFace2node,surf_ids) ; + Loci::writeVOG(output_name, newPos, newCl, newCr, newFace2node,surf_ids, MPI_COMM_WORLD) ; gettimeofday(&time_new_grid_write_end,NULL) ; gettimeofday(&time_prog_end,NULL) ; diff --git a/src/FVMtools/msh2vog.cc b/src/FVMtools/msh2vog.cc index a72a99d6..91b43483 100644 --- a/src/FVMtools/msh2vog.cc +++ b/src/FVMtools/msh2vog.cc @@ -1000,7 +1000,7 @@ int main(int ac, char *av[]) { if(MPI_rank == 0) cerr << "writing VOG file" << endl ; - Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids) ; + Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids, MPI_COMM_WORLD) ; Loci::Finalize() ; return 0 ; } diff --git a/src/FVMtools/plot3d2vog.cc b/src/FVMtools/plot3d2vog.cc index 5e0193bb..da441f4d 100644 --- a/src/FVMtools/plot3d2vog.cc +++ b/src/FVMtools/plot3d2vog.cc @@ -2056,7 +2056,7 @@ if(Lref == "") } } ENDFORALL ; - Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids) ; + Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids, MPI_COMM_WORLD) ; Loci::Finalize() ; return 0 ; diff --git a/src/FVMtools/ugrid2vog.cc b/src/FVMtools/ugrid2vog.cc index 2a2b56a4..5c27212d 100644 --- a/src/FVMtools/ugrid2vog.cc +++ b/src/FVMtools/ugrid2vog.cc @@ -3412,6 +3412,6 @@ int main(int ac, char* av[]) { if(MPI_rank == 0) cerr << "writing VOG file" << endl ; - Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids) ; + Loci::writeVOG(outfile, pos, cl, cr, face2node,surf_ids, MPI_COMM_WORLD) ; Loci::Finalize() ; } diff --git a/src/FVMtools/vog2surf.cc b/src/FVMtools/vog2surf.cc index 9b2e887c..f761d0b9 100644 --- a/src/FVMtools/vog2surf.cc +++ b/src/FVMtools/vog2surf.cc @@ -101,7 +101,7 @@ void readSurfaces(string filename, // read in boundary names. vector > boundary_ids ; - Loci::readBCfromVOG(filename,boundary_ids) ; + Loci::readBCfromVOG(filename,boundary_ids,MPI_COMM_WORLD) ; map surf_id ; for(size_t i=0;i 1) { // cerr << "vog2surf is not parallel! Run on only one processor!" << endl ; // Loci::Abort() ; diff --git a/src/FVMtools/vogcut.cc b/src/FVMtools/vogcut.cc index b9b348cb..2420121e 100644 --- a/src/FVMtools/vogcut.cc +++ b/src/FVMtools/vogcut.cc @@ -977,7 +977,7 @@ int main(int ac, char *av[]) { } //read in boundary names vector > boundary_ids; - Loci::readBCfromVOG(input_file,boundary_ids) ; + Loci::readBCfromVOG(input_file,boundary_ids,MPI_COMM_WORLD) ; std::map bc_map; for(unsigned int i = 0; i < boundary_ids.size(); i++){ bc_map[boundary_ids[i].first] = boundary_ids[i].second; @@ -1020,7 +1020,7 @@ int main(int ac, char *av[]) { string bcstr(bcname) ; surf_ids.push_back(pair(new_bc_id, bcstr)) ; } - Loci::writeVOG(output_file, act_pos_dat, cl, cr, face2node,surf_ids, volTags) ; + Loci::writeVOG(output_file, act_pos_dat, cl, cr, face2node,surf_ids, volTags, MPI_COMM_WORLD) ; // Close everything up H5Fclose(input_fid) ; Loci::Finalize() ; diff --git a/src/FVMtools/vogmerge.cc b/src/FVMtools/vogmerge.cc index ae56ec71..012e54b0 100644 --- a/src/FVMtools/vogmerge.cc +++ b/src/FVMtools/vogmerge.cc @@ -1297,7 +1297,7 @@ int main(int ac, char *av[]) { vector > > boundary_ids(num_inputs); for(int i=0;i > &input, - entitySet input_image, - entitySet input_preimage, - const std::vector &init_ptn) ; } namespace VOG { @@ -282,7 +276,7 @@ namespace VOG { vector ptn(MPI_processes) ; // entity Partition // Get entity distributions nodes = pos.domain() ; - entitySet allNodes = Loci::all_collect_entitySet(nodes) ; + entitySet allNodes = Loci::all_collect_entitySet(nodes, MPI_COMM_WORLD) ; vector nodesizes(MPI_processes) ; int size = nodes.size() ; MPI_Allgather(&size,1,MPI_INT,&nodesizes[0],1,MPI_INT,MPI_COMM_WORLD) ; @@ -300,7 +294,7 @@ namespace VOG { vector ptn(MPI_processes) ; // entity Partition faces = face2node.domain() ; - entitySet allFaces = Loci::all_collect_entitySet(faces) ; + entitySet allFaces = Loci::all_collect_entitySet(faces, MPI_COMM_WORLD) ; vector facesizes(MPI_processes) ; int size = faces.size() ; MPI_Allgather(&size,1,MPI_INT,&facesizes[0],1,MPI_INT,MPI_COMM_WORLD) ; @@ -319,7 +313,7 @@ namespace VOG { entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells) ; + entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells, MPI_COMM_WORLD) ; int mn = geom_cells.Min() ; int mx = geom_cells.Max() ; vector pl = simplePartitionVec(mn,mx,MPI_processes) ; @@ -339,7 +333,7 @@ namespace VOG { // Get entity distributions nodes = pos.domain() ; - entitySet allNodes = Loci::all_collect_entitySet(nodes) ; + entitySet allNodes = Loci::all_collect_entitySet(nodes, MPI_COMM_WORLD) ; vector nodesizes(MPI_processes) ; int size = nodes.size() ; MPI_Allgather(&size,1,MPI_INT,&nodesizes[0],1,MPI_INT,MPI_COMM_WORLD) ; @@ -350,7 +344,7 @@ namespace VOG { } faces = face2node.domain() ; - entitySet allFaces = Loci::all_collect_entitySet(faces) ; + entitySet allFaces = Loci::all_collect_entitySet(faces, MPI_COMM_WORLD) ; vector facesizes(MPI_processes) ; size = faces.size() ; MPI_Allgather(&size,1,MPI_INT,&facesizes[0],1,MPI_INT,MPI_COMM_WORLD) ; @@ -362,7 +356,7 @@ namespace VOG { entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells) ; + entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells, MPI_COMM_WORLD) ; int mn = geom_cells.Min() ; int mx = geom_cells.Max() ; vector pl = simplePartitionVec(mn,mx,MPI_processes) ; @@ -403,7 +397,7 @@ namespace VOG { if(MPI_processes > 1) { Loci::storeRepP sp = tmp_pos.Rep() ; - Loci::fill_clone(sp, total_dom, init_ptn) ; + Loci::fill_clone(sp, total_dom, init_ptn, MPI_COMM_WORLD) ; } entitySet face_dom = face2node.domain() ; @@ -459,8 +453,8 @@ namespace VOG { Loci::storeRepP cp_sp = cpos.Rep() ; Loci::storeRepP cn_sp = cnum.Rep() ; entitySet clone_cells = tmp_cells - (cells&init_ptn[MPI_rank]) ; - std::vector v_cpos = Loci::send_global_clone_non(cp_sp, clone_cells, init_ptn) ; - std::vector v_cnum = Loci::send_global_clone_non(cn_sp, clone_cells, init_ptn) ; + std::vector v_cpos = Loci::send_global_clone_non(cp_sp, clone_cells, init_ptn, MPI_COMM_WORLD) ; + std::vector v_cnum = Loci::send_global_clone_non(cn_sp, clone_cells, init_ptn, MPI_COMM_WORLD) ; for(int i = 0; i < Loci::MPI_processes; ++i) { entitySet dom = v_cpos[i]->domain() & cpos.domain() ; dstore > tmp_cpos(v_cpos[i]) ; @@ -470,8 +464,8 @@ namespace VOG { cnum[di] += tmp_cnum[di] ; } ENDFORALL ; } - Loci::fill_clone(cp_sp, clone_cells, init_ptn) ; - Loci::fill_clone(cn_sp, clone_cells, init_ptn) ; + Loci::fill_clone(cp_sp, clone_cells, init_ptn, MPI_COMM_WORLD) ; + Loci::fill_clone(cn_sp, clone_cells, init_ptn, MPI_COMM_WORLD) ; FORALL(tmp_cells,cc) { cpos[cc] = cpos[cc]/cnum[cc] ; } ENDFORALL ; @@ -615,7 +609,7 @@ namespace VOG { + cr.image(interior_faces) ; clone_cells -= geom_cells ; Loci::storeRepP cp_sp = color.Rep() ; - Loci::fill_clone(cp_sp, clone_cells, cptn) ; + Loci::fill_clone(cp_sp, clone_cells, cptn, MPI_COMM_WORLD) ; FORALL(interior_faces,fc) { int color_l = color[cl[fc]] ; @@ -755,7 +749,7 @@ namespace VOG { entitySet faces = face2node.domain() ; entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells) ; + entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells, MPI_COMM_WORLD) ; // create splits for sorting cell data const int p = Loci::MPI_processes ; @@ -974,7 +968,7 @@ namespace VOG { // Get entity distributions entitySet nodes = pos.domain() ; - entitySet allNodes = Loci::all_collect_entitySet(nodes) ; + entitySet allNodes = Loci::all_collect_entitySet(nodes, MPI_COMM_WORLD) ; vector nodesizes(MPI_processes) ; int size = nodes.size() ; MPI_Allgather(&size,1,MPI_INT,&nodesizes[0],1,MPI_INT,MPI_COMM_WORLD) ; @@ -985,7 +979,7 @@ namespace VOG { } entitySet faces = face2node.domain() ; - entitySet allFaces = Loci::all_collect_entitySet(faces) ; + entitySet allFaces = Loci::all_collect_entitySet(faces, MPI_COMM_WORLD) ; vector facesizes(MPI_processes) ; size = faces.size() ; MPI_Allgather(&size,1,MPI_INT,&facesizes[0],1,MPI_INT,MPI_COMM_WORLD) ; @@ -997,7 +991,7 @@ namespace VOG { entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells) ; + entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells, MPI_COMM_WORLD) ; int mn = geom_cells.Min() ; int mx = geom_cells.Max() ; vector pl = simplePartitionVec(mn,mx,MPI_processes) ; @@ -1085,7 +1079,7 @@ namespace VOG { entitySet cimage = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - cell2cell.setRep(MapRepP(cell2cell.Rep())->expand(cimage,cptn)) ; + cell2cell.setRep(MapRepP(cell2cell.Rep())->expand(cimage,cptn, MPI_COMM_WORLD)) ; REPORTMEM() ; FORALL(faces,fc) { @@ -1259,7 +1253,7 @@ namespace VOG { entitySet clone_nodes = node_access - pos.domain() ; Loci::storeRepP nk_sp = node_key.Rep() ; std::vector v_nk = - Loci::send_global_clone_non(nk_sp,clone_nodes,nptn) ; + Loci::send_global_clone_non(nk_sp,clone_nodes,nptn, MPI_COMM_WORLD) ; for(int i=0;idomain() & pos.domain() ; dstore tmp_nk(v_nk[i]) ; @@ -1318,7 +1312,7 @@ namespace VOG { node2node[nd] = nmapping[nd][0] ; } ENDFORALL ; - node2node.setRep(MapRepP(node2node.Rep())->expand(node_access,nptn)) ; + node2node.setRep(MapRepP(node2node.Rep())->expand(node_access,nptn, MPI_COMM_WORLD)) ; REPORTMEM() ; FORALL(face2node.domain(),fc) { for(int i=0;i #include #include +#include #include #include #include @@ -37,6 +38,7 @@ using std::istream ; using std::ofstream ; namespace Loci { + extern fact_db *exec_current_fact_db ; extern ofstream debugout ; @@ -45,16 +47,18 @@ namespace Loci { using std::vector ; using std::sort ; //**************************************************************************/ - storeRepP dMapRepI::expand(entitySet &out_of_dom, std::vector &ptn) { - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + storeRepP dMapRepI::expand(entitySet &out_of_dom, std::vector &ptn, MPI_Comm comm) { + int np ; + MPI_Comm_size(comm, &np) ; + int *recv_count = new int[np] ; + int *send_count = new int[np] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; entitySet::const_iterator ei ; std::vector::const_iterator vi ; int size_send = 0 ; - std::vector > copy(MPI_processes), send_clone(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + std::vector > copy(np), send_clone(np) ; + for(int i = 0; i < np; ++i) { entitySet tmp = out_of_dom & ptn[i] ; for(ei = tmp.begin(); ei != tmp.end(); ++ei) copy[i].push_back(*ei) ; @@ -64,28 +68,28 @@ namespace Loci { } int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_buf = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = copy[i].begin(); vi != copy[i].end(); ++vi) { send_buf[size_send] = *vi ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - for(int i = 0; i < MPI_processes; ++i) { + comm) ; + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i].push_back(recv_buf[j]) ; @@ -94,26 +98,26 @@ namespace Loci { entitySet dom = domain() ; - std::vector map_entities(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) + std::vector map_entities(np) ; + for(int i = 0; i < np; ++i) for(vi = send_clone[i].begin(); vi != send_clone[i].end(); ++vi) if(dom.inSet(*vi)) (map_entities[i])[*vi] = attrib_data[*vi] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { send_count[i] = 2 * map_entities[i].size() ; size_send += send_count[i] ; } int *send_map = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_map = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(HASH_MAP(int, int)::const_iterator miv = map_entities[i].begin(); miv != map_entities[i].end(); ++miv) { send_map[size_send] = miv->first ; ++size_send ; @@ -122,15 +126,15 @@ namespace Loci { } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_map,send_count, send_displacement , MPI_INT, recv_map, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; HASH_MAP(int, int) hm ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]-1; j+=2) { hm[recv_map[j]] = recv_map[j+1]; diff --git a/src/System/DMultiMap.cc b/src/System/DMultiMap.cc index afffa3ad..35e85ede 100644 --- a/src/System/DMultiMap.cc +++ b/src/System/DMultiMap.cc @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -35,6 +36,8 @@ using std::istream ; namespace Loci { + extern fact_db *exec_current_fact_db ; + using std::pair ; using std::make_pair ; using std::vector ; @@ -62,17 +65,18 @@ namespace Loci return static_map.Rep() ; } - storeRepP dmultiMapRepI::expand(entitySet &out_of_dom, std::vector &ptn) { - - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + storeRepP dmultiMapRepI::expand(entitySet &out_of_dom, std::vector &ptn, MPI_Comm comm) { + int np ; + MPI_Comm_size(comm, &np) ; + int *recv_count = new int[np] ; + int *send_count = new int[np] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; entitySet::const_iterator ei ; std::vector >::const_iterator vi ; int size_send = 0 ; - std::vector > > copy(MPI_processes), send_clone(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + std::vector > > copy(np), send_clone(np) ; + for(int i = 0; i < np; ++i) { entitySet tmp = out_of_dom & ptn[i] ; for(ei = tmp.begin(); ei != tmp.end(); ++ei) copy[i].push_back(*ei) ; @@ -82,37 +86,37 @@ namespace Loci } int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_buf = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = copy[i].begin(); vi != copy[i].end(); ++vi) { send_buf[size_send] = *vi ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - for(int i = 0; i < MPI_processes; ++i) { + comm) ; + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i].push_back(recv_buf[j]) ; sort(send_clone[i].begin(), send_clone[i].end()) ; } - std::vector > > comm_list(MPI_processes) ; + std::vector > > comm_list(np) ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = send_clone[i].begin(); vi != send_clone[i].end(); ++vi) { comm_list[i].push_back(*vi) ; int lsz = attrib_data[*vi].size() ; @@ -121,21 +125,21 @@ namespace Loci comm_list[i].push_back(attrib_data[*vi][l]) ; } - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { send_count[i] = comm_list[i].size() ; } size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += send_count[i] ; int *send_map = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_map = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { int lsz = comm_list[i].size() ; for(int l=0;lsetDomainKeySpace(getDomainKeySpace()) ; MapRepP(hm.Rep())->setRangeKeySpace(getRangeKeySpace()) ; std::vector > ss ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]-1; ++j) { int count = recv_map[j+1] ; diff --git a/src/System/FVMGridReader.cc b/src/System/FVMGridReader.cc index b0d974e5..254fe1bc 100644 --- a/src/System/FVMGridReader.cc +++ b/src/System/FVMGridReader.cc @@ -100,7 +100,8 @@ namespace Loci { vector &procid, MPI_Comm comm) ; - bool redistribute_cell_weight(storeRepP old_store, storeRepP new_store); + bool redistribute_cell_weight(storeRepP old_store, storeRepP new_store, + MPI_Comm comm); extern vector simplePartition(int mn, int mx, MPI_Comm comm); extern bool load_cell_weights ; @@ -115,12 +116,16 @@ namespace Loci { template void readVectorDist(hid_t group_id, const char *vector_name, vector sizes, - vector &v) { + vector &v, + MPI_Comm comm) { + int r = 0, p = 0 ; + MPI_Comm_rank(comm, &r) ; + MPI_Comm_size(comm, &p) ; hid_t dataset = 0 ; hid_t dspace = 0 ; - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || r == 0) { dataset = H5Dopen(group_id,vector_name,H5P_DEFAULT) ; if(dataset < 0) { cerr << "unable to open dataset" << endl ; @@ -128,12 +133,12 @@ namespace Loci { } dspace = H5Dget_space(dataset) ; } - v.resize(sizes[MPI_rank]) ; + v.resize(sizes[r]) ; if(use_parallel_io) { // each process read in vector - long lsz = sizes[MPI_rank] ; + long lsz = sizes[r] ; hsize_t dimension = lsz ; hsize_t count = dimension ; @@ -141,7 +146,7 @@ namespace Loci { hsize_t start = 0 ; //each process has different start - for(int i=0; i tmp(sz) ; @@ -213,23 +218,27 @@ namespace Loci { H5Sclose(memspace) ; } // send to remote processor - MPI_Send(&tmp[0],sz*sizeof(T),MPI_BYTE,1,0,MPI_COMM_WORLD) ; + MPI_Send(&tmp[0],sz*sizeof(T),MPI_BYTE,1,0, + comm) ; } H5Dclose(dataset) ; H5Sclose(dspace) ; } else { - long size = sizes[MPI_rank] ; + long size = sizes[r] ; // if(size > 0) { MPI_Status status ; - MPI_Recv(&v[0],size*sizeof(T),MPI_BYTE,MPI_rank-1,0,MPI_COMM_WORLD,&status) ; + MPI_Recv(&v[0],size*sizeof(T),MPI_BYTE,r-1,0, + comm,&status) ; // } - for(int i=MPI_rank+1;i tmp(lsz) ; - MPI_Recv(&tmp[0],lsz*sizeof(T),MPI_BYTE,MPI_rank-1,0,MPI_COMM_WORLD,&status) ; - MPI_Send(&tmp[0],lsz*sizeof(T),MPI_BYTE,MPI_rank+1,0,MPI_COMM_WORLD) ; + MPI_Recv(&tmp[0],lsz*sizeof(T),MPI_BYTE,r-1,0, + comm,&status) ; + MPI_Send(&tmp[0],lsz*sizeof(T),MPI_BYTE,r+1,0, + comm) ; } } } @@ -398,7 +407,11 @@ namespace Loci { } bool readBCfromVOG(string filename, - vector > &boundary_ids) { + vector > &boundary_ids, + MPI_Comm comm) { + int r = 0 ; + MPI_Comm_rank(comm, &r) ; + /*process 0 read in boundary_ids and broadcast it to all prcesses*/ hid_t file_id = 0 ; @@ -406,7 +419,7 @@ namespace Loci { /* Save old error handler */ H5E_auto_t old_func = 0 ; void *old_client_data = 0 ; - if(MPI_rank == 0) { + if(r == 0) { H5Eget_auto(H5E_DEFAULT,&old_func, &old_client_data); /* Turn off error handling */ H5Eset_auto(H5E_DEFAULT,NULL, NULL); @@ -449,10 +462,10 @@ namespace Loci { } // Share boundary tag data with all other processors int bsz = boundary_ids.size() ; - MPI_Bcast(&bsz,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&bsz,1,MPI_INT,0,comm) ; if(bsz > 0) { string buf ; - if(MPI_rank == 0) { + if(r == 0) { ostringstream oss ; for(int i=0;i &boundary_names, store &boundary_tags, vector > &volTags, - int max_alloc, string filename) { + int max_alloc, string filename, + MPI_Comm &comm) { + int r = 0, p = 0 ; + MPI_Comm_rank(comm, &r) ; + MPI_Comm_size(comm, &p) ; - local_nodes.resize(Loci::MPI_processes); - local_faces.resize(Loci::MPI_processes); - local_cells.resize(Loci::MPI_processes); + local_nodes.resize(p); + local_faces.resize(p); + local_cells.resize(p); hid_t file_id = 0 ; hid_t face_g = 0 ; @@ -588,25 +605,25 @@ namespace Loci { H5E_auto_t old_func = 0 ; void *old_client_data = 0 ; vector > boundary_ids ; - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || r == 0) { H5Eget_auto(H5E_DEFAULT,&old_func, &old_client_data); /* Turn off error handling */ H5Eset_auto(H5E_DEFAULT,NULL, NULL); } // file_id = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT) ; - file_id = readVOGOpen(filename) ; - if(MPI_rank == 0 && file_id <= 0) { + file_id = readVOGOpen(filename,comm) ; + if(r == 0 && file_id <= 0) { failure = 1 ; } int fail_state = 0 ; - MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,comm) ; if(fail_state != 0) { return false ; } - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || r == 0) { face_g = H5Gopen(file_id,"face_info",H5P_DEFAULT) ; node_g = H5Gopen(file_id,"node_info",H5P_DEFAULT) ; // Check to see if the file has surface info @@ -639,10 +656,10 @@ namespace Loci { // In the serial I/O case we need to broadcast // Share boundary tag data with all other processors int bsz = boundary_ids.size() ; - MPI_Bcast(&bsz,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&bsz,1,MPI_INT,0,comm) ; if(bsz > 0) { string buf ; - if(MPI_rank == 0) { + if(r == 0) { ostringstream oss ; for(int i=0;i > tpos(mxsz) ; // Shift remaining ones into place - for(int i=MPI_rank+1;i cluster_sizes ; // Now read in face clusters long nclusters = 0 ; - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || r == 0) { dataset = H5Dopen(face_g,"cluster_sizes",H5P_DEFAULT) ; dspace = H5Dget_space(dataset) ; if(dataset <=0 || dspace <=0) @@ -906,29 +926,29 @@ namespace Loci { nclusters = size ; } - MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,comm) ; if(fail_state != 0) { return false ; } if(!use_parallel_io) { - MPI_Bcast(&nclusters,1,MPI_LONG,0,MPI_COMM_WORLD) ; + MPI_Bcast(&nclusters,1,MPI_LONG,0,comm) ; } - vector cluster_dist(MPI_processes,0) ; + vector cluster_dist(p,0) ; #ifdef DEBUG long sum = 0 ; #endif - for(int i=0;ii?1:0); + for(int i=0;ii?1:0); #ifdef DEBUG sum += cluster_dist[i] ; #endif } FATAL(sum != nclusters) ; REPORTMEM() ; - readVectorDist(face_g,"cluster_sizes",cluster_dist,cluster_sizes) ; + readVectorDist(face_g,"cluster_sizes",cluster_dist,cluster_sizes,comm) ; long cluster_info_size = 0 ; for(size_t i=0;i > volDat ; - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || r == 0) { readVolTags(file_id,volDat) ; } if(!use_parallel_io) { int nvtags = volDat.size() ; - MPI_Bcast(&nvtags,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&nvtags,1,MPI_INT,0,comm) ; for(int i=0;i(name,set)) ; } delete[] ibuf ; @@ -987,14 +1007,14 @@ namespace Loci { } volTags.swap(volDat) ; - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || r == 0) { H5Gclose(face_g) ; H5Fclose(file_id) ; /* Restore previous error handler */ H5Eset_auto(H5E_DEFAULT,old_func, old_client_data); } - MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&failure,&fail_state,1,MPI_INT,MPI_MAX,comm) ; if(fail_state != 0) { return false ; } @@ -1013,16 +1033,16 @@ namespace Loci { } // Now get a face allocation for each processor - vector faces_pp(MPI_processes,0) ; + vector faces_pp(p,0) ; MPI_Allgather(&tot_faces,1,MPI_INT,&faces_pp[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int face_accum = 0 ; int faces_base = max_alloc + nnodes ; - if(Loci::MPI_processes > 1) { + if(p > 1) { faces_base = max_alloc ; } - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < p; ++i) { local_faces[i] = EMPTY ; if(faces_pp[i] > 0) { local_faces[i] = interval(faces_base + face_accum, @@ -1031,23 +1051,23 @@ namespace Loci { face_accum += faces_pp[i] ; } int cells_base = faces_base + face_accum ; - if(Loci::MPI_processes > 1) { + if(p > 1) { cells_base = max_alloc+nnodes ; } store counts ; - counts.allocate(local_faces[MPI_rank]) ; - tot_faces = local_faces[MPI_rank].Min() ; + counts.allocate(local_faces[r]) ; + tot_faces = local_faces[r].Min() ; for(size_t i=0;i > &input, - entitySet input_image, - entitySet input_preimage, - const std::vector &init_ptn) ; - #ifdef LOCI_USE_METIS vector newMetisPartitionOfCells(const vector &local_cells, const Map &cl, const Map &cr, const store &boundary_tags, - storeRepP cell_weight_store) { + storeRepP cell_weight_store, + MPI_Comm comm) { + int r = 0, p = 0 ; + MPI_Comm_rank(comm, &r) ; + MPI_Comm_size(comm, &p) ; entitySet dom = cl.domain() & cr.domain() ; entitySet refset = boundary_tags.domain() ; entitySet bcfaces = cr.preimage(refset).first ; @@ -1185,7 +1203,7 @@ namespace Loci { multiMap cell2cell ; entitySet all_cells ; - for(int i=0;i >().swap(rawMap) ; // Free up memory from rawMap int count = 0 ; - idx_t size_map = local_cells[Loci::MPI_rank].size() ; + idx_t size_map = local_cells[r].size() ; vector size_adj(size_map) ; count = 0 ; - for(entitySet::const_iterator ei=local_cells[Loci::MPI_rank].begin(); - ei != local_cells[Loci::MPI_rank].end(); ++ei) { + for(entitySet::const_iterator ei=local_cells[r].begin(); + ei != local_cells[r].end(); ++ei) { size_adj[count] = cell2cell[*ei].size() ; ++count ; } @@ -1205,14 +1223,14 @@ namespace Loci { vector part(size_map) ; vector xadj(size_map+1) ; idx_t edgecut ; - vector vdist(Loci::MPI_processes + 1) ; + vector vdist(p + 1) ; int cmin = local_cells[0].Min(); - for(int i=0; i vwgt(ncon*size_map) ; int cnt = 0 ; - for(entitySet::const_iterator ei=local_cells[Loci::MPI_rank].begin(); - ei!=local_cells[Loci::MPI_rank].end();++ei,cnt+=ncon) { + for(entitySet::const_iterator ei=local_cells[r].begin(); + ei!=local_cells[r].end();++ei,cnt+=ncon) { // first weight for cell is 1 (the cell computation) vwgt[cnt] = 1 ; // the second weight is from the store cell_weights[*ei] @@ -1419,17 +1438,17 @@ namespace Loci { &part[0],&mc) ; } - if(Loci::MPI_rank == 0) { + if(r == 0) { Loci::debugout << " Parmetis Edge cut " << edgecut << endl ; } //find the partition ptn given by Metis vector ptn ; - for(int i = 0; i < Loci::MPI_processes; ++i) { + for(int i = 0; i < p; ++i) { ptn.push_back(EMPTY) ; } - cmin = local_cells[Loci::MPI_rank].Min() ; + cmin = local_cells[r].Min() ; for(int i=0;i &ptn, const vector &ptn_t, entitySet new_alloc, - storeRepP inRep,storeRepP outRep) { - vector rdom(MPI_processes) ; - vector send_sizes(MPI_processes) ; + storeRepP inRep,storeRepP outRep, + MPI_Comm comm = MPI_COMM_WORLD) { + int p = 1 ; + MPI_Comm_size(comm, &p) ; + vector rdom(p) ; + vector send_sizes(p) ; // Determine how to redistribute current domain to new processors entitySet::const_iterator ei = new_alloc.begin() ; - for(int i=0;ipack_size(ptn[i]) ; sequence s ; for(entitySet::const_iterator si=ptn_t[i].begin();si!=ptn_t[i].end();++si) { @@ -1459,13 +1481,13 @@ namespace Loci { } WARN(ei != new_alloc.end()) ; - vector recv_sizes(MPI_processes) ; + vector recv_sizes(p) ; MPI_Alltoall(&send_sizes[0],1,MPI_INT, &recv_sizes[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int size_send = 0 ; int size_recv = 0 ; - for(int i=0;i memspace(size_send+size_recv,0) ; unsigned char *send_store = &memspace[0] ; unsigned char *recv_store = &memspace[size_send] ; - vector send_displacement(MPI_processes,0) ; - vector recv_displacement(MPI_processes,0) ; + vector send_displacement(p,0) ; + vector recv_displacement(p,0) ; send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < p; ++i) { send_displacement[i] = send_displacement[i-1] + send_sizes[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_sizes[i-1] ; } int loc_pack = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < p; ++i) { inRep->pack(&send_store[0], loc_pack, size_send, ptn[i]) ; } @@ -1493,9 +1515,9 @@ namespace Loci { MPI_PACKED, &recv_store[0], &recv_sizes[0], &recv_displacement[0], MPI_PACKED, - MPI_COMM_WORLD) ; + comm) ; loc_pack = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < p; ++i) { outRep->unpack(&recv_store[0], loc_pack, size_recv, rdom[i]) ; } @@ -1525,6 +1547,7 @@ namespace Loci { store &boundary_tags, entitySet bcsurfset, fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; int faceKeySpace = tmp_face2node.Rep()->getDomainKeySpace() ; pos.allocate(nodes) ; @@ -1534,11 +1557,11 @@ namespace Loci { cr.Rep()->setDomainKeySpace(faceKeySpace) ; entitySet old_nodes = t_pos.domain() ; - redistribute_container(node_ptn,node_ptn_t,nodes,t_pos.Rep(),pos.Rep()) ; + redistribute_container(node_ptn,node_ptn_t,nodes,t_pos.Rep(),pos.Rep(),comm) ; t_pos.allocate(EMPTY) ; - redistribute_container(face_ptn,face_ptn_t,faces,tmp_cr.Rep(),cr.Rep()) ; + redistribute_container(face_ptn,face_ptn_t,faces,tmp_cr.Rep(),cr.Rep(),comm) ; tmp_cr.allocate(EMPTY) ; - redistribute_container(face_ptn,face_ptn_t,faces,tmp_cl.Rep(),cl.Rep()) ; + redistribute_container(face_ptn,face_ptn_t,faces,tmp_cl.Rep(),cl.Rep(),comm) ; tmp_cl.allocate(EMPTY) ; cl.Rep()->setDomainKeySpace(faceKeySpace) ; @@ -1555,12 +1578,12 @@ namespace Loci { } store count_reorder ; count_reorder.allocate(faces) ; - redistribute_container(face_ptn,face_ptn_t,faces,count.Rep(),count_reorder.Rep()) ; + redistribute_container(face_ptn,face_ptn_t,faces,count.Rep(),count_reorder.Rep(),comm) ; face2node.allocate(count_reorder) ; face2node.Rep()->setDomainKeySpace(tmp_face2node.Rep()->getDomainKeySpace()) ; redistribute_container(face_ptn,face_ptn_t,faces,tmp_face2node.Rep(), - face2node.Rep()) ; + face2node.Rep(),comm) ; tmp_face2node.allocate(EMPTY) ; entitySet geom_cells = (cr.image(faces)+cl.image(faces))-bcsurfset ; @@ -1619,28 +1642,28 @@ namespace Loci { using std::cout ; using std::endl ; - vector saddr(MPI_processes) ; - for(int i=0;i saddr(facts.get_comm_size()) ; + for(int i=0;i raddr(MPI_processes) ; + vector raddr(facts.get_comm_size()) ; MPI_Alltoall(&saddr[0],1,MPI_INT, &raddr[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int b = *nodes.begin() ; int sum = 0 ; - for(int i=0;i bcptn = all_collect_vectors(bcsurf_ptn[MPI_rank],MPI_COMM_WORLD) ; - vector bcalloc = all_collect_vectors(bcsurfset,MPI_COMM_WORLD) ; - for(int i=0;i bcptn = all_collect_vectors(bcsurf_ptn[facts.get_comm_rank()],comm) ; + vector bcalloc = all_collect_vectors(bcsurfset,comm) ; + for(int i=0;iimage(f2n->domain())-old_nodes ; entitySet old_dom = orig_cells+old_nodes ; - vector old_ptn = all_collect_vectors(old_dom) ; + vector old_ptn = all_collect_vectors(old_dom, comm) ; { storeRepP PRep = remap.Rep() ; - fill_clone(PRep,out_of_dom,old_ptn) ; + fill_clone(PRep,out_of_dom,old_ptn,comm) ; } MapRepP(face2node.Rep())->compose(remap,faces) ; @@ -1767,45 +1790,50 @@ namespace Loci { void assignOwner(vector > > &scratchPad, vector ptn, - vector &out_ptn) ; + vector &out_ptn, + MPI_Comm comm = MPI_COMM_WORLD) ; void fill_clone_proc(map &mapdata, entitySet &out_of_dom, - std::vector &init_ptn) { + std::vector &init_ptn, + MPI_Comm comm) { + int r = 0, p = 0 ; + MPI_Comm_rank(comm, &r) ; + MPI_Comm_size(comm, &p) ; REPORTMEM() ; - vector recv_req(MPI_processes) ; - for(int i=0;i recv_req(p) ; + for(int i=0;i send_set(MPI_processes) ; - for(int i=0;i send_set(p) ; + for(int i=0;i partitionNodes(vector face_ptn, MapRepP face2node,entitySet old_node_dom) { + vector partitionNodes(vector face_ptn, MapRepP face2node,entitySet old_node_dom, + MPI_Comm comm = MPI_COMM_WORLD) { + int p = 1 ; + MPI_Comm_size(comm, &p) ; // find node_ptn that best matches the face partition. Loop over faces entitySet fdom ; - for(int i=0;i procmap ; procmap.allocate(fdom) ; - for(int i=0;i node_ptn_old = all_collect_vectors(old_node_dom) ; - vector node_ptn(MPI_processes) ; - assignOwner(scratchPad,node_ptn_old,node_ptn) ; + vector node_ptn_old = all_collect_vectors(old_node_dom, comm) ; + vector node_ptn(p) ; + assignOwner(scratchPad,node_ptn_old,node_ptn,comm) ; return node_ptn ; } - vector transposePtn(const vector &ptn) { - vector send_sz(MPI_processes) ; - for(int i=0;i transposePtn(const vector &ptn, + MPI_Comm comm = MPI_COMM_WORLD) { + int p = 1 ; + MPI_Comm_size(comm, &p) ; + vector send_sz(p) ; + for(int i=0;i recv_sz(MPI_processes) ; + vector recv_sz(p) ; MPI_Alltoall(&send_sz[0],1,MPI_INT, &recv_sz[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int size_send = 0 ; int size_recv = 0 ; - for(int i=0;iallocate(new_alloc) ; int *send_store = new int[size_send] ; int *recv_store = new int[size_recv] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + int *send_displacement = new int[p] ; + int *recv_displacement = new int[p] ; send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < p; ++i) { send_displacement[i] = send_displacement[i-1] + send_sz[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_sz[i-1] ; } - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < p; ++i) { for(size_t j=0;j ptn_t(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + vector ptn_t(p) ; + for(int i = 0; i < p; ++i) { for(int j=0;j > > &scratchPad, vector ptn, - vector &out_ptn) { + vector &out_ptn, + MPI_Comm comm) { std::sort(scratchPad.begin(),scratchPad.end()) ; vector > >::iterator i1,i2 ; @@ -2106,14 +2145,17 @@ namespace Loci { scratchPad.erase(i1,scratchPad.end()) ; } - vector > > nsplits(MPI_processes-1) ; - for(int i=1;i > > nsplits(p-1) ; + for(int i=1;i &cell_weights, vector &cell_ptn, vector &face_ptn, - vector &node_ptn) { + vector &node_ptn, + MPI_Comm comm) { - vector tmp(MPI_processes) ; // Initialize partition vectors + int p = 1 ; + MPI_Comm_size(comm,&p) ; + + vector tmp(p) ; // Initialize partition vectors cell_ptn = tmp ; face_ptn = tmp ; node_ptn = tmp ; @@ -2373,7 +2423,7 @@ namespace Loci { Loci::MapRepP(face2node.Rep())->image(ifdom) + pos.domain() ; Loci::storeRepP sp = tmp_pos.Rep() ; vector ptn = local_nodes ; - fill_clone(sp,total_dom,ptn) ; + fill_clone(sp,total_dom,ptn,comm) ; int i=0 ; vector > fcenter(fsz) ; FORALL(ifdom,fc) { @@ -2397,9 +2447,9 @@ namespace Loci { } vector3d pmax = pmaxl, pmin=pminl ; MPI_Allreduce(&(pmaxl.x),&(pmax.x),3,MPI_FLOAT,MPI_MAX, - MPI_COMM_WORLD) ; + comm) ; MPI_Allreduce(&(pminl.x),&(pmin.x),3,MPI_FLOAT,MPI_MIN, - MPI_COMM_WORLD) ; + comm) ; // compute scaling factor so that we can scale the floating // point numbers to cover the integers double s = 4e9/max(max(max(pmax.x-pmin.x,1e-3f),pmax.y-pmin.y), @@ -2425,10 +2475,10 @@ namespace Loci { dstore cell_weights_tmp ; int minc = cdom.Min(); int mincg = minc ; - MPI_Allreduce(&minc,&mincg,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&minc,&mincg,1,MPI_INT,MPI_MIN,comm) ; int maxc = cdom.Max() ; int maxcg = maxc ; - MPI_Allreduce(&maxc,&maxcg,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&maxc,&maxcg,1,MPI_INT,MPI_MAX,comm) ; FORALL(cdom,pi) { cell_weights_tmp[pi] = cell_weights[pi] ; } ENDFORALL ; @@ -2442,7 +2492,7 @@ namespace Loci { vector ptn = local_cells ; // gather weights from other processors so we can use the cl and // cr maps - fill_clone(sp,total_dom,ptn) ; + fill_clone(sp,total_dom,ptn,comm) ; int i = 0; FORALL(ifdom,fc) { // the face weight will be the maximum of the to cell weights @@ -2461,7 +2511,7 @@ namespace Loci { i++ ; } ENDFORALL ; - Loci::parSampleSort(key_list,MPI_COMM_WORLD) ; + Loci::parSampleSort(key_list,comm) ; // Now analyze weights to see if we need to do weighted partitions // If we have a large spread of weights, then divide into large and small @@ -2474,10 +2524,10 @@ namespace Loci { wminl = min(wminl,double(key_list[ii].weight)) ; } double wsum=wsuml, usum=usuml, wmax=wmaxl, wmin=wminl ; - MPI_Allreduce(&wsuml,&wsum,1,MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD) ; - MPI_Allreduce(&usuml,&usum,1,MPI_DOUBLE,MPI_SUM, MPI_COMM_WORLD) ; - MPI_Allreduce(&wmaxl,&wmax,1,MPI_DOUBLE,MPI_MAX, MPI_COMM_WORLD) ; - MPI_Allreduce(&wminl,&wmin,1,MPI_DOUBLE,MPI_MIN, MPI_COMM_WORLD) ; + MPI_Allreduce(&wsuml,&wsum,1,MPI_DOUBLE,MPI_SUM, comm) ; + MPI_Allreduce(&usuml,&usum,1,MPI_DOUBLE,MPI_SUM, comm) ; + MPI_Allreduce(&wmaxl,&wmax,1,MPI_DOUBLE,MPI_MAX, comm) ; + MPI_Allreduce(&wminl,&wmin,1,MPI_DOUBLE,MPI_MIN, comm) ; double wavg = wsum/usum ; @@ -2496,24 +2546,26 @@ namespace Loci { // Partition low processors - vector w_lowv(MPI_processes) ; + vector w_lowv(p) ; MPI_Allgather(&w_low,1,MPI_DOUBLE,&w_lowv[0],1,MPI_DOUBLE, - MPI_COMM_WORLD) ; - vector w_highv(MPI_processes) ; + comm) ; + vector w_highv(p) ; MPI_Allgather(&w_high,1,MPI_DOUBLE,&w_highv[0],1,MPI_DOUBLE, - MPI_COMM_WORLD) ; - vector c_lowv(MPI_processes) ; + comm) ; + vector c_lowv(p) ; MPI_Allgather(&c_low,1,MPI_DOUBLE,&c_lowv[0],1,MPI_DOUBLE, - MPI_COMM_WORLD) ; - vector c_highv(MPI_processes) ; + comm) ; + vector c_highv(p) ; MPI_Allgather(&c_high,1,MPI_DOUBLE,&c_highv[0],1,MPI_DOUBLE, - MPI_COMM_WORLD) ; + comm) ; double proc_sumw_low = 0 ; double tot_sumw_low = 0 ; double proc_sumw_high = 0 ; double tot_sumw_high = 0 ; - for(int i=0;i procs(key_list.size()) ; // processors are assigned for the low and high groups by simple @@ -2532,35 +2584,35 @@ namespace Loci { w_high = proc_sumw_high ; for(size_t ii=0;ii wsump(MPI_processes,0) ; - vector csump(MPI_processes,0) ; + vector wsump(p,0) ; + vector csump(p,0) ; for(size_t ii=0;ii wsumr(MPI_processes,0) ; - MPI_Allreduce(&wsump[0],&wsumr[0],MPI_processes,MPI_DOUBLE, - MPI_SUM,MPI_COMM_WORLD) ; + vector wsumr(p,0) ; + MPI_Allreduce(&wsump[0],&wsumr[0],p,MPI_DOUBLE, + MPI_SUM,comm) ; - vector csumr(MPI_processes,0) ; - MPI_Allreduce(&csump[0],&csumr[0],MPI_processes,MPI_DOUBLE, - MPI_SUM,MPI_COMM_WORLD) ; + vector csumr(p,0) ; + MPI_Allreduce(&csump[0],&csumr[0],p,MPI_DOUBLE, + MPI_SUM,comm) ; debugout << "processor weights =" ; - for(int i=0;i fsplits(MPI_processes) ; - MPI_Allgather(&fsizes,1,MPI_INT,&fsplits[0],1,MPI_INT,MPI_COMM_WORLD) ; + vector fsplits(p) ; + MPI_Allgather(&fsizes,1,MPI_INT,&fsplits[0],1,MPI_INT,comm) ; - for(int i=0;i > splitters(MPI_processes-1) ; - for(int i=0;i > splitters(p-1) ; + for(int i=0;i &ptn, entitySet local_set) { + void randomPartition(vector &ptn, entitySet local_set, + MPI_Comm comm) { + int p = 1 ; + MPI_Comm_size(comm,&p) ; + int r = 0 ; + MPI_Comm_rank(comm,&r) ; + // Sort random keys vector key_list(local_set.size()) ; int i=0; @@ -2632,28 +2690,28 @@ namespace Loci { key_list[i].id = ii ; i++ ; } ENDFORALL ; - balanceDistribution(key_list,MPI_COMM_WORLD) ; - Loci::parSampleSort(key_list,MPI_COMM_WORLD) ; - balanceDistribution(key_list,MPI_COMM_WORLD) ; + balanceDistribution(key_list,comm) ; + Loci::parSampleSort(key_list,comm) ; + balanceDistribution(key_list,comm) ; // compute interior face to processor mapping vector > proc_pairs(key_list.size()) ; for(size_t ii=0;ii(key_list[ii].id,MPI_rank) ; + proc_pairs[ii] = pair(key_list[ii].id,r) ; } // Now we need to sort the data back to the processor that owns // the corresponding face int cmin = local_set.Min() ; - vector csplits(MPI_processes) ; - MPI_Allgather(&cmin,1,MPI_INT,&csplits[0],1,MPI_INT,MPI_COMM_WORLD) ; - vector > splitters(MPI_processes-1) ; - for(int i=0;i csplits(p) ; + MPI_Allgather(&cmin,1,MPI_INT,&csplits[0],1,MPI_INT,comm) ; + vector > splitters(p-1) ; + for(int i=0;i &local_cells, vector &cell_ptn, vector &face_ptn, - vector &node_ptn) { + vector &node_ptn, + MPI_Comm comm) { - vector tmp(MPI_processes) ; // Initialize partition vectors + int p=1 ; + MPI_Comm_size(comm,&p) ; + + vector tmp(p) ; // Initialize partition vectors cell_ptn = tmp ; face_ptn = tmp ; node_ptn = tmp ; - randomPartition(cell_ptn,local_cells[MPI_rank]) ; - randomPartition(face_ptn,local_faces[MPI_rank]) ; - randomPartition(node_ptn,local_nodes[MPI_rank]) ; + int r = 0 ; + MPI_Comm_rank(comm,&r) ; + randomPartition(cell_ptn,local_cells[r],comm) ; + randomPartition(face_ptn,local_faces[r],comm) ; + randomPartition(node_ptn,local_nodes[r],comm) ; } void ORB_Partition_Mesh(const vector &local_nodes, @@ -2682,12 +2746,15 @@ namespace Loci { const store > &pos, const Map &cl, const Map &cr, const multiMap &face2node, - const store &boundary_tags, + const store &boundary_tags, vector &cell_ptn, vector &face_ptn, - vector &node_ptn) { + vector &node_ptn, + MPI_Comm comm) { - vector tmp(MPI_processes) ; // Initialize partition vectors + int p = 1 ; + MPI_Comm_size(comm,&p) ; + vector tmp(p) ; // Initialize partition vectors cell_ptn = tmp ; face_ptn = tmp ; node_ptn = tmp ; @@ -2704,7 +2771,7 @@ namespace Loci { Loci::MapRepP(face2node.Rep())->image(fdom) + pos.domain() ; Loci::storeRepP sp = tmp_pos.Rep() ; vector ptn = local_nodes ; - fill_clone(sp,total_dom,ptn) ; + fill_clone(sp,total_dom,ptn,comm) ; vector > fcenter(fdom.size()) ; int i=0 ; FORALL(fdom,fc) { @@ -2719,7 +2786,8 @@ namespace Loci { // perform ORB partition of faces vector fprocmap ; - ORBPartition(fcenter,fprocmap,MPI_COMM_WORLD) ; + ORBPartition(fcenter,fprocmap, + comm) ; i=0 ; // Create face_ptn ; FORALL(fdom,fc) { @@ -2756,6 +2824,7 @@ namespace Loci { bool readFVMGrid(fact_db &facts, string filename, storeRepP cellwts) { + MPI_Comm comm = facts.get_comm() ; double t1 = MPI_Wtime() ; REPORTMEM() ; @@ -2787,7 +2856,7 @@ namespace Loci { if(useVOG) { if(!readGridVOG(local_nodes, local_faces, local_cells, t_pos, tmp_cl, tmp_cr, tmp_face2node, tmp_boundary_names, tmp_boundary_tags, - volTags, max_alloc, filename)) { + volTags, max_alloc, filename, comm)) { return false; } } else { @@ -2803,7 +2872,7 @@ namespace Loci { entitySet global_boundary_cells = tmp_boundary_tags.domain() ; REPORTMEM() ; - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { int npnts = local_nodes[0].size(); int nfaces = local_faces[0].size(); @@ -2855,25 +2924,25 @@ namespace Loci { // check if the file exists store cell_weights ; int file_exists = 1 ; - if(Loci::MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { struct stat buf ; if(stat(cell_weight_file.c_str(),&buf) == -1 || !S_ISREG(buf.st_mode)) { file_exists = 0 ; } } - MPI_Bcast(&file_exists,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&file_exists,1,MPI_INT,0,comm) ; if(file_exists == 1) { debugout << "reading cell weights" << endl ; - if(Loci::MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { std::cout << "Space Filling Curve partition reading additional " << "cell weights from: " << cell_weight_file << std::endl ; } // create a hdf5 handle hid_t file_id = Loci::hdf5OpenFile(cell_weight_file.c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT) ; + H5F_ACC_RDONLY, H5P_DEFAULT, comm) ; if(file_id < 0) { std::cerr << "...file reading failed..., Aborting" << std::endl ; Loci::Abort() ; @@ -2881,17 +2950,17 @@ namespace Loci { // read readContainerRAW(file_id,"cellweights", cell_weights.Rep(), - MPI_COMM_WORLD) ; - Loci::hdf5CloseFile(file_id) ; + comm) ; + Loci::hdf5CloseFile(file_id, comm) ; cellwts = cell_weights.Rep() ; } } partitionerSelector partitioner_type = partitionerMethod ; if(partitioner_type == GRAPH) { - int lcpp = local_cells[MPI_rank].size() ; + int lcpp = local_cells[facts.get_comm_rank()].size() ; int mincpp = lcpp ; - MPI_Allreduce(&lcpp,&mincpp,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&lcpp,&mincpp,1,MPI_INT,MPI_MIN,comm) ; if(mincpp < metis_cpp_threshold) { partitioner_type = SFC ; // Space filling curve partitioner debugout << "switching from metis to space filling curve partitioner" @@ -2907,16 +2976,16 @@ namespace Loci { { debugout << "Using ORB partition" << endl ; ORB_Partition_Mesh(local_nodes, local_faces, local_cells, - t_pos, tmp_cl, tmp_cr, tmp_face2node, - tmp_boundary_tags, - cell_ptn,face_ptn,node_ptn) ; + t_pos, tmp_cl, tmp_cr, tmp_face2node, + tmp_boundary_tags, + cell_ptn,face_ptn,node_ptn,comm) ; } break ; case RANDOM: { debugout << "Using random partition" << endl ; - RND_Partition_Mesh(local_nodes, local_faces, local_cells, - cell_ptn,face_ptn,node_ptn) ; + RND_Partition_Mesh(local_nodes, local_faces, local_cells, + cell_ptn,face_ptn,node_ptn,comm) ; } break ; case SFC: @@ -2926,40 +2995,41 @@ namespace Loci { // read in additional vertex weights if any if(cellwts != 0) { //debugout << "getting cell_weights_store" << endl ; - entitySet dom = local_cells[MPI_rank]; + entitySet dom = local_cells[facts.get_comm_rank()]; cell_weights.allocate(dom); - redistribute_cell_weight(cellwts, cell_weights.Rep()); + redistribute_cell_weight(cellwts, cell_weights.Rep(),comm); - int offset = local_cells[Loci::MPI_rank].Min() + int offset = local_cells[facts.get_comm_rank()].Min() - cell_weights.domain().Min() ; storeRepP sp = cell_weights.Rep(); sp->shift(offset) ; - if(cell_weights.domain() != local_cells[Loci::MPI_rank]) { + if(cell_weights.domain() != local_cells[facts.get_comm_rank()]) { cerr << "cell_weights=" << cell_weights.domain() - << ", local_cells = " << local_cells[Loci::MPI_rank] << endl ; + << ", local_cells = " << local_cells[facts.get_comm_rank()] << endl ; cerr << "cell weights partition inconsistent!" << endl ; Loci::Abort() ; } } SFC_Partition_Mesh(local_nodes, local_faces, local_cells, t_pos, - tmp_cl, tmp_cr, tmp_face2node, tmp_boundary_tags, - cell_weights, cell_ptn, face_ptn, node_ptn) ; + tmp_cl, tmp_cr, tmp_face2node, tmp_boundary_tags, + cell_weights, cell_ptn, face_ptn, node_ptn, + facts.get_comm()) ; } break ; case SIMPLE: // Simple partition { debugout << "Using simple partition method" << endl ; - cell_ptn = vector(MPI_processes) ; - cell_ptn[MPI_rank] = local_cells[MPI_rank] ; + cell_ptn = vector(facts.get_comm_size()) ; + cell_ptn[facts.get_comm_rank()] = local_cells[facts.get_comm_rank()] ; REPORTMEM() ; - face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags) ; + face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags,comm) ; REPORTMEM() ; node_ptn = partitionNodes(face_ptn, MapRepP(tmp_face2node.Rep()), - t_pos.domain()) ; + t_pos.domain(),comm) ; } break ; #ifdef LOCI_USE_METIS @@ -2970,13 +3040,13 @@ namespace Loci { #else debugout << "Using METIS graph partitioner" << endl ; #endif - cell_ptn = newMetisPartitionOfCells(local_cells,tmp_cl,tmp_cr,tmp_boundary_tags,cellwts) ; + cell_ptn = newMetisPartitionOfCells(local_cells,tmp_cl,tmp_cr,tmp_boundary_tags,cellwts,facts.get_comm()) ; REPORTMEM() ; - face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags) ; + face_ptn = partitionFaces(cell_ptn,tmp_cl,tmp_cr,tmp_boundary_tags,comm) ; REPORTMEM() ; node_ptn = partitionNodes(face_ptn, MapRepP(tmp_face2node.Rep()), - t_pos.domain()) ; + t_pos.domain(),comm) ; } break ; #endif @@ -2986,13 +3056,14 @@ namespace Loci { store cell_weights ; SFC_Partition_Mesh(local_nodes, local_faces, local_cells, t_pos, - tmp_cl, tmp_cr, tmp_face2node, tmp_boundary_tags, - cell_weights, cell_ptn, face_ptn, node_ptn) ; + tmp_cl, tmp_cr, tmp_face2node, tmp_boundary_tags, + cell_weights, cell_ptn, face_ptn, node_ptn, + facts.get_comm()) ; } break ; } - vector bcsurf_ptn(MPI_processes) ; + vector bcsurf_ptn(facts.get_comm_size()) ; entitySet refset = tmp_boundary_tags.domain() ; // round robin allocate boundary faces @@ -3000,29 +3071,29 @@ namespace Loci { int cnt = 0 ; //int refsetsz = refset.size() ; FORALL(refset,ii) { - if(cnt == MPI_rank) { + if(cnt == facts.get_comm_rank()) { bcsurf_ptn[cnt] += ii ; } cnt++ ; - if(cnt == MPI_processes) { + if(cnt == facts.get_comm_size()) { cnt = 0 ; } } ENDFORALL ; REPORTMEM() ; - vector cell_ptn_t = transposePtn(cell_ptn) ; - vector face_ptn_t = transposePtn(face_ptn) ; - vector node_ptn_t = transposePtn(node_ptn) ; + vector cell_ptn_t = transposePtn(cell_ptn,comm) ; + vector face_ptn_t = transposePtn(face_ptn,comm) ; + vector node_ptn_t = transposePtn(node_ptn,comm) ; int newnodes = 0 ; - for(int p=0;p node_alloc(newnodes) ; int i=0; - for(int p=0;p face_alloc(newfaces) ; i = 0 ; - for(int p=0;p cell_alloc(newcells) ; i = 0 ; - for(int p=0;p cell_weights ; - if(Loci::MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { std::cout << "setupFVMGridWithWeightInFile(): reading cell weights from: " << weightfile << std::endl ; } // create a hdf5 handle hid_t file_id = Loci::hdf5OpenFile(weightfile.c_str(), - H5F_ACC_RDONLY, H5P_DEFAULT) ; + H5F_ACC_RDONLY, H5P_DEFAULT, comm) ; if(file_id < 0) { std::cerr << "...file reading failed..., Aborting" << std::endl ; Loci::Abort() ; } readContainerRAW(file_id,"cellweights", cell_weights.Rep(), - MPI_COMM_WORLD) ; + comm) ; - Loci::hdf5CloseFile(file_id) ; + Loci::hdf5CloseFile(file_id, comm) ; cellwts = cell_weights.Rep() ; } diff --git a/src/System/FVMGridWriter.cc b/src/System/FVMGridWriter.cc index 0785a2b7..7b010b47 100644 --- a/src/System/FVMGridWriter.cc +++ b/src/System/FVMGridWriter.cc @@ -222,13 +222,16 @@ namespace Loci { - void writeVOGSurf(hid_t file_id, vector > surface_ids) { + void writeVOGSurf(hid_t file_id, vector > surface_ids, + MPI_Comm comm) { /* parallel io included, all attribute and group creations need to be called collectively by all ranks, including H5Awrite */ + int r = 0; + MPI_Comm_rank(comm,&r) ; hid_t group_id = 0 ; if(surface_ids.size() == 0) return; - if(MPI_rank == 0 || use_parallel_io) { + if(r == 0 || use_parallel_io) { vector > surface_ids_mod= surface_ids ; for(size_t i=0;i >& volTags){ - if(MPI_rank == 0 || (use_parallel_io && output_fid > 0)){ + void writeVOGTag(hid_t output_fid, + vector >& volTags, + MPI_Comm comm){ + int r = 0; + MPI_Comm_rank(comm,&r) ; + if(r == 0 || (use_parallel_io && output_fid > 0)){ hid_t cell_info = H5Gcreate(output_fid,"cell_info", H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; @@ -291,7 +298,8 @@ namespace Loci { H5Awrite(att_id,H5T_NATIVE_INT,&num) ; H5Aclose(att_id) ; - Loci::HDF5_WriteDomain(vol_id,volTags[i].second, MPI_COMM_WORLD) ; + Loci::HDF5_WriteDomain(vol_id,volTags[i].second, + comm) ; H5Gclose(vol_id) ; @@ -300,9 +308,14 @@ namespace Loci { } } - void writeVOGNode(hid_t file_id, store > &pos) {//parallel io included + void writeVOGNode(hid_t file_id, store > &pos, + MPI_Comm comm) + { //parallel io included + + int r = 0 ; + MPI_Comm_size(comm,&r) ; hid_t group_id = 0 ; - if(MPI_rank == 0 || use_parallel_io){ + if(r == 0 || use_parallel_io){ group_id = H5Gcreate(file_id,"node_info",H5P_DEFAULT, H5P_DEFAULT,H5P_DEFAULT) ; @@ -319,20 +332,20 @@ namespace Loci { // if(use_parallel_io) // writeUnorderedVectorP(group_id,"positions",vpos) ; // else - writeUnorderedVector(group_id,"positions",vpos) ; + writeUnorderedVector(group_id,"positions",vpos, comm) ; - if(MPI_rank == 0 || use_parallel_io) H5Gclose(group_id) ; + if(r == 0 || use_parallel_io) H5Gclose(group_id) ; long long local_num_nodes = pos.domain().size() ; long long num_nodes = 0 ; MPI_Allreduce(&local_num_nodes,&num_nodes,1,MPI_LONG_LONG_INT, - MPI_SUM,MPI_COMM_WORLD) ; + MPI_SUM, comm) ; - if(MPI_rank == 0 || use_parallel_io) { + if(r == 0 || use_parallel_io) { group_id = H5Gcreate(file_id,"file_info", H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; - if(MPI_rank == 0) cerr << "num_nodes = " << num_nodes << endl ; + if(r == 0) cerr << "num_nodes = " << num_nodes << endl ; hsize_t dims = 1 ; hid_t dataspace_id = H5Screate_simple(1,&dims,NULL) ; @@ -346,11 +359,12 @@ namespace Loci { } } - void writeVOGFace(hid_t file_id, Map &cl, Map &cr, multiMap &face2node) { + void writeVOGFace(hid_t file_id, Map &cl, Map &cr, multiMap &face2node, + MPI_Comm comm) { // Compute cell set entitySet tmp_cells = cl.image(cl.domain())+cr.image(cr.domain()) ; entitySet loc_geom_cells = tmp_cells & interval(0,Loci::UNIVERSE_MAX) ; - entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells) ; + entitySet geom_cells = Loci::all_collect_entitySet(loc_geom_cells,comm) ; Map tmp_cl, tmp_cr; multiMap tmp_face2node; @@ -364,13 +378,15 @@ namespace Loci { // Reduce these variables MPI_Allreduce(&local_num_faces,&num_faces,1,MPI_LONG_LONG_INT, - MPI_SUM,MPI_COMM_WORLD) ; + MPI_SUM, comm) ; hid_t group_id = 0 ; - if(MPI_rank == 0 || use_parallel_io) { + int r = 0 ; + MPI_Comm_rank(comm,&r) ; + if(r == 0 || use_parallel_io) { group_id = H5Gopen(file_id,"file_info",H5P_DEFAULT) ; - if(MPI_rank == 0)cerr << "num_cells = " << num_cells << endl + if(r == 0)cerr << "num_cells = " << num_cells << endl << "num_faces = " << num_faces << endl ; hsize_t dims = 1 ; @@ -450,17 +466,20 @@ namespace Loci { } - Loci::writeUnorderedVector(group_id,"cluster_sizes",cluster_sizes) ; - Loci::writeUnorderedVector(group_id,"cluster_info",cluster_info) ; + Loci::writeUnorderedVector(group_id,"cluster_sizes",cluster_sizes, comm) ; + Loci::writeUnorderedVector(group_id,"cluster_info",cluster_info, comm) ; - if(MPI_rank == 0 || use_parallel_io) { + if(r == 0 || use_parallel_io) { H5Gclose(group_id) ; } } - void writeVOGClose(hid_t file_id) {//parallel io included - if(MPI_rank == 0 || use_parallel_io) H5Fclose(file_id) ; + void writeVOGClose(hid_t file_id, + MPI_Comm &comm) {//parallel io included + int r = 0 ; + MPI_Comm_rank(comm,&r) ; + if(r == 0 || use_parallel_io) H5Fclose(file_id) ; } @@ -468,30 +487,32 @@ namespace Loci { void writeVOG(string filename,store > &pos, Map &cl, Map &cr, multiMap &face2node, - vector > surface_ids) { + vector > surface_ids, + MPI_Comm comm) { // write grid file hid_t file_id = 0; - file_id = writeVOGOpen(filename) ; + file_id = writeVOGOpen(filename,comm) ; - writeVOGSurf(file_id,surface_ids) ; - writeVOGNode(file_id,pos) ; - writeVOGFace(file_id,cl,cr,face2node) ; - writeVOGClose(file_id) ; //parallel io included + writeVOGSurf(file_id,surface_ids,comm) ; + writeVOGNode(file_id,pos,comm) ; + writeVOGFace(file_id,cl,cr,face2node,comm) ; + writeVOGClose(file_id,comm) ; //parallel io included } void writeVOG(string filename,store > &pos, Map &cl, Map &cr, multiMap &face2node, vector >& surface_ids, - vector >& volTags ) { + vector >& volTags, + MPI_Comm comm) { // write grid file - hid_t file_id = writeVOGOpen(filename) ; + hid_t file_id = writeVOGOpen(filename,comm) ; - writeVOGTag(file_id, volTags) ; - writeVOGSurf(file_id,surface_ids) ; - writeVOGNode(file_id,pos) ; - writeVOGFace(file_id,cl,cr,face2node) ; - writeVOGClose(file_id) ; //parallel io included + writeVOGTag(file_id, volTags,comm) ; + writeVOGSurf(file_id,surface_ids,comm) ; + writeVOGNode(file_id,pos,comm) ; + writeVOGFace(file_id,cl,cr,face2node,comm) ; + writeVOGClose(file_id,comm) ; //parallel io included } diff --git a/src/System/FVMStuff.cc b/src/System/FVMStuff.cc index b2553244..34378e62 100644 --- a/src/System/FVMStuff.cc +++ b/src/System/FVMStuff.cc @@ -56,7 +56,7 @@ namespace Loci{ /// numbering. storeRepP get_node_remap(fact_db &facts, entitySet nodes, int keyspace) { REPORTMEM() ; - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { int minNode = nodes.Min() ; Map nm ; @@ -81,13 +81,15 @@ namespace Loci{ newnum.allocate(nodes) ; // Expand g2f to include clone regions - entitySet out_of_dom = gnodes - init_ptn[MPI_rank] ; - g2f.setRep(MapRepP(g2f.Rep())->expand(out_of_dom, init_ptn)) ; + entitySet out_of_dom = gnodes - init_ptn[facts.get_comm_rank()] ; + g2f.setRep(MapRepP(g2f.Rep())->expand(out_of_dom, init_ptn, + facts.get_comm())) ; entitySet fnodes = g2f.image(gnodes) ; int minNode_local = fnodes.Min() ; int minNode = minNode_local ; - MPI_Allreduce(&minNode_local,&minNode,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&minNode_local, &minNode, 1, MPI_INT, + MPI_MIN, facts.get_comm()) ; FORALL(nodes,i) { newnum[i] = g2f[l2g[i]]-minNode+1 ; @@ -103,7 +105,7 @@ namespace Loci{ /// @param[in] facts Fact database. /// @param[in] nodes The local store domain that needs to be output. storeRepP get_output_node_remap(fact_db &facts, entitySet nodes) { - int p = MPI_processes ; + int p = facts.get_comm_size() ; if(p == 1) { int index = 1; @@ -143,7 +145,7 @@ namespace Loci{ int gsz = gnodes.size() ; int color = (gsz>0)?1:0 ; MPI_Comm groupcomm ; - MPI_Comm_split(MPI_COMM_WORLD,color,p,&groupcomm) ; + MPI_Comm_split(facts.get_comm(), color, p, &groupcomm) ; if(color==0) { MPI_Comm_free(&groupcomm) ; #ifdef VERBOSE @@ -728,32 +730,33 @@ namespace Loci{ // write grid topology file hid_t file_id = 0, group_id = 0 ; - file_id=writeVOGOpen(filename) ; - if(use_parallel_io ||MPI_rank == 0 ) { + MPI_Comm comm = facts.get_comm() ; + file_id=writeVOGOpen(filename,comm) ; + if(use_parallel_io ||facts.get_comm_rank() == 0 ) { group_id = H5Gcreate(file_id, "elements", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) ; } - writeUnorderedVector(group_id, "tetrahedra",tets) ; - writeUnorderedVector(group_id, "tetrahedra_ids",tets_ids) ; + writeUnorderedVector(group_id, "tetrahedra",tets, facts.get_comm()) ; + writeUnorderedVector(group_id, "tetrahedra_ids",tets_ids, facts.get_comm()) ; - writeUnorderedVector(group_id, "hexahedra",hexs) ; - writeUnorderedVector(group_id, "hexahedra_ids",hexs_ids) ; + writeUnorderedVector(group_id, "hexahedra",hexs, facts.get_comm()) ; + writeUnorderedVector(group_id, "hexahedra_ids",hexs_ids, facts.get_comm()) ; - writeUnorderedVector(group_id, "prism",prsm) ; - writeUnorderedVector(group_id, "prism_ids",prsm_ids) ; + writeUnorderedVector(group_id, "prism",prsm, facts.get_comm()) ; + writeUnorderedVector(group_id, "prism_ids",prsm_ids, facts.get_comm()) ; - writeUnorderedVector(group_id, "pyramid",pyrm) ; - writeUnorderedVector(group_id, "pyramid_ids",pyrm_ids) ; + writeUnorderedVector(group_id, "pyramid",pyrm, facts.get_comm()) ; + writeUnorderedVector(group_id, "pyramid_ids",pyrm_ids, facts.get_comm()) ; - writeUnorderedVector(group_id, "GeneralCellNfaces",generalCellNfaces) ; - writeUnorderedVector(group_id, "GeneralCellNsides",generalCellNsides) ; - writeUnorderedVector(group_id, "GeneralCellNodes", generalCellNodes) ; - writeUnorderedVector(group_id, "GeneralCell_ids", generalCell_ids) ; + writeUnorderedVector(group_id, "GeneralCellNfaces",generalCellNfaces, facts.get_comm()) ; + writeUnorderedVector(group_id, "GeneralCellNsides",generalCellNsides, facts.get_comm()) ; + writeUnorderedVector(group_id, "GeneralCellNodes", generalCellNodes, facts.get_comm()) ; + writeUnorderedVector(group_id, "GeneralCell_ids", generalCell_ids, facts.get_comm()) ; - if(use_parallel_io || MPI_rank == 0) { + if(use_parallel_io || facts.get_comm_rank() == 0) { H5Gclose(group_id) ; group_id = H5Gcreate(file_id, "boundaries", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) ; @@ -762,7 +765,7 @@ namespace Loci{ const_store boundary_names(bnamesRep) ; entitySet boundaries = boundary_names.domain() ; - if(MPI_processes > 1) { + if(facts.get_comm_size() > 1) { entitySet local_boundaries ; int bkeyspace = bnamesRep->getDomainKeySpace() ; std::vector init_ptn = facts.get_init_ptn(bkeyspace) ; @@ -770,7 +773,7 @@ namespace Loci{ fact_db::distribute_infoP df = facts.get_distribute_info() ; l2g = df->l2g.Rep() ; FORALL(boundaries,bb) { - if(init_ptn[MPI_rank].inSet(l2g[bb])) { + if(init_ptn[facts.get_comm_rank()].inSet(l2g[bb])) { local_boundaries += bb ; } } ENDFORALL ; @@ -782,7 +785,7 @@ namespace Loci{ } ENDFORALL ; - bnames = MPIConcatStrings(bnames,MPI_COMM_WORLD) ; + bnames = MPIConcatStrings(bnames, facts.get_comm()) ; vector bnamelist ; for(size_t i=0;iimage(localCells)) & ref.domain() ; Map l2f ; - if(MPI_processes > 1) { + if(facts.get_comm_size() > 1) { fact_db::distribute_infoP df = facts.get_distribute_info() ; - int kd = getKeyDomain(fset, df, MPI_COMM_WORLD) ; + int kd = getKeyDomain(fset, df, facts.get_comm()) ; if(kd < 0) { cerr << "gridTopology, boundary faces not in single keyspace!" @@ -826,7 +829,7 @@ namespace Loci{ for(size_t i=0;i boundary_names(bnamesRep) ; entitySet boundaries = boundary_names.domain() ; - if(MPI_processes > 1) { + if(facts.get_comm_size() > 1) { entitySet local_boundaries ; int bkeyspace = bnamesRep->getDomainKeySpace() ; std::vector init_ptn = facts.get_init_ptn(bkeyspace) ; @@ -933,7 +936,7 @@ namespace Loci{ fact_db::distribute_infoP df = facts.get_distribute_info() ; l2g = df->l2g.Rep() ; FORALL(boundaries,bb) { - if(init_ptn[MPI_rank].inSet(l2g[bb])) { + if(init_ptn[facts.get_comm_rank()].inSet(l2g[bb])) { local_boundaries += bb ; } } ENDFORALL ; @@ -946,7 +949,7 @@ namespace Loci{ } ENDFORALL ; - bnames = MPIConcatStrings(bnames,MPI_COMM_WORLD) ; + bnames = MPIConcatStrings(bnames, facts.get_comm()) ; vector bnamelist ; for(size_t i=0;iimage(bfaces) ; // get the nodes that belong to this processor - if(MPI_processes > 1) { + if(facts.get_comm_size() > 1) { entitySet dom = nodes_local ; fact_db::distribute_infoP dist = facts.get_distribute_info() ; entitySet my_entities = dist->my_entities ; @@ -1091,7 +1097,8 @@ namespace Loci{ faceorder.allocate(bfaces) ; int sz = bfaces.size() ; int off = 0 ; - MPI_Scan(&sz,&off,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Scan(&sz, &off, 1, MPI_INT, MPI_SUM, + facts.get_comm()) ; off -= sz ; FORALL(bfaces,fc) { faceorder[fc] = off ; @@ -1153,14 +1160,14 @@ namespace Loci{ #endif //write out vectors - writeUnorderedVector(file_id,"triangles",Trias) ; - writeUnorderedVector(file_id,"triangles_ord",tria_ids) ; - writeUnorderedVector(file_id,"quads",Quads) ; - writeUnorderedVector(file_id,"quads_ord",quad_ids) ; + writeUnorderedVector(file_id,"triangles",Trias, facts.get_comm()) ; + writeUnorderedVector(file_id,"triangles_ord",tria_ids, facts.get_comm()) ; + writeUnorderedVector(file_id,"quads",Quads, facts.get_comm()) ; + writeUnorderedVector(file_id,"quads_ord",quad_ids, facts.get_comm()) ; - writeUnorderedVector(file_id,"nside_sizes",nsizes) ; - writeUnorderedVector(file_id,"nside_nodes",nsidenodes) ; - writeUnorderedVector(file_id,"nside_ord",genc_ids) ; + writeUnorderedVector(file_id,"nside_sizes",nsizes, facts.get_comm()) ; + writeUnorderedVector(file_id,"nside_nodes",nsidenodes, facts.get_comm()) ; + writeUnorderedVector(file_id,"nside_ord",genc_ids, facts.get_comm()) ; #ifdef VERBOSE debugout << "time to write unordered vectors = " << s.stop() << endl ; @@ -1501,7 +1508,7 @@ namespace Loci{ nsizes[i] = cp.faceLoops[i].size() ; } - writeUnorderedVector(bc_id,"nside_sizes",nsizes) ; + writeUnorderedVector(bc_id,"nside_sizes",nsizes, facts.get_comm()) ; //write out face nodes vector nsidenodes ; @@ -1514,7 +1521,7 @@ namespace Loci{ } } - writeUnorderedVector(bc_id,"nside_nodes",nsidenodes) ; + writeUnorderedVector(bc_id,"nside_nodes",nsidenodes, facts.get_comm()) ; #ifdef VERBOSE debugout << "time to write cut plane topology=" << s.stop() << endl ; @@ -1580,8 +1587,8 @@ namespace Loci{ if(facts.is_distributed_start()) { int pk = pos.Rep()->getDomainKeySpace() ; init_ptn = facts.get_init_ptn(pk) ; - if(GLOBAL_OR(tmp_out)) { - fill_clone(sp, out_of_dom, init_ptn) ; + if(GLOBAL_OR(tmp_out, facts.get_comm())) { + fill_clone(sp, out_of_dom, init_ptn,facts.get_comm()) ; } } @@ -1659,10 +1666,12 @@ namespace Loci{ vector p1closest(p1.size()) ; - parallelNearestNeighbors(p2,p2id,p1,p1closest,MPI_COMM_WORLD) ; + parallelNearestNeighbors(p2, p2id, p1, p1closest, + facts.get_comm()) ; vector p2closest(p2.size()) ; - parallelNearestNeighbors(p1,p1id,p2,p2closest,MPI_COMM_WORLD) ; + parallelNearestNeighbors(p1, p1id, p2, p2closest, + facts.get_comm()) ; for(size_t i=0;i init_ptn ; if(facts.is_distributed_start()) { init_ptn = facts.get_init_ptn(fk) ; - fill_clone(sp, p1map, init_ptn) ; + fill_clone(sp, p1map, init_ptn,facts.get_comm()) ; } bool periodic_problem = false ; for(size_t i=0;igetDomainKeySpace() ; @@ -1894,15 +1903,15 @@ namespace Loci{ // Create constraints for each std::map::const_iterator mi ; for(mi=BCsets.begin();mi!=BCsets.end();++mi) { - if(GLOBAL_OR(mi->second.size())) { + if(GLOBAL_OR(mi->second.size(), facts.get_comm())) { constraint bc_constraint ; bc_constraint = mi->second ; std::string constraint_name = mi->first + std::string("_BC") ; bc_constraint.Rep()->setDomainKeySpace(fk) ; facts.create_fact(constraint_name,bc_constraint) ; - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { std::cout << constraint_name << ' ' << mi->second << endl ; - } else if(MPI_rank == 0) { + } else if(facts.get_comm_rank() == 0) { std::cout << "setting boundary condition " << constraint_name << endl ; } @@ -1969,10 +1978,12 @@ namespace Loci{ int p2inp = p2.bset.size() ; int p1size ; int p2size ; - MPI_Allreduce(&p1inp,&p1size,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; - MPI_Allreduce(&p2inp,&p2size,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&p1inp, &p1size, 1, MPI_INT, + MPI_SUM, facts.get_comm()) ; + MPI_Allreduce(&p2inp, &p2size, 1, MPI_INT, + MPI_SUM, facts.get_comm()) ; if(p1size != p2size) { - if(MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { cerr << "periodic boundaries " << p1.name << " do not match in number of faces" << endl ; cerr << "master has " << p1size << " faces and slave has " @@ -2031,6 +2042,7 @@ namespace Loci{ /// @param[out] normal Face normals. void getFaceCenter(fact_db &facts, store > &fcenter, store &area, store > &normal) { + MPI_Comm comm = facts.get_comm() ; // Compute face centers store > pos ; pos = facts.get_variable("pos") ; @@ -2049,7 +2061,7 @@ namespace Loci{ // gather nodal data needed for the computation. std::map g2l ; getLocalContextMap(g2l,node_access) ; - gatherData(posdata,pos,node_access,node_ptn) ; + gatherData(posdata,pos,node_access,node_ptn,comm) ; fcenter.allocate(fdom) ; area.allocate(fdom) ; normal.allocate(fdom) ; @@ -2135,7 +2147,7 @@ namespace Loci{ /// @param[in] area Face areas. void getCellCenter(fact_db &facts, store > &ccenter, store > &fcenter, store &area) { - + MPI_Comm comm = facts.get_comm() ; multiMap upper,lower,boundary_map ; upper = facts.get_variable("upper") ; lower = facts.get_variable("lower") ; @@ -2153,8 +2165,8 @@ namespace Loci{ std::vector face_ptn = facts.get_init_ptn(fkeyspace) ; std::map g2l ; getLocalContextMap(g2l,faceimage) ; - gatherData(fcenterdata,fcenter,faceimage,face_ptn) ; - gatherData(areadata,area,faceimage,face_ptn) ; + gatherData(fcenterdata,fcenter,faceimage,face_ptn,comm) ; + gatherData(areadata,area,faceimage,face_ptn,comm) ; // compute wireframe centroid FORALL(cells,cc) { vector3d csum = vector3d (0,0,0) ; @@ -2197,6 +2209,7 @@ namespace Loci{ } void get_full_cellStencil(multiMap &cellStencil,fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; using std::vector ; using std::pair ; Map cl,cr ; @@ -2215,14 +2228,14 @@ namespace Loci{ // loop before, so gather with clone cells int gkeyspace = geom_cells_c.getDomainKeySpace() ; std::vector ptn = facts.get_init_ptn(gkeyspace) ; - geom_cells = distribute_entitySet(geom_cells,ptn) ; + geom_cells = distribute_entitySet(geom_cells,ptn,comm) ; entitySet geom_cell_expand = - dist_expand_entitySet(geom_cells,cellmask,ptn) ; + dist_expand_entitySet(geom_cells,cellmask,ptn,comm) ; Loci::protoMap f2cell ; #ifdef DEBUG // Check to see if there are any ORPHAN cells in geom_cells. - entitySet accessedSet = distribute_entitySet(cellmask,ptn) ; + entitySet accessedSet = distribute_entitySet(cellmask,ptn,comm) ; WARN(GLOBAL_OR((geom_cells-accessedSet) != EMPTY)) #endif // Get mapping from face to geometric cells @@ -2240,22 +2253,22 @@ namespace Loci{ // Equijoin on first of pairs to get node to neighboring cell mapping // This will give us a mapping from nodes to neighboring cells Loci::protoMap n2c ; - Loci::equiJoinFF(f2node,f2cell,n2c) ; + Loci::equiJoinFF(f2node,f2cell,n2c,comm) ; // In case there are processors that have no n2c's allocated to them // re-balance map distribution - Loci::balanceDistribution(n2c,MPI_COMM_WORLD) ; + Loci::balanceDistribution(n2c, facts.get_comm()) ; // Equijoin node2cell with itself to get cell to cell map of // all cells that share one or more nodes Loci::protoMap n2cc = n2c ; Loci::protoMap c2c ; - Loci::equiJoinFF(n2c,n2cc,c2c) ; + Loci::equiJoinFF(n2c,n2cc,c2c,comm) ; // Remove self references Loci::removeIdentity(c2c) ; // Create cell stencil map from protoMap - distributed_inverseMap(cellStencil,c2c,geom_cells,geom_cells,ptn) ; + distributed_inverseMap(cellStencil,c2c,geom_cells,geom_cells,ptn,comm) ; #ifdef DEBUG entitySet degen_cells ; @@ -2266,11 +2279,11 @@ namespace Loci{ degen_cells += ii ; } } ENDFORALL ; - entitySet tot_degen = all_collect_entitySet(degen_cells) ; + entitySet tot_degen = all_collect_entitySet(degen_cells,facts.get_comm()) ; WARN(tot_degen!=EMPTY) ; if(tot_degen != EMPTY) { debugout << "tot_degen=" << tot_degen << endl ; - if(MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "tot_degen = " << tot_degen << endl ; } FORALL(faces,fc) { @@ -2311,7 +2324,7 @@ namespace Loci{ } void get_stable_cellStencil(multiMap &cellStencilFiltered, fact_db & facts) { - + MPI_Comm comm = facts.get_comm() ; // get full stencil multiMap cellStencil ; get_full_cellStencil(cellStencil,facts) ; @@ -2339,8 +2352,8 @@ namespace Loci{ getLocalContextMap(fg2l,faceimage) ; std::vector > fcenterdata ; std::vector > fnormaldata ; - gatherData(fcenterdata,fcenter,faceimage,face_ptn) ; - gatherData(fnormaldata,normal,faceimage,face_ptn) ; + gatherData(fcenterdata,fcenter,faceimage,face_ptn,comm) ; + gatherData(fnormaldata,normal,faceimage,face_ptn,comm) ; int ckeyspace = upper.getDomainKeySpace() ; std::vector cell_ptn = facts.get_init_ptn(ckeyspace) ; @@ -2350,7 +2363,7 @@ namespace Loci{ std::vector > ccenterdata ; std::map cg2l ; getLocalContextMap(cg2l,cellImage) ; - gatherData(ccenterdata,ccenter,cellImage,cell_ptn) ; + gatherData(ccenterdata,ccenter,cellImage,cell_ptn,comm) ; store sizes ; sizes.allocate(cells) ; @@ -2439,6 +2452,7 @@ namespace Loci{ } void get_neigh_cellStencil(multiMap &cellStencil,fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; using std::vector ; using std::pair ; Map cl,cr ; @@ -2455,14 +2469,14 @@ namespace Loci{ int gkeyspace = geom_cells_c.getDomainKeySpace() ; std::vector ptn = facts.get_init_ptn(gkeyspace) ; - geom_cells = distribute_entitySet(geom_cells,ptn) ; + geom_cells = distribute_entitySet(geom_cells,ptn,comm) ; entitySet geom_cell_expand = - dist_expand_entitySet(geom_cells,cellmask,ptn) ; + dist_expand_entitySet(geom_cells,cellmask,ptn,comm) ; Loci::protoMap f2cell ; #ifdef DEBUG // Check to see if there are any ORPHAN cells in geom_cells. - entitySet accessedSet = distribute_entitySet(cellmask,ptn) ; + entitySet accessedSet = distribute_entitySet(cellmask,ptn,comm) ; WARN(GLOBAL_OR((geom_cells-accessedSet) != EMPTY)) #endif @@ -2478,12 +2492,12 @@ namespace Loci{ // join f2cell to get f Loci::protoMap f2celll = f2cell ; Loci::protoMap f2f ; - Loci::equiJoinFF(f2cell,f2celll,f2f) ; + Loci::equiJoinFF(f2cell,f2celll,f2f,comm) ; Loci::removeIdentity(f2f) ; - Loci::balanceDistribution(f2f,MPI_COMM_WORLD) ; + Loci::balanceDistribution(f2f, comm) ; // Create cell stencil map from protoMap - distributed_inverseMap(cellStencil,f2f,geom_cells,geom_cells,ptn) ; + distributed_inverseMap(cellStencil,f2f,geom_cells,geom_cells,ptn,comm) ; } @@ -2491,7 +2505,8 @@ namespace Loci{ /// @param[out] cellStencilSymm /// @param[in] facts void get_symm_cellStencil(multiMap &cellStencilSymm, fact_db & facts) { - if(Loci::MPI_rank==0) { + MPI_Comm comm = facts.get_comm() ; + if(facts.get_comm_rank()==0) { cout <<"Generating symm stencil" << endl ; } using std::vector ; @@ -2521,8 +2536,8 @@ namespace Loci{ std::map fg2l ; // face global to local getLocalContextMap(fg2l,faceimage) ; vector > fcenterdata, fnormaldata ; - gatherData(fcenterdata,fcenter,faceimage,face_ptn) ; - gatherData(fnormaldata,normal,faceimage,face_ptn) ; + gatherData(fcenterdata,fcenter,faceimage,face_ptn,comm) ; + gatherData(fnormaldata,normal,faceimage,face_ptn,comm) ; int ckeyspace = upper.getDomainKeySpace() ; std::vector cell_ptn = facts.get_init_ptn(ckeyspace) ; @@ -2532,7 +2547,7 @@ namespace Loci{ vector > ccenterdata ; std::map cg2l ; getLocalContextMap(cg2l,cellImage) ; - gatherData(ccenterdata,ccenter,cellImage,cell_ptn) ; + gatherData(ccenterdata,ccenter,cellImage,cell_ptn,comm) ; multiMap neighStencil; get_neigh_cellStencil(neighStencil,facts); @@ -2540,7 +2555,7 @@ namespace Loci{ vector > ncenterdata ; std::map ncg2l ; getLocalContextMap(ncg2l,neighCellImage) ; - gatherData(ncenterdata,ccenter,neighCellImage,cell_ptn) ; + gatherData(ncenterdata,ccenter,neighCellImage,cell_ptn,comm) ; store sizes ; sizes.allocate(cells) ; @@ -2713,7 +2728,8 @@ namespace Loci{ } void get_symmF_cellStencil(multiMap &cellStencilSymmF, fact_db & facts) { - if(Loci::MPI_rank==0) { + MPI_Comm comm = facts.get_comm() ; + if(facts.get_comm_rank()==0) { cout << "Generating symmF stencil" << endl ; } using std::vector ; @@ -2744,8 +2760,8 @@ namespace Loci{ getLocalContextMap(fg2l,faceimage) ; vector > fcenterdata ; vector > fnormaldata ; - gatherData(fcenterdata,fcenter,faceimage,face_ptn) ; - gatherData(fnormaldata,normal,faceimage,face_ptn) ; + gatherData(fcenterdata,fcenter,faceimage,face_ptn,comm) ; + gatherData(fnormaldata,normal,faceimage,face_ptn,comm) ; int ckeyspace = upper.getDomainKeySpace() ; std::vector cell_ptn = facts.get_init_ptn(ckeyspace) ; @@ -2755,7 +2771,7 @@ namespace Loci{ vector > ccenterdata ; std::map cg2l ; getLocalContextMap(cg2l,cellImage) ; - gatherData(ccenterdata,ccenter,cellImage,cell_ptn) ; + gatherData(ccenterdata,ccenter,cellImage,cell_ptn,comm) ; multiMap neighStencil; get_neigh_cellStencil(neighStencil,facts); @@ -2763,7 +2779,7 @@ namespace Loci{ vector > ncenterdata ; std::map ncg2l ; getLocalContextMap(ncg2l,neighCellImage) ; - gatherData(ncenterdata,ccenter,neighCellImage,cell_ptn) ; + gatherData(ncenterdata,ccenter,neighCellImage,cell_ptn,comm) ; store sizes ; sizes.allocate(cells) ; @@ -3043,7 +3059,8 @@ namespace Loci{ } void get_symmC_cellStencil(multiMap &cellStencilSymmC, fact_db & facts) { - if(Loci::MPI_rank==0) { + MPI_Comm comm = facts.get_comm() ; + if(facts.get_comm_rank()==0) { cout <<"Generating symmC stencil" << endl ; } using std::vector ; @@ -3074,8 +3091,8 @@ namespace Loci{ getLocalContextMap(fg2l,faceimage) ; vector > fcenterdata ; vector > fnormaldata ; - gatherData(fcenterdata,fcenter,faceimage,face_ptn) ; - gatherData(fnormaldata,normal,faceimage,face_ptn) ; + gatherData(fcenterdata,fcenter,faceimage,face_ptn,comm) ; + gatherData(fnormaldata,normal,faceimage,face_ptn,comm) ; int ckeyspace = upper.getDomainKeySpace() ; std::vector cell_ptn = facts.get_init_ptn(ckeyspace) ; @@ -3085,7 +3102,7 @@ namespace Loci{ vector > ccenterdata ; std::map cg2l ; getLocalContextMap(cg2l,cellImage) ; - gatherData(ccenterdata,ccenter,cellImage,cell_ptn) ; + gatherData(ccenterdata,ccenter,cellImage,cell_ptn,comm) ; multiMap neighStencil; get_neigh_cellStencil(neighStencil,facts); @@ -3093,7 +3110,7 @@ namespace Loci{ vector > ncenterdata ; std::map ncg2l ; getLocalContextMap(ncg2l,neighCellImage) ; - gatherData(ncenterdata,ccenter,neighCellImage,cell_ptn) ; + gatherData(ncenterdata,ccenter,neighCellImage,cell_ptn,comm) ; store sizes ; sizes.allocate(cells) ; @@ -3241,6 +3258,7 @@ namespace Loci{ void createLowerUpper(fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; constraint geom_cells,interior_faces,boundary_faces; constraint faces = facts.get_variable("faces") ; geom_cells = facts.get_variable("geom_cells") ; @@ -3262,10 +3280,10 @@ namespace Loci{ std::vector fptn = facts.get_init_ptn(fkeyspace) ; entitySet - global_interior_faces = (distribute_entitySet(ifaces,fptn) & (*faces)) ; + global_interior_faces = (distribute_entitySet(ifaces,fptn,comm) & (*faces)) ; entitySet - global_boundary_faces = (distribute_entitySet(bfaces,fptn) & (*faces)) ; + global_boundary_faces = (distribute_entitySet(bfaces,fptn,comm) & (*faces)) ; Map cl,cr ; cl = facts.get_variable("cl") ; @@ -3273,7 +3291,8 @@ namespace Loci{ // Note, all_collect_entitySet has the potential to be inefficient, // but not in the present case. These low level utilities need to be // rethought. - entitySet global_geom_cells = all_collect_entitySet(*geom_cells) ; + entitySet global_geom_cells = all_collect_entitySet(*geom_cells, + comm) ; multiMap lower,upper,boundary_map ; distributed_inverseMap(upper, cl, global_geom_cells, global_interior_faces, facts,ckeyspace) ; @@ -3533,49 +3552,45 @@ namespace Loci{ namespace { - /// A utility that returns the global sum. - int global_sum(int l) { + int global_sum(int l, MPI_Comm comm) { int g ; - MPI_Allreduce(&l, &g, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD) ; + MPI_Allreduce(&l, &g, 1, MPI_INT, MPI_SUM, comm) ; return g ; } /// A utility function that takes an entitySet from a processor and /// returns a vector of entitySet gathered from all processors. - vector gather_all_entitySet(const entitySet& eset) { + vector gather_all_entitySet(const entitySet& eset, + MPI_Comm comm) { + int np = 0 ; + MPI_Comm_size(comm, &np) ; int local_size = eset.size() ; - int global_size = global_sum(local_size) ; - // compute receive counts from all processors - int* recv_counts = new int[MPI_processes] ; + int global_size = global_sum(local_size, comm) ; + int* recv_counts = new int[np] ; MPI_Allgather(&local_size, 1, MPI_INT, - recv_counts, 1, MPI_INT, MPI_COMM_WORLD) ; - // then compute receive displacement - int* recv_displs = new int[MPI_processes] ; + recv_counts, 1, MPI_INT, comm) ; + int* recv_displs = new int[np] ; recv_displs[0] = 0 ; - for(int i=1;i - vector ret(MPI_processes) ; + vector ret(np) ; int k = 0 ; - for(int i=0;i 1) { + if(facts.get_comm_size() > 1) { // then we will need to remove duplicates along the boundaries // we send the first element in the vector to the left neighbor // processor (my_id - 1) and each processor compares its last @@ -3883,24 +3900,27 @@ namespace Loci{ sendbuf[1] = std::numeric_limits::max() ; } MPI_Status status ; - if(MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { // rank 0 only receives from 1, no sending needed MPI_Recv(recvbuf, 2, MPI_INT, 1/*source*/, 0/*msg tag*/, - MPI_COMM_WORLD, &status) ; - } else if(MPI_rank == MPI_processes-1) { + facts.get_comm(), &status) ; + } else if(facts.get_comm_rank() == facts.get_comm_size()-1) { // the last processes only sends to the second last processes, // no receiving is needed MPI_Send(sendbuf, 2, MPI_INT, - MPI_rank-1/*dest*/, 0/*msg tag*/, MPI_COMM_WORLD) ; + facts.get_comm_rank()-1/*dest*/, 0/*msg tag*/, + facts.get_comm()) ; } else { // others will send to MPI_rank-1 and receive from MPI_rank+1 - MPI_Sendrecv(sendbuf, 2, MPI_INT, MPI_rank-1/*dest*/,0/*msg tag*/, - recvbuf, 2, MPI_INT, MPI_rank+1/*source*/,0/*tag*/, - MPI_COMM_WORLD, &status) ; + MPI_Sendrecv(sendbuf, 2, MPI_INT, + facts.get_comm_rank()-1/*dest*/, 0/*msg tag*/, + recvbuf, 2, MPI_INT, + facts.get_comm_rank()+1/*source*/, 0/*tag*/, + facts.get_comm(), &status) ; } // then compare the results with last element in local emap - if( (MPI_rank != MPI_processes-1) && (!emap.empty())){ + if( (facts.get_comm_rank() != facts.get_comm_size()-1) && (!emap.empty())){ const pair& last = emap.back() ; if( (recvbuf[0] == last.first) && (recvbuf[1] == last.second)) { @@ -3962,8 +3982,8 @@ namespace Loci{ // instead it is the el Map image distribution entitySet el_image = el.image(el.domain()) ; vector el_image_partitions = - gather_all_entitySet(el_image) ; - distributed_inverseMap(n2e, el, el_image, edges, el_image_partitions) ; + gather_all_entitySet(el_image, comm) ; + distributed_inverseMap(n2e, el, el_image, edges, el_image_partitions,comm) ; // Now create face2edge map with same size as face2node multiMap face2edge ; @@ -4005,7 +4025,7 @@ namespace Loci{ // we then expand the n2e map entitySet nodes_out_domain = nodes_accessed - n2e.domain() ; n2e.setRep(MapRepP(n2e.Rep())->expand(nodes_out_domain, - el_image_partitions)) ; + el_image_partitions,comm)) ; // okay, then we are going to expand the edge map // first count all the edges we need entitySet edges_accessed ; @@ -4030,7 +4050,7 @@ namespace Loci{ edges_accessed += e ; } } - vector edge_partitions = gather_all_entitySet(edge.domain()) ; + vector edge_partitions = gather_all_entitySet(edge.domain(), facts.get_comm()) ; entitySet edges_out_domain = edges_accessed - edge.domain() ; // but since there is no expand method implemented for // MapVec at this time, we will just do a hack to convert @@ -4050,7 +4070,7 @@ namespace Loci{ edge2[*ei][1] = edge[*ei][1] ; } edge2.setRep(MapRepP(edge2.Rep())->expand(edges_out_domain, - edge_partitions)) ; + edge_partitions,comm)) ; // we are now ready for the face2edge map // Now loop over faces, for each face search for matching edge and @@ -4102,7 +4122,7 @@ namespace Loci{ //sort edge2node according to fileNumbering - if(MPI_processes > 1){ + if(facts.get_comm_size() > 1){ //create Map node_l2f entitySet nodes ; Map node_l2f ; @@ -4120,7 +4140,7 @@ namespace Loci{ g2f = df->g2fv[nkeyspace].Rep() ; //don't use nodes & init_ptn to define local nodes, //because nodes may not cover all nodes in init_ptn - entitySet localNodes = pos->domain()&init_ptn[MPI_rank] ; + entitySet localNodes = pos->domain()&init_ptn[facts.get_comm_rank()] ; node_l2f.allocate(localNodes); FORALL(localNodes, d){ node_l2f[d] = g2f[d] ; @@ -4128,7 +4148,7 @@ namespace Loci{ entitySet out_of_dom = nodes - localNodes ; // vector tmp_ptn = gather_all_entitySet(localNodes) ; - node_l2f.setRep(MapRepP(node_l2f.Rep())->expand(out_of_dom, init_ptn)) ; + node_l2f.setRep(MapRepP(node_l2f.Rep())->expand(out_of_dom, init_ptn, comm)) ; //end of create Map @@ -4152,8 +4172,9 @@ namespace Loci{ }ENDFORALL; - if(GLOBAL_OR(edge2global.empty())) { - parallel_balance_pair2_vector(edge2global, MPI_COMM_WORLD) ; + if(GLOBAL_OR(edge2global.empty(), facts.get_comm())) { + parallel_balance_pair2_vector(edge2global, + facts.get_comm()) ; } // Sort edges and remove duplicates sort(edge2global.begin(),edge2global.end()) ; @@ -4166,22 +4187,23 @@ namespace Loci{ // less than the total number of processes, we split the communicator // so that only those do have elements would participate in the // parallel sample sorting - if(GLOBAL_OR(edge2global.empty())) { + if(GLOBAL_OR(edge2global.empty(), facts.get_comm())) { MPI_Comm sub_comm ; int color = edge2global.empty() ; - MPI_Comm_split(MPI_COMM_WORLD, color, MPI_rank, &sub_comm) ; + MPI_Comm_split(facts.get_comm(), color, facts.get_comm_rank(), + &sub_comm) ; if(!edge2global.empty()) { par_sort2(edge2global, sub_comm) ; } MPI_Comm_free(&sub_comm) ; } else { - par_sort2(edge2global, MPI_COMM_WORLD) ; + par_sort2(edge2global, facts.get_comm()) ; } // remove duplicates again in the new sorted vector uend2 = unique(edge2global.begin(), edge2global.end()) ; edge2global.erase(uend2, edge2global.end()) ; #ifdef BOUNDARY_DUPLICATE_DETECT - if(MPI_processes > 1) { + if(facts.get_comm_size() > 1) { // then we will need to remove duplicates along the boundaries // we send the first element in the vector to the left neighbor // processor (my_id - 1) and each processor compares its last @@ -4208,24 +4230,27 @@ namespace Loci{ sendbuf[2] = std::numeric_limits::max() ; } MPI_Status status ; - if(MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { // rank 0 only receives from 1, no sending needed MPI_Recv(recvbuf, 3, MPI_INT, 1/*source*/, 0/*msg tag*/, - MPI_COMM_WORLD, &status) ; - } else if(MPI_rank == MPI_processes-1) { + facts.get_comm(), &status) ; + } else if(facts.get_comm_rank() == facts.get_comm_size()-1) { // the last processes only sends to the second last processes, // no receiving is needed MPI_Send(sendbuf, 3, MPI_INT, - MPI_rank-1/*dest*/, 0/*msg tag*/, MPI_COMM_WORLD) ; + facts.get_comm_rank()-1/*dest*/, 0/*msg tag*/, + facts.get_comm()) ; } else { // others will send to MPI_rank-1 and receive from MPI_rank+1 - MPI_Sendrecv(sendbuf, 3, MPI_INT, MPI_rank-1/*dest*/,0/*msg tag*/, - recvbuf, 3, MPI_INT, MPI_rank+1/*source*/,0/*tag*/, - MPI_COMM_WORLD, &status) ; + MPI_Sendrecv(sendbuf, 3, MPI_INT, + facts.get_comm_rank()-1/*dest*/, 0/*msg tag*/, + recvbuf, 3, MPI_INT, + facts.get_comm_rank()+1/*source*/, 0/*tag*/, + facts.get_comm(), &status) ; } // then compare the results with last element in local emap - if( (MPI_rank != MPI_processes-1) && (!edge2global.empty())) { + if( (facts.get_comm_rank() != facts.get_comm_size()-1) && (!edge2global.empty())) { const pair, Entity>& last = edge2global.back() ; if( (recvbuf[0] == last.first.first) && (recvbuf[1] == last.first.second)&& @@ -4238,11 +4263,13 @@ namespace Loci{ int local_num_edge = edge2global.size() ; - vector edge_sizes(MPI_processes) ; - MPI_Allgather(&local_num_edge,1,MPI_INT,&edge_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + vector edge_sizes(facts.get_comm_size()) ; + MPI_Allgather(&local_num_edge, 1, MPI_INT, + &edge_sizes[0], 1, MPI_INT, + facts.get_comm()) ; int file_num_offset = 0 ; - for(int i = 0; i < MPI_rank; i++) { + for(int i = 0; i < facts.get_comm_rank(); i++) { file_num_offset += edge_sizes[i] ; } @@ -4268,7 +4295,7 @@ namespace Loci{ file2global, input_image, input_preimage, - init_ptne); + init_ptne,facts.get_comm()); if(global2file.domain() != edges){ @@ -4301,6 +4328,7 @@ namespace Loci{ void setupOverset(fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; storeRepP sp = facts.get_variable("componentGeometry") ; if(sp == 0) { return ; @@ -4331,8 +4359,9 @@ namespace Loci{ pos = facts.get_variable("pos") ; entitySet dom = pos.domain() ; - vector posptn = all_collect_vectors(dom,MPI_COMM_WORLD) ; - entitySet surfNodes = dist_collect_entitySet(nodeSet,posptn) ; + vector posptn = + all_collect_vectors(dom, facts.get_comm()) ; + entitySet surfNodes = dist_collect_entitySet(nodeSet,posptn,comm) ; // Now get volume tags variableSet vars = facts.get_extensional_facts() ; @@ -4364,8 +4393,8 @@ namespace Loci{ for(mi=volMap.begin();mi!=volMap.end();++mi) { int ckeyspace = cl.getRangeKeySpace() ; std::vector cptn = facts.get_init_ptn(ckeyspace) ; - entitySet volgather = distribute_entitySet(mi->second,cptn) ; - volgather = dist_expand_entitySet(volgather,domc,cptn) ; + entitySet volgather = distribute_entitySet(mi->second,cptn,comm) ; + volgather = dist_expand_entitySet(volgather,domc,cptn,comm) ; volSets.push_back(volgather) ; } @@ -4386,7 +4415,7 @@ namespace Loci{ int nkeyspace = pos.getDomainKeySpace() ; std::vector nptn = facts.get_init_ptn(nkeyspace) ; - nodes = distribute_entitySet(nodes,nptn) ; + nodes = distribute_entitySet(nodes,nptn,comm) ; nodesets.push_back(nodes) ; } @@ -4399,7 +4428,7 @@ namespace Loci{ for(int i=0;i #include #include +#include #include #include namespace Loci { + extern fact_db *exec_current_fact_db ; using std::pair ; using std::make_pair ; @@ -385,17 +387,19 @@ namespace Loci { return make_pair(domain,domain) ; } - storeRepP MapRepI::expand(entitySet &out_of_dom, std::vector &ptn) { + storeRepP MapRepI::expand(entitySet &out_of_dom, std::vector &ptn, MPI_Comm comm) { storeRepP sp ; - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + int np ; + MPI_Comm_size(comm, &np) ; + int *recv_count = new int[np] ; + int *send_count = new int[np] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; entitySet::const_iterator ei ; std::vector::const_iterator vi ; int size_send = 0 ; - std::vector > copy(MPI_processes), send_clone(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + std::vector > copy(np), send_clone(np) ; + for(int i = 0; i < np; ++i) { entitySet tmp = out_of_dom & ptn[i] ; for(ei = tmp.begin(); ei != tmp.end(); ++ei) copy[i].push_back(*ei) ; @@ -405,54 +409,54 @@ namespace Loci { } int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_buf = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = copy[i].begin(); vi != copy[i].end(); ++vi) { send_buf[size_send] = *vi ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - for(int i = 0; i < MPI_processes; ++i) { + comm) ; + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i].push_back(recv_buf[j]) ; std::sort(send_clone[i].begin(), send_clone[i].end()) ; } - std::vector map_entities(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) + std::vector map_entities(np) ; + for(int i = 0; i < np; ++i) for(vi = send_clone[i].begin(); vi != send_clone[i].end(); ++vi) if(store_domain.inSet(*vi)) (map_entities[i])[*vi] = base_ptr[*vi] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { send_count[i] = 2 * map_entities[i].size() ; size_send += send_count[i] ; } int *send_map = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_map = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(HASH_MAP(int, int)::const_iterator miv = map_entities[i].begin(); miv != map_entities[i].end(); ++miv) { send_map[size_send] = miv->first ; ++size_send ; @@ -461,15 +465,15 @@ namespace Loci { } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_map,send_count, send_displacement , MPI_INT, recv_map, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; HASH_MAP(int, int) hm ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]-1; j+=2) { hm[recv_map[j]] = recv_map[j+1]; @@ -619,16 +623,18 @@ namespace Loci { allocate(count) ; } - storeRepP multiMapRepI::expand(entitySet &out_of_dom, std::vector &ptn) { - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + storeRepP multiMapRepI::expand(entitySet &out_of_dom, std::vector &ptn, MPI_Comm comm) { + int np ; + MPI_Comm_size(comm, &np) ; + int *recv_count = new int[np] ; + int *send_count = new int[np] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; entitySet::const_iterator ei ; std::vector::const_iterator vi ; int size_send = 0 ; - std::vector > copy(MPI_processes), send_clone(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + std::vector > copy(np), send_clone(np) ; + for(int i = 0; i < np; ++i) { entitySet tmp = out_of_dom & ptn[i] ; for(ei = tmp.begin(); ei != tmp.end(); ++ei) copy[i].push_back(*ei) ; @@ -638,35 +644,35 @@ namespace Loci { } int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_buf = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = copy[i].begin(); vi != copy[i].end(); ++vi) { send_buf[size_send] = *vi ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - for(int i = 0; i < MPI_processes; ++i) { + comm) ; + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i].push_back(recv_buf[j]) ; std::sort(send_clone[i].begin(), send_clone[i].end()) ; } - std::vector) > map_entities(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) + std::vector) > map_entities(np) ; + for(int i = 0; i < np; ++i) for(vi = send_clone[i].begin(); vi != send_clone[i].end(); ++vi) { if(store_domain.inSet(*vi)) { for(const int* j = begin(*vi); j != end(*vi); ++j) @@ -674,23 +680,23 @@ namespace Loci { } } - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { send_count[i] = 2 * map_entities[i].size() ; for(HASH_MAP(int, std::vector)::iterator hi = map_entities[i].begin(); hi != map_entities[i].end(); ++hi) send_count[i] += hi->second.size() ; } size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += send_count[i] ; int *send_map = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_map = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(HASH_MAP(int, std::vector )::const_iterator miv = map_entities[i].begin(); miv != map_entities[i].end(); ++miv) { send_map[size_send] = miv->first ; ++size_send ; @@ -703,16 +709,16 @@ namespace Loci { } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_map,send_count, send_displacement , MPI_INT, recv_map, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; HASH_MAP(int, std::vector ) hm ; std::vector ss ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]-1; ++j) { int count = recv_map[j+1] ; diff --git a/src/System/categories.cc b/src/System/categories.cc index ba81856c..204af39f 100644 --- a/src/System/categories.cc +++ b/src/System/categories.cc @@ -22,6 +22,7 @@ #include #include #include "Tools/debugger.h" +#include #include using std::vector ; @@ -68,8 +69,9 @@ namespace Loci { // over their domains. void getVariableAssociations(map &vm, fact_db &facts, int kd) { vector ptn = facts.get_init_ptn(kd) ; + MPI_Comm comm = facts.get_comm() ; entitySet total_entities ; - entitySet my_entities = ptn[MPI_rank] ; + entitySet my_entities = ptn[facts.get_comm_rank()] ; vm.clear() ; variableSet vars = facts.get_typed_variables() ; for(variableSet::const_iterator vi=vars.begin();vi!=vars.end();++vi) { @@ -102,11 +104,12 @@ namespace Loci { if(mp->getRangeKeySpace() == kd) { entitySet dom = my_entities & p->domain() ; entitySet image_dom = mp->image(dom) ; - image_dom += distribute_entitySet(image_dom, ptn) ; + image_dom += distribute_entitySet(image_dom, ptn, comm) ; entitySet testSet = image_dom - total_entities ; int size = testSet.size() ; int tsize = 0 ; - MPI_Allreduce(&size,&tsize,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&size, &tsize, 1, MPI_INT, + MPI_SUM, comm) ; if(tsize != 0) { std::string name = "image_" ; name.append(vi->get_info().name) ; @@ -118,7 +121,7 @@ namespace Loci { } } variable v_leftout("SENTINEL_LEFTOUT") ; - vm[v_leftout] = ptn[MPI_rank]-total_entities ; + vm[v_leftout] = ptn[facts.get_comm_rank()]-total_entities ; } // Compute the distict intervals that are required to describe all of @@ -218,7 +221,9 @@ namespace Loci { void getGlobalCategories(vector &clist, vector &vlist, const map &cmap, - const map &vm) { + const map &vm, + fact_db &facts) { + int r = facts.get_comm_rank() ; vector vnlist ; map::const_iterator ii ; for(ii=vm.begin();ii!=vm.end();++ii) { @@ -249,7 +254,7 @@ namespace Loci { int comm_root = 0 ; while(comm_root != -1) { vector buffer ; - if(comm_root == MPI_rank) { + if(comm_root == r) { set::const_iterator si ; for(si=cat_set.begin();si!=cat_set.end();++si) { entitySet s = *si ; @@ -259,10 +264,12 @@ namespace Loci { } } int buf_size = buffer.size() ; - MPI_Bcast(&buf_size,1,MPI_INT,comm_root,MPI_COMM_WORLD) ; - if(comm_root != MPI_rank) + MPI_Comm comm = facts.get_comm() ; + MPI_Bcast(&buf_size, 1, MPI_INT, comm_root, comm) ; + if(comm_root != r) buffer = vector(buf_size) ; - MPI_Bcast(&buffer[0],buf_size,MPI_INT,comm_root,MPI_COMM_WORLD) ; + MPI_Bcast(&buffer[0], buf_size, MPI_INT, comm_root, + comm) ; for(size_t ex=0;ex::const_iterator si ; @@ -294,14 +302,16 @@ namespace Loci { // Collect the sizes of the distributed variable sets void getVarSizes(vector > &var_sizes, const vector &var_list, - const map &vm) { + const map &vm, + fact_db &facts) { var_sizes.clear() ; for(size_t i=0;i::const_iterator vinfo = vm.find(v) ; int local_size = vinfo->second.size() ; int vsize = 0 ; - MPI_Allreduce(&local_size,&vsize,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&local_size, &vsize, 1, MPI_INT, MPI_SUM, + facts.get_comm()) ; var_sizes.push_back(pair(v,vsize)) ; } std::stable_sort(var_sizes.begin(),var_sizes.end(),compare_var_sizes) ; @@ -334,10 +344,10 @@ namespace Loci { vector cat_list ; vector var_list ; - getGlobalCategories(cat_list,var_list,cmap,vm) ; + getGlobalCategories(cat_list,var_list,cmap,vm,facts) ; vector > var_sizes ; - getVarSizes(var_sizes,var_list,vm) ; + getVarSizes(var_sizes,var_list,vm,facts) ; vector > cat_keys ; getCategoryKeys(cat_keys,var_sizes,cat_list) ; diff --git a/src/System/comp_loop.cc b/src/System/comp_loop.cc index db11240c..561ab354 100644 --- a/src/System/comp_loop.cc +++ b/src/System/comp_loop.cc @@ -412,7 +412,7 @@ namespace Loci { vinfo.offset=1 ; li->v = variable(vinfo) ; } - execute_comm2::inc_comm_step() ; + execute_comm2::inc_comm_step(facts.get_comm()) ; //executeP exec_comm = //new execute_comm(advance_variables_barrier, facts); executeP exec_comm2 = diff --git a/src/System/comp_recurse.cc b/src/System/comp_recurse.cc index 1944ac73..f74e92b2 100644 --- a/src/System/comp_recurse.cc +++ b/src/System/comp_recurse.cc @@ -46,7 +46,7 @@ namespace Loci { } void impl_recurse_compiler::set_var_existence(fact_db &facts, sched_db &scheds) { - + #ifdef VERBOSE debugout << "set var existence for recursive impl rule " << impl << endl ; #endif @@ -194,7 +194,7 @@ namespace Loci { << ", newdomain = " << newdomain << endl ; #endif if(domain == ~EMPTY) { - if(MPI_processes == 1) + if(facts.get_comm_size() == 1) cerr << "problem in recurse compiler for rule = "<< impl << endl ; else debugout << "problem in recurse compiler for rule = "<< impl << endl ; @@ -404,7 +404,7 @@ namespace Loci { create += send_entitySet(create,facts) ; create += fill_entitySet(create,facts) ; scheds.set_existential_info(rvar,impl,create) ; - + } void impl_recurse_compiler::process_var_requests(fact_db &facts, sched_db &scheds) { @@ -439,7 +439,7 @@ namespace Loci { // list clist = sort_comm(request_comm,facts) ; // scheds.update_comm_info_list(clist, sched_db::RECURSE_CLIST); } - + } executeP impl_recurse_compiler::create_execution_schedule(fact_db &facts, sched_db &scheds) { @@ -456,12 +456,12 @@ namespace Loci { entitySet my_entities = ~EMPTY ; if(facts.isDistributed()) { fact_db::distribute_infoP d = facts.get_distribute_info() ; - + std::vector > pre_send_entities = barrier_existential_rule_analysis(recurse_vars,facts, scheds) ; scheds.update_send_entities(pre_send_entities, sched_db::RECURSE_PRE); my_entities = d->my_entities ; - + } control_set.clear() ; @@ -575,7 +575,7 @@ namespace Loci { } if(fctrl.use_constraints) { if((srcs & fctrl.constraints) != fctrl.constraints) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "recursive rule: " << *fi << " cannot supply all entitites in constraint" << endl ; cerr << "constraints = " << fctrl.constraints << endl ; @@ -596,7 +596,7 @@ namespace Loci { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "detailed report:" << endl ; cerr<< "fctrl.nr_sources = " << fctrl.nr_sources << endl ; } else { @@ -608,7 +608,7 @@ namespace Loci { entitySet sources = vmap_source_exist(*si,facts, scheds) ; sources &= my_entities ; if((sources & constraints) != constraints) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "sources & constraints != constraints for input" << endl << sources << " -- " << *si << endl ; @@ -632,7 +632,7 @@ namespace Loci { entitySet exist = scheds.variable_existence(*vi) ; entitySet fails = working & ~exist ; if(fails != EMPTY) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "expecting to find variable " << *vi << " at entities " << fails << endl << *vi << " exists at entities " << exist << endl ; @@ -687,7 +687,7 @@ namespace Loci { if(!facts.isDistributed()) deltar = deltas ; else - MPI_Allreduce(&deltas,&deltar, 1, MPI_INT, MPI_SUM, MPI_COMM_WORLD) ; + MPI_Allreduce(&deltas,&deltar, 1, MPI_INT, MPI_SUM, facts.get_comm()) ; if(deltar != 0) finished = false ; } @@ -741,7 +741,7 @@ namespace Loci { recurse_entities[*vi] += scheds.get_existential_info(*vi,*ri) ; } } - + std::vector > pre_send_entities = scheds.get_send_entities(recurse_vars, sched_db::RECURSE_PRE); for(vi=pre_send_entities.begin();vi!=pre_send_entities.end();++vi) { @@ -893,8 +893,8 @@ namespace Loci { if(facts.isDistributed()) { list pre_clist = scheds.get_comm_info_list(recurse_vars, facts, sched_db::RECURSE_PRE_CLIST); list pre_plist = scheds.get_comm_info_list(recurse_vars, facts, sched_db::RECURSE_PRE_PLIST); - - execute_comm2::inc_comm_step() ; + MPI_Comm comm = facts.get_comm() ; + execute_comm2::inc_comm_step(comm) ; if(!pre_plist.empty()) { //executeP exec_commp = new execute_comm(pre_plist, facts); executeP exec_commp2 = new execute_comm2(pre_plist, facts); @@ -902,7 +902,7 @@ namespace Loci { //el->append_list(exec_commp) ; } - execute_comm2::inc_comm_step() ; + execute_comm2::inc_comm_step(comm) ; if(!pre_clist.empty()) { //executeP exec_commc = new execute_comm(pre_clist, facts); executeP exec_commc2 = new execute_comm2(pre_clist, facts); @@ -926,7 +926,7 @@ namespace Loci { vi!=recurse_vars.end(); ++vi) { vector > &commv = send_req_var[*vi] ; - execute_comm2::inc_comm_step() ; + execute_comm2::inc_comm_step(facts.get_comm()) ; if(idx plist = put_precomm_info(*sei, facts) ; - execute_comm2::inc_comm_step() ; + execute_comm2::inc_comm_step(facts.get_comm()) ; if(!plist.empty()) { //executeP exec_comm = new execute_comm(plist,facts); executeP exec_comm2 = new execute_comm2(plist,facts); @@ -997,12 +997,12 @@ namespace Loci { } while(!finished) ; if(facts.isDistributed()) { - list post_clist = scheds.get_comm_info_list(recurse_vars, facts, sched_db::RECURSE_POST_CLIST); - - execute_comm2::inc_comm_step() ; + list post_clist = scheds.get_comm_info_list(recurse_vars, facts, sched_db::RECURSE_POST_CLIST); + + execute_comm2::inc_comm_step(facts.get_comm()) ; if(!post_clist.empty()) { - executeP exec_comm2 = new execute_comm2(post_clist, facts); - el->append_list(exec_comm2) ; + executeP exec_comm2 = new execute_comm2(post_clist, facts); + el->append_list(exec_comm2) ; } // Make sure to request any variables communicated so that // the space is allocated. This is a hack that should be diff --git a/src/System/comp_reduce.cc b/src/System/comp_reduce.cc index 7f7f5e9e..5df9d49d 100644 --- a/src/System/comp_reduce.cc +++ b/src/System/comp_reduce.cc @@ -362,7 +362,7 @@ namespace Loci { } #ifdef GROUP_ALLREDUCE - groupAllReduce(sp,join_ops,MPI_COMM_WORLD) ; + groupAllReduce(sp, join_ops, facts.get_comm()) ; #else // Old Code (Works only for fixed sized types) int size = 0; @@ -378,7 +378,9 @@ namespace Loci { } global_join_ops = join_ops; - MPI_Allreduce(&send_ptr[0], &result_ptr[0], size, MPI_PACKED, create_join_op, MPI_COMM_WORLD) ; + MPI_Allreduce(&send_ptr[0], &result_ptr[0], size, + MPI_PACKED, create_join_op, + facts.get_comm()) ; position = 0; for(size_t i = 0; i < sp.size(); i++) { sp[i]->unpack(&result_ptr[0], position, size, seq) ; @@ -439,7 +441,7 @@ namespace Loci { vector > jop ; for(size_t i=0;i 1) { + if(verbose || facts.get_comm_size() > 1) { ostringstream oss ; for(size_t i = 0; i < reduce_vars.size(); i++) oss << "reduce param " << reduce_vars[i] << std::endl; @@ -724,7 +726,7 @@ namespace Loci { for(int i=0;i 0) { #ifdef DEBUG @@ -793,7 +798,9 @@ namespace Loci { int loc_unpack = 0; if(rerecv_procs.inSet(recv_info[i].first)) { int temp ; - MPI_Unpack(recv_ptr[i], r_size[i], &loc_unpack, &temp, sizeof(int), MPI_BYTE, MPI_COMM_WORLD) ; + MPI_Unpack(recv_ptr[i], r_size[i], &loc_unpack, + &temp, sizeof(int), MPI_BYTE, + facts.get_comm()) ; if(temp > maxr_size[i]) maxr_size[i] = temp ; } @@ -819,7 +826,10 @@ namespace Loci { for(int i = 0; i < rerecv_size; i++) { int proc = recv_info[recv_index[i]].first ; recv_ptr[recv_index[i]] = new unsigned char[maxr_size[recv_index[i]]] ; - MPI_Irecv(recv_ptr[recv_index[i]], maxr_size[recv_index[i]], MPI_PACKED, proc, 2, MPI_COMM_WORLD, &re_request[i]) ; + MPI_Irecv(recv_ptr[recv_index[i]], + maxr_size[recv_index[i]], MPI_PACKED, + proc, 2, facts.get_comm(), + &re_request[i]) ; } for(int i=0;i 0) { @@ -934,14 +946,15 @@ namespace Loci { executeP exec_comm_reduce = new execute_comm_reduce(rlist, facts, join_op); el->append_list(exec_comm_reduce); } - execute_comm2::inc_comm_step() ; + MPI_Comm comm = facts.get_comm() ; + execute_comm2::inc_comm_step(comm) ; if(!clist.empty()) { //executeP exec_comm = new execute_comm(clist, facts); executeP exec_comm2 = new execute_comm2(clist, facts); el->append_list(exec_comm2) ; //el->append_list(exec_comm) ; } - if(verbose || MPI_processes > 1) { + if(verbose || facts.get_comm_size() > 1) { ostringstream oss ; oss << "reduce store " << reduce_var ; executeP exec_msg = new execute_msg(oss.str()); @@ -950,7 +963,7 @@ namespace Loci { return executeP(el) ; } - if(verbose || MPI_processes > 1) { + if(verbose || facts.get_comm_size() > 1) { ostringstream oss ; oss << "reduce store " << reduce_var ; executeP exec_msg2 = new execute_msg(oss.str()); diff --git a/src/System/comp_tools.cc b/src/System/comp_tools.cc index 5c005ac5..4ca4fb1b 100644 --- a/src/System/comp_tools.cc +++ b/src/System/comp_tools.cc @@ -154,7 +154,7 @@ namespace Loci { } if(rinfo.constraints.begin() != rinfo.constraints.end()) if((sources & constraints) != constraints) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "Warning, rule " << r << " cannot supply all entities of constraint" << endl ; cerr << "constraints = " << constraints << endl ; @@ -171,7 +171,7 @@ namespace Loci { entitySet sources = vmap_source_exist(*si,facts, scheds) ; sources &= my_entities ; if((sources & constraints) != constraints) { - if(MPI_processes == 1) + if(facts.get_comm_size() == 1) cerr << "sources & constraints != constraints for input" << endl << sources << " -- " << *si << endl ; @@ -194,7 +194,7 @@ namespace Loci { entitySet exist = scheds.variable_existence(*vi) ; entitySet fails = working & ~exist ; if(fails != EMPTY) { - if(MPI_processes == 1) + if(facts.get_comm_size() == 1) cerr << "expecting to find variable " << *vi << " at entities " << fails << endl << *vi << " exists at entities " << exist << endl ; else debugout << "expecting to find variable " << *vi << " at entities " << fails << endl << *vi << " exists at entities " << exist << endl ; @@ -694,7 +694,7 @@ namespace Loci { if(rinfo.constraints.begin() != rinfo.constraints.end()) if((srcs & cnstrnts & filter) != (cnstrnts & filter)) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "Warning, reduction rule:" << apply << "cannot supply all entities of constraint" << endl ; cerr << "constraints = " << (cnstrnts&filter) << endl ; @@ -713,7 +713,7 @@ namespace Loci { entitySet sources = vmap_source_exist(*si,facts, scheds) ; sources &= filter; if((sources & cnstrnts & filter) != (cnstrnts&filter)) { - if(MPI_processes == 1) + if(facts.get_comm_size() == 1) cerr << "sources & constraints != constraints for input" << endl << (sources & filter) << " -- " << *si << endl ; @@ -738,7 +738,7 @@ namespace Loci { entitySet exist = scheds.variable_existence(*vi) ; entitySet fails = working & ~exist ; if(fails != EMPTY) { - if(MPI_processes == 1) + if(facts.get_comm_size() == 1) cerr << "expecting to find variable " << *vi << " at entities " << fails << endl << *vi << " exists at entities " << exist << endl ; else debugout << "expecting to find variable " << *vi << " at entities " << fails << endl << *vi << " exists at entities " << exist << endl ; @@ -807,7 +807,7 @@ namespace Loci { // if the constraints is not "~EMPTY" if(!rinfo.constraints.empty() && constraints != all_entities) { if((sources & constraints) != constraints) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "Warning, rule " << r << " cannot supply all entities of constraint" << endl ; cerr << "constraints = " << constraints << endl ; @@ -956,8 +956,9 @@ namespace Loci { MPI_Status *status = new MPI_Status[d->xmit.size()] ; for(size_t i=0;ixmit.size();++i) { - MPI_Irecv(recv_buffer[i], recv_size[i], MPI_INT, d->xmit[i].proc, 2, - MPI_COMM_WORLD, &recv_request[i] ) ; + MPI_Irecv(recv_buffer[i], recv_size[i], MPI_INT, + d->xmit[i].proc, 2, + facts.get_comm(), &recv_request[i] ) ; } for(size_t i=0;icopy.size();++i) { int j=sesz ; @@ -979,8 +980,8 @@ namespace Loci { } } int send_size = j ; - MPI_Send(send_buffer[i],send_size, MPI_INT, d->copy[i].proc, - 2,MPI_COMM_WORLD) ; + MPI_Send(send_buffer[i], send_size, MPI_INT, + d->copy[i].proc, 2, facts.get_comm()) ; } if(d->xmit.size() > 0) { @@ -1290,8 +1291,9 @@ namespace Loci { MPI_Status *status = new MPI_Status[d->xmit.size()] ; for(size_t i=0;ixmit.size();++i) { - MPI_Irecv(recv_buffer[i], recv_size[i], MPI_INT, d->xmit[i].proc, 3, - MPI_COMM_WORLD, &recv_request[i] ) ; + MPI_Irecv(recv_buffer[i], recv_size[i], MPI_INT, + d->xmit[i].proc, 3, + facts.get_comm(), &recv_request[i] ) ; } for(size_t i=0;icopy.size();++i) { @@ -1309,8 +1311,8 @@ namespace Loci { send_buffer[i][j++] = l2g[*ei] ; } int send_size = 2*temp.size() ; - MPI_Send(send_buffer[i],send_size, MPI_INT, d->copy[i].proc, - 3,MPI_COMM_WORLD) ; + MPI_Send(send_buffer[i], send_size, MPI_INT, + d->copy[i].proc, 3, facts.get_comm()) ; } if(d->xmit.size() > 0) { @@ -1523,8 +1525,9 @@ namespace Loci { // post the actual recv requests for(size_t i=0;i& units = send_info[i].units ; if(resend_flag[i]) // pack the message size - MPI_Pack(&send_info[i].max_send_size,1,MPI_INT,send_info[i].buf, - send_info[i].send_size,&pack_offset,MPI_COMM_WORLD) ; + MPI_Pack(&send_info[i].max_send_size, 1, MPI_INT, + send_info[i].buf, send_info[i].send_size, + &pack_offset, facts.get_comm()) ; else for(size_t k=0;k recv_info[i].recv_size) recv_info[i].recv_size = rs ; } else @@ -1638,8 +1644,9 @@ namespace Loci { int proc = rerecv[i].proc ; int idx = rerecv[i].idx ; recv_info[idx].buf = new unsigned char[recv_info[idx].recv_size] ; - MPI_Irecv(recv_info[idx].buf,recv_info[idx].recv_size, - MPI_PACKED,proc,tag2,MPI_COMM_WORLD,&recv_req[i]) ; + MPI_Irecv(recv_info[idx].buf, recv_info[idx].recv_size, + MPI_PACKED, proc, tag2, + facts.get_comm(), &recv_req[i]) ; } // do the repack/resend @@ -1657,8 +1664,9 @@ namespace Loci { for(size_t i=0;i maxr_size[i]) maxr_size[i] = temp ; } @@ -1971,7 +1984,10 @@ namespace Loci { for(int i = 0; i < rerecv_size; i++) { int proc = recv_info[recv_index[i]].first ; recv_ptr[recv_index[i]] = new unsigned char[maxr_size[recv_index[i]]] ; - MPI_Irecv(recv_ptr[recv_index[i]], maxr_size[recv_index[i]], MPI_PACKED, proc, 2, MPI_COMM_WORLD, &re_request[i]) ; + MPI_Irecv(recv_ptr[recv_index[i]], + maxr_size[recv_index[i]], MPI_PACKED, + proc, 2, facts.get_comm(), + &re_request[i]) ; } for(int i=0;i 0) { @@ -2153,7 +2171,7 @@ namespace Loci { for(int i=0;i 0) { @@ -2306,18 +2325,19 @@ namespace Loci { std::list plist = scheds.get_comm_info_list(barrier_vars, facts, sched_db::BARRIER_PLIST); - execute_comm2::inc_comm_step() ; + MPI_Comm comm = facts.get_comm() ; + execute_comm2::inc_comm_step(comm) ; if(!plist.empty()) { executeP tmp2 = new execute_comm2(plist, facts) ; el->append_list(tmp2) ; } - execute_comm2::inc_comm_step() ; + execute_comm2::inc_comm_step(comm) ; if(!clist.empty()) { executeP tmp2 = new execute_comm2(clist, facts) ; el->append_list(tmp2) ; } } - if(MPI_processes==1 && !isGPUSync) { + if(facts.get_comm_size()==1 && !isGPUSync) { ostringstream oss ; oss << "Sync: " << barrier_vars << endl ; executeP exec_thrd_sync = new execute_msg(oss.str()) ; @@ -2844,7 +2864,7 @@ namespace Loci { entitySet duplication_clist_procs = sending_comm_processors(duplication_clist_entities, facts); vector temp = send_entitySetv(original_clist_entities, facts); - for(int i = 0; i < Loci::MPI_processes; i++) { + for(int i = 0; i < facts.get_comm_size(); i++) { if(temp[i].size() > 0) { original_clist_procs += i; original_clist_entities += temp[i]; @@ -2852,7 +2872,7 @@ namespace Loci { } temp = send_entitySetv(duplication_clist_entities, facts); - for(int i = 0; i < Loci::MPI_processes; i++) { + for(int i = 0; i < facts.get_comm_size(); i++) { if(temp[i].size() > 0) { duplication_clist_procs += i; duplication_clist_entities += temp[i]; @@ -2887,7 +2907,7 @@ namespace Loci { entitySet original_plist_procs = sending_comm_processors(original_plist_entities, facts); temp = send_entitySetv(original_plist_entities, facts); - for(int i = 0; i < Loci::MPI_processes; i++) { + for(int i = 0; i < facts.get_comm_size(); i++) { if(temp[i].size() > 0) { original_plist_procs += i; original_plist_entities += temp[i]; @@ -3029,7 +3049,8 @@ namespace Loci { time[2] = duplication_comm_time; time[3] = duplication_comp_time; - MPI_Allreduce(time, max_time, 4, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(time, max_time, 4, MPI_DOUBLE, + MPI_MAX, facts.get_comm()) ; if((max_time[0] + max_time[1]) - (max_time[2] + max_time[3]) > -0.000000000000000001) //scheds.set_duplicate_variable(v, true); diff --git a/src/System/comp_tools.h b/src/System/comp_tools.h index 1c5e5fb9..192792f3 100644 --- a/src/System/comp_tools.h +++ b/src/System/comp_tools.h @@ -550,9 +550,9 @@ namespace Loci { virtual void Print(std::ostream& s) const ; virtual string getName() {return "execute_comm2";}; virtual void dataCollate(collectData& data_collector) const ; - static void inc_comm_step() { + static void inc_comm_step(MPI_Comm comm) { int lt=tag_base, gt=0 ; - MPI_Allreduce(<, >, 1, MPI_INT, MPI_MAX, MPI_COMM_WORLD) ; + MPI_Allreduce(<, >, 1, MPI_INT, MPI_MAX, comm) ; tag_base = gt ; if(tag_base > 32500) { tag_base=1500 ; // recycle tags diff --git a/src/System/depend_graph2.cc b/src/System/depend_graph2.cc index 58da1f9d..0ac8531e 100644 --- a/src/System/depend_graph2.cc +++ b/src/System/depend_graph2.cc @@ -21,6 +21,7 @@ #include #include "dist_tools.h" #include "loci_globs.h" +#include #include #include using std::map ; diff --git a/src/System/dist_tools.cc b/src/System/dist_tools.cc index 71803fd1..544f089c 100644 --- a/src/System/dist_tools.cc +++ b/src/System/dist_tools.cc @@ -222,9 +222,10 @@ namespace Loci { entitySet tmp_dom = p->domain() ; MapRepP mp = MapRepP(p->getRep()) ; - entitySet glob_dom = collectSet(tmp_dom,locdom[kd],MPI_COMM_WORLD) ; + entitySet glob_dom = collectSet(tmp_dom, + locdom[kd], facts.get_comm()) ; entitySet tmp_out = (glob_dom & locdom[kd]) - tmp_dom ; - storeRepP sp = mp->expand(tmp_out, ptn) ; + storeRepP sp = mp->expand(tmp_out, ptn, facts.get_comm()) ; if(sp->domain() != tmp_dom) { //facts.update_fact(variable(*vi), sp) ; sp->setDomainKeySpace(kd) ; @@ -277,7 +278,7 @@ namespace Loci { domain += mp->domain(); } } - domain = all_collect_entitySet(domain); + domain = all_collect_entitySet(domain,facts.get_comm()); tmp = domain; } @@ -288,7 +289,7 @@ namespace Loci { image += mp->image(tmp); } } - tmp = all_collect_entitySet(image); + tmp = all_collect_entitySet(image,facts.get_comm()); image = EMPTY; } @@ -319,27 +320,28 @@ namespace Loci { std::set >::const_iterator smi ; for(smi = maps.begin(); smi != maps.end(); ++smi) { - std::vector preimage_vec = all_collect_vectors(domain); + std::vector preimage_vec = all_collect_vectors(domain, facts.get_comm()); const vector &mv = *smi ; for(int i = mv.size() -1; i >= 0; --i) { variableSet v = mv[i] ; v &= vars ; - std::vector tmp_preimage_vec(MPI_processes); + std::vector tmp_preimage_vec(facts.get_comm_size()); for(variableSet::const_iterator vi = v.begin(); vi != v.end(); ++vi) { storeRepP p = facts.get_variable(*vi) ; if(isMAP(p)) { MapRepP mp = MapRepP(p->getRep()) ; - for(int j = 0; j < MPI_processes; j++) { + for(int j = 0; j < facts.get_comm_size(); j++) { tmp_preimage_vec[j] += mp->preimage(preimage_vec[j]).second; } } } - for(int j = 0; j < MPI_processes; j++) { - preimage_vec[j] = all_collect_entitySet(tmp_preimage_vec[j]); + for(int j = 0; j < facts.get_comm_size(); j++) { + preimage_vec[j] = all_collect_entitySet(tmp_preimage_vec[j], + facts.get_comm()); } if(i == 0) { - context += preimage_vec[MPI_rank]; + context += preimage_vec[facts.get_comm_rank()]; } } } @@ -372,11 +374,10 @@ namespace Loci { if(isMAP(p)) { entitySet tmp_dom = p->domain() ; MapRepP mp = MapRepP(p->getRep()) ; - entitySet glob_dom = collectSet(tmp_dom,locdom,MPI_COMM_WORLD) - - ; + entitySet glob_dom = collectSet(tmp_dom, + locdom, facts.get_comm()) ; entitySet tmp_out = (glob_dom & locdom) - tmp_dom ; - storeRepP sp = mp->expand(tmp_out, ptn) ; + storeRepP sp = mp->expand(tmp_out, ptn,facts.get_comm()) ; if(sp->domain() != tmp_dom) { //facts.update_fact(variable(*vi), sp) ; facts.replace_fact(*vi,sp) ; @@ -416,10 +417,11 @@ namespace Loci { storeRepP tmp_sp = facts.get_variable(*vi) ; if(tmp_sp->RepType() == CONSTRAINT) { entitySet tmp_dom = tmp_sp->domain() ; - if(GLOBAL_OR(tmp_dom != ~EMPTY,MPI_COMM_WORLD)) { + if(GLOBAL_OR(tmp_dom != ~EMPTY, facts.get_comm())) { std::vector &ptn = facts.get_init_ptn(tmp_sp->getDomainKeySpace()) ; - entitySet global_tmp_dom = distribute_entitySet(tmp_dom,ptn) ; + entitySet global_tmp_dom = distribute_entitySet(tmp_dom,ptn, + facts.get_comm()) ; //global_tmp_dom = all_collect_entitySet(tmp_dom) ; constraint tmp ; @@ -467,11 +469,11 @@ namespace Loci { context_maps = classify_moderate_maps(facts, context_maps); //Find out entities that can produce target variable entities owned by a processor - facts.global_comp_entities += context_for_map_output(ptn[MPI_rank], facts, context_maps); + facts.global_comp_entities += context_for_map_output(ptn[facts.get_comm_rank()], facts, context_maps); //Add entities so that maximum depth of duplication can be achieved if(multilevel_duplication) { - entitySet mySet = ptn[MPI_rank]; + entitySet mySet = ptn[facts.get_comm_rank()]; bool continue_adding = true; #ifdef VERBOSE int num_levels = 1; @@ -481,7 +483,7 @@ namespace Loci { facts, context_maps) ; entitySet added_entities = context_for_map_output(mySet, facts, context_maps); added_entities -= facts.global_comp_entities; - if(GLOBAL_AND(added_entities==EMPTY)) + if(GLOBAL_AND(added_entities==EMPTY, facts.get_comm())) continue_adding = false; else { facts.global_comp_entities += added_entities; @@ -500,7 +502,7 @@ namespace Loci { } else { for(int i=0;iimage(tmp_mp->domain()) ; std::vector &ptn = facts.get_init_ptn(kd_range) ; - keyspace_expand[kd_range]+=image-ptn[MPI_rank] ; + keyspace_expand[kd_range]+=image-ptn[facts.get_comm_rank()] ; } for(auto vi = constraintVars.begin();vi!=constraintVars.end();++vi) { @@ -531,7 +533,8 @@ namespace Loci { int kd = tmp_sp->getDomainKeySpace() ; std::vector &ptn = facts.get_init_ptn(kd) ; entitySet dom = tmp_sp->domain() ; - dom = dist_expand_entitySet(dom,keyspace_expand[kd],ptn) ; + dom = dist_expand_entitySet(dom,keyspace_expand[kd],ptn, + facts.get_comm()) ; constraint tmp ; *tmp = dom ; tmp.Rep()->setDomainKeySpace(kd) ; @@ -542,17 +545,17 @@ namespace Loci { vector > copyv(nkd), send_clonev(nkd) ; for(int kd=0;kd copy(MPI_processes), send_clone(MPI_processes) ; + vector copy(facts.get_comm_size()), send_clone(facts.get_comm_size()) ; entitySet tmp_copy ; std::vector &ptn = facts.get_init_ptn(kd) ; - tmp_copy = image[kd] - ptn[MPI_rank] ; + tmp_copy = image[kd] - ptn[facts.get_comm_rank()] ; - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes]; - int *recv_displacement = new int[MPI_processes]; + int *recv_count = new int[facts.get_comm_size()] ; + int *send_count = new int[facts.get_comm_size()] ; + int *send_displacement = new int[facts.get_comm_size()]; + int *recv_displacement = new int[facts.get_comm_size()]; int size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < facts.get_comm_size(); ++i) { copy[i] = tmp_copy & ptn[i] ; send_count[i] = copy[i].size() ; size_send += send_count[i] ; @@ -560,30 +563,30 @@ namespace Loci { int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + facts.get_comm()) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < facts.get_comm_size(); ++i) { size_send += recv_count[i] ; } int *recv_buf = new int[size_send] ; size_send = 0 ; entitySet::const_iterator ei ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < facts.get_comm_size(); ++i) for(ei = copy[i].begin(); ei != copy[i].end(); ++ei) { send_buf[size_send] = *ei ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < facts.get_comm_size(); ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - std::vector add(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + facts.get_comm()) ; + std::vector add(facts.get_comm_size()) ; + for(int i = 0; i < facts.get_comm_size(); ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i] += recv_buf[j] ; @@ -604,7 +607,7 @@ namespace Loci { variableSet vars = facts.get_typed_variables() ; double start = MPI_Wtime() ; - int myid = MPI_rank ; + int myid = facts.get_comm_rank() ; int size = 0 ; Map l2g ; Map l2f ; @@ -619,14 +622,15 @@ namespace Loci { categories(facts,iv,kd) ; // FIX THIS std::vector &ptn = facts.get_init_ptn(kd) ; for(size_t i=0;i rset(p) ; - MPI_Allgather(&iset.first,1,MPI_2INT,&rset[0].first,1,MPI_2INT,MPI_COMM_WORLD) ; + MPI_Allgather(&iset.first,1,MPI_2INT,&rset[0].first,1,MPI_2INT,comm) ; // Remove redundant info @@ -1163,7 +1169,7 @@ namespace Loci { s.start() ; // If the remainder is empty, then we are finished - if(GLOBAL_AND((inSet==EMPTY),MPI_COMM_WORLD)) { + if(GLOBAL_AND((inSet==EMPTY),comm)) { return retval ; } @@ -1174,7 +1180,7 @@ namespace Loci { (iset2.second-iset2.first) < (retval[i].second-retval[i].first)) iset2 = retval[i] ; - MPI_Allgather(&iset2.first,1,MPI_2INT,&rset[0].first,1,MPI_2INT,MPI_COMM_WORLD) ; + MPI_Allgather(&iset2.first,1,MPI_2INT,&rset[0].first,1,MPI_2INT,comm) ; // Remove redundant info @@ -1184,7 +1190,7 @@ namespace Loci { // If the remainder is empty, then we are finished - if(GLOBAL_AND((inSet==EMPTY),MPI_COMM_WORLD)) { + if(GLOBAL_AND((inSet==EMPTY),comm)) { return retval ; } @@ -1201,7 +1207,7 @@ namespace Loci { vector recv_sz(p) ; MPI_Alltoall(&send_sz[0],1,MPI_INT, &recv_sz[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int size_send = 0 ; int size_recv = 0 ; @@ -1228,7 +1234,7 @@ namespace Loci { MPI_Alltoallv(&send_store[0],&send_sz[0], &send_displacement[0], MPI_2INT, &recv_store[0],&recv_sz[0], &recv_displacement[0], MPI_2INT, - MPI_COMM_WORLD) ; + comm) ; // Sort received intervals to increase performance of union code sort(recv_store.begin(),recv_store.end()) ; @@ -1238,7 +1244,11 @@ namespace Loci { return retval ; } - +#ifndef LOCI_STRICT_COMM + entitySet dist_collect_entitySet(entitySet inSet, const vector &ptn) { + dist_collect_entitySet(inSet,ptn,MPI_COMM_WORLD) ; + } +#endif // ----------------------------------------------------------------------- /// @brief dist_expand_entitySet() is given an input set and a set of /// entities that are in the clone region. The ownership of entities @@ -1253,7 +1263,7 @@ namespace Loci { /// @param [ptn] partition function that describes which processor owns /// which entities. entitySet dist_expand_entitySet(entitySet inSet, entitySet clone, - const vector &ptn) { + const vector &ptn, MPI_Comm comm) { // We don't need to send data to ourselves clone -= inSet ; @@ -1274,7 +1284,7 @@ namespace Loci { // Send clone requests to partners vector recv_req(ptn.size()*2) ; MPI_Alltoall(&send_req[0],2,MPI_INT, &recv_req[0],2,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; // From the clone requests, find a set that we need to send to the // corresponding processor @@ -1289,7 +1299,7 @@ namespace Loci { // Share the size information with partners vector recv_sizes(ptn.size()) ; MPI_Alltoall(&send_sizes[0],1,MPI_INT, &recv_sizes[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; // Compress all of the recieves to a single buffer, compute offsets // within each buffer for the recieves @@ -1312,14 +1322,14 @@ namespace Loci { int recv_offset = recv_offsets[i]*2 ; MPI_Irecv(&(recv_buffer[recv_offset]),recv_size,MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[cnt++]) ; + comm,&recv_Requests[cnt++]) ; } } // Send the data to partner for(size_t i=0;i < ptn.size();++i) { if(send_sizes[i] != 0) { - MPI_Send(&(send_data[i][0]),send_sizes[i]*2,MPI_INT,i,3,MPI_COMM_WORLD) ; + MPI_Send(&(send_data[i][0]),send_sizes[i]*2,MPI_INT,i,3,comm) ; } } @@ -1335,7 +1345,12 @@ namespace Loci { return inSet ; } - +#ifndef LOCI_STRICT_COMM + entitySet dist_expand_entitySet(entitySet inSet, entitySet clone, + const vector &ptn) { + dist_expand_entitySet(inSet,clone,ptn,MPI_COMM_WORLD) ; + } +#endif inline bool spec_ival_compare(const interval &i1, const interval &i2) { if(i1.first < i2.first) @@ -1346,15 +1361,15 @@ namespace Loci { } // Return union of all entitySets from all processors - entitySet all_gather_entitySet(const entitySet &e) { - const int p = MPI_processes ; + entitySet all_gather_entitySet(const entitySet &e, MPI_Comm comm) { + int p = 0 ; MPI_Comm_size(comm, &p) ; if(p == 1) return e ; int send_count = 2*e.num_intervals() ; vector recv_count(p) ; MPI_Allgather(&send_count,1,MPI_INT,&recv_count[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; vector recv_disp(p) ; recv_disp[0] = 0 ; for(int i=1;i &ptn) { - - const int p = MPI_processes ; + entitySet distribute_entitySet(entitySet e,const vector &ptn, MPI_Comm comm) { + int p = 0 ; MPI_Comm_size(comm, &p) ; // Single processor, do nothing if(p == 1) return e ; // if any processor contains universal set, all processors hold the // universal set - if(GLOBAL_OR(e == ~EMPTY)) + if(GLOBAL_OR(e == ~EMPTY, comm)) return ~EMPTY ; + int r = 0 ; MPI_Comm_rank(comm, &r) ; // if no communication need, then just return the input set - if(GLOBAL_AND((e & ptn[MPI_rank]) == e)) + if(GLOBAL_AND((e & ptn[r]) == e, comm)) return e ; - // Use recursive splitting to get set to owning processor. - const int r = MPI_rank ; - int nump = p ; // number of processors in the current group int start = 0 ; // starting processor number for the current group @@ -1438,23 +1450,23 @@ namespace Loci { // next lower processor so it gets shared with the other partition if(r == oddproc) { int set_ivls = give.num_intervals() ; - MPI_Send(&set_ivls,1,MPI_INT,r-1,11,MPI_COMM_WORLD); + MPI_Send(&set_ivls,1,MPI_INT,r-1,11,comm); if(set_ivls > 0) { vector ivl_list(set_ivls) ; for(int i=0;i 0) { vector ivl_list(set_ivls) ; - MPI_Recv(&ivl_list[0],2*set_ivls,MPI_INT,r+1,12,MPI_COMM_WORLD, + MPI_Recv(&ivl_list[0],2*set_ivls,MPI_INT,r+1,12,comm, &status) ; for(int i=0;i 0) { vector ivl_list(set_ivls) ; for(int i=0;i 0) { vector ivl_list(set_ivls) ; MPI_Recv(&ivl_list[0],2*set_ivls,MPI_INT,partner,12, - MPI_COMM_WORLD, &status) ; + comm, &status) ; for(int i=0;i 0) { vector ivl_list(set_ivls) ; - MPI_Recv(&ivl_list[0],2*set_ivls,MPI_INT,partner,12,MPI_COMM_WORLD, + MPI_Recv(&ivl_list[0],2*set_ivls,MPI_INT,partner,12,comm, &status) ; for(int i=0;i 0) { vector ivl_list(set_ivls) ; @@ -1510,7 +1522,7 @@ namespace Loci { ivl_list[i] = give[i] ; MPI_Send(&ivl_list[0],2*set_ivls,MPI_INT,partner,12, - MPI_COMM_WORLD) ; + comm) ; } } } @@ -1519,13 +1531,13 @@ namespace Loci { start = nstart ; } - return e & ptn[MPI_rank] ; + return e & ptn[r] ; } // Collect largest interval of entitySet from all processors - entitySet collectLargest(const entitySet &e) { - const int p = MPI_processes ; + entitySet collectLargest(const entitySet &e, MPI_Comm comm) { + int p = 0 ; MPI_Comm_size(comm, &p) ; // Else we compute set //First get largest interval interval ivl_large(1,-1) ; @@ -1538,7 +1550,7 @@ namespace Loci { vector ivl_large_p(p) ; MPI_Allgather(&ivl_large,2,MPI_INT,&(ivl_large_p[0]),2,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; entitySet lset ; for(int i=0;iRepType() == PARAMETER) - write_parameterS(group_id,qrep,MPI_COMM_WORLD) ; + write_parameterS(group_id,qrep,comm) ; else - write_storeS(group_id,qrep,qrep->domain(),offset,MPI_COMM_WORLD) ; + write_storeS(group_id,qrep,qrep->domain(),offset,comm) ; } - void write_containerP(hid_t group_id, storeRepP qrep) { + void write_containerP(hid_t group_id, storeRepP qrep, MPI_Comm comm) { #ifndef H5_HAVE_PARALLEL - write_containerS(group_id, qrep); + write_containerS(group_id, qrep, comm); #else int offset = 0 ; if(qrep->RepType() == PARAMETER) - write_parameterP(group_id,qrep,MPI_COMM_WORLD) ; + write_parameterP(group_id,qrep,comm) ; else - write_storeP(group_id,qrep,qrep->domain(),offset,MPI_COMM_WORLD) ; + write_storeP(group_id,qrep,qrep->domain(),offset,comm) ; #endif } @@ -912,7 +910,7 @@ namespace Loci { void read_storeS(hid_t group_id, storeRepP qrep, int &offset, MPI_Comm comm) { #ifdef io_performance - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(comm); Loci::stopWatch s; s.start(); #endif @@ -1123,7 +1121,7 @@ namespace Loci { delete [] tmp_buf ; delete [] tmp_int ; #ifdef io_performance - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(comm); double wall_time = s.stop(); if(prank == 0) std::cerr << " serial time to read_store " << " " << wall_time << endl; #endif @@ -1135,7 +1133,7 @@ namespace Loci { read_storeS(group_id, qrep, offset, comm); #else #ifdef io_performance - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(comm); Loci::stopWatch s; s.start(); #endif @@ -1242,7 +1240,7 @@ namespace Loci { H5Dclose(dataset) ; H5Sclose(dataspace) ; #ifdef io_performance - MPI_Barrier(MPI_COMM_WORLD); + MPI_Barrier(comm); double wall_time = s.stop(); if(prank == 0) std::cerr << " parallel time to read_store " << " " << wall_time << endl; #endif @@ -1314,56 +1312,60 @@ namespace Loci { - entitySet findBoundingSet(entitySet in) { + entitySet findBoundingSet(entitySet in, MPI_Comm comm) { Entity max_val = in.Max() ; Entity min_val = in.Min() ; Entity gmin_val = min_val; Entity gmax_val = max_val ; - MPI_Allreduce(&min_val,&gmin_val,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; - MPI_Allreduce(&max_val,&gmax_val,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&min_val, &gmin_val, 1, MPI_INT, + MPI_MIN, comm) ; + MPI_Allreduce(&max_val, &gmax_val, 1, MPI_INT, + MPI_MAX, comm) ; return entitySet(interval(gmin_val,gmax_val)) ; } - vector transposeSeq(const vector sv) { - vector send_sz(MPI_processes) ; - for(int i=0;i transposeSeq(const vector sv, MPI_Comm comm) { + int np ; MPI_Comm_size(comm, &np) ; + vector send_sz(np) ; + for(int i=0;i recv_sz(MPI_processes) ; - MPI_Alltoall(&send_sz[0],1,MPI_INT, - &recv_sz[0],1,MPI_INT, - MPI_COMM_WORLD) ; + vector recv_sz(np) ; + MPI_Alltoall(&send_sz[0], 1, MPI_INT, + &recv_sz[0], 1, MPI_INT, + comm) ; int size_send = 0 ; int size_recv = 0 ; - for(int i=0;iallocate(new_alloc) ; int *send_store = new int[size_send] ; int *recv_store = new int[size_recv] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_sz[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_sz[i-1] ; } - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(size_t j=0;j sv_t(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) + vector sv_t(np) ; + for(int i = 0; i < np; ++i) for(int j=0;j recv_seqs = transposeSeq(send_seqs) ; + vector recv_seqs = transposeSeq(send_seqs, comm) ; // shift by the offset @@ -1564,8 +1566,9 @@ namespace Loci { // fact_db pointer (facts) // MPI Communicator storeRepP Local2FileOrder_output(storeRepP sp, entitySet dom, - fact_db& facts, MPI_Comm comm) { + fact_db& facts) { + MPI_Comm comm = facts.get_comm() ; // Get number of processors int p = 0 ; MPI_Comm_size(comm,&p) ; @@ -1626,7 +1629,7 @@ namespace Loci { } //Get the sequences of where we place the data when we receive it - vector recv_seqs = transposeSeq(send_seqs) ; + vector recv_seqs = transposeSeq(send_seqs, comm) ; // don't need shift by the offset because global number is unique int offset = out_ptn[prank].Min() ; @@ -1768,7 +1771,7 @@ namespace Loci { // Transpose the send requests to get the sending sequences // from this processor - vector send_seq = transposeSeq(send_req) ; + vector send_seq = transposeSeq(send_req, comm) ; vector send_sets(p) ; for(int i=0;iRepType() != PARAMETER && MPI_processes != 1) { - // parallel store write.. reorder to file numbering then write out - // reorder from local to file numbering + if(var->RepType() != PARAMETER && facts.get_comm_size() != 1) { int offset = 0 ; entitySet dom = var->domain() ; - // Create container vardist that is ordered across processors in the - // file numbering, the domain of this container shifted by offset - // is the actual file numbering. - storeRepP vardist = Local2FileOrder(var,dom,offset,dist,MPI_COMM_WORLD) ; - // Write out container that has been distributed in the file numbering + storeRepP vardist = + Local2FileOrder(var, dom, offset, dist, comm) ; if(use_parallel_io) - pio::write_storeP(group_id,vardist,vardist->domain(),offset,MPI_COMM_WORLD) ; + pio::write_storeP(group_id, vardist, + vardist->domain(), offset, comm) ; else - pio::write_storeS(group_id,vardist,vardist->domain(),offset,MPI_COMM_WORLD) ; + pio::write_storeS(group_id, vardist, + vardist->domain(), offset, comm) ; } else { // No need to reorder container if parameter or only one // processor, so just write out the container. if(use_parallel_io) - pio::write_containerP(group_id, var) ; + pio::write_containerP(group_id, var, comm) ; else - pio::write_containerS(group_id, var) ; + pio::write_containerS(group_id, var, comm) ; } - if(MPI_rank == 0||use_parallel_io) + if(facts.get_comm_rank() == 0||use_parallel_io) H5Gclose(group_id) ; } int getMinFileNumberFromLocal(entitySet read_set, - fact_db::distribute_infoP dist ) { - + fact_db::distribute_infoP dist, + MPI_Comm comm) { int minIDfl = std::numeric_limits::max() ; - int kd = getKeyDomain(read_set, dist, MPI_COMM_WORLD) ; + int kd = getKeyDomain(read_set, dist, comm) ; if(kd< 0) { cerr << "read_set not in single keyspace!" << endl ; kd = 0 ; } - // local 2 file Map l2f ; l2f = dist->l2f.Rep() ; - // Compute map from local numbering to file numbering FORALL(read_set,ii) { minIDfl = min(minIDfl,l2f[ii]) ; } ENDFORALL ; int minIDf = minIDfl ; - MPI_Allreduce(&minIDfl,&minIDf,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&minIDfl, &minIDf, 1, MPI_INT, + MPI_MIN, comm) ; return minIDf ; } - //serial/parallel io void read_container_redistribute(hid_t file_id, std::string vname, storeRepP var, entitySet read_set, fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; hid_t group_id = 0; - if(use_parallel_io || MPI_rank == 0) + if(use_parallel_io || facts.get_comm_rank() == 0) group_id = H5Gopen(file_id, vname.c_str(),H5P_DEFAULT) ; if(var->RepType() == PARAMETER) { - read_parameter(group_id, var, MPI_COMM_WORLD) ; - if(use_parallel_io || MPI_rank == 0) + read_parameter(group_id, var, comm) ; + if(use_parallel_io || facts.get_comm_rank() == 0) H5Gclose(group_id) ; return ; } @@ -1949,27 +1949,25 @@ namespace Loci { // Read in store in file numbering int offset = 0 ; storeRepP new_store = var->new_store(EMPTY) ; - if(use_parallel_io ) - pio::read_storeP( group_id, new_store,offset,MPI_COMM_WORLD) ; - else pio::read_storeS( group_id, new_store,offset,MPI_COMM_WORLD) ; + if(use_parallel_io) + pio::read_storeP(group_id, new_store, offset, comm) ; + else + pio::read_storeS(group_id, new_store, offset, comm) ; - if(use_parallel_io || MPI_rank == 0) + if(use_parallel_io || facts.get_comm_rank() == 0) H5Gclose(group_id) ; - // map from file number to local numbering fact_db::distribute_infoP dist = facts.get_distribute_info() ; if(dist != 0) { - // Correct offset if file numbering changes. Assume read_set is being - // read in over the same set int minID = offset ; - MPI_Bcast(&minID,1,MPI_INT,0,MPI_COMM_WORLD) ; - const int minIDf = getMinFileNumberFromLocal(read_set,dist) ; + MPI_Bcast(&minID, 1, MPI_INT, 0, comm) ; + const int minIDf = getMinFileNumberFromLocal(read_set,dist,comm) ; const int correct = minIDf - minID ; offset += correct ; - // Allocate space for reordered container storeRepP result = var->new_store(read_set) ; - File2LocalOrder(result,read_set,new_store,offset,dist,MPI_COMM_WORLD) ; + File2LocalOrder(result, read_set, new_store, + offset, dist, comm) ; // Copy results into container if(read_set == EMPTY) { read_set = result->domain() ; @@ -1995,12 +1993,13 @@ namespace Loci { void writeSetIds(hid_t file_id, entitySet local_set, fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; vector ids(local_set.size()) ; int c = 0 ; - if(MPI_processes > 1) { + if(facts.get_comm_size() > 1) { fact_db::distribute_infoP df = facts.get_distribute_info() ; - int kd = getKeyDomain(local_set, df, MPI_COMM_WORLD) ; + int kd = getKeyDomain(local_set, df, comm) ; if(kd < 0) { cerr << "unable to find distribute info in writeSetIds" << endl; kd = 0 ; @@ -2020,7 +2019,7 @@ namespace Loci { } ENDFORALL ; } - writeUnorderedVector(file_id,"entityIds",ids) ; + writeUnorderedVector(file_id,"entityIds",ids, comm) ; } @@ -2033,20 +2032,21 @@ namespace Loci { // file_id = H5Fcreate(filename,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT) ;//error - file_id = writeVOGOpen(filename); - if(MPI_rank == 0 || use_parallel_io) { + file_id = writeVOGOpen(filename, facts.get_comm()); + if(facts.get_comm_rank() == 0 || use_parallel_io) { group_id = H5Gcreate(file_id,"dataInfo", H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT) ; } writeSetIds(group_id,set,facts) ; - if(MPI_rank == 0 || use_parallel_io) + if(facts.get_comm_rank() == 0 || use_parallel_io) H5Gclose(group_id) ; return file_id ; } - void closeUnorderedFile(hid_t file_id) { - if(MPI_rank == 0 || use_parallel_io) + void closeUnorderedFile(hid_t file_id, MPI_Comm comm) { + int r ; MPI_Comm_rank(comm, &r) ; + if(r == 0 || use_parallel_io) H5Fclose(file_id) ; } @@ -2160,7 +2160,7 @@ namespace Loci { offsets[i+1] = offsets[i]+skipsz[i] ; //Get the sequences of where we place the data when we receive it - vector recv_seqs = transposeSeq(send_seqs) ; + vector recv_seqs = transposeSeq(send_seqs, comm) ; if(global2local.size() > 0) { // now map the recv seqs to local ordering int Imn = 0 ; @@ -2523,7 +2523,8 @@ namespace Loci { } - void getL2FMap(Map &l2f, entitySet dom, fact_db::distribute_infoP dist) { + void getL2FMap(Map &l2f, entitySet dom, fact_db::distribute_infoP dist, + MPI_Comm comm) { if(dist == 0) { l2f.allocate(dom) ; FORALL(dom,ii) { @@ -2538,7 +2539,7 @@ namespace Loci { mnl = min(mnl,l2f[ii]) ; } ENDFORALL ; int mn=mnl ; - MPI_Allreduce(&mnl,&mn,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&mnl, &mn, 1, MPI_INT, MPI_MIN, comm) ; FORALL(dom,ii) { l2f[ii] -= mn ; } ENDFORALL ; @@ -2693,7 +2694,7 @@ namespace Loci { } //Get the sequences of where we place the data when we receive it - vector send_seqs = transposeSeq(recv_seqs) ; + vector send_seqs = transposeSeq(recv_seqs, comm) ; // Compute allocation domain @@ -2796,7 +2797,7 @@ namespace Loci { } //Get the sequences of where we place the data when we receive it - vector send_seqs = transposeSeq(recv_seqs) ; + vector send_seqs = transposeSeq(recv_seqs, comm) ; // Compute allocation domain diff --git a/src/System/dynamic_exec.cc b/src/System/dynamic_exec.cc index edf63b7a..3f0c74ac 100644 --- a/src/System/dynamic_exec.cc +++ b/src/System/dynamic_exec.cc @@ -391,9 +391,10 @@ namespace Loci // first we need to get the size of the iterate space int lsz = exec_set.size() ; int gsz = 0 ; - MPI_Allreduce(&lsz,&gsz,1,MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&lsz, &gsz, 1, MPI_INT, MPI_SUM, + facts.get_comm()) ; // Now compute chomp size - int p = MPI_processes ; + int p = facts.get_comm_size() ; // Chunk size int CHUNKING_FACTOR=256 ; int csz = max(gsz/(p*CHUNKING_FACTOR),10) ; @@ -455,9 +456,9 @@ namespace Loci // ======================================================================== // proc count, rank - nProcs = Loci::MPI_processes; + nProcs = facts.get_comm_size() ; - myRank = Loci::MPI_rank; + myRank = facts.get_comm_rank() ; foreMan = 0; // first items, item counts @@ -473,7 +474,8 @@ namespace Loci yMapSave.swap(tmp) ; } - MPI_Allgather (i2, 2, MPI_INT, &yMapSave[0], 2, MPI_INT, MPI_COMM_WORLD); + MPI_Allgather (i2, 2, MPI_INT, &yMapSave[0], 2, MPI_INT, + facts.get_comm()) ; allItems = 0; for (int proc = 0; proc < nProcs; proc++) allItems = allItems + yMapSave[2 * proc + 1]; @@ -540,7 +542,7 @@ namespace Loci debugout << "Sending data to " << dest << endl ; #endif MPI_Send (&buf[0], buf_size, MPI_PACKED, dest, TAG_INPUT, - MPI_COMM_WORLD); + facts1->get_comm()) ; } } else if(recvChunks.size() != 0) { // First post Irecvs @@ -559,7 +561,7 @@ namespace Loci debugout << "recieving data from " << src << endl ; #endif MPI_Irecv (&buf[offset], buf_size, MPI_PACKED, src, TAG_INPUT, - MPI_COMM_WORLD, &req_list[i]); + facts1->get_comm(), &req_list[i]) ; offset += buf_size ; } @@ -670,11 +672,11 @@ namespace Loci debugout << "recieving computed data from " << dest << endl ; #endif MPI_Irecv (&buf[offset1], buf_size, MPI_PACKED, dest, TAG_OUTPUT, - MPI_COMM_WORLD, &req_list[i*2]); + facts1->get_comm(), &req_list[i*2]) ; offset1 += buf_size ; int nchunks = sendChunks[i].chunkList.size() ; - MPI_Irecv(×[offset2],nchunks,MPI_FLOAT,dest,TAG_TIMES, - MPI_COMM_WORLD, &req_list[i*2+1]); + MPI_Irecv(×[offset2], nchunks, MPI_FLOAT, dest, TAG_TIMES, + facts1->get_comm(), &req_list[i*2+1]) ; offset2 += nchunks ; } @@ -730,11 +732,11 @@ namespace Loci debugout << "sending computed data to " << dest << endl ; #endif MPI_Send (&buf[0], buf_size, MPI_PACKED, dest, TAG_OUTPUT, - MPI_COMM_WORLD); + facts1->get_comm()) ; int nchunks = recvChunks[i].chunkList.size() ; - MPI_Send(&remote_times[cnt2],nchunks,MPI_FLOAT,dest,TAG_TIMES, - MPI_COMM_WORLD) ; + MPI_Send(&remote_times[cnt2], nchunks, MPI_FLOAT, dest, + TAG_TIMES, facts1->get_comm()) ; cnt2 += nchunks ; } // Release space for the remote computation @@ -757,12 +759,14 @@ namespace Loci // Estimate parallel efficiency //======================================================= float total_time = 0 ; - MPI_Allreduce(&exec_time,&total_time,1,MPI_FLOAT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&exec_time, &total_time, 1, MPI_FLOAT, MPI_SUM, + facts1->get_comm()) ; float max_time = 0 ; - MPI_Allreduce(&exec_time,&max_time,1,MPI_FLOAT,MPI_MAX,MPI_COMM_WORLD) ; - double ef = total_time/(max_time*float(MPI_processes)) ; + MPI_Allreduce(&exec_time, &max_time, 1, MPI_FLOAT, MPI_MAX, + facts1->get_comm()) ; + double ef = total_time/(max_time*float(facts1->get_comm_size())) ; - if(MPI_rank == 0) + if(facts1->get_comm_rank() == 0) Loci::debugout << "parallel efficiency of IWS scheduled computation is " << ef*100.0 << "%, time =" << max_time << endl ; //======================================================= @@ -780,8 +784,9 @@ namespace Loci #ifdef VERBOSE debugout << "reducing " << time << endl ; #endif - MPI_Allreduce(&time,&total_time,1,MPI_FLOAT,MPI_SUM,MPI_COMM_WORLD) ; - float mean_time = total_time/float(MPI_processes) ; + MPI_Allreduce(&time, &total_time, 1, MPI_FLOAT, MPI_SUM, + facts1->get_comm()) ; + float mean_time = total_time/float(facts1->get_comm_size()) ; float diff_time = time-mean_time ; if(fabs(diff_time) < mean_time*eff_tol) // If close to average time, zero diff_time = 0 ; @@ -844,21 +849,21 @@ namespace Loci diff_time += (chunkData[send_list[i]].chunkTime[0]+ chunkData[send_list[i]].chunkTime[1]) ; } - vector time_xfers(MPI_processes) ; - MPI_Allgather(&diff_time, 1, MPI_FLOAT, &time_xfers[0],1,MPI_FLOAT, - MPI_COMM_WORLD) ; + vector time_xfers(facts1->get_comm_size()) ; + MPI_Allgather(&diff_time, 1, MPI_FLOAT, &time_xfers[0], 1, + MPI_FLOAT, facts1->get_comm()) ; #ifdef VERBOSE - if(MPI_rank == 0) { + if(facts1->get_comm_rank() == 0) { debugout << "mean_time = " << mean_time << endl ; debugout << "time xfers ="; - for(int i=0;iget_comm_size();++i) debugout << " " << time_xfers[i] ; debugout << endl ; } #endif vector > send_chunks ; vector > recv_chunks ; - for(int i=0;iget_comm_size();++i) { if(time_xfers[i] > 0.0) send_chunks.push_back(pair(time_xfers[i],i)) ; if(time_xfers[i] < 0.0) @@ -877,7 +882,7 @@ namespace Loci if(ptime - send_chunks[j].first > -mean_time*eff_tol) { ptime -= send_chunks[j].first ; //assign all chunks from this processor - if(send_chunks[j].second == MPI_rank) { + if(send_chunks[j].second == facts1->get_comm_rank()) { #ifdef VERBOSE debugout << "adding " << send_list.size() << "chunks to sendto" << endl ; @@ -901,52 +906,52 @@ namespace Loci } #ifdef VERBOSE - if(MPI_rank == 0) { + if(facts1->get_comm_rank() == 0) { debugout << "time xfers2 ="; - for(int i=0;iget_comm_size();++i) debugout << " " << time_xfers[i] ; debugout << endl ; } #endif bool rebalance = false ; - for(int i=0;iget_comm_size();++i) if(time_xfers[i] > 0) rebalance = true ; bool sources = false ; - for(int i=0;iget_comm_size();++i) if(time_xfers[i] < 0) sources = true ; if(rebalance && sources) { // we have residual work to distribute int nchunks = send_list.size() ; - vector chunk_groups(MPI_processes) ; - MPI_Allgather(&nchunks, 1, MPI_INT, &chunk_groups[0],1,MPI_INT, - MPI_COMM_WORLD) ; + vector chunk_groups(facts1->get_comm_size()) ; + MPI_Allgather(&nchunks, 1, MPI_INT, &chunk_groups[0], 1, + MPI_INT, facts1->get_comm()) ; vector chunk_times(nchunks) ; for(int i=0;i chunk_displ(MPI_processes) ; + vector chunk_displ(facts1->get_comm_size()) ; chunk_displ[0] = 0 ; - for(int i=1;iget_comm_size();++i) chunk_displ[i] = chunk_displ[i-1]+chunk_groups[i-1] ; - int ntchunks = (chunk_displ[MPI_processes-1]+ - chunk_groups[MPI_processes-1]) ; + int ntchunks = (chunk_displ[facts1->get_comm_size()-1]+ + chunk_groups[facts1->get_comm_size()-1]) ; vector chunk_time_gather(ntchunks) ; - MPI_Allgatherv(&chunk_times[0],nchunks,MPI_FLOAT, + MPI_Allgatherv(&chunk_times[0], nchunks, MPI_FLOAT, &chunk_time_gather[0], &chunk_groups[0], &chunk_displ[0], MPI_FLOAT, - MPI_COMM_WORLD) ; + facts1->get_comm()) ; vector > > chunkQueue(ntchunks) ; int cnk = 0 ; - for(int i=0;iget_comm_size();++i) { for(int j=0;j chunk_info(i,j) ; float chunk_time = chunk_time_gather[chunk_displ[i]+j] ; @@ -956,7 +961,7 @@ namespace Loci } } cnk = 0 ; - for(int i=0;iget_comm_size();++i) { if(chunk_groups[i] > 1) { std::sort(&chunkQueue[cnk],&chunkQueue[cnk+chunk_groups[i]]) ; } @@ -966,7 +971,7 @@ namespace Loci // sort(chunkQueue.begin(),chunkQueue.end()) ; int chunkQueueStart = chunkQueue.size()-1 ; - for(int i=MPI_processes-1;i>=0;--i) + for(int i=facts1->get_comm_size()-1;i>=0;--i) if(time_xfers[i] < 0) { vector sendto_p ; float time_x = -time_xfers[i] ; @@ -976,7 +981,7 @@ namespace Loci // assign chunk time_x -= chunkQueue[j].first ; const int cp = chunkQueue[j].second.first ; - if(cp == MPI_rank) + if(cp == facts1->get_comm_rank()) sendto_p.push_back(send_list[chunkQueue[j].second.second]) ; time_xfers[cp] -= chunkQueue[j].first ; if(time_xfers[cp] < mean_time*eff_tol) @@ -999,7 +1004,7 @@ namespace Loci } - if(MPI_rank == 0) { + if(facts1->get_comm_rank() == 0) { bool found = false ; for(int j=chunkQueueStart;j >=0;--j) if((chunkQueue[j].first > 0)) { @@ -1018,9 +1023,9 @@ namespace Loci } #ifdef VERBOSE - if(MPI_rank == 0) { + if(facts1->get_comm_rank() == 0) { debugout << "time xfers3 ="; - for(int i=0;iget_comm_size();++i) debugout << " " << time_xfers[i] ; debugout << endl ; } @@ -1055,9 +1060,9 @@ namespace Loci int local_chk_info[2],global_chk_info[2] ; local_chk_info[0] = selfChunks.size() ; local_chk_info[1] = numChunks ; - MPI_Allreduce(&local_chk_info[0],&global_chk_info[0],2,MPI_INT, - MPI_SUM, MPI_COMM_WORLD) ; - if(MPI_rank == 0) { + MPI_Allreduce(&local_chk_info[0], &global_chk_info[0], 2, + MPI_INT, MPI_SUM, facts1->get_comm()) ; + if(facts1->get_comm_rank() == 0) { debugout << "IWS Schedule: Communicating " << global_chk_info[1]-global_chk_info[0] << " chunks, " @@ -1091,18 +1096,18 @@ namespace Loci sendChunks.swap(local_send_list) ; // now invert sendChunks - vector sendSizes(MPI_processes,0) ; + vector sendSizes(facts1->get_comm_size(),0) ; for(size_t i=0;i recvSizes(MPI_processes,0) ; - MPI_Alltoall(&sendSizes[0],1,MPI_INT, - &recvSizes[0],1,MPI_INT, - MPI_COMM_WORLD) ; + vector recvSizes(facts1->get_comm_size(),0) ; + MPI_Alltoall(&sendSizes[0], 1, MPI_INT, + &recvSizes[0], 1, MPI_INT, + facts1->get_comm()) ; numRemoteChunks = 0 ; vector local_recv_list ; - for(int i=0;iget_comm_size();++i) { numRemoteChunks += recvSizes[i] ; if(recvSizes[i]!=0) { chunkCommInfo tmp ; @@ -1118,23 +1123,25 @@ namespace Loci } for(size_t i=0;iget_comm(), &tStatus) ; } for(size_t i=0;iget_comm()) ; } for(size_t i=0;iget_comm(), &tStatus) ; } for(size_t i=0;iget_comm()) ; } recvChunks.swap(local_recv_list) ; #ifdef VERBOSE @@ -1152,11 +1159,13 @@ namespace Loci int nsendrecvl = recvChunks.size()+sendChunks.size() ; int nsendrecvg = 0 ; int nsendrecvs = 0 ; - MPI_Allreduce(&nsendrecvl,&nsendrecvg,1,MPI_INT,MPI_MAX, MPI_COMM_WORLD) ; - MPI_Allreduce(&nsendrecvl,&nsendrecvs,1,MPI_INT,MPI_SUM, MPI_COMM_WORLD) ; - if(MPI_rank == 0) { + MPI_Allreduce(&nsendrecvl, &nsendrecvg, 1, MPI_INT, MPI_MAX, + facts1->get_comm()) ; + MPI_Allreduce(&nsendrecvl, &nsendrecvs, 1, MPI_INT, MPI_SUM, + facts1->get_comm()) ; + if(facts1->get_comm_rank() == 0) { debugout << "IWS Schedule: Each processor communicating with an average of " - << double(nsendrecvs)/double(MPI_processes) + << double(nsendrecvs)/double(facts1->get_comm_size()) << " processors." << endl << "IWS Schedule: Maximum number of communicating partners: " << nsendrecvg << " processors." << endl ; @@ -1169,8 +1178,9 @@ namespace Loci double timesin[2] = {comm_time,sched_time} ; double timesmx[2] = {0,0} ; - MPI_Allreduce(×in[0],×mx[0],2,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; - if(MPI_rank == 0) + MPI_Allreduce(×in[0], ×mx[0], 2, MPI_DOUBLE, MPI_MAX, + facts1->get_comm()) ; + if(facts1->get_comm_rank() == 0) Loci::debugout << "comm_time = " << timesmx[0] << ",sched_time=" << timesmx[1] << endl ; #endif //======================================================= @@ -1197,9 +1207,10 @@ namespace Loci double mu) { int bufInfo[3], pSize, tPos; int size ; - MPI_Pack_size(3,MPI_INT,MPI_COMM_WORLD,&size) ; + MPI_Comm comm = facts1->get_comm() ; + MPI_Pack_size(3, MPI_INT, comm, &size) ; pSize = size ; - MPI_Pack_size(1,MPI_DOUBLE,MPI_COMM_WORLD,&size) ; + MPI_Pack_size(1, MPI_DOUBLE, comm, &size) ; pSize += size ; GetBuffer (pSize); @@ -1207,9 +1218,9 @@ namespace Loci bufInfo[1] = chunkSize; bufInfo[2] = chunkDest; tPos = 0; - MPI_Pack (bufInfo, 3, MPI_INT, buf, pSize, &tPos, MPI_COMM_WORLD); - MPI_Pack (&mu, 1, MPI_DOUBLE, buf, pSize, &tPos, MPI_COMM_WORLD); - MPI_Send (buf, pSize, MPI_PACKED, dest, action, MPI_COMM_WORLD); + MPI_Pack (bufInfo, 3, MPI_INT, buf, pSize, &tPos, comm) ; + MPI_Pack (&mu, 1, MPI_DOUBLE, buf, pSize, &tPos, comm) ; + MPI_Send (buf, pSize, MPI_PACKED, dest, action, comm) ; #if DEBUGOUT //if (myRank == foreMan) Loci::debugout << "Info: action=" << action << ", " << myRank << "->" @@ -1224,17 +1235,18 @@ namespace Loci int bufInfo[3], pSize, tPos; MPI_Status tStatus; + MPI_Comm comm = facts1->get_comm() ; int size ; - MPI_Pack_size(3,MPI_INT,MPI_COMM_WORLD,&size) ; + MPI_Pack_size(3, MPI_INT, comm, &size) ; pSize = size ; - MPI_Pack_size(1,MPI_DOUBLE,MPI_COMM_WORLD,&size) ; + MPI_Pack_size(1, MPI_DOUBLE, comm, &size) ; pSize += size ; GetBuffer (pSize); - MPI_Recv (buf, pSize, MPI_PACKED, src, action, MPI_COMM_WORLD, &tStatus); + MPI_Recv (buf, pSize, MPI_PACKED, src, action, comm, &tStatus) ; tPos = 0; - MPI_Unpack (buf, pSize, &tPos, bufInfo, 3, MPI_INT, MPI_COMM_WORLD); - MPI_Unpack (buf, pSize, &tPos, mu, 1, MPI_DOUBLE, MPI_COMM_WORLD); + MPI_Unpack (buf, pSize, &tPos, bufInfo, 3, MPI_INT, comm) ; + MPI_Unpack (buf, pSize, &tPos, mu, 1, MPI_DOUBLE, comm) ; *chunkStart = bufInfo[0]; *chunkSize = bufInfo[1]; *chunkDest = bufInfo[2]; @@ -1245,8 +1257,9 @@ namespace Loci int position, pSize ; int bufInfo[3]; + MPI_Comm comm = facts1->get_comm() ; int size ; - MPI_Pack_size(3,MPI_INT,MPI_COMM_WORLD,&size) ; + MPI_Pack_size(3, MPI_INT, comm, &size) ; pSize = size ; pSize += inputPackSize (tStart, tSize); @@ -1256,14 +1269,14 @@ namespace Loci bufInfo[1] = tSize; bufInfo[2] = pSize; position = 0; - MPI_Pack (bufInfo, 3, MPI_INT, buf, pSize, &position, MPI_COMM_WORLD); + MPI_Pack (bufInfo, 3, MPI_INT, buf, pSize, &position, comm) ; entitySet myent = interval (tStart, tStart + tSize - 1); for (variableSet::const_iterator vi = inputs.begin (); vi != inputs.end (); ++vi) { storeRepP s_ptr = facts1->get_variable (*vi); s_ptr->pack (&buf[0], position, pSize, myent); } - MPI_Send (buf, pSize, MPI_PACKED, dest, WORK_REMOTE, MPI_COMM_WORLD); + MPI_Send (buf, pSize, MPI_PACKED, dest, WORK_REMOTE, comm) ; #if DEBUGOUT Loci::debugout << "SendInput " << myRank << "->" << dest << ", chunk=" @@ -1277,12 +1290,13 @@ namespace Loci MPI_Status tStatus; int bufInfo[3]; + MPI_Comm comm = facts1->get_comm() ; GetBuffer (msgSize); - MPI_Recv (buf, msgSize, MPI_PACKED, src, WORK_REMOTE, MPI_COMM_WORLD, - &tStatus); + MPI_Recv (buf, msgSize, MPI_PACKED, src, WORK_REMOTE, comm, + &tStatus) ; //unpack chunk info position = 0; - MPI_Unpack (buf, msgSize, &position, bufInfo, 3, MPI_INT, MPI_COMM_WORLD); + MPI_Unpack (buf, msgSize, &position, bufInfo, 3, MPI_INT, comm) ; *tStart = bufInfo[0]; *tSize = bufInfo[1]; size = bufInfo[2]; @@ -1315,10 +1329,11 @@ namespace Loci // size += s_ptr->pack_size (interval (0, tSize - 1)); // } // pSize = size + 3 * sizeof (int) + sizeof (double); + MPI_Comm comm = facts1->get_comm() ; int size ; - MPI_Pack_size(3,MPI_INT,MPI_COMM_WORLD,&size) ; + MPI_Pack_size(3, MPI_INT, comm, &size) ; pSize = size ; - MPI_Pack_size(1,MPI_DOUBLE,MPI_COMM_WORLD,&size) ; + MPI_Pack_size(1, MPI_DOUBLE, comm, &size) ; pSize += size ; pSize += outputPackSize (0, tSize); @@ -1328,8 +1343,8 @@ namespace Loci bufInfo[1] = tSize; bufInfo[2] = pSize; position = 0; - MPI_Pack (bufInfo, 3, MPI_INT, buf, pSize, &position, MPI_COMM_WORLD); - MPI_Pack (tTime, 1, MPI_DOUBLE, buf, pSize, &position, MPI_COMM_WORLD); + MPI_Pack (bufInfo, 3, MPI_INT, buf, pSize, &position, comm) ; + MPI_Pack (tTime, 1, MPI_DOUBLE, buf, pSize, &position, comm) ; //Pack outputs entitySet myent2 = interval (0, tSize - 1); for (variableSet::const_iterator vi = outputs.begin (); @@ -1338,7 +1353,7 @@ namespace Loci s_ptr->pack (&buf[0], position, pSize, myent2); } //Send outputs - MPI_Send (buf, pSize, MPI_PACKED, dest, RECV_OUTPUT, MPI_COMM_WORLD); + MPI_Send (buf, pSize, MPI_PACKED, dest, RECV_OUTPUT, comm) ; #if DEBUGOUT Loci::debugout << "SendOutput " << myRank << "->" << dest << ", chunk=" @@ -1352,13 +1367,14 @@ namespace Loci int position, tStart, tSize, size; int bufInfo[3]; + MPI_Comm comm = facts1->get_comm() ; GetBuffer (msgSize); - MPI_Recv (buf, msgSize, MPI_PACKED, src, RECV_OUTPUT, MPI_COMM_WORLD, - &tStatus); + MPI_Recv (buf, msgSize, MPI_PACKED, src, RECV_OUTPUT, comm, + &tStatus) ; //unpack chunk info position = 0; - MPI_Unpack (buf, msgSize, &position, bufInfo, 3, MPI_INT, MPI_COMM_WORLD); - MPI_Unpack (buf, msgSize, &position, tTime, 1, MPI_DOUBLE, MPI_COMM_WORLD); + MPI_Unpack (buf, msgSize, &position, bufInfo, 3, MPI_INT, comm) ; + MPI_Unpack (buf, msgSize, &position, tTime, 1, MPI_DOUBLE, comm) ; tStart = bufInfo[0]; tSize = bufInfo[1]; size = bufInfo[2]; @@ -1573,7 +1589,8 @@ namespace Loci #if DEBUGOUT Loci::debugout << "MPI_Probe() ..." << endl; #endif - MPI_Probe (MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, &mStatus); + MPI_Probe (MPI_ANY_SOURCE, MPI_ANY_TAG, + facts1->get_comm(), &mStatus) ; //timerDiff = MPI_Wtime () - timerStart; //idleTime += timerDiff; action = RETRIEVE_MSG; @@ -1588,8 +1605,8 @@ namespace Loci // ======================================================= case TEST4_MSG: // test for a message - MPI_Iprobe (MPI_ANY_SOURCE, MPI_ANY_TAG, MPI_COMM_WORLD, - &MsgInQueue, &mStatus); + MPI_Iprobe (MPI_ANY_SOURCE, MPI_ANY_TAG, + facts1->get_comm(), &MsgInQueue, &mStatus) ; if (MsgInQueue) action = RETRIEVE_MSG; else @@ -1949,7 +1966,7 @@ namespace Loci MPI_Allreduce (&workTime[0], &ewt[0], nProcs, MPI_DOUBLE, MPI_SUM, - MPI_COMM_WORLD); + facts1->get_comm()) ; // running average of work times @@ -1967,7 +1984,7 @@ namespace Loci ideal1 /= nProcs; #else // simple BARRIER - MPI_Barrier (MPI_COMM_WORLD); + MPI_Barrier (facts1->get_comm()) ; #endif } diff --git a/src/System/fact_db.cc b/src/System/fact_db.cc index c2aafeb5..a0d25103 100644 --- a/src/System/fact_db.cc +++ b/src/System/fact_db.cc @@ -53,10 +53,13 @@ namespace Loci { int factdb_allocated_base = 0 ; fact_db::fact_db() { + comm_ = MPI_COMM_WORLD ; + comm_rank_ = MPI_rank ; + comm_size_ = MPI_processes ; distributed_info = 0 ; maximum_allocated = factdb_allocated_base+0 ; vector baseline_ptn ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < comm_size_; ++i) { baseline_ptn.push_back(EMPTY) ; } init_ptn.push_back(baseline_ptn) ; @@ -72,8 +75,17 @@ namespace Loci { fact_db::~fact_db() {} + void fact_db::set_comm(MPI_Comm c) { + comm_ = c ; + MPI_Comm_rank(c, &comm_rank_) ; + MPI_Comm_size(c, &comm_size_) ; + } + void fact_db::copy_all_from(const fact_db& f) { + comm_ = f.comm_ ; + comm_rank_ = f.comm_rank_ ; + comm_size_ = f.comm_size_ ; init_ptn = f.init_ptn ; gmax_alloc = f.gmax_alloc ; keyDomainName = f.keyDomainName ; @@ -101,6 +113,7 @@ namespace Loci { df = new distribute_info ; df->myid = fdf->myid ; df->isDistributed = fdf->isDistributed ; + df->comm = fdf->comm ; // we make a deep copy of the maps entitySet l2g_alloc = fdf->l2g.domain() ; df->l2g.allocate(l2g_alloc) ; @@ -158,7 +171,7 @@ namespace Loci { distributed_info->g2lv.push_back(dMap()) ; } vector baseline_ptn ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < comm_size_; ++i) { baseline_ptn.push_back(EMPTY) ; } while(init_ptn.size() < keyDomainName.size()) @@ -353,10 +366,11 @@ namespace Loci { if(gmax_alloc.size() == 0) { getKeyDomain("Main") ; } - if(MPI_processes > 1) { + if(comm_size_ > 1) { if(!dist_from_start) { dist_from_start = 1 ; distributed_info = new distribute_info; + distributed_info->comm = comm_ ; int num_keyspace = gmax_alloc.size() ; for(int i=0;ig2fv.push_back(dMap()) ; @@ -369,26 +383,26 @@ namespace Loci { } - int* send_buf = new int[MPI_processes] ; - int* size_send = new int[MPI_processes] ; - int* size_recv = new int[MPI_processes] ; - int* recv_buf = new int[MPI_processes] ; - for(int i = 0; i < MPI_processes; ++i) { + int* send_buf = new int[comm_size_] ; + int* size_send = new int[comm_size_] ; + int* size_recv = new int[comm_size_] ; + int* recv_buf = new int[comm_size_] ; + for(int i = 0; i < comm_size_; ++i) { send_buf[i] = gmax_alloc[kd] ; size_send[i] = size ; } - MPI_Alltoall(send_buf, 1, MPI_INT, recv_buf, 1, MPI_INT, MPI_COMM_WORLD) ; - MPI_Alltoall(size_send, 1, MPI_INT, size_recv, 1, MPI_INT, MPI_COMM_WORLD) ; - std::sort(recv_buf, recv_buf+MPI_processes) ; - gmax_alloc[kd] = recv_buf[MPI_processes-1] ; + MPI_Alltoall(send_buf, 1, MPI_INT, recv_buf, 1, MPI_INT, comm_) ; + MPI_Alltoall(size_send, 1, MPI_INT, size_recv, 1, MPI_INT, comm_) ; + std::sort(recv_buf, recv_buf+comm_size_) ; + gmax_alloc[kd] = recv_buf[comm_size_-1] ; int local_max = gmax_alloc[kd] ; int global_max = 0 ; - for(int i = 0; i < MPI_rank; ++i) + for(int i = 0; i < comm_rank_; ++i) local_max += size_recv[i] ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < comm_size_; ++i) global_max += size_recv[i] ; - for(int i = 0 ; i < MPI_processes; ++i) { + for(int i = 0 ; i < comm_size_; ++i) { int local = gmax_alloc[kd] ; for(int j = 0; j < i; ++j) local += size_recv[j] ; @@ -422,7 +436,7 @@ namespace Loci { } void fact_db::update_remap(const std::vector > &remap_update, size_t kd) { - if(Loci::MPI_processes > 1) { + if(comm_size_ > 1) { warn(!dist_from_start); fatal(distributed_info == NULL); @@ -971,6 +985,7 @@ namespace Loci { fact_db::distribute_infoP df = new fact_db::distribute_info; df->myid = 0 ; df->isDistributed = 0 ; + df->comm = facts.get_comm() ; entitySet dom ; for(variableSet::const_iterator vi=vars.begin();vi!=vars.end();++vi) { storeRepP p = facts.get_variable(*vi) ; @@ -1027,7 +1042,7 @@ namespace Loci { void fact_db::write_hdf5(const char *filename, variableSet &vars) { hid_t file_id=0 ; file_id = Loci::hdf5CreateFile(filename, H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, comm_) ; for(variableSet::const_iterator vi = vars.begin(); vi != vars.end(); ++vi) { storeRepP p = get_variable(*vi) ; @@ -1035,7 +1050,7 @@ namespace Loci { writeContainer(file_id,variable(*vi).get_info().name,p,*this) ; } } - hdf5CloseFile(file_id); + hdf5CloseFile(file_id, comm_); } @@ -1043,14 +1058,14 @@ namespace Loci { void fact_db::read_hdf5(const char *filename, variableSet &vars) { hid_t file_id=0 ; - file_id = hdf5OpenFile(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + file_id = hdf5OpenFile(filename, H5F_ACC_RDONLY, H5P_DEFAULT, comm_); for(variableSet::const_iterator vi = vars.begin(); vi != vars.end(); ++vi) { storeRepP p = get_variable(*vi) ; if(isSTORE(p)) { readContainer(file_id,variable(*vi).get_info().name,p,EMPTY,*this) ; } } - hdf5CloseFile(file_id); + hdf5CloseFile(file_id, comm_); } diff --git a/src/System/gmap.cc b/src/System/gmap.cc index 4dea47b0..7dfbcfa8 100644 --- a/src/System/gmap.cc +++ b/src/System/gmap.cc @@ -32,6 +32,7 @@ #include #include #include +#include using std::cerr ; using std::endl ; using std::ostream ; @@ -159,7 +160,7 @@ namespace Loci { gEntitySet local_input_image = global_image ; - local_input_image &= init_ptn[MPI_rank] ; + local_input_image &= init_ptn[get_exec_rank()] ; for(size_t i=0;i #include #include +#include using std::cerr ; using std::endl ; using std::ostream ; @@ -155,8 +156,8 @@ namespace Loci { - int *send_displs = new int[MPI_processes] ; - int *recv_displs = new int[MPI_processes] ; + int *send_displs = new int[np] ; + int *recv_displs = new int[np] ; send_displs[0] = 0 ; recv_displs[0] = 0 ; for(int i=1;i #include "gpuMap.h" +#include using std::bad_alloc ; using std::map ; @@ -297,7 +298,7 @@ namespace Loci { return defermap->preimage(codomain) ; } - storeRepP gpuMapRepI::expand(entitySet &out_of_dom, std::vector &ptn) { + storeRepP gpuMapRepI::expand(entitySet &out_of_dom, std::vector &ptn, MPI_Comm comm) { cerr << "expand should not be called for gpuMap" << endl ; debugger_() ; return getRep() ; @@ -622,7 +623,7 @@ namespace Loci { // cout << "COLLAPSE: " << *rsi << endl ; // } } - if(MPI_rank == 0 && loopVarBase!=EMPTY) + if(facts.get_comm_rank() == 0 && loopVarBase!=EMPTY) cout << "gpu looping vars = " << loopVarBase < stmp(stencil.size()*4) ; for(size_t i=0;i0 && access[0] < 0) ; - const int p = MPI_processes ; + const int p = exec_current_fact_db->get_comm_size() ; // Now communicate the accessed info vector req_sizes(p,0) ; // Count accesses to each processor @@ -786,7 +791,7 @@ namespace Loci { vector snd_sizes(p,0) ; MPI_Alltoall(&req_sizes[0],1,MPI_INT,&snd_sizes[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int snd_tot_size = 0 ; for(int i=0;i bsize_list(p) ; - MPI_Allgather(&bsize,1,MPI_INT,&bsize_list[0],1,MPI_INT,MPI_COMM_WORLD) ; + MPI_Allgather(&bsize,1,MPI_INT,&bsize_list[0],1,MPI_INT,comm) ; #ifdef VERBOSE debugout << "bsizes = " << endl ; @@ -1267,7 +1272,7 @@ namespace Loci { lb_sizes[i] = boxes[i].stop-boxes[i].start+1 ; } - allGatherVec(b_sizes,lb_sizes,MPI_COMM_WORLD) ; + allGatherVec(b_sizes,lb_sizes,comm) ; vector box_sp_tmp(b_sizes.size()) ; // source processor of block box_sp.swap(box_sp_tmp) ; diff --git a/src/System/mod_db.cc b/src/System/mod_db.cc index d3966215..a83264e6 100644 --- a/src/System/mod_db.cc +++ b/src/System/mod_db.cc @@ -22,6 +22,7 @@ #include "loci_globs.h" #include #include +#include #include #include //#define VERBOSE @@ -139,15 +140,15 @@ namespace Loci { << endl ; #endif if((msi = mod_map.find(tmp_str)) == mod_map.end()) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) cout << "Loading in rules from " << tmp_str << endl ; md.m_library = dlopen_helper(tmp_str) ; if(md.m_library == 0) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) cerr << "unable to open " << tmp_str.c_str() << endl ; const char *error = dlerror() ; if(error) - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) cerr << "reason for failure is " << error << endl ; Loci::Abort() ; } @@ -323,7 +324,7 @@ namespace Loci { parse_str(load, str_vec) ; for(size_t i = 0; i < str_vec.size(); ++i) if(str_set.find(str_vec[i]) == str_set.end()) { - if(Loci::MPI_rank == 0) + if(Loci::MPI_rank == 0) cout << "loading in rules from " << str_vec[i] <<" for module " << to_str << endl ; load_module(str_vec[i], to_str, rdb, str_set) ; } @@ -381,7 +382,7 @@ namespace Loci { unnamedVarList.push_back(nonameset) ; for(size_t i = 0; i < str_vec.size(); ++i) if(str_set.find(str_vec[i]) == str_set.end()) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) cout << "loading in rules from " << str_vec[i] <<" for module " << to_str << endl ; load_module(str_vec[i], to_str, problem_name, facts, rdb, str_set) ; } diff --git a/src/System/multiStoreIO.cc b/src/System/multiStoreIO.cc index b93e75e1..4c3633e1 100644 --- a/src/System/multiStoreIO.cc +++ b/src/System/multiStoreIO.cc @@ -39,19 +39,20 @@ namespace Loci { fmin_local = min(fmin_local,fid) ; fmax_local = max(fmax_local,fid) ; } ENDFORALL ; + MPI_Comm comm = facts.get_comm() ; int fmin_global = fmin_local ; - MPI_Allreduce(&fmin_local,&fmin_global,1,MPI_INT, - MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&fmin_local, &fmin_global, 1, MPI_INT, + MPI_MIN, comm) ; fmin = fmin_global ; int fmax_global = fmax_local ; - MPI_Allreduce(&fmax_local,&fmax_global,1,MPI_INT, - MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&fmax_local, &fmax_global, 1, MPI_INT, + MPI_MAX, comm) ; // Find distribution of file numbers to processors int nfilenums = fmax_global-fmin_global+1 ; - const int p = MPI_processes ; - delta = (nfilenums+p-1)/MPI_processes ; - const int r = MPI_rank ; + const int p = facts.get_comm_size() ; + delta = (nfilenums+p-1)/p ; + const int r = facts.get_comm_rank() ; fstart = delta*r ; // start of file number on this processor fsz = delta ; // size of file numbers on this processor if(delta*(r+1) > nfilenums) @@ -70,7 +71,7 @@ namespace Loci { const_Map l2f ; l2f = df->l2f.Rep() ; - const int p = MPI_processes ; + const int p = facts.get_comm_size() ; // Collect information about mapping between file number and // global-processor numbering // Find file number to processor mapping @@ -80,10 +81,11 @@ namespace Loci { int ps = fid/delta ; sendto[ps]++ ; } ENDFORALL ; + MPI_Comm comm = facts.get_comm() ; std::vector recvfrom(p,0) ; - MPI_Alltoall(&sendto[0],1,MPI_INT,&recvfrom[0],1,MPI_INT, - MPI_COMM_WORLD) ; - const int r = MPI_rank ; + MPI_Alltoall(&sendto[0], 1, MPI_INT, &recvfrom[0], 1, MPI_INT, + comm) ; + const int r = facts.get_comm_rank() ; std::vector sendbuf(read_set.size()*3) ; std::vector offsets(p,0) ; @@ -118,13 +120,13 @@ namespace Loci { for(int i=0;i 0) { MPI_Irecv(&recvbuf[recv_offsets[i]*3],recvfrom[i]*3,MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&sendbuf[send_offsets[i]*3],sendto[i]*3,MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } std::vector statuslist(nreq) ; @@ -154,7 +156,7 @@ namespace Loci { } // Handle single processor case - const int p = MPI_processes ; + const int p = facts.get_comm_size() ; if(p == 1) { int imin = read_set.Min() ; for(int i=0;i recv_req(p,0) ; - MPI_Alltoall(&send_req[0],1,MPI_INT,&recv_req[0],1,MPI_INT, - MPI_COMM_WORLD) ; + MPI_Alltoall(&send_req[0], 1, MPI_INT, &recv_req[0], 1, MPI_INT, + comm) ; std::vector sendbuffer(fileID.size()) ; std::vector soffsets(p+1,0) ; std::vector roffsets(p+1,0) ; @@ -224,13 +226,13 @@ namespace Loci { for(int i=0;i 0) { MPI_Irecv(&recvbuffer[roffsets[i]],recv_req[i],MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&sendbuffer[soffsets[i]],send_req[i],MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } std::vector statuslist(nreq) ; @@ -259,13 +261,13 @@ namespace Loci { for(int i=0;i 0) { MPI_Irecv(&datarecv[soffsets[i]*2],send_req[i]*2,MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&datasend[roffsets[i]*2],recv_req[i]*2,MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } MPI_Waitall(nreq,&recv_Requests[0],&statuslist[0]) ; @@ -291,14 +293,16 @@ namespace Loci { std::vector &recv_sz, std::vector &recv_local_num, const std::vector &local_num, - const std::vector &procID) { - const int p = MPI_processes ; + const std::vector &procID, + MPI_Comm comm) { + int p = 0 ; + MPI_Comm_size(comm, &p) ; for(int i=0;i send_data(procID.size()) ; std::vector soffsets(p+1,0) ; for(int i=0;i 0) { MPI_Irecv(&recv_data[roffsets[i]],recv_sz[i],MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&send_data[soffsets[i]],send_sz[i],MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } std::vector statuslist(nreq) ; @@ -344,8 +348,10 @@ namespace Loci { const std::vector &recv_sz, const std::vector &recv_local_num, const std::vector &counts, - const std::vector &procID) { - const int p = MPI_processes ; + const std::vector &procID, + MPI_Comm comm) { + int p = 0 ; + MPI_Comm_size(comm, &p) ; std::vector soffsets(p+1,0) ; std::vector roffsets(p+1,0) ; for(int i=0;i 0) { MPI_Irecv(&rbuf[roffsets[i]],recv_sz[i],MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&sbuf[soffsets[i]],send_sz[i],MPI_INT,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } std::vector statuslist(nreq) ; diff --git a/src/System/param_rule.cc b/src/System/param_rule.cc index 416f7a56..13946890 100644 --- a/src/System/param_rule.cc +++ b/src/System/param_rule.cc @@ -23,6 +23,7 @@ using std::string ; using std::ostringstream ; #include +#include //#define VERBOSE diff --git a/src/System/rule.cc b/src/System/rule.cc index 2ac6bc2e..3d60af53 100644 --- a/src/System/rule.cc +++ b/src/System/rule.cc @@ -211,19 +211,6 @@ namespace Loci { space_dist = false ; } - rule_implP rule_impl::add_namespace(const string& n) const { - rule_implP with_namespace = new_rule_impl(); - - variableSet vars = with_namespace->get_var_list() ; - std::map new_vars; - for(variableSet::variableSetIterator i=vars.begin();i!=vars.end();++i) { - new_vars[*i] = i->add_namespace(n); - } - - with_namespace->rename_vars(new_vars); - return with_namespace ; - } - namespace { inline void fill_descriptors(set &v, const exprList &in) { @@ -693,58 +680,6 @@ namespace Loci { } - void - rule_impl::split_constraints(const variableSet& dc) { - - vector new_constraints ; - - set::iterator si=rule_info.constraints.begin() ; - set::iterator si_bak ; - - // split the constraints field into static & dynamic ones - while(si != rule_info.constraints.end()) { - variableSet local_dc ; - // get the constraints variables - for(variableSet::const_iterator vi=si->var.begin(); - vi!=si->var.end();++vi) { - // since a constraints variables may appear in other forms - // e.g., having an offset, or an assign operator, e.g., - // if we have a dynamic constraints A{n}, it may appear in - // a rule as "A{n=0}" or "A{n+1}". therefore, we need to - // drop possible assigns and offsets. - variable nv = (vi->new_offset(0)).drop_assign() ; - if(dc.inSet(nv)) - local_dc += *vi ; - } - if(local_dc != EMPTY) { - // make a copy of vmap_info - vmap_info dc_copy(*si) ; - // modify its var field - dc_copy.var = local_dc ; - // insert it into the dynamic_constraints set - rule_info.dynamic_constraints.insert(dc_copy) ; - - // construct a new copy of static constraints - variableSet local_sc = variableSet(si->var - local_dc) ; - if(local_sc != EMPTY) { - vmap_info sc_copy(*si) ; - sc_copy.var = local_sc ; - new_constraints.push_back(sc_copy) ; - } - // then erase original copy in constraints set - si_bak = si ; - ++si_bak ; - rule_info.constraints.erase(si) ; - si = si_bak ; - } else - ++si ; - } - // finally push the new static constraints ones to the set - rule_info.constraints.insert(new_constraints.begin(), - new_constraints.end()) ; - } - - variableSet rule_impl::get_var_list() { storeIMap::iterator sp ; set::const_iterator i ; @@ -791,33 +726,6 @@ namespace Loci { return vset ; } - void rule_impl::set_variable_times(time_ident tl) { - set::const_iterator i ; - set tmp ; - for(i=rule_info.sources.begin();i!=rule_info.sources.end();++i) - tmp.insert(convert_vmap_info(*i,tl)) ; - rule_info.sources.swap(tmp) ; - tmp.clear() ; - for(i=rule_info.targets.begin();i!=rule_info.targets.end();++i) - tmp.insert(convert_vmap_info(*i,tl)) ; - rule_info.targets.swap(tmp) ; - tmp.clear() ; - for(i=rule_info.constraints.begin(); - i!=rule_info.constraints.end();++i) - tmp.insert(convert_vmap_info(*i,tl)) ; - rule_info.constraints.swap(tmp) ; - tmp.clear() ; - rule_info.conditionals = convert_set(rule_info.conditionals,tl) ; - storeIMap tmp2 ; - storeIMap::iterator j ; - for(j=var_table.begin();j!=var_table.end();++j) { - variable v(j->first,tl) ; - // std::pair vsp = make_pair(v, j->second) ; - // tmp2.insert(vsp) ; - tmp2.insert(std::pair(v,j->second)) ; - } - var_table.swap(tmp2) ; - } void rule_impl::copy_store_from(rule_impl &f) { storeIMap::iterator sp ; @@ -1122,7 +1030,6 @@ namespace Loci { return ; } rule_impl = fi.rule_impl->new_rule_impl() ; - //rule_impl->set_variable_times(tl) ; variableSet vset = rule_impl->get_var_list() ; std::map rm ; for(variableSet::const_iterator vsi = vset.begin(); vsi != vset.end(); ++vsi) { @@ -1260,7 +1167,6 @@ namespace Loci { return ; } rule_impl = fi.rule_impl->new_rule_impl() ; - //rule_impl->set_variable_times(tl) ; variableSet vset = rule_impl->get_var_list() ; std::map rm ; for(variableSet::const_iterator vsi = vset.begin(); diff --git a/src/System/sched_comp.cc b/src/System/sched_comp.cc index c65b81f0..194dbe04 100644 --- a/src/System/sched_comp.cc +++ b/src/System/sched_comp.cc @@ -275,7 +275,7 @@ namespace Loci { const variableSet& given, const variableSet& target) { #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Start!" << endl ; #endif /*********************************** @@ -335,21 +335,21 @@ namespace Loci { top_down_visit(reduceV) ; #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Passed Information Collection!" << endl ; #endif det1 = MPI_Wtime() ; if(use_dynamic_memory) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) if(!in_internal_query) cout << "USING DYNAMIC MEMORY MANAGEMENT" << endl ; } // chomping searching and graph editing if(use_chomp) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) if(!in_internal_query) cout << "USING CHOMPING" << " (chomping size: " @@ -501,7 +501,7 @@ namespace Loci { det2 = MPI_Wtime() ; #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Passed Memory " << "Management Decoration!" << endl ; #endif @@ -511,7 +511,7 @@ namespace Loci { top_down_visit(dagcV) ; #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Passed Graph Cycle Check!" << endl ; #endif @@ -618,7 +618,7 @@ namespace Loci { compChompVisitor compchompv(reduceV.get_reduceInfo()) ; top_down_visit(compchompv) ; #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Passed Chomping Compilation!" << endl ; #endif } @@ -629,7 +629,7 @@ namespace Loci { schedst = MPI_Wtime() ; if(randomized_memory_greedy_schedule && (!in_internal_query)){ - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) if(!in_internal_query) cout << "graph scheduling... (randomized memory greedy)" << endl ; @@ -662,7 +662,7 @@ namespace Loci { MPI_Allreduce(local_sizes, global_sizes, num_vars, MPI_DOUBLE, - MPI_MAX,MPI_COMM_WORLD) ; + MPI_MAX, facts.get_comm()) ; id = 0; for(variableSet::const_iterator mi=all_vars.begin();mi!=all_vars.end();++mi) { @@ -682,7 +682,7 @@ namespace Loci { top_down_visit(mgs) ; }else if(!memory_greedy_schedule) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) if(!in_internal_query) cout << "graph scheduling... (computation greedy)" << endl ; @@ -692,7 +692,7 @@ namespace Loci { } else { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) if(!in_internal_query) cout << "graph scheduling... (memory greedy)" << endl ; @@ -701,7 +701,7 @@ namespace Loci { top_down_visit(gsv) ; } #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Passed Graph Scheduling!" << endl ; #endif @@ -714,7 +714,7 @@ namespace Loci { #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Passed Schedule Assembly!" << endl ; #endif @@ -733,7 +733,7 @@ namespace Loci { Loci::debugout << "Time taken for graph scheduling = " << schedet-schedst << " sceonds " << endl ; #ifdef COMPILE_PROGRESS - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Graph Compile Phase] Graph Compile Phase End!" << endl ; #endif } @@ -944,7 +944,7 @@ namespace Loci { } else if(facts.is_distributed_start()) - if((MPI_processes > 1)) + if((facts.get_comm_size() > 1)) schedule->append_list(new allocate_all_vars(facts,scheds,alloc,false)) ; schedule->append_list(fact_db_comm->create_execution_schedule(facts, scheds)); @@ -1145,7 +1145,7 @@ namespace Loci { // since the fact_db facts is in global numbering state #ifdef RENUMBER if(clone.is_distributed_start()) { - if((MPI_processes > 1)) + if((facts.get_comm_size() > 1)) get_clone(clone, par_rdb) ; else Loci::serial_freeze(clone) ; @@ -1174,7 +1174,7 @@ namespace Loci { vi2!=all_queries.end();++vi2) { storeRepP srp = clone.get_variable(*vi2) ; if(srp->RepType() == Loci::CONSTRAINT) - if(GLOBAL_AND(srp->domain()==EMPTY)) { + if(GLOBAL_AND(srp->domain()==EMPTY, facts.get_comm())) { empty_constraints += *vi2 ; // we don't create empty constraints in // the global fact database @@ -1187,7 +1187,7 @@ namespace Loci { vi2!=all_queries.end();++vi2) { storeRepP srp = clone.get_variable(*vi2) ; if(srp->RepType() == Loci::CONSTRAINT) - if(GLOBAL_AND(srp->domain()==EMPTY)) { + if(GLOBAL_AND(srp->domain()==EMPTY, facts.get_comm())) { empty_constraints += *vi2 ; continue ; } @@ -1215,7 +1215,7 @@ namespace Loci { // since the fact_db facts is in global numbering state #ifdef RENUMBER if(clone.is_distributed_start()) { - if((MPI_processes > 1)) + if((facts.get_comm_size() > 1)) get_clone(clone, par_rdb) ; else Loci::serial_freeze(clone) ; @@ -1247,7 +1247,7 @@ namespace Loci { vi2!=queries.end();++vi2) { storeRepP srp = clone.get_variable(*vi2) ; if(srp->RepType() == Loci::CONSTRAINT) - if(GLOBAL_AND(srp->domain()==EMPTY)) { + if(GLOBAL_AND(srp->domain()==EMPTY, facts.get_comm())) { empty_constraints += *vi2 ; // we don't create empty constraints in // the global fact database @@ -1265,7 +1265,7 @@ namespace Loci { vi2!=queries.end();++vi2) { storeRepP srp = clone.get_variable(*vi2) ; if(srp->RepType() == Loci::CONSTRAINT) - if(GLOBAL_AND(srp->domain()==EMPTY)) { + if(GLOBAL_AND(srp->domain()==EMPTY, facts.get_comm())) { empty_constraints += *vi2 ; continue ; } diff --git a/src/System/sched_db.cc b/src/System/sched_db.cc index d610bcde..35b244f4 100644 --- a/src/System/sched_db.cc +++ b/src/System/sched_db.cc @@ -24,6 +24,7 @@ #include #include +#include using std::string ; using std::map ; using std::make_pair ; @@ -41,8 +42,6 @@ using std::ofstream ; //#define VERBOSE namespace Loci { - extern int MPI_processes ; - extern int MPI_rank ; extern ofstream debugout ; extern bool rule_has_mapping_in_output(rule r); @@ -151,7 +150,7 @@ namespace Loci { if(all_vars.inSet(v)) { if(all_vars.inSet(alias)) { - if(MPI_processes == 1) { + if(facts.get_comm_size() == 1) { cerr << "alias already in fact_db!" << endl ; cerr << "error found in alias_variable("<RepType() != Loci::CONSTRAINT) continue ; - if(GLOBAL_AND(srp->domain() == EMPTY)) + if(GLOBAL_AND(srp->domain() == EMPTY, facts.get_comm())) empty_constraints += *vi ; } ruleSet del_rules ; @@ -692,6 +692,7 @@ namespace Loci { sched_db &scheds, const variableSet& target, int nth) { + exec_current_fact_db = &facts ; variableSet parVars = target ; parVars += facts.get_extensional_facts() ; rule_db par_rdb ; @@ -709,13 +710,14 @@ namespace Loci { //deco_depend_gr(gr,given) ; ////////////////////////////////////////////////////////////////// #ifdef ENABLE_RELATION_GEN - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "Stationary Relation Generation..." << endl ; stationary_relation_gen(par_rdb, facts, target) ; + exec_current_fact_db = &facts ; #endif // then we need to perform global -> local renumbering if(facts.is_distributed_start()) { - if((MPI_processes > 1)) + if((facts.get_comm_size() > 1)) get_clone(facts, par_rdb) ; else Loci::serial_freeze(facts) ; @@ -725,7 +727,7 @@ namespace Loci { // then we can generate the dependency graph variableSet given = facts.get_typed_variables() ; - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "generating dependency graph..." << endl ; @@ -740,13 +742,13 @@ namespace Loci { // If graph is empty, return a null schedule if(gr.get_target_vertices() == EMPTY) { - if(Loci::MPI_rank == 0) + if(facts.get_comm_rank() == 0) cerr << "Warning: empty dependency graph!" << endl ; return executeP(0) ; } //////////////////////////////////////////////////////////////////////// std::string dottycmd = "dotty " ; - if(Loci::MPI_rank==0) { + if(facts.get_comm_rank()==0) { if(show_graphs) { cout << "creating visualization file for dependency graph..." << endl ; create_digraph_dot_file(gr,"dependgr.dot") ; @@ -765,7 +767,7 @@ namespace Loci { //////////////////// scheds.init(facts) ; - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "setting up variable types..." << endl ; set_var_types(facts,gr,scheds) ; @@ -773,12 +775,12 @@ namespace Loci { //scheds.print_summary(facts,cout) ; ////////////// - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "decomposing graph..." << endl ; decomposed_graph decomp(gr,given,target) ; ////////////////////////////////////////////////////////////////// - if(Loci::MPI_rank==0) { + if(facts.get_comm_rank()==0) { if(show_graphs) { cout << "creating visualization file for decomposed graph..." << '\n' ; cout << "visualizing decomposed graph..." << '\n' ; @@ -882,7 +884,7 @@ namespace Loci { Loci::debugout << "Time taken for graph processing = " << sw.stop() << " seconds " << endl ; - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "existential analysis..." << endl ; sw.start() ; @@ -899,7 +901,7 @@ namespace Loci { } */ /////////////////////////////////// - if(Loci::MPI_rank==0) { + if(facts.get_comm_rank()==0) { #ifdef PTHREADS if(threading_pointwise || threading_global_reduction || threading_local_reduction || threading_chomping @@ -930,18 +932,18 @@ namespace Loci { Loci::debugout << "Time taken for create execution schedule = " << sw.stop() << " seconds " << endl ; - if(GLOBAL_OR(scheds.errors_found())) { - if(MPI_rank == 0) { + if(GLOBAL_OR(scheds.errors_found(), facts.get_comm())) { + if(facts.get_comm_rank() == 0) { cerr << "error in generating schedule, dumping schedule files" << endl ; - if(MPI_processes != 1) + if(facts.get_comm_size() != 1) cerr << "see debug files for more information" << endl ; } ostringstream oss ; oss << "debug/schedule" ; - if(MPI_processes > 1) { - oss << "-" << MPI_rank ; + if(facts.get_comm_size() > 1) { + oss << "-" << facts.get_comm_rank() ; } string sched_filename = oss.str() ; @@ -1490,6 +1492,7 @@ bool operator <(const timingData &d) const { sched_db &scheds, const variableSet& target, int nth) { + exec_current_fact_db = &facts ; // since this function is always executed inside // the create_execution_schedule function so the // fact database is always in the local number state @@ -1498,7 +1501,7 @@ bool operator <(const timingData &d) const { variableSet given = facts.get_typed_variables() ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) { + if(facts.get_comm_rank()==0) { cout << "[Internal] generating dependency graph..." << endl ; } #endif @@ -1506,7 +1509,7 @@ bool operator <(const timingData &d) const { // the dependency graph digraph gr ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "\t[Internal] (recursive backward searching version)" << endl ; #endif given -= variable("EMPTY") ; @@ -1529,19 +1532,19 @@ bool operator <(const timingData &d) const { scheds.init(facts) ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "[Internal] setting up variable types..." << endl ; #endif set_var_types(facts,gr,scheds) ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "[Internal] decomposing graph..." << endl ; #endif decomposed_graph decomp(gr,given,target) ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) { + if(facts.get_comm_rank()==0) { cerr << "[Internal] setting initial variables..." << endl ; } #endif @@ -1585,29 +1588,29 @@ bool operator <(const timingData &d) const { // Loci::debugout << " initial_vars = " << initial_vars << endl ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cerr << "[Internal] compiling graph..." << endl; #endif graph_compiler compile_graph(decomp, initial_vars) ; compile_graph.compile(facts,scheds,given,target) ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "[Internal] existential analysis..." << endl ; #endif compile_graph.existential_analysis(facts, scheds) ; #ifdef INTERNAL_VERBOSE - if(Loci::MPI_rank==0) + if(facts.get_comm_rank()==0) cout << "[Internal] creating execution schedule..." << endl; #endif executeP sched = compile_graph.execution_schedule (facts,scheds,initial_vars) ; - if(GLOBAL_OR(scheds.errors_found())) { - if(MPI_rank == 0) { + if(GLOBAL_OR(scheds.errors_found(), facts.get_comm())) { + if(facts.get_comm_rank() == 0) { cerr << "[Internal] error in generating schedule" << endl ; - if(MPI_processes != 1) + if(facts.get_comm_size() != 1) cerr << "[Internal] see debug files for more information" << endl ; cerr << "[Internal] Aborting..." << endl ; @@ -1629,7 +1632,7 @@ bool operator <(const timingData &d) const { stopWatch sw ; sw.start() ; - if(MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { cout << "[Internal] Quering facts: " << query << endl ; } // Here, we won't erase the intentional facts since @@ -1650,11 +1653,12 @@ bool operator <(const timingData &d) const { // If a schedule was generated, execute it #ifdef INTERNAL_VERBOSE - if(MPI_rank == 0) + if(facts.get_comm_rank() == 0) cout << "[Internal] begin query execution" << endl ; #endif exec_current_fact_db = &local_facts ; schedule->execute(local_facts, local_scheds) ; + exec_current_fact_db = &facts ; for(variableSet::const_iterator vi=query.begin(); vi!=query.end();++vi) { @@ -1663,7 +1667,8 @@ bool operator <(const timingData &d) const { } double tlocal = sw.stop() ; double tglobal = 0 ; - MPI_Allreduce(&tlocal,&tglobal, 1, MPI_DOUBLE, MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&tlocal, &tglobal, 1, MPI_DOUBLE, + MPI_MAX, facts.get_comm()) ; debugout << "time to execute internal query " << tglobal < 1) { - oss << "-" << MPI_rank ; + if(local_facts.get_comm_size() > 1) { + oss << "-" << local_facts.get_comm_rank() ; } string sched_filename = oss.str() ; @@ -1846,13 +1852,14 @@ bool operator <(const timingData &d) const { collectTiming timeProf ; schedule->dataCollate(timeProf) ; - timeProf.balanceAnalysis(MPI_COMM_WORLD) ; + timeProf.balanceAnalysis(local_facts.get_comm()) ; double compute_time_local = timeProf.getComputeTime() ; double prof_exec_time = timeProf.getTotalTime() ; double compute_time_total = 0 ; - MPI_Allreduce(&compute_time_local,&compute_time_total, 1, MPI_DOUBLE, - MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&compute_time_local, &compute_time_total, + 1, MPI_DOUBLE, MPI_SUM, + local_facts.get_comm()) ; timeProf.PrintSummary(debugout) ; @@ -1938,33 +1945,37 @@ bool operator <(const timingData &d) const { << LociInputVarsSize << " bytes (" << LociInputVarsSize/(1024*1024) << "MB)" << endl ; - if(MPI_processes > 1) { + if(local_facts.get_comm_size() > 1) { // code to find out the largest memory bounds on all processes double LargestPeakMemory = 0 ; MPI_Allreduce(&LociAppPeakMemory, &LargestPeakMemory, 1, MPI_DOUBLE, - MPI_MAX,MPI_COMM_WORLD) ; + MPI_MAX, + local_facts.get_comm()) ; double totalPeakMemory = 0 ; MPI_Allreduce(&LociAppPeakMemory, &totalPeakMemory, 1, MPI_DOUBLE, - MPI_SUM,MPI_COMM_WORLD) ; - double avgPeakMemory = totalPeakMemory/MPI_processes ; + MPI_SUM, + local_facts.get_comm()) ; + double avgPeakMemory = totalPeakMemory/local_facts.get_comm_size() ; double LargestPeakMemoryBeanCounting = 0 ; MPI_Allreduce(&LociAppPeakMemoryBeanCounting, &LargestPeakMemoryBeanCounting, 1, MPI_DOUBLE, - MPI_MAX,MPI_COMM_WORLD) ; + MPI_MAX, + local_facts.get_comm()) ; double totalPeakMemoryBeanCounting = 0 ; MPI_Allreduce(&LociAppPeakMemoryBeanCounting, &totalPeakMemoryBeanCounting, 1, MPI_DOUBLE, - MPI_SUM,MPI_COMM_WORLD) ; - double avgPeakMemoryBeanCounting = totalPeakMemoryBeanCounting/MPI_processes ; + MPI_SUM, + local_facts.get_comm()) ; + double avgPeakMemoryBeanCounting = totalPeakMemoryBeanCounting/local_facts.get_comm_size() ; Loci::debugout << endl ; Loci::debugout << "The global largest Peak Memory: " @@ -2005,139 +2016,162 @@ bool operator <(const timingData &d) const { { double mytime = ta_expand.getTime() ; double maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_expand2.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand time (2): " << mytime << ", max = " << maxtime << endl ; mytime = ta_expand_start.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand start block time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_expand_cache.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand cache management time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_expand_collect_img.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand image collection time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_expand_missing.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand missing domain comp time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_expand_comm.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total expand comm time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context_nonepd.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context non-expand block time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context_nonepd_domt.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context non-epd block (dom) time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context_nonepd_ints.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context non-epd block (int) time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context_epdend.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context expand-end block time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context_epdmid.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context expand-mid block time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_context_epdsta.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total context expand-sta block time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_output_oh.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total output overhead: " << mytime << ", max = " << maxtime << endl ; mytime = ta_compute.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total compute time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_record_erase.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total record erase time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_erase.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total erase time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_invalidate.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total invalidate time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_keyremoval.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total key removal time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_insertion.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total insertion time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_keyinsert.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total key insert time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_dist.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total distribution time: " << mytime << ", max = " << maxtime << endl ; Loci::debugout << "[dynamic] total distribution number: " @@ -2145,49 +2179,57 @@ bool operator <(const timingData &d) const { mytime = ta_key_dist.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total key distribution time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_dist_renumber.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total distribution renumber time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_push.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total push time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_pw_push.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total pointwise push time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_param_push.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total param push time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_param_pack.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total param pack time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_param_unpack.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total param unpack time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_param_reduce.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total param reduce time: " << mytime << ", max = " << maxtime << endl ; Loci::debugout << "[dynamic] total param reduction num: " @@ -2195,31 +2237,36 @@ bool operator <(const timingData &d) const { mytime = ta_dist_keys.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total dist keys time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_dist_all2all.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total dist all2all time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_dist_pack.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total dist pack time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_dist_unpack.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total dist unpack time: " << mytime << ", max = " << maxtime << endl ; mytime = ta_dctrl.getTime() ; maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1,MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, MPI_DOUBLE, + MPI_MAX, local_facts.get_comm()) ; Loci::debugout << "[dynamic] total dcontrol reset time: " << mytime << ", max = " << maxtime << endl ; @@ -2234,26 +2281,29 @@ bool operator <(const timingData &d) const { #ifdef PARTICLE_PERF { double pmaxtime = 0 ; - MPI_Allreduce(&pwalltime,&pmaxtime,1, - MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&pwalltime, &pmaxtime, 1, + MPI_DOUBLE, MPI_MAX, + local_facts.get_comm()) ; Loci::debugout << "Particle Walltime: " << pwalltime << ", max = " << pmaxtime << endl ; } #endif // communicate the execution time - if(MPI_processes > 1) { + if(local_facts.get_comm_size() > 1) { double mytime = exec_time ; double maxtime = 0 ; - MPI_Allreduce(&mytime,&maxtime,1, - MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&mytime, &maxtime, 1, + MPI_DOUBLE, MPI_MAX, + local_facts.get_comm()) ; Loci::debugout << "Global max time taken for execution" << " of the schedule = " << maxtime << " seconds " << endl ; mytime = prof_exec_time ; - MPI_Allreduce(&mytime,&maxtime,1, - MPI_DOUBLE,MPI_MAX,MPI_COMM_WORLD) ; - if(MPI_rank == 0 ) { - double eff = compute_time_total/(double(MPI_processes)*maxtime) ; + MPI_Allreduce(&mytime, &maxtime, 1, + MPI_DOUBLE, MPI_MAX, + local_facts.get_comm()) ; + if(local_facts.get_comm_rank() == 0 ) { + double eff = compute_time_total/(double(local_facts.get_comm_size())*maxtime) ; cout << "Schedule execution complete, estimated parallel efficiency = " << ceil(1000.0*eff)/10.0 << "%." << endl ; } diff --git a/src/System/thread.cc b/src/System/thread.cc index f52110db..1a91a0b6 100644 --- a/src/System/thread.cc +++ b/src/System/thread.cc @@ -612,7 +612,7 @@ namespace Loci { whole_entities = d->my_entities; } else { // note, this probably needs to be fixed for big mode - whole_entities = facts.get_init_ptn(0)[Loci::MPI_rank] ; + whole_entities = facts.get_init_ptn(0)[facts.get_comm_rank()] ; } // we will partition the entities among all the work threads. diff --git a/src/System/visit_assemble.cc b/src/System/visit_assemble.cc index 34e0b923..946656cf 100644 --- a/src/System/visit_assemble.cc +++ b/src/System/visit_assemble.cc @@ -22,6 +22,7 @@ #include "visit_tools.h" #include #include "comp_tools.h" +#include #include "dist_tools.h" @@ -199,11 +200,11 @@ namespace Loci { string filename = oss.str() ; hid_t file_id=0 ; file_id = hdf5CreateFile(filename.c_str(),H5F_ACC_TRUNC, - H5P_DEFAULT, H5P_DEFAULT) ; + H5P_DEFAULT, H5P_DEFAULT, facts.get_comm()) ; writeContainer(file_id,v.get_info().name,st,facts) ; - hdf5CloseFile(file_id) ; + hdf5CloseFile(file_id, facts.get_comm()) ; } else { - if(MPI_rank == 0) { + if(facts.get_comm_rank() == 0) { ostringstream oss ; oss << "dump_vars/"< #include diff --git a/src/System/visit_viz.cc b/src/System/visit_viz.cc index 43f9b408..74bad905 100644 --- a/src/System/visit_viz.cc +++ b/src/System/visit_viz.cc @@ -21,6 +21,7 @@ #include "visitor.h" #include "visit_tools.h" #include "comp_tools.h" +#include #include diff --git a/src/include/Config/conf.h b/src/include/Config/conf.h index f71b4ff0..4ba2a036 100644 --- a/src/include/Config/conf.h +++ b/src/include/Config/conf.h @@ -140,4 +140,10 @@ #define MPI_NO_CPPBIND #endif +#ifdef LOCI_STRICT_COMM +#define LOCI_DEFAULT_COMM +#else +#define LOCI_DEFAULT_COMM = MPI_COMM_WORLD +#endif + #endif diff --git a/src/include/DMap.h b/src/include/DMap.h index 9a909415..ca1a9502 100644 --- a/src/include/DMap.h +++ b/src/include/DMap.h @@ -57,10 +57,10 @@ namespace Loci { virtual storeRepP MapRemap(const dMap &dm, const dMap &rm) const ; virtual storeRepP redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD) ; + MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual void compose(const dMap &m, const entitySet &context) ; virtual void copy(storeRepP &st, const entitySet &context) ; virtual void gather(const dMap &m, storeRepP &st, @@ -91,7 +91,7 @@ namespace Loci { virtual void writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, hsize_t dimension, const char* name, entitySet& en, hid_t xfer_plist_id) const ; #endif - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP freeze() ; virtual storeRepP thaw() ; virtual block_hash *get_attrib_data() { return &attrib_data; } diff --git a/src/include/DMapVec_def.h b/src/include/DMapVec_def.h index bb2fe2a3..e988b047 100644 --- a/src/include/DMapVec_def.h +++ b/src/include/DMapVec_def.h @@ -86,7 +86,7 @@ namespace Loci { virtual void writehdf5P(hid_t group_id, hid_t dataspace,hid_t dataset, hsize_t dimension, const char* name, entitySet &en, hid_t xfer_plist_id) const ; #endif - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual DatatypeP getType() ; virtual storeRepP freeze() ; virtual storeRepP thaw() ; diff --git a/src/include/DMapVec_impl.h b/src/include/DMapVec_impl.h index 411fd6d7..990b79e3 100644 --- a/src/include/DMapVec_impl.h +++ b/src/include/DMapVec_impl.h @@ -23,6 +23,7 @@ #include #include +#include namespace Loci { @@ -226,16 +227,18 @@ void dMapVecRepI::writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, //------------------------------------------------------------------------ template - storeRepP dMapVecRepI::expand(entitySet &out_of_dom, std::vector &init_ptn) { - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + storeRepP dMapVecRepI::expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm) { + int np ; + MPI_Comm_size(comm, &np) ; + int *recv_count = new int[np] ; + int *send_count = new int[np] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; entitySet::const_iterator ei ; std::vector::const_iterator vi ; int size_send = 0 ; - std::vector > copy(MPI_processes), send_clone(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + std::vector > copy(np), send_clone(np) ; + for(int i = 0; i < np; ++i) { entitySet tmp = out_of_dom & init_ptn[i] ; for(ei = tmp.begin(); ei != tmp.end(); ++ei) copy[i].push_back(*ei) ; @@ -245,28 +248,28 @@ void dMapVecRepI::writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, } int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_buf = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = copy[i].begin(); vi != copy[i].end(); ++vi) { send_buf[size_send] = *vi ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - for(int i = 0; i < MPI_processes; ++i) { + comm) ; + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i].push_back(recv_buf[j]) ; @@ -274,8 +277,8 @@ void dMapVecRepI::writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, } - std::vector map_entities(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i){ + std::vector map_entities(np) ; + for(int i = 0; i < np; ++i){ entitySet dom; for(unsigned int j = 0; j ::writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, map_entities[i] = dom; } - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { send_count[i] = map_entities[i].size()*(1+M) ; } size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += send_count[i] ; int *send_map = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_map = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(entitySet::const_iterator miv = map_entities[i].begin(); miv != map_entities[i].end(); ++miv) { send_map[size_send] = *miv ; ++size_send ; @@ -311,17 +314,17 @@ void dMapVecRepI::writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_map,send_count, send_displacement , MPI_INT, recv_map, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; HASH_MAP(int, std::vector ) hm ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i];) { for(unsigned int k = 0; k < M; ++k) diff --git a/src/include/DMultiMap.h b/src/include/DMultiMap.h index 1ff8992e..b3c16060 100644 --- a/src/include/DMultiMap.h +++ b/src/include/DMultiMap.h @@ -84,7 +84,7 @@ namespace Loci { virtual void writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, hsize_t dimension, const char* name, entitySet& en, hid_t xfer_plist_id) const ; #endif - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP thaw() ; virtual storeRepP freeze() ; block_hash > > *get_attrib_data() {return &attrib_data;} diff --git a/src/include/DStore_def.h b/src/include/DStore_def.h index c9c7b797..2e458d7f 100644 --- a/src/include/DStore_def.h +++ b/src/include/DStore_def.h @@ -90,10 +90,10 @@ namespace Loci { virtual storeRepP remap(const dMap &m) const ; virtual storeRepP redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD) ; + MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP freeze() ; virtual storeRepP thaw() ; virtual void copy(storeRepP &st, const entitySet &context) ; diff --git a/src/include/FVMAdapt/defines.h b/src/include/FVMAdapt/defines.h index 06c1aa69..c450175d 100644 --- a/src/include/FVMAdapt/defines.h +++ b/src/include/FVMAdapt/defines.h @@ -414,9 +414,9 @@ void colorMatrix(Map &cl, Map &cr, multiMap &face2node); namespace Loci { - extern hid_t writeVOGOpen(string filename); - extern void writeVOGSurf(hid_t file_id, std::vector > surface_ids); - extern void writeVOGClose(hid_t file_id) ; + extern hid_t writeVOGOpen(string filename, MPI_Comm comm); + extern void writeVOGSurf(hid_t file_id, std::vector > surface_ids, MPI_Comm comm); + extern void writeVOGClose(hid_t file_id, MPI_Comm comm) ; extern bool readVolTags(hid_t input_fid, std::vector > &volDat); @@ -426,7 +426,7 @@ namespace Loci { const_store &inner_nodes_face, const_store &inner_nodes_edge ); - extern void writeVOGFace(hid_t file_id, Map &cl, Map &cr, multiMap &face2node) ; + extern void writeVOGFace(hid_t file_id, Map &cl, Map &cr, multiMap &face2node, MPI_Comm comm) ; extern unsigned long readAttributeLong(hid_t group, const char *name); bool setupFVMGridFromContainer(fact_db &facts, diff --git a/src/include/LociGridReaders.h b/src/include/LociGridReaders.h index 27980271..94b4bb14 100644 --- a/src/include/LociGridReaders.h +++ b/src/include/LociGridReaders.h @@ -222,7 +222,8 @@ namespace Loci { /// @param[out] boundary_ids Vector of pairs of BC ID and name. /// @return true if success bool readBCfromVOG(std::string filename, - std::vector > &boundary_ids) ; + std::vector > &boundary_ids, + MPI_Comm comm LOCI_DEFAULT_COMM) ; /// Sets up facts related to grid boundary conditions. /// @param[in,out] facts Fact database to set up grid into. @@ -239,11 +240,13 @@ namespace Loci { extern void writeVOG(std::string filename,store > &pos, Map &cl, Map &cr, multiMap &face2node, - std::vector > surfaceids) ; + std::vector > surfaceids, + MPI_Comm comm LOCI_DEFAULT_COMM) ; extern void writeVOG(std::string filename,store > &pos, Map &cl, Map &cr, multiMap &face2node, std::vector >& surfaceids, - std::vector >& volTags) ; + std::vector >& volTags, + MPI_Comm comm LOCI_DEFAULT_COMM) ; /// Reads grid structures from grid file in the .vog format. /// @param[out] local_nodes Partition of nodes diff --git a/src/include/Map.h b/src/include/Map.h index cbb4ebf8..7d2b6555 100644 --- a/src/include/Map.h +++ b/src/include/Map.h @@ -89,7 +89,7 @@ namespace Loci { #endif Entity * get_base_ptr() const { Entity * p = 0 ; if(alloc_id>=0) p = ((Entity *)storeAllocateData[alloc_id].base_ptr) -storeAllocateData[alloc_id].base_offset ; return p ; } - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual DatatypeP getType() ; virtual frame_info get_frame_info() ; } ; diff --git a/src/include/MapVec_def.h b/src/include/MapVec_def.h index d1eeb579..3b297e4b 100644 --- a/src/include/MapVec_def.h +++ b/src/include/MapVec_def.h @@ -84,7 +84,7 @@ namespace Loci { virtual void writehdf5P(hid_t group_id, hid_t dataspace, hid_t dataset, hsize_t dimension, const char* name, entitySet& en, hid_t xfer_plist_id) const ; #endif VEC * get_base_ptr() const { VEC * p = 0 ; if(alloc_id>=0) p = ((VEC *)storeAllocateData[alloc_id].base_ptr) - storeAllocateData[alloc_id].base_offset ; return p ; } - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP freeze() ; virtual storeRepP thaw() ; virtual DatatypeP getType() ; diff --git a/src/include/MapVec_impl.h b/src/include/MapVec_impl.h index 5aba6672..b2652ca0 100644 --- a/src/include/MapVec_impl.h +++ b/src/include/MapVec_impl.h @@ -23,6 +23,7 @@ #include #include +#include namespace Loci { @@ -358,17 +359,19 @@ namespace Loci { template storeRepP MapVecRepI::expand(entitySet &out_of_dom, - std::vector &ptn) { + std::vector &ptn, MPI_Comm comm) { + int np ; + MPI_Comm_size(comm, &np) ; cerr<<"start mapVec expand" << endl; - int *recv_count = new int[MPI_processes] ; - int *send_count = new int[MPI_processes] ; - int *send_displacement = new int[MPI_processes] ; - int *recv_displacement = new int[MPI_processes] ; + int *recv_count = new int[np] ; + int *send_count = new int[np] ; + int *send_displacement = new int[np] ; + int *recv_displacement = new int[np] ; entitySet::const_iterator ei ; std::vector::const_iterator vi ; int size_send = 0 ; - std::vector > copy(MPI_processes), send_clone(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i) { + std::vector > copy(np), send_clone(np) ; + for(int i = 0; i < np; ++i) { entitySet tmp = out_of_dom & ptn[i] ; for(ei = tmp.begin(); ei != tmp.end(); ++ei) copy[i].push_back(*ei) ; @@ -378,35 +381,35 @@ namespace Loci { } int *send_buf = new int[size_send] ; MPI_Alltoall(send_count, 1, MPI_INT, recv_count, 1, MPI_INT, - MPI_COMM_WORLD) ; + comm) ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) size_send += recv_count[i] ; int *recv_buf = new int[size_send] ; size_send = 0 ; - for(int i = 0; i < MPI_processes; ++i) + for(int i = 0; i < np; ++i) for(vi = copy[i].begin(); vi != copy[i].end(); ++vi) { send_buf[size_send] = *vi ; ++size_send ; } send_displacement[0] = 0 ; recv_displacement[0] = 0 ; - for(int i = 1; i < MPI_processes; ++i) { + for(int i = 1; i < np; ++i) { send_displacement[i] = send_displacement[i-1] + send_count[i-1] ; recv_displacement[i] = recv_displacement[i-1] + recv_count[i-1] ; } MPI_Alltoallv(send_buf,send_count, send_displacement , MPI_INT, recv_buf, recv_count, recv_displacement, MPI_INT, - MPI_COMM_WORLD) ; - for(int i = 0; i < MPI_processes; ++i) { + comm) ; + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i]; ++j) send_clone[i].push_back(recv_buf[j]) ; std::sort(send_clone[i].begin(), send_clone[i].end()) ; } - std::vector map_entities(MPI_processes) ; - for(int i = 0; i < MPI_processes; ++i){ + std::vector map_entities(np) ; + for(int i = 0; i < np; ++i){ entitySet dom; for(unsigned int j = 0; j ) hm ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i];) { for(int k = 0; k < M; ++k) @@ -469,7 +472,7 @@ namespace Loci { for(int p = 0; p < M ; ++p) new_map[*ei][p] = base_ptr[*ei][p] ; - for(int i = 0; i < MPI_processes; ++i) { + for(int i = 0; i < np; ++i) { for(int j = recv_displacement[i]; j < recv_displacement[i]+recv_count[i];) { for(unsigned int k = 0; k < M; ++k) diff --git a/src/include/Map_rep.h b/src/include/Map_rep.h index 0e819afe..8376bd47 100644 --- a/src/include/Map_rep.h +++ b/src/include/Map_rep.h @@ -44,7 +44,7 @@ namespace Loci { virtual storeRepP get_map() = 0 ; virtual void compose(const dMap &m, const entitySet &context) = 0 ; virtual store_type RepType() const ; - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) = 0 ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) = 0 ; virtual storeRepP MapRemap(const dMap &dm, const dMap &im) const = 0 ; virtual void shift(int_type) {std::cerr<<"shift for Map has not been implemented!"< &init_ptn) {return new_store(EMPTY); } + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) {return new_store(EMPTY); } virtual storeRepP freeze() {return getRep() ; } virtual storeRepP thaw() {return getRep() ; } virtual DatatypeP getType() { return DatatypeP(new AtomicType(INT)) ; } diff --git a/src/include/dist_internal.h b/src/include/dist_internal.h index af18513e..0abd5b1a 100644 --- a/src/include/dist_internal.h +++ b/src/include/dist_internal.h @@ -46,9 +46,11 @@ namespace Loci { void write_vector_intP(hid_t group_id, const char* name, std::vector& vint, MPI_Comm comm) ; } std::vector all_collect_sizes(int size,MPI_Comm comm) ; +#ifndef LOCI_STRICT_COMM inline std::vector all_collect_sizes(int size) { return all_collect_sizes(size,MPI_COMM_WORLD) ; } +#endif } diff --git a/src/include/distribute.h b/src/include/distribute.h index 7a880f00..972e39bb 100644 --- a/src/include/distribute.h +++ b/src/include/distribute.h @@ -49,6 +49,7 @@ namespace Loci { extern int MPI_rank ; void Init(int* argc, char*** argv) ; + void SetDefaultComm(MPI_Comm comm) ; void Finalize() ; void Abort() ; size_t MPI_process_mem_avail() ; @@ -567,13 +568,13 @@ namespace Loci { } ; - dMap send_map(Map &dm, entitySet &out_of_dom, std::vector &init_ptn) ; + dMap send_map(Map &dm, entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; - std::vector send_global_map(Map &attrib_data, entitySet &out_of_dom, std::vector &init_ptn) ; - void fill_clone(storeRepP& sp, entitySet &out_of_dom, std::vector &init_ptn) ; + std::vector send_global_map(Map &attrib_data, entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; + void fill_clone(storeRepP& sp, entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; - storeRepP send_clone_non(storeRepP& sp, entitySet &out_of_dom, std::vector &init_ptn) ; - std::vector send_global_clone_non(storeRepP &sp , entitySet &out_of_dom, std::vector &init_ptn) ; + storeRepP send_clone_non(storeRepP& sp, entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; + std::vector send_global_clone_non(storeRepP &sp , entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; std::vector > transpose_entitySet(const std::vector > &in, @@ -585,11 +586,18 @@ namespace Loci { transpose_sequence(const std::vector& in, MPI_Comm comm) ; std::vector all_collect_vectors(entitySet &e,MPI_Comm comm) ; +#ifndef LOCI_STRICT_COMM std::vector all_collect_vectors(entitySet &e) ; +#endif + + entitySet dist_collect_entitySet(entitySet inSet, const std::vector &ptn, MPI_Comm comm ) ; +#ifndef LOCI_STRICT_COMM + entitySet dist_collect_entitySet(entitySet inSet, const std::vector &ptn) ; +#endif - entitySet distribute_entitySet(entitySet e,const std::vector &ptn) ; + entitySet distribute_entitySet(entitySet e,const std::vector &ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; - entitySet all_collect_entitySet(const entitySet &e) ; + entitySet all_collect_entitySet(const entitySet &e, MPI_Comm comm LOCI_DEFAULT_COMM) ; // This is equivalent to but more efficient than // collectSet(entitySet iset) { @@ -598,10 +606,10 @@ namespace Loci { entitySet collectSet(const entitySet iset, const entitySet domain, MPI_Comm comm) ; - int GLOBAL_OR(int b, MPI_Comm comm=MPI_COMM_WORLD) ; - int GLOBAL_AND(int b, MPI_Comm comm=MPI_COMM_WORLD) ; - int GLOBAL_MAX(int b, MPI_Comm comm=MPI_COMM_WORLD) ; - int GLOBAL_MIN(int b, MPI_Comm comm=MPI_COMM_WORLD) ; + int GLOBAL_OR(int b, MPI_Comm comm LOCI_DEFAULT_COMM) ; + int GLOBAL_AND(int b, MPI_Comm comm LOCI_DEFAULT_COMM) ; + int GLOBAL_MAX(int b, MPI_Comm comm LOCI_DEFAULT_COMM) ; + int GLOBAL_MIN(int b, MPI_Comm comm LOCI_DEFAULT_COMM) ; // We've added these back as they seem to be used // in the fuel cell program//from distribute.h ////// diff --git a/src/include/distribute_container.h b/src/include/distribute_container.h index ef58b533..0a803e19 100644 --- a/src/include/distribute_container.h +++ b/src/include/distribute_container.h @@ -44,31 +44,36 @@ namespace Loci { std::vector > &input, entitySet input_image, entitySet input_preimage, - const std::vector &init_ptn) ; + const std::vector &init_ptn, + MPI_Comm comm LOCI_DEFAULT_COMM) ; - dMap distribute_dMap(dMap m, const std::vector &init_ptn) ; + dMap distribute_dMap(dMap m, const std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; void distributed_inverseMap(dmultiMap &result, const dMap &input_map, const entitySet &input_image, const entitySet &input_preimage, - std::vector &init_ptn) ; + std::vector &init_ptn, + MPI_Comm comm LOCI_DEFAULT_COMM) ; void distributed_inverseMap(dmultiMap &result, const Map &input_map, const entitySet &input_image, const entitySet &input_preimage, - std::vector &init_ptn) ; + std::vector &init_ptn, + MPI_Comm comm LOCI_DEFAULT_COMM) ; void distributed_inverseMap(dmultiMap &result, const dmultiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, - std::vector &init_ptn) ; + std::vector &init_ptn, + MPI_Comm comm LOCI_DEFAULT_COMM) ; void distributed_inverseMap(dmultiMap &result, const multiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, - std::vector &init_ptn); + std::vector &init_ptn, + MPI_Comm comm LOCI_DEFAULT_COMM); void distributed_inverseMap(dmultiMap &result, const const_dMap &input_map, @@ -135,19 +140,19 @@ namespace Loci { } } - void distributed_inverseMap(multiMap &result, const dMap &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn) ; + void distributed_inverseMap(multiMap &result, const dMap &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; - void distributed_inverseMap(multiMap &result, const Map &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn) ; + void distributed_inverseMap(multiMap &result, const Map &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; - void distributed_inverseMap(multiMap &result, const dmultiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn) ; + void distributed_inverseMap(multiMap &result, const dmultiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; - void distributed_inverseMap(multiMap &result, const multiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn); + void distributed_inverseMap(multiMap &result, const multiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM); inline void distributed_inverseMap(multiMap &result, const Map &input_map, const entitySet &input_image, const entitySet &input_preimage, fact_db &facts, size_t kd) { if(facts.is_distributed_start()) { std::vector init_ptn = facts.get_init_ptn(kd) ; - Loci::distributed_inverseMap(result, input_map, input_image, input_preimage, init_ptn) ; + Loci::distributed_inverseMap(result, input_map, input_image, input_preimage, init_ptn,facts.get_comm()) ; } else { Loci::inverseMap(result,input_map,input_image,input_preimage) ; } @@ -156,7 +161,7 @@ namespace Loci { inline void distributed_inverseMap(multiMap &result, const multiMap &input_map, const entitySet &input_image, const entitySet &input_preimage, fact_db &facts,size_t kd) { if(facts.is_distributed_start()) { std::vector init_ptn = facts.get_init_ptn(kd) ; - Loci::distributed_inverseMap(result, input_map, input_image, input_preimage, init_ptn) ; + Loci::distributed_inverseMap(result, input_map, input_image, input_preimage, init_ptn,facts.get_comm()) ; } else { Loci::inverseMap(result,input_map,input_image,input_preimage) ; } @@ -165,7 +170,7 @@ namespace Loci { inline void distributed_inverseMap(multiMap &result, const dMap &input_map, const entitySet &input_image, const entitySet &input_preimage, fact_db &facts, size_t kd) { if(facts.is_distributed_start()) { std::vector init_ptn = facts.get_init_ptn(kd) ; - Loci::distributed_inverseMap(result, input_map, input_image, input_preimage, init_ptn) ; + Loci::distributed_inverseMap(result, input_map, input_image, input_preimage, init_ptn,facts.get_comm()) ; } else { Loci::inverseMap(result,input_map,input_image,input_preimage) ; } @@ -179,22 +184,23 @@ namespace Loci { return v1.first < v2.first ; } - inline void equiJoinFF(protoMap &in1, protoMap &in2, protoMap &out) { + inline void equiJoinFF(protoMap &in1, protoMap &in2, protoMap &out, + MPI_Comm comm LOCI_DEFAULT_COMM) { std::sort(in1.begin(),in1.end(),equiFF) ; std::sort(in2.begin(),in2.end(),equiFF) ; int p = 0 ; - MPI_Comm_size(MPI_COMM_WORLD,&p) ; + MPI_Comm_size(comm,&p) ; // Sort inputs using same splitters (this will make sure that // data that needs to be on the same processor ends up on the // same processor if(p != 1) { std::vector > splitters ; - parGetSplitters(splitters,in1,equiFF,MPI_COMM_WORLD) ; + parGetSplitters(splitters,in1,equiFF,comm) ; - parSplitSort(in1,splitters,equiFF,MPI_COMM_WORLD) ; - parSplitSort(in2,splitters,equiFF,MPI_COMM_WORLD) ; + parSplitSort(in1,splitters,equiFF,comm) ; + parSplitSort(in2,splitters,equiFF,comm) ; } // Find pairs where first entry are the same and create joined protomap @@ -211,7 +217,7 @@ namespace Loci { } // Remove duplicates from protomap - parSampleSort(out,equiFF,MPI_COMM_WORLD) ; + parSampleSort(out,equiFF,comm) ; std::sort(out.begin(),out.end()) ; out.erase(std::unique(out.begin(),out.end()),out.end()) ; } @@ -272,7 +278,8 @@ namespace Loci { template void gatherData(std::vector &result, store &data, entitySet get, - std::vector &ptn) { + std::vector &ptn, + MPI_Comm comm LOCI_DEFAULT_COMM) { using std::vector ; { vector tmp(get.size()) ; result.swap(tmp) ;} @@ -285,7 +292,7 @@ namespace Loci { recv_sets[i] = get&ptn[i] ; vector send_sets = - transpose_entitySet(recv_sets, MPI_COMM_WORLD) ; + transpose_entitySet(recv_sets, comm) ; int send_sz = 0 ; int reqs = 0 ; for(int i=0;i void writeUnorderedVectorS(hid_t group_id, const char *element_name, std::vector &v) { writeUnorderedVectorS(group_id,element_name,v,MPI_COMM_WORLD) ; } +#endif //----------------------------------------------------------------------- template void writeUnorderedVectorP(hid_t group_id, @@ -576,6 +584,7 @@ namespace Loci { } //----------------------------------------------------------------------- +#ifndef LOCI_STRICT_COMM template void writeUnorderedVectorP(hid_t group_id, const char *element_name, std::vector &v @@ -585,6 +594,7 @@ namespace Loci { v, MPI_COMM_WORLD ); } +#endif @@ -683,7 +693,7 @@ namespace Loci { readUnorderedVectorS(group_id,element_name,v,prime_comm) ; #else int procs = 1 ; - MPI_Comm_size(MPI_COMM_WORLD,&procs) ; + MPI_Comm_size(prime_comm,&procs) ; if(procs == 1) return readVectorSerial(group_id,element_name,v) ; @@ -764,6 +774,7 @@ namespace Loci { //----------------------------------------------------------------------- +#ifndef LOCI_STRICT_COMM template void writeUnorderedVector(hid_t group_id, const char *element_name, std::vector &v @@ -773,17 +784,19 @@ namespace Loci { v, MPI_COMM_WORLD ); } +#endif //----------------------------------------------------------------------- template void writeUnorderedStore(hid_t file_id, const_store &s, entitySet set, - const char *name) { + const char *name, + MPI_Comm comm LOCI_DEFAULT_COMM) { std::vector v(set.size()) ; size_t c = 0 ; FORALL(set,ii) { v[c++] = s[ii] ; } ENDFORALL ; - writeUnorderedVector(file_id,name,v) ; + writeUnorderedVector(file_id,name,v,comm) ; } //----------------------------------------------------------------------- @@ -926,7 +939,7 @@ namespace Loci { // fact_db pointer (facts) // MPI Communicator storeRepP Local2FileOrder_output(storeRepP sp, entitySet dom, - fact_db& facts, MPI_Comm comm); + fact_db& facts); //serial/parallel io template void writeCutPlaneNodalVal(hid_t file_id, @@ -965,7 +978,7 @@ namespace Loci { //transform the store into output order store gedge_pos; storeRepP geposRep = Local2FileOrder_output(edge_pos.Rep(), edgesCut, - facts, MPI_COMM_WORLD); + facts); if(geposRep == NULL){ gedge_pos .allocate(EMPTY); @@ -1151,7 +1164,7 @@ namespace Loci { storeRepP input, int offset, fact_db::distribute_infoP dist, MPI_Comm comm) ; - void getL2FMap(Map &l2f, entitySet dom, fact_db::distribute_infoP dist) ; + void getL2FMap(Map &l2f, entitySet dom, fact_db::distribute_infoP dist, MPI_Comm comm LOCI_DEFAULT_COMM) ; void FindSimpleDistribution(entitySet dom, const Map &l2f, std::vector &splits, MPI_Comm comm) ; void memoryBalancedDistribution(std::vector &splits_out, diff --git a/src/include/distribute_long.h b/src/include/distribute_long.h index f2ca26e8..3ff5b8ed 100644 --- a/src/include/distribute_long.h +++ b/src/include/distribute_long.h @@ -53,14 +53,14 @@ namespace Loci { return false ; } - template T g_GLOBAL_MAX(T b, MPI_Comm comm=MPI_COMM_WORLD) { + template T g_GLOBAL_MAX(T b, MPI_Comm comm LOCI_DEFAULT_COMM) { T result ; MPI_Datatype MPI_T_type = MPI_traits::get_MPI_type() ; MPI_Allreduce(&b, &result, 1, MPI_T_type, MPI_MAX, comm) ; return result ; } - template T g_GLOBAL_MIN(T b, MPI_Comm comm=MPI_COMM_WORLD) { + template T g_GLOBAL_MIN(T b, MPI_Comm comm LOCI_DEFAULT_COMM) { T result ; MPI_Datatype MPI_T_type = MPI_traits::get_MPI_type() ; MPI_Allreduce(&b, &result, 1, MPI_T_type, MPI_MIN, comm) ; @@ -68,7 +68,7 @@ namespace Loci { } // Collect largest interval of entitySet from all processors - template genIntervalSet g_collectLargest(const genIntervalSet&e,MPI_Comm comm=MPI_COMM_WORLD) { + template genIntervalSet g_collectLargest(const genIntervalSet&e,MPI_Comm comm LOCI_DEFAULT_COMM) { int p = 1; MPI_Comm_size(comm,&p) ; @@ -94,7 +94,7 @@ namespace Loci { } // Return union of all entitySets from all processors, the actual user interface is g_all_collect_entitySet() - template genIntervalSet g_all_gather_entitySet(const genIntervalSet &e,MPI_Comm comm = MPI_COMM_WORLD ) { + template genIntervalSet g_all_gather_entitySet(const genIntervalSet &e,MPI_Comm comm LOCI_DEFAULT_COMM ) { int p = 1; MPI_Comm_size(comm,&p) ; if(p == 1) @@ -129,7 +129,7 @@ namespace Loci { } //Return union of all entitySets from all processors, - template genIntervalSet g_all_collect_entitySet(const genIntervalSet &e,MPI_Comm comm = MPI_COMM_WORLD ) { + template genIntervalSet g_all_collect_entitySet(const genIntervalSet &e,MPI_Comm comm LOCI_DEFAULT_COMM ) { int p = 1 ; MPI_Comm_size(comm,&p) ; // no operation for single processor @@ -192,13 +192,13 @@ namespace Loci { //Return union of all entitySets from all processors that belongs to this processor template genIntervalSet g_dist_collect_entitySet(const genIntervalSet &inSet, const std::vector > &ptn, - MPI_Comm comm = MPI_COMM_WORLD ) { + MPI_Comm comm LOCI_DEFAULT_COMM ) { const int r = MPI_rank ; genIntervalSet retval = inSet & ptn[r] ; //set from me // Check for empty and universal set int sbits = ((inSet != genIntervalSet::EMPTY)?1:0)| ((retval != ptn[r])?2:0) ; int rbits = sbits ; - MPI_Allreduce(&sbits, &rbits, 1, MPI_INT, MPI_BOR, MPI_COMM_WORLD) ; + MPI_Allreduce(&sbits, &rbits, 1, MPI_INT, MPI_BOR, comm) ; if((rbits & 1) == 0) // EMPTY set return genIntervalSet::EMPTY ; if((rbits & 2) == 0) // UNIVERSE set @@ -257,9 +257,11 @@ namespace Loci { } +#ifndef LOCI_STRICT_COMM template std::vector > g_all_collect_vectors(genIntervalSet &e) { return g_all_collect_vectors(e,MPI_COMM_WORLD) ; } +#endif } diff --git a/src/include/distributed_inverse_long.h b/src/include/distributed_inverse_long.h index ab45f082..9886aad9 100644 --- a/src/include/distributed_inverse_long.h +++ b/src/include/distributed_inverse_long.h @@ -35,11 +35,13 @@ #include #include #include +#include namespace Loci { - template void distributed_inverse_map(std::vector &recv_store, // void distributed_inverse_map(std::vector &recv_store, std::vector > &input, - const std::vector > &init_ptn){ + const std::vector > &init_ptn, + MPI_Comm comm LOCI_DEFAULT_COMM){ // Sort input according to second field sort(input.begin(),input.end(),field_sort2) ; @@ -66,7 +68,7 @@ namespace Loci { vector recv_sz(MPI_processes) ; MPI_Alltoall(&send_sz[0],1,MPI_INT, &recv_sz[0],1,MPI_INT, - MPI_COMM_WORLD) ; + comm) ; int size_send = 0 ; int size_recv = 0 ; for(int i=0;i::get_MPI_type() ; MPI_Alltoallv(&send_store[0], &send_sz[0], send_displacement, MPI_T_type, &recv_store[0], &recv_sz[0], recv_displacement, MPI_T_type, - MPI_COMM_WORLD) ; + comm) ; delete[] recv_displacement ; delete[] send_displacement ; diff --git a/src/include/fact_db.h b/src/include/fact_db.h index 696d9c55..8106d8fa 100644 --- a/src/include/fact_db.h +++ b/src/include/fact_db.h @@ -84,7 +84,8 @@ namespace Loci { int xmit_total_size ; // dMap remap ; std::vector g2fv ; // Global to file numbering - distribute_info() {} ; + MPI_Comm comm ; + distribute_info() : comm(MPI_COMM_WORLD) {} ; } ; std::vector > init_ptn ; /// Global numbering partition indexed by key space @@ -107,7 +108,16 @@ namespace Loci { v = mi->second ; return v ; } + MPI_Comm get_comm() const { return comm_ ; } + void set_comm(MPI_Comm c) ; + int get_comm_rank() const { return comm_rank_ ; } + int get_comm_size() const { return comm_size_ ; } + private: + MPI_Comm comm_ ; + int comm_rank_ ; + int comm_size_ ; + struct fact_info { store_refP data_rep ; } ; @@ -370,14 +380,29 @@ namespace Loci { extern fact_db *exec_current_fact_db ; +#ifndef LOCI_STRICT_COMM + inline MPI_Comm get_exec_comm() { + return (exec_current_fact_db != 0) ? + exec_current_fact_db->get_comm() : MPI_COMM_WORLD ; + } + + inline int get_exec_rank() { + return (exec_current_fact_db != 0) ? + exec_current_fact_db->get_comm_rank() : MPI_rank ; + } + + inline int get_exec_size() { + return (exec_current_fact_db != 0) ? + exec_current_fact_db->get_comm_size() : MPI_processes ; + } +#endif + inline entitySet collect_entitySet(entitySet e) { return collect_entitySet(e,*exec_current_fact_db) ; } - entitySet all_collect_entitySet(const entitySet &e) ; - inline entitySet all_collect_entitySet(entitySet localset,fact_db &facts) { if(facts.is_distributed_start()) - return Loci::all_collect_entitySet(localset) ; + return Loci::all_collect_entitySet(localset, facts.get_comm()) ; return localset ; } } diff --git a/src/include/gblackbox.h b/src/include/gblackbox.h index 58620994..147baaed 100644 --- a/src/include/gblackbox.h +++ b/src/include/gblackbox.h @@ -55,21 +55,21 @@ namespace Loci { //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and // redistributes the container according to the domain partition virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // this redistribute version takes an additional remap // argument, after redistribution, the new container is remapped // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const; + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const; virtual gstore_type RepType() const; virtual gEntitySet domain() const; virtual gStoreRepP remap(const gMap &m) const; diff --git a/src/include/gconstraint.h b/src/include/gconstraint.h index 6fdef52a..cfc31882 100644 --- a/src/include/gconstraint.h +++ b/src/include/gconstraint.h @@ -54,14 +54,14 @@ namespace Loci { //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and // redistributes the container according to the domain partition virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // this redistribute version takes an additional remap @@ -69,7 +69,7 @@ namespace Loci { // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const; + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const; virtual gStoreRepP remap(const gMap &m) const ; @@ -131,14 +131,14 @@ namespace Loci { //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm);} // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and // redistributes the container according to the domain partition virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->split_redistribute(dom_ptn, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->split_redistribute(dom_ptn, comm);} // this redistribute version takes an additional remap @@ -146,7 +146,7 @@ namespace Loci { // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap,comm);} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap,comm);} virtual gStoreRepP remap(const gMap &m) const{return Rep()->remap(m) ;} gConstraint & operator=(const gConstraint &p) diff --git a/src/include/gmap.h b/src/include/gmap.h index ceb2c731..92bd3bf5 100644 --- a/src/include/gmap.h +++ b/src/include/gmap.h @@ -68,24 +68,24 @@ namespace Loci { virtual gStoreRepP clone() const{return new gMapRepI(*this); } virtual gStoreRepP remap(const gMap &m) const ; virtual void inplace_remap(const gMap &m); - virtual void inplace_compose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD); + virtual void inplace_compose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM); virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual gStoreRepP redistribute(const std::vector& dom_ptn, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const ; + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const ; // virtual gStoreRepP // redistribute_omd(const std::vector& dom_ptn, - // const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const ; + // const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; virtual gStoreRepP local_inverse() const; virtual gStoreRepP distributed_inverse(gEntitySet global_image, gEntitySet global_preimage, @@ -137,7 +137,7 @@ namespace Loci { //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual void* get_attrib_data() {return &attrib_data; } virtual const void* get_attrib_data() const{return &attrib_data; } @@ -218,21 +218,21 @@ namespace Loci { return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); @@ -241,14 +241,14 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm) ;} virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_ptn, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_ptn, comm) ;} //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const{return gMapRepP(Rep())->expand(dom, init_ptn, comm);} + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const{return gMapRepP(Rep())->expand(dom, init_ptn, comm);} @@ -326,13 +326,13 @@ namespace Loci { } virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm) ;} virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap, comm) ;} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap, comm) ;} virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_ptn, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_ptn, comm) ;} virtual gStoreRepP remap(const gMap &m) const{return Rep()->remap(m);} virtual gStoreRepP distributed_inverse(gEntitySet global_image, @@ -345,21 +345,21 @@ namespace Loci { return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); diff --git a/src/include/gmap_rep.h b/src/include/gmap_rep.h index 46c2bf40..0220bc32 100644 --- a/src/include/gmap_rep.h +++ b/src/include/gmap_rep.h @@ -50,14 +50,14 @@ namespace Loci { return gStoreRepP(0); } //this method remap the SECOND field of this using m - virtual void inplace_compose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD) = 0 ; + virtual void inplace_compose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM) = 0 ; //this method remap the SECOND field of this using m and return a new map - // virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD) = 0 ; + // virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM) = 0 ; virtual gstore_type RepType() const {return GMAP;} //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const = 0 ; + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const = 0 ; virtual gStoreRepP local_inverse() const = 0; } ; diff --git a/src/include/gmapvec_def.h b/src/include/gmapvec_def.h index b71c7a0a..f6764dc4 100644 --- a/src/include/gmapvec_def.h +++ b/src/include/gmapvec_def.h @@ -71,20 +71,20 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual gStoreRepP redistribute(const std::vector& dom_ptn, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const ; + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const ; - virtual void inplace_compose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD ) ; - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const ; + virtual void inplace_compose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM ) ; + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; virtual gStoreRepP local_inverse() const; virtual gStoreRepP distributed_inverse(gEntitySet global_image, @@ -138,7 +138,7 @@ namespace Loci { //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const ; + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual void* get_attrib_data() {return &attrib_data; } virtual const void* get_attrib_data() const{return &attrib_data; } @@ -228,19 +228,19 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm) ;} virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_ptn, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_ptn, comm) ;} //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const{ + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const{ return gMapRepP(Rep())->expand(dom, init_ptn, comm);} - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); @@ -248,14 +248,14 @@ namespace Loci { } //duplication removal is needed in this method - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); @@ -315,13 +315,13 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm) ;} virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap, comm) ;} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap, comm) ;} virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_ptn, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_ptn, comm) ;} virtual gStoreRepP remap(const gMap &m) const{return Rep()->remap(m);} virtual gstore_type RepType() const {return Rep()->RepType();} @@ -367,21 +367,21 @@ namespace Loci { else return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); diff --git a/src/include/gmultimap.h b/src/include/gmultimap.h index fc482653..02476a85 100644 --- a/src/include/gmultimap.h +++ b/src/include/gmultimap.h @@ -79,20 +79,20 @@ namespace Loci { #endif virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const ; + MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual gStoreRepP redistribute(const std::vector& dom_ptn, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const ; + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const ; - virtual void inplace_compose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD ) ; - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const ; + virtual void inplace_compose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM ) ; + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; virtual gStoreRepP local_inverse() const; virtual gStoreRepP distributed_inverse(gEntitySet global_image, @@ -149,7 +149,7 @@ namespace Loci { //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const ; + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const ; virtual void* get_attrib_data() {return &attrib_data; } virtual const void* get_attrib_data() const{return &attrib_data; } @@ -238,18 +238,18 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm) ;} virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_ptn, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_ptn, comm) ;} //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const{ + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const{ return gMapRepP(Rep())->expand(dom, init_ptn, comm);} - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); @@ -257,14 +257,14 @@ namespace Loci { } //duplication removal is needed in this method - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); @@ -336,13 +336,13 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm) ;} virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap, comm) ;} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap, comm) ;} virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_ptn, comm) ;} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_ptn, comm) ;} virtual gStoreRepP remap(const gMap &m) const{return Rep()->remap(m);} virtual gstore_type RepType() const {return Rep()->RepType();} @@ -400,21 +400,21 @@ namespace Loci { else return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); else return gStoreRepP(0); } - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD )const{ + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM )const{ CPTR p(Rep()) ; if(p != 0) return p->recompose(m, comm); @@ -422,7 +422,7 @@ namespace Loci { } //different from traditional maps, this method is const method //dom is the domain after expansion, not out_of_dom - virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm=MPI_COMM_WORLD)const{ + virtual gStoreRepP expand(gEntitySet &dom, std::vector &init_ptn,MPI_Comm comm LOCI_DEFAULT_COMM)const{ return gMapRepP(Rep())->expand(dom, init_ptn, comm);} gKeySpaceP get_domain_space()const{return static_cast(Rep())->get_domain_space() ;} diff --git a/src/include/gmultistore_def.h b/src/include/gmultistore_def.h index c67a5d7d..690e140e 100644 --- a/src/include/gmultistore_def.h +++ b/src/include/gmultistore_def.h @@ -110,20 +110,20 @@ namespace Loci { //For stores,recompose will compose a store whose domain is the domain of m, //whose data is the data of the SECOND field of m. //for example, pos.recompose(face2node) will produce the positions for each face - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD)const ; + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM)const ; // this method redistributes the stores according to the split of local domain over a group of process //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // this redistribute version takes an additional remap // argument, after redistribution, the new store is remapped // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split,const gMap &remap, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and @@ -133,7 +133,7 @@ namespace Loci { // containers will need to implement this method later. virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; //binary search function, for random access //mixed equality and equivalance @@ -327,15 +327,15 @@ namespace Loci { //For stores,recompose will compose a store whose domain is the domain of m, //whose data is the data of the SECOND field of m. //for example, pos.recompose(face2node) will produce the positions for each face - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD)const{ + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM)const{ return Rep()->recompose(m, comm);} virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm);} virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap,comm);} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap,comm);} // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and // redistributes the gMultiStores according to the domain partition @@ -344,7 +344,7 @@ namespace Loci { // containers will need to implement this method later. virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->split_redistribute(dom_ptn, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->split_redistribute(dom_ptn, comm);} //insert elements into store void insert(gEntity e, const T &val){ diff --git a/src/include/gparameter_def.h b/src/include/gparameter_def.h index a21799a0..f7c6598a 100644 --- a/src/include/gparameter_def.h +++ b/src/include/gparameter_def.h @@ -81,24 +81,24 @@ namespace Loci { //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and // redistributes the container according to the domain partition virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // this redistribute version takes an additional remap // argument, after redistribution, the new container is remapped // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const; + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const; // virtual gStoreRepP // redistribute_omd(const std::vector& dom_ptn, - // const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + // const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual gstore_type RepType() const { return GPARAMETER ;} virtual gEntitySet domain() const { return store_domain ;} @@ -212,14 +212,14 @@ namespace Loci { //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm);} // this redistribute version takes an additional remap // argument, after redistribution, the new container is remapped // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap,comm);} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap,comm);} // the redistribute takes a vector of gEntitySets as domain @@ -227,7 +227,7 @@ namespace Loci { // redistributes container according to the domain partition virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->split_redistribute(dom_ptn, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->split_redistribute(dom_ptn, comm);} // the remap method merely renumbers the container diff --git a/src/include/gpuMap.h b/src/include/gpuMap.h index 5e74d381..72c76a96 100644 --- a/src/include/gpuMap.h +++ b/src/include/gpuMap.h @@ -89,7 +89,7 @@ namespace Loci { #endif Entity * get_base_ptr() const { Entity * p = 0 ; if(alloc_id>=0) p = ((Entity *)GPUstoreAllocateData[alloc_id].base_ptr) -GPUstoreAllocateData[alloc_id].base_offset ; return p ; } - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual DatatypeP getType() ; virtual frame_info get_frame_info() ; virtual void copyFrom(const storeRepP &fromMap, entitySet set) ; diff --git a/src/include/gpuMapVec_def.h b/src/include/gpuMapVec_def.h index 524b643d..999c19fc 100644 --- a/src/include/gpuMapVec_def.h +++ b/src/include/gpuMapVec_def.h @@ -82,7 +82,7 @@ namespace Loci { return base_ptr; } - virtual storeRepP expand(entitySet & out_of_dom, std::vector & init_ptn); + virtual storeRepP expand(entitySet & out_of_dom, std::vector & init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM); virtual storeRepP freeze(); virtual storeRepP thaw(); virtual DatatypeP getType(); diff --git a/src/include/gpuMapVec_impl.h b/src/include/gpuMapVec_impl.h index baa14a50..850fdf39 100644 --- a/src/include/gpuMapVec_impl.h +++ b/src/include/gpuMapVec_impl.h @@ -210,7 +210,7 @@ namespace Loci { template inline storeRepP gpuMapVecRepI::expand( - entitySet & out_of_dom, std::vector & init_ptn + entitySet & out_of_dom, std::vector & init_ptn, MPI_Comm comm ) { cerr << "gpuMapVecRepI::expand is not implemented" << endl; Loci::Abort(); diff --git a/src/include/gpuparameter_def.h b/src/include/gpuparameter_def.h index e953095d..467ac959 100644 --- a/src/include/gpuparameter_def.h +++ b/src/include/gpuparameter_def.h @@ -93,10 +93,10 @@ namespace Loci { // virtual gStoreRepP copy2gstore()const; virtual storeRepP redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD) ; + MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual void shift(int_type offset) ; virtual ~gpuparamRepI() ; virtual store_type RepType() const ; diff --git a/src/include/gstore_def.h b/src/include/gstore_def.h index 6ca7d9b3..8bc399fc 100644 --- a/src/include/gstore_def.h +++ b/src/include/gstore_def.h @@ -133,8 +133,8 @@ namespace Loci { const char* name, IDENTITY_CONVERTER c, const gEntitySet &en) const; void hdf5write(hid_t group_id, hid_t dataspace, hid_t dataset, hsize_t dimension, const char* name, USER_DEFINED_CONVERTER c, const gEntitySet &en) const; - virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; - virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm=MPI_COMM_WORLD )const ; + virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; + virtual gStoreRepP recompose(const gMultiMap &m, MPI_Comm comm LOCI_DEFAULT_COMM )const ; public: typedef typename std::vector >::iterator iterator ; typedef typename std::vector >::const_iterator const_iterator ; @@ -153,21 +153,21 @@ namespace Loci { //For stores,recompose will compose a store whose domain is the domain of m, //whose data is the data of the SECOND field of m. //for example, pos.recompose(face2node) will produce the positions for each face - virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD)const ; + virtual gStoreRepP recompose( gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM)const ; // this method redistributes the stores according to the split of local domain over a group of process //dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // this redistribute version takes an additional remap // argument, after redistribution, the new store is remapped // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split,const gMap &remap, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and @@ -177,7 +177,7 @@ namespace Loci { // containers will need to implement this method later. virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const; + MPI_Comm comm LOCI_DEFAULT_COMM)const; //binary search function, for random access //mixed equality and equivalance @@ -340,7 +340,7 @@ namespace Loci { //For stores,recompose will compose a store whose domain is the domain of m, //whose data is the data of the SECOND field of m. //for example, pos.recompose(face2node) will produce the positions for each face - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD)const{ + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM)const{ return Rep()->recompose(m, comm);} // virtual gStoreRepP recompose(const gMap &m, MPI_Comm comm=MPI_COMM_WORLD )const{ // return Rep()->recompose(m, comm);} @@ -351,10 +351,10 @@ namespace Loci { virtual gStoreRepP redistribute(const std::vector& dom_split, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split, comm);} virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->redistribute(dom_split,remap,comm);} + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->redistribute(dom_split,remap,comm);} // the redistribute takes a vector of gEntitySets as domain // distribution over a group of processes and // redistributes the gStores according to the domain partition @@ -363,7 +363,7 @@ namespace Loci { // containers will need to implement this method later. virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const{return Rep()->split_redistribute(dom_ptn, comm);} + MPI_Comm comm LOCI_DEFAULT_COMM)const{return Rep()->split_redistribute(dom_ptn, comm);} //insert elements into store void insert(gEntity e, const T &val){ diff --git a/src/include/gstore_impl.h b/src/include/gstore_impl.h index 9b3ced64..ba04d06d 100644 --- a/src/include/gstore_impl.h +++ b/src/include/gstore_impl.h @@ -167,7 +167,7 @@ namespace Loci { gStoreRepP gStoreRepI::recompose(const gMultiMap &remap, MPI_Comm comm)const{ gEntitySet new_dom = remap.image(); gEntitySet old_dom = domain(); - vector ptn = g_all_collect_vectors(new_dom); + vector ptn = g_all_collect_vectors(new_dom, comm); gStore expanded_store;//the value in expanded store is unique in each process expanded_store = split_redistribute(ptn, comm); expanded_store.local_sort(); @@ -218,7 +218,7 @@ namespace Loci { gEntitySet new_dom = remap.image(); gEntitySet old_dom = domain(); - vector ptn = g_all_collect_vectors(new_dom); + vector ptn = g_all_collect_vectors(new_dom, comm); gStore expanded_store;//the value in expanded store is unique in each process expanded_store = split_redistribute(ptn, comm); expanded_store.local_sort(); diff --git a/src/include/gstore_rep.h b/src/include/gstore_rep.h index b630bfee..89e1ec6a 100644 --- a/src/include/gstore_rep.h +++ b/src/include/gstore_rep.h @@ -93,7 +93,7 @@ namespace Loci { //for example, pos.recompose(face2node) will producea store //of the positions for each face //For maps, in necessary, expanded map will be generated from m - virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm=MPI_COMM_WORLD)const{ + virtual gStoreRepP recompose(gStoreRepP &m, MPI_Comm comm LOCI_DEFAULT_COMM)const{ std::cerr << "gStoreRep.recompose() is not implemented yet" << std::endl ; abort() ; @@ -109,7 +109,7 @@ namespace Loci { // containers will need to implement this method later. virtual gStoreRepP redistribute(const std::vector& dom_split,//send split - MPI_Comm comm=MPI_COMM_WORLD)const { + MPI_Comm comm LOCI_DEFAULT_COMM)const { std::cerr << "gStoreRep.redistribute() is not implemented yet" << std::endl ; abort() ; @@ -121,7 +121,7 @@ namespace Loci { //first the local domain is splitted according to dom_ptn, then redistribute is performed virtual gStoreRepP split_redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD)const { + MPI_Comm comm LOCI_DEFAULT_COMM)const { std::cerr << "gStoreRep.split_redistribute() is not implemented yet" << std::endl ; abort() ; @@ -134,7 +134,7 @@ namespace Loci { // dom_split: the send split of local domain virtual gStoreRepP redistribute(const std::vector& dom_split, - const gMap& remap, MPI_Comm comm=MPI_COMM_WORLD)const { + const gMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM)const { std::cerr << "gStoreRep.redistribute() is not implemented yet" << std::endl ; abort() ; diff --git a/src/include/mpi_containerIO.h b/src/include/mpi_containerIO.h index 263bfd63..02939e96 100644 --- a/src/include/mpi_containerIO.h +++ b/src/include/mpi_containerIO.h @@ -59,7 +59,8 @@ namespace Loci { MPI_Comm comm) ; extern int getMinFileNumberFromLocal(entitySet read_set, - fact_db::distribute_infoP dist ) ; + fact_db::distribute_infoP dist, + MPI_Comm comm LOCI_DEFAULT_COMM) ; //only process 0 will call this function to write header, if ordered, also write domain eset void pmpi_WriteHeader(MPI_File fh, const entitySet &eset, const store_header& header ); @@ -105,7 +106,7 @@ namespace Loci { /* write out the store that already in file numbering */ - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = facts.get_comm() ; int prank = 0 ; int np = 0 ; MPI_Comm_rank(comm,&prank) ; @@ -178,7 +179,7 @@ namespace Loci { /* write out the storeVec that already in file numbering */ - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = facts.get_comm() ; int prank = 0 ; int np = 0 ; MPI_Comm_rank(comm,&prank) ; @@ -250,7 +251,7 @@ namespace Loci { const const_store &var, const entitySet& write_set, fact_db &facts, int xfer_type, bool ordered) { #ifndef MPI_STUBB - MPI_Comm comm = MPI_COMM_WORLD ; + MPI_Comm comm = facts.get_comm() ; int prank = 0 ; int np = 0 ; MPI_Comm_rank(comm,&prank) ; @@ -375,7 +376,7 @@ namespace Loci { const entitySet& write_set, fact_db &facts, int xfer_type, bool ordered) { #ifndef MPI_STUBB - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = facts.get_comm() ; int prank = 0 ; int np = 0 ; MPI_Comm_rank(comm,&prank) ; @@ -503,7 +504,7 @@ namespace Loci { entitySet read_set, fact_db &facts, int xfer_type) { #ifndef MPI_STUBB - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = facts.get_comm() ; int prank = 0 ; int np = 0 ; MPI_Comm_rank(comm,&prank) ; @@ -549,10 +550,10 @@ namespace Loci { std::vector send_sz(np,0) ; std::vector recv_sz(np,0) ; std::vector recv_local_num ; - distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID) ; + distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID,comm) ; std::vector recv_data ; - sendData(recv_data,send_sz,recv_sz,recv_local_num,data,procID) ; + sendData(recv_data,send_sz,recv_sz,recv_local_num,data,procID,comm) ; std::vector alloc_set = recv_local_num ; @@ -621,7 +622,7 @@ namespace Loci { // read in over the same set int minID = offset ; MPI_Bcast(&minID,1,MPI_INT,0,comm) ; - const int minIDf = getMinFileNumberFromLocal(read_set,dist) ; + const int minIDf = getMinFileNumberFromLocal(read_set,dist,comm) ; const int correct = minIDf - minID ; offset += correct ; @@ -662,7 +663,7 @@ namespace Loci { entitySet read_set, fact_db &facts, int xfer_type) { #ifndef MPI_STUBB - MPI_Comm comm = MPI_COMM_WORLD; + MPI_Comm comm = facts.get_comm() ; int prank = 0 ; int np = 0 ; MPI_Comm_rank(comm,&prank) ; @@ -713,10 +714,10 @@ namespace Loci { std::vector send_sz(np,0) ; std::vector recv_sz(np,0) ; std::vector recv_local_num ; - distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID) ; + distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID,comm) ; std::vector recv_data ; - sendMultiData(recv_data,send_sz,recv_sz,recv_local_num,data,procID, vec_size) ; + sendMultiData(recv_data,send_sz,recv_sz,recv_local_num,data,procID, vec_size,comm) ; std::vector alloc_set = recv_local_num ; @@ -785,7 +786,7 @@ namespace Loci { // read in over the same set int minID = offset ; MPI_Bcast(&minID,1,MPI_INT,0,comm) ; - const int minIDf = getMinFileNumberFromLocal(read_set,dist) ; + const int minIDf = getMinFileNumberFromLocal(read_set,dist,comm) ; const int correct = minIDf - minID ; offset += correct ; diff --git a/src/include/mpi_multiStoreIO.h b/src/include/mpi_multiStoreIO.h index 41831b1c..6c286809 100644 --- a/src/include/mpi_multiStoreIO.h +++ b/src/include/mpi_multiStoreIO.h @@ -306,10 +306,12 @@ namespace Loci { void pmpi_writeMultiStoreP(std::string filename, const const_multiStore &var, entitySet write_set, fact_db &facts, int xfer_type) { + MPI_Comm comm = facts.get_comm() ; MPI_File fh = 0; MPI_Offset moffset = 0; - MPI_File_open( MPI_COMM_WORLD, filename.c_str(), - MPI_MODE_WRONLY | MPI_MODE_CREATE, PHDF5_MPI_Info, &fh) ; + MPI_File_open( comm, filename.c_str(), + MPI_MODE_WRONLY | MPI_MODE_CREATE, + PHDF5_MPI_Info, &fh) ; std::vector sizes_local ;//unique sizes on this process, such as 0, 35, 105,... @@ -354,8 +356,10 @@ namespace Loci { //no groups, write out two vectors counts and fileids - pmpi_writeUnorderedVectorP(fh, moffset, counts, MPI_COMM_WORLD, xfer_type) ; - pmpi_writeUnorderedVectorP(fh, moffset, fileids, MPI_COMM_WORLD, xfer_type) ; + pmpi_writeUnorderedVectorP(fh, moffset, counts, comm, + xfer_type) ; + pmpi_writeUnorderedVectorP(fh, moffset, fileids, comm, + xfer_type) ; std::sort(counts.begin(),counts.end()) ; std::vector::const_iterator lu = std::unique(counts.begin(),counts.end()) ; @@ -374,7 +378,8 @@ namespace Loci { std::vector sizes ; //unique sizes for all processors int cmin = -1 ; do { - MPI_Allreduce(&sizes_local[iloc],&cmin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&sizes_local[iloc],&cmin,1,MPI_INT, + MPI_MIN,comm) ; if(cmin != std::numeric_limits::max()) sizes.push_back(cmin) ; if(sizes_local[iloc] == cmin) @@ -413,12 +418,13 @@ namespace Loci { std::vector block_data_elems(block_sizes.size(),0) ;//global::for each block, how many elements need xfer MPI_Allreduce(&num_local_xfers[0],&block_data_elems[0],block_sizes.size(), - MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_INT,MPI_SUM,comm) ; // write out write schedule information - pmpi_writeVectorSerialP(fh, moffset, block_sizes, MPI_COMM_WORLD) ; - pmpi_writeVectorSerialP(fh, moffset, block_data_elems, MPI_COMM_WORLD) ; + pmpi_writeVectorSerialP(fh, moffset, block_sizes, comm) ; + pmpi_writeVectorSerialP(fh, moffset, + block_data_elems, comm) ; // pre_time += sp.stop() ; // Now write out the main data block @@ -432,8 +438,8 @@ namespace Loci { int p = 0; int r = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &r); - MPI_Comm_size(MPI_COMM_WORLD, &p); + MPI_Comm_rank(comm, &r); + MPI_Comm_size(comm, &p); if(r == 0)MPI_File_write_at(fh, moffset, &total_size, sizeof(unsigned long), MPI_BYTE, MPI_STATUS_IGNORE); moffset += (MPI_Offset)sizeof(unsigned long); @@ -449,7 +455,7 @@ namespace Loci { std::vector all_local_sizes(block_sizes.size()*p) ; MPI_Allgather(&num_local_xfers[0],block_sizes.size(),MPI_INT, &all_local_sizes[0],block_sizes.size(),MPI_INT, - MPI_COMM_WORLD) ; + comm) ; /* Create a large dataset for all processes */ @@ -502,7 +508,7 @@ namespace Loci { int lsz = (count==0)?0:1 ; int gsz = lsz ; - MPI_Allreduce(&lsz,&gsz,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&lsz,&gsz,1,MPI_INT,MPI_MAX,comm) ; if(gsz != 0) { if(xfer_type == DXFER_COLLECTIVE_IO) @@ -530,19 +536,22 @@ namespace Loci { fact_db &facts, int xfer_type) { //group_id is known to the world + MPI_Comm comm = facts.get_comm() ; MPI_File fh = 0; MPI_Offset moffset = 0; - MPI_File_open( MPI_COMM_WORLD, filename.c_str(), + MPI_File_open( comm, filename.c_str(), MPI_MODE_RDONLY, PHDF5_MPI_Info, &fh) ; int mpi_size; - MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + MPI_Comm_size(comm, &mpi_size); int r = MPI_rank ; std::vector counts ; std::vector fileID ; - pmpi_readUnorderedVectorP(fh, moffset, counts, MPI_COMM_WORLD, xfer_type) ; - pmpi_readUnorderedVectorP(fh, moffset, fileID, MPI_COMM_WORLD, xfer_type) ; + pmpi_readUnorderedVectorP(fh, moffset, counts, comm, + xfer_type) ; + pmpi_readUnorderedVectorP(fh, moffset, fileID, comm, + xfer_type) ; // Now we need to create a mapping from fileID to local and processor // number @@ -554,9 +563,9 @@ namespace Loci { std::vector send_sz(p,0) ; std::vector recv_sz(p,0) ; std::vector recv_local_num ; - distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID) ; + distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID,comm) ; std::vector recv_count ; - sendData(recv_count,send_sz,recv_sz,recv_local_num,counts,procID) ; + sendData(recv_count,send_sz,recv_sz,recv_local_num,counts,procID,comm) ; std::vector alloc_set = recv_local_num ; @@ -580,25 +589,27 @@ namespace Loci { std::vector block_sets ; - pmpi_readVectorSerialP(fh, moffset, block_schedule, MPI_COMM_WORLD) ; - pmpi_readVectorSerialP(fh, moffset, block_sets, MPI_COMM_WORLD) ; + pmpi_readVectorSerialP(fh, moffset, block_schedule, + comm) ; + pmpi_readVectorSerialP(fh, moffset, block_sets, comm) ; int bsize = block_schedule.size() ; - MPI_Bcast(&bsize,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&bsize,1,MPI_INT,0,comm) ; if(0 != r) { std::vector bsched(bsize) ; block_schedule.swap(bsched) ; std::vector bsets(bsize) ; block_sets.swap(bsets) ; } - MPI_Bcast(&block_schedule[0],bsize,MPI_INT,0,MPI_COMM_WORLD) ; - MPI_Bcast(&block_sets[0],bsize,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&block_schedule[0],bsize,MPI_INT,0,comm) ; + MPI_Bcast(&block_sets[0],bsize,MPI_INT,0,comm) ; //prime processes Open data array unsigned long dimension = 0 ; //read in dimension here if(r == 0) MPI_File_read_at(fh, moffset, &dimension, sizeof(unsigned long), MPI_BYTE, MPI_STATUS_IGNORE); - MPI_Bcast(&dimension,sizeof(unsigned long), MPI_BYTE,0,MPI_COMM_WORLD) ; + MPI_Bcast(&dimension,sizeof(unsigned long), + MPI_BYTE,0,comm) ; moffset += (MPI_Offset)sizeof(unsigned long); @@ -621,7 +632,8 @@ namespace Loci { send_sz_blk[procID[j]]++ ; } - MPI_Allgather(&lbsz,1,MPI_INT,&file_read_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + MPI_Allgather(&lbsz,1,MPI_INT,&file_read_sizes[0], + 1,MPI_INT,comm) ; int tot = 0 ; for(int j=0;j statuslist(nreq) ; diff --git a/src/include/multiMap.h b/src/include/multiMap.h index 7c4281d6..b7536c58 100644 --- a/src/include/multiMap.h +++ b/src/include/multiMap.h @@ -88,7 +88,7 @@ namespace Loci { virtual DatatypeP getType() ; virtual frame_info get_frame_info() ; private: - virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn) ; + virtual storeRepP expand(entitySet &out_of_dom, std::vector &init_ptn, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP freeze() ; virtual storeRepP thaw() ; } ; diff --git a/src/include/multiStoreIO.h b/src/include/multiStoreIO.h index ec1d86e8..718e891d 100644 --- a/src/include/multiStoreIO.h +++ b/src/include/multiStoreIO.h @@ -45,11 +45,14 @@ namespace Loci { hid_t hdf5PCreateFile(const char *name, unsigned flags, hid_t create_id, hid_t access_id, size_t file_size_estimate,MPI_Comm comm) ;//obselete +#ifndef LOCI_STRICT_COMM inline hid_t hdf5PCreateFile(const char *name, unsigned flags, hid_t create_id, hid_t access_id,size_t file_size_estimate) { //obselete return hdf5PCreateFile(name,flags,create_id,access_id,file_size_estimate, MPI_COMM_WORLD) ; } +#endif +#ifndef LOCI_STRICT_COMM inline hid_t hdf5POpenFile(const char *name, unsigned flags, hid_t access_id) {//obselete #ifndef H5_HAVE_PARALLEL if(Loci::MPI_rank==0) @@ -60,6 +63,7 @@ namespace Loci { return H5Fopen(name,flags,access_id) ; #endif } +#endif inline hid_t hdf5POpenFile(const char *name, unsigned flags, hid_t access_id, MPI_Comm comm) { //obselete @@ -110,7 +114,8 @@ namespace Loci { std::vector &recv_sz, std::vector &recv_local_num, const std::vector &local_num, - const std::vector &procID) ; + const std::vector &procID, + MPI_Comm comm LOCI_DEFAULT_COMM) ; template< class T > @@ -119,7 +124,8 @@ namespace Loci { const std::vector &recv_sz, const std::vector &recv_local_num, const std::vector &counts, - const std::vector &procID) { + const std::vector &procID, + MPI_Comm comm LOCI_DEFAULT_COMM) { const int p = MPI_processes ; std::vector soffsets(p+1,0) ; std::vector roffsets(p+1,0) ; @@ -148,13 +154,13 @@ namespace Loci { for(int i=0;i 0) { MPI_Irecv(&rbuf[roffsets[i]],recv_sz[i]*sizeof(T),MPI_BYTE,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&sbuf[soffsets[i]],send_sz[i]*sizeof(T),MPI_BYTE,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } std::vector statuslist(nreq) ; @@ -170,7 +176,8 @@ namespace Loci { const std::vector &recv_local_num, const std::vector &counts, const std::vector &procID, - int vec_size) { + int vec_size, + MPI_Comm comm LOCI_DEFAULT_COMM) { const int p = MPI_processes ; std::vector soffsets(p+1,0) ; std::vector roffsets(p+1,0) ; @@ -202,13 +209,13 @@ namespace Loci { for(int i=0;i 0) { MPI_Irecv(&rbuf[roffsets[i]],recv_sz[i]*sizeof(T)*vec_size,MPI_BYTE,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int i=0;i 0) { MPI_Isend(&sbuf[soffsets[i]],send_sz[i]*sizeof(T)*vec_size,MPI_BYTE,i,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } std::vector statuslist(nreq) ; @@ -218,7 +225,7 @@ namespace Loci { } - inline size_t containerSizeEstimateKb(storeRepP p) { + inline size_t containerSizeEstimateKb(storeRepP p, MPI_Comm comm LOCI_DEFAULT_COMM) { entitySet dom = p->domain() ; int szkb = 0 ; for(size_t i=0;i::const_iterator lu = std::unique(counts.begin(),counts.end()) ; int nunique = lu-counts.begin() ; @@ -303,12 +313,14 @@ namespace Loci { // schedule blocking for writing // Find global sizes array + MPI_Comm comm = facts.get_comm() ; sizes_local.push_back(std::numeric_limits::max()) ; int iloc = 0 ; std::vector sizes ; int cmin = -1 ; do { - MPI_Allreduce(&sizes_local[iloc],&cmin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&sizes_local[iloc],&cmin,1,MPI_INT, + MPI_MIN,comm) ; if(cmin != std::numeric_limits::max()) sizes.push_back(cmin) ; if(sizes_local[iloc] == cmin) @@ -346,8 +358,9 @@ namespace Loci { } ENDFORALL ; std::vector block_data_elems(block_sizes.size(),0) ; - MPI_Allreduce(&num_local_xfers[0],&block_data_elems[0],block_sizes.size(), - MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&num_local_xfers[0],&block_data_elems[0], + block_sizes.size(), + MPI_INT,MPI_SUM,comm) ; int p = MPI_processes ; int r = MPI_rank ; @@ -370,7 +383,8 @@ namespace Loci { max_local_size = max(max_local_size,local_size) ; } int max_size = max_local_size ; - MPI_Allreduce(&max_local_size,&max_size,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&max_local_size,&max_size,1,MPI_INT, + MPI_MAX,comm) ; if(r == 0) max_local_size = max_size ; @@ -378,7 +392,7 @@ namespace Loci { MPI_Gather(&num_local_xfers[0],block_sizes.size(),MPI_INT, &all_local_sizes[0],block_sizes.size(),MPI_INT, - 0, MPI_COMM_WORLD) ; + 0, comm) ; const int nadvance = 3 ; std::vector > buffer(nadvance) ; @@ -422,10 +436,11 @@ namespace Loci { int id = k ; static int flag = 0 ; - MPI_Irecv(&buffer[id][0],sizeof(T)*count,MPI_BYTE,k,0,MPI_COMM_WORLD, + MPI_Irecv(&buffer[id][0],sizeof(T)*count, + MPI_BYTE,k,0,comm, &recv_Requests[id]) ; - // MPI_Isend(&flag,1,MPI_INT,k,0,MPI_COMM_WORLD,&send_Requests[id]) ; - MPI_Send(&flag,1,MPI_INT,k,0,MPI_COMM_WORLD) ; + // MPI_Isend(&flag,1,MPI_INT,k,0,comm,&send_Requests[id]) ; + MPI_Send(&flag,1,MPI_INT,k,0,comm) ; } } @@ -458,10 +473,11 @@ namespace Loci { int id = 0 ; static int flag = 0 ; - MPI_Irecv(&buffer[id][0],sizeof(T)*count,MPI_BYTE,nadvance,0,MPI_COMM_WORLD, + MPI_Irecv(&buffer[id][0],sizeof(T)*count, + MPI_BYTE,nadvance,0,comm, &recv_Requests[id]) ; - // MPI_Isend(&flag,1,MPI_INT,nadvance,0,MPI_COMM_WORLD,&send_Requests[id]) ; - MPI_Send(&flag,1,MPI_INT,nadvance,0,MPI_COMM_WORLD) ; + // MPI_Isend(&flag,1,MPI_INT,nadvance,0,comm,&send_Requests[id]) ; + MPI_Send(&flag,1,MPI_INT,nadvance,0,comm) ; } } for(int k=1;k::const_iterator lu = std::unique(counts.begin(),counts.end()) ; int nunique = lu-counts.begin() ; @@ -630,12 +650,14 @@ namespace Loci { // schedule blocking for writing // Find global sizes array + MPI_Comm comm = facts.get_comm() ; sizes_local.push_back(std::numeric_limits::max()) ; int iloc = 0 ; std::vector sizes ; //unique sizes for all processors int cmin = -1 ; do { - MPI_Allreduce(&sizes_local[iloc],&cmin,1,MPI_INT,MPI_MIN,MPI_COMM_WORLD) ; + MPI_Allreduce(&sizes_local[iloc],&cmin,1,MPI_INT, + MPI_MIN,comm) ; if(cmin != std::numeric_limits::max()) sizes.push_back(cmin) ; if(sizes_local[iloc] == cmin) @@ -673,12 +695,15 @@ namespace Loci { } ENDFORALL ; std::vector block_data_elems(block_sizes.size(),0) ;//global::for each block, how many elements need xfer - MPI_Allreduce(&num_local_xfers[0],&block_data_elems[0],block_sizes.size(), - MPI_INT,MPI_SUM,MPI_COMM_WORLD) ; + MPI_Allreduce(&num_local_xfers[0],&block_data_elems[0], + block_sizes.size(), + MPI_INT,MPI_SUM,comm) ; // write out write schedule information - Loci::pio::writeVectorSerialP(group_id,"block_schedule",block_sizes, MPI_COMM_WORLD) ; - Loci::pio::writeVectorSerialP(group_id,"block_sets", block_data_elems, MPI_COMM_WORLD) ; + Loci::pio::writeVectorSerialP(group_id,"block_schedule", + block_sizes, comm) ; + Loci::pio::writeVectorSerialP(group_id,"block_sets", + block_data_elems, comm) ; // pre_time += sp.stop() ; // Now write out the main data block @@ -708,13 +733,13 @@ namespace Loci { int p = 0; int r = 0; - MPI_Comm_rank(MPI_COMM_WORLD, &r); - MPI_Comm_size(MPI_COMM_WORLD, &p); + MPI_Comm_rank(comm, &r); + MPI_Comm_size(comm, &p); //allgather local sizes over std::vector all_local_sizes(block_sizes.size()*p) ; MPI_Allgather(&num_local_xfers[0],block_sizes.size(),MPI_INT, &all_local_sizes[0],block_sizes.size(),MPI_INT, - MPI_COMM_WORLD) ; + comm) ; hid_t dataspace =-1; hid_t dataset =-1; @@ -807,7 +832,7 @@ namespace Loci { int lsz = (count==0)?0:1 ; int gsz = lsz ; - MPI_Allreduce(&lsz,&gsz,1,MPI_INT,MPI_MAX,MPI_COMM_WORLD) ; + MPI_Allreduce(&lsz,&gsz,1,MPI_INT,MPI_MAX,comm) ; if(gsz != 0) { #ifdef DEBUG ret = @@ -840,6 +865,7 @@ namespace Loci { multiStore &var, entitySet read_set, fact_db &facts) { + MPI_Comm comm = facts.get_comm() ; hid_t group_id = -1; int r = MPI_rank ; if(0 == r) { @@ -848,8 +874,8 @@ namespace Loci { std::vector counts ; std::vector fileID ; - readUnorderedVectorS(group_id,"counts",counts,MPI_COMM_WORLD) ; - readUnorderedVectorS(group_id,"fileID",fileID,MPI_COMM_WORLD) ; + readUnorderedVectorS(group_id,"counts",counts,comm) ; + readUnorderedVectorS(group_id,"fileID",fileID,comm) ; // Now we need to create a mapping from fileID to local and processor // number @@ -861,9 +887,10 @@ namespace Loci { std::vector send_sz(p,0) ; std::vector recv_sz(p,0) ; std::vector recv_local_num ; - distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID) ; + distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID, + comm) ; std::vector recv_count ; - sendData(recv_count,send_sz,recv_sz,recv_local_num,counts,procID) ; + sendData(recv_count,send_sz,recv_sz,recv_local_num,counts,procID,comm) ; // distributeMapMultiStore(send_sz,recv_sz,recv_count,counts,procID) ; std::vector alloc_set = recv_local_num ; @@ -890,15 +917,15 @@ namespace Loci { readVectorSerial(group_id,"block_sets", block_sets) ; } int bsize = block_schedule.size() ; - MPI_Bcast(&bsize,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&bsize,1,MPI_INT,0,comm) ; if(0 != r) { std::vector bsched(bsize) ; block_schedule.swap(bsched) ; std::vector bsets(bsize) ; block_sets.swap(bsets) ; } - MPI_Bcast(&block_schedule[0],bsize,MPI_INT,0,MPI_COMM_WORLD) ; - MPI_Bcast(&block_sets[0],bsize,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&block_schedule[0],bsize,MPI_INT,0,comm) ; + MPI_Bcast(&block_sets[0],bsize,MPI_INT,0,comm) ; // Open data array hid_t dataset = 0, dataspace = 0; @@ -931,7 +958,8 @@ namespace Loci { send_sz_blk[procID[j]]++ ; } - MPI_Gather(&lbsz,1,MPI_INT,&file_read_sizes[0],1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Gather(&lbsz,1,MPI_INT,&file_read_sizes[0],1,MPI_INT, + 0,comm) ; int tot = 0 ; for(int j=0;j 0) { - MPI_Recv(&read_buffer[0], msgsz*sizeof(T),MPI_BYTE,0,0, - MPI_COMM_WORLD,&mstat) ; + MPI_Recv(&read_buffer[0], msgsz*sizeof(T), + MPI_BYTE,0,0,comm,&mstat) ; } } @@ -1029,7 +1057,7 @@ namespace Loci { MPI_Irecv(&rdata[roffset[j]*block_schedule[i]], recv_sz_blk[j]*block_schedule[i]*sizeof(T), MPI_BYTE,j,3, - MPI_COMM_WORLD,&recv_Requests[req]) ; + comm,&recv_Requests[req]) ; req++ ; } for(int j=0;j statuslist(nreq) ; @@ -1075,6 +1103,7 @@ namespace Loci { pio::readMultiStoreS(file_id,vname,var,read_set,facts) ; #else //group_id is known to the world + MPI_Comm comm = facts.get_comm() ; hid_t group_id = -1; int r = MPI_rank ; //int mpi_rank = MPI_rank; @@ -1083,8 +1112,10 @@ namespace Loci { std::vector counts ; std::vector fileID ; - Loci::pio::readUnorderedVectorP(group_id,"counts",counts,MPI_COMM_WORLD) ; - Loci::pio::readUnorderedVectorP(group_id,"fileID",fileID,MPI_COMM_WORLD) ; + Loci::pio::readUnorderedVectorP(group_id,"counts",counts, + comm) ; + Loci::pio::readUnorderedVectorP(group_id,"fileID",fileID, + comm) ; // Now we need to create a mapping from fileID to local and processor // number @@ -1098,7 +1129,7 @@ namespace Loci { std::vector recv_local_num ; distributeMapMultiStore(send_sz,recv_sz,recv_local_num,local_num,procID) ; std::vector recv_count ; - sendData(recv_count,send_sz,recv_sz,recv_local_num,counts,procID) ; + sendData(recv_count,send_sz,recv_sz,recv_local_num,counts,procID,comm) ; // distributeMapMultiStore(send_sz,recv_sz,recv_count,counts,procID) ; std::vector alloc_set = recv_local_num ; @@ -1128,15 +1159,15 @@ namespace Loci { } int bsize = block_schedule.size() ; - MPI_Bcast(&bsize,1,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&bsize,1,MPI_INT,0,comm) ; if(0 != r) { std::vector bsched(bsize) ; block_schedule.swap(bsched) ; std::vector bsets(bsize) ; block_sets.swap(bsets) ; } - MPI_Bcast(&block_schedule[0],bsize,MPI_INT,0,MPI_COMM_WORLD) ; - MPI_Bcast(&block_sets[0],bsize,MPI_INT,0,MPI_COMM_WORLD) ; + MPI_Bcast(&block_schedule[0],bsize,MPI_INT,0,comm) ; + MPI_Bcast(&block_sets[0],bsize,MPI_INT,0,comm) ; //prime processes Open data array hid_t dataset = 0, dataspace = 0; @@ -1171,7 +1202,8 @@ namespace Loci { send_sz_blk[procID[j]]++ ; } - MPI_Allgather(&lbsz,1,MPI_INT,&file_read_sizes[0],1,MPI_INT,MPI_COMM_WORLD) ; + MPI_Allgather(&lbsz,1,MPI_INT,&file_read_sizes[0],1,MPI_INT, + comm) ; int tot = 0 ; for(int j=0;j statuslist(nreq) ; diff --git a/src/include/parameter.h b/src/include/parameter.h index 7597ed56..357dacb2 100644 --- a/src/include/parameter.h +++ b/src/include/parameter.h @@ -76,13 +76,13 @@ namespace Loci { virtual void guarantee_domain(const entitySet& include) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD) ; + MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP redistribute_omd(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual void shift(int_type offset) ; virtual ~paramRepI() ; virtual store_type RepType() const ; diff --git a/src/include/partition.h b/src/include/partition.h index 43ae6fd2..9feab0e4 100644 --- a/src/include/partition.h +++ b/src/include/partition.h @@ -71,7 +71,7 @@ namespace Loci { //given send_split, return recv_split; //or given recv_split, return send_split //allow overlap between processes - std::vector transposePtn(const std::vector &ptn, MPI_Comm comm =MPI_COMM_WORLD ); + std::vector transposePtn(const std::vector &ptn, MPI_Comm comm LOCI_DEFAULT_COMM); } diff --git a/src/include/rule.h b/src/include/rule.h index dafe93bd..52c0cbaa 100644 --- a/src/include/rule.h +++ b/src/include/rule.h @@ -213,9 +213,6 @@ namespace Loci { // to reflect the substitution of constraints for maps. void replace_map_constraints(fact_db& facts) ; - void split_constraints(const variableSet& dc) ; - - void set_variable_times(time_ident tl) ; void copy_store_from(rule_impl &f) ; void Print(std::ostream &s) const ; @@ -235,7 +232,6 @@ namespace Loci { // the postlude is similarly defined as the prelude method virtual void postlude(const sequence&) {} virtual CPTR get_joiner() = 0 ; - virtual rule_implP add_namespace(const std::string& n) const ; std::string get_comments() const {return rule_comments ;} std::string get_fileloc() const { return fileloc; } } ; @@ -722,10 +718,6 @@ namespace Loci { int ident() const { return rule::rdb->get_id(*this) ; } rule_implP get_rule_implP() const ; - // SH - namespace support - rule_implP add_namespace(const std::string& n) const { - return rule_impl->add_namespace(n) ; - } } ; private: friend struct rule::info ; @@ -789,12 +781,6 @@ namespace Loci { rule(const std::string &s) { create_rdb(); id = rdb->get_id(info(s)) ; } - // SH - Namespace support - // We use a new rule_implP identical to the original but with namespace'd variables to construct a new rule (rule(rule_implP&)) - // get_rule_implP() gives us our rule_implP for this rule, the rule_implP adds the namespace to a copy of itself - rule add_namespace(const std::string& n) const { - return rule(get_rule_implP()->add_namespace(n)); - } rule parent() const { return rule(*this,time().parent()) ; } /* diff --git a/src/include/store_rep.h b/src/include/store_rep.h index 49a736dd..ede5e61c 100644 --- a/src/include/store_rep.h +++ b/src/include/store_rep.h @@ -359,7 +359,7 @@ namespace Loci { // containers will need to implement this method later. virtual storeRepP redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD) { + MPI_Comm comm LOCI_DEFAULT_COMM) { std::cerr << "storeRep.redistribute() is not implemented yet" << std::endl ; abort() ; @@ -369,7 +369,7 @@ namespace Loci { // argument, upon redistribution, the new store is remapped virtual storeRepP redistribute(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) { + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) { std::cerr << "storeRep.redistribute() is not implemented yet" << std::endl ; abort() ; @@ -502,10 +502,10 @@ namespace Loci { virtual void guarantee_domain(const entitySet& include) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - MPI_Comm comm=MPI_COMM_WORLD) ; + MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual storeRepP redistribute(const std::vector& dom_ptn, - const dMap& remap, MPI_Comm comm=MPI_COMM_WORLD) ; + const dMap& remap, MPI_Comm comm LOCI_DEFAULT_COMM) ; virtual int getDomainKeySpace() const { return Rep()->getDomainKeySpace() ; } virtual void setDomainKeySpace(int v) { Rep()->setDomainKeySpace(v) ; }