New constructor for MPMD::Copier

Src/Base/AMReX_MPMD.H

@@ -26,18 +26,27 @@ int MyProgId (); //! Program ID
 class Copier
 {
 public:
-    Copier (BoxArray const& ba, DistributionMapping const& dm);
+    Copier (bool);
+
+    Copier (BoxArray const& ba, DistributionMapping const& dm,
+            bool send_ba = false);
 
     template <typename FAB>
     void send (FabArray<FAB> const& mf, int icomp, int ncomp) const;
 
     template <typename FAB>
     void recv (FabArray<FAB>& mf, int icomp, int ncomp) const;
 
+    [[nodiscard]] BoxArray const& boxArray () const;
+
+    [[nodiscard]] DistributionMapping const& DistributionMap () const;
+
 private:
     std::map<int,FabArrayBase::CopyComTagsContainer> m_SndTags;
     std::map<int,FabArrayBase::CopyComTagsContainer> m_RcvTags;
     bool m_is_thread_safe;
+    BoxArray m_ba;
+    DistributionMapping m_dm;
 };
 
 template <typename FAB>
@@ -176,6 +185,7 @@ void Copier::recv (FabArray<FAB>& mf, int icomp, int ncomp) const
     }
 }
 
+
 }
 
 #endif

Src/Base/AMReX_MPMD.cpp

@@ -139,7 +139,9 @@ int MyProgId ()
     return (myproc == ParallelDescriptor::MyProc()) ? 0 : 1;
 }
 
-Copier::Copier (BoxArray const& ba, DistributionMapping const& dm)
+Copier::Copier (BoxArray const& ba, DistributionMapping const& dm,
+        bool send_ba)
+        : m_ba(ba), m_dm(dm)
 {
     int rank_offset = myproc - ParallelDescriptor::MyProc();
     int this_root, other_root;
@@ -152,7 +154,6 @@ Copier::Copier (BoxArray const& ba, DistributionMapping const& dm)
     }
 
     Vector<Box> bv = ba.boxList().data();
-
     int this_nboxes = static_cast<int>(ba.size());
     Vector<int> procs = dm.ProcessorMap();
     if (rank_offset != 0) {
@@ -163,34 +164,46 @@ Copier::Copier (BoxArray const& ba, DistributionMapping const& dm)
 
     Vector<Box> obv;
     Vector<int> oprocs;
-    int other_nboxes;
+    int other_nboxes = this_nboxes;
     if (myproc == this_root) {
         if (rank_offset == 0) // the first program
         {
             MPI_Send(&this_nboxes, 1, MPI_INT, other_root, 0, MPI_COMM_WORLD);
-            MPI_Recv(&other_nboxes, 1, MPI_INT, other_root, 1, MPI_COMM_WORLD,
+            if (!send_ba)
+            {
+                MPI_Recv(&other_nboxes, 1, MPI_INT, other_root, 1, MPI_COMM_WORLD,
                      MPI_STATUS_IGNORE);
-            obv.resize(other_nboxes);
+                obv.resize(other_nboxes);
+            }
             MPI_Send(bv.data(), this_nboxes,
                      ParallelDescriptor::Mpi_typemap<Box>::type(),
                      other_root, 2, MPI_COMM_WORLD);
-            MPI_Recv(obv.data(), other_nboxes,
+            if (!send_ba)
+            {
+                MPI_Recv(obv.data(), other_nboxes,
                      ParallelDescriptor::Mpi_typemap<Box>::type(),
                      other_root, 3, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
-            oprocs.resize(other_nboxes);
+            }
             MPI_Send(procs.data(), this_nboxes, MPI_INT, other_root, 4, MPI_COMM_WORLD);
+            oprocs.resize(other_nboxes);
             MPI_Recv(oprocs.data(), other_nboxes, MPI_INT, other_root, 5, MPI_COMM_WORLD,
                      MPI_STATUS_IGNORE);
         }
         else // the second program
         {
-            MPI_Recv(&other_nboxes, 1, MPI_INT, other_root, 0, MPI_COMM_WORLD,
+            if (!send_ba)
+            {
+                MPI_Recv(&other_nboxes, 1, MPI_INT, other_root, 0, MPI_COMM_WORLD,
                      MPI_STATUS_IGNORE);
+                obv.resize(other_nboxes);
+            }
             MPI_Send(&this_nboxes, 1, MPI_INT, other_root, 1, MPI_COMM_WORLD);
-            obv.resize(other_nboxes);
-            MPI_Recv(obv.data(), other_nboxes,
+            if (!send_ba)
+            {
+                MPI_Recv(obv.data(), other_nboxes,
                      ParallelDescriptor::Mpi_typemap<Box>::type(),
                      other_root, 2, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+            }
             MPI_Send(bv.data(), this_nboxes,
                      ParallelDescriptor::Mpi_typemap<Box>::type(),
                      other_root, 3, MPI_COMM_WORLD);
@@ -201,30 +214,48 @@ Copier::Copier (BoxArray const& ba, DistributionMapping const& dm)
         }
     }
 
-    ParallelDescriptor::Bcast(&other_nboxes, 1);
-    if (obv.empty()) {
-        obv.resize(other_nboxes);
+    if (!send_ba) {
+        ParallelDescriptor::Bcast(&other_nboxes, 1);
+        if (obv.empty()){
+            obv.resize(other_nboxes);
+        }
+        ParallelDescriptor::Bcast(obv.data(), obv.size());
+    }
+
+    if (oprocs.empty()) {
         oprocs.resize(other_nboxes);
     }
-    ParallelDescriptor::Bcast(obv.data(), obv.size());
     ParallelDescriptor::Bcast(oprocs.data(), oprocs.size());
 
-    BoxArray oba(BoxList(std::move(obv)));
+    BoxArray oba;
+    if (!obv.empty()) {
+        oba.define(BoxList(std::move(obv)));
+    }
 
     // At this point, ba and bv hold our boxes, and oba holds the other
     // program's boxes. procs holds mpi ranks of our boxes, and oprocs holds
     // mpi ranks of the other program's boxes.  All mpi ranks are in
     // MPI_COMM_WORLD.
 
     // Build communication meta-data
-    AMREX_ALWAYS_ASSERT(ba.ixType() == oba.ixType());
-    m_is_thread_safe = ba.ixType().cellCentered();
+    if (!send_ba){
+        AMREX_ALWAYS_ASSERT(ba.ixType() == oba.ixType());
+        m_is_thread_safe = ba.ixType().cellCentered();
+    }else{
+        m_is_thread_safe = true;
+    }
 
     std::vector<std::pair<int,Box> > isects;
 
     for (int i = 0; i < this_nboxes; ++i) {
         if (procs[i] == myproc) {
-            oba.intersections(bv[i], isects);
+            if (!send_ba){
+                oba.intersections(bv[i], isects);
+            }
+            else{
+                isects.resize(0);
+                isects.emplace_back(i,bv[i]);
+            }
             for (auto const& isec : isects) {
                 const int oi = isec.first;
                 const Box& bx = isec.second;
@@ -235,14 +266,112 @@ Copier::Copier (BoxArray const& ba, DistributionMapping const& dm)
         }
     }
 
-    for (auto& kv : m_SndTags) {
-        std::sort(kv.second.begin(), kv.second.end());
+    if (!send_ba){
+        for (auto& kv : m_SndTags) {
+            std::sort(kv.second.begin(), kv.second.end());
+        }
+        for (auto& kv : m_RcvTags) {
+            std::sort(kv.second.begin(), kv.second.end());
+        }
+    }
+}
+
+Copier::Copier (bool)
+    : m_is_thread_safe(true)
+{
+    int rank_offset = myproc - ParallelDescriptor::MyProc();
+    int this_root, other_root;
+    if (rank_offset == 0) { // First program
+        this_root = 0;
+        other_root = ParallelDescriptor::NProcs();
+    } else {
+        this_root = rank_offset;
+        other_root = 0;
+    }
+
+    Vector<Box> bv;
+    int this_nboxes;
+
+    if (myproc == this_root) {
+        int tags[2];
+        if (rank_offset == 0) // the first program
+        {
+            tags[0] = 1;
+            tags[1] = 3;
+        }
+        else // the second program
+        {
+            tags[0] = 0;
+            tags[1] = 2;
+        }
+
+        MPI_Recv(&this_nboxes, 1, MPI_INT, other_root, tags[0], MPI_COMM_WORLD,
+                 MPI_STATUS_IGNORE);
+        bv.resize(this_nboxes);
+        MPI_Recv(bv.data(), this_nboxes,
+                 ParallelDescriptor::Mpi_typemap<Box>::type(),
+                 other_root, tags[1], MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+    }
+
+    ParallelDescriptor::Bcast(&this_nboxes, 1);
+    if (bv.empty()) {
+        bv.resize(this_nboxes);
+    }
+
+    ParallelDescriptor::Bcast(bv.data(), bv.size());
+    m_ba.define(BoxList(std::move(bv)));
+    m_dm.define(m_ba);
+    Vector<int> procs = m_dm.ProcessorMap();
+    if (rank_offset != 0) {
+        for (int i = 0; i < this_nboxes; ++i) {
+            procs[i] += rank_offset;
+        }
+    }
+
+    Vector<int> oprocs(this_nboxes);
+    if (myproc == this_root) {
+        if (rank_offset == 0) // the first program
+        {
+            MPI_Send(procs.data(), this_nboxes, MPI_INT, other_root, 4, MPI_COMM_WORLD);
+            MPI_Recv(oprocs.data(), this_nboxes, MPI_INT, other_root, 5, MPI_COMM_WORLD,
+                     MPI_STATUS_IGNORE);
+        }
+        else // the second program
+        {
+            MPI_Recv(oprocs.data(), this_nboxes, MPI_INT, other_root, 4, MPI_COMM_WORLD,
+                     MPI_STATUS_IGNORE);
+            MPI_Send(procs.data(), this_nboxes, MPI_INT, other_root, 5, MPI_COMM_WORLD);
+        }
     }
-    for (auto& kv : m_RcvTags) {
-        std::sort(kv.second.begin(), kv.second.end());
+
+    ParallelDescriptor::Bcast(oprocs.data(), oprocs.size());
+
+    // procs holds mpi ranks of our boxes, and oprocs holds
+    // mpi ranks of the other program's boxes.  All mpi ranks are in
+    // MPI_COMM_WORLD.
+
+    // Build communication meta-data
+
+    for (int i = 0; i < this_nboxes; ++i) {
+        if (procs[i] == myproc) {
+            const Box& bx = m_ba[i];
+            const int orank = oprocs[i];
+            m_SndTags[orank].emplace_back(bx, bx, i, i);
+            m_RcvTags[orank].emplace_back(bx, bx, i, i);
+        }
     }
 }
 
+BoxArray const& Copier::boxArray () const
+{
+    return m_ba;
+}
+
+DistributionMapping const& Copier::DistributionMap () const
+{
+    return m_dm;
+}
+
 }
 
 #endif