CMakeLists.txt

@@ -5,7 +5,7 @@ set(PROJECT_NAME entity)
 
 project(
   ${PROJECT_NAME}
-  VERSION 1.1.0
+  VERSION 1.1.1
   LANGUAGES CXX C)
 add_compile_options("-D ENTITY_VERSION=\"${PROJECT_VERSION}\"")
 execute_process(COMMAND

README.md

@@ -12,6 +12,8 @@ Our [detailed documentation](https://entity-toolkit.github.io/) includes everyth
 
 ## Core developers (alphabetical)
 
+👀 __Yangyang Cai__ {[@StaticObserver](https://github.com/StaticObserver): GRPIC}
+
 💁‍♂️ __Alexander Chernoglazov__ {[@SChernoglazov](https://github.com/SChernoglazov): PIC}
 
 🍵 __Benjamin Crinquand__ {[@bcrinquand](https://github.com/bcrinquand): GRPIC, cubed-sphere}

src/framework/domain/metadomain.cpp

@@ -390,7 +390,7 @@ namespace ntt {
 #if defined(MPI_ENABLED)
     auto dx_mins        = std::vector<real_t>(g_ndomains);
     dx_mins[g_mpi_rank] = dx_min;
-    MPI_Allgather(&dx_mins[g_mpi_rank],
+    MPI_Allgather(&dx_min,
                   1,
                   mpi::get_type<real_t>(),
                   dx_mins.data(),

src/framework/domain/output.cpp

@@ -26,6 +26,10 @@
 #include <Kokkos_ScatterView.hpp>
 #include <Kokkos_StdAlgorithms.hpp>
 
+#if defined(MPI_ENABLED)
+  #include <mpi.h>
+#endif // MPI_ENABLED
+
 #include <algorithm>
 #include <iterator>
 #include <vector>
@@ -458,34 +462,55 @@ namespace ntt {
                       ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) {
           buff_x3 = array_t<real_t*> { "x3", nout };
         }
-        // clang-format off
-        Kokkos::parallel_for(
-          "PrtlToPhys",
-          nout,
-          kernel::PrtlToPhys_kernel<S, M>(prtl_stride,
-                                          buff_x1, buff_x2, buff_x3,
-                                          buff_ux1, buff_ux2, buff_ux3,
-                                          buff_wei,
-                                          species.i1, species.i2, species.i3,
-                                          species.dx1, species.dx2, species.dx3,
-                                          species.ux1, species.ux2, species.ux3,
-                                          species.phi, species.weight,
-                                          local_domain->mesh.metric));
-        // clang-format on
-        g_writer.writeParticleQuantity(buff_wei, prtl.name("W", 0));
-        g_writer.writeParticleQuantity(buff_ux1, prtl.name("U", 1));
-        g_writer.writeParticleQuantity(buff_ux2, prtl.name("U", 2));
-        g_writer.writeParticleQuantity(buff_ux3, prtl.name("U", 3));
+        if (nout > 0) {
+          // clang-format off
+          Kokkos::parallel_for(
+            "PrtlToPhys",
+            nout,
+            kernel::PrtlToPhys_kernel<S, M>(prtl_stride,
+                                            buff_x1, buff_x2, buff_x3,
+                                            buff_ux1, buff_ux2, buff_ux3,
+                                            buff_wei,
+                                            species.i1, species.i2, species.i3,
+                                            species.dx1, species.dx2, species.dx3,
+                                            species.ux1, species.ux2, species.ux3,
+                                            species.phi, species.weight,
+                                            local_domain->mesh.metric));
+          // clang-format on
+        }
+        std::size_t offset   = 0;
+        std::size_t glob_tot = nout;
+#if defined(MPI_ENABLED)
+        auto glob_nout = std::vector<std::size_t>(g_ndomains);
+        MPI_Allgather(&nout,
+                      1,
+                      mpi::get_type<std::size_t>(),
+                      glob_nout.data(),
+                      1,
+                      mpi::get_type<std::size_t>(),
+                      MPI_COMM_WORLD);
+        glob_tot = 0;
+        for (auto r = 0; r < g_mpi_size; ++r) {
+          if (r < g_mpi_rank) {
+            offset += glob_nout[r];
+          }
+          glob_tot += glob_nout[r];
+        }
+#endif // MPI_ENABLED
+        g_writer.writeParticleQuantity(buff_wei, glob_tot, offset, prtl.name("W", 0));
+        g_writer.writeParticleQuantity(buff_ux1, glob_tot, offset, prtl.name("U", 1));
+        g_writer.writeParticleQuantity(buff_ux2, glob_tot, offset, prtl.name("U", 2));
+        g_writer.writeParticleQuantity(buff_ux3, glob_tot, offset, prtl.name("U", 3));
         if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or
                       M::Dim == Dim::_3D) {
-          g_writer.writeParticleQuantity(buff_x1, prtl.name("X", 1));
+          g_writer.writeParticleQuantity(buff_x1, glob_tot, offset, prtl.name("X", 1));
         }
         if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) {
-          g_writer.writeParticleQuantity(buff_x2, prtl.name("X", 2));
+          g_writer.writeParticleQuantity(buff_x2, glob_tot, offset, prtl.name("X", 2));
         }
         if constexpr (M::Dim == Dim::_3D or
                       ((D == Dim::_2D) and (M::CoordType != Coord::Cart))) {
-          g_writer.writeParticleQuantity(buff_x3, prtl.name("X", 3));
+          g_writer.writeParticleQuantity(buff_x3, glob_tot, offset, prtl.name("X", 3));
         }
       }
     } // end shouldWrite("particles", step, time)

src/framework/parameters.cpp

@@ -408,11 +408,16 @@ namespace ntt {
         toml::find_or(raw_data, "output", "fields", "stride", defaults::output::flds_stride));
 
     // particles
-    const auto prtl_out = toml::find_or(raw_data,
-                                        "output",
-                                        "particles",
-                                        "species",
-                                        std::vector<unsigned short> {});
+    auto prtl_out = toml::find_or(raw_data,
+                                  "output",
+                                  "particles",
+                                  "species",
+                                  std::vector<unsigned short> {});
+    if (prtl_out.size() == 0) {
+      for (unsigned short i = 0; i < species.size(); ++i) {
+        prtl_out.push_back(i + 1);
+      }
+    }
     set("output.particles.species", prtl_out);
     set("output.particles.stride",
         toml::find_or(raw_data,

src/output/writer.cpp

@@ -130,17 +130,20 @@ namespace out {
     for (const auto& prtl : m_prtl_writers) {
       for (auto d { 0u }; d < dim; ++d) {
         m_io.DefineVariable<real_t>(prtl.name("X", d + 1),
-                                    {},
-                                    {},
+                                    { adios2::UnknownDim },
+                                    { adios2::UnknownDim },
                                     { adios2::UnknownDim });
       }
       for (auto d { 0u }; d < Dim::_3D; ++d) {
         m_io.DefineVariable<real_t>(prtl.name("U", d + 1),
-                                    {},
-                                    {},
+                                    { adios2::UnknownDim },
+                                    { adios2::UnknownDim },
                                     { adios2::UnknownDim });
       }
-      m_io.DefineVariable<real_t>(prtl.name("W", 0), {}, {}, { adios2::UnknownDim });
+      m_io.DefineVariable<real_t>(prtl.name("W", 0),
+                                  { adios2::UnknownDim },
+                                  { adios2::UnknownDim },
+                                  { adios2::UnknownDim });
     }
   }
 
@@ -216,9 +219,13 @@ namespace out {
   }
 
   void Writer::writeParticleQuantity(const array_t<real_t*>& array,
+                                     std::size_t             glob_total,
+                                     std::size_t             loc_offset,
                                      const std::string&      varname) {
     auto var = m_io.InquireVariable<real_t>(varname);
-    var.SetSelection(adios2::Box<adios2::Dims>({}, { array.extent(0) }));
+    var.SetShape({ glob_total });
+    var.SetSelection(
+      adios2::Box<adios2::Dims>({ loc_offset }, { array.extent(0) }));
     auto array_h = Kokkos::create_mirror_view(array);
     Kokkos::deep_copy(array_h, array);
     m_writer.Put<real_t>(var, array_h);

src/output/writer.h

@@ -114,7 +114,10 @@ namespace out {
                     const ndfield_t<D, N>&,
                     const std::vector<std::size_t>&);
 
-    void writeParticleQuantity(const array_t<real_t*>&, const std::string&);
+    void writeParticleQuantity(const array_t<real_t*>&,
+                               std::size_t,
+                               std::size_t,
+                               const std::string&);
     void writeSpectrum(const array_t<real_t*>&, const std::string&);
     void writeSpectrumBins(const array_t<real_t*>&, const std::string&);