AMReX/PICSAR: Update to New Particle APIs (#3848)

* AMReX: Weekly Update * Particles: Update WarpX APIs
ECP-WarpX · Apr 18, 2023 · f71597f · f71597f
1 parent 296c381
commit f71597f
Show file tree

Hide file tree

Showing 13 changed files with 30 additions and 17 deletions.
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -110,7 +110,7 @@ jobs:
         which nvcc || echo "nvcc not in PATH!"
 
         git clone https://github.com/AMReX-Codes/amrex.git ../amrex
-        cd ../amrex && git checkout --detach 27ef4ea56945af7f250a6cec9634ebdcaf863b0b && cd -
+        cd ../amrex && git checkout --detach c1e8e73b63dc8b6732c6cdfe51914f7ae144575b && cd -
         make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 2
 
   build_nvhpc21-11-nvcc:

diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini
@@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more
 
 [AMReX]
 dir = /home/regtester/git/amrex/
-branch = 27ef4ea56945af7f250a6cec9634ebdcaf863b0b
+branch = c1e8e73b63dc8b6732c6cdfe51914f7ae144575b
 
 [source]
 dir = /home/regtester/git/WarpX

diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini
@@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det
 
 [AMReX]
 dir = /home/regtester/AMReX_RegTesting/amrex/
-branch = 27ef4ea56945af7f250a6cec9634ebdcaf863b0b
+branch = c1e8e73b63dc8b6732c6cdfe51914f7ae144575b
 
 [source]
 dir = /home/regtester/AMReX_RegTesting/warpx

diff --git a/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp b/Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.cpp
@@ -91,7 +91,8 @@ FieldProbeParticleContainer::AddNParticles (int lev,
      * (particle_tile).
      */
 
-    using PinnedTile = ParticleTile<NStructReal, NStructInt, NArrayReal, NArrayInt,
+    using PinnedTile = ParticleTile<amrex::Particle<NStructReal, NStructInt>,
+                                    NArrayReal, NArrayInt,
                                     amrex::PinnedArenaAllocator>;
     PinnedTile pinned_tile;
     pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps());

diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H
@@ -50,8 +50,8 @@ public:
      * \brief Method to initialize runtime attributes. Does nothing for LaserParticleContainer.
      */
     virtual void DefaultInitializeRuntimeAttributes (
-                        amrex::ParticleTile<NStructReal, NStructInt, NArrayReal,
-                                            NArrayInt,amrex::PinnedArenaAllocator>& /*pinned_tile*/,
+                        amrex::ParticleTile<amrex::Particle<NStructReal, NStructInt>,
+                                            NArrayReal, NArrayInt, amrex::PinnedArenaAllocator>& /*pinned_tile*/,
                         const int /*n_external_attr_real*/,
                         const int /*n_external_attr_int*/,
                         const amrex::RandomEngine& /*engine*/) override final {}

diff --git a/Source/Particles/ParticleIO.H b/Source/Particles/ParticleIO.H
@@ -10,9 +10,11 @@
 #include "Particles/WarpXParticleContainer.H"
 
 #include <AMReX_AmrParticles.H>
+#include <AMReX_ParIter.H>
 #include <AMReX_Gpu.H>
 #include <AMReX_REAL.H>
 
+
 enum struct ConvertDirection{WarpX_to_SI, SI_to_WarpX};
 
 /** Convert particle momentum to/from SI
@@ -45,12 +47,14 @@ particlesConvertUnits (ConvertDirection convert_direction, T_ParticleContainer*
         factor = 1._rt/mass;
     }
 
+    using PinnedParIter = typename T_ParticleContainer::ParIterType;
+
     const int nLevels = pc->finestLevel();
     for (int lev=0; lev<=nLevels; lev++){
 #ifdef AMREX_USE_OMP
 #pragma omp parallel if (Gpu::notInLaunchRegion())
 #endif
-        for (ParIter pti(*pc, lev); pti.isValid(); ++pti)
+        for (PinnedParIter pti(*pc, lev); pti.isValid(); ++pti)
         {
             // - momenta are stored as a struct of array, in `attribs`
             // The GetStructOfArrays is called directly since the convenience routine GetAttribs

diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H
@@ -234,8 +234,9 @@ public:
      * @param[in] engine the random engine, used in initialization of QED optical depths
      */
     virtual void DefaultInitializeRuntimeAttributes (
-                    amrex::ParticleTile<NStructReal, NStructInt, NArrayReal,
-                                        NArrayInt,amrex::PinnedArenaAllocator>& pinned_tile,
+                    amrex::ParticleTile<amrex::Particle<NStructReal, NStructInt>,
+                                        NArrayReal, NArrayInt,
+                                        amrex::PinnedArenaAllocator>& pinned_tile,
                     const int n_external_attr_real,
                     const int n_external_attr_int,
                     const amrex::RandomEngine& engine) override final;

diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp
@@ -673,8 +673,9 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot,
 
 void
 PhysicalParticleContainer::DefaultInitializeRuntimeAttributes (
-                    amrex::ParticleTile<NStructReal, NStructInt, NArrayReal,
-                                        NArrayInt,amrex::PinnedArenaAllocator>& pinned_tile,
+                    amrex::ParticleTile<amrex::Particle<NStructReal, NStructInt>,
+                                        NArrayReal, NArrayInt,
+                                        amrex::PinnedArenaAllocator>& pinned_tile,
                     const int n_external_attr_real,
                     const int n_external_attr_int,
                     const amrex::RandomEngine& engine)

diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H
@@ -25,6 +25,7 @@
 
 #include <AMReX_Array.H>
 #include <AMReX_FArrayBox.H>
+#include <AMReX_GpuAllocators.H>
 #include <AMReX_GpuContainers.H>
 #include <AMReX_INT.H>
 #include <AMReX_ParIter.H>
@@ -144,8 +145,9 @@ public:
      * class.
      */
     virtual void DefaultInitializeRuntimeAttributes (
-                        amrex::ParticleTile<NStructReal, NStructInt, NArrayReal,
-                                            NArrayInt,amrex::PinnedArenaAllocator>& pinned_tile,
+                        amrex::ParticleTile<amrex::Particle<NStructReal, NStructInt>,
+                                            NArrayReal, NArrayInt,
+                                            amrex::PinnedArenaAllocator>& pinned_tile,
                         const int n_external_attr_real,
                         const int n_external_attr_int,
                         const amrex::RandomEngine& engine) = 0;

diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp
@@ -180,8 +180,9 @@ WarpXParticleContainer::AddNParticles (int /*lev*/,
     // Redistribute() will move them to proper places.
     auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0);
 
-    using PinnedTile = amrex::ParticleTile<NStructReal, NStructInt, NArrayReal, NArrayInt,
-                                    amrex::PinnedArenaAllocator>;
+    using PinnedTile = amrex::ParticleTile<Particle<NStructReal, NStructInt>,
+                                           NArrayReal, NArrayInt,
+                                           amrex::PinnedArenaAllocator>;
     PinnedTile pinned_tile;
     pinned_tile.define(NumRuntimeRealComps(), NumRuntimeIntComps());
 

diff --git a/Source/Particles/WarpXParticleContainer_fwd.H b/Source/Particles/WarpXParticleContainer_fwd.H
@@ -8,6 +8,9 @@
 #ifndef WARPX_WarpXParticleContainer_fwd_H_
 #define WARPX_WarpXParticleContainer_fwd_H_
 
+#include <AMReX_GpuAllocators.H>
+
+
 enum struct ParticleBC;
 enum struct ConvertDirection;
 

diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake
@@ -240,7 +240,7 @@ set(WarpX_amrex_src ""
 set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git"
     CACHE STRING
     "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)")
-set(WarpX_amrex_branch "27ef4ea56945af7f250a6cec9634ebdcaf863b0b"
+set(WarpX_amrex_branch "c1e8e73b63dc8b6732c6cdfe51914f7ae144575b"
     CACHE STRING
     "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)")
 

diff --git a/run_test.sh b/run_test.sh
@@ -71,7 +71,7 @@ python3 -m pip install --upgrade -r warpx/Regression/requirements.txt
 
 # Clone AMReX and warpx-data
 git clone https://github.com/AMReX-Codes/amrex.git
-cd amrex && git checkout --detach 27ef4ea56945af7f250a6cec9634ebdcaf863b0b && cd -
+cd amrex && git checkout --detach c1e8e73b63dc8b6732c6cdfe51914f7ae144575b && cd -
 # warpx-data contains various required data sets
 git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git