Various fixes (AMReX-Codes#2338)

* AMReX_EB_LeastSquares_2D_K.H: fix warning * Tests/Amr/Advection_AmrLevel: should use pinned memory on host * Tests/AsyncOut/multifab: avoid managed memory * Tests/EB_CNS: fix warning on extra semicolon * Tests/GPU/AnyOf: avoid touching managed memory on host * Tests/LinearSolvers/CellOverset: fix compilation * Tests/LinearSolvers/EBTensor: initialize data on device * Tests/Parser: print max rel. error * Tests/Particles/AsyncIO: fix typo in inputs * Tests/Particles/DenseBins: do not test serial version for dpcpp * Tests/Particles/GhostsAndVirtuals: do not do randomperbox test for dpcpp
kaiserpeng · Sep 18, 2021 · 677bc09 · 677bc09
1 parent f47f884
commit 677bc09
Show file tree

Hide file tree

Showing 14 changed files with 50 additions and 32 deletions.
diff --git a/Src/EB/AMReX_EB_LeastSquares_2D_K.H b/Src/EB/AMReX_EB_LeastSquares_2D_K.H
@@ -693,7 +693,7 @@ Real grad_y_of_phi_on_centroids_extdir(int i,int j,int k,int n,
                     rhs(irow) += Amatrix(a_ind,irow)*phi_val;
 
                     if (flag(ii,jj,k).isSingleValued() &&
-                        is_eb_dirichlet && is_eb_inhomog && Amatrix(a_ind+9,irow)){
+                        is_eb_dirichlet && is_eb_inhomog && Amatrix(a_ind+9,irow) != Real(0.0)){
                       rhs(irow) += Amatrix(a_ind+9,irow)*phieb(ii,jj,k,n);
                     }
                 }

diff --git a/Tests/Amr/Advection_AmrLevel/Exec/UniformVelocity/face_velocity_2d_K.H b/Tests/Amr/Advection_AmrLevel/Exec/UniformVelocity/face_velocity_2d_K.H
@@ -17,8 +17,8 @@ void get_face_velocity(const amrex::Real /*time*/,
 {
     using namespace amrex;
 
-    vx.setVal<RunOn::Device>(AmrLevelAdv::d_prob_parm->adv_vel[0]);
-    vy.setVal<RunOn::Device>(AmrLevelAdv::d_prob_parm->adv_vel[1]);
+    vx.setVal<RunOn::Device>(AmrLevelAdv::h_prob_parm->adv_vel[0]);
+    vy.setVal<RunOn::Device>(AmrLevelAdv::h_prob_parm->adv_vel[1]);
 
     return;
 }

diff --git a/Tests/Amr/Advection_AmrLevel/Exec/UniformVelocity/face_velocity_3d_K.H b/Tests/Amr/Advection_AmrLevel/Exec/UniformVelocity/face_velocity_3d_K.H
@@ -18,9 +18,9 @@ void get_face_velocity(const amrex::Real /*time*/,
 {
     using namespace amrex;
 
-    vx.setVal<RunOn::Device>(AmrLevelAdv::d_prob_parm->adv_vel[0]);
-    vy.setVal<RunOn::Device>(AmrLevelAdv::d_prob_parm->adv_vel[1]);
-    vz.setVal<RunOn::Device>(AmrLevelAdv::d_prob_parm->adv_vel[2]);
+    vx.setVal<RunOn::Device>(AmrLevelAdv::h_prob_parm->adv_vel[0]);
+    vy.setVal<RunOn::Device>(AmrLevelAdv::h_prob_parm->adv_vel[1]);
+    vz.setVal<RunOn::Device>(AmrLevelAdv::h_prob_parm->adv_vel[2]);
 
     return;
 }

diff --git a/Tests/AsyncOut/multifab/main.cpp b/Tests/AsyncOut/multifab/main.cpp
@@ -63,7 +63,8 @@ void main_main ()
     DistributionMapping dm(ba);
 
     Vector<MultiFab> mfs(nwrites);
-    Gpu::ManagedVector< Array4<Real> > arrs(nwrites);
+    Gpu::HostVector< Array4<Real> > h_arrs(nwrites);
+    Gpu::DeviceVector< Array4<Real> > d_arrs(nwrites);
 
     for (int m = 0; m < nwrites; ++m) {
         mfs[m].define(ba, dm, 1, 0);
@@ -72,10 +73,12 @@ void main_main ()
     for (MFIter mfi(mfs[0]); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.validbox();
         for (int m = 0; m < nwrites; ++m) {
-            arrs[m] = mfs[m].array(mfi);
+            h_arrs[m] = mfs[m].array(mfi);
         }
 
-        auto arrs_ptr = arrs.dataPtr();
+        amrex::Gpu::copyAsync(Gpu::hostToDevice, h_arrs.begin(), h_arrs.end(), d_arrs.begin());
+
+        auto arrs_ptr = d_arrs.dataPtr();
 
         amrex::ParallelForRNG(bx,
         [=] AMREX_GPU_DEVICE (int i, int j, int k, RandomEngine const& engine) noexcept

diff --git a/Tests/EB_CNS/Source/CNS.H b/Tests/EB_CNS/Source/CNS.H
@@ -107,7 +107,7 @@ public:
     // Cleanup data descriptors at end of run.
     static void variableCleanUp ();
 
-    static int numGrow() { return NUM_GROW; };
+    static int numGrow() { return NUM_GROW; }
 
     static int numState () { return NUM_STATE; }
 

diff --git a/Tests/GPU/AnyOf/main.cpp b/Tests/GPU/AnyOf/main.cpp
@@ -72,7 +72,8 @@ void main_main ()
     }
 
     // Redo, confirming works for a single item.
-    vec[int(nitem-1)] = 1.0;
+    auto pvec = vec.data();
+    amrex::single_task([=] AMREX_GPU_DEVICE () { pvec[nitem-1] = 1.0; });
 
     {
         BL_PROFILE("Vector AnyOf - Just 1");
@@ -115,10 +116,13 @@ void main_main ()
                            });
 
             // Redo, confirming works for a single value.
-            if (bx.contains(IntVect{ncell/3,ncell/2,ncell-1}))
+            amrex::single_task([=] AMREX_GPU_DEVICE ()
             {
-                arr(ncell/3,ncell/2,ncell-1) = 1.0;
-            }
+                if (bx.contains(IntVect{ncell/3,ncell/2,ncell-1}))
+                {
+                    arr(ncell/3,ncell/2,ncell-1) = 1.0;
+                }
+            });
 
             bool anyof_C = Reduce::AnyOf(bx,
                            [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept

diff --git a/Tests/LinearSolvers/CellOverset/MyTest.cpp b/Tests/LinearSolvers/CellOverset/MyTest.cpp
@@ -34,9 +34,16 @@ MyTest::solve ()
 
     std::unique_ptr<MLABecLaplacian> mlabec;
     if (do_overset) {
-        mlabec = std::make_unique<MLABecLaplacian>({geom}, {grids}, {dmap}, {&oversetmask}, info);
+        mlabec = std::make_unique<MLABecLaplacian>(Vector<Geometry>{geom},
+                                                   Vector<BoxArray>{grids},
+                                                   Vector<DistributionMapping>{dmap},
+                                                   Vector<iMultiFab const*>{&oversetmask},
+                                                   info);
     } else {
-        mlabec = std::make_unique<MLABecLaplacian>({geom}, {grids}, {dmap}, info);
+        mlabec = std::make_unique<MLABecLaplacian>(Vector<Geometry>{geom},
+                                                   Vector<BoxArray>{grids},
+                                                   Vector<DistributionMapping>{dmap},
+                                                   info);
     }
 
     mlabec->setDomainBC(mlmg_lobc, mlmg_hibc);

diff --git a/Tests/LinearSolvers/EBTensor/MyTest.cpp b/Tests/LinearSolvers/EBTensor/MyTest.cpp
@@ -169,7 +169,7 @@ MyTest::initData ()
     eta.define(grids, dmap, 1, 1, MFInfo(), *factory);
 
     const auto& dx = geom.CellSizeArray();
-    const auto& problo = geom.ProbLo();
+    const auto& problo = geom.ProbLoArray();
 
     amrex::Real R;
 
@@ -186,14 +186,11 @@ MyTest::initData ()
 
     for (MFIter mfi(exact); mfi.isValid(); ++mfi) {
         const Box& bx = mfi.fabbox();
-        const auto lo = amrex::lbound(bx);
-        const auto hi = amrex::ubound(bx);
         const Array4<Real> velfab = exact.array(mfi);
         const Array4<Real> rhsfab = rhs.array(mfi);
         const Array4<Real> etafab = eta.array(mfi);
-        for         (int k = lo.z; k <= hi.z; ++k) {
-            for     (int j = lo.y; j <= hi.y; ++j) {
-                for (int i = lo.x; i <= hi.x; ++i) {
+        amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k)
+             {
                     AMREX_D_TERM(Real x = (i+0.5)*dx[0] + problo[0];,
                                  Real y = (j+0.5)*dx[1] + problo[1];,
                                  Real z = (k+0.5)*dx[2] + problo[2];)
@@ -228,9 +225,9 @@ MyTest::initData ()
                         z < -1.0 || z > 1.0)
 #endif
                     {
-                        AMREX_D_TERM(x = std::max(-1.0,std::min(1.0,x));,
-                                     y = std::max(-1.0,std::min(1.0,y));,
-                                     z = std::max(-1.0,std::min(1.0,z));)
+                        AMREX_D_TERM(x = amrex::max(-1.0,amrex::min(1.0,x));,
+                                     y = amrex::max(-1.0,amrex::min(1.0,y));,
+                                     z = amrex::max(-1.0,amrex::min(1.0,z));)
 #if (AMREX_SPACEDIM == 2)
                             init(x,y,R2,u,v,urhs,vrhs,seta);
 #elif (AMREX_SPACEDIM == 3)
@@ -246,9 +243,7 @@ MyTest::initData ()
                                      velfab(i,j,k,1) = v;,
                                      velfab(i,j,k,2) = w;);
                     }
-                }
-            }
-        }
+             });
     }
 
     amrex::EB_set_covered(exact, 0.0);

diff --git a/Tests/LinearSolvers/EBTensor/MyTest_2D_K.H b/Tests/LinearSolvers/EBTensor/MyTest_2D_K.H
@@ -3,6 +3,7 @@
 
 #include <AMReX_FArrayBox.H>
 
+AMREX_GPU_DEVICE
 inline void init (amrex::Real x, amrex::Real y, amrex::Real R2,
                   amrex::Real& u, amrex::Real& v,
                   amrex::Real& urhs, amrex::Real& vrhs,

diff --git a/Tests/LinearSolvers/EBTensor/MyTest_3D_K.H b/Tests/LinearSolvers/EBTensor/MyTest_3D_K.H
@@ -3,6 +3,7 @@
 
 #include <AMReX_FArrayBox.H>
 
+AMREX_GPU_DEVICE
 inline void init (amrex::Real x, amrex::Real y, amrex::Real z, amrex::Real R2,
                   amrex::Real& u, amrex::Real& v, amrex::Real& w,
                   amrex::Real& urhs, amrex::Real& vrhs, amrex::Real& wrhs,

diff --git a/Tests/Parser/main.cpp b/Tests/Parser/main.cpp
@@ -28,20 +28,23 @@ int test1 (std::string const& f,
     GpuArray<Real,1> dx{(hi[0]-lo[0]) / (N-1)};
 
     int nfail = 0;
+    Real max_relerror = 0.;
     for (int i = 0; i < N; ++i) {
         Real x = lo[0] + i*dx[0];
         double result = exe(x);
         double benchmark = fb(x);
         double abserror = std::abs(result-benchmark);
         double relerror = abserror / (1.e-50 + std::max(std::abs(result),std::abs(benchmark)));
         if (abserror > abstol && relerror > reltol) {
-            amrex::Print() << "    f(" << x << ") = " << result << ", "
-                           << benchmark << "\n";
+            amrex::Print() << "\n    f(" << x << ") = " << result << ", "
+                           << benchmark;
+            max_relerror = std::max(max_relerror, relerror);
             ++nfail;
         }
     }
     if (nfail > 0) {
-        amrex::Print() << "    failed " << nfail << " times\n";
+        amrex::Print() << "\n    failed " << nfail << " times.  Max rel. error: "
+                       << max_relerror << "\n";
         return 1;
     } else {
         amrex::Print() << "    pass\n";

diff --git a/Tests/Particles/AsyncIO/inputs b/Tests/Particles/AsyncIO/inputs
@@ -26,4 +26,4 @@ verbose = true   # set to true to get more verbosity
 # Number of levels
 nlevs = 2
 
-amrex.async_io=1
+amrex.async_out=1
diff --git a/Tests/Particles/DenseBins/main.cpp b/Tests/Particles/DenseBins/main.cpp
@@ -109,7 +109,9 @@ void testDenseBins ()
     amrex::Vector<int> items(nitems);
     initData(nbins, items);
 
+#ifndef AMREX_USE_DPCPP
     testSerial(nbins, items);
+#endif
 #ifdef AMREX_USE_OMP
     testOpenMP(nbins, items);
 #endif

diff --git a/Tests/Particles/GhostsAndVirtuals/main.cpp b/Tests/Particles/GhostsAndVirtuals/main.cpp
@@ -763,8 +763,10 @@ int main(int argc, char* argv[])
   amrex::Print()<<"Original test"<<std::endl;
   test_ghosts_and_virtuals(parms);
 
+#ifndef AMREX_USE_DPCPP
   amrex::Print()<<"RandomPerBox test"<<std::endl;
   test_ghosts_and_virtuals_randomperbox(parms);
+#endif
 
   amrex::Print()<<"OnePerCell test"<<std::endl;
   test_ghosts_and_virtuals_onepercell(parms);