Add more build options for linear solvers (AMReX-Codes#4181)

Add two new CMake build options, AMReX_LINEAR_SOLVERS_INCFLO and AMReX_LINEAR_SOLVERS_EM. Both options depend on AMReX_LINEAR_SOLVERS, and are enabled by default when AMReX_LINEAR_SOLVERS is enabled. By default, the cell-centered MLABecLaplacian and MLPoisson solvers are always included. However, these new build options provide finer control over which solvers to build. * AMReX_LINEAR_SOLVERS_INCFLO can be used to control the inclusion of solvers commonly used by incompressible flow codes, such as nodal projection and Navier-Stokes tensor solvers. * AMReX_LINEAR_SOLVERS_EM can be used to control the inclusion of solvers commonly used by electromagnetic applications, such as the curl curl and tensor Laplacian solvers used by WarpX. These options are also available in GNU Make as USE_LINEAR_SOLVERS_INCFLO and USE_LINEAR_SOLVER_EM.
clarkse · Oct 7, 2024 · e122280 · e122280
1 parent 6239d25
commit e122280
Show file tree

Hide file tree

Showing 26 changed files with 1,565 additions and 1,451 deletions.
diff --git a/.github/workflows/apps.yml b/.github/workflows/apps.yml
@@ -52,7 +52,9 @@ jobs:
         export AMREX_HOME=${PWD}
         export MICROPHYSICS_HOME=${PWD}/Microphysics
         cd Castro/Exec/hydro_tests/Sedov/
-        make -j4 CCACHE=ccache USE_MPI=FALSE
+        make -j4 CCACHE=ccache USE_MPI=FALSE \
+            USE_LINEAR_SOLVERS_INCFLO=FALSE \
+            USE_LINEAR_SOLVERS_EM=FALSE
 
         ccache -s
         du -hs ~/.cache/ccache
@@ -92,7 +94,8 @@ jobs:
             -DWarpX_QED=OFF                       \
             -DWarpX_OPENPMD=OFF                   \
             -DCMAKE_VERBOSE_MAKEFILE=ON           \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache  \
+            -DAMReX_LINEAR_SOLVER_INCFLO=OFF
         cmake --build WarpX/build -j 4
 
         ccache -s

diff --git a/Docs/sphinx_documentation/source/BuildingAMReX.rst b/Docs/sphinx_documentation/source/BuildingAMReX.rst
@@ -463,6 +463,12 @@ The list of available options is reported in the :ref:`table <tab:cmakevar>` bel
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_LINEAR_SOLVERS         |  Build AMReX linear solvers                     | YES                     | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_LINEAR_SOLVERS_INCFLO  |  Build AMReX linear solvers for incompressible  | YES                     | YES, NO               |
+   |                              |  flow                                           |                         |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_LINEAR_SOLVERS_EM      |  Build AMReX linear solvers for electromagnetic | YES                     | YES, NO               |
+   |                              |  solvers                                        |                         |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_AMRDATA                |  Build data services                            | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_AMRLEVEL               |  Build AmrLevel class                           | YES                     | YES, NO               |
@@ -681,6 +687,10 @@ A list of AMReX component names and related configure options are shown in the t
    +------------------------------+-----------------+
    | AMReX_LINEAR_SOLVERS         | LSOLVERS        |
    +------------------------------+-----------------+
+   | AMReX_LINEAR_SOLVERS_INCFLO  | LSOLVERS_INCFLO |
+   +------------------------------+-----------------+
+   | AMReX_LINEAR_SOLVERS_EM      | LSOLVERS_EM     |
+   +------------------------------+-----------------+
    | AMReX_AMRDATA                | AMRDATA         |
    +------------------------------+-----------------+
    | AMReX_AMRLEVEL               | AMRLEVEL        |

diff --git a/Src/Extern/HYPRE/AMReX_Habec_2D_K.H b/Src/Extern/HYPRE/AMReX_Habec_2D_K.H
@@ -6,7 +6,7 @@
 #include <AMReX_EBMultiFabUtil.H>
 #include <AMReX_MultiCutFab.H>
 #include <AMReX_EBFabFactory.H>
-#include <AMReX_MLEBABecLap_K.H>
+#include <AMReX_MLLinOp_K.H>
 #endif
 
 namespace amrex {

diff --git a/Src/Extern/HYPRE/AMReX_Habec_3D_K.H b/Src/Extern/HYPRE/AMReX_Habec_3D_K.H
@@ -6,7 +6,7 @@
 #include <AMReX_EBMultiFabUtil.H>
 #include <AMReX_MultiCutFab.H>
 #include <AMReX_EBFabFactory.H>
-#include <AMReX_MLEBABecLap_K.H>
+#include <AMReX_MLLinOp_K.H>
 #endif
 
 namespace amrex {

diff --git a/Src/LinearSolvers/CMakeLists.txt b/Src/LinearSolvers/CMakeLists.txt
@@ -16,6 +16,8 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        MLMG/AMReX_MLLinOp_K.H
        MLMG/AMReX_MLCellLinOp.H
        MLMG/AMReX_MLNodeLinOp.H
+       MLMG/AMReX_MLNodeLinOp_K.H
+       MLMG/AMReX_MLNodeLinOp_${D}D_K.H
        MLMG/AMReX_MLNodeLinOp.cpp
        MLMG/AMReX_MLCellABecLap.H
        MLMG/AMReX_MLCellABecLap_K.H
@@ -31,30 +33,6 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        MLMG/AMReX_MLPoisson.H
        MLMG/AMReX_MLPoisson_K.H
        MLMG/AMReX_MLPoisson_${D}D_K.H
-       MLMG/AMReX_MLNodeLaplacian.H
-       MLMG/AMReX_MLNodeLaplacian.cpp
-       MLMG/AMReX_MLNodeLaplacian_sync.cpp
-       MLMG/AMReX_MLNodeLaplacian_sten.cpp
-       MLMG/AMReX_MLNodeLaplacian_misc.cpp
-       MLMG/AMReX_MLNodeLap_K.H
-       MLMG/AMReX_MLNodeLap_${D}D_K.H
-       MLMG/AMReX_MLNodeTensorLaplacian.H
-       MLMG/AMReX_MLNodeTensorLaplacian.cpp
-       MLMG/AMReX_MLNodeTensorLap_K.H
-       MLMG/AMReX_MLNodeTensorLap_${D}D_K.H
-       MLMG/AMReX_MLTensorOp.H
-       MLMG/AMReX_MLTensorOp.cpp
-       MLMG/AMReX_MLTensorOp_grad.cpp
-       MLMG/AMReX_MLTensor_K.H
-       MLMG/AMReX_MLTensor_${D}D_K.H
-       MLMG/AMReX_MLEBNodeFDLaplacian.H
-       MLMG/AMReX_MLEBNodeFDLaplacian.cpp
-       MLMG/AMReX_MLEBNodeFDLap_K.H
-       MLMG/AMReX_MLEBNodeFDLap_${D}D_K.H
-       MLMG/AMReX_MLNodeABecLaplacian.H
-       MLMG/AMReX_MLNodeABecLaplacian.cpp
-       MLMG/AMReX_MLNodeABecLap_K.H
-       MLMG/AMReX_MLNodeABecLap_${D}D_K.H
        AMReX_GMRES.H
        AMReX_GMRES_MLMG.H
        )
@@ -68,30 +46,72 @@ foreach(D IN LISTS AMReX_SPACEDIM)
           )
     endif ()
 
-    if (NOT D EQUAL 1)
+    if (AMReX_LINEAR_SOLVERS_EM)
+       if (NOT D EQUAL 1 AND AMReX_LINEAR_SOLVERS_EM)
+          target_sources(amrex_${D}d
+             PRIVATE
+             MLMG/AMReX_MLCurlCurl.H
+             MLMG/AMReX_MLCurlCurl.cpp
+             MLMG/AMReX_MLCurlCurl_K.H
+             )
+       endif ()
+
        target_sources(amrex_${D}d
           PRIVATE
-          MLMG/AMReX_MLCurlCurl.H
-          MLMG/AMReX_MLCurlCurl.cpp
-          MLMG/AMReX_MLCurlCurl_K.H
-          )
+          MLMG/AMReX_MLEBNodeFDLaplacian.H
+          MLMG/AMReX_MLEBNodeFDLaplacian.cpp
+          MLMG/AMReX_MLEBNodeFDLap_K.H
+          MLMG/AMReX_MLEBNodeFDLap_${D}D_K.H
+          MLMG/AMReX_MLNodeTensorLaplacian.H
+          MLMG/AMReX_MLNodeTensorLaplacian.cpp
+          MLMG/AMReX_MLNodeTensorLap_K.H
+          MLMG/AMReX_MLNodeTensorLap_${D}D_K.H
+       )
+    endif ()
+
+    if (AMReX_LINEAR_SOLVERS_INCFLO)
+       target_sources(amrex_${D}d
+          PRIVATE
+          MLMG/AMReX_MLNodeABecLaplacian.H
+          MLMG/AMReX_MLNodeABecLaplacian.cpp
+          MLMG/AMReX_MLNodeABecLap_K.H
+          MLMG/AMReX_MLNodeABecLap_${D}D_K.H
+          MLMG/AMReX_MLNodeLaplacian.H
+          MLMG/AMReX_MLNodeLaplacian.cpp
+          MLMG/AMReX_MLNodeLaplacian_sync.cpp
+          MLMG/AMReX_MLNodeLaplacian_sten.cpp
+          MLMG/AMReX_MLNodeLaplacian_misc.cpp
+          MLMG/AMReX_MLNodeLap_K.H
+          MLMG/AMReX_MLNodeLap_${D}D_K.H
+          MLMG/AMReX_MLTensorOp.H
+          MLMG/AMReX_MLTensorOp.cpp
+          MLMG/AMReX_MLTensorOp_grad.cpp
+          MLMG/AMReX_MLTensor_K.H
+          MLMG/AMReX_MLTensor_${D}D_K.H
+       )
     endif ()
 
     if (AMReX_EB AND NOT D EQUAL 1)
        target_sources(amrex_${D}d
           PRIVATE
-          MLMG/AMReX_MLNodeLaplacian_eb.cpp
           MLMG/AMReX_MLEBABecLap.H
           MLMG/AMReX_MLEBABecLap.cpp
           MLMG/AMReX_MLEBABecLap_F.cpp
           MLMG/AMReX_MLEBABecLap_K.H
           MLMG/AMReX_MLEBABecLap_${D}D_K.H
-          MLMG/AMReX_MLEBTensorOp.H
-          MLMG/AMReX_MLEBTensorOp.cpp
-          MLMG/AMReX_MLEBTensorOp_bc.cpp
-          MLMG/AMReX_MLEBTensor_K.H
-          MLMG/AMReX_MLEBTensor_${D}D_K.H
           )
+
+       if (AMReX_LINEAR_SOLVERS_INCFLO)
+          target_sources(amrex_${D}d
+             PRIVATE
+             MLMG/AMReX_MLNodeLaplacian_eb.cpp
+             MLMG/AMReX_MLEBTensorOp.H
+             MLMG/AMReX_MLEBTensorOp.cpp
+             MLMG/AMReX_MLEBTensorOp_bc.cpp
+             MLMG/AMReX_MLEBTensor_K.H
+             MLMG/AMReX_MLEBTensor_${D}D_K.H
+             )
+       endif ()
     endif ()
 
     if (AMReX_FORTRAN)
@@ -102,7 +122,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
           )
     endif ()
 
-    if (AMReX_HYPRE)
+    if (AMReX_HYPRE AND AMReX_LINEAR_SOLVERS_INCFLO)
        target_sources(amrex_${D}d
           PRIVATE
           MLMG/AMReX_MLNodeLaplacian_hypre.cpp

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_K.H
@@ -6,13 +6,6 @@
 
 #include <AMReX_EBCellFlag.H>
 
-namespace amrex {
-    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
-    Real get_dx_eb (Real kappa) noexcept {
-        return amrex::max(Real(0.3),(kappa*kappa-Real(0.25))/(Real(2.0)*kappa));
-    }
-}
-
 #if (AMREX_SPACEDIM == 2)
 #include <AMReX_MLEBABecLap_2D_K.H>
 #else

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.cpp
@@ -1,6 +1,6 @@
 #include <AMReX_MLEBNodeFDLaplacian.H>
 #include <AMReX_MLEBNodeFDLap_K.H>
-#include <AMReX_MLNodeLap_K.H>
+#include <AMReX_MLNodeLinOp_K.H>
 #include <AMReX_MLNodeTensorLap_K.H>
 #include <AMReX_MultiFabUtil.H>
 

diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp_K.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp_K.H
@@ -1053,6 +1053,15 @@ void mllinop_apply_innu_zhi (int i, int j, int k,
     }
 }
 
+#ifdef AMREX_USE_EB
+
+AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+Real get_dx_eb (Real kappa) noexcept {
+    return amrex::max(Real(0.3),(kappa*kappa-Real(0.25))/(Real(2.0)*kappa));
+}
+
+#endif
+
 }
 
 #endif