From 486cd438aea970962a02c799c910eb8ddcd7dd71 Mon Sep 17 00:00:00 2001 From: "CCSE@LBNL" Date: Sun, 14 Mar 2021 08:38:31 -0700 Subject: [PATCH] Remove tabs and trailing white spaces (#1860) The changes are purely white spaces only. `git diff -w development..HEAD` shows nothing. It was done with ``` find . -type d \( -name .git \ -o -path ./paper \ \) -prune -o \ -type f \( -name "*.H" -o -name "*.h" -o -name "*.hh" -o -name "*.hpp" \ -o -name "*.c" -o -name "*.cc" -o -name "*.cpp" -o -name "*.cxx" \ -o -name "*.f" -o -name "*.F" -o -name "*.f90" -o -name "*.F90" \ -o -name "*.py" \ -o -name "*.md" -o -name "*.rst" \ -o -name "*.sh" \ -o -name "*.tex" \ -o -name "*.txt" \ \) \ -exec grep -Iq . {} \; \ -exec sed -i 's/[[:blank:]]\+$//g' {} + \ -exec sed -i 's/\t/\ \ \ \ \ \ \ \ /g' {} + ``` We will add a CI check as a follow-up. --- CONTRIBUTING.md | 8 +- Docs/Migration/Migration.md | 10 +- .../source/AMReX_Profiling_Tools_Chapter.rst | 8 +- Docs/sphinx_documentation/source/AmrCore.rst | 46 +- Docs/sphinx_documentation/source/AmrLevel.rst | 6 +- Docs/sphinx_documentation/source/Basics.rst | 8 +- Docs/sphinx_documentation/source/DualGrid.rst | 4 +- Docs/sphinx_documentation/source/EB.rst | 28 +- .../source/External_Profiling_Tools.rst | 30 +- Docs/sphinx_documentation/source/Fortran.rst | 12 +- Docs/sphinx_documentation/source/GPU.rst | 14 +- .../source/GPU_Chapter.rst | 8 +- .../source/GettingStarted.rst | 2 +- .../source/GridCreation.rst | 46 +- Docs/sphinx_documentation/source/IO.rst | 10 +- .../source/InputsLoadBalancing.rst | 6 +- .../source/Inputs_Chapter.rst | 6 +- .../source/Introduction.rst | 2 +- .../source/LinearSolvers.rst | 164 +- .../source/LinearSolvers_Chapter.rst | 16 +- .../source/LoadBalancing.rst | 14 +- .../source/ManagingGridHierarchy_Chapter.rst | 18 +- Docs/sphinx_documentation/source/Particle.rst | 20 +- Docs/sphinx_documentation/source/SWFFT.rst | 6 +- Docs/sphinx_documentation/source/Testing.rst | 20 +- .../source/Visualization.rst | 6 +- Docs/sphinx_documentation/source/index.rst | 2 +- Docs/sphinx_tutorials/source/AMR_Tutorial.rst | 12 +- .../source/Basic_Tutorial.rst | 18 +- .../source/Blueprint_Tutorial.rst | 2 +- Docs/sphinx_tutorials/source/GPU_Tutorial.rst | 14 +- .../source/LinearSolvers_Tutorial.rst | 8 +- Docs/sphinx_tutorials/source/MUI_Tutorial.rst | 6 +- .../source/Particles_Tutorial.rst | 30 +- Docs/sphinx_tutorials/source/index.rst | 8 +- README.md | 2 +- Src/Amr/AMReX_Amr.H | 14 +- Src/Amr/AMReX_Amr.cpp | 632 +++--- Src/Amr/AMReX_AmrLevel.H | 38 +- Src/Amr/AMReX_AmrLevel.cpp | 422 ++-- Src/Amr/AMReX_AuxBoundaryData.H | 4 +- Src/Amr/AMReX_AuxBoundaryData.cpp | 18 +- Src/Amr/AMReX_Derive.H | 20 +- Src/Amr/AMReX_Derive.cpp | 16 +- Src/Amr/AMReX_StateData.H | 60 +- Src/Amr/AMReX_StateData.cpp | 412 ++-- Src/Amr/AMReX_StateDescriptor.H | 38 +- Src/Amr/AMReX_StateDescriptor.cpp | 64 +- Src/Amr/AMReX_extrapolater_3D_K.H | 312 +-- Src/AmrCore/AMReX_AmrCore.cpp | 126 +- Src/AmrCore/AMReX_AmrMesh.cpp | 180 +- Src/AmrCore/AMReX_AmrParGDB.H | 26 +- Src/AmrCore/AMReX_AmrParticles.H | 4 +- Src/AmrCore/AMReX_Cluster.cpp | 16 +- Src/AmrCore/AMReX_ErrorList.H | 2 +- Src/AmrCore/AMReX_ErrorList.cpp | 24 +- Src/AmrCore/AMReX_FillPatchUtil.H | 4 +- Src/AmrCore/AMReX_FillPatchUtil_2d.F90 | 16 +- Src/AmrCore/AMReX_FillPatchUtil_3d.F90 | 32 +- Src/AmrCore/AMReX_FillPatchUtil_F.H | 2 +- Src/AmrCore/AMReX_FillPatchUtil_I.H | 30 +- Src/AmrCore/AMReX_FluxRegister.cpp | 162 +- Src/AmrCore/AMReX_INTERP_1D.F90 | 40 +- Src/AmrCore/AMReX_INTERP_2D.F90 | 108 +- Src/AmrCore/AMReX_INTERP_F.H | 82 +- Src/AmrCore/AMReX_Interp_1D_C.H | 2 +- Src/AmrCore/AMReX_Interp_2D_C.H | 2 +- Src/AmrCore/AMReX_Interp_3D_C.H | 4 +- Src/AmrCore/AMReX_Interpolater.H | 4 +- Src/AmrCore/AMReX_Interpolater.cpp | 40 +- Src/AmrCore/AMReX_TagBox.cpp | 32 +- Src/Base/AMReX.H | 4 +- Src/Base/AMReX.cpp | 70 +- Src/Base/AMReX_BCRec.cpp | 2 +- Src/Base/AMReX_BLBackTrace.cpp | 126 +- Src/Base/AMReX_BLProfiler.H | 18 +- Src/Base/AMReX_BLProfiler.cpp | 142 +- Src/Base/AMReX_BLutil_F.F90 | 2 +- Src/Base/AMReX_BaseFab.H | 46 +- Src/Base/AMReX_BaseFab.cpp | 2 +- Src/Base/AMReX_BaseUmap_f.H | 4 +- Src/Base/AMReX_BlockMutex.H | 6 +- Src/Base/AMReX_BlockMutex.cpp | 2 +- Src/Base/AMReX_Box.H | 28 +- Src/Base/AMReX_Box.cpp | 46 +- Src/Base/AMReX_BoxArray.H | 6 +- Src/Base/AMReX_BoxArray.cpp | 258 +-- Src/Base/AMReX_BoxDomain.cpp | 6 +- Src/Base/AMReX_BoxList.H | 4 +- Src/Base/AMReX_BoxList.cpp | 48 +- Src/Base/AMReX_COORDSYS_1D_C.H | 8 +- Src/Base/AMReX_COORDSYS_2D_C.H | 2 +- Src/Base/AMReX_CoordSys.cpp | 26 +- Src/Base/AMReX_CudaGraph.H | 2 +- Src/Base/AMReX_CuptiTrace.H | 2 +- Src/Base/AMReX_CuptiTrace.cpp | 26 +- Src/Base/AMReX_DistributionMapping.H | 28 +- Src/Base/AMReX_DistributionMapping.cpp | 276 +-- Src/Base/AMReX_Extension.H | 12 +- Src/Base/AMReX_FACopyDescriptor.H | 322 +-- Src/Base/AMReX_FArrayBox.H | 12 +- Src/Base/AMReX_FArrayBox.cpp | 38 +- Src/Base/AMReX_FBI.H | 46 +- Src/Base/AMReX_FILCC_1D.F90 | 4 +- Src/Base/AMReX_FILCC_2D.F90 | 4 +- Src/Base/AMReX_FILCC_3D.F90 | 4 +- Src/Base/AMReX_FPC.cpp | 2 +- Src/Base/AMReX_FabArray.H | 204 +- Src/Base/AMReX_FabArrayBase.H | 92 +- Src/Base/AMReX_FabArrayBase.cpp | 1052 +++++----- Src/Base/AMReX_FabArrayCommI.H | 100 +- Src/Base/AMReX_FabArrayUtility.H | 14 +- Src/Base/AMReX_FabConv.cpp | 12 +- Src/Base/AMReX_FileSystem.cpp | 2 +- Src/Base/AMReX_Geometry.H | 16 +- Src/Base/AMReX_Geometry.cpp | 18 +- Src/Base/AMReX_GpuAllocators.H | 20 +- Src/Base/AMReX_GpuComplex.H | 96 +- Src/Base/AMReX_GpuContainers.H | 36 +- Src/Base/AMReX_GpuDevice.cpp | 18 +- Src/Base/AMReX_GpuLaunch.H | 10 +- Src/Base/AMReX_GpuMemory.H | 8 +- Src/Base/AMReX_GpuUtility.H | 2 +- Src/Base/AMReX_IArrayBox.H | 4 +- Src/Base/AMReX_IArrayBox.cpp | 4 +- Src/Base/AMReX_INT.H | 6 +- Src/Base/AMReX_IntConv.H | 4 +- Src/Base/AMReX_IntConv.cpp | 4 +- Src/Base/AMReX_IntVect.H | 22 +- Src/Base/AMReX_Lazy.H | 2 +- Src/Base/AMReX_Lazy.cpp | 24 +- Src/Base/AMReX_MFCopyDescriptor.cpp | 50 +- Src/Base/AMReX_MFIter.H | 12 +- Src/Base/AMReX_MFIter.cpp | 268 +-- Src/Base/AMReX_Machine.cpp | 4 +- Src/Base/AMReX_MemPool.cpp | 40 +- Src/Base/AMReX_MemProfiler.H | 4 +- Src/Base/AMReX_MemProfiler.cpp | 410 ++-- Src/Base/AMReX_MultiFab.H | 234 +-- Src/Base/AMReX_MultiFab.cpp | 84 +- Src/Base/AMReX_MultiFabUtil.cpp | 60 +- Src/Base/AMReX_MultiFabUtil_1D_C.H | 6 +- Src/Base/AMReX_NFiles.H | 6 +- Src/Base/AMReX_NFiles.cpp | 28 +- Src/Base/AMReX_NonLocalBC.H | 84 +- Src/Base/AMReX_NonLocalBC.cpp | 4 +- Src/Base/AMReX_NonLocalBCImpl.H | 2 +- Src/Base/AMReX_PODVector.H | 208 +- Src/Base/AMReX_ParallelContext.cpp | 2 +- Src/Base/AMReX_ParallelDescriptor.H | 276 +-- Src/Base/AMReX_ParallelDescriptor.cpp | 122 +- Src/Base/AMReX_ParallelDescriptor_F.F90 | 2 +- Src/Base/AMReX_ParmParse.H | 6 +- Src/Base/AMReX_ParmParse.cpp | 504 ++--- Src/Base/AMReX_Partition.H | 92 +- Src/Base/AMReX_Periodicity.H | 12 +- Src/Base/AMReX_Periodicity.cpp | 22 +- Src/Base/AMReX_PlotFileUtil.H | 86 +- Src/Base/AMReX_PlotFileUtil.cpp | 162 +- Src/Base/AMReX_Print.H | 140 +- Src/Base/AMReX_REAL.H | 2 +- Src/Base/AMReX_RealBox.cpp | 2 +- Src/Base/AMReX_RealVect.H | 14 +- Src/Base/AMReX_Reduce.H | 2 +- Src/Base/AMReX_SPACE.H | 2 +- Src/Base/AMReX_Scan.H | 8 +- Src/Base/AMReX_Slopes_K.H | 10 +- Src/Base/AMReX_ThirdPartyProfiling.H | 2 +- Src/Base/AMReX_TinyProfiler.H | 4 +- Src/Base/AMReX_TinyProfiler.cpp | 146 +- Src/Base/AMReX_TypeTraits.H | 6 +- Src/Base/AMReX_Utility.H | 54 +- Src/Base/AMReX_Utility.cpp | 48 +- Src/Base/AMReX_Vector.H | 18 +- Src/Base/AMReX_VectorIO.H | 8 +- Src/Base/AMReX_VectorIO.cpp | 8 +- Src/Base/AMReX_VisMF.H | 4 +- Src/Base/AMReX_VisMF.cpp | 492 ++--- Src/Base/AMReX_bc_types_mod.F90 | 2 +- Src/Base/AMReX_ccse-mpi.H | 2 +- Src/Base/AMReX_error_fi.cpp | 6 +- Src/Base/AMReX_error_mod.F90 | 4 +- Src/Base/AMReX_filcc_mod.F90 | 148 +- Src/Base/AMReX_iMultiFab.H | 88 +- Src/Base/AMReX_iMultiFab.cpp | 84 +- Src/Base/AMReX_mempool_mod.F90 | 6 +- Src/Base/AMReX_parmparse_fi.cpp | 46 +- Src/Base/AMReX_parmparse_mod.F90 | 8 +- Src/Boundary/AMReX_BndryData.H | 6 +- Src/Boundary/AMReX_BndryData.cpp | 16 +- Src/Boundary/AMReX_BndryRegister.H | 24 +- Src/Boundary/AMReX_BndryRegister.cpp | 36 +- Src/Boundary/AMReX_FabSet.H | 12 +- Src/Boundary/AMReX_FabSet.cpp | 28 +- Src/Boundary/AMReX_InterpBndryData.H | 4 +- Src/Boundary/AMReX_InterpBndryData.cpp | 6 +- Src/Boundary/AMReX_MacBndry.H | 2 +- Src/Boundary/AMReX_MacBndry.cpp | 8 +- Src/Boundary/AMReX_Mask.H | 6 +- Src/Boundary/AMReX_MultiMask.H | 8 +- Src/Boundary/AMReX_MultiMask.cpp | 6 +- Src/Boundary/AMReX_YAFluxRegister.cpp | 4 +- Src/EB/AMReX_EB2_3D_C.H | 24 +- Src/EB/AMReX_EB2_IF_Complement.H | 2 +- Src/EB/AMReX_EB2_IF_Lathe.H | 4 +- Src/EB/AMReX_EB2_IF_Sphere.H | 2 +- Src/EB/AMReX_EB2_Level.cpp | 2 +- Src/EB/AMReX_EBFluxRegister.cpp | 8 +- Src/EB/AMReX_EBInterpolater.cpp | 8 +- Src/EB/AMReX_EBMultiFabUtil.H | 6 +- Src/EB/AMReX_EBMultiFabUtil.cpp | 38 +- Src/EB/AMReX_EBMultiFabUtil_2D_C.H | 20 +- Src/EB/AMReX_EBMultiFabUtil_3D_C.H | 114 +- Src/EB/AMReX_EBToPVD.H | 10 +- Src/EB/AMReX_EBToPVD.cpp | 20 +- Src/EB/AMReX_EB_LeastSquares_2D_K.H | 2 +- Src/EB/AMReX_EB_LeastSquares_3D_K.H | 104 +- Src/EB/AMReX_EB_STL_utils.H | 4 +- Src/EB/AMReX_EB_STL_utils.cpp | 8 +- Src/EB/AMReX_EB_slopes_K.H | 94 +- Src/EB/AMReX_EB_triGeomOps_K.H | 16 +- Src/EB/AMReX_MultiCutFab.H | 2 +- Src/EB/AMReX_WriteEBSurface.H | 2 +- Src/EB/AMReX_WriteEBSurface.cpp | 10 +- Src/EB/AMReX_ebcellflag_mod.F90 | 4 +- .../AMReX_Conduit_Blueprint_ParticlesI.H | 20 +- Src/Extern/HYPRE/AMReX_Hypre.H | 8 +- Src/Extern/HYPRE/AMReX_Hypre.cpp | 6 +- Src/Extern/HYPRE/AMReX_HypreABecLap.H | 6 +- Src/Extern/HYPRE/AMReX_HypreABecLap.cpp | 8 +- Src/Extern/HYPRE/AMReX_HypreABecLap2.H | 2 +- Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp | 8 +- Src/Extern/HYPRE/AMReX_HypreABecLap3.H | 2 +- Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp | 12 +- Src/Extern/PETSc/AMReX_PETSc.H | 10 +- Src/Extern/PETSc/AMReX_PETSc.cpp | 42 +- Src/Extern/ProfParser/AMReX_AVGDOWN_1D.F | 20 +- Src/Extern/ProfParser/AMReX_AVGDOWN_2D.F | 26 +- Src/Extern/ProfParser/AMReX_AVGDOWN_3D.F | 20 +- Src/Extern/ProfParser/AMReX_AVGDOWN_F.H | 34 +- Src/Extern/ProfParser/AMReX_BLProfStats.H | 8 +- Src/Extern/ProfParser/AMReX_BLProfStats.cpp | 16 +- Src/Extern/ProfParser/AMReX_BLProfUtilities.H | 14 +- .../ProfParser/AMReX_BLProfUtilities.cpp | 20 +- Src/Extern/ProfParser/AMReX_BLWritePlotFile.H | 12 +- .../ProfParser/AMReX_BLWritePlotFile.cpp | 72 +- Src/Extern/ProfParser/AMReX_CommProfStats.H | 74 +- Src/Extern/ProfParser/AMReX_CommProfStats.cpp | 482 ++--- .../ProfParser/AMReX_ProfParserBatch.cpp | 94 +- .../ProfParser/AMReX_RegionsProfStats.H | 4 +- .../ProfParser/AMReX_RegionsProfStats.cpp | 398 ++-- Src/Extern/SWFFT/AlignedAllocator.h | 6 +- Src/Extern/SWFFT/CheckDecomposition.c | 502 ++--- Src/Extern/SWFFT/Dfft.H | 36 +- Src/Extern/SWFFT/DfftC.cpp | 10 +- Src/Extern/SWFFT/Distribution.H | 26 +- Src/Extern/SWFFT/DistributionC.cpp | 8 +- Src/Extern/SWFFT/Error.h | 6 +- Src/Extern/SWFFT/TimingStats.h | 10 +- Src/Extern/SWFFT/complex-type.h | 4 +- Src/Extern/SWFFT/distribution.c | 1458 +++++++------- Src/Extern/SWFFT/distribution_c.h | 20 +- Src/Extern/amrdata/AMReX_AmrData.H | 66 +- Src/Extern/amrdata/AMReX_AmrData.cpp | 650 +++--- Src/Extern/amrdata/AMReX_DataServices.H | 8 +- Src/Extern/amrdata/AMReX_DataServices.cpp | 200 +- Src/Extern/amrdata/AMReX_FABUTIL_1D.F | 50 +- Src/Extern/amrdata/AMReX_FABUTIL_2D.F | 52 +- Src/Extern/amrdata/AMReX_FABUTIL_3D.F | 162 +- Src/Extern/amrdata/AMReX_XYPlotDataList.H | 14 +- Src/Extern/amrdata/AMReX_XYPlotDataList.cpp | 20 +- Src/Extern/amrdata/CMakeLists.txt | 2 +- Src/Extern/hpgmg/BL_HPGMG.H | 104 +- Src/Extern/hpgmg/BL_HPGMG.cpp | 4 +- Src/F_Interfaces/AmrCore/AMReX_FAmrCore.H | 8 +- Src/F_Interfaces/AmrCore/AMReX_FAmrCore.cpp | 38 +- Src/F_Interfaces/AmrCore/AMReX_amrcore_fi.cpp | 86 +- .../AmrCore/AMReX_amrcore_mod.F90 | 4 +- .../AmrCore/AMReX_fillpatch_fi.cpp | 86 +- .../AmrCore/AMReX_fillpatch_mod.F90 | 6 +- .../AmrCore/AMReX_fluxregister_fi.cpp | 16 +- .../AmrCore/AMReX_fluxregister_mod.F90 | 10 +- Src/F_Interfaces/AmrCore/AMReX_tagbox_fi.cpp | 18 +- Src/F_Interfaces/AmrCore/AMReX_tagbox_mod.F90 | 2 +- Src/F_Interfaces/Base/AMReX_FPhysBC.cpp | 4 +- Src/F_Interfaces/Base/AMReX_box_fi.cpp | 4 +- Src/F_Interfaces/Base/AMReX_box_mod.F90 | 2 +- Src/F_Interfaces/Base/AMReX_boxarray_fi.cpp | 26 +- Src/F_Interfaces/Base/AMReX_distromap_fi.cpp | 16 +- Src/F_Interfaces/Base/AMReX_distromap_mod.F90 | 2 +- Src/F_Interfaces/Base/AMReX_geometry_fi.cpp | 28 +- Src/F_Interfaces/Base/AMReX_geometry_mod.F90 | 4 +- Src/F_Interfaces/Base/AMReX_init_mod.F90 | 2 +- Src/F_Interfaces/Base/AMReX_multifab_fi.cpp | 240 +-- Src/F_Interfaces/Base/AMReX_multifab_mod.F90 | 12 +- .../Base/AMReX_multifabutil_fi.cpp | 6 +- Src/F_Interfaces/Base/AMReX_parallel_mod.F90 | 4 +- Src/F_Interfaces/Base/AMReX_physbc_fi.cpp | 4 +- Src/F_Interfaces/Base/AMReX_physbc_mod.F90 | 2 +- Src/F_Interfaces/Base/AMReX_plotfile_fi.cpp | 30 +- Src/F_Interfaces/Base/AMReX_vismf_fi.cpp | 6 +- .../LinearSolvers/AMReX_linop_mod.F90 | 2 +- .../LinearSolvers/AMReX_multigrid_fi.cpp | 4 +- .../LinearSolvers/AMReX_multigrid_mod.F90 | 4 +- .../LinearSolvers/AMReX_poisson_mod.F90 | 2 +- Src/F_Interfaces/Octree/AMReX_octree_fi.cpp | 2 +- .../Particle/AMReX_particlecontainer_fi.cpp | 6 +- Src/LinearSolvers/MLMG/AMReX_MLABecLap_2D_K.H | 2 +- Src/LinearSolvers/MLMG/AMReX_MLABecLap_3D_K.H | 36 +- .../MLMG/AMReX_MLABecLaplacian.cpp | 14 +- Src/LinearSolvers/MLMG/AMReX_MLALap_1D_K.H | 12 +- Src/LinearSolvers/MLMG/AMReX_MLALaplacian.cpp | 6 +- Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H | 8 +- Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp | 64 +- .../MLMG/AMReX_MLCellABecLap.cpp | 2 +- Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp | 14 +- Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H | 2 +- Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp | 42 +- .../MLMG/AMReX_MLEBABecLap_2D_K.H | 46 +- .../MLMG/AMReX_MLEBABecLap_3D_K.H | 36 +- Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp | 278 +-- .../MLMG/AMReX_MLEBTensor_2D_K.H | 88 +- .../MLMG/AMReX_MLEBTensor_3D_K.H | 174 +- Src/LinearSolvers/MLMG/AMReX_MLLinOp.H | 16 +- Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp | 52 +- Src/LinearSolvers/MLMG/AMReX_MLLinOp_nd.F90 | 30 +- Src/LinearSolvers/MLMG/AMReX_MLMG.cpp | 2 +- Src/LinearSolvers/MLMG/AMReX_MLMGBndry.cpp | 2 +- Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H | 20 +- Src/LinearSolvers/MLMG/AMReX_MLNodeLap_3D_K.H | 60 +- .../MLMG/AMReX_MLNodeLaplacian.cpp | 64 +- Src/LinearSolvers/MLMG/AMReX_MLPoisson.cpp | 8 +- Src/LinearSolvers/MLMG/AMReX_MLPoisson_1D_K.H | 4 +- Src/LinearSolvers/MLMG/AMReX_MLTensorOp.H | 6 +- Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp | 34 +- Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H | 34 +- .../Projections/AMReX_MacProjector.H | 2 +- .../Projections/AMReX_NodalProjector.cpp | 2 +- Src/Particle/AMReX_ArrayOfStructs.H | 2 +- Src/Particle/AMReX_BinIterator.H | 20 +- Src/Particle/AMReX_NeighborParticlesGPUImpl.H | 68 +- Src/Particle/AMReX_ParGDB.H | 36 +- Src/Particle/AMReX_Particle.H | 4 +- Src/Particle/AMReX_ParticleCommunication.cpp | 24 +- Src/Particle/AMReX_ParticleContainerBase.H | 26 +- Src/Particle/AMReX_ParticleContainerI.H | 150 +- Src/Particle/AMReX_ParticleHDF5.H | 288 +-- Src/Particle/AMReX_ParticleIO.H | 78 +- Src/Particle/AMReX_ParticleInit.H | 208 +- Src/Particle/AMReX_ParticleMPIUtil.H | 2 +- Src/Particle/AMReX_ParticleMesh.H | 22 +- Src/Particle/AMReX_ParticleReduce.H | 124 +- Src/Particle/AMReX_ParticleTransformation.H | 58 +- Src/Particle/AMReX_ParticleUtil.H | 42 +- Src/Particle/AMReX_Particle_mod_K.H | 84 +- Src/Particle/AMReX_Particles.H | 56 +- Src/Particle/AMReX_SparseBins.H | 54 +- Src/Particle/AMReX_TracerParticle_mod_K.H | 38 +- Src/Particle/AMReX_TracerParticles.H | 16 +- Src/Particle/AMReX_TracerParticles.cpp | 100 +- Src/SDC/AMReX_SDCquadrature.F90 | 10 +- Src/SDC/AMReX_SDCstruct.H | 2 +- Src/SDC/AMReX_SDCstruct.cpp | 114 +- Tests/AsyncOut/multifab/main.cpp | 8 +- Tests/DivFreePatch/main.cpp | 20 +- Tests/FillBoundaryComparison/main.cpp | 182 +- Tests/GPU/AnyOf/main.cpp | 12 +- Tests/GPU/Fuse/main.cpp | 2 +- Tests/GPU/Vector/main.cpp | 16 +- Tests/HDF5Benchmark/main.cpp | 4 +- Tests/LinearSolvers/ABecLaplacian_C/MyTest.H | 2 +- .../LinearSolvers/ABecLaplacian_C/MyTest.cpp | 38 +- .../ABecLaplacian_C/initProb.cpp | 2 +- .../ABecLaplacian_C/initProb_K.H | 8 +- .../ABecLaplacian_C/scalingtest/run-16.sh | 2 +- .../ABecLaplacian_C/scalingtest/run-32.sh | 2 +- .../ABecLaplacian_C/scalingtest/run-8.sh | 2 +- .../threadmultiple_test/knl-mpi-1.sh | 4 +- .../threadmultiple_test/knl-mpi-16.sh | 4 +- .../threadmultiple_test/knl-mpi-2.sh | 6 +- .../threadmultiple_test/knl-mpi-32.sh | 4 +- .../threadmultiple_test/knl-mpi-4.sh | 4 +- .../threadmultiple_test/knl-mpi-64.sh | 6 +- .../threadmultiple_test/knl-mpi-8.sh | 6 +- .../threadmultiple_test/knl-omp-1.sh | 4 +- .../threadmultiple_test/knl-omp-16.sh | 4 +- .../threadmultiple_test/knl-omp-2.sh | 6 +- .../threadmultiple_test/knl-omp-32.sh | 4 +- .../threadmultiple_test/knl-omp-4.sh | 4 +- .../threadmultiple_test/knl-omp-64.sh | 6 +- .../threadmultiple_test/knl-omp-8.sh | 6 +- .../ABecLaplacian_F/init_prob.F90 | 22 +- .../LinearSolvers/ABecLaplacian_F/mytest.F90 | 10 +- Tests/LinearSolvers/CellEB/MyTest.H | 2 +- Tests/LinearSolvers/CellEB/MyTest.cpp | 10 +- Tests/LinearSolvers/CellEB/initEB.cpp | 2 +- Tests/LinearSolvers/CellEB2/MyTest.H | 2 +- Tests/LinearSolvers/CellEB2/MyTest.cpp | 16 +- Tests/LinearSolvers/CellEB2/initEB.cpp | 4 +- .../LinearSolvers/EBConvergenceTest/BC_2D.F90 | 4 +- .../LinearSolvers/EBConvergenceTest/BC_3D.F90 | 30 +- .../EBConvergenceTest/Convergence_Data_Gen.sh | 18 +- .../LinearSolvers/EBConvergenceTest/MyTest.H | 2 +- .../EBConvergenceTest/MyTest.cpp | 84 +- .../EBConvergenceTest/MyTest_F.H | 58 +- Tests/LinearSolvers/EBConvergenceTest/RHS.F90 | 254 +-- .../LinearSolvers/EBConvergenceTest/acoef.F90 | 40 +- .../LinearSolvers/EBConvergenceTest/bcoef.F90 | 64 +- Tests/LinearSolvers/EBTensor/MyTest.cpp | 2 +- .../LinearSolvers/EBTensor/MyTestPlotfile.cpp | 2 +- Tests/LinearSolvers/EBTensor/MyTest_2D_K.H | 2 +- Tests/LinearSolvers/EBflux_grad/MyTest.H | 6 +- Tests/LinearSolvers/EBflux_grad/MyTest.cpp | 12 +- Tests/LinearSolvers/EBflux_grad/initEB.cpp | 2 +- Tests/LinearSolvers/LeastSquares/MyTest.cpp | 4 +- Tests/LinearSolvers/LeastSquares/README.md | 2 +- Tests/LinearSolvers/LeastSquares/initData.cpp | 40 +- Tests/LinearSolvers/LeastSquares/initEB.cpp | 30 +- .../LeastSquares/initPoiseuilleData.cpp | 4 +- .../LinearSolvers/MAC_Projection_EB/main.cpp | 48 +- Tests/LinearSolvers/MLMG/fort_3d.F90 | 14 +- Tests/LinearSolvers/MLMG/init_prob.cpp | 2 +- Tests/LinearSolvers/MLMG/main.cpp | 26 +- .../MultiComponent/MCNodalLinOp.H | 142 +- .../MultiComponent/MCNodalLinOp.cpp | 1066 +++++----- Tests/LinearSolvers/MultiComponent/main.cpp | 392 ++-- .../Nodal_Projection_EB/main.cpp | 102 +- Tests/LinearSolvers/NodeEB/MyTest.cpp | 16 +- Tests/MultiBlock/Advection/main.cpp | 6 +- Tests/MultiBlock/IndexType/main.cpp | 8 +- Tests/Particles/AssignDensity/main.cpp | 22 +- .../AssignMultiLevelDensity/main.cpp | 40 +- Tests/Particles/AsyncIO/main.cpp | 4 +- Tests/Particles/GhostsAndVirtuals/main.cpp | 4 +- Tests/Particles/InitFromAscii/main.cpp | 18 +- Tests/Particles/Intersection/main.cpp | 32 +- Tests/Particles/NeighborParticles/CheckPair.H | 4 +- Tests/Particles/NeighborParticles/Constants.H | 2 +- .../NeighborParticles/MDParticleContainer.cpp | 36 +- Tests/Particles/NeighborParticles/README.md | 8 +- Tests/Particles/ParticleIterator/main.cpp | 12 +- Tests/Particles/ParticleMesh/main.cpp | 40 +- .../ParticleTransformations/main.cpp | 104 +- Tests/Particles/SparseBins/main.cpp | 16 +- Tests/Particles/TypeDescriptor/main.cpp | 82 +- .../AMRProfParser/TestCodes/AMRProfTest0.cpp | 6 +- .../AMRProfParser/TestCodes/ProfWaitTest.cpp | 16 +- Tools/AMRProfParser/TestCodes/SendTest0.cpp | 4 +- .../AMRProfParser/TestCodes/TokenizeTest.cpp | 20 +- .../AmrDeriveTecplot/AmrDeriveTecplot.cpp | 112 +- Tools/C_util/AppendToPlotFile.cpp | 48 +- .../AugmentPlotfile/AugmentPlotfile.cpp | 6 +- .../AugmentPlotfile/AugmentPlotfile_F.H | 4 +- Tools/C_util/Convergence/Add.cpp | 4 +- Tools/C_util/Convergence/ComparePlotfiles.cpp | 28 +- .../C_util/Convergence/ComputeAmrDataNorms.H | 10 +- .../Convergence/ComputeAmrDataNorms.cpp | 370 ++-- Tools/C_util/Convergence/DebugDump.H | 16 +- Tools/C_util/Convergence/DebugOut.H | 10 +- Tools/C_util/Convergence/DebugOut.cpp | 26 +- Tools/C_util/Convergence/DiffFab.cpp | 16 +- .../Convergence/DiffSameDomainRefined.cpp | 42 +- .../DiffSameDomainRefinedComposite.cpp | 56 +- .../Convergence/DiffSameDomainRefinedFD.cpp | 44 +- .../Convergence/DiffSameDomainRefinedStag.cpp | 96 +- Tools/C_util/Convergence/DiffSameGrid.cpp | 34 +- Tools/C_util/Convergence/DiffSameGrid2.cpp | 138 +- .../Convergence/DiffSameGridRefined.cpp | 42 +- Tools/C_util/Convergence/DiffUniform.cpp | 106 +- Tools/C_util/Convergence/PltFileNorm.cpp | 88 +- Tools/C_util/Convergence/PltFileNormB.cpp | 20 +- .../Convergence/PltFileScalConvRate.cpp | 60 +- .../Convergence/RichardsonConvergenceTest.cpp | 2 +- Tools/C_util/DiffMultiFab/diffmultifab.cpp | 44 +- Tools/C_util/Statistics/AVGDOWN_2D.F | 12 +- Tools/C_util/Statistics/AVGDOWN_3D.F | 12 +- Tools/C_util/Statistics/AVGDOWN_F.H | 16 +- Tools/C_util/Statistics/ComputeAmrDataStat.H | 142 +- .../C_util/Statistics/ComputeAmrDataStat.cpp | 1762 ++++++++--------- Tools/C_util/Statistics/PltFileFluxAve.H | 42 +- Tools/C_util/Statistics/PltFileFluxAve.cpp | 384 ++-- Tools/C_util/Statistics/PltFileList.cpp | 142 +- Tools/C_util/Statistics/PltFileStat.cpp | 620 +++--- Tools/C_util/Statistics/PltFileXAve.H | 38 +- Tools/C_util/Statistics/PltFileXAve.cpp | 354 ++-- Tools/C_util/TV_TempWrite.H | 68 +- Tools/C_util/ViewMF/MFNorm.cpp | 10 +- Tools/C_util/ViewMF/checkMFghostcells.cpp | 14 +- Tools/C_util/ViewMF/main.cpp | 94 +- Tools/C_util/ViewMF/mfMinMax.cpp | 8 +- Tools/C_util/ViewMF/viewMF.cpp | 10 +- Tools/C_util/ViewMF/viewMFcol.cpp | 14 +- Tools/C_util/ViewMF/viewMFdiff.cpp | 46 +- Tools/C_util/ViewMF/viewMFdiffHardWire.cpp | 46 +- Tools/C_util/WritePlotFile.H | 20 +- Tools/C_util/WritePlotFile.cpp | 254 +-- Tools/C_util/dbgTools/crsGrids.cpp | 2 +- Tools/C_util/dbgTools/intersectGrids.cpp | 4 +- Tools/CompileTesting/compiletesting.py | 6 +- Tools/EBSurfaceTools/ConvertEBSurface.cpp | 6 +- Tools/EBSurfaceTools/isoToVTK.py | 6 +- Tools/F_scripts/fcheck.py | 22 +- Tools/GNUMake/README.md | 6 +- Tools/Plotfile/AMReX_PPMUtil.cpp | 2 +- Tools/Plotfile/fcompare.cpp | 6 +- Tools/Postprocessing/C_Src/HorizontalAvg.cpp | 18 +- Tools/Postprocessing/C_Src/IntegrateComp.cpp | 8 +- .../Postprocessing/C_Src/MultiFabToMatLab.cpp | 60 +- .../Postprocessing/C_Src/PlotfileToMatLab.cpp | 4 +- Tools/Postprocessing/C_Src/PlotfileToTurb.cpp | 2 +- .../C_Src/PtwisePltTransform.cpp | 10 +- .../C_Src/PtwisePltTransform_nd.f90 | 2 +- .../C_Src/WritePlotfileToASCII.cpp | 14 +- Tools/Postprocessing/C_Src/cube_extract.cpp | 36 +- .../Postprocessing/C_Src/particle_compare.cpp | 84 +- Tools/Postprocessing/python/column_depth.py | 6 +- Tools/Postprocessing/python/conv_slopes.py | 24 +- .../Postprocessing/python/dumpparthistory.py | 28 +- Tools/Postprocessing/python/eos_data.txt | 46 +- Tools/Postprocessing/python/helmeos.py | 26 +- Tools/Postprocessing/python/parseparticles.py | 52 +- Tools/Postprocessing/python/test_helmeos.py | 6 +- .../python/test_parseparticles.py | 8 +- .../amrex_binary_particles_to_vtp.py | 20 +- .../write_pview_file.py | 6 +- Tools/Py_util/plotsinglevar.py | 94 +- Tools/Release/ppCleanup.py | 14 +- Tools/Release/ppCleanupDir.py | 2 +- Tools/Release/release.py | 16 +- Tools/typechecker/typechecker.py | 18 +- Tutorials/Amr/Advection_AmrCore/Exec/Prob.H | 4 +- .../Advection_AmrCore/Exec/paraview_amr101.py | 2 +- Tutorials/Amr/Advection_AmrCore/README.md | 32 +- .../Source/AdvancePhiAllLevels.cpp | 22 +- .../Source/AdvancePhiAtLevel.cpp | 50 +- .../Amr/Advection_AmrCore/Source/AmrCoreAdv.H | 18 +- .../Advection_AmrCore/Source/AmrCoreAdv.cpp | 140 +- .../Source/DefineVelocity.cpp | 2 +- .../Advection_AmrCore/Source/Src_K/Adv_K.H | 4 +- .../Source/Src_K/compute_flux_3D_K.H | 2 +- .../Advection_AmrCore/Source/face_velocity.H | 6 +- .../Amr/Advection_AmrCore/Source/main.cpp | 16 +- .../Exec/SingleVortex/Prob.f90 | 4 +- .../Exec/SingleVortex/face_velocity_2d.f90 | 6 +- .../Exec/SingleVortex/face_velocity_3d.f90 | 4 +- .../Exec/UniformVelocity/Prob.f90 | 14 +- .../Amr/Advection_AmrLevel/Source/Adv_F.H | 62 +- .../Advection_AmrLevel/Source/AmrLevelAdv.H | 14 +- .../Advection_AmrLevel/Source/AmrLevelAdv.cpp | 310 +-- .../Advection_AmrLevel/Source/LevelBldAdv.cpp | 4 +- .../Source/Src_2d/Adv_2d.f90 | 10 +- .../Source/Src_2d/compute_flux_2d.f90 | 2 +- .../Source/Src_2d/slope_2d.f90 | 12 +- .../Source/Src_3d/Adv_3d.f90 | 10 +- .../Source/Src_3d/compute_flux_3d.f90 | 8 +- .../Source/Src_3d/slope_3d.f90 | 18 +- .../Source/Src_nd/Tagging_nd.f90 | 8 +- .../Source/Src_nd/tagging_params.f90 | 2 +- .../Amr/Advection_AmrLevel/Source/main.cpp | 38 +- Tutorials/Basic/Build_with_libamrex/main.cpp | 6 +- .../Build_with_libamrex/test_parameters.cpp | 2 +- .../Basic/HeatEquation_EX1_C/Source/main.cpp | 16 +- .../Basic/HeatEquation_EX1_C/Source/myfunc.H | 4 +- .../HeatEquation_EX1_C/Source/myfunc.cpp | 6 +- .../HeatEquation_EX1_CF/Source/advance.cpp | 14 +- .../HeatEquation_EX1_CF/Source/advance_3d.f90 | 2 +- .../Source/init_phi_2d.f90 | 6 +- .../Source/init_phi_3d.f90 | 6 +- .../Basic/HeatEquation_EX1_CF/Source/main.cpp | 16 +- .../Basic/HeatEquation_EX1_CF/Source/myfunc.H | 4 +- .../HeatEquation_EX1_CF/Source/myfunc_F.H | 6 +- .../Basic/HeatEquation_EX1_F/advance.f90 | 24 +- Tutorials/Basic/HeatEquation_EX1_F/fmain.F90 | 4 +- .../Basic/HeatEquation_EX1_F/init_phi.f90 | 14 +- .../Basic/HeatEquation_EX2_C/Source/main.cpp | 100 +- .../Basic/HeatEquation_EX2_C/Source/myfunc.H | 6 +- .../HeatEquation_EX2_C/Source/myfunc.cpp | 64 +- .../HeatEquation_EX2_C/Source/mykernel.H | 134 +- .../HeatEquation_EX2_CF/Source/advance.cpp | 14 +- .../HeatEquation_EX2_CF/Source/advance_3d.f90 | 2 +- .../Source/init_phi_2d.f90 | 6 +- .../Source/init_phi_3d.f90 | 6 +- .../Basic/HeatEquation_EX2_CF/Source/main.cpp | 18 +- .../Basic/HeatEquation_EX2_CF/Source/myfunc.H | 6 +- .../HeatEquation_EX2_CF/Source/myfunc_F.H | 6 +- .../HeatEquation_EX3_C/Source/advance.cpp | 82 +- .../HeatEquation_EX3_C/Source/init_phi_2d.f90 | 6 +- .../HeatEquation_EX3_C/Source/init_phi_3d.f90 | 6 +- .../Basic/HeatEquation_EX3_C/Source/main.cpp | 22 +- .../Basic/HeatEquation_EX3_C/Source/myfunc.H | 8 +- .../HeatEquation_EX3_C/Source/myfunc_F.H | 2 +- Tutorials/Basic/main_C/main.cpp | 6 +- Tutorials/Basic/main_F/main.F90 | 4 +- .../AssignMultiLevelDensity/main.cpp | 60 +- .../CellSortedParticles/CellSortedPC.H | 12 +- .../CellSortedParticles/CellSortedPC.cpp | 90 +- .../CellSortedParticles/cell_sorted_3d.F90 | 38 +- .../CellSortedParticles/cell_sorted_F.H | 2 +- .../Blueprint/CellSortedParticles/main.cpp | 26 +- .../HeatEquation_EX1_C/Source/advance.cpp | 14 +- .../HeatEquation_EX1_C/Source/advance_3d.f90 | 2 +- .../HeatEquation_EX1_C/Source/init_phi_2d.f90 | 6 +- .../HeatEquation_EX1_C/Source/init_phi_3d.f90 | 6 +- .../HeatEquation_EX1_C/Source/main.cpp | 42 +- .../HeatEquation_EX1_C/Source/myfunc.H | 4 +- .../HeatEquation_EX1_C/Source/myfunc_F.H | 6 +- Tutorials/Blueprint/README.txt | 8 +- .../EB/CNS/Exec/Combustor/bc_fill_nd.F90 | 2 +- Tutorials/EB/CNS/Exec/Pulse/cns_prob.F90 | 2 +- Tutorials/EB/CNS/Exec/ShockRef/cns_prob.F90 | 2 +- Tutorials/EB/CNS/Exec/Sod/cns_prob.F90 | 2 +- Tutorials/EB/CNS/Source/CNS.H | 4 +- Tutorials/EB/CNS/Source/CNS.cpp | 48 +- Tutorials/EB/CNS/Source/CNS_F.H | 14 +- Tutorials/EB/CNS/Source/CNS_advance.cpp | 8 +- Tutorials/EB/CNS/Source/CNS_init_eb2.cpp | 16 +- Tutorials/EB/CNS/Source/CNS_io.cpp | 6 +- Tutorials/EB/CNS/Source/CNS_setup.cpp | 18 +- .../EB/CNS/Source/diffusion/cns_diff_mod.F90 | 2 +- .../CNS/Source/diffusion/cns_eb_diff_wall.F90 | 8 +- .../EB/CNS/Source/diffusion/diff_coef_mod.F90 | 2 +- Tutorials/EB/CNS/Source/fortran/CNS_divop.F90 | 14 +- Tutorials/EB/CNS/Source/fortran/CNS_dudt.F90 | 12 +- Tutorials/EB/CNS/Source/fortran/CNS_f.F90 | 4 +- Tutorials/EB/CNS/Source/fortran/CNS_nd.F90 | 4 +- .../EB/CNS/Source/fortran/bc_fill_nd.F90 | 2 +- .../EB/CNS/Source/hydro/Hyp_gamma_MOL.F90 | 20 +- .../EB/CNS/Source/hydro/Hyp_gamma_MOL_EB.F90 | 20 +- Tutorials/EB/CNS/Source/hydro/analriem3d.F90 | 8 +- .../EB/CNS/Source/hydro/cns_eb_hyp_wall.F90 | 2 +- .../CNS/Source/hydro/slope_mol_3d_gamma.F90 | 86 +- .../Source/hydro/slope_mol_3d_gamma_EB.F90 | 50 +- Tutorials/EB/CNS/Source/main.cpp | 50 +- Tutorials/EB/GeometryGeneration/main.cpp | 6 +- Tutorials/EB/MacProj/main.cpp | 8 +- Tutorials/EB/Poisson/main.cpp | 12 +- Tutorials/ForkJoin/MLMG/ff.f90 | 10 +- .../Advection_F/Exec/SingleVortex/Prob_2d.f90 | 18 +- .../Advection_F/Exec/SingleVortex/Prob_3d.f90 | 8 +- .../Exec/SingleVortex/face_velocity_2d.F90 | 12 +- .../Exec/SingleVortex/face_velocity_3d.F90 | 6 +- .../Source/Src_2d/advect_2d_mod.F90 | 44 +- .../Source/Src_2d/compute_flux_2d.f90 | 2 +- .../Advection_F/Source/Src_2d/slope_2d.f90 | 12 +- .../Source/Src_3d/advect_3d_mod.F90 | 32 +- .../Source/Src_3d/compute_flux_3d.f90 | 8 +- .../Advection_F/Source/Src_3d/slope_3d.f90 | 18 +- .../Advection_F/Source/amr_data_mod.F90 | 6 +- .../Advection_F/Source/averagedown_mod.F90 | 4 +- .../Advection_F/Source/bc_mod.F90 | 2 +- .../Advection_F/Source/compute_dt_mod.F90 | 10 +- .../Advection_F/Source/evolve_mod.F90 | 18 +- .../Advection_F/Source/fillpatch_mod.F90 | 6 +- .../Advection_F/Source/fmain.F90 | 2 +- .../Advection_F/Source/initdata.F90 | 14 +- .../Advection_F/Source/my_amr_mod.F90 | 14 +- .../Advection_F/Source/plotfile_mod.F90 | 4 +- .../Advection_F/Source/tagging_mod.F90 | 2 +- .../Exec/SingleVortex/Prob.f90 | 10 +- .../Exec/SingleVortex/face_velocity_2d.F90 | 12 +- .../Source/Src_2d/advect_2d_mod.F90 | 22 +- .../Source/Src_2d/compute_flux_2d.f90 | 2 +- .../Source/Src_2d/slope_2d.f90 | 12 +- .../Source/amr_data_mod.F90 | 2 +- .../Source/averagedown_mod.F90 | 2 +- .../Advection_octree_F/Source/bc_mod.F90 | 2 +- .../Source/compute_dt_mod.F90 | 8 +- .../Advection_octree_F/Source/evolve_mod.F90 | 8 +- .../Source/fillpatch_mod.F90 | 8 +- .../Advection_octree_F/Source/my_amr_mod.F90 | 14 +- .../Exec/SingleVortex/Prob.f90 | 10 +- .../Exec/SingleVortex/face_velocity_2d.F90 | 12 +- .../Source/Src_2d/advect_2d_mod.F90 | 6 +- .../Source/Src_2d/compute_flux_2d.f90 | 6 +- .../Source/Src_2d/slope_2d.f90 | 12 +- .../Source/amr_data_mod.F90 | 2 +- .../Source/averagedown_mod.F90 | 2 +- .../Advection_octree_F2/Source/bc_mod.F90 | 2 +- .../Source/compute_dt_mod.F90 | 8 +- .../Advection_octree_F2/Source/evolve_mod.F90 | 8 +- .../Source/fillpatch_mod.F90 | 8 +- .../Advection_octree_F2/Source/my_amr_mod.F90 | 14 +- Tutorials/GPU/CNS/Source/CNS.H | 4 +- Tutorials/GPU/CNS/Source/CNS.cpp | 50 +- Tutorials/GPU/CNS/Source/CNS_K.H | 2 +- Tutorials/GPU/CNS/Source/CNS_advance.cpp | 2 +- Tutorials/GPU/CNS/Source/CNS_io.cpp | 4 +- Tutorials/GPU/CNS/Source/CNS_setup.cpp | 16 +- Tutorials/GPU/CNS/Source/hydro/CNS_hydro_K.H | 2 +- Tutorials/GPU/CNS/Source/main.cpp | 48 +- Tutorials/GPU/EBCNS/Source/CNS.H | 4 +- Tutorials/GPU/EBCNS/Source/CNS.cpp | 32 +- Tutorials/GPU/EBCNS/Source/CNS_advance.cpp | 2 +- Tutorials/GPU/EBCNS/Source/CNS_init_eb2.cpp | 16 +- Tutorials/GPU/EBCNS/Source/CNS_io.cpp | 50 +- Tutorials/GPU/EBCNS/Source/CNS_setup.cpp | 16 +- .../GPU/EBCNS/Source/hydro/CNS_hydro_K.H | 2 +- Tutorials/GPU/EBCNS/Source/main.cpp | 48 +- Tutorials/GPU/Launch/MyKernel_F.F90 | 2 +- Tutorials/GPU/ParallelScan/main.cpp | 6 +- Tutorials/MUI/Source_01/init_phi_3d.f90 | 8 +- Tutorials/MUI/Source_01/main_01.cpp | 246 +-- Tutorials/MUI/Source_02/main_02.cpp | 208 +- Tutorials/MUI/doc/MUIcouplingNotes.tex | 4 +- .../CellSortedParticles/CellSortedPC.H | 12 +- .../CellSortedParticles/CellSortedPC.cpp | 88 +- .../CellSortedParticles/cell_sorted_3d.F90 | 38 +- .../CellSortedParticles/cell_sorted_F.H | 2 +- .../Particles/CellSortedParticles/main.cpp | 24 +- .../Exec/CUDA/EMParticleContainer.cpp | 8 +- .../ElectromagneticPIC/Exec/CUDA/em_pic_K.H | 18 +- .../Exec/OpenACC/EMParticleContainer.cpp | 32 +- .../Exec/OpenACC/em_pic_3d.F90 | 16 +- .../Exec/OpenMP/EMParticleContainer.cpp | 8 +- .../Source/EMParticleContainer.H | 12 +- .../Source/EMParticleContainerInit.cpp | 42 +- .../ElectromagneticPIC/Source/main.cpp | 20 +- .../ElectrostaticParticleContainer.H | 14 +- .../ElectrostaticParticleContainer.cpp | 98 +- .../ElectrostaticPIC/electrostatic_pic_2d.f90 | 52 +- .../ElectrostaticPIC/electrostatic_pic_3d.f90 | 60 +- .../ElectrostaticPIC/electrostatic_pic_F.H | 48 +- Tutorials/Particles/ElectrostaticPIC/main.cpp | 86 +- Tutorials/Particles/NeighborList/CheckPair.H | 4 +- Tutorials/Particles/NeighborList/Constants.H | 2 +- .../NeighborList/MDParticleContainer.H | 2 +- .../NeighborList/MDParticleContainer.cpp | 74 +- Tutorials/Particles/NeighborList/README.md | 8 +- Tutorials/Particles/NeighborList/main.cpp | 38 +- .../SDC/MISDC_ADR_2d/Source/SDC_sweeper.cpp | 178 +- .../SDC/MISDC_ADR_2d/Source/functions_2d.f90 | 20 +- .../SDC/MISDC_ADR_2d/Source/init_phi_2d.f90 | 12 +- Tutorials/SDC/MISDC_ADR_2d/Source/main.cpp | 192 +- Tutorials/SDC/MISDC_ADR_2d/Source/myfunc.H | 40 +- Tutorials/SDC/MISDC_ADR_2d/Source/myfunc_F.H | 38 +- Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.H | 2 +- Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.cpp | 8 +- .../SWFFT/SWFFT_poisson/SWFFT_Test_F.F90 | 10 +- Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.H | 8 +- Tutorials/SWFFT/SWFFT_poisson/main.cpp | 4 +- .../SWFFT/SWFFT_poisson/swfft_solver.cpp | 22 +- Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.H | 2 +- Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.cpp | 28 +- Tutorials/SWFFT/SWFFT_simple/SWFFT_Test_F.H | 8 +- Tutorials/SWFFT/SWFFT_simple/main.cpp | 4 +- .../SWFFT/SWFFT_simple/swfft_compute.cpp | 26 +- 746 files changed, 17275 insertions(+), 17275 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e7cec8a90a3..2ec258c7301 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,9 +22,9 @@ Development generally follows the following ideas: since these will be used for a squashed commit message.* Please note the following: - If you choose to make contributions to the code - then you hereby grant a non-exclusive, royalty-free perpetual license - to install, use, modify, prepare derivative works, + If you choose to make contributions to the code + then you hereby grant a non-exclusive, royalty-free perpetual license + to install, use, modify, prepare derivative works, incorporate into other computer software, distribute, and sublicense such enhancements or derivative works thereof, in binary and source code form. @@ -53,7 +53,7 @@ your fork. First, let us setup your local git repo. Make your own fork of the main (`upstream`) repository: on the [AMReX Github page](https://github.com/AMReX-Codes/amrex), press the -fork button. +fork button. If you already had a fork of AMReX prior to 4/17/2020, we recommend deleting it and re-forking. This is due to a history re-write on the main repository. Note that you will lose any branches diff --git a/Docs/Migration/Migration.md b/Docs/Migration/Migration.md index 4a4664f0ccb..17e372d73b3 100644 --- a/Docs/Migration/Migration.md +++ b/Docs/Migration/Migration.md @@ -16,7 +16,7 @@ environment variable `AMREX_HOME` is set to the AMReX directory. ## Step 0 Make sure your code works with the [latest version of BoxLib on -github](https://github.com/BoxLib-Codes/BoxLib). +github](https://github.com/BoxLib-Codes/BoxLib). ## Step 1 @@ -59,7 +59,7 @@ has been removed from AMReX. This step has to be done manually. function does not change the pointers even though they may modify the data pointed by the pointers. AMReX provides a number of functions that can convert from `unique_ptr` to `T*`. For - example, + example, `Array GetArrOfPtrs (const Array >& a)`. These functions are in `Src/C_BaseLib/Array.H` as of writing, and they will be moved to `Src/Base/AMReX_Array.H`. @@ -97,14 +97,14 @@ into the `amrex` namespace. In this step, you can use `Tools/Migration/step-5-amrex-namespace/amrex-namespace.sh` to replace `BoxLib::` with `amrex::` for those already in `BoxLib` namespace. However, the rest of work is expected to be performed manually, -because C++ is too a complicated language for shell scripting. +because C++ is too a complicated language for shell scripting. For most `.cpp` files, you can put a `using namespace amrex;` line after the last `include` line, or `using amrex::MultiFab` etc., or you can add `amrex::` to wherever needed. Note that having both `using namespace amrex` and `using namespace std` in one file may cause conflicts because some names like `min` and `max` exist in both -namespace. +namespace. For header files, it is considered bad practice to have `using namespace amrex` because it pollutes the namespace. So you need to @@ -113,7 +113,7 @@ manually add `amrex::` in front of AMReX names likes `MultiFab` and ### Step 6 -AMReX `migration/6-distributionmap` branch should be used in this step. +AMReX `migration/6-distributionmap` branch should be used in this step. In BoxLib, there is a `DistributionMapping` cache implemented with `std::map` with the number of `Box`es as the key. Utilizing the diff --git a/Docs/sphinx_documentation/source/AMReX_Profiling_Tools_Chapter.rst b/Docs/sphinx_documentation/source/AMReX_Profiling_Tools_Chapter.rst index c015f0a9960..235e383e02c 100644 --- a/Docs/sphinx_documentation/source/AMReX_Profiling_Tools_Chapter.rst +++ b/Docs/sphinx_documentation/source/AMReX_Profiling_Tools_Chapter.rst @@ -11,13 +11,13 @@ varying levels of profiling detail. Here are links to short courses (slides) on how to use the profiling tools. More details can be found in the documentation below. -Lecture 1: `Introduction and TINYPROFILER `_ +Lecture 1: `Introduction and TINYPROFILER `_ -Lecture 2: `Introduction to Full Profiling `_ +Lecture 2: `Introduction to Full Profiling `_ -Lecture 3: `Using ProfVis -- GUI Features `_ +Lecture 3: `Using ProfVis -- GUI Features `_ -Lecture 4: `Batch Options and Advanced Profiling Flags `_ +Lecture 4: `Batch Options and Advanced Profiling Flags `_ .. toctree:: :maxdepth: 1 diff --git a/Docs/sphinx_documentation/source/AmrCore.rst b/Docs/sphinx_documentation/source/AmrCore.rst index b0d115dbe61..117063174b9 100644 --- a/Docs/sphinx_documentation/source/AmrCore.rst +++ b/Docs/sphinx_documentation/source/AmrCore.rst @@ -25,9 +25,9 @@ .. _fig:Adv: -.. table:: Time sequence (:math:`t=0,0.5,1,1.5,2` s) of advection of a Gaussian profile using the SingleVortex tutorial. The analytic velocity field distorts the profile, and then restores the profile to the original configuration. The red, green, and blue boxes indicate grids at AMR levels :math:`\ell=0,1`, and :math:`2`. +.. table:: Time sequence (:math:`t=0,0.5,1,1.5,2` s) of advection of a Gaussian profile using the SingleVortex tutorial. The analytic velocity field distorts the profile, and then restores the profile to the original configuration. The red, green, and blue boxes indicate grids at AMR levels :math:`\ell=0,1`, and :math:`2`. :align: center - + +-----+-----+-----+-----+-----+ | |a| | |b| | |c| | |d| | |e| | +-----+-----+-----+-----+-----+ @@ -69,7 +69,7 @@ The protected data members are: int finest_level; // Current finest level. Vector n_error_buf; // Buffer cells around each tagged cell. - Vector blocking_factor; // Blocking factor in grid generation + Vector blocking_factor; // Blocking factor in grid generation // (by level). Vector max_grid_size; // Maximum allowable grid size (by level). Real grid_eff; // Grid efficiency. @@ -77,12 +77,12 @@ The protected data members are: bool use_fixed_coarse_grids; int use_fixed_upto_level; - bool refine_grid_layout; // chop up grids to have the number of + bool refine_grid_layout; // chop up grids to have the number of // grids no less the number of procs Vector geom; Vector dmap; - Vector grids; + Vector grids; The following parameters are frequently set via the inputs file or the command line. Their usage is described in the section on :ref:`sec:grid_creation` @@ -135,22 +135,22 @@ class :cpp:`AmrCoreAdv`. :: //! Tag cells for refinement. TagBoxArray tags is built on level lev grids. - virtual void ErrorEst (int lev, TagBoxArray& tags, Real time, + virtual void ErrorEst (int lev, TagBoxArray& tags, Real time, int ngrow) override = 0; //! Make a new level from scratch using provided BoxArray and DistributionMapping. //! Only used during initialization. - virtual void MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, + virtual void MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, const DistributionMapping& dm) override = 0; - //! Make a new level using provided BoxArray and DistributionMapping and fill + //! Make a new level using provided BoxArray and DistributionMapping and fill // with interpolated coarse level data. - virtual void MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba, + virtual void MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba, const DistributionMapping& dm) = 0; - //! Remake an existing level using provided BoxArray and DistributionMapping + //! Remake an existing level using provided BoxArray and DistributionMapping // and fill with existing fine and coarse data. - virtual void RemakeLevel (int lev, Real time, const BoxArray& ba, + virtual void RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMapping& dm) = 0; //! Delete level data @@ -215,7 +215,7 @@ single-level routines :cpp:`MultiFab::FillBoundary` and :cpp:`FillDomainBoundary to fill interior, periodic, and physical boundary ghost cells. In principle, you can write a single-level application that calls :cpp:`FillPatchSingleLevel()` instead of using :cpp:`MultiFab::FillBoundary` and :cpp:`FillDomainBoundary()`. - + A :cpp:`FillPatchUtil` uses an :cpp:`Interpolator`. This is largely hidden from application codes. AMReX_Interpolater.cpp/H contains the virtual base class :cpp:`Interpolater`, which provides an interface for coarse-to-fine spatial interpolation operators. The fillpatch routines described @@ -245,7 +245,7 @@ Using FluxRegisters ------------------- AMReX_FluxRegister.cpp/H contains the class :cpp:`FluxRegister`, which is -derived from the class :cpp:`BndryRegister` (in ``amrex/Src/Boundary/AMReX_BndryRegister``). +derived from the class :cpp:`BndryRegister` (in ``amrex/Src/Boundary/AMReX_BndryRegister``). In the most general terms, a FluxRegister is a special type of BndryRegister that stores and manipulates data (most often fluxes) at coarse-fine interfaces. A simple usage scenario comes from a conservative discretization of a hyperbolic @@ -402,7 +402,7 @@ Code Structure Source code tree for the AmrAdvection_AmrCore example. -The figure shows the :ref:`fig:AmrAdvection_AmrCore_flowchart` +The figure shows the :ref:`fig:AmrAdvection_AmrCore_flowchart` - amrex/Src/ @@ -499,11 +499,11 @@ These fluxes are used to set or increment the flux registers. // the flux registers from the coarse or fine grid perspective // NOTE: the flux register associated with flux_reg[lev] is associated // with the lev/lev-1 interface (and has grid spacing associated with lev-1) - if (do_reflux) { + if (do_reflux) { if (flux_reg[lev+1]) { for (int i = 0; i < BL_SPACEDIM; ++i) { flux_reg[lev+1]->CrseInit(fluxes[i],i,0,0,fluxes[i].nComp(), -1.0); - } + } } if (flux_reg[lev]) { for (int i = 0; i < BL_SPACEDIM; ++i) { @@ -558,7 +558,7 @@ advanced a multiple of :cpp:`regrid_int`, we call the :cpp:`regrid` function. // regrid could add newly refine levels // (if finest_level < max_level) // so we save the previous finest level index - int old_finest = finest_level; + int old_finest = finest_level; regrid(lev, time); // if there are newly created levels, set the time step @@ -618,19 +618,19 @@ interface to a Fortran routine that tags cells (in this case, :fortran:`state_er #endif { Vector itags; - + for (MFIter mfi(state,true); mfi.isValid(); ++mfi) { const Box& tilebox = mfi.tilebox(); TagBox& tagfab = tags[mfi]; - + // We cannot pass tagfab to Fortran because it is BaseFab. // So we are going to get a temporary integer array. // set itags initially to 'untagged' everywhere // we define itags over the tilebox region tagfab.get_itags(itags, tilebox); - + // data pointer and index space int* tptr = itags.dataPtr(); const int* tlo = tilebox.loVect(); @@ -639,8 +639,8 @@ interface to a Fortran routine that tags cells (in this case, :fortran:`state_er // tag cells for refinement state_error(tptr, ARLIM_3D(tlo), ARLIM_3D(thi), BL_TO_FORTRAN_3D(state[mfi]), - &tagval, &clearval, - ARLIM_3D(tilebox.loVect()), ARLIM_3D(tilebox.hiVect()), + &tagval, &clearval, + ARLIM_3D(tilebox.loVect()), ARLIM_3D(tilebox.hiVect()), ZFILL(dx), ZFILL(prob_lo), &time, &phierr[lev]); // // Now update the tags in the TagBox in the tilebox region @@ -665,7 +665,7 @@ is simple: dx,problo,time,phierr) bind(C, name="state_error") implicit none - + integer :: lo(3),hi(3) integer :: state_lo(3),state_hi(3) integer :: tag_lo(3),tag_hi(3) diff --git a/Docs/sphinx_documentation/source/AmrLevel.rst b/Docs/sphinx_documentation/source/AmrLevel.rst index de4180d25e3..66f73896379 100644 --- a/Docs/sphinx_documentation/source/AmrLevel.rst +++ b/Docs/sphinx_documentation/source/AmrLevel.rst @@ -111,10 +111,10 @@ the :cpp:`AmrLevelAdv` class, e.g., for (int i = 0; i < BL_SPACEDIM; ++i) { lo_bc[i] = hi_bc[i] = INT_DIR; // periodic boundaries } - + BCRec bc(lo_bc, hi_bc); - desc_lst.setComponent(Phi_Type, 0, "phi", bc, + desc_lst.setComponent(Phi_Type, 0, "phi", bc, StateDescriptor::BndryFunc(nullfill)); } @@ -168,7 +168,7 @@ The figure above shows the :ref:`fig:AmrAdvection_AmrLevel_flowchart` main() Amr amr; amr.init() - loop { + loop { amr.coarseTimeStep() /* compute dt */ timeStep() diff --git a/Docs/sphinx_documentation/source/Basics.rst b/Docs/sphinx_documentation/source/Basics.rst index ba5044d8236..1b6455877d2 100644 --- a/Docs/sphinx_documentation/source/Basics.rst +++ b/Docs/sphinx_documentation/source/Basics.rst @@ -365,7 +365,7 @@ Sharing the Command Line ------------------------ In some cases we want AMReX to only read part of the command line -- this happens, for example, when we -are going to use AMReX in cooperation with another code package and that code also takes command-line +are going to use AMReX in cooperation with another code package and that code also takes command-line arguments. .. highlight:: console @@ -1518,7 +1518,7 @@ together. In this section, we will show how you can operate on the // Call function f1 to work on the region specified by box. // Note that the whole region of the Fab includes ghost // cells (if there are any), and is thus larger than or - // equal to "box". + // equal to "box". f1(box, a); } @@ -2214,7 +2214,7 @@ macro on loops that are not safe for vectorization may lead to errors, so if unsure about the independence of the iterations of a loop, test and verify before adding the macro. -These loops should usually use :cpp:`i <= hi.x`, not :cpp:`i < hi.x`, when +These loops should usually use :cpp:`i <= hi.x`, not :cpp:`i < hi.x`, when defining the loop bounds. If not, the highest index cells will be left out of the calculation. @@ -2611,7 +2611,7 @@ For example, :: mpiexec -n 4 valgrind --leak-check=yes --track-origins=yes --log-file=vallog.%p ./foo.exe ... - + Breaking into Debuggers ======================= diff --git a/Docs/sphinx_documentation/source/DualGrid.rst b/Docs/sphinx_documentation/source/DualGrid.rst index c9849ef743b..4eee16860da 100644 --- a/Docs/sphinx_documentation/source/DualGrid.rst +++ b/Docs/sphinx_documentation/source/DualGrid.rst @@ -15,12 +15,12 @@ the mesh work and particle work have very different requirements for load balanc Rather than using a combined work estimate to create the same grids for mesh and particle data, we have the option to pursue a "dual grid" approach. -With this approach the mesh (:cpp:`MultiFab`) and particle (:cpp:`ParticleContainer`) data +With this approach the mesh (:cpp:`MultiFab`) and particle (:cpp:`ParticleContainer`) data are allocated on different :cpp:`BoxArrays` with different :cpp:`DistributionMappings`. This enables separate load balancing strategies to be used for the mesh and particle work. -The cost of this strategy, of course, is the need to copy mesh data onto temporary +The cost of this strategy, of course, is the need to copy mesh data onto temporary :cpp:`MultiFabs` defined on the particle :cpp:`BoxArrays` when mesh-particle communication is required. diff --git a/Docs/sphinx_documentation/source/EB.rst b/Docs/sphinx_documentation/source/EB.rst index e6a759dae5e..b5d300aeed7 100644 --- a/Docs/sphinx_documentation/source/EB.rst +++ b/Docs/sphinx_documentation/source/EB.rst @@ -226,7 +226,7 @@ procedure for this goes as follows: Real operator() (const Array& p) const; -- Make a :cpp:`EB2::GeometryShop` object using the implicit function. +- Make a :cpp:`EB2::GeometryShop` object using the implicit function. - Build an :cpp:`EB2::IndexSpace` with the :cpp:`EB2::GeometryShop` object and a :cpp:`Geometry` object that contains the information about the domain and the @@ -382,7 +382,7 @@ format of basic AMReX objects such as :cpp:`BaseFab`, :cpp:`FArrayBox`, const Vector& a_ngrow, EBSupport a_support); -or +or .. highlight: c++ @@ -405,11 +405,11 @@ needed. - :cpp:`EBSupport:full`: volume plus area fraction, boundary centroid and face centroid -:cpp:`EBFArrayBoxFactory` is derived from :cpp:`FabFactory`. +:cpp:`EBFArrayBoxFactory` is derived from :cpp:`FabFactory`. :cpp:`MultiFab` constructors have an optional argument :cpp:`const FabFactory&`. We can use :cpp:`EBFArrayBoxFactory` to build :cpp:`MultiFab`\ s that carry EB data. Member function of -:cpp:`FabArray` +:cpp:`FabArray` .. highlight: c++ @@ -503,7 +503,7 @@ it to determine if a box contains cut cells. auto const& flags = factory->getMultiEBCellFlagFab(); MultiCutFab const& centroid = factory->getCentroid(); - + for (MFIter mfi ...) { const Box& bx = mfi.tilebox(); FabType t = flags[mfi].getType(bx); @@ -544,20 +544,20 @@ testing cell types and getting neighbor information. For example else if (is_single_valued_cell(flags(i,j,k))) then ! this is a cut cell end if - end do - end do - end do + end do + end do + end do Linear Solvers ============== Linear solvers for the canonical form (equation :eq:`eqn::abeclap`) -have been discussed in chapter :ref:`Chap:LinearSolvers`. +have been discussed in chapter :ref:`Chap:LinearSolvers`. -AMReX supports multi-level -1) cell-centered solvers with homogeneous Neumann, homogeneous Dirichlet, -or inhomogeneous Dirichlet boundary conditions on the EB faces, and +AMReX supports multi-level +1) cell-centered solvers with homogeneous Neumann, homogeneous Dirichlet, +or inhomogeneous Dirichlet boundary conditions on the EB faces, and 2) nodal solvers with homogeneous Neumann boundary conditions on the EB faces. To use a cell-centered solver with EB, one builds a linear operator @@ -601,9 +601,9 @@ where phi_on_eb is the MultiFab holding the Dirichlet values in every cut cell, and coeff again is a real number (i.e. the value is the same at every cell) or a MultiFab holding the coefficient of the gradient at each cell with an EB face. -Currently there are options to define the face-based coefficients on +Currently there are options to define the face-based coefficients on face centers vs face centroids, and to interpret the solution variable -as being defined on cell centers vs cell centroids. +as being defined on cell centers vs cell centroids. The default is for the solution variable to be defined at cell centers; to tell the solver to interpret the solution variable as living diff --git a/Docs/sphinx_documentation/source/External_Profiling_Tools.rst b/Docs/sphinx_documentation/source/External_Profiling_Tools.rst index 71222d7588e..1dd03f2d7e7 100644 --- a/Docs/sphinx_documentation/source/External_Profiling_Tools.rst +++ b/Docs/sphinx_documentation/source/External_Profiling_Tools.rst @@ -135,20 +135,20 @@ Example MPI profile output: ##IPMv2.0.5######################################################## # - # command : /global/cscratch1/sd/cchan2/projects/lbl/BoxLib/Tests/LinearSolvers/C_CellMG/./main3d.intel.MPI.OMP.ex.ipm inputs.3d.25600 - # start : Tue Aug 15 17:34:23 2017 host : nid11311 + # command : /global/cscratch1/sd/cchan2/projects/lbl/BoxLib/Tests/LinearSolvers/C_CellMG/./main3d.intel.MPI.OMP.ex.ipm inputs.3d.25600 + # start : Tue Aug 15 17:34:23 2017 host : nid11311 # stop : Tue Aug 15 17:34:35 2017 wallclock : 11.54 # mpi_tasks : 128 on 32 nodes %comm : 32.51 # mem [GB] : 126.47 gflop/sec : 0.00 # # : [total] min max - # wallclock : 1188.42 9.28 8.73 11.54 - # MPI : 386.31 3.02 2.51 4.78 + # wallclock : 1188.42 9.28 8.73 11.54 + # MPI : 386.31 3.02 2.51 4.78 # %wall : - # MPI : 32.52 24.36 41.44 + # MPI : 32.52 24.36 41.44 # #calls : # MPI : 5031172 39306 23067 57189 - # mem [GB] : 126.47 0.99 0.98 1.00 + # mem [GB] : 126.47 0.99 0.98 1.00 # # [time] [count] <%wall> # MPI_Allreduce 225.72 567552 18.99 @@ -252,7 +252,7 @@ generated figures. Some examples are shown here. .. table:: Example of performance graphs generated by IPM :align: center - + +-----------------------------------------------------+------------------------------------------------------+ | |a| | |b| | +-----------------------------------------------------+------------------------------------------------------+ @@ -301,7 +301,7 @@ that is viewed in the Nsight Systems GUI, typically on a local workstation or ma To generate a qdrep file, run nsys with the ``-o`` option: -.. highlight:: console +.. highlight:: console :: @@ -379,7 +379,7 @@ Nsight Systems GUI Tips * When analyzing an AMReX application in the Nsight Systems GUI using NVTX regions or ``TINY_PROFILE=TRUE``, AMReX users may find it useful to turn on the feature "Rename CUDA Kernels by NVTX". This will change the CUDA kernel names to match the inner-most NVTX region in which they were launched instead of the typical - mangled compiler name. This will make identifying AMReX CUDA kernels in Nsight Systems reports considerably easier. + mangled compiler name. This will make identifying AMReX CUDA kernels in Nsight Systems reports considerably easier. This feature can be found in the GUI's drop down menu, under: @@ -411,7 +411,7 @@ The standard way to run Nsight Compute on an AMReX application is to specify an that will be transferred to a local workstation of machine for viewing in the Nsight Compute GUI. Nsight Compute can be told to return a report file using the ``-o`` flag. In addition, when running with Nsight compute on an AMReX application, it is important to turn off the floating -point exception trap, as it causes a runtime error. So, an entire AMReX application can be +point exception trap, as it causes a runtime error. So, an entire AMReX application can be analyzed with Nsight Compute by running: .. highlight:: console @@ -433,7 +433,7 @@ through the ``--nvtx``, ``--nvtx-include`` and ``--nvtx-exclude`` flags. For exa ncu --nvtx --nvtx-include "Hydro()" --nvtx-exclude "StencilA(),StencilC()" -o kernels ${EXE} ${INPUTS} amrex.fpe_trap_invalid=0 will return a file named ``kernels`` which contains an analysis of the CUDA kernels launched inside -the ``Hydro()`` region, ignoring any kernels launched inside ``StencilA()`` and ``StencilC()``. +the ``Hydro()`` region, ignoring any kernels launched inside ``StencilA()`` and ``StencilC()``. When using the NVTX regions built into AMReX's TinyProfiler, be aware that the application must be built with ``TINY_PROFILE=TRUE`` and the NVTX region names are identical to the TinyProfiler timer names. @@ -472,9 +472,9 @@ analysis will be critical to achieve results within a reasonable time. For examp ncu --section SpeedOfLight_RooflineChart --nvtx --nvtx-include "MLMG()" -c 10 -o roofline ${EXE} ${INPUTS} amrex.fpe_trap_invalid=0 will perform a roofline analysis of the first ten kernels inside of the region ``MLMG()``, and report -their relative performance in the file ``roofline``, which can be read by the Nsight Compute GUI. +their relative performance in the file ``roofline``, which can be read by the Nsight Compute GUI. -For further information on the roofline model, refer to the scientific literature, `Wikipedia +For further information on the roofline model, refer to the scientific literature, `Wikipedia overview `_, NERSC -`documentation `_ and -`tutorials `_. +`documentation `_ and +`tutorials `_. diff --git a/Docs/sphinx_documentation/source/Fortran.rst b/Docs/sphinx_documentation/source/Fortran.rst index 9631374e53b..51129c21ff5 100644 --- a/Docs/sphinx_documentation/source/Fortran.rst +++ b/Docs/sphinx_documentation/source/Fortran.rst @@ -18,7 +18,7 @@ functionality, the "AmrCore" directory wraps around the :cpp:`AmrCore` class support for octree type of AMR grids. Each directory has a "Make.package" file that can be included in make files (see ``amrex/Tutorials/Basic/HelloWorld_F`` and ``amrex/Tutorials/Amr/Advection_F`` for examples). The libamrex approach includes the -Fortran interface by default. +Fortran interface by default. A simple example can be found at ``amrex/Tutorials/Basic/HelloWorld_F/``. The source code is shown below in its entirety. @@ -176,7 +176,7 @@ There are many type-bound procedures for :fortran:`amrex_multifab`. For example ncomp ! Return the number of components nghost ! Return the number of ghost cells - setval ! Set the data to the given value + setval ! Set the data to the given value copy ! Copy data from given amrex_multifab to this amrex_multifab Note that the copy function here only works on copying data from another @@ -275,7 +275,7 @@ example, ! Then mf2 becomes a shallow copy of mf1. ! mf1 is still the owner of the data. call amrex_multifab_destroy(mf1) - ! mf2 no longer contains a valid pointer because mf1 has been destroyed. + ! mf2 no longer contains a valid pointer because mf1 has been destroyed. call amrex_multifab_destroy(mf2) ! But we still need to destroy it. If we need to transfer the ownership, :fortran:`amrex_multifab`, @@ -313,7 +313,7 @@ infrastructure. With AMR, the main program might look like below, program main use amrex_amr_module - implicit none + implicit none call amrex_init() call amrex_amrcore_init() call my_amr_init() ! user's own code, not part of AMReX @@ -364,13 +364,13 @@ interfaces: real(amrex_real), intent(in), value :: time type(c_ptr), intent(in), value :: ba, dm end subroutine amrex_make_level_proc - + subroutine amrex_clear_level_proc (lev) bind(c) import implicit none integer, intent(in) , value :: lev end subroutine amrex_clear_level_proc - + subroutine amrex_error_est_proc (lev, tags, time, tagval, clearval) bind(c) import implicit none diff --git a/Docs/sphinx_documentation/source/GPU.rst b/Docs/sphinx_documentation/source/GPU.rst index 84685ee7614..4669e7cae17 100644 --- a/Docs/sphinx_documentation/source/GPU.rst +++ b/Docs/sphinx_documentation/source/GPU.rst @@ -677,8 +677,8 @@ implementation is reproduced here: :: Real MultiFab::Dot (const MultiFab& x, int xcomp, - const MultiFab& y, int ycomp, - int numcomp, int nghost, bool local) { + const MultiFab& y, int ycomp, + int numcomp, int nghost, bool local) { Real sm = amrex::ReduceSum(x, y, nghost, [=] AMREX_GPU_HOST_DEVICE (Box const& bx, FArrayBox const& xfab, FArrayBox const& yfab) -> Real { @@ -688,7 +688,7 @@ implementation is reproduced here: if (!local) ParallelAllReduce::Sum(sm, ParallelContext::CommunicatorSub()); return sm; - } + } :cpp:`amrex::ReduceSum` takes two :cpp:`MultiFab`\ s, ``x`` and ``y`` and returns the sum of the value returned from the given lambda function. @@ -1598,9 +1598,9 @@ Basic Gpu Debugging :: - Gpu::setLaunchRegion(0); - ... ; - Gpu::setLaunchRegion(1); + Gpu::setLaunchRegion(0); + ... ; + Gpu::setLaunchRegion(1); Note that functions, ``amrex::launch`` and ``amrex::ParallelFor``, do not respect the launch region flag. Only the macros (e.g., @@ -1613,7 +1613,7 @@ Cuda-specific tests :: - nvprof ./main3d.xxx + nvprof ./main3d.xxx - Run under ``nvprof -o profile%p.nvvp ./main3d.xxxx`` for a small problem and examine page faults using nvvp diff --git a/Docs/sphinx_documentation/source/GPU_Chapter.rst b/Docs/sphinx_documentation/source/GPU_Chapter.rst index 0a8e38fabcd..9f18eb8eff6 100644 --- a/Docs/sphinx_documentation/source/GPU_Chapter.rst +++ b/Docs/sphinx_documentation/source/GPU_Chapter.rst @@ -3,15 +3,15 @@ GPU === -In this chapter, we will present the GPU support in AMReX. AMReX targets +In this chapter, we will present the GPU support in AMReX. AMReX targets NVIDIA, AMD and Intel GPUs using their native vendor language and therefore requires CUDA, HIP/ROCm and DPC++/SYCL, for NVIDIA, AMD and Intel GPUs, respectively. Users can also use OpenMP and/or OpenACC in their applications. AMReX supports NVIDIA GPUs with compute capability >= 6 and CUDA >= 10 -as well as CUDA 9.1. While HIP and DPC++ compilers are in development in -preparation for Frontier and Aurora, AMReX only supports the latest -publically released versions of those compilers on the Iris and Tulip testbeds. +as well as CUDA 9.1. While HIP and DPC++ compilers are in development in +preparation for Frontier and Aurora, AMReX only supports the latest +publically released versions of those compilers on the Iris and Tulip testbeds. For complete details of CUDA, HIP, DPC++, OpenMP and OpenACC languages, see their respective documentations. diff --git a/Docs/sphinx_documentation/source/GettingStarted.rst b/Docs/sphinx_documentation/source/GettingStarted.rst index 534f798dc27..6d33f0a025a 100644 --- a/Docs/sphinx_documentation/source/GettingStarted.rst +++ b/Docs/sphinx_documentation/source/GettingStarted.rst @@ -184,7 +184,7 @@ and then taking the divergence to update the cells, .. math:: - \phi_{i,\,j}^{n+1} = \phi_{i,\,j}^n + \phi_{i,\,j}^{n+1} = \phi_{i,\,j}^n + \frac{\Delta t}{\Delta x}\left(F_{i+^1\!/_2,\,j}-F_{i-^1\!/_2,\,j}\right) + \frac{\Delta t}{\Delta y}\left(F_{i,\,j+^1\!/_2}-F_{i,\,j-^1\!/_2}\right) diff --git a/Docs/sphinx_documentation/source/GridCreation.rst b/Docs/sphinx_documentation/source/GridCreation.rst index c718e7edacc..1b8336b164a 100644 --- a/Docs/sphinx_documentation/source/GridCreation.rst +++ b/Docs/sphinx_documentation/source/GridCreation.rst @@ -10,53 +10,53 @@ Grid Creation ------------- To run an AMReX-based application you must specifiy the domain size by -specifying :cpp:`n_cell` -- this is the number of cells spanning the domain +specifying :cpp:`n_cell` -- this is the number of cells spanning the domain in each coordinate direction at level 0. -Users often specify :cpp:`max_grid_size` as well. The default load balancing algorithm then divides the +Users often specify :cpp:`max_grid_size` as well. The default load balancing algorithm then divides the domain in every direction so that each grid is no longer than :cpp:`max_grid_size` in that direction. If not specified by the user, :cpp:`max_grid_size` defaults to 128 in 2D and 32 in 3D (in each coordinate direction). -Another popular input is :cpp:`blocking_factor`. The value of :cpp:`blocking_factor` -constrains grid creation in that in that each grid must be divisible by :cpp:`blocking_factor`. +Another popular input is :cpp:`blocking_factor`. The value of :cpp:`blocking_factor` +constrains grid creation in that in that each grid must be divisible by :cpp:`blocking_factor`. Note that both the domain (at each level) and :cpp:`max_grid_size` must be divisible by :cpp:`blocking_factor`, and that :cpp:`blocking_factor` must be either 1 or a power of 2 (otherwise the gridding algorithm would not in fact create grids divisible by :cpp:`blocking_factor` because of how :cpp:`blocking_factor` is used in the gridding algorithm). If not specified by the user, :cpp:`blocking_factor` defaults to 8 in each coordinate direction. -The typical purpose of :cpp:`blocking_factor` is to ensure that the grids will be +The typical purpose of :cpp:`blocking_factor` is to ensure that the grids will be sufficiently coarsenable for good multigrid performance. -There is one more default behavior to be aware of. There is a boolean :cpp:`refine_grid_layout` +There is one more default behavior to be aware of. There is a boolean :cpp:`refine_grid_layout` that defaults to true but can be over-ridden at run-time. -If :cpp:`refine_grid_layout` is true and the number of grids created is less than the number of processors +If :cpp:`refine_grid_layout` is true and the number of grids created is less than the number of processors (Ngrids < Nprocs), then grids will be further subdivided until Ngrids >= Nprocs. -Caveat: if subdividing the grids to achieve Ngrids >= Nprocs would violate the -:cpp:`blocking_factor` criterion then additional grids are not created and the +Caveat: if subdividing the grids to achieve Ngrids >= Nprocs would violate the +:cpp:`blocking_factor` criterion then additional grids are not created and the number of grids will remain less than the number of processors Note that :cpp:`n_cell` must be given as three separate integers, one for each coordinate direction. -However, :cpp:`max_grid_size` and :cpp:`blocking_factor` can be specified as a single value -applying to all coordinate directions, or as separate values for each direction. +However, :cpp:`max_grid_size` and :cpp:`blocking_factor` can be specified as a single value +applying to all coordinate directions, or as separate values for each direction. - - If :cpp:`max_grid_size` (or :cpp:`blocking_factor`) is specified as multiple integers then the first + - If :cpp:`max_grid_size` (or :cpp:`blocking_factor`) is specified as multiple integers then the first integer applies to level 0, the second to level 1, etc. If you don't specify as many integers as there are levels, the final value will be used for the remaining levels. - - If different values of :cpp:`max_grid_size` (or :cpp:`blocking_factor`) are wanted for each coordinate direction, - then :cpp:`max_grid_size_x`, :cpp:`max_grid_size_y` and :cpp:`max_grid_size_z` - (or :cpp:`blocking_factor_x`, :cpp:`blocking_factor_y` and :cpp:`blocking_factor_z`) must be used. + - If different values of :cpp:`max_grid_size` (or :cpp:`blocking_factor`) are wanted for each coordinate direction, + then :cpp:`max_grid_size_x`, :cpp:`max_grid_size_y` and :cpp:`max_grid_size_z` + (or :cpp:`blocking_factor_x`, :cpp:`blocking_factor_y` and :cpp:`blocking_factor_z`) must be used. If you don't specify as many integers as there are levels, the final value will be used for the remaining levels. Additional notes: - - To create identical grids of a specific size, e.g. of length *m* in each direction, + - To create identical grids of a specific size, e.g. of length *m* in each direction, then set :cpp:`max_grid_size` = *m* and :cpp:`blocking_factor` = *m*. - - Note that :cpp:`max_grid_size` is just an upper bound; with :cpp:`n_cell = 48` + - Note that :cpp:`max_grid_size` is just an upper bound; with :cpp:`n_cell = 48` and :cpp:`max_grid_size = 32`, we will typically have one grid of length 32 and one of length 16. The grid creation process at level 0 proceeds as follows (if not using the KD-tree approach): @@ -73,10 +73,10 @@ The grid creation process at level 0 proceeds as follows (if not using the KD-tr (unless doing so would violate the :cpp:`blocking_factor` criterion). The creation of grids at levels > 0 begins by tagging cells at the coarser level and follows -the Berger-Rigoutsis clustering algorithm with the additional constraints of satisfying -the :cpp:`blocking_factor` and :cpp:`max_grid_size` criteria. An additional parameter -becomes relevant here: the "grid efficiency", specified as :cpp:`amr.grid_eff` in the inputs file. -This threshhold value, which defaults to 0.7 (or 70%), is used to ensure that +the Berger-Rigoutsis clustering algorithm with the additional constraints of satisfying +the :cpp:`blocking_factor` and :cpp:`max_grid_size` criteria. An additional parameter +becomes relevant here: the "grid efficiency", specified as :cpp:`amr.grid_eff` in the inputs file. +This threshhold value, which defaults to 0.7 (or 70%), is used to ensure that grids do not contain too large a fraction of un-tagged cells. We note that the grid creation process attempts to satisfy the :cpp:`amr.grid_eff` constraint but will not do so if it means violating the :cpp:`blocking_factor` criterion. @@ -84,7 +84,7 @@ violating the :cpp:`blocking_factor` criterion. Users often like to ensure that coarse/fine boundaries are not too close to tagged cells; the way to do this is to set :cpp:`amr.n_error_buf` to a large integer value (the default is 1). This parameter is used to increase the number of tagged cells before the grids are defined; -if cell "*(i,j,k)*" satisfies the tagging criteria, then, for example, if :cpp:`amr.n_error_buf` is 3, -all cells in the 7x7x7 box from lower corner "*(i-3,j-3,k-3)*" to "*(i+3,j+3,k+3)*" will be tagged. +if cell "*(i,j,k)*" satisfies the tagging criteria, then, for example, if :cpp:`amr.n_error_buf` is 3, +all cells in the 7x7x7 box from lower corner "*(i-3,j-3,k-3)*" to "*(i+3,j+3,k+3)*" will be tagged. diff --git a/Docs/sphinx_documentation/source/IO.rst b/Docs/sphinx_documentation/source/IO.rst index 8bce21ceb37..5002c7d22ee 100644 --- a/Docs/sphinx_documentation/source/IO.rst +++ b/Docs/sphinx_documentation/source/IO.rst @@ -10,7 +10,7 @@ Plotfile ======== AMReX has its own native plotfile format. Many visualization tools are -available for AMReX plotfiles (see the chapter on :ref:`Chap:Visualization`). +available for AMReX plotfiles (see the chapter on :ref:`Chap:Visualization`). AMReX provides the following two functions for writing a generic AMReX plotfile. Many AMReX application codes may have their own plotfile routines that store additional information such as compiler options, git hashes of the @@ -52,7 +52,7 @@ making such strings. const std::string& pfname = amrex::Concatenate("plt",istep); // plt00258 // By default there are 5 digits, but we can change it to say 4. - const std::string& pfname2 = amrex::Concatenate("plt",istep,4); // plt0258 + const std::string& pfname2 = amrex::Concatenate("plt",istep,4); // plt0258 istep =1234567; // Having more than 5 digits is OK. const std::string& pfname3 = amrex::Concatenate("plt",istep); // plt1234567 @@ -106,7 +106,7 @@ However we have provided an example restart capability in the tutorial Refer to the functions :cpp:`ReadCheckpointFile()` and :cpp:`WriteCheckpointFile()` in this tutorial. -A checkpoint file is actually a directory with name, e.g., +A checkpoint file is actually a directory with name, e.g., ``chk00010`` containing a ``Header`` (text) file, along with subdirectories ``Level_0``, ``Level_1``, etc. containing the :cpp:`MultiFab` data at each level of refinement. @@ -183,7 +183,7 @@ can also be used. For example, HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); std::string HeaderFileName(checkpointname + "/Header"); HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | - std::ofstream::trunc | + std::ofstream::trunc | std::ofstream::binary); if( ! HeaderFile.good()) { @@ -306,7 +306,7 @@ read the file from the disk and broadcast it to others as } } -The following code how to read in a :cpp:`BoxArray`, create a +The following code how to read in a :cpp:`BoxArray`, create a :cpp:`DistributionMapping`, build :cpp:`MultiFab` and :cpp:`FluxRegister` data, and read in a :cpp:`MultiFab` from a checkpoint file, on a level-by-level basis: diff --git a/Docs/sphinx_documentation/source/InputsLoadBalancing.rst b/Docs/sphinx_documentation/source/InputsLoadBalancing.rst index fbde3f3b922..d7db65dab5e 100644 --- a/Docs/sphinx_documentation/source/InputsLoadBalancing.rst +++ b/Docs/sphinx_documentation/source/InputsLoadBalancing.rst @@ -18,13 +18,13 @@ The following inputs must be preceded by "amr" and determine how we create the g | max_grid_size_z | Maximum number of cells at level 0 in each grid in z-direction | Int | 32 | +-------------------+-----------------------------------------------------------------------+-------------+-----------+ | blocking_factor_x | Each grid must be divisible by blocking_factor_x in x-direction | Int | 8 | -| | (must be 1 or power of 2) | | | +| | (must be 1 or power of 2) | | | +-------------------+-----------------------------------------------------------------------+-------------+-----------+ | blocking_factor_y | Each grid must be divisible by blocking_factor_y in y-direction | Int | 8 | -| | (must be 1 or power of 2) | | | +| | (must be 1 or power of 2) | | | +-------------------+-----------------------------------------------------------------------+-------------+-----------+ | blocking_factor_z | Each grid must be divisible by blocking_factor_z in z-direction | Int | 8 | -| | (must be 1 or power of 2) | | | +| | (must be 1 or power of 2) | | | +-------------------+-----------------------------------------------------------------------+-------------+-----------+ The following inputs must be preceded by "particles" diff --git a/Docs/sphinx_documentation/source/Inputs_Chapter.rst b/Docs/sphinx_documentation/source/Inputs_Chapter.rst index 4944c187bb8..0a64aeb492c 100644 --- a/Docs/sphinx_documentation/source/Inputs_Chapter.rst +++ b/Docs/sphinx_documentation/source/Inputs_Chapter.rst @@ -7,8 +7,8 @@ Run-time Inputs :maxdepth: 1 InputsProblemDefinition - InputsTimeStepping - InputsLoadBalancing - InputsPlotFiles + InputsTimeStepping + InputsLoadBalancing + InputsPlotFiles InputsCheckpoint diff --git a/Docs/sphinx_documentation/source/Introduction.rst b/Docs/sphinx_documentation/source/Introduction.rst index d627355f810..89d12d0adf2 100644 --- a/Docs/sphinx_documentation/source/Introduction.rst +++ b/Docs/sphinx_documentation/source/Introduction.rst @@ -30,4 +30,4 @@ Key features of AMReX include: - Plotfile format supported by AmrVis, VisIt, ParaView, and yt. AMReX is developed at LBNL, NREL, and ANL as part of the Block-Structured AMR -Co-Design Center in DOE's Exascale Computing Project. +Co-Design Center in DOE's Exascale Computing Project. diff --git a/Docs/sphinx_documentation/source/LinearSolvers.rst b/Docs/sphinx_documentation/source/LinearSolvers.rst index 78c44528ebf..cd8c678831a 100644 --- a/Docs/sphinx_documentation/source/LinearSolvers.rst +++ b/Docs/sphinx_documentation/source/LinearSolvers.rst @@ -170,14 +170,14 @@ The supported BC types at the physical domain boundaries are - :cpp:`LinOpBCType::reflect_odd` for reflection with sign changed. -2) Cell-centered solvers only: -if we want to do a linear solve where the boundary conditions on the +2) Cell-centered solvers only: +if we want to do a linear solve where the boundary conditions on the coarsest AMR level of the solve come from a coarser level (e.g. the -base AMR level of the solve is > 0 and does not cover the entire domain), -we must explicitly provide the coarser data. Boundary conditions from a -coarser level are always Dirichlet. +base AMR level of the solve is > 0 and does not cover the entire domain), +we must explicitly provide the coarser data. Boundary conditions from a +coarser level are always Dirichlet. -Note that this step, if needed, must be performed before the step below. +Note that this step, if needed, must be performed before the step below. The :cpp:`MLLinOp` member function for this step is .. highlight:: c++ @@ -189,12 +189,12 @@ The :cpp:`MLLinOp` member function for this step is Here :cpp:`const MultiFab* crse` contains the Dirichlet boundary values at the coarse resolution, and :cpp:`int crse_ratio` (e.g., 2) is the refinement ratio between the coarsest solver level and the AMR -level below it. The MultiFab crse does not need to have ghost cells itself. -If the coarse grid bc's for the solve are identically zero, :cpp:`nullptr` +level below it. The MultiFab crse does not need to have ghost cells itself. +If the coarse grid bc's for the solve are identically zero, :cpp:`nullptr` can be passed instead of :cpp:`crse`. -3) Cell-centered solvers only: -before the solve one must always call the :cpp:`MLLinOp` member function +3) Cell-centered solvers only: +before the solve one must always call the :cpp:`MLLinOp` member function .. highlight:: c++ @@ -202,16 +202,16 @@ before the solve one must always call the :cpp:`MLLinOp` member function virtual void setLevelBC (int amrlev, const MultiFab* levelbcdata) = 0; -If we want to supply any inhomogeneous Dirichlet or Neumann boundary -conditions at the domain boundaries, we must supply those values -in ``MultiFab* levelbcdata``, which must have at least one ghost cell. +If we want to supply any inhomogeneous Dirichlet or Neumann boundary +conditions at the domain boundaries, we must supply those values +in ``MultiFab* levelbcdata``, which must have at least one ghost cell. Note that the argument :cpp:`amrlev` is relative to the solve, not necessarily the full AMR hierarchy; amrlev = 0 refers to the coarsest level of the solve. If the boundary condition is Dirichlet the ghost cells outside the domain boundary of ``levelbcdata`` must hold the value of the solution -at the domain boundary; +at the domain boundary; if the boundary condition is Neumann those ghost cells must hold the value of the gradient of the solution normal to the boundary (e.g. it would hold dphi/dx on both the low and high faces in the x-direction). @@ -221,11 +221,11 @@ we can pass :cpp:`nullptr` instead of a MultiFab. We can use the solution array itself to hold these values; the values are copied to internal arrays and will not be over-written -when the solution array itself is being updated by the solver. +when the solution array itself is being updated by the solver. Note, however, that this call does not provide an initial guess for the solve. -It should be emphasized that the data in ``levelbcdata`` for -Dirichlet or Neumann boundaries are assumed to be exactly on the face +It should be emphasized that the data in ``levelbcdata`` for +Dirichlet or Neumann boundaries are assumed to be exactly on the face of the physical domain; storing these values in the ghost cell of a cell-centered array is a convenience of implementation. @@ -292,10 +292,10 @@ Available choices are if cg fails. The matrix must be symmetric. - :cpp:`MLMG::BottomSolver::hypre`: One of the solvers available through hypre; - see the section below on External Solvers + see the section below on External Solvers - :cpp:`MLMG::BottomSolver::petsc`: Currently for cell-centered only. - + - :cpp:`LPInfo::setAgglomeration(bool)` (by default true) can be used continue to coarsen the multigrid by copying what would have been the bottom solver to a new :cpp:`MultiFab` with a new :cpp:`BoxArray` with @@ -318,12 +318,12 @@ We have the option using the :cpp:`MLMG` member method void setMaxOrder (int maxorder); -to set the order of the cell-centered linear operator stencil at physical boundaries +to set the order of the cell-centered linear operator stencil at physical boundaries with Dirichlet boundary conditions and at coarse-fine boundaries. In both of these -cases, the boundary value is not defined at the center of the ghost cell. +cases, the boundary value is not defined at the center of the ghost cell. The order determines the number of interior cells that are used in the extrapolation -of the boundary value from the cell face to the center of the ghost cell, where -the extrapolated value is then used in the regular stencil. For example, +of the boundary value from the cell face to the center of the ghost cell, where +the extrapolated value is then used in the regular stencil. For example, :cpp:`maxorder = 2` uses the boundary value and the first interior value to extrapolate to the ghost cell center; :cpp:`maxorder = 3` uses the boundary value and the first two interior values. @@ -344,8 +344,8 @@ Embedded Boundaries AMReX supports multi-level solvers for use with embedded boundaries. These include -1) cell-centered solvers with homogeneous Neumann, homogeneous Dirichlet, -or inhomogeneous Dirichlet boundary conditions on the EB faces, and +1) cell-centered solvers with homogeneous Neumann, homogeneous Dirichlet, +or inhomogeneous Dirichlet boundary conditions on the EB faces, and 2) nodal solvers with homogeneous Neumann boundary conditions on the EB faces. To use a cell-centered solver with EB, one builds a linear operator @@ -389,9 +389,9 @@ where phi_on_eb is the MultiFab holding the Dirichlet values in every cut cell, and coeff again is a real number (i.e. the value is the same at every cell) or a MultiFab holding the coefficient of the gradient at each cell with an EB face. -Currently there are options to define the face-based coefficients on +Currently there are options to define the face-based coefficients on face centers vs face centroids, and to interpret the solution variable -as being defined on cell centers vs cell centroids. +as being defined on cell centers vs cell centroids. The default is for the solution variable to be defined at cell centers; to tell the solver to interpret the solution variable as living @@ -417,7 +417,7 @@ External Solvers ================ AMReX provides interfaces to the `hypre `_ preconditioners and solvers, including BoomerAMG, GMRES (all variants), PCG, and BICGStab as -solvers, and BoomerAMG and Euclid as preconditioners. These can be called as +solvers, and BoomerAMG and Euclid as preconditioners. These can be called as as bottom solvers for both cell-centered and node-based problems. If it is built with Hypre support, AMReX initializes Hypre by default in @@ -440,18 +440,18 @@ residual correction form of the original problem. To build Hypre, follow the nex 1.- git clone https://github.com/hypre-space/hypre.git 2.- cd hypre/src 3.- ./configure - (if you want to build hypre with long long int, do ./configure --enable-bigint ) + (if you want to build hypre with long long int, do ./configure --enable-bigint ) 4.- make install 5.- Create an environment variable with the HYPRE directory -- HYPRE_DIR=/hypre_path/hypre/src/hypre -To use hypre, one must include ``amrex/Src/Extern/HYPRE`` in the build system. +To use hypre, one must include ``amrex/Src/Extern/HYPRE`` in the build system. For examples of using hypre, we refer the reader to ``Tutorials/LinearSolvers/ABecLaplacian_C`` or ``Tutorials/LinearSolvers/NodalProjection_EB``. -Caveat: to use hypre for the nodal solver, you must either build with USE_EB = TRUE, +Caveat: to use hypre for the nodal solver, you must either build with USE_EB = TRUE, or explicitly set the coarsening strategy in the calling routine to be ``RAP`` rather than ``Sigma`` -by adding +by adding .. highlight:: c++ @@ -463,7 +463,7 @@ where :cpp:`nodal_projector` is the :cpp:`NodalProjector` object we have built. The following parameter should be set to True if the problem to be solved has a singular matrix. -In this case, the solution is only defined to within a constant. Setting this parameter to True +In this case, the solution is only defined to within a constant. Setting this parameter to True replaces one row in the matrix sent to hypre from AMReX by a row that sets the value at one cell to 0. - :cpp:`hypre.adjust_singular_matrix`: Default is False. @@ -517,7 +517,7 @@ problems. To build PETSc, follow the next steps: 1.- git clone https://github.com/petsc/petsc.git 2.- cd petsc 3.- ./configure --download-hypre=yes --prefix=build_dir - 4.- Follow the steps given by petsc + 4.- Follow the steps given by petsc 5.- Create an environment variable with the PETSC directory -- PETSC_DIR=/petsc_path/petsc/build_dir @@ -528,45 +528,45 @@ reader to ``Tutorials/LinearSolvers/ABecLaplacian_C``. MAC Projection ========================= -Some codes define a velocity field :math:`U = (u,v,w)` on faces, i.e. +Some codes define a velocity field :math:`U = (u,v,w)` on faces, i.e. :math:`u` is defined on x-faces, :math:`v` is defined on y-faces, -and :math:`w` is defined on z-faces. We refer to the exact projection -of this velocity field as a MAC projection, in which we solve +and :math:`w` is defined on z-faces. We refer to the exact projection +of this velocity field as a MAC projection, in which we solve .. math:: D( \beta \nabla \phi) = D(U^*) - S -for :math:`\phi` and then set +for :math:`\phi` and then set .. math:: U = U^* - \beta \nabla \phi -where :math:`U^*` is a vector field (typically velocity) that we want to satisfy +where :math:`U^*` is a vector field (typically velocity) that we want to satisfy :math:`D(U) = S`. For incompressible flow, :math:`S = 0`. The MacProjection class can be defined and used to perform the MAC projection without explicitly calling the solver directly. In addition to solving the variable coefficient Poisson equation, the MacProjector internally computes the divergence of the vector field, :math:`D(U^*)`, to compute the right-hand-side, and after the solve, subtracts the weighted gradient term to -make the vector field result satisfy the divergence constraint. +make the vector field result satisfy the divergence constraint. In the simplest form of the call, :math:`S` is assumed to be zero and does not need to be specified. Typically, the user does not allocate the solution array, but it is also possible to create and pass -in the solution array and have :math:`\phi` returned as well as :math:`U`. +in the solution array and have :math:`\phi` returned as well as :math:`U`. Caveat: Currently the MAC projection only works when the base level covers the full domain; it does not yet have the interface to pass boundary conditions for a fine level that come from coarser data. Also note that any Dirichlet or Neumann boundary conditions at domain boundaries -are assumed to be homogeneous. The call to the :cpp:`MLLinOp` member function +are assumed to be homogeneous. The call to the :cpp:`MLLinOp` member function :cpp:`setLevelBC` occurs inside the MacProjection class; one does not need to call that explicitly when using the MacProjection class. -The code below is taken from -``Tutorials/LinearSolvers/MAC_Projection_EB/main.cpp`` and demonstrates how to set up +The code below is taken from +``Tutorials/LinearSolvers/MAC_Projection_EB/main.cpp`` and demonstrates how to set up the MACProjector object and use it to perform a MAC projection. .. highlight:: c++ @@ -580,16 +580,16 @@ the MACProjector object and use it to perform a MAC projection. vel[idim].define (amrex::convert(grids,IntVect::TheDimensionVector(idim)), dmap, 1, 1, MFInfo(), factory); beta[idim].define(amrex::convert(grids,IntVect::TheDimensionVector(idim)), dmap, 1, 0, - MFInfo(), factory); + MFInfo(), factory); beta[idim].setVal(1.0); // set beta to 1 } - // If we want to use phi elsewhere, we must create an array in which to return the solution + // If we want to use phi elsewhere, we must create an array in which to return the solution // MultiFab phi_inout(grids, dmap, 1, 1, MFInfo(), factory); // If we want to supply a non-zero S we must allocate and fill it outside the solver // MultiFab S(grids, dmap, 1, 0, MFInfo(), factory); - // Set S here ... + // Set S here ... // set initial velocity to U=(1,0,0) AMREX_D_TERM(vel[0].setVal(1.0);, @@ -614,13 +614,13 @@ the MACProjector object and use it to perform a MAC projection. // lp_info, // structure for passing info to the operator // {&S}); // defines the specified RHS divergence - // Set bottom-solver to use hypre instead of native BiCGStab - if (use_hypre_as_full_solver || use_hypre_as_bottom_solver) + // Set bottom-solver to use hypre instead of native BiCGStab + if (use_hypre_as_full_solver || use_hypre_as_bottom_solver) macproj.setBottomSolver(MLMG::BottomSolver::hypre); // Set boundary conditions. // Here we use Neumann on the low x-face, Dirichlet on the high x-face, - // and periodic in the other two directions + // and periodic in the other two directions // (the first argument is for the low end, the second is for the high end) // Note that Dirichlet and Neumann boundary conditions are assumed to be homogeneous. macproj.setDomainBC({AMREX_D_DECL(LinOpBCType::Neumann, @@ -644,7 +644,7 @@ the MACProjector object and use it to perform a MAC projection. // at face centers (MLMG::Location::FaceCenter) or face centroids (MLMG::Location::FaceCentroid) macproj.project(reltol,abstol,MLMG::Location::FaceCenter); - // If we want to use phi elsewhere, we can pass in an array in which to return the solution + // If we want to use phi elsewhere, we can pass in an array in which to return the solution // macproj.project({&phi_inout},reltol,abstol,MLMG::Location::FaceCenter); See ``Tutorials/LinearSolvers/MAC_Projection_EB`` for the complete working example. @@ -653,19 +653,19 @@ Nodal Projection ================ Some codes define a velocity field :math:`U = (u,v,w)` with all -components co-located on cell centers. The nodal solver in AMReX +components co-located on cell centers. The nodal solver in AMReX can be used to compute an approximate projection of the cell-centered velocity field, with pressure and velocity divergence defined on nodes. When we use the nodal solver this way, and subtract only the cell average of the gradient from the velocity, it is effectively an approximate projection. -As with the MAC projection, consider that we want to solve +As with the MAC projection, consider that we want to solve .. math:: D( \beta \nabla \phi) = D(U^*) - S -for :math:`\phi` and then set +for :math:`\phi` and then set .. math:: @@ -684,7 +684,7 @@ gradient term to make the vector field result satisfy the divergence constraint. .. highlight:: c++ :: - + // // Given a cell-centered velocity (vel) field, a cell-centered // scalar field (sigma) field, and a source term S (either node- @@ -695,15 +695,15 @@ gradient term to make the vector field result satisfy the divergence constraint. // and then perform the projection: // // vel = vel - sigma * grad(phi) - // + // // // Create the EB factory - // + // EBFArrayBoxFactory factory(eb_level, geom, grids, dmap, ng_ebs, ebs); // - // Create the cell-centered velocity field we want to project + // Create the cell-centered velocity field we want to project // MultiFab vel(grids, dmap, AMREX_SPACEDIM, 1, MFInfo(), factory); @@ -713,7 +713,7 @@ gradient term to make the vector field result satisfy the divergence constraint. // // Setup linear operator, AKA the nodal Laplacian - // + // LPInfo lp_info; // If we want to use hypre to solve the full problem we do not need to coarsen the GMG stencils @@ -740,14 +740,14 @@ gradient term to make the vector field result satisfy the divergence constraint. matrix.setHarmonicAverage(false); // - // Compute RHS + // Compute RHS // // NOTE: it's up to the user to compute the RHS. as opposed // to the MAC projection case !!! // // NOTE: do this operation AFTER setting up the linear operator so // that compRHS method can be used - // + // // RHS is nodal const BoxArray & nd_grids = amrex::convert(grids, IntVect{1,1,1}); // nodal grids @@ -793,9 +793,9 @@ gradient term to make the vector field result satisfy the divergence constraint. nodal_solver.setVerbose(mg_verbose); nodal_solver.setBottomVerbose(mg_bottom_verbose); - // Set bottom-solver to use hypre instead of native BiCGStab + // Set bottom-solver to use hypre instead of native BiCGStab // ( we could also have set this to cg, bicgcg, cgbicg) - // if (use_hypre_as_full_solver || use_hypre_as_bottom_solver) + // if (use_hypre_as_full_solver || use_hypre_as_bottom_solver) // nodal_solver.setBottomSolver(MLMG::BottomSolver::hypre); // Define the relative tolerance @@ -811,7 +811,7 @@ gradient term to make the vector field result satisfy the divergence constraint. // // Create cell-centered MultiFab to hold value of -sigma*grad(phi) at cell-centers - // + // // MultiFab fluxes(grids, dmap, AMREX_SPACEDIM, 1, MFInfo(), factory); fluxes.setVal(0.0); @@ -820,8 +820,8 @@ gradient term to make the vector field result satisfy the divergence constraint. nodal_solver.getFluxes( {&fluxes} ); // - // Apply projection explicitly -- vel = vel - sigma * grad(phi) - // + // Apply projection explicitly -- vel = vel - sigma * grad(phi) + // MultiFab::Add( *vel, *fluxes, 0, 0, AMREX_SPACEDIM, 0); See ``Tutorials/LinearSolvers/Nodal_Projection_EB`` for the complete working example. @@ -852,17 +852,17 @@ and in 3-d Cartesian component form as ( (\eta w_x)_x + (\eta w_y)_y + (\eta w_z)_z ) + ( (\eta u_z)_x + (\eta v_z)_y + (\eta w_z)_z ) + ( (\kappa - \frac{2}{3} \eta) (u_x+v_y+w_z) )_z -Here :math:`\eta` is the dynamic viscosity and :math:`\kappa` is the bulk viscosity. +Here :math:`\eta` is the dynamic viscosity and :math:`\kappa` is the bulk viscosity. We evaluate the following terms from the above using the ``MLABecLaplacian`` and ``MLEBABecLaplacian`` operators; .. math:: - ( (\frac{4}{3} \eta + \kappa) u_x)_x + ( \eta u_y)_y + (\eta u_z)_z + ( (\frac{4}{3} \eta + \kappa) u_x)_x + ( \eta u_y)_y + (\eta u_z)_z - (\eta v_x)_x + ( (\frac{4}{3} \eta + \kappa) v_y)_y + (\eta v_z)_z + (\eta v_x)_x + ( (\frac{4}{3} \eta + \kappa) v_y)_y + (\eta v_z)_z - (\eta w_x)_x + ( \eta w_y)_y + ( (\frac{4}{3} \eta + \kappa) w_z)_z + (\eta w_x)_x + ( \eta w_y)_y + ( (\frac{4}{3} \eta + \kappa) w_z)_z the following cross-terms are evaluated separately using the ``MLTensorOp`` and ``MLEBTensorOp`` operators. @@ -977,22 +977,22 @@ An example (implemented in the ``MultiComponent`` tutorial) might be: .. _fig::refluxfreecoarsefine: .. figure:: ./LinearSolvers/refluxfreecoarsefine.png - :height: 2cm - :align: center - - : Reflux-free coarse-fine boundary update. - Level 2 ghost nodes (small dark blue) are interpolated from coarse boundary. - Level 1 ghost nodes are updated during the relaxation along with all the other interior fine nodes. - Coarse nodes (large blue) on the coarse/fine boundary are updated by restricting with interior nodes - and the first level of ghost nodes. - Coarse nodes underneath level 2 ghost nodes are not updated. - The remaining coarse nodes are updates by restriction. - + :height: 2cm + :align: center + + : Reflux-free coarse-fine boundary update. + Level 2 ghost nodes (small dark blue) are interpolated from coarse boundary. + Level 1 ghost nodes are updated during the relaxation along with all the other interior fine nodes. + Coarse nodes (large blue) on the coarse/fine boundary are updated by restricting with interior nodes + and the first level of ghost nodes. + Coarse nodes underneath level 2 ghost nodes are not updated. + The remaining coarse nodes are updates by restriction. + The MC nodal operator can inherit from the ``MCNodeLinOp`` class. ``Fapply``, ``Fsmooth``, and ``Fflux`` must update level 1 ghost nodes that are inside the domain. `interpolation` and `restriction` can be implemented as usual. `reflux` is a straightforward restriction from fine to coarse, using level 1 ghost nodes for restriction as described above. - + See ``Tutorials/LinearSolvers/MultiComponent`` for a complete working example. .. solver reuse diff --git a/Docs/sphinx_documentation/source/LinearSolvers_Chapter.rst b/Docs/sphinx_documentation/source/LinearSolvers_Chapter.rst index e421128bbeb..c7a81e0a7b5 100644 --- a/Docs/sphinx_documentation/source/LinearSolvers_Chapter.rst +++ b/Docs/sphinx_documentation/source/LinearSolvers_Chapter.rst @@ -8,7 +8,7 @@ Linear Solvers AMReX supports both single-level solves and composite solves on multiple AMR levels, with the scalar solution to the linear system defined on either cell centers or nodes. -AMReX also supports solution of linear systems with embedded boundaries. +AMReX also supports solution of linear systems with embedded boundaries. (See chapter :ref:`Chap:EB` for more details on the embedded boundary representation of complex geometry.) @@ -21,23 +21,23 @@ that solve linear systems in the canonical form .. math:: (A \alpha - B \nabla \cdot \beta \nabla ) \phi = f, :label: eqn::abeclap -where :math:`A` and :math:`B` are scalar constants, +where :math:`A` and :math:`B` are scalar constants, :math:`\alpha` and :math:`\beta` are scalar fields, -:math:`\phi` is the unknown, +:math:`\phi` is the unknown, and :math:`f` is the right-hand side of the equation. Note that Poisson's equation :math:`\nabla^2 \phi = f` is a special case of the canonical form. The solution :math:`\phi` is at either -cell centers or nodes. +cell centers or nodes. -For the cell-centered solver, :math:`\alpha`, :math:`\phi` and :math:`f` +For the cell-centered solver, :math:`\alpha`, :math:`\phi` and :math:`f` are represented by cell-centered MultiFabs, and :math:`\beta` is represented by ``AMREX_SPACEDIM`` face type -MultiFabs, i.e. there are separate MultiFabs for the :math:`\beta` +MultiFabs, i.e. there are separate MultiFabs for the :math:`\beta` coefficient in each coordinate direction. -For the nodal solver, :math:`A` and :math:`\alpha` are assumed to be zero, +For the nodal solver, :math:`A` and :math:`\alpha` are assumed to be zero, :math:`\phi` and :math:`f` are nodal, -and :math:`\beta` (whcih we later refer to as :math:`\sigma`) is cell-centered. +and :math:`\beta` (whcih we later refer to as :math:`\sigma`) is cell-centered. In addition to these solvers, AMReX has support for tensor solves used to calculate the viscous terms that appear in the compressible Navier-Stokes diff --git a/Docs/sphinx_documentation/source/LoadBalancing.rst b/Docs/sphinx_documentation/source/LoadBalancing.rst index fa384ea1659..3ef142714be 100644 --- a/Docs/sphinx_documentation/source/LoadBalancing.rst +++ b/Docs/sphinx_documentation/source/LoadBalancing.rst @@ -9,13 +9,13 @@ Load Balancing -------------- -The process of load balancing is typically independent of the process of grid creation; -the inputs to load balancing are a given set of grids with a set of weights +The process of load balancing is typically independent of the process of grid creation; +the inputs to load balancing are a given set of grids with a set of weights assigned to each grid. (The exception to this is the KD-tree approach in which the grid creation process is governed by trying to balance the work in each grid.) -Single-level load balancing algorithms are sequentially applied to each AMR level independently, -and the resulting distributions are mapped onto the ranks taking into account the weights +Single-level load balancing algorithms are sequentially applied to each AMR level independently, +and the resulting distributions are mapped onto the ranks taking into account the weights already assigned to them (assign heaviest set of grids to the least loaded rank). Note that the load of each process is measured by how much memory has already been allocated, not how much memory will be allocated. Therefore the following code is not recommended because it tends to generate @@ -45,11 +45,11 @@ Instead, one should do, Distribution options supported by AMReX include the following; the default is SFC: -- Knapsack: the default weight of a grid in the knapsack algorithm is the number of grid cells, - but AMReX supports the option to pass an array of weights -- one per grid -- or alternatively +- Knapsack: the default weight of a grid in the knapsack algorithm is the number of grid cells, + but AMReX supports the option to pass an array of weights -- one per grid -- or alternatively to pass in a MultiFab of weights per cell which is used to compute the weight per grid. -- SFC: enumerate grids with a space-filling Z-morton curve, then partition the +- SFC: enumerate grids with a space-filling Z-morton curve, then partition the resulting ordering across ranks in a way that balances the load. - Round-robin: sort grids and assign them to ranks in round-robin fashion -- specifically diff --git a/Docs/sphinx_documentation/source/ManagingGridHierarchy_Chapter.rst b/Docs/sphinx_documentation/source/ManagingGridHierarchy_Chapter.rst index 61fb62364ef..68c09b9cbea 100644 --- a/Docs/sphinx_documentation/source/ManagingGridHierarchy_Chapter.rst +++ b/Docs/sphinx_documentation/source/ManagingGridHierarchy_Chapter.rst @@ -4,7 +4,7 @@ Gridding and Load Balancing =========================== -AMReX provides a great deal of generality when it comes to how to decompose the +AMReX provides a great deal of generality when it comes to how to decompose the computational domain into individual logically rectangular grids, and how to distribute those grids to MPI ranks. We use the phrase "load balancing" here to refer to the combined process of grid creation (and re-creation when regridding) and distribution of grids to MPI ranks. @@ -13,24 +13,24 @@ Even for single-level calculations, AMReX provides the flexibility to have diffe more than one grid per MPI rank, and different strategies for distributing the grids to MPI ranks. For multi-level calculations, the same principles for load balancing apply as in single-level calculations, -but there is additional complexity in how to tag cells for refinement and how to create the +but there is additional complexity in how to tag cells for refinement and how to create the union of grids at levels > 0 where that union most likely does not cover the computational domain. -See :ref:`sec:grid_creation` for grids are created, i.e. how the :cpp:`BoxArray` on which +See :ref:`sec:grid_creation` for grids are created, i.e. how the :cpp:`BoxArray` on which :cpp:`MultiFabs` will be built is defined at each level. See :ref:`sec:load_balancing` for the strategies AMReX supports for distributing -grids to MPI ranks, i.e. defining the :cpp:`DistributionMapping` with which -:cpp:`MultiFabs` at that level will be built. +grids to MPI ranks, i.e. defining the :cpp:`DistributionMapping` with which +:cpp:`MultiFabs` at that level will be built. -We also note that we can create separate grids, and map them in different ways to MPI ranks, for +We also note that we can create separate grids, and map them in different ways to MPI ranks, for different types of data in a single calculation. We refer to this as the "dual grid approach" and the most common usage is to load balance mesh and particle data separately. See :ref:`sec:dual_grid` for more about this approach. -When running on multicore machines with OpenMP, we can also control the distribution of -work by setting the size of grid tiles (by defining :cpp:`fabarray_mfiter.tile_size`), and if relevant, of -particle tiles (by defining :cpp:`particle.tile_size`). We can also specify the strategy for assigning +When running on multicore machines with OpenMP, we can also control the distribution of +work by setting the size of grid tiles (by defining :cpp:`fabarray_mfiter.tile_size`), and if relevant, of +particle tiles (by defining :cpp:`particle.tile_size`). We can also specify the strategy for assigning tiles to OpenMP threads. See :ref:`sec:basics:mfiter:tiling` for more about tiling. .. toctree:: diff --git a/Docs/sphinx_documentation/source/Particle.rst b/Docs/sphinx_documentation/source/Particle.rst index 4d9746242b2..d120720b557 100644 --- a/Docs/sphinx_documentation/source/Particle.rst +++ b/Docs/sphinx_documentation/source/Particle.rst @@ -40,7 +40,7 @@ flags. Our particle struct would be set up like: Particle<4, 2> p; -and the order of the particle components in would be (assuming :cpp:`BL_SPACEDIM` is 3): +and the order of the particle components in would be (assuming :cpp:`BL_SPACEDIM` is 3): :cpp:`x y z m vx vy vz id cpu flag1 flag2`. [3]_ Setting Particle data @@ -291,7 +291,7 @@ Adding particle components at runtime In addition to the components specified as template parameters, you can also add additional :cpp:`Real` and :cpp:`int` components at runtime. These components -will be stored in Struct-of-Array style. To add a runtime component, use the +will be stored in Struct-of-Array style. To add a runtime component, use the :cpp:`AddRealComp` and :cpp:`AddIntComp` methods of :cpp:`ParticleContainer`, like so: .. highlight:: c++ @@ -316,8 +316,8 @@ When you are using runtime components, it is crucial that when you are adding particles to the container, you call the :cpp:`DefineAndReturnParticleTile` method for each tile prior to adding any particles. This will make sure the space for the new components has been allocated. For example, in the above section -on :ref:`initializing particle data `, we accessed -the particle tile data using the :cpp:`GetParticles` method. If we runtime components +on :ref:`initializing particle data `, we accessed +the particle tile data using the :cpp:`GetParticles` method. If we runtime components are used, :cpp:`DefineAndReturnParticleTile` should be used instead: .. highlight:: c++ @@ -330,7 +330,7 @@ are used, :cpp:`DefineAndReturnParticleTile` should be used instead: // instead of this... // auto& particles = GetParticles(lev)[std::make_pair(mfi.index(), // mfi.LocalTileIndex())]; - + // we do this... auto& particle_tile = DefineAndReturnParticleTile(lev, mfi); @@ -625,8 +625,8 @@ mesh data IO. For example: will create a plot file called "plt00000" and write the mesh data in :cpp:`output` to it, and then write the particle data in a subdirectory called "particle0". There is also the :cpp:`WriteAsciiFile` method, which writes the particles in a human-readable text format. This is mainly useful for testing and debugging. -The binary file format is currently readable by :cpp:`yt`. In additional, there is a Python conversion script in -``amrex/Tools/Py_util/amrex_particles_to_vtp`` that can convert both the ASCII and the binary particle files to a +The binary file format is currently readable by :cpp:`yt`. In additional, there is a Python conversion script in +``amrex/Tools/Py_util/amrex_particles_to_vtp`` that can convert both the ASCII and the binary particle files to a format readable by Paraview. See the chapter on :ref:`Chap:Visualization` for more information on visualizing AMReX datasets, including those with particles. Inputs parameters @@ -636,7 +636,7 @@ Inputs parameters There are several runtime parameters users can set in their :cpp:`inputs` files that control the behavior of the AMReX particle classes. These are summarized below. They should be preceded by -"particles" in your inputs deck. +"particles" in your inputs deck. The first set of parameters concerns the tiling capability of the ParticleContainer. If you are seeing poor performance with OpenMP, the first thing to look at is whether there are enough tiles available for each thread to work on. @@ -645,7 +645,7 @@ with OpenMP, the first thing to look at is whether there are enough tiles availa | | Description | Type | Default | +===================+=======================================================================+=============+=============+ | do_tiling | Whether to use tiling for particles. Should be on when using OpenMP, | Bool | False | -| | and off when running on GPUs. | | | +| | and off when running on GPUs. | | | +-------------------+-----------------------------------------------------------------------+-------------+-------------+ | tile_size | If tiling is on, the maximum tile_size to in each direction | Ints | 1024000,8,8 | +-------------------+-----------------------------------------------------------------------+-------------+-------------+ @@ -654,7 +654,7 @@ The next set concerns runtime parameters that control the particle IO. Parallel too many MPI tasks touch the disk at once. Additionally, performance can degrade if all MPI tasks try writing to the same file, or if too many small files are created. In general, the "correct" values of these parameters will depend on the size of your problem (i.e., number of boxes, number of MPI tasks), as well as the system you are using. If you are experiencing -problems with particle IO, you could try varying some / all of these parameters. +problems with particle IO, you could try varying some / all of these parameters. +-------------------+-----------------------------------------------------------------------+-------------+-------------+ | | Description | Type | Default | diff --git a/Docs/sphinx_documentation/source/SWFFT.rst b/Docs/sphinx_documentation/source/SWFFT.rst index 697ee168931..3b8e77ed8b8 100644 --- a/Docs/sphinx_documentation/source/SWFFT.rst +++ b/Docs/sphinx_documentation/source/SWFFT.rst @@ -7,8 +7,8 @@ .. role:: underline :class: underline -.. _swfftdoc: - +.. _swfftdoc: + SWFFT ======= @@ -53,7 +53,7 @@ The following figures illustrate how data is distributed from block structured g .. table:: SWFFT Redistribution from :math:`4 \times 4 \times 4` Box Array into Pencils :align: center - + +---------------------------------------------------------+------------------------------------------------------+ | |a| | |b| | +---------------------------------------------------------+------------------------------------------------------+ diff --git a/Docs/sphinx_documentation/source/Testing.rst b/Docs/sphinx_documentation/source/Testing.rst index 58ee2420a7d..5b6f32363a7 100644 --- a/Docs/sphinx_documentation/source/Testing.rst +++ b/Docs/sphinx_documentation/source/Testing.rst @@ -9,7 +9,7 @@ AMReX as a library for a common set of configuration options. This operation is through Travis-CI. This layer of testing is deliberately limited, so that it can be run quickly on every commit. For more extensive testing, we rely on the nightly regression results. - + Nightly Regression Testing ========================== @@ -45,7 +45,7 @@ likely want to change include: to control where the generated output will be written, and :: - + [AMReX] dir = /path/to/amrex # the path to the amrex repository you want to test branch = "development" @@ -57,20 +57,20 @@ The test runner is a Python script and can be invoked like so: :: python regtest.py AMReX-Tests.ini - + Before you can use it, you must first generate a set of "benchmarks" - i.e. known "good" answers to the tests that will be run. If you are testing a pull request, you can generate these by running the script with the a recent version of the :cpp:`development` branch of AMReX. You can generate the benchmarks like so: :: - + python regtest.py --make_benchmarks 'generating initial benchmarks' AMReX-Tests.ini Once that is finished, you can switch over to the branch you want to test in :cpp:`AMReX-Tests.ini`, and then re-run the script without the :cpp:`--make_benchmarks` option: :: - + python regtest.py --make_benchmarks 'generating initial benchmarks' AMReX-Tests.ini The script will generate a set of html pages in the directory specified in your :cpp:`AMReX-Tests.ini` @@ -83,15 +83,15 @@ For a complete set of script options, run python regtest.py --help A particularly useful option lets you run just a subset of the complete test suite. To run only one test, you can do: - + :: - + python regtest.py --single_test AMReX-Tests.ini To run an enumerated list of tests, do: - + :: - + python regtest.py --tests ' ' AMReX-Tests.ini @@ -103,7 +103,7 @@ do is start from an existing test and modify it. For example, this entry: :: - [MLMG_FI_PoisCom] + [MLMG_FI_PoisCom] buildDir = Tutorials/LinearSolvers/ABecLaplacian_F inputFile = inputs-rt-poisson-com dim = 3 diff --git a/Docs/sphinx_documentation/source/Visualization.rst b/Docs/sphinx_documentation/source/Visualization.rst index 56c9982d8e3..e563816887d 100644 --- a/Docs/sphinx_documentation/source/Visualization.rst +++ b/Docs/sphinx_documentation/source/Visualization.rst @@ -209,9 +209,9 @@ then select movie.visit. Create an image to your liking and press the the movie, choose "File" :math:`\rightarrow` "Save movie ...", and follow the on-screen instructions. -Caveat: +Caveat: -The Visit reader determines "Cycle" from the name of the plotfile (directory), +The Visit reader determines "Cycle" from the name of the plotfile (directory), specifically from the integer that follows the string "plt" in the plotfile name. So ... if you call it plt00100 or myplt00100 or this_is_my_plt00100 then it will @@ -220,7 +220,7 @@ correctly recognize and print Cycle: 100. If you call it plt00100_old it will also correctly recognize and print Cycle: 100 But, if you do not have "plt" followed immediately by the number, -e.g. you name it pltx00100, then VisIt will not be able to correctly recognize +e.g. you name it pltx00100, then VisIt will not be able to correctly recognize and print the value for "Cycle". (It will still read and display the data itself.) .. _section-1: diff --git a/Docs/sphinx_documentation/source/index.rst b/Docs/sphinx_documentation/source/index.rst index 14d2722949e..fed94168042 100644 --- a/Docs/sphinx_documentation/source/index.rst +++ b/Docs/sphinx_documentation/source/index.rst @@ -57,7 +57,7 @@ Documentation on migration from BoxLib is available in the AMReX repository at D External_Profiling_Tools_Chapter External_Frameworks_Chapter Regression_Testing_Chapter - + .. toctree:: :maxdepth: 1 :caption: API diff --git a/Docs/sphinx_tutorials/source/AMR_Tutorial.rst b/Docs/sphinx_tutorials/source/AMR_Tutorial.rst index dfdecaec29b..8d1702423ce 100644 --- a/Docs/sphinx_tutorials/source/AMR_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/AMR_Tutorial.rst @@ -7,7 +7,7 @@ Tutorials/Amr ============= -For each of these tutorials, +For each of these tutorials, plotfiles are generated that can be viewed with amrvis2d / amrvis3d (CCSE's native vis / spreadsheet tool, downloadable separately from ccse.lbl.gov) or with VisIt. @@ -29,7 +29,7 @@ directories. Notably, this example does not use source code from amrex/Src/Amr (see the tutorial Advection_AmrLevel). -The directory Exec includes a makefile and a sample inputs file. +The directory Exec includes a makefile and a sample inputs file. **Advection_AmrLevel** ---------------------- @@ -42,8 +42,8 @@ It is an AMReX based code designed to run in parallel using MPI/OMP. This example uses source code from the amrex/Src/Base, Boundary, AmrCore, and Amr directories. -The directories Exec/SingleVortex and Exec/UniformVelocity each include -a makefile and a sample inputs file. +The directories Exec/SingleVortex and Exec/UniformVelocity each include +a makefile and a sample inputs file. **Advection_F** ---------------- @@ -53,7 +53,7 @@ field that is specified on faces. It is a AMReX based code designed to run in parallel using MPI/OMP. It uses the Fortran interfaces of AMReX. -The directory Exec/SingleVortex includes a makefile and a sample inputs file. +The directory Exec/SingleVortex includes a makefile and a sample inputs file. **Advection_octree_F** ---------------------- @@ -65,4 +65,4 @@ It is a AMReX based code designed to run in parallel using MPI/OMP. It uses the Fortran interfaces of AMReX. The grids have an octree structure with a grid size of 8. No subcycling is used. -The directory Exec/SingleVortex includes a makefile and a sample inputs file. +The directory Exec/SingleVortex includes a makefile and a sample inputs file. diff --git a/Docs/sphinx_tutorials/source/Basic_Tutorial.rst b/Docs/sphinx_tutorials/source/Basic_Tutorial.rst index 9688e7a98b6..1380f29c8d4 100644 --- a/Docs/sphinx_tutorials/source/Basic_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/Basic_Tutorial.rst @@ -7,7 +7,7 @@ Tutorials/Basic ========================== -The tutorials in amrex/Tutorials/Basic demonstrate the most fundamental +The tutorials in amrex/Tutorials/Basic demonstrate the most fundamental operations supported by AMReX. **HelloWorld** @@ -17,12 +17,12 @@ HelloWorld_C and HelloWorld_F demonstrate the GNU Make system -- with a sample Make.package and GNUmakefile -- and the amrex::Initialize and amrex::Finalize functions. -In addition, in HelloWorld_C, the amrex::Print() operation, -which only prints from the I/O processor, is used to print out -the AMReX version (as defined by amrex::Version()) being used. +In addition, in HelloWorld_C, the amrex::Print() operation, +which only prints from the I/O processor, is used to print out +the AMReX version (as defined by amrex::Version()) being used. HelloWorld_F is a simple example of how to use the F_Interface routines, -which are Fortran wrappers for the underlying C++ data strutures and +which are Fortran wrappers for the underlying C++ data strutures and iterators. Here, for example, rather than calling amrex::Print() in C++, we test on whether amrex_parallel_ioprocessor() is true, and if so, invoke the usual Fortran print call. @@ -54,11 +54,11 @@ main_C and main_F introduce the following: **Build_with_libamrex** ----------------------- -This tutorial builds on the main_C example and demonstrates how to build the executable when we +This tutorial builds on the main_C example and demonstrates how to build the executable when we want to link local files with the pre-built amrex library (libamrex.a) that has been installed elsewhere. -We separate main.cpp from the main_C example into two separate files (main.cpp and -test_parameters.cpp), replace MyAmr.H by MyParams.H and add a Fortran file my_func.f90. -The GNUmakefile here assumes that you have already built the AMReX library; for instructions on how to do +We separate main.cpp from the main_C example into two separate files (main.cpp and +test_parameters.cpp), replace MyAmr.H by MyParams.H and add a Fortran file my_func.f90. +The GNUmakefile here assumes that you have already built the AMReX library; for instructions on how to do that see the Building_libamrex_ chapter. .. Building_libamrex: https://amrex-codes.github.io/amrex/docs_html/BuildingAMReX.html diff --git a/Docs/sphinx_tutorials/source/Blueprint_Tutorial.rst b/Docs/sphinx_tutorials/source/Blueprint_Tutorial.rst index 3ab89cc25ec..01c96735269 100644 --- a/Docs/sphinx_tutorials/source/Blueprint_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/Blueprint_Tutorial.rst @@ -10,7 +10,7 @@ Tutorials/Blueprint These tests, ``AssignMultiLevelDensity`` and ``HeatEquation_EX1_C``, demonstrate how to convert AMReX Mesh data into an in-memory Conduit Mesh Blueprint description for consumption by the ALPINE Ascent -in situ visualization and analysis tool. These are variants, respectively, of +in situ visualization and analysis tool. These are variants, respectively, of ``amrex/Tests/Particles/AssignMultiLevelDensity`` and ``amrex/Tutorials/Basic/HeatEquation_EX1_C``. diff --git a/Docs/sphinx_tutorials/source/GPU_Tutorial.rst b/Docs/sphinx_tutorials/source/GPU_Tutorial.rst index c85402e3838..9040c883e1e 100644 --- a/Docs/sphinx_tutorials/source/GPU_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/GPU_Tutorial.rst @@ -9,7 +9,7 @@ Tutorials/GPU The tutorials in ``amrex/Tutorials/GPU`` demonstrate the implementation of AMReX's GPU toolkit as well as provide GPU ported versions of CPU -tutorials to help applications convert to GPUs. +tutorials to help applications convert to GPUs. **Your first AMReX GPU application** ------------------------------------ @@ -18,7 +18,7 @@ This is a step-by-step guide to preparing, compiling and running your first AMReX GPU program. This guide will use ``Tutorials/GPU/Launch``, and instructions will focus on ORNL's systems: -1. Before compiling, the ``pgi`` and ``cuda`` software must be available. +1. Before compiling, the ``pgi`` and ``cuda`` software must be available. On ORNL systems, the modules can be loaded directly by typing: .. highlight:: console @@ -30,25 +30,25 @@ and instructions will focus on ORNL's systems: 2. Go to ``Tutorials/GPU/Launch`` to compile the executable. Compile with ``make USE_CUDA=TRUE COMP=pgi USE_MPI=TRUE USE_OMP=FALSE``, or edit the ``GNUmakefile`` to match this configuration and run ``make``. This - should result in an executable: ``main3d.pgi.MPI.CUDA.ex``. + should result in an executable: ``main3d.pgi.MPI.CUDA.ex``. 3. On Summit systems, this executable can be submitted by using one of the run scripts found in ``Tutorials/GPU``. ``run.script`` can be used to run on Summitdev, and ``run.summit`` can be used for Summit. To change the number of ranks and GPUs used in the simulation, change the number of resource sets, ``n`` in the ``jsrun`` line. For the first ``Launch`` tutorial, use ``n=1`` - and set ``INPUTS=""`` because no input file is used in this example. + and set ``INPUTS=""`` because no input file is used in this example. When ready, submit the job script (on Summit: ``bsub run.script``). -Congratulations! You've accelerated AMReX using GPUs! +Congratulations! You've accelerated AMReX using GPUs! **Launch** ---------- Launch shows multiple examples of how GPU work can be offloaded using the tools available in AMReX. It includes examples of multiple AMReX macro launch methods, -launching a Fortran function using CUDA and launching work using OpenACC and -OpenMP offloading. This tutorial will be regularly updated with AMReX's +launching a Fortran function using CUDA and launching work using OpenACC and +OpenMP offloading. This tutorial will be regularly updated with AMReX's preferred GPU launch methodologies. **CNS** diff --git a/Docs/sphinx_tutorials/source/LinearSolvers_Tutorial.rst b/Docs/sphinx_tutorials/source/LinearSolvers_Tutorial.rst index 3b88bb98280..b9ce54f5b4f 100644 --- a/Docs/sphinx_tutorials/source/LinearSolvers_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/LinearSolvers_Tutorial.rst @@ -7,15 +7,15 @@ Tutorials/LinearSolvers ========================== -There are five examples in the Tutorials/LinearSolvers directory. +There are five examples in the Tutorials/LinearSolvers directory. ``ABecLaplacian_C`` demonstrates how to solve with cell-centered data in a C++ framework. -This example shows how to use either hypre or PETSc as a bottom-solver (or to solve +This example shows how to use either hypre or PETSc as a bottom-solver (or to solve the equation at the finest level if you set the "max coarsening level" to 0. ``ABecLaplacian_F`` demonstrates how to solve with cell-centered data using the Fortran interfaces. -``NodalPoisson`` demonstrates how to set up and solve a variable coefficient Poisson equation +``NodalPoisson`` demonstrates how to set up and solve a variable coefficient Poisson equation with the rhs and solution data on nodes. ``MAC_Projection_EB`` demonstrates how to set up and perform an EB-aware MAC projection @@ -26,4 +26,4 @@ of a specified cell-centered velocity field around nine cylindrical objects. ``MultiComponent`` demonstrates how to solve using a multi-component operator with nodal data. The operator is of the form :math:`D(\mathbf{\phi})_i = \alpha_{ij}\nabla^2\phi_j`, where :math:`\mathbf{\phi}` is the multi-component solution variable and :math:`\alpha_{ij}` is a matrix of coefficients. -The operator (``MCNodalLinOp``) is implemented using the "reflux-free" method. +The operator (``MCNodalLinOp``) is implemented using the "reflux-free" method. diff --git a/Docs/sphinx_tutorials/source/MUI_Tutorial.rst b/Docs/sphinx_tutorials/source/MUI_Tutorial.rst index 397a46990db..ddbac9bf4c9 100644 --- a/Docs/sphinx_tutorials/source/MUI_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/MUI_Tutorial.rst @@ -11,7 +11,7 @@ The goal of this tutorial is to incorporate the MxUI/MUI (Multiscale Universal I In this tutorial, two different C++ codes are built separately. Each has different spatial dimensions: one is built in 3D (``AMREX_SPACEDIM = 3``), and the other in 2D (``AMREX_SPACEDIM = 2``). Each code is compiled separately within their respective "exec" directories ``Exec_01`` & ``Exec_02``}, after which the two executables are run together using the following command, specifying the number of MPI processes to designate to each executable: -.. highlight:: console +.. highlight:: console :: @@ -23,11 +23,11 @@ on a single line within the ``Exec_coupled`` directory. ``N1`` and ``n2`` are th The figure below shows one possible grid structure of the 2D (red grid) and 3D (multicolored blocks) setup. .. |a| image:: ./SWFFT/iface_rect.png - :width: 60% + :width: 60% .. table:: :align: center - + +------------------------------------------------------+ | |a| | +------------------------------------------------------+ diff --git a/Docs/sphinx_tutorials/source/Particles_Tutorial.rst b/Docs/sphinx_tutorials/source/Particles_Tutorial.rst index 2639d085a68..d5fd1e8866c 100644 --- a/Docs/sphinx_tutorials/source/Particles_Tutorial.rst +++ b/Docs/sphinx_tutorials/source/Particles_Tutorial.rst @@ -7,26 +7,26 @@ Tutorials/Particles ========================== -There are several tutorials in amrex/Tutorials/Particles that demonstrate the basic usage of -AMReX's particle data structures. +There are several tutorials in amrex/Tutorials/Particles that demonstrate the basic usage of +AMReX's particle data structures. **ElectrostaticPIC** -------------------- This tutorial demonstrates how to perform an electrostatic Particle-in-Cell calculation -using AMReX. The code initializes a single particle in a conducting box (i.e. Dirichlet -zero boundary conditions) that is slightly off-center in one direction. Because of the +using AMReX. The code initializes a single particle in a conducting box (i.e. Dirichlet +zero boundary conditions) that is slightly off-center in one direction. Because of the boundary conditions, the particle sees an image charge and is accelerated in this direction. The code is currently set up to use one level of static mesh refinement. The charge density, -electric field, and electrostatic potential are all defined on the mesh nodes. To solve +electric field, and electrostatic potential are all defined on the mesh nodes. To solve Poisson's equation, we use AMReX's Fortran-based multigrid solver. The Fortran routines for -performing charge deposition, field gathering, and the particle push are all defined in +performing charge deposition, field gathering, and the particle push are all defined in :cpp:`electrostatic_pic_2d.f90` and :cpp:`electrostatic_pic_3d.f90` for 2D and 3D, respectively. The particle container in this example using a Struct-of-Arrays layout, with :cpp:`1 + 2*BL_SPACEDIM` -real components to store the particle weight, velocity, and the electric field interpolated -to the particle position. To see how to set up such a particle container, see +real components to store the particle weight, velocity, and the electric field interpolated +to the particle position. To see how to set up such a particle container, see :cpp:`ElectrostaticParticleContainer.H`. **ElectromagneticPIC** @@ -40,16 +40,16 @@ time domain method. This tutorial also demonstrates how to offload calculations involving particle data onto the GPU using OpenACC. To compile with GPU support, use the pgi compiler, and set -:cpp:`USE_ACC = TRUE`, and :cpp:`USE_CUDA = TRUE`, :cpp:`USE_OMP = FALSE`. +:cpp:`USE_ACC = TRUE`, and :cpp:`USE_CUDA = TRUE`, :cpp:`USE_OMP = FALSE`. You can choose between two problem types by toggling the :cpp:`problem_type` parameter in the provided inputs file. Choosing the uniform plasma setup provides a nearly perfectly load balanced problem setup that is useful for performance testing. Choosing the Langmuir wave problem will automatically compare the simulated fields to the exact solution. - -Currently, this tutorial does not use mesh refinement. - + +Currently, this tutorial does not use mesh refinement. + **NeighborList** ---------------- @@ -62,8 +62,8 @@ defined in :cpp:`neighbor_list_2d.f90` and :cpp:`neighbor_list_3d.f90` demonstra the resulting data structure. The particles in this example store velocity and acceleration in addition to the default -components. They are initially placed at cell centers and given random velocities. When a -particle reaches the domain boundary, it is specularly reflected back into the domain. To +components. They are initially placed at cell centers and given random velocities. When a +particle reaches the domain boundary, it is specularly reflected back into the domain. To see how the particle data structures are set up, see :cpp:`NeighborListParticleContainer.cpp`. The file called :cpp:`inputs` can be used to run this tutorial with a single level, and @@ -79,6 +79,6 @@ potentially interact particles that are in the same cell as each other. Every ti whether it's still in the same cell or not. If it isn't, we mark the particle as unsorted. We then call Redistribute() as normal, and then insert the unsorted particles into the proper cells. Care is taken so that, if the Redistribute call changes the order of the particles in the Container, the indices -in the cell lists are updated accordingly. +in the cell lists are updated accordingly. This Tutorial is currently single-level only. diff --git a/Docs/sphinx_tutorials/source/index.rst b/Docs/sphinx_tutorials/source/index.rst index b8e01abc7a0..1d2217b6635 100644 --- a/Docs/sphinx_tutorials/source/index.rst +++ b/Docs/sphinx_tutorials/source/index.rst @@ -11,11 +11,11 @@ applications. AMReX is freely available at https://github.com/AMReX-Codes/amrex. There is extensive documentation for AMReX at :ref:`amrex_docs:amrex_doc_indx` AMReX Tutorials are a set of small stand-alone example codes that demonstrate how -to use different parts of the AMReX functionality. +to use different parts of the AMReX functionality. -We are always happy to have users contribute to AMReX Tutorials as well as -the AMReX source code. To contribute, issue a pull request against the development branch -(details at https://help.github.com/articles/creating-a-pull-request/). +We are always happy to have users contribute to AMReX Tutorials as well as +the AMReX source code. To contribute, issue a pull request against the development branch +(details at https://help.github.com/articles/creating-a-pull-request/). The amrex/Tutorials directory is broken into the following categories: diff --git a/README.md b/README.md index 816c5be9d2d..525306be6a7 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Citing](http://joss.theoj.org/papers/10.21105/joss.01370/status.svg)](https://doi.org/10.21105/joss.01370) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2555438.svg)](https://doi.org/10.5281/zenodo.2555438) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.2555438.svg)](https://doi.org/10.5281/zenodo.2555438) ![CI: CMake on development](https://github.com/AMReX-codes/amrex/workflows/cmake/badge.svg?branch=development) ![CI: Travis on development](https://img.shields.io/travis/AMReX-codes/amrex/development) diff --git a/Src/Amr/AMReX_Amr.H b/Src/Amr/AMReX_Amr.H index af7c8ef912b..f3bce8c8057 100644 --- a/Src/Amr/AMReX_Amr.H +++ b/Src/Amr/AMReX_Amr.H @@ -213,9 +213,9 @@ public: Real coarseTimeStepDt (Real stop_time); //! Retrieve derived data. User is responsible for deleting pointer. std::unique_ptr derive (const std::string& name, - Real time, - int lev, - int ngrow); + Real time, + int lev, + int ngrow); //! Name of the restart chkpoint file. const std::string& theRestartFile () const noexcept { return restart_chkfile; } //! Name of the restart plotfile. @@ -369,13 +369,13 @@ protected: // pure virtural function in AmrCore virtual void MakeNewLevelFromScratch (int /*lev*/, Real /*time*/, const BoxArray& /*ba*/, const DistributionMapping& /*dm*/) override - { amrex::Abort("How did we get here!"); } + { amrex::Abort("How did we get here!"); } virtual void MakeNewLevelFromCoarse (int /*lev*/, Real /*time*/, const BoxArray& /*ba*/, const DistributionMapping& /*dm*/) override - { amrex::Abort("How did we get here!"); } + { amrex::Abort("How did we get here!"); } virtual void RemakeLevel (int /*lev*/, Real /*time*/, const BoxArray& /*ba*/, const DistributionMapping& /*dm*/) override - { amrex::Abort("How did we get here!"); } + { amrex::Abort("How did we get here!"); } virtual void ClearLevel (int /*lev*/) override - { amrex::Abort("How did we get here!"); } + { amrex::Abort("How did we get here!"); } //! Whether to write a plotfile now bool writePlotNow () noexcept; diff --git a/Src/Amr/AMReX_Amr.cpp b/Src/Amr/AMReX_Amr.cpp index ab4ed5a34a3..d50087faf24 100644 --- a/Src/Amr/AMReX_Amr.cpp +++ b/Src/Amr/AMReX_Amr.cpp @@ -278,7 +278,7 @@ Amr::InitAmr () pp.query("regrid_file" , regrid_grids_file); pp.query("message_int", message_int); - + if (pp.contains("run_log")) { std::string log_file_name; @@ -304,7 +304,7 @@ Amr::InitAmr () datalog.resize(num_datalogs); datalogname.resize(num_datalogs); pp.queryarr("data_log",datalogname,0,num_datalogs); - for (int i = 0; i < num_datalogs; i++) + for (int i = 0; i < num_datalogs; i++) setRecordDataInfo(i,datalogname[i]); } @@ -337,7 +337,7 @@ Amr::InitAmr () // for (int i = 0; i < nlev; i++) { - + // Something nonzero so old & new will differ #ifdef AMREX_USE_FLOAT dt_level[i] = 1.e30f; @@ -351,18 +351,18 @@ Amr::InitAmr () } // Make the default regrid_int = 1 for all levels. - if (max_level > 0) + if (max_level > 0) { regrid_int.resize(max_level); for (int i = 0; i < max_level; i++) regrid_int[i] = 1; } - + // // Setup plot and checkpoint controls. // initPltAndChk(); - + // // Setup insitu controls // @@ -376,7 +376,7 @@ Amr::InitAmr () // // Read in the regrid interval if max_level > 0. // - if (max_level > 0) + if (max_level > 0) { int numvals = pp.countval("regrid_int"); if (numvals == 1) @@ -401,7 +401,7 @@ Amr::InitAmr () { amrex::Error("You did not specify enough values of regrid_int"); } - else + else { // // Otherwise we expect a vector of max_level values @@ -554,7 +554,7 @@ Amr::isStatePlotVar (const std::string& name) { if (*li == name) { return true; - } + } } return false; } @@ -581,8 +581,8 @@ Amr::fillStatePlotVarList () for (int comp(0); comp < desc_lst[typ].nComp(); ++comp) { if (desc_lst[typ].getType() == IndexType::TheCellType()) { state_plot_vars.push_back(desc_lst[typ].name(comp)); - } - } + } + } } } @@ -601,8 +601,8 @@ Amr::fillStateSmallPlotVarList () for (int comp(0); comp < desc_lst[typ].nComp(); ++comp) { if (desc_lst[typ].getType() == IndexType::TheCellType()) { state_small_plot_vars.push_back(desc_lst[typ].name(comp)); - } - } + } + } } } @@ -644,7 +644,7 @@ Amr::isDerivePlotVar (const std::string& name) noexcept { if (*li == name) { return true; - } + } } return false; @@ -659,13 +659,13 @@ Amr::isDeriveSmallPlotVar (const std::string& name) noexcept { if (*li == name) { return true; - } + } } return false; } -void +void Amr::fillDerivePlotVarList () { derive_plot_vars.clear(); @@ -682,7 +682,7 @@ Amr::fillDerivePlotVarList () } } -void +void Amr::fillDeriveSmallPlotVarList () { derive_small_plot_vars.clear(); @@ -874,7 +874,7 @@ Amr::writePlotFile () file_name_digits); if (verbose > 0) { - amrex::Print() << "PLOTFILE: file = " << pltfile << '\n'; + amrex::Print() << "PLOTFILE: file = " << pltfile << '\n'; } if (record_run_info && ParallelDescriptor::IOProcessor()) { @@ -913,7 +913,7 @@ Amr::writeSmallPlotFile () file_name_digits); if (verbose > 0) { - amrex::Print() << "SMALL PLOTFILE: file = " << pltfile << '\n'; + amrex::Print() << "SMALL PLOTFILE: file = " << pltfile << '\n'; } if (record_run_info && ParallelDescriptor::IOProcessor()) { @@ -1111,7 +1111,7 @@ Amr::checkInput () } if(verbose > 0) { - amrex::Print() << "Successfully read inputs file ... " << '\n'; + amrex::Print() << "Successfully read inputs file ... " << '\n'; } } @@ -1146,7 +1146,7 @@ Amr::init (Real strt_time, #ifdef BL_COMM_PROFILING Vector probDomain(maxLevel()+1); for(int i(0); i < probDomain.size(); ++i) { - probDomain[i] = Geom(i).Domain(); + probDomain[i] = Geom(i).Domain(); } BL_COMM_PROFILE_INITAMR(finest_level, max_level, ref_ratio, probDomain); #endif @@ -1169,7 +1169,7 @@ Amr::readProbinFile (int& a_init) probin_file_name[i] = probin_file[i]; if (verbose > 0) - amrex::Print() << "Starting to call amrex_probinit ... \n"; + amrex::Print() << "Starting to call amrex_probinit ... \n"; const int nAtOnce = probinit_natonce; const int MyProc = ParallelDescriptor::MyProc(); @@ -1191,18 +1191,18 @@ Amr::readProbinFile (int& a_init) #ifdef AMREX_DIMENSION_AGNOSTIC amrex_probinit(&a_init, - probin_file_name.dataPtr(), - &probin_file_length, - AMREX_ZFILL(Geom(0).ProbLo()), - AMREX_ZFILL(Geom(0).ProbHi())); + probin_file_name.dataPtr(), + &probin_file_length, + AMREX_ZFILL(Geom(0).ProbLo()), + AMREX_ZFILL(Geom(0).ProbHi())); #else amrex_probinit(&a_init, - probin_file_name.dataPtr(), - &probin_file_length, - Geom(0).ProbLo(), - Geom(0).ProbHi()); + probin_file_name.dataPtr(), + &probin_file_length, + Geom(0).ProbLo(), + Geom(0).ProbHi()); #endif piEnd = amrex::second(); @@ -1233,12 +1233,12 @@ Amr::readProbinFile (int& a_init) ParallelDescriptor::ReduceRealMax(piTotal, IOProc); ParallelDescriptor::ReduceRealMax(piTotalAll, IOProc); - amrex::Print() << "amrex_probinit max time = " << piTotal << '\n' - << "amrex_probinit total time = " << piTotalAll << '\n'; + amrex::Print() << "amrex_probinit max time = " << piTotal << '\n' + << "amrex_probinit total time = " << piTotalAll << '\n'; } if (verbose > 0) - amrex::Print() << "Successfully run amrex_probinit\n"; + amrex::Print() << "Successfully run amrex_probinit\n"; } #endif @@ -1252,10 +1252,10 @@ Amr::initialInit (Real strt_time, InitializeInit(strt_time, stop_time, lev0_grids, pmap); // This is a subtlety, but in the case where we are initializing the data - // from a plotfile, we want to use the time read in from the plotfile as + // from a plotfile, we want to use the time read in from the plotfile as // the start time instead of using "strt_time". - // The Amr data "cumtime" has been set in InitializeInit; if we are restarting - // from a plotfile, then cumtime must be re-defined in that initialization routine. + // The Amr data "cumtime" has been set in InitializeInit; if we are restarting + // from a plotfile, then cumtime must be re-defined in that initialization routine. // Thus here we pass "cumtime" rather than "strt_time" to FinalizeInit. FinalizeInit (cumtime, stop_time); } @@ -1301,11 +1301,11 @@ Amr::FinalizeInit (Real strt_time, // Compute dt and set time levels of all grid data. // amr_level[0]->computeInitialDt(finest_level, - sub_cycle, - n_cycle, - ref_ratio, - dt_level, - stop_time); + sub_cycle, + n_cycle, + ref_ratio, + dt_level, + stop_time); // // The following was added for multifluid. // @@ -1350,7 +1350,7 @@ Amr::FinalizeInit (Real strt_time, printGridInfo(amrex::OutStream(),0,finest_level); } else if (verbose > 0) - { + { amrex::Print() << "INITIAL GRIDS \n"; printGridSummary(amrex::OutStream(),0,finest_level); } @@ -1378,7 +1378,7 @@ Amr::restart (const std::string& filename) VisMF::SetMFFileInStreams(mffile_nstreams); if (verbose > 0) { - amrex::Print() << "restarting calculation from file: " << filename << "\n"; + amrex::Print() << "restarting calculation from file: " << filename << "\n"; } if (record_run_info && ParallelDescriptor::IOProcessor()) { @@ -1418,14 +1418,14 @@ Amr::restart (const std::string& filename) std::string faHeaderName; fais >> faHeaderName; if( ! fais.eof()) { - std::string faHeaderFullName(filename + '/' + faHeaderName + "_H"); - Vector &tempCharArray = faHeaderMap[faHeaderFullName]; - ParallelDescriptor::ReadAndBcastFile(faHeaderFullName, tempCharArray); - if(verbose > 2) { - amrex::Print() - << ":::: faHeaderName faHeaderFullName tempCharArray.size() = " << faHeaderName - << " " << faHeaderFullName << " " << tempCharArray.size() << "\n"; - } + std::string faHeaderFullName(filename + '/' + faHeaderName + "_H"); + Vector &tempCharArray = faHeaderMap[faHeaderFullName]; + ParallelDescriptor::ReadAndBcastFile(faHeaderFullName, tempCharArray); + if(verbose > 2) { + amrex::Print() + << ":::: faHeaderName faHeaderFullName tempCharArray.size() = " << faHeaderName + << " " << faHeaderFullName << " " << tempCharArray.size() << "\n"; + } } } StateData::SetFAHeaderMapPtr(&faHeaderMap); @@ -1498,7 +1498,7 @@ Amr::restart (const std::string& filename) } Vector n_cycle_in; - n_cycle_in.resize(mx_lev+1); + n_cycle_in.resize(mx_lev+1); for (int i(0); i <= mx_lev; ++i) { is >> n_cycle_in[i]; } bool any_changed = false; @@ -1506,9 +1506,9 @@ Amr::restart (const std::string& filename) if (n_cycle[i] != n_cycle_in[i]) { any_changed = true; if (verbose > 0) { - amrex::Print() << "Warning: n_cycle has changed at level " << i << - " from " << n_cycle_in[i] << " to " << n_cycle[i] << "\n"; - } + amrex::Print() << "Warning: n_cycle has changed at level " << i << + " from " << n_cycle_in[i] << " to " << n_cycle[i] << "\n"; + } } } @@ -1517,8 +1517,8 @@ Amr::restart (const std::string& filename) { level_count[0] = regrid_int[0]; if (verbose > 0) { - amrex::Print() << "Warning: This forces a full regrid \n"; - } + amrex::Print() << "Warning: This forces a full regrid \n"; + } } @@ -1543,8 +1543,8 @@ Amr::restart (const std::string& filename) for (int i(1); i <= finest_level; ++i) { if (dt_level[i] != dt_level[i-1]) { - amrex::Error("restart: must have same dt at all levels if not subcycling"); - } + amrex::Error("restart: must have same dt at all levels if not subcycling"); + } } } } @@ -1553,9 +1553,9 @@ Amr::restart (const std::string& filename) { if (regrid_int[0] > 0) { level_count[0] = regrid_int[0]; - } else { + } else { amrex::Error("restart: can't have regrid_on_restart and regrid_int <= 0"); - } + } } checkInput(); @@ -1564,10 +1564,10 @@ Amr::restart (const std::string& filename) // for (int lev(0); lev <= finest_level; ++lev) { - amr_level[lev].reset((*levelbld)()); + amr_level[lev].reset((*levelbld)()); amr_level[lev]->restart(*this, is); - this->SetBoxArray(lev, amr_level[lev]->boxArray()); - this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); + this->SetBoxArray(lev, amr_level[lev]->boxArray()); + this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); } // // Build any additional data structures. @@ -1585,7 +1585,7 @@ Amr::restart (const std::string& filename) int new_finest_level = std::min(max_level,finest_level); finest_level = new_finest_level; - + // These are just used to hold the extra stuff we have to read in. Geometry geom_dummy; Real real_dummy; @@ -1620,9 +1620,9 @@ Amr::restart (const std::string& filename) if (regrid_on_restart && max_level > 0) { if (regrid_int[0] > 0) { level_count[0] = regrid_int[0]; - } else { + } else { amrex::Error("restart: can't have regrid_on_restart and regrid_int <= 0"); - } + } } checkInput(); @@ -1632,10 +1632,10 @@ Amr::restart (const std::string& filename) // for (int lev = 0; lev <= new_finest_level; lev++) { - amr_level[lev].reset((*levelbld)()); + amr_level[lev].reset((*levelbld)()); amr_level[lev]->restart(*this, is); - this->SetBoxArray(lev, amr_level[lev]->boxArray()); - this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); + this->SetBoxArray(lev, amr_level[lev]->boxArray()); + this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); } // // Build any additional data structures. @@ -1680,7 +1680,7 @@ Amr::restart (const std::string& filename) ParallelDescriptor::ReduceRealMax(dRestartTime,ParallelDescriptor::IOProcessorNumber()); - amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n'; + amrex::Print() << "Restart time = " << dRestartTime << " seconds." << '\n'; } BL_PROFILE_REGION_STOP("Amr::restart()"); } @@ -1711,7 +1711,7 @@ Amr::checkPoint () const std::string& ckfile = amrex::Concatenate(check_file_root,level_steps[0],file_name_digits); if(verbose > 0) { - amrex::Print() << "CHECKPOINT: file = " << ckfile << "\n"; + amrex::Print() << "CHECKPOINT: file = " << ckfile << "\n"; } if(record_run_info && ParallelDescriptor::IOProcessor()) { @@ -1739,7 +1739,7 @@ Amr::checkPoint () if (precreateDirectories) { // ---- make all directories at once amrex::UtilRenameDirectoryToOld(ckfile, false); // dont call barrier amrex::UtilCreateCleanDirectory(ckfileTemp, false); // dont call barrier - for (int i(0); i <= finest_level; ++i) + for (int i(0); i <= finest_level; ++i) { amr_level[i]->CreateLevelDirectory(ckfileTemp); } @@ -1765,11 +1765,11 @@ Amr::checkPoint () // Only the IOProcessor() writes to the header file. // HeaderFile.open(HeaderFileName.c_str(), std::ios::out | std::ios::trunc | - std::ios::binary); + std::ios::binary); if ( ! HeaderFile.good()) { amrex::FileOpenFailed(HeaderFileName); - } + } old_prec = HeaderFile.precision(17); @@ -1810,20 +1810,20 @@ Amr::checkPoint () } if (ParallelDescriptor::IOProcessor()) { - const Vector &FAHeaderNames = StateData::FabArrayHeaderNames(); - if(FAHeaderNames.size() > 0) { - std::string FAHeaderFilesName = ckfileTemp + "/FabArrayHeaders.txt"; - std::ofstream FAHeaderFile(FAHeaderFilesName.c_str(), - std::ios::out | std::ios::trunc | - std::ios::binary); - if ( ! FAHeaderFile.good()) { - amrex::FileOpenFailed(FAHeaderFilesName); - } - - for(int i(0); i < FAHeaderNames.size(); ++i) { - FAHeaderFile << FAHeaderNames[i] << '\n'; - } - } + const Vector &FAHeaderNames = StateData::FabArrayHeaderNames(); + if(FAHeaderNames.size() > 0) { + std::string FAHeaderFilesName = ckfileTemp + "/FabArrayHeaders.txt"; + std::ofstream FAHeaderFile(FAHeaderFilesName.c_str(), + std::ios::out | std::ios::trunc | + std::ios::binary); + if ( ! FAHeaderFile.good()) { + amrex::FileOpenFailed(FAHeaderFilesName); + } + + for(int i(0); i < FAHeaderNames.size(); ++i) { + FAHeaderFile << FAHeaderNames[i] << '\n'; + } + } } if(ParallelDescriptor::IOProcessor()) { @@ -1831,7 +1831,7 @@ Amr::checkPoint () if( ! HeaderFile.good()) { amrex::Error("Amr::checkpoint() failed"); - } + } } last_checkpoint = level_steps[0]; @@ -1841,9 +1841,9 @@ Amr::checkPoint () auto dCheckPointTime = amrex::second() - dCheckPointTime0; ParallelDescriptor::ReduceRealMax(dCheckPointTime, - ParallelDescriptor::IOProcessorNumber()); + ParallelDescriptor::IOProcessorNumber()); - amrex::Print() << "checkPoint() time = " << dCheckPointTime << " secs." << '\n'; + amrex::Print() << "checkPoint() time = " << dCheckPointTime << " secs." << '\n'; } if (AsyncOut::UseAsyncOut()) { @@ -1873,14 +1873,14 @@ Amr::RegridOnly (Real time, bool do_io) BL_ASSERT(regrid_on_restart == 1); if (max_level == 0) - { + { regrid_level_0_on_restart(); } else - { + { int lev_top = std::min(finest_level, max_level-1); for (int i = 0; i <= lev_top; i++) - { + { regrid(i,time); } } @@ -1909,7 +1909,7 @@ Amr::timeStep (int level, BL_PROFILE("Amr::timeStep()"); BL_COMM_PROFILE_NAMETAG("Amr::timeStep TOP"); - // This is used so that the AmrLevel functions can know which level is being advanced + // This is used so that the AmrLevel functions can know which level is being advanced // when regridding is called with possible lbase > level. which_level_being_advanced = level; @@ -1922,7 +1922,7 @@ Amr::timeStep (int level, // if (max_level == 0 && regrid_on_restart) { - regrid_level_0_on_restart(); + regrid_level_0_on_restart(); } else { @@ -1942,18 +1942,18 @@ Amr::timeStep (int level, { int post_regrid_flag = 1; amr_level[0]->computeNewDt(finest_level, - sub_cycle, - n_cycle, - ref_ratio, - dt_min, - dt_level, - stop_time, - post_regrid_flag); + sub_cycle, + n_cycle, + ref_ratio, + dt_min, + dt_level, + stop_time, + post_regrid_flag); } for (int k(i); k <= finest_level; ++k) { level_count[k] = 0; - } + } if (old_finest < finest_level) { @@ -1969,7 +1969,7 @@ Amr::timeStep (int level, } if (old_finest > finest_level) { lev_top = std::min(finest_level, max_level - 1); - } + } } if (max_level == 0 && loadbalance_level0_int > 0 && loadbalance_with_workestimates) @@ -1986,16 +1986,16 @@ Amr::timeStep (int level, // if (plotfile_on_restart && ! (restart_chkfile.empty()) ) { - plotfile_on_restart = 0; - writePlotFile(); + plotfile_on_restart = 0; + writePlotFile(); } // // Advance grids at this level. // if (verbose > 0) { - amrex::Print() << "[Level " << level << " step " << level_steps[level]+1 << "] " - << "ADVANCE with dt = " << dt_level[level] << "\n"; + amrex::Print() << "[Level " << level << " step " << level_steps[level]+1 << "] " + << "ADVANCE with dt = " << dt_level[level] << "\n"; } Real dt_new = amr_level[level]->advance(time,dt_level[level],iteration,niter); @@ -2008,28 +2008,28 @@ Amr::timeStep (int level, if (verbose > 0) { - amrex::Print() << "[Level " << level << " step " << level_steps[level] << "] " - << "Advanced " << amr_level[level]->countCells() << " cells\n"; + amrex::Print() << "[Level " << level << " step " << level_steps[level] << "] " + << "Advanced " << amr_level[level]->countCells() << " cells\n"; } // If the level signified that it wants a regrid after the advance has // occurred, do that now. if (amr_level[level]->postStepRegrid()) { - int old_finest = finest_level; + int old_finest = finest_level; - regrid(level, time); + regrid(level, time); - if (old_finest < finest_level) - { - // - // The new levels will not have valid time steps. - // - for (int k = old_finest + 1; k <= finest_level; ++k) - { - dt_level[k] = dt_level[k-1] / n_cycle[k]; - } - } + if (old_finest < finest_level) + { + // + // The new levels will not have valid time steps. + // + for (int k = old_finest + 1; k <= finest_level; ++k) + { + dt_level[k] = dt_level[k-1] / n_cycle[k]; + } + } } // @@ -2086,22 +2086,22 @@ Amr::coarseTimeStep (Real stop_time) { int post_regrid_flag = 0; amr_level[0]->computeNewDt(finest_level, - sub_cycle, - n_cycle, - ref_ratio, - dt_min, - dt_level, - stop_time, - post_regrid_flag); + sub_cycle, + n_cycle, + ref_ratio, + dt_min, + dt_level, + stop_time, + post_regrid_flag); } else { amr_level[0]->computeInitialDt(finest_level, - sub_cycle, - n_cycle, - ref_ratio, - dt_level, - stop_time); + sub_cycle, + n_cycle, + ref_ratio, + dt_level, + stop_time); } BL_PROFILE_REGION_START(stepName.str()); @@ -2117,15 +2117,15 @@ Amr::coarseTimeStep (Real stop_time) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); run_stop = amrex::second() - run_strt; - const int istep = level_steps[0]; + const int istep = level_steps[0]; #ifdef BL_LAZY - Lazy::QueueReduction( [=] () mutable { + Lazy::QueueReduction( [=] () mutable { #endif - ParallelDescriptor::ReduceRealMax(run_stop,IOProc); - amrex::Print() << "\n[STEP " << istep << "] Coarse TimeStep time: " << run_stop << '\n'; + ParallelDescriptor::ReduceRealMax(run_stop,IOProc); + amrex::Print() << "\n[STEP " << istep << "] Coarse TimeStep time: " << run_stop << '\n'; #ifdef BL_LAZY - }); + }); #endif #ifndef AMREX_MEM_PROFILING @@ -2133,25 +2133,25 @@ Amr::coarseTimeStep (Real stop_time) Long max_fab_kilobytes = min_fab_kilobytes; #ifdef BL_LAZY - Lazy::QueueReduction( [=] () mutable { + Lazy::QueueReduction( [=] () mutable { #endif - ParallelDescriptor::ReduceLongMin(min_fab_kilobytes, IOProc); - ParallelDescriptor::ReduceLongMax(max_fab_kilobytes, IOProc); + ParallelDescriptor::ReduceLongMin(min_fab_kilobytes, IOProc); + ParallelDescriptor::ReduceLongMax(max_fab_kilobytes, IOProc); - amrex::Print() << "[STEP " << istep << "] FAB kilobyte spread across MPI nodes: [" - << min_fab_kilobytes << " ... " << max_fab_kilobytes << "]\n"; + amrex::Print() << "[STEP " << istep << "] FAB kilobyte spread across MPI nodes: [" + << min_fab_kilobytes << " ... " << max_fab_kilobytes << "]\n"; #ifdef BL_LAZY - amrex::Print() << "\n"; - }); + amrex::Print() << "\n"; + }); #endif #endif } #ifdef AMREX_MEM_PROFILING { - std::ostringstream ss; - ss << "[STEP " << level_steps[0] << "]"; - MemProfiler::report(ss.str()); + std::ostringstream ss; + ss << "[STEP " << level_steps[0] << "]"; + MemProfiler::report(ss.str()); } #endif @@ -2164,10 +2164,10 @@ Amr::coarseTimeStep (Real stop_time) if (verbose > 0) { - amrex::Print() - << "\nSTEP = " << level_steps[0] - << " TIME = " << cumtime - << " DT = " << dt_level[0] << "\n\n"; + amrex::Print() + << "\nSTEP = " << level_steps[0] + << " TIME = " << cumtime + << " DT = " << dt_level[0] << "\n\n"; } if (record_run_info && ParallelDescriptor::IOProcessor()) { @@ -2219,40 +2219,40 @@ Amr::coarseTimeStep (Real stop_time) } - int to_stop = 0; + int to_stop = 0; int to_checkpoint = 0; int to_plot = 0; int to_small_plot = 0; if (message_int > 0 && level_steps[0] % message_int == 0) { - if (ParallelDescriptor::IOProcessor()) - { - FILE *fp; - if ((fp=fopen("dump_and_continue","r")) != 0) - { - remove("dump_and_continue"); - to_checkpoint = 1; - fclose(fp); - } - else if ((fp=fopen("stop_run","r")) != 0) - { - remove("stop_run"); - to_stop = 1; - fclose(fp); - } - else if ((fp=fopen("dump_and_stop","r")) != 0) - { - remove("dump_and_stop"); - to_checkpoint = 1; - to_stop = 1; - fclose(fp); - } - - if ((fp=fopen("plot_and_continue","r")) != 0) - { - remove("plot_and_continue"); - to_plot = 1; - fclose(fp); - } + if (ParallelDescriptor::IOProcessor()) + { + FILE *fp; + if ((fp=fopen("dump_and_continue","r")) != 0) + { + remove("dump_and_continue"); + to_checkpoint = 1; + fclose(fp); + } + else if ((fp=fopen("stop_run","r")) != 0) + { + remove("stop_run"); + to_stop = 1; + fclose(fp); + } + else if ((fp=fopen("dump_and_stop","r")) != 0) + { + remove("dump_and_stop"); + to_checkpoint = 1; + to_stop = 1; + fclose(fp); + } + + if ((fp=fopen("plot_and_continue","r")) != 0) + { + remove("plot_and_continue"); + to_plot = 1; + fclose(fp); + } if ((fp=fopen("small_plot_and_continue","r")) != 0) { @@ -2260,32 +2260,32 @@ Amr::coarseTimeStep (Real stop_time) to_small_plot = 1; fclose(fp); } - } + } int packed_data[4]; - packed_data[0] = to_stop; - packed_data[1] = to_checkpoint; + packed_data[0] = to_stop; + packed_data[1] = to_checkpoint; packed_data[2] = to_plot; packed_data[3] = to_small_plot; - ParallelDescriptor::Bcast(packed_data, 4, ParallelDescriptor::IOProcessorNumber()); - to_stop = packed_data[0]; - to_checkpoint = packed_data[1]; + ParallelDescriptor::Bcast(packed_data, 4, ParallelDescriptor::IOProcessorNumber()); + to_stop = packed_data[0]; + to_checkpoint = packed_data[1]; to_plot = packed_data[2]; to_small_plot = packed_data[3]; } if(to_stop == 1 && to_checkpoint == 0) { // prevent main from writing files - last_checkpoint = level_steps[0]; - last_plotfile = level_steps[0]; + last_checkpoint = level_steps[0]; + last_plotfile = level_steps[0]; } if (to_checkpoint && write_plotfile_with_checkpoint) { - to_plot = 1; - to_small_plot = 1; + to_plot = 1; + to_small_plot = 1; } if ((check_int > 0 && level_steps[0] % check_int == 0) || check_test == 1 - || to_checkpoint) + || to_checkpoint) { checkPoint(); } @@ -2308,15 +2308,15 @@ Amr::coarseTimeStep (Real stop_time) { ParallelDescriptor::Barrier("Amr::coarseTimeStep::to_stop"); if(ParallelDescriptor::IOProcessor()) { - if (to_checkpoint) - { - amrex::ErrorStream() << "Stopped by user w/ checkpoint" << std::endl; - } - else - { - amrex::ErrorStream() << "Stopped by user w/o checkpoint" << std::endl; - } - } + if (to_checkpoint) + { + amrex::ErrorStream() << "Stopped by user w/ checkpoint" << std::endl; + } + else + { + amrex::ErrorStream() << "Stopped by user w/o checkpoint" << std::endl; + } + } } } @@ -2388,10 +2388,10 @@ Amr::writePlotNow() noexcept } - return ( (plot_int > 0 && level_steps[0] % plot_int == 0) || + return ( (plot_int > 0 && level_steps[0] % plot_int == 0) || plot_test == 1 || amr_level[0]->writePlotNow()); -} +} bool Amr::writeSmallPlotNow() noexcept @@ -2430,9 +2430,9 @@ Amr::writeSmallPlotNow() noexcept } if (num_per_old != num_per_new) - { + { plot_test = 1; - } + } } @@ -2461,13 +2461,13 @@ Amr::writeSmallPlotNow() noexcept } - return ( (small_plot_int > 0 && level_steps[0] % small_plot_int == 0) || + return ( (small_plot_int > 0 && level_steps[0] % small_plot_int == 0) || plot_test == 1 || amr_level[0]->writeSmallPlotNow()); -} +} void -Amr::defBaseLevel (Real strt_time, +Amr::defBaseLevel (Real strt_time, const BoxArray* lev0_grids, const Vector* pmap) { @@ -2501,13 +2501,13 @@ Amr::defBaseLevel (Real strt_time, lev0 = *lev0_grids; - if (refine_grid_layout) { - ChopGrids(0,lev0,ParallelDescriptor::NProcs()); - } + if (refine_grid_layout) { + ChopGrids(0,lev0,ParallelDescriptor::NProcs()); + } } else { - lev0 = MakeBaseGrids(); + lev0 = MakeBaseGrids(); } this->SetBoxArray(0, lev0); @@ -2533,7 +2533,7 @@ Amr::regrid (int lbase, if (lbase > std::min(finest_level,max_level-1)) return; if (verbose > 0) - amrex::Print() << "Now regridding at level lbase = " << lbase << "\n"; + amrex::Print() << "Now regridding at level lbase = " << lbase << "\n"; // // Compute positions of new grids. @@ -2551,12 +2551,12 @@ Amr::regrid (int lbase, bool grids_unchanged = finest_level == new_finest; for (int lev = start, End = std::min(finest_level,new_finest); lev <= End; lev++) { - if (new_grid_places[lev] == amr_level[lev]->boxArray()) { - new_grid_places[lev] = amr_level[lev]->boxArray(); // to avoid duplicates - new_dmap[lev] = amr_level[lev]->DistributionMap(); - } else { - grids_unchanged = false; - } + if (new_grid_places[lev] == amr_level[lev]->boxArray()) { + new_grid_places[lev] = amr_level[lev]->boxArray(); // to avoid duplicates + new_dmap[lev] = amr_level[lev]->DistributionMap(); + } else { + grids_unchanged = false; + } } // @@ -2564,11 +2564,11 @@ Amr::regrid (int lbase, // if (use_efficient_regrid == 1 && grids_unchanged ) { - if (verbose > 0) { - amrex::Print() << "Regridding at level lbase = " << lbase - << " but grids unchanged\n"; - } - return; + if (verbose > 0) { + amrex::Print() << "Regridding at level lbase = " << lbase + << " but grids unchanged\n"; + } + return; } // @@ -2584,9 +2584,9 @@ Amr::regrid (int lbase, // Reclaim all remaining storage for levels > new_finest. // for(int lev = new_finest + 1; lev <= finest_level; ++lev) { - amr_level[lev].reset(); - this->ClearBoxArray(lev); - this->ClearDistributionMap(lev); + amr_level[lev].reset(); + this->ClearBoxArray(lev); + this->ClearDistributionMap(lev); } finest_level = new_finest; @@ -2603,11 +2603,11 @@ Amr::regrid (int lbase, new_dmap[lev] = makeLoadBalanceDistributionMap(lev, time, new_grid_places[lev]); } else if (new_dmap[lev].empty()) { - new_dmap[lev].define(new_grid_places[lev]); - } + new_dmap[lev].define(new_grid_places[lev]); + } AmrLevel* a = (*levelbld)(*this,lev,Geom(lev),new_grid_places[lev], - new_dmap[lev],cumtime); + new_dmap[lev],cumtime); if (initial) { @@ -2617,8 +2617,8 @@ Amr::regrid (int lbase, // be officially inserted into the hierarchy prior to the call. // amr_level[lev].reset(a); - this->SetBoxArray(lev, amr_level[lev]->boxArray()); - this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); + this->SetBoxArray(lev, amr_level[lev]->boxArray()); + this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); amr_level[lev]->initData(); } else if (amr_level[lev]) @@ -2630,17 +2630,17 @@ Amr::regrid (int lbase, // a->init(*amr_level[lev]); amr_level[lev].reset(a); - this->SetBoxArray(lev, amr_level[lev]->boxArray()); - this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); - } + this->SetBoxArray(lev, amr_level[lev]->boxArray()); + this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); + } else { a->init(); amr_level[lev].reset(a); if (lev > 0) level_steps[lev] = level_steps[lev-1] * n_cycle[lev-1]; - this->SetBoxArray(lev, amr_level[lev]->boxArray()); - this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); + this->SetBoxArray(lev, amr_level[lev]->boxArray()); + this->SetDistributionMap(lev, amr_level[lev]->DistributionMap()); } } @@ -2648,10 +2648,10 @@ Amr::regrid (int lbase, // // Check at *all* levels whether we need to do anything special now that the grids - // at levels lbase+1 and higher may have changed. + // at levels lbase+1 and higher may have changed. // for(int lev(0); lev <= new_finest; ++lev) { - amr_level[lev]->post_regrid(lbase,new_finest); + amr_level[lev]->post_regrid(lbase,new_finest); } // @@ -2784,46 +2784,46 @@ Amr::regrid_level_0_on_restart() // Now refine these boxes back to level 0. // lev0.refine(2); - + // - // If use_efficient_regrid flag is set, then test to see whether we in fact + // If use_efficient_regrid flag is set, then test to see whether we in fact // have just changed the level 0 grids. If not, then don't do anything more here. // - if ( !( (use_efficient_regrid == 1) && (lev0 == amr_level[0]->boxArray()) ) ) - { - // - // Construct skeleton of new level. - // - DistributionMapping dm(lev0); - AmrLevel* a = (*levelbld)(*this,0,Geom(0),lev0,dm,cumtime); - - a->init(*amr_level[0]); - amr_level[0].reset(a); - - this->SetBoxArray(0, amr_level[0]->boxArray()); - this->SetDistributionMap(0, amr_level[0]->DistributionMap()); - - amr_level[0]->post_regrid(0,0); - - if (ParallelDescriptor::IOProcessor()) - { - if (verbose > 1) - { - printGridInfo(amrex::OutStream(),0,finest_level); - } - else if (verbose > 0) - { - printGridSummary(amrex::OutStream(),0,finest_level); - } - } - - if (record_grid_info && ParallelDescriptor::IOProcessor()) - printGridInfo(gridlog,0,finest_level); + if ( !( (use_efficient_regrid == 1) && (lev0 == amr_level[0]->boxArray()) ) ) + { + // + // Construct skeleton of new level. + // + DistributionMapping dm(lev0); + AmrLevel* a = (*levelbld)(*this,0,Geom(0),lev0,dm,cumtime); + + a->init(*amr_level[0]); + amr_level[0].reset(a); + + this->SetBoxArray(0, amr_level[0]->boxArray()); + this->SetDistributionMap(0, amr_level[0]->DistributionMap()); + + amr_level[0]->post_regrid(0,0); + + if (ParallelDescriptor::IOProcessor()) + { + if (verbose > 1) + { + printGridInfo(amrex::OutStream(),0,finest_level); + } + else if (verbose > 0) + { + printGridSummary(amrex::OutStream(),0,finest_level); + } + } + + if (record_grid_info && ParallelDescriptor::IOProcessor()) + printGridInfo(gridlog,0,finest_level); } else { - if (verbose > 0) - amrex::Print() << "Regridding at level 0 but grids unchanged \n"; + if (verbose > 0) + amrex::Print() << "Regridding at level 0 but grids unchanged \n"; } } @@ -2858,7 +2858,7 @@ Amr::printGridInfo (std::ostream& os, const Box& b = bs[k]; os << ' ' << lev << ": " << b << " "; - + for (int i = 0; i < AMREX_SPACEDIM; i++) os << b.length(i) << ' '; @@ -2882,7 +2882,7 @@ Amr::grid_places (int lbase, if (lbase == 0) { - new_grids[0] = MakeBaseGrids(); + new_grids[0] = MakeBaseGrids(); } if ( time == 0. && !initial_grids_file.empty() && !use_fixed_coarse_grids) @@ -2929,7 +2929,7 @@ Amr::grid_places (int lbase, new_grids[lev].maxSize(max_grid_size[lev]); } } - else if ( !regrid_grids_file.empty() ) // Use grids in regrid_grids_file + else if ( !regrid_grids_file.empty() ) // Use grids in regrid_grids_file { new_finest = std::min(max_level,(finest_level+1)); new_finest = std::min(new_finest,regrid_ba.size()); @@ -2956,12 +2956,12 @@ Amr::grid_places (int lbase, auto stoptime = amrex::second() - strttime; #ifdef BL_LAZY - Lazy::QueueReduction( [=] () mutable { + Lazy::QueueReduction( [=] () mutable { #endif - ParallelDescriptor::ReduceRealMax(stoptime,ParallelDescriptor::IOProcessorNumber()); - amrex::Print() << "grid_places() time: " << stoptime << " new finest: " << new_finest<< '\n'; + ParallelDescriptor::ReduceRealMax(stoptime,ParallelDescriptor::IOProcessorNumber()); + amrex::Print() << "grid_places() time: " << stoptime << " new finest: " << new_finest<< '\n'; #ifdef BL_LAZY - }); + }); #endif } } @@ -3006,18 +3006,18 @@ Amr::bldFineLevels (Real strt_time) // finest_level = new_finest; - DistributionMapping new_dm {new_grids[new_finest]}; + DistributionMapping new_dm {new_grids[new_finest]}; AmrLevel* level = (*levelbld)(*this, new_finest, Geom(new_finest), new_grids[new_finest], - new_dm, + new_dm, strt_time); amr_level[new_finest].reset(level); - this->SetBoxArray(new_finest, new_grids[new_finest]); - this->SetDistributionMap(new_finest, new_dm); + this->SetBoxArray(new_finest, new_grids[new_finest]); + this->SetDistributionMap(new_finest, new_dm); amr_level[new_finest]->initData(); } @@ -3026,33 +3026,33 @@ Amr::bldFineLevels (Real strt_time) // Iterate grids to ensure fine grids encompass all interesting gunk. // but only iterate if we did not provide a grids file. // - if ( regrid_grids_file.empty() || (strt_time == 0.0 && !initial_grids_file.empty()) ) + if ( regrid_grids_file.empty() || (strt_time == 0.0 && !initial_grids_file.empty()) ) { - bool grids_the_same; + bool grids_the_same; - const int MaxCnt = 4; + const int MaxCnt = 4; - int count = 0; + int count = 0; - do - { - for (int i = 0; i <= finest_level; i++) { - new_grids[i] = amr_level[i]->boxArray(); - } + do + { + for (int i = 0; i <= finest_level; i++) { + new_grids[i] = amr_level[i]->boxArray(); + } - regrid(0,strt_time,true); + regrid(0,strt_time,true); - grids_the_same = true; + grids_the_same = true; - for (int i = 0; i <= finest_level && grids_the_same; i++) { - if (!(new_grids[i] == amr_level[i]->boxArray())) { - grids_the_same = false; - } - } + for (int i = 0; i <= finest_level && grids_the_same; i++) { + if (!(new_grids[i] == amr_level[i]->boxArray())) { + grids_the_same = false; + } + } - count++; - } - while (!grids_the_same && count < MaxCnt); + count++; + } + while (!grids_the_same && count < MaxCnt); } } @@ -3076,12 +3076,12 @@ Amr::initSubcycle () amrex::Error("nosub <= 0 not allowed.\n"); subcycling_mode = "None"; } - else + else { subcycling_mode = "Auto"; pp.query("subcycling_mode",subcycling_mode); } - + if (subcycling_mode == "None") { sub_cycle = false; @@ -3138,7 +3138,7 @@ Amr::initSubcycle () for (int i = 1; i <= max_level; i++) { n_cycle[i] = MaxRefRatio(i-1); - } + } } else if (subcycling_mode == "Optimal") { @@ -3148,7 +3148,7 @@ Amr::initSubcycle () for (int i = 1; i <= max_level; i++) { n_cycle[i] = MaxRefRatio(i-1); - } + } } else { @@ -3172,7 +3172,7 @@ Amr::initPltAndChk () // if (plot_nfiles == -1) plot_nfiles = ParallelDescriptor::NProcs(); if (checkpoint_nfiles == -1) checkpoint_nfiles = ParallelDescriptor::NProcs(); - + check_file_root = "chk"; pp.query("check_file",check_file_root); @@ -3185,7 +3185,7 @@ Amr::initPltAndChk () if (check_int > 0 && check_per > 0) { if (ParallelDescriptor::IOProcessor()) - amrex::Warning("Warning: both amr.check_int and amr.check_per are > 0."); + amrex::Warning("Warning: both amr.check_int and amr.check_per are > 0."); } plot_file_root = "plt"; @@ -3262,7 +3262,7 @@ Real Amr::computeOptimalSubcycling(int n, int* best, Real* dt_max, Real* est_work, int* cycle_max) { BL_ASSERT(cycle_max[0] == 1); - // internally these represent the total number of steps at a level, + // internally these represent the total number of steps at a level, // not the number of cycles std::vector cycles(n); #ifdef AMREX_USE_FLOAT @@ -3293,7 +3293,7 @@ Amr::computeOptimalSubcycling(int n, int* best, Real* dt_max, Real* est_work, in work += cycles[i]*est_work[i]; } ratio = work/dt; - if (ratio < best_ratio) + if (ratio < best_ratio) { for (int i = 0; i < n; i++) best[i] = cycles[i]; @@ -3315,7 +3315,7 @@ const Vector& Amr::getInitialBA() noexcept } #ifdef AMREX_PARTICLES -void +void Amr::RedistributeParticles () { amr_level[0]->particle_redistribute(0,true); diff --git a/Src/Amr/AMReX_AmrLevel.H b/Src/Amr/AMReX_AmrLevel.H index c6af088c2d2..5182310e24a 100644 --- a/Src/Amr/AMReX_AmrLevel.H +++ b/Src/Amr/AMReX_AmrLevel.H @@ -64,7 +64,7 @@ public: void SetLevelDirectoryCreated(bool ldc) noexcept { levelDirectoryCreated = ldc; } /** * \brief A string written as the first item in writePlotFile() at - * level zero. + * level zero. * It is so we can distinguish between different types of * plot files. * This default "HyperCLaw-V1.1" is for VisIt software and some of our @@ -93,13 +93,13 @@ public: std::ostream& os); /** - * \brief Write small plot file stuff to specified directory. - * Unlike writePlotFile, this is NOT a pure virtual function + * \brief Write small plot file stuff to specified directory. + * Unlike writePlotFile, this is NOT a pure virtual function * so implementation by derived classes is optional. */ virtual void writeSmallPlotFile (const std::string& /*dir*/, std::ostream& /*os*/, - VisMF::How /*how*/ = VisMF::NFiles) {} + VisMF::How /*how*/ = VisMF::NFiles) {} //! Write current state to checkpoint file. virtual void checkPoint (const std::string& dir, std::ostream& os, @@ -114,7 +114,7 @@ public: //! Restart from a checkpoint file. virtual void restart (Amr& papa, std::istream& is, - bool bReadSpecial = false); + bool bReadSpecial = false); //! Old checkpoint may have different number of states than the new source code. virtual void set_state_in_checkpoint (Vector& state_in_checkpoint); @@ -191,7 +191,7 @@ public: virtual int okToContinue () { return 1; } /** * \brief Should I regrid with this level as base level? - * This test is only evaluated if regrid_int > 0 and + * This test is only evaluated if regrid_int > 0 and * level_count >= regrid_int as well. Defaults to true. */ virtual int okToRegrid (); @@ -264,7 +264,7 @@ public: int clearval, int tagval, Real time, - int n_error_buf = 0, + int n_error_buf = 0, int ngrow = 0) = 0; //! Interpolate from coarse level to the valid area in dest. void FillCoarsePatch (MultiFab& dest, @@ -273,7 +273,7 @@ public: int state_idx, int scomp, int ncomp, - int nghost = 0); + int nghost = 0); //! Function to set physical boundary conditions. virtual void setPhysBoundaryValues (FArrayBox& dest, int state_indx, @@ -288,8 +288,8 @@ public: * grown BoxArray. */ virtual std::unique_ptr derive (const std::string& name, - Real time, - int ngrow); + Real time, + int ngrow); /** * \brief This version of derive() fills the dcomp'th component of mf * with the derived quantity. @@ -322,9 +322,9 @@ public: //! Boundary condition access function. Vector getBCArray (int State_Type, - int gridno, - int scomp, - int ncomp); + int gridno, + int scomp, + int ncomp); //! Get state data at specified index and time. MultiFab& get_data (int state_indx, Real time) noexcept; //! Hack to allow override of (non-fine-fine) fillpatched boundary data @@ -334,7 +334,7 @@ public: int dcomp, int ncomp, Real time) const; - /** + /** * \brief Called in grid_places after other tagging routines to modify * the list of tagged points. Default implementation does nothing. */ @@ -357,7 +357,7 @@ public: /** * \brief Returns one the TimeLevel enums. * Asserts that time is between AmrOldTime and AmrNewTime. - */ + */ TimeLevel which_time (int state_indx, Real time) const noexcept; //! Does the AmrLevel want Amr to write a plotfile now? @@ -367,7 +367,7 @@ public: virtual bool writeSmallPlotNow (); #ifdef AMREX_PARTICLES - //! This function can be called from the parent + //! This function can be called from the parent virtual void particle_redistribute (int /*lbase*/ = 0, bool /*a_init*/ = false) {;} #endif @@ -388,7 +388,7 @@ public: int scomp, int ncomp, int dcomp=0); - + #ifdef AMREX_USE_EB static void SetEBMaxGrowCells (int nbasic, int nvolume, int nfull) noexcept { m_eb_basic_grow_cells = nbasic; @@ -410,14 +410,14 @@ protected: int lev, const Geometry& level_geom, const BoxArray& bl, - const DistributionMapping& dm, + const DistributionMapping& dm, Real time); AmrLevel (const AmrLevel&) = delete; AmrLevel& operator = (const AmrLevel&) = delete; //! Common code used by all constructors. - void finishConstructor (); + void finishConstructor (); // // The Data. diff --git a/Src/Amr/AMReX_AmrLevel.cpp b/Src/Amr/AMReX_AmrLevel.cpp index d85ebe57bd8..a47e07fdc37 100644 --- a/Src/Amr/AMReX_AmrLevel.cpp +++ b/Src/Amr/AMReX_AmrLevel.cpp @@ -38,10 +38,10 @@ AmrLevel::postCoarseTimeStep (Real time) BL_ASSERT(level == 0); // sync up statedata time for (int lev = 0; lev <= parent->finestLevel(); ++lev) { - AmrLevel& amrlevel = parent->getLevel(lev); - for (int i = 0; i < amrlevel.state.size(); ++i) { - amrlevel.state[i].syncNewTimeLevel(time); - } + AmrLevel& amrlevel = parent->getLevel(lev); + for (int i = 0; i < amrlevel.state.size(); ++i) { + amrlevel.state[i].syncNewTimeLevel(time); + } } } @@ -78,7 +78,7 @@ AmrLevel::AmrLevel (Amr& papa, int lev, const Geometry& level_geom, const BoxArray& ba, - const DistributionMapping& dm, + const DistributionMapping& dm, Real time) : geom(level_geom), @@ -126,7 +126,7 @@ AmrLevel::AmrLevel (Amr& papa, MultiFab::RegionTag level_tag("AmrLevel_Level_" + std::to_string(lev)); state[i].define(geom.Domain(), grids, - dm, + dm, desc_lst[i], time, parent->dtLevel(lev), @@ -155,25 +155,25 @@ AmrLevel::writePlotFile (const std::string& dir, for (int typ = 0; typ < desc_lst.size(); typ++) { for (int comp = 0; comp < desc_lst[typ].nComp();comp++) - { + { if (parent->isStatePlotVar(desc_lst[typ].name(comp)) && desc_lst[typ].getType() == IndexType::TheCellType()) - { + { plot_var_map.push_back(std::pair(typ,comp)); - } - } + } + } } std::vector derive_names; const std::list& dlist = derive_lst.dlist(); for (std::list::const_iterator it = dlist.begin(); - it != dlist.end(); - ++it) + it != dlist.end(); + ++it) { if (parent->isDerivePlotVar(it->name())) { derive_names.push_back(it->name()); - } + } } int n_data_items = plot_var_map.size() + derive_names.size(); @@ -200,14 +200,14 @@ AmrLevel::writePlotFile (const std::string& dir, os << n_data_items << '\n'; - // - // Names of variables - // - for (i =0; i < static_cast(plot_var_map.size()); i++) + // + // Names of variables + // + for (i =0; i < static_cast(plot_var_map.size()); i++) { - int typ = plot_var_map[i].first; - int comp = plot_var_map[i].second; - os << desc_lst[typ].name(comp) << '\n'; + int typ = plot_var_map[i].first; + int comp = plot_var_map[i].second; + os << desc_lst[typ].name(comp) << '\n'; } // derived @@ -270,13 +270,13 @@ AmrLevel::writePlotFile (const std::string& dir, // Only the I/O processor makes the directory if it doesn't already exist. // if ( ! levelDirectoryCreated) { - if (ParallelDescriptor::IOProcessor()) { - if ( ! amrex::UtilCreateDirectory(FullPath, 0755)) { - amrex::CreateDirectoryFailed(FullPath); - } - } - // Force other processors to wait until directory is built. - ParallelDescriptor::Barrier(); + if (ParallelDescriptor::IOProcessor()) { + if ( ! amrex::UtilCreateDirectory(FullPath, 0755)) { + amrex::CreateDirectoryFailed(FullPath); + } + } + // Force other processors to wait until directory is built. + ParallelDescriptor::Barrier(); } if (ParallelDescriptor::IOProcessor()) @@ -325,21 +325,21 @@ AmrLevel::writePlotFile (const std::string& dir, // for (i = 0; i < static_cast(plot_var_map.size()); i++) { - int typ = plot_var_map[i].first; - int comp = plot_var_map[i].second; - this_dat = &state[typ].newData(); - MultiFab::Copy(plotMF,*this_dat,comp,cnt,1,nGrow); - cnt++; + int typ = plot_var_map[i].first; + int comp = plot_var_map[i].second; + this_dat = &state[typ].newData(); + MultiFab::Copy(plotMF,*this_dat,comp,cnt,1,nGrow); + cnt++; } // derived if (derive_names.size() > 0) { - for (auto const& dname : derive_names) - { + for (auto const& dname : derive_names) + { derive(dname, cur_time, plotMF, cnt); - cnt++; - } + cnt++; + } } #ifdef AMREX_USE_EB @@ -382,7 +382,7 @@ AmrLevel::writePlotFilePost (const std::string& /*dir*/, void AmrLevel::restart (Amr& papa, std::istream& is, - bool bReadSpecial) + bool bReadSpecial) { BL_PROFILE("AmrLevel::restart()"); parent = &papa; @@ -417,9 +417,9 @@ AmrLevel::restart (Amr& papa, Vector state_in_checkpoint(ndesc, 1); if (ndesc > nstate) { - set_state_in_checkpoint(state_in_checkpoint); + set_state_in_checkpoint(state_in_checkpoint); } else { - BL_ASSERT(nstate == ndesc); + BL_ASSERT(nstate == ndesc); } dmap.define(grids); @@ -443,12 +443,12 @@ AmrLevel::restart (Amr& papa, state.resize(ndesc); for (int i = 0; i < ndesc; ++i) { - if (state_in_checkpoint[i]) { - state[i].restart(is, geom.Domain(), grids, dmap, *m_factory, - desc_lst[i], papa.theRestartFile()); - } + if (state_in_checkpoint[i]) { + state[i].restart(is, geom.Domain(), grids, dmap, *m_factory, + desc_lst[i], papa.theRestartFile()); + } } - + if (parent->useFixedCoarseGrids()) constructAreaNotToTag(); post_step_regrid = 0; @@ -619,8 +619,8 @@ AmrLevel::getEdgeBoxArray (int dir) const noexcept { BL_ASSERT(dir >=0 && dir < AMREX_SPACEDIM); if (edge_grids[dir].empty()) { - edge_grids[dir] = grids; - edge_grids[dir].surroundingNodes(dir); + edge_grids[dir] = grids; + edge_grids[dir].surroundingNodes(dir); } return edge_grids[dir]; } @@ -629,8 +629,8 @@ const BoxArray& AmrLevel::getNodalBoxArray () const noexcept { if (nodal_grids.empty()) { - nodal_grids = grids; - nodal_grids.surroundingNodes(); + nodal_grids = grids; + nodal_grids.surroundingNodes(); } return nodal_grids; } @@ -848,7 +848,7 @@ FillPatchIteratorHelper::Initialize (int boxGrow, } } - // cells outside physical boundaries are not included in unfilledThisLevel + // cells outside physical boundaries are not included in unfilledThisLevel bool Done = false; @@ -890,7 +890,7 @@ FillPatchIteratorHelper::Initialize (int boxGrow, { const Box& cbox = m_map->CoarseBox(fbx,fine_ratio); - crse_boxes.back() = cbox; + crse_boxes.back() = cbox; if (is_periodic && !thePDomain.contains(cbox)) { @@ -942,15 +942,15 @@ FillPatchIteratorHelper::Initialize (int boxGrow, BL_ASSERT(CrseBoxes[i].intersects(thePDomain)); theState.InterpAddBox(m_mfcd, - m_mfid[l], - &tempUnfillable, - FBIDs[i], - CrseBoxes[i], - m_time, - m_scomp, - 0, - m_ncomp, - extrap); + m_mfid[l], + &tempUnfillable, + FBIDs[i], + CrseBoxes[i], + m_time, + m_scomp, + 0, + m_ncomp, + extrap); unfillableThisLevel.catenate(tempUnfillable); } @@ -994,7 +994,7 @@ FillPatchIterator::Initialize (int boxGrow, m_range = desc.sameInterps(scomp,ncomp); m_fabs.define(m_leveldata.boxArray(),m_leveldata.DistributionMap(), - m_ncomp,boxGrow,MFInfo(),m_leveldata.Factory()); + m_ncomp,boxGrow,MFInfo(),m_leveldata.Factory()); const Geometry& geom = m_amrlevel.Geom(); @@ -1008,19 +1008,19 @@ FillPatchIterator::Initialize (int boxGrow, const int SComp = m_range[i].first; const int NComp = m_range[i].second; - if (level == 0) - { - FillFromLevel0(time, idx, SComp, DComp, NComp); - } - else - { - if (level == 1 || - amrex::ProperlyNested(m_amrlevel.crse_ratio, - m_amrlevel.parent->blockingFactor(m_amrlevel.level), - boxGrow, boxType, desc.interp(SComp))) - { - FillFromTwoLevels(time, idx, SComp, DComp, NComp); - } else { + if (level == 0) + { + FillFromLevel0(time, idx, SComp, DComp, NComp); + } + else + { + if (level == 1 || + amrex::ProperlyNested(m_amrlevel.crse_ratio, + m_amrlevel.parent->blockingFactor(m_amrlevel.level), + boxGrow, boxType, desc.interp(SComp))) + { + FillFromTwoLevels(time, idx, SComp, DComp, NComp); + } else { #ifdef AMREX_USE_EB if (EB2::TopIndexSpaceIfPresent()) { @@ -1028,21 +1028,21 @@ FillPatchIterator::Initialize (int boxGrow, } #endif - static bool first = true; - if (first) { - first = false; - if (ParallelDescriptor::IOProcessor() && amrex::Verbose()) { - IntVect new_blocking_factor = m_amrlevel.parent->blockingFactor(m_amrlevel.level); + static bool first = true; + if (first) { + first = false; + if (ParallelDescriptor::IOProcessor() && amrex::Verbose()) { + IntVect new_blocking_factor = m_amrlevel.parent->blockingFactor(m_amrlevel.level); new_blocking_factor *= 2; - for (int j = 0; j < 10; ++j) { - if (amrex::ProperlyNested(m_amrlevel.crse_ratio, - new_blocking_factor, - boxGrow, boxType, desc.interp(SComp))) { - break; - } else { - new_blocking_factor *= 2; - } - } + for (int j = 0; j < 10; ++j) { + if (amrex::ProperlyNested(m_amrlevel.crse_ratio, + new_blocking_factor, + boxGrow, boxType, desc.interp(SComp))) { + break; + } else { + new_blocking_factor *= 2; + } + } amrex::Print() << "WARNING: Grids are not properly nested. We might have to use\n" << " two coarse levels to do fillpatch. Consider using\n"; if (new_blocking_factor < IntVect{AMREX_D_DECL(128,128,128)}) { @@ -1050,32 +1050,32 @@ FillPatchIterator::Initialize (int boxGrow, } else { amrex::Print() << " larger amr.blocking_factor.\n"; } - } - } - - FillPatchIteratorHelper* fph = 0; - fph = new FillPatchIteratorHelper(m_amrlevel, - m_leveldata, - boxGrow, - time, - idx, - SComp, - NComp, - desc.interp(SComp)); - + } + } + + FillPatchIteratorHelper* fph = 0; + fph = new FillPatchIteratorHelper(m_amrlevel, + m_leveldata, + boxGrow, + time, + idx, + SComp, + NComp, + desc.interp(SComp)); + #if defined(AMREX_CRSEGRNDOMP) || (!defined(AMREX_XSDK) && defined(CRSEGRNDOMP)) #ifdef AMREX_USE_OMP #pragma omp parallel #endif #endif - for (MFIter mfi(m_fabs); mfi.isValid(); ++mfi) - { - fph->fill(m_fabs[mfi],DComp,mfi.index()); - } - - delete fph; - } - } + for (MFIter mfi(m_fabs); mfi.isValid(); ++mfi) + { + fph->fill(m_fabs[mfi],DComp,mfi.index()); + } + + delete fph; + } + } DComp += NComp; } @@ -1121,7 +1121,7 @@ FillPatchIterator::FillFromTwoLevels (Real time, int idx, int scomp, int dcomp, const Geometry& geom_fine = fine_level.geom; const Geometry& geom_crse = crse_level.geom; - + Vector smf_crse; Vector stime_crse; StateData& statedata_crse = crse_level.state[idx]; @@ -1136,14 +1136,14 @@ FillPatchIterator::FillFromTwoLevels (Real time, int idx, int scomp, int dcomp, const StateDescriptor& desc = AmrLevel::desc_lst[idx]; - amrex::FillPatchTwoLevels(m_fabs, time, - smf_crse, stime_crse, + amrex::FillPatchTwoLevels(m_fabs, time, + smf_crse, stime_crse, smf_fine, stime_fine, - scomp, dcomp, ncomp, + scomp, dcomp, ncomp, geom_crse, geom_fine, physbcf_crse, scomp, physbcf_fine, scomp, - crse_level.fineRatio(), + crse_level.fineRatio(), desc.interp(scomp), desc.getBCs(),scomp); } @@ -1295,7 +1295,7 @@ FillPatchIteratorHelper::fill (FArrayBox& fab, BL_ASSERT(CrseBoxes[i].ok()); CrseFabs[i].reset(new FArrayBox(CrseBoxes[i],m_ncomp)); CrseFabs[i]->setComplement(std::numeric_limits::quiet_NaN(), domain_box, 0, m_ncomp); - } + } for (int i = 0; i < NC; i++) { @@ -1304,14 +1304,14 @@ FillPatchIteratorHelper::fill (FArrayBox& fab, // to ensure we've filled the FABs at the coarse level. // TheState.InterpFillFab(m_mfcd, - m_mfid[l], - FBIDs[i], - *CrseFabs[i], - m_time, - 0, - 0, - m_ncomp, - extrap); + m_mfid[l], + FBIDs[i], + *CrseFabs[i], + m_time, + 0, + 0, + m_ncomp, + extrap); } } // @@ -1406,17 +1406,17 @@ FillPatchIteratorHelper::fill (FArrayBox& fab, // for (int j = 0; j < NC; j++) { crsefab.copy(*CrseFabs[j]); - } + } // // Get boundary conditions for the fine patch. // amrex::setBC(crsefab.box(), - cDomain, - m_scomp, - 0, - m_ncomp, - theBCs, - bcr); + cDomain, + m_scomp, + 0, + m_ncomp, + theBCs, + bcr); // // Interpolate up to fine patch. // @@ -1436,9 +1436,9 @@ FillPatchIteratorHelper::fill (FArrayBox& fab, // // Copy intersect finefab into next level m_cboxes. // - for (int j = 0, K = FinerCrseFabs.size(); j < K; ++j) { - FinerCrseFabs[j]->copy(finefab); - } + for (int j = 0, K = FinerCrseFabs.size(); j < K; ++j) { + FinerCrseFabs[j]->copy(finefab); + } } CrseFabs.clear(); @@ -1513,7 +1513,7 @@ AmrLevel::FillCoarsePatch (MultiFab& mf, int idx, int scomp, int ncomp, - int nghost) + int nghost) { BL_PROFILE("AmrLevel::FillCoarsePatch()"); @@ -1536,9 +1536,9 @@ AmrLevel::FillCoarsePatch (MultiFab& mf, Box domain_g = pdomain; for (int i = 0; i < AMREX_SPACEDIM; ++i) { - if (geom.isPeriodic(i)) { - domain_g.grow(i,nghost); - } + if (geom.isPeriodic(i)) { + domain_g.grow(i,nghost); + } } std::vector< std::pair > ranges = desc.sameInterps(scomp,ncomp); @@ -1552,11 +1552,11 @@ AmrLevel::FillCoarsePatch (MultiFab& mf, Interpolater* mapper = desc.interp(SComp); BoxArray crseBA(mf_BA.size()); - + for (int j = 0, N = crseBA.size(); j < N; ++j) { BL_ASSERT(mf_BA[j].ixType() == desc.getType()); - const Box& bx = amrex::grow(mf_BA[j],nghost) & domain_g; + const Box& bx = amrex::grow(mf_BA[j],nghost) & domain_g; crseBA.set(j,mapper->CoarseBox(bx, crse_ratio)); } @@ -1571,50 +1571,50 @@ AmrLevel::FillCoarsePatch (MultiFab& mf, crseMF.define(crseBA,mf_DM,NComp,0); } - if ( level == 1 - || amrex::ProperlyNested(crse_ratio, parent->blockingFactor(level), - nghost, mf_BA.ixType(), mapper) ) - { - StateData& statedata = clev.state[idx]; - - Vector smf; - Vector stime; - statedata.getData(smf,stime,time); + if ( level == 1 + || amrex::ProperlyNested(crse_ratio, parent->blockingFactor(level), + nghost, mf_BA.ixType(), mapper) ) + { + StateData& statedata = clev.state[idx]; - StateDataPhysBCFunct physbcf(statedata,SComp,cgeom); + Vector smf; + Vector stime; + statedata.getData(smf,stime,time); + + StateDataPhysBCFunct physbcf(statedata,SComp,cgeom); crseMF.setDomainBndry(std::numeric_limits::quiet_NaN(), cgeom); - amrex::FillPatchSingleLevel(crseMF,time,smf,stime,SComp,0,NComp,cgeom,physbcf,SComp); - } - else - { - FillPatch(clev,crseMF,0,time,idx,SComp,NComp,0); - } + amrex::FillPatchSingleLevel(crseMF,time,smf,stime,SComp,0,NComp,cgeom,physbcf,SComp); + } + else + { + FillPatch(clev,crseMF,0,time,idx,SComp,NComp,0); + } #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(mf); mfi.isValid(); ++mfi) - { + for (MFIter mfi(mf); mfi.isValid(); ++mfi) + { const Box& dbx = amrex::grow(mfi.validbox(),nghost) & domain_g; - + Vector bcr(ncomp); - - amrex::setBC(crseMF[mfi].box(),cdomain,SComp,0,NComp,desc.getBCs(),bcr); - - mapper->interp(crseMF[mfi], - 0, - mf[mfi], - DComp, - NComp, - dbx, - crse_ratio, - cgeom, - geom, - bcr, - SComp, - idx, RunOn::Gpu); - } + + amrex::setBC(crseMF[mfi].box(),cdomain,SComp,0,NComp,desc.getBCs(),bcr); + + mapper->interp(crseMF[mfi], + 0, + mf[mfi], + DComp, + NComp, + dbx, + crse_ratio, + cgeom, + geom, + bcr, + SComp, + idx, RunOn::Gpu); + } if (nghost > 0) { StateDataPhysBCFunct physbcf(state[idx],SComp,geom); @@ -1648,13 +1648,13 @@ AmrLevel::derive (const std::string& name, Real time, int ngrow) BoxArray dstBA(srcBA); dstBA.convert(rec->deriveType()); - int ngrow_src = ngrow; - { - Box bx0 = srcBA[0]; - Box bx1 = rec->boxMap()(bx0); - int g = bx0.smallEnd(0) - bx1.smallEnd(0); - ngrow_src += g; - } + int ngrow_src = ngrow; + { + Box bx0 = srcBA[0]; + Box bx1 = rec->boxMap()(bx0); + int g = bx0.smallEnd(0) - bx1.smallEnd(0); + ngrow_src += g; + } MultiFab srcMF(srcBA, dmap, rec->numState(), ngrow_src, MFInfo(), *m_factory); @@ -1692,9 +1692,9 @@ AmrLevel::derive (const std::string& name, Real time, int ngrow) Real* ddat = (*mf)[mfi].dataPtr(); const int* dlo = (*mf)[mfi].loVect(); const int* dhi = (*mf)[mfi].hiVect(); - const Box& gtbx = mfi.growntilebox(); - const int* lo = gtbx.loVect(); - const int* hi = gtbx.hiVect(); + const Box& gtbx = mfi.growntilebox(); + const int* lo = gtbx.loVect(); + const int* hi = gtbx.hiVect(); int n_der = rec->numDerive(); Real* cdat = srcMF[mfi].dataPtr(); const int* clo = srcMF[mfi].loVect(); @@ -1801,13 +1801,13 @@ AmrLevel::derive (const std::string& name, Real time, MultiFab& mf, int dcomp) const BoxArray& srcBA = state[index].boxArray(); - int ngrow_src = ngrow; - { - Box bx0 = srcBA[0]; - Box bx1 = rec->boxMap()(bx0); - int g = bx0.smallEnd(0) - bx1.smallEnd(0); - ngrow_src += g; - } + int ngrow_src = ngrow; + { + Box bx0 = srcBA[0]; + Box bx1 = rec->boxMap()(bx0); + int g = bx0.smallEnd(0) - bx1.smallEnd(0); + ngrow_src += g; + } MultiFab srcMF(srcBA,dmap,rec->numState(),ngrow_src, MFInfo(), *m_factory); @@ -1985,14 +1985,14 @@ AmrLevel::setPlotVariables () if (pp.contains("plot_vars")) { std::string nm; - + int nPltVars = pp.countval("plot_vars"); - + for (int i = 0; i < nPltVars; i++) { pp.get("plot_vars", nm, i); - if (nm == "ALL") + if (nm == "ALL") parent->fillStatePlotVarList(); else if (nm == "NONE") parent->clearStatePlotVarList(); @@ -2000,25 +2000,25 @@ AmrLevel::setPlotVariables () parent->addStatePlotVar(nm); } } - else + else { // // The default is to add them all. // parent->fillStatePlotVarList(); } - + if (pp.contains("derive_plot_vars")) { std::string nm; - + int nDrvPltVars = pp.countval("derive_plot_vars"); - + for (int i = 0; i < nDrvPltVars; i++) { pp.get("derive_plot_vars", nm, i); - if (nm == "ALL") + if (nm == "ALL") parent->fillDerivePlotVarList(); else if (nm == "NONE") parent->clearDerivePlotVarList(); @@ -2026,7 +2026,7 @@ AmrLevel::setPlotVariables () parent->addDerivePlotVar(nm); } } - else + else { // // The default is to add none of them. @@ -2043,9 +2043,9 @@ AmrLevel::setSmallPlotVariables () if (pp.contains("small_plot_vars")) { std::string nm; - + int nPltVars = pp.countval("small_plot_vars"); - + for (int i = 0; i < nPltVars; i++) { pp.get("small_plot_vars", nm, i); @@ -2058,7 +2058,7 @@ AmrLevel::setSmallPlotVariables () parent->addStateSmallPlotVar(nm); } } - else + else { // // The default is to use none. @@ -2069,14 +2069,14 @@ AmrLevel::setSmallPlotVariables () if (pp.contains("derive_small_plot_vars")) { std::string nm; - + int nDrvPltVars = pp.countval("derive_small_plot_vars"); - + for (int i = 0; i < nDrvPltVars; i++) { pp.get("derive_small_plot_vars", nm, i); - if (nm == "ALL") + if (nm == "ALL") parent->fillDeriveSmallPlotVarList(); else if (nm == "NONE") parent->clearDeriveSmallPlotVarList(); @@ -2084,14 +2084,14 @@ AmrLevel::setSmallPlotVariables () parent->addDeriveSmallPlotVar(nm); } } - else + else { // // The default is to add none of them. // parent->clearDeriveSmallPlotVarList(); } - + } AmrLevel::TimeLevel @@ -2105,7 +2105,7 @@ AmrLevel::which_time (int indx, Real time) const noexcept const Real epsilon = 0.001_rt * (newtime - oldtime); BL_ASSERT(time >= oldtime-epsilon && time <= newtime+epsilon); - + if (time >= oldtime-epsilon && time <= oldtime+epsilon) { return AmrOldTime; @@ -2171,8 +2171,8 @@ void AmrLevel::constructAreaNotToTag () if (parent->useFixedUpToLevel()==level) { // We use the next coarser level shrunk by one blockingfactor - // as the region in which we allow tagging. - // Why level-1? Because we always use the full domain at level 0 + // as the region in which we allow tagging. + // Why level-1? Because we always use the full domain at level 0 // and therefore level 0 in initialba is level 1 in the AMR hierarchy, etc. const Vector& initialba = parent->getInitialBA(); Box tagarea(initialba[level-1].minimalBox()); @@ -2200,7 +2200,7 @@ void AmrLevel::constructAreaNotToTag () void AmrLevel::FillPatch (AmrLevel& amrlevel, - MultiFab& leveldata, + MultiFab& leveldata, int boxGrow, Real time, int index, diff --git a/Src/Amr/AMReX_AuxBoundaryData.H b/Src/Amr/AMReX_AuxBoundaryData.H index 9c284d24a18..32a10beef36 100644 --- a/Src/Amr/AMReX_AuxBoundaryData.H +++ b/Src/Amr/AMReX_AuxBoundaryData.H @@ -32,7 +32,7 @@ public: int src_comp, int dst_comp, int num_comp, - int src_ng = 0); + int src_ng = 0); size_t size () const noexcept { @@ -45,7 +45,7 @@ public: int num_comp); void initialize (const BoxArray& grids, - int n_grow, + int n_grow, int n_comp, const Geometry& geom); diff --git a/Src/Amr/AMReX_AuxBoundaryData.cpp b/Src/Amr/AMReX_AuxBoundaryData.cpp index 145270895d2..4b6a2f1e0d2 100644 --- a/Src/Amr/AMReX_AuxBoundaryData.cpp +++ b/Src/Amr/AMReX_AuxBoundaryData.cpp @@ -55,8 +55,8 @@ AuxBoundaryData::AuxBoundaryData (const AuxBoundaryData& rhs) void AuxBoundaryData::initialize (const BoxArray& ba, - int n_grow, - int n_comp, + int n_grow, + int n_comp, const Geometry& geom) { BL_ASSERT(!m_initialized); @@ -109,15 +109,15 @@ AuxBoundaryData::initialize (const BoxArray& ba, { const int IOProc = ParallelDescriptor::IOProcessorNumber(); auto run_time = amrex::second() - strt_time; - const int sz = nba.size(); + const int sz = nba.size(); #ifdef BL_LAZY - Lazy::QueueReduction( [=] () mutable { + Lazy::QueueReduction( [=] () mutable { #endif - ParallelDescriptor::ReduceRealMax(run_time,IOProc); - amrex::Print() << "AuxBoundaryData::initialize() size = " << sz << ", time = " << run_time << '\n'; + ParallelDescriptor::ReduceRealMax(run_time,IOProc); + amrex::Print() << "AuxBoundaryData::initialize() size = " << sz << ", time = " << run_time << '\n'; #ifdef BL_LAZY - }); + }); #endif } @@ -143,13 +143,13 @@ AuxBoundaryData::copyFrom (const MultiFab& mf, int src_comp, int dst_comp, int num_comp, - int src_ng) + int src_ng) { BL_ASSERT(m_initialized); if (!m_empty && mf.size() > 0) { - m_fabs.copy(mf,src_comp,dst_comp,num_comp,src_ng,0); + m_fabs.copy(mf,src_comp,dst_comp,num_comp,src_ng,0); } } diff --git a/Src/Amr/AMReX_Derive.H b/Src/Amr/AMReX_Derive.H index 9886c3cb1e9..5748801e685 100644 --- a/Src/Amr/AMReX_Derive.H +++ b/Src/Amr/AMReX_Derive.H @@ -75,13 +75,13 @@ extern "C" * \param grid_no */ typedef void (*DeriveFunc3D)(amrex::Real* data, const int* dlo, const int* dhi, const int* nvar, - const amrex::Real* compdat, const int* clo, const int* chi, const int* ncomp, - const int* lo, const int* hi, - const int* domain_lo, const int* domain_hi, - const amrex::Real* delta, const amrex::Real* xlo, - const amrex::Real* time, const amrex::Real* dt, - const int* bcrec, - const int* level, const int* grid_no) ; + const amrex::Real* compdat, const int* clo, const int* chi, const int* ncomp, + const int* lo, const int* hi, + const int* domain_lo, const int* domain_hi, + const amrex::Real* delta, const amrex::Real* xlo, + const amrex::Real* time, const amrex::Real* dt, + const int* bcrec, + const int* level, const int* grid_no) ; } typedef void (*DeriveFuncFab) (const amrex::Box& bx, amrex::FArrayBox& derfab, int dcomp, int ncomp, @@ -252,7 +252,7 @@ protected: DeriveRec (const std::string& name, IndexType result_type, int nvar_derive, - Vector& var_names, + Vector& var_names, DeriveFunc der_func, DeriveBoxMap box_map, Interpolater* interp = &pc_interp); @@ -260,7 +260,7 @@ protected: DeriveRec (const std::string& name, IndexType result_type, int nvar_derive, - Vector& var_names, + Vector& var_names, DeriveFunc3D der_func_3d, DeriveBoxMap box_map, Interpolater* interp = &pc_interp); @@ -268,7 +268,7 @@ protected: DeriveRec (const std::string& name, IndexType result_type, int nvar_derive, - Vector& var_names, + Vector& var_names, DeriveFuncFab der_func_fab, DeriveBoxMap box_map, Interpolater* interp = &pc_interp); diff --git a/Src/Amr/AMReX_Derive.cpp b/Src/Amr/AMReX_Derive.cpp index de7893f7279..c0d1fb54d7e 100644 --- a/Src/Amr/AMReX_Derive.cpp +++ b/Src/Amr/AMReX_Derive.cpp @@ -83,7 +83,7 @@ DeriveRec::DeriveRec (const std::string& a_name, DeriveRec::DeriveRec (const std::string& a_name, IndexType result_type, int nvar_derive, - Vector& var_names, + Vector& var_names, DeriveFunc der_func, DeriveBoxMap box_map, Interpolater* a_interp) @@ -100,7 +100,7 @@ DeriveRec::DeriveRec (const std::string& a_name, DeriveRec::DeriveRec (const std::string& a_name, IndexType result_type, int nvar_derive, - Vector& var_names, + Vector& var_names, DeriveFunc3D der_func_3d, DeriveBoxMap box_map, Interpolater* a_interp) @@ -117,7 +117,7 @@ DeriveRec::DeriveRec (const std::string& a_name, DeriveRec::DeriveRec (const std::string& a_name, IndexType result_type, int nvar_derive, - Vector& var_names, + Vector& var_names, DeriveFuncFab der_func_fab, DeriveBoxMap box_map, Interpolater* a_interp) @@ -131,7 +131,7 @@ DeriveRec::DeriveRec (const std::string& a_name, bx_map(box_map) {} -DeriveRec::~DeriveRec () +DeriveRec::~DeriveRec () { delete [] bcr; delete [] bcr3D; @@ -226,7 +226,7 @@ void DeriveRec::addRange (const DescriptorList& d_list, int state_indx, int src_comp, - int num_comp) + int num_comp) { StateRange* r = new StateRange; @@ -328,7 +328,7 @@ const std::string& DeriveRec::variableName(int comp) const noexcept { - if (comp < variable_names.size()) + if (comp < variable_names.size()) return variable_names[comp]; return derive_name; @@ -422,7 +422,7 @@ DeriveList::dlist () } bool -DeriveList::canDerive (const std::string& name) const +DeriveList::canDerive (const std::string& name) const { for (std::list::const_iterator li = lst.begin(), End = lst.end(); li != End; @@ -449,7 +449,7 @@ DeriveList::get (const std::string& name) const void DeriveList::addComponent (const std::string& name, - const DescriptorList& d_list, + const DescriptorList& d_list, int state_indx, int s_comp, int n_comp) diff --git a/Src/Amr/AMReX_StateData.H b/Src/Amr/AMReX_StateData.H index 3d35ffae9a7..6014815658b 100644 --- a/Src/Amr/AMReX_StateData.H +++ b/Src/Amr/AMReX_StateData.H @@ -54,7 +54,7 @@ public: */ StateData (const Box& p_domain, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, const StateDescriptor* d, Real cur_time, Real dt, @@ -85,7 +85,7 @@ public: */ void define (const Box& p_domain, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, const StateDescriptor& d, Real cur_time, Real dt, @@ -193,25 +193,25 @@ public: Vector& mfid); void InterpAddBox (MultiFabCopyDescriptor& multiFabCopyDesc, - Vector& mfid, - BoxList* returnedUnfillableBoxes, - Vector& returnedFillBoxIds, - const Box& subbox, - Real time, - int src_comp, - int dest_comp, - int num_comp, - bool extrap = false); + Vector& mfid, + BoxList* returnedUnfillableBoxes, + Vector& returnedFillBoxIds, + const Box& subbox, + Real time, + int src_comp, + int dest_comp, + int num_comp, + bool extrap = false); void InterpFillFab (MultiFabCopyDescriptor& fabCopyDesc, - const Vector& mfid, - const Vector& fillBoxIds, - FArrayBox& dest, - Real time, - int src_comp, - int dest_comp, - int num_comp, - bool extrap = false); + const Vector& mfid, + const Vector& fillBoxIds, + FArrayBox& dest, + Real time, + int src_comp, + int dest_comp, + int num_comp, + bool extrap = false); /** @@ -268,9 +268,9 @@ public: * \param restart_file */ void restart (std::istream& is, - const Box& p_domain, - const BoxArray& grds, - const DistributionMapping& dm, + const Box& p_domain, + const BoxArray& grds, + const DistributionMapping& dm, const FabFactory& factroy, const StateDescriptor& d, const std::string& restart_file); @@ -282,7 +282,7 @@ public: * \param rhs */ void restart (const StateDescriptor& d, - const StateData& rhs); + const StateData& rhs); /** * \brief Returns the StateDescriptor. @@ -313,16 +313,16 @@ public: * \brief Returns the current time. */ Real curTime () const noexcept { - return (desc->timeType() == StateDescriptor::Point) ? - new_time.stop : 0.5_rt*(new_time.start + new_time.stop); + return (desc->timeType() == StateDescriptor::Point) ? + new_time.stop : 0.5_rt*(new_time.start + new_time.stop); } /** * \brief Returns the previous time. */ Real prevTime () const noexcept { - return (desc->timeType() == StateDescriptor::Point) ? - old_time.stop : 0.5_rt*(old_time.start + old_time.stop); + return (desc->timeType() == StateDescriptor::Point) ? + old_time.stop : 0.5_rt*(old_time.start + old_time.stop); } /** @@ -385,8 +385,8 @@ public: bool hasNewData () const noexcept { return new_data != nullptr; } void getData (Vector& data, - Vector& datatime, - Real time) const; + Vector& datatime, + Real time) const; /** * \brief Get the Arena used. @@ -461,7 +461,7 @@ class StateDataPhysBCFunct public: StateDataPhysBCFunct (StateData& sd, int sc, const Geometry& geom_); - + void operator() (MultiFab& mf, int dcomp, int ncomp, IntVect const& nghost, Real time, int bccomp); diff --git a/Src/Amr/AMReX_StateData.cpp b/Src/Amr/AMReX_StateData.cpp index bf74c619ccf..185d9deaeb7 100644 --- a/Src/Amr/AMReX_StateData.cpp +++ b/Src/Amr/AMReX_StateData.cpp @@ -28,7 +28,7 @@ Vector StateData::fabArrayHeaderNames; std::map > *StateData::faHeaderMap; -StateData::StateData () +StateData::StateData () : desc(nullptr), new_time{INVALID_TIME,INVALID_TIME}, old_time{INVALID_TIME,INVALID_TIME}, @@ -38,7 +38,7 @@ StateData::StateData () StateData::StateData (const Box& p_domain, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, const StateDescriptor* d, Real cur_time, Real dt, @@ -58,7 +58,7 @@ StateData::StateData (StateData&& rhs) noexcept new_data(std::move(rhs.new_data)), old_data(std::move(rhs.old_data)), arena(rhs.arena) -{ +{ } void @@ -89,7 +89,7 @@ StateData::operator= (StateData const& rhs) void StateData::define (const Box& p_domain, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, const StateDescriptor& d, Real time, Real dt, @@ -136,15 +136,15 @@ void StateData::copyOld (const StateData& state) { const MultiFab& MF = state.oldData(); - + int nc = MF.nComp(); int ng = MF.nGrow(); - + BL_ASSERT(nc == (*old_data).nComp()); BL_ASSERT(ng == (*old_data).nGrow()); - + MultiFab::Copy(*old_data, state.oldData(), 0, 0, nc, ng); - + old_time = state.old_time; } @@ -155,10 +155,10 @@ StateData::copyNew (const StateData& state) int nc = MF.nComp(); int ng = MF.nGrow(); - + BL_ASSERT(nc == (*new_data).nComp()); BL_ASSERT(ng == (*new_data).nGrow()); - + MultiFab::Copy(*new_data, state.newData(), 0, 0, nc, ng); new_time = state.new_time; @@ -174,9 +174,9 @@ StateData::reset () void StateData::restart (std::istream& is, - const Box& p_domain, - const BoxArray& grds, - const DistributionMapping& dm, + const Box& p_domain, + const BoxArray& grds, + const DistributionMapping& dm, const FabFactory& factory, const StateDescriptor& d, const std::string& chkfile) @@ -196,18 +196,18 @@ StateData::restart (std::istream& is, } { - Box domain_in; - BoxArray grids_in; - is >> domain_in; - grids_in.readFrom(is); - BL_ASSERT(domain_in == domain); - BL_ASSERT(amrex::match(grids_in,grids)); + Box domain_in; + BoxArray grids_in; + is >> domain_in; + grids_in.readFrom(is); + BL_ASSERT(domain_in == domain); + BL_ASSERT(amrex::match(grids_in,grids)); } restartDoit(is, chkfile); } -void +void StateData::restartDoit (std::istream& is, const std::string& chkfile) { BL_PROFILE("StateData::restartDoit()"); @@ -230,7 +230,7 @@ StateData::restartDoit (std::istream& is, const std::string& chkfile) *m_factory)); } // - // If no data is written then we just allocate the MF instead of reading it in. + // If no data is written then we just allocate the MF instead of reading it in. // This assumes that the application will do something with it. // We set it to zero in case a compiler complains about uninitialized data. // @@ -242,44 +242,44 @@ StateData::restartDoit (std::istream& is, const std::string& chkfile) std::string FullPathName; for(int ns(1); ns <= nsets; ++ns) { - MultiFab *whichMF = nullptr; - if(ns == 1) { - whichMF = new_data.get(); - } else if(ns == 2) { - whichMF = old_data.get(); - } else { - amrex::Abort("**** Error in StateData::restart: invalid nsets."); - } - - is >> mf_name; - // - // Note that mf_name is relative to the Header file. - // We need to prepend the name of the chkfile directory. - // - FullPathName = chkfile; - if ( ! chkfile.empty() && chkfile[chkfile.length()-1] != '/') { - FullPathName += '/'; - } - FullPathName += mf_name; - - // ---- check for preread header - std::string FullHeaderPathName(FullPathName + "_H"); - const char *faHeader = 0; - if(faHeaderMap != 0) { - std::map >::iterator fahmIter; - fahmIter = faHeaderMap->find(FullHeaderPathName); - if(fahmIter != faHeaderMap->end()) { - faHeader = fahmIter->second.dataPtr(); - } - } - - VisMF::Read(*whichMF, FullPathName, faHeader); + MultiFab *whichMF = nullptr; + if(ns == 1) { + whichMF = new_data.get(); + } else if(ns == 2) { + whichMF = old_data.get(); + } else { + amrex::Abort("**** Error in StateData::restart: invalid nsets."); + } + + is >> mf_name; + // + // Note that mf_name is relative to the Header file. + // We need to prepend the name of the chkfile directory. + // + FullPathName = chkfile; + if ( ! chkfile.empty() && chkfile[chkfile.length()-1] != '/') { + FullPathName += '/'; + } + FullPathName += mf_name; + + // ---- check for preread header + std::string FullHeaderPathName(FullPathName + "_H"); + const char *faHeader = 0; + if(faHeaderMap != 0) { + std::map >::iterator fahmIter; + fahmIter = faHeaderMap->find(FullHeaderPathName); + if(fahmIter != faHeaderMap->end()) { + faHeader = fahmIter->second.dataPtr(); + } + } + + VisMF::Read(*whichMF, FullPathName, faHeader); } } -void +void StateData::restart (const StateDescriptor& d, - const StateData& rhs) + const StateData& rhs) { desc = &d; arena = nullptr; @@ -352,14 +352,14 @@ StateData::syncNewTimeLevel (Real time) Real teps = (new_time.stop - old_time.stop)*1.e-3_rt; if (time > new_time.stop-teps && time < new_time.stop+teps) { - if (desc->timeType() == StateDescriptor::Point) - { - new_time.start = new_time.stop = time; - } - else - { - new_time.stop = time; - } + if (desc->timeType() == StateDescriptor::Point) + { + new_time.start = new_time.stop = time; + } + else + { + new_time.stop = time; + } } } @@ -438,7 +438,7 @@ StateData::FillBoundary (FArrayBox& dest, { BL_PROFILE("StateData::FillBoundary(dx)"); BL_ASSERT(dest.box().ixType() == desc->getType()); - + if (domain.contains(dest.box())) return; const Box& bx = dest.box(); @@ -492,7 +492,7 @@ StateData::FillBoundary (FArrayBox& dest, // // Use the "group" boundary fill routine. // - desc->bndryFill(sc)(dat,dlo,dhi,plo,phi,dx,xlo,&time,bcrs.dataPtr(),groupsize); + desc->bndryFill(sc)(dat,dlo,dhi,plo,phi,dx,xlo,&time,bcrs.dataPtr(),groupsize); i += groupsize; } else @@ -528,7 +528,7 @@ StateData::FillBoundary (Box const& bx, { BL_PROFILE("StateData::FillBoundary(geom)"); BL_ASSERT(bx.ixType() == desc->getType()); - + if (domain.contains(bx)) return; Vector bcr(num_comp); @@ -553,7 +553,7 @@ StateData::FillBoundary (Box const& bx, // // Use the "group" boundary fill routine. // - desc->bndryFill(sc)(bx,dest,dc,groupsize,geom,time,bcr,0,sc); + desc->bndryFill(sc)(bx,dest,dc,groupsize,geom,time,bcr,0,sc); i += groupsize; } else @@ -583,15 +583,15 @@ StateData::RegisterData (MultiFabCopyDescriptor& multiFabCopyDesc, void StateData::InterpAddBox (MultiFabCopyDescriptor& multiFabCopyDesc, - Vector& mfid, - BoxList* unfillableBoxes, - Vector& returnedFillBoxIds, - const Box& subbox, - Real time, - int src_comp, - int dest_comp, - int num_comp, - bool extrap) + Vector& mfid, + BoxList* unfillableBoxes, + Vector& returnedFillBoxIds, + const Box& subbox, + Real time, + int src_comp, + int dest_comp, + int num_comp, + bool extrap) { if (desc->timeType() == StateDescriptor::Point) { @@ -608,18 +608,18 @@ StateData::InterpAddBox (MultiFabCopyDescriptor& multiFabCopyDesc, else { amrex::InterpAddBox(multiFabCopyDesc, - unfillableBoxes, - returnedFillBoxIds, - subbox, - mfid[MFOLDDATA], - mfid[MFNEWDATA], - old_time.start, - new_time.start, - time, - src_comp, - dest_comp, - num_comp, - extrap); + unfillableBoxes, + returnedFillBoxIds, + subbox, + mfid[MFOLDDATA], + mfid[MFNEWDATA], + old_time.start, + new_time.start, + time, + src_comp, + dest_comp, + num_comp, + extrap); } } else @@ -657,14 +657,14 @@ StateData::InterpAddBox (MultiFabCopyDescriptor& multiFabCopyDesc, void StateData::InterpFillFab (MultiFabCopyDescriptor& multiFabCopyDesc, - const Vector& mfid, - const Vector& fillBoxIds, - FArrayBox& dest, - Real time, - int src_comp, - int dest_comp, - int num_comp, - bool extrap) + const Vector& mfid, + const Vector& fillBoxIds, + FArrayBox& dest, + Real time, + int src_comp, + int dest_comp, + int num_comp, + bool extrap) { BL_PROFILE("StateData::InterpFillFab()"); if (desc->timeType() == StateDescriptor::Point) @@ -676,17 +676,17 @@ StateData::InterpFillFab (MultiFabCopyDescriptor& multiFabCopyDesc, else { amrex::InterpFillFab(multiFabCopyDesc, - fillBoxIds, - mfid[MFOLDDATA], - mfid[MFNEWDATA], - dest, - old_time.start, - new_time.start, - time, - src_comp, - dest_comp, - num_comp, - extrap); + fillBoxIds, + mfid[MFOLDDATA], + mfid[MFNEWDATA], + dest, + old_time.start, + new_time.start, + time, + src_comp, + dest_comp, + num_comp, + extrap); } } else @@ -712,35 +712,35 @@ StateData::InterpFillFab (MultiFabCopyDescriptor& multiFabCopyDesc, void StateData::getData (Vector& data, - Vector& datatime, - Real time) const + Vector& datatime, + Real time) const { data.clear(); datatime.clear(); if (desc->timeType() == StateDescriptor::Point) { - BL_ASSERT(new_data != nullptr); + BL_ASSERT(new_data != nullptr); if (old_data == nullptr) { - data.push_back(new_data.get()); - datatime.push_back(new_time.start); + data.push_back(new_data.get()); + datatime.push_back(new_time.start); } else { - const Real teps = (new_time.start - old_time.start)*1.e-3_rt; - if (time > new_time.start-teps && time < new_time.start+teps) { - data.push_back(new_data.get()); - datatime.push_back(new_time.start); - } else if (time > old_time.start-teps && time < old_time.start+teps) { - data.push_back(old_data.get()); - datatime.push_back(old_time.start); - } else { - data.push_back(old_data.get()); - data.push_back(new_data.get()); - datatime.push_back(old_time.start); - datatime.push_back(new_time.start); - } + const Real teps = (new_time.start - old_time.start)*1.e-3_rt; + if (time > new_time.start-teps && time < new_time.start+teps) { + data.push_back(new_data.get()); + datatime.push_back(new_time.start); + } else if (time > old_time.start-teps && time < old_time.start+teps) { + data.push_back(old_data.get()); + datatime.push_back(old_time.start); + } else { + data.push_back(old_data.get()); + data.push_back(new_data.get()); + datatime.push_back(old_time.start); + datatime.push_back(new_time.start); + } } } else @@ -749,15 +749,15 @@ StateData::getData (Vector& data, if (time > new_time.start-teps && time < new_time.stop+teps) { - data.push_back(new_data.get()); - datatime.push_back(time); + data.push_back(new_data.get()); + datatime.push_back(time); } else if (old_data != nullptr && time > old_time.start-teps && time < old_time.stop+teps) { - data.push_back(old_data.get()); - datatime.push_back(time); + data.push_back(old_data.get()); + datatime.push_back(time); } else { @@ -799,18 +799,18 @@ StateData::checkPoint (const std::string& name, << new_time.start << '\n' << new_time.stop << '\n'; - if (desc->store_in_checkpoint()) + if (desc->store_in_checkpoint()) { if (dump_old) { os << 2 << '\n' << mf_name_new << '\n' << mf_name_old << '\n'; - fabArrayHeaderNames.push_back(mf_name_new); - fabArrayHeaderNames.push_back(mf_name_old); + fabArrayHeaderNames.push_back(mf_name_new); + fabArrayHeaderNames.push_back(mf_name_old); } else { os << 1 << '\n' << mf_name_new << '\n'; - fabArrayHeaderNames.push_back(mf_name_new); + fabArrayHeaderNames.push_back(mf_name_new); } } else @@ -821,24 +821,24 @@ StateData::checkPoint (const std::string& name, if (desc->store_in_checkpoint()) { - BL_ASSERT(new_data); - std::string mf_fullpath_new(fullpathname + NewSuffix); - if (AsyncOut::UseAsyncOut()) { - VisMF::AsyncWrite(*new_data,mf_fullpath_new); - } else { - VisMF::Write(*new_data,mf_fullpath_new,how); - } - - if (dump_old) - { - BL_ASSERT(old_data); - std::string mf_fullpath_old(fullpathname + OldSuffix); - if (AsyncOut::UseAsyncOut()) { - VisMF::AsyncWrite(*old_data,mf_fullpath_old); - } else { - VisMF::Write(*old_data,mf_fullpath_old,how); - } - } + BL_ASSERT(new_data); + std::string mf_fullpath_new(fullpathname + NewSuffix); + if (AsyncOut::UseAsyncOut()) { + VisMF::AsyncWrite(*new_data,mf_fullpath_new); + } else { + VisMF::Write(*new_data,mf_fullpath_new,how); + } + + if (dump_old) + { + BL_ASSERT(old_data); + std::string mf_fullpath_old(fullpathname + OldSuffix); + if (AsyncOut::UseAsyncOut()) { + VisMF::AsyncWrite(*old_data,mf_fullpath_old); + } else { + VisMF::Write(*old_data,mf_fullpath_old,how); + } + } } } @@ -884,61 +884,61 @@ StateDataPhysBCFunct::operator() (MultiFab& mf, int dest_comp, int num_comp, Int #endif #endif { - FArrayBox tmp; + FArrayBox tmp; - for (MFIter mfi(mf); mfi.isValid(); ++mfi) - { + for (MFIter mfi(mf); mfi.isValid(); ++mfi) + { FArrayBox& dest = mf[mfi]; Array4 const& desta = dest.array(); const Box& bx = dest.box(); - - bool has_phys_bc = false; - bool is_periodic = false; - for (int i = 0; i < AMREX_SPACEDIM; ++i) { - bool touch = bx.smallEnd(i) < domainlo[i] || bx.bigEnd(i) > domainhi[i]; - if (geom.isPeriodic(i)) { - is_periodic = is_periodic || touch; - } else { - has_phys_bc = has_phys_bc || touch; - } - } - - if (has_phys_bc) - { + + bool has_phys_bc = false; + bool is_periodic = false; + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + bool touch = bx.smallEnd(i) < domainlo[i] || bx.bigEnd(i) > domainhi[i]; + if (geom.isPeriodic(i)) { + is_periodic = is_periodic || touch; + } else { + has_phys_bc = has_phys_bc || touch; + } + } + + if (has_phys_bc) + { if (has_bndryfunc_fab) { statedata->FillBoundary(bx, dest, time, geom, dest_comp, src_comp, num_comp); } else { statedata->FillBoundary(dest, time, dx, prob_domain, dest_comp, src_comp, num_comp); } - - if (is_periodic) // fix up corner - { - Box GrownDomain = domain; - - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - if (!geom.isPeriodic(dir)) - { - const int lo = domainlo[dir] - bx.smallEnd(dir); - const int hi = bx.bigEnd(dir) - domainhi[dir]; - if (lo > 0) GrownDomain.growLo(dir,lo); - if (hi > 0) GrownDomain.growHi(dir,hi); - } - } - - for (int dir = 0; dir < AMREX_SPACEDIM; dir++) - { - if (!geom.isPeriodic(dir)) continue; - - Box lo_slab = bx; - Box hi_slab = bx; - lo_slab.shift(dir, domain.length(dir)); - hi_slab.shift(dir,-domain.length(dir)); - lo_slab &= GrownDomain; - hi_slab &= GrownDomain; - - if (lo_slab.ok()) - { + + if (is_periodic) // fix up corner + { + Box GrownDomain = domain; + + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + if (!geom.isPeriodic(dir)) + { + const int lo = domainlo[dir] - bx.smallEnd(dir); + const int hi = bx.bigEnd(dir) - domainhi[dir]; + if (lo > 0) GrownDomain.growLo(dir,lo); + if (hi > 0) GrownDomain.growHi(dir,hi); + } + } + + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) + { + if (!geom.isPeriodic(dir)) continue; + + Box lo_slab = bx; + Box hi_slab = bx; + lo_slab.shift(dir, domain.length(dir)); + hi_slab.shift(dir,-domain.length(dir)); + lo_slab &= GrownDomain; + hi_slab &= GrownDomain; + + if (lo_slab.ok()) + { if (run_on_gpu) { tmp.resize(lo_slab,num_comp); @@ -1001,10 +1001,10 @@ StateDataPhysBCFunct::operator() (MultiFab& mf, int dest_comp, int num_comp, Int } dest.copy(tmp, lo_slab, 0, db, dest_comp, num_comp); } - } - - if (hi_slab.ok()) - { + } + + if (hi_slab.ok()) + { if (run_on_gpu) { tmp.resize(hi_slab,num_comp); @@ -1067,11 +1067,11 @@ StateDataPhysBCFunct::operator() (MultiFab& mf, int dest_comp, int num_comp, Int } dest.copy(tmp, hi_slab, 0, db, dest_comp, num_comp); } - } - } - } - } - } + } + } + } + } + } } } diff --git a/Src/Amr/AMReX_StateDescriptor.H b/Src/Amr/AMReX_StateDescriptor.H index d06de01349f..38061caf101 100644 --- a/Src/Amr/AMReX_StateDescriptor.H +++ b/Src/Amr/AMReX_StateDescriptor.H @@ -82,23 +82,23 @@ public: */ virtual ~BndryFunc (); - /** - * Fill boundary cells using "regular" function. - * - * \param data - * \param lo - * \param hi - * \param dom_lo - * \param dom_hi - * \param dx - * \param grd_lo - * \param time - * \param bc - */ - virtual void operator () (Real* data, const int* lo, const int* hi, - const int* dom_lo, const int* dom_hi, - const Real* dx, const Real* grd_lo, - const Real* time, const int* bc) const; + /** + * Fill boundary cells using "regular" function. + * + * \param data + * \param lo + * \param hi + * \param dom_lo + * \param dom_hi + * \param dx + * \param grd_lo + * \param time + * \param bc + */ + virtual void operator () (Real* data, const int* lo, const int* hi, + const int* dom_lo, const int* dom_hi, + const Real* dx, const Real* grd_lo, + const Real* time, const int* bc) const; /** * \brief Fill boundary cells using "group" function. @@ -438,8 +438,8 @@ private: static int bf_ext_dir_threadsafe; static bool bf_thread_safety (const int* lo,const int* hi, - const int* dom_lo, const int* dom_hi, - const int* bc, int ng); + const int* dom_lo, const int* dom_hi, + const int* bc, int ng); }; /** diff --git a/Src/Amr/AMReX_StateDescriptor.cpp b/Src/Amr/AMReX_StateDescriptor.cpp index e66d5566e43..8f05bf365e8 100644 --- a/Src/Amr/AMReX_StateDescriptor.cpp +++ b/Src/Amr/AMReX_StateDescriptor.cpp @@ -21,16 +21,16 @@ StateDescriptor::BndryFunc::~BndryFunc () {} bool StateDescriptor::bf_thread_safety (const int* /*lo*/,const int* /*hi*/, - const int* /*dom_lo*/, const int* /*dom_hi*/, - const int* bc, int ng) + const int* /*dom_lo*/, const int* /*dom_hi*/, + const int* bc, int ng) { bool thread_safe = true; if (!bf_ext_dir_threadsafe) { - bool has_ext_dir = false; - for (int i=0; i<2*AMREX_SPACEDIM*ng && !has_ext_dir; ++i) { - has_ext_dir = bc[i]==BCType::ext_dir; - } - if (has_ext_dir) thread_safe = false; + bool has_ext_dir = false; + for (int i=0; i<2*AMREX_SPACEDIM*ng && !has_ext_dir; ++i) { + has_ext_dir = bc[i]==BCType::ext_dir; + } + if (has_ext_dir) thread_safe = false; } return thread_safe; } @@ -46,19 +46,19 @@ StateDescriptor::BndryFunc::operator () (Real* data,const int* lo,const int* hi, bool thread_safe = bf_thread_safety(lo, hi, dom_lo, dom_hi, a_bc, 1); if (thread_safe) { if (m_func != 0) - m_func(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); + m_func(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); else - m_func3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), - AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); + m_func3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), + AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); } else { #ifdef AMREX_USE_OMP #pragma omp critical (bndryfunc) #endif if (m_func != 0) - m_func(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); + m_func(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); else - m_func3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), - AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); + m_func3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), + AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); } } @@ -73,19 +73,19 @@ StateDescriptor::BndryFunc::operator () (Real* data,const int* lo,const int* hi, bool thread_safe = bf_thread_safety(lo, hi, dom_lo, dom_hi, a_bc, ng); if (thread_safe) { if (m_gfunc != 0) - m_gfunc(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); - else - m_gfunc3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), - AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); + m_gfunc(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); + else + m_gfunc3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), + AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); } else { #ifdef AMREX_USE_OMP #pragma omp critical (bndryfunc) #endif if (m_gfunc != 0) - m_gfunc(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); - else - m_gfunc3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), - AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); + m_gfunc(data,AMREX_ARLIM(lo),AMREX_ARLIM(hi),dom_lo,dom_hi,dx,grd_lo,time,a_bc); + else + m_gfunc3D(data,AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),AMREX_ARLIM_3D(dom_lo),AMREX_ARLIM_3D(dom_hi), + AMREX_ZFILL(dx),AMREX_ZFILL(grd_lo),time,a_bc); } } @@ -164,7 +164,7 @@ DescriptorList::addDescriptor (int indx, IndexType typ, StateDescriptor::TimeCenter ttyp, int nextra, - int num_comp, + int num_comp, Interpolater* interp, bool extrap, bool a_store_in_checkpoint) @@ -172,7 +172,7 @@ DescriptorList::addDescriptor (int indx, if (indx >= desc.size()) desc.resize(indx+1); desc[indx].reset(new StateDescriptor(typ,ttyp,indx,nextra,num_comp,interp,extrap,a_store_in_checkpoint)); -} +} StateDescriptor::StateDescriptor () noexcept @@ -189,7 +189,7 @@ StateDescriptor::StateDescriptor () noexcept StateDescriptor::StateDescriptor (IndexType btyp, StateDescriptor::TimeCenter ttyp, int ident, - int nextra, + int nextra, int num_comp, Interpolater* a_interp, bool a_extrap, @@ -205,7 +205,7 @@ StateDescriptor::StateDescriptor (IndexType btyp, m_store_in_checkpoint(a_store_in_checkpoint) { BL_ASSERT (num_comp > 0); - + names.resize(num_comp); bc.resize(num_comp); bc_func.resize(num_comp); @@ -319,7 +319,7 @@ StateDescriptor::define (IndexType btyp, int num_comp, Interpolater* a_interp, bool a_extrap, - bool a_store_in_checkpoint) + bool a_store_in_checkpoint) { type = btyp; t_type = ttyp; @@ -331,7 +331,7 @@ StateDescriptor::define (IndexType btyp, m_store_in_checkpoint = a_store_in_checkpoint; BL_ASSERT (num_comp > 0); - + names.resize(num_comp); bc.resize(num_comp); bc_func.resize(num_comp); @@ -347,7 +347,7 @@ StateDescriptor::setComponent (int comp, const std::string& nm, const BCRec& bcr, const StateDescriptor::BndryFunc& func, - Interpolater* a_interp, + Interpolater* a_interp, int max_map_start_comp_, int min_map_end_comp_) { @@ -408,10 +408,10 @@ StateDescriptor::setUpMaps (int& use_default_map, const Interpolater* default_map, int start_comp, int num_comp, - Interpolater**& maps, + Interpolater**& maps, int& nmaps, int*& map_start_comp, - int*& map_num_comp, + int*& map_num_comp, int*& max_start_comp, int*& min_end_comp) const { @@ -427,7 +427,7 @@ StateDescriptor::setUpMaps (int& use_default_map, // Interpolater* map = mapper_comp[start_comp]; if (!map) map = (Interpolater*) default_map; - nmaps = 1; + nmaps = 1; int icomp = start_comp+1; use_default_map = 1; @@ -508,7 +508,7 @@ StateDescriptor::setUpMaps (int& use_default_map, } void -StateDescriptor::cleanUpMaps (Interpolater**& maps, +StateDescriptor::cleanUpMaps (Interpolater**& maps, int*& map_start_comp, int*& map_num_comp, int*& max_start_comp, diff --git a/Src/Amr/AMReX_extrapolater_3D_K.H b/Src/Amr/AMReX_extrapolater_3D_K.H index 8b0ade7cb58..74d0d5c50fe 100644 --- a/Src/Amr/AMReX_extrapolater_3D_K.H +++ b/Src/Amr/AMReX_extrapolater_3D_K.H @@ -41,15 +41,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else if ( ( mask(i+1,j+1,k) == finecell ) || ( mask(i+1,j,k+1) == finecell ) || ( mask(i,j+1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) - + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) + + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + mask(i,j+1,k+1) * data(i,j+1,k+1,n) ) / ( mask(i+1,j+1,k) + mask(i+1,j,k+1) + mask(i,j+1,k+1) ); } else { @@ -66,15 +66,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else if ( ( mask(i+1,j+1,k) == finecell ) || ( mask(i+1,j,k-1) == finecell ) || ( mask(i,j+1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) - + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) + + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + mask(i,j+1,k-1) * data(i,j+1,k-1,n) ) / ( mask(i+1,j+1,k) + mask(i+1,j,k-1) + mask(i,j+1,k-1) ); } else { @@ -91,15 +91,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k+1) ); } else if ( ( mask(i+1,j-1,k) == finecell ) || ( mask(i+1,j,k+1) == finecell ) || ( mask(i,j-1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) - + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) + + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + mask(i,j-1,k+1) * data(i,j-1,k+1,n) ) / ( mask(i+1,j-1,k) + mask(i+1,j,k+1) + mask(i,j-1,k+1) ); } else { @@ -116,15 +116,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) ); } else if ( ( mask(i+1,j-1,k) == finecell ) || ( mask(i+1,j,k-1) == finecell ) || ( mask(i,j-1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) - + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) + + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + mask(i,j-1,k-1) * data(i,j-1,k-1,n) ) / ( mask(i+1,j-1,k) + mask(i+1,j,k-1) + mask(i,j-1,k-1) ); } else { @@ -141,15 +141,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else if ( ( mask(i-1,j+1,k) == finecell ) || ( mask(i-1,j,k+1) == finecell ) || ( mask(i,j+1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) - + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) + + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + mask(i,j+1,k+1) * data(i,j+1,k+1,n) ) / ( mask(i-1,j+1,k) + mask(i-1,j,k+1) + mask(i,j+1,k+1) ); } else { @@ -166,15 +166,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else if ( ( mask(i-1,j+1,k) == finecell ) || ( mask(i-1,j,k-1) == finecell ) || ( mask(i,j+1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) - + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) + + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + mask(i,j+1,k-1) * data(i,j+1,k-1,n) ) / ( mask(i-1,j+1,k) + mask(i-1,j,k-1) + mask(i,j+1,k-1) ); } else { @@ -191,15 +191,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j,k+1) ); } else if ( ( mask(i-1,j-1,k) == finecell ) || ( mask(i-1,j,k+1) == finecell ) || ( mask(i,j-1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) - + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) + + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + mask(i,j-1,k+1) * data(i,j-1,k+1,n) ) / ( mask(i-1,j-1,k) + mask(i-1,j,k+1) + mask(i,j-1,k+1) ); } else { @@ -216,15 +216,15 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) ); } else if ( ( mask(i-1,j-1,k) == finecell ) || ( mask(i-1,j,k-1) == finecell ) || ( mask(i,j-1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) - + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) + + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + mask(i,j-1,k-1) * data(i,j-1,k-1,n) ) / ( mask(i-1,j-1,k) + mask(i-1,j,k-1) + mask(i,j-1,k-1) ); } else { @@ -244,9 +244,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i+1,j+1,k,n) @@ -270,9 +270,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i+1,j-1,k,n) @@ -296,9 +296,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i-1,j+1,k,n) @@ -322,9 +322,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i-1,j-1,k,n) @@ -348,9 +348,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) @@ -374,9 +374,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) @@ -400,9 +400,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -426,9 +426,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -452,9 +452,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) @@ -478,9 +478,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) @@ -504,9 +504,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -530,9 +530,9 @@ amrex_first_order_extrap_cpu(amrex::Box const& bx, ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -664,152 +664,152 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, if ( mask(i,j,k) == crsecell ) { // Corners // xlo, ylo, zlo - if ( ( i == lo.x-1) && ( j == lo.y-1 ) && ( k == lo.z-1 ) ) { + if ( ( i == lo.x-1) && ( j == lo.y-1 ) && ( k == lo.z-1 ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else if ( ( mask(i+1,j+1,k) == finecell ) || ( mask(i+1,j,k+1) == finecell ) || ( mask(i,j+1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) - + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) + + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + mask(i,j+1,k+1) * data(i,j+1,k+1,n) ) / ( mask(i+1,j+1,k) + mask(i+1,j,k+1) + mask(i,j+1,k+1) ); } else { data(i,j,k,n) = data(i+1,j+1,k+1,n); } // xlo, ylo, zhi - } else if ( ( i == lo.x-1) && ( j == lo.y-1 ) && ( k == hi.z+1 ) ) { + } else if ( ( i == lo.x-1) && ( j == lo.y-1 ) && ( k == hi.z+1 ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else if ( ( mask(i+1,j+1,k) == finecell ) || ( mask(i+1,j,k-1) == finecell ) || ( mask(i,j+1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) - + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + data(i,j,k,n) = ( mask(i+1,j+1,k) * data(i+1,j+1,k,n) + + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + mask(i,j+1,k-1) * data(i,j+1,k-1,n) ) / ( mask(i+1,j+1,k) + mask(i+1,j,k-1) + mask(i,j+1,k-1) ); } else { data(i,j,k,n) = data(i+1,j+1,k-1,n); } // xlo, yhi, zlo - } else if ( ( i == lo.x-1) && ( j == hi.y+1 ) && ( k == lo.z-1 ) ) { + } else if ( ( i == lo.x-1) && ( j == hi.y+1 ) && ( k == lo.z-1 ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k+1) ); } else if ( ( mask(i+1,j-1,k) == finecell ) || ( mask(i+1,j,k+1) == finecell ) || ( mask(i,j-1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) - + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) + + mask(i+1,j,k+1) * data(i+1,j,k+1,n) + mask(i,j-1,k+1) * data(i,j-1,k+1,n) ) / ( mask(i+1,j-1,k) + mask(i+1,j,k+1) + mask(i,j-1,k+1) ); } else { data(i,j,k,n) = data(i+1,j-1,k+1,n); } // xlo, yhi, zhi - } else if ( ( i == lo.x-1) && ( j == hi.y+1 ) && ( k == hi.z+1 ) ) { + } else if ( ( i == lo.x-1) && ( j == hi.y+1 ) && ( k == hi.z+1 ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) ); } else if ( ( mask(i+1,j-1,k) == finecell ) || ( mask(i+1,j,k-1) == finecell ) || ( mask(i,j-1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) - + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) + + mask(i+1,j,k-1) * data(i+1,j,k-1,n) + mask(i,j-1,k-1) * data(i,j-1,k-1,n) ) / ( mask(i+1,j-1,k) + mask(i+1,j,k-1) + mask(i,j-1,k-1) ); } else { data(i,j,k,n) = data(i+1,j-1,k-1,n); } // xhi, ylo, zlo - } else if ( ( i == hi.x+1) && ( j == lo.y-1 ) && ( k == lo.z-1 ) ) { + } else if ( ( i == hi.x+1) && ( j == lo.y-1 ) && ( k == lo.z-1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else if ( ( mask(i-1,j+1,k) == finecell ) || ( mask(i-1,j,k+1) == finecell ) || ( mask(i,j+1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) - + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) + + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + mask(i,j+1,k+1) * data(i,j+1,k+1,n) ) / ( mask(i-1,j+1,k) + mask(i-1,j,k+1) + mask(i,j+1,k+1) ); } else { data(i,j,k,n) = data(i-1,j+1,k+1,n); } // xhi, ylo, zhi - } else if ( ( i == hi.x+1) && ( j == lo.y-1 ) && ( k == hi.z+1 ) ) { + } else if ( ( i == hi.x+1) && ( j == lo.y-1 ) && ( k == hi.z+1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else if ( ( mask(i-1,j+1,k) == finecell ) || ( mask(i-1,j,k-1) == finecell ) || ( mask(i,j+1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) - + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) + + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + mask(i,j+1,k-1) * data(i,j+1,k-1,n) ) / ( mask(i-1,j+1,k) + mask(i-1,j,k-1) + mask(i,j+1,k-1) ); } else { data(i,j,k,n) = data(i-1,j+1,k-1,n); } // xhi, yhi, zlo - } else if ( ( i == hi.x+1) && ( j == hi.y+1 ) && ( k == lo.z-1 ) ) { + } else if ( ( i == hi.x+1) && ( j == hi.y+1 ) && ( k == lo.z-1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j,k+1) ); } else if ( ( mask(i-1,j-1,k) == finecell ) || ( mask(i-1,j,k+1) == finecell ) || ( mask(i,j-1,k+1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) - + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) + + mask(i-1,j,k+1) * data(i-1,j,k+1,n) + mask(i,j-1,k+1) * data(i,j-1,k+1,n) ) / ( mask(i-1,j-1,k) + mask(i-1,j,k+1) + mask(i,j-1,k+1) ); } else { data(i,j,k,n) = data(i-1,j-1,k+1,n); } // xhi, yhi, zhi - } else if ( ( i == hi.x+1) && ( j == hi.y+1 ) && ( k == hi.z+1 ) ) { + } else if ( ( i == hi.x+1) && ( j == hi.y+1 ) && ( k == hi.z+1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) + data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) ); } else if ( ( mask(i-1,j-1,k) == finecell ) || ( mask(i-1,j,k-1) == finecell ) || ( mask(i,j-1,k-1) == finecell ) ) { - data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) - + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) + + mask(i-1,j,k-1) * data(i-1,j,k-1,n) + mask(i,j-1,k-1) * data(i,j-1,k-1,n) ) / ( mask(i-1,j-1,k) + mask(i-1,j,k-1) + mask(i,j-1,k-1) ); } else { @@ -817,16 +817,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, } // Edges // xlo, ylo, z-valid - } else if ( ( i == lo.x-1) && ( j == lo.y-1 ) && + } else if ( ( i == lo.x-1) && ( j == lo.y-1 ) && ( k >= lo.z ) && ( k <= hi.z ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i+1,j+1,k,n) @@ -837,16 +837,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( 1 + mask(i+1,j,k-1) + mask(i+1,j,k+1) + mask(i,j+1,k-1) + mask(i,j+1,k+1) ); } // xlo, yhi, z-valid - } else if ( ( i == lo.x-1) && ( j == hi.y+1 ) && + } else if ( ( i == lo.x-1) && ( j == hi.y+1 ) && ( k >= lo.z ) && ( k <= hi.z ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i+1,j-1,k,n) @@ -857,16 +857,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( 1 + mask(i+1,j,k-1) + mask(i+1,j,k+1) + mask(i,j-1,k-1) + mask(i,j-1,k+1) ); } // xhi, ylo, z-valid - } else if ( ( i == hi.x+1) && ( j == lo.y-1 ) && + } else if ( ( i == hi.x+1) && ( j == lo.y-1 ) && ( k >= lo.z ) && ( k <= hi.z ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i-1,j+1,k,n) @@ -877,16 +877,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( 1 + mask(i-1,j,k-1) + mask(i-1,j,k+1) + mask(i,j+1,k-1) + mask(i,j+1,k+1) ); } // xhi, yhi, z-valid - } else if ( ( i == hi.x+1) && ( j == hi.y+1 ) && + } else if ( ( i == hi.x+1) && ( j == hi.y+1 ) && ( k >= lo.z ) && ( k <= hi.z ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( data(i-1,j-1,k,n) @@ -897,16 +897,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( 1 + mask(i-1,j,k-1) + mask(i-1,j,k+1) + mask(i,j-1,k-1) + mask(i,j-1,k+1) ); } // xlo, y-valid, zlo - } else if ( ( i == lo.x-1) && ( j >= lo.y ) && + } else if ( ( i == lo.x-1) && ( j >= lo.y ) && ( j <= hi.y ) && ( k == lo.z-1 ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) @@ -917,16 +917,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i+1,j-1,k) + mask(i+1,j+1,k) + 1 + mask(i,j-1,k+1) + mask(i,j+1,k+1) ); } // xlo, y-valid, zhi - } else if ( ( i == lo.x-1) && ( j >= lo.y ) && + } else if ( ( i == lo.x-1) && ( j >= lo.y ) && ( j <= hi.y ) && ( k == hi.z+1 ) ) { if ( ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i+1,j-1,k) * data(i+1,j-1,k,n) @@ -937,16 +937,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i+1,j-1,k) + mask(i+1,j+1,k) + 1 + mask(i,j-1,k-1) + mask(i,j+1,k-1) ); } // xhi, y-valid, zlo - } else if ( ( i == hi.x+1) && ( j >= lo.y ) && + } else if ( ( i == hi.x+1) && ( j >= lo.y ) && ( j <= hi.y ) && ( k == lo.z-1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -957,16 +957,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i-1,j-1,k) + mask(i-1,j+1,k) + 1 + mask(i,j-1,k+1) + mask(i,j+1,k+1) ); } // xhi, y-valid, zhi - } else if ( ( i == hi.x+1) && ( j >= lo.y ) && + } else if ( ( i == hi.x+1) && ( j >= lo.y ) && ( j <= hi.y ) && ( k == hi.z+1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i,j-1,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -977,16 +977,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i-1,j-1,k) + mask(i-1,j+1,k) + 1 + mask(i,j-1,k-1) + mask(i,j+1,k-1) ); } // x-valid, ylo, zlo - } else if ( ( i >= lo.x) && ( i <= hi.x ) && + } else if ( ( i >= lo.x) && ( i <= hi.x ) && ( j == lo.y-1 ) && ( k == lo.z-1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) @@ -997,16 +997,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i-1,j+1,k) + mask(i+1,j+1,k) + mask(i-1,j,k+1) + mask(i+1,j,k+1) + 1 ); } // x-valid, ylo, zhi - } else if ( ( i >= lo.x) && ( i <= hi.x ) && + } else if ( ( i >= lo.x) && ( i <= hi.x ) && ( j == lo.y-1 ) && ( k == hi.z+1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i+1,j,k) == finecell ) || ( mask(i,j+1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j+1,k) * data(i,j+1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j+1,k) * data(i,j+1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j+1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i-1,j+1,k) * data(i-1,j+1,k,n) @@ -1017,16 +1017,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i-1,j+1,k) + mask(i+1,j+1,k) + mask(i-1,j,k-1) + mask(i+1,j,k-1) + 1 ); } // x-valid, yhi, zlo - } else if ( ( i >= lo.x) && ( i <= hi.x ) && + } else if ( ( i >= lo.x) && ( i <= hi.x ) && ( j == hi.y+1 ) && ( k == lo.z-1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k+1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k+1) * data(i,j,k+1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k+1) * data(i,j,k+1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k+1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) @@ -1037,16 +1037,16 @@ amrex_first_order_extrap_gpu(int i, int j, int k, int n, / ( mask(i-1,j-1,k) + mask(i+1,j-1,k) + mask(i-1,j,k+1) + mask(i+1,j,k+1) + 1 ); } // x-valid, yhi, zhi - } else if ( ( i >= lo.x) && ( i <= hi.x ) && + } else if ( ( i >= lo.x) && ( i <= hi.x ) && ( j == hi.y+1 ) && ( k == hi.z+1 ) ) { if ( ( mask(i-1,j,k) == finecell ) || ( mask(i+1,j,k) == finecell ) || ( mask(i,j-1,k) == finecell ) || ( mask(i,j,k-1) == finecell ) ) { data(i,j,k,n) = ( mask(i-1,j,k) * data(i-1,j,k,n) - + mask(i+1,j,k) * data(i+1,j,k,n) - + mask(i,j-1,k) * data(i,j-1,k,n) - + mask(i,j,k-1) * data(i,j,k-1,n) ) + + mask(i+1,j,k) * data(i+1,j,k,n) + + mask(i,j-1,k) * data(i,j-1,k,n) + + mask(i,j,k-1) * data(i,j,k-1,n) ) / ( mask(i-1,j,k) + mask(i+1,j,k) + mask(i,j-1,k) + mask(i,j,k-1) ); } else { data(i,j,k,n) = ( mask(i-1,j-1,k) * data(i-1,j-1,k,n) diff --git a/Src/AmrCore/AMReX_AmrCore.cpp b/Src/AmrCore/AMReX_AmrCore.cpp index e64e16eef94..e897219053c 100644 --- a/Src/AmrCore/AMReX_AmrCore.cpp +++ b/Src/AmrCore/AMReX_AmrCore.cpp @@ -80,7 +80,7 @@ AmrCore::regrid (int lbase, Real time, bool) if (lev <= finest_level) // an old level { bool ba_changed = (new_grids[lev] != grids[lev]); - if (ba_changed || coarse_ba_changed) { + if (ba_changed || coarse_ba_changed) { BoxArray level_grids = grids[lev]; DistributionMapping level_dmap = dmap[lev]; if (ba_changed) { @@ -95,9 +95,9 @@ AmrCore::regrid (int lbase, Real time, bool) } } coarse_ba_changed = ba_changed;; - } - else // a new level - { + } + else // a new level + { DistributionMapping new_dmap(new_grids[lev]); const auto old_num_setdm = num_setdm; MakeNewLevelFromCoarse(lev, time, new_grids[lev], new_dmap); @@ -105,13 +105,13 @@ AmrCore::regrid (int lbase, Real time, bool) if (old_num_setdm == num_setdm) { SetDistributionMap(lev, new_dmap); } - } + } } for (int lev = new_finest+1; lev <= finest_level; ++lev) { - ClearLevel(lev); - ClearBoxArray(lev); - ClearDistributionMap(lev); + ClearLevel(lev); + ClearBoxArray(lev); + ClearDistributionMap(lev); } finest_level = new_finest; @@ -140,71 +140,71 @@ AmrCore::printGridSummary (std::ostream& os, int min_lev, int max_lev) const noe << " % of domain" << '\n'; - if (numgrid > 1) { - Long vmin = std::numeric_limits::max(); - Long vmax = -1; - int lmax = -1; - int smin = std::numeric_limits::max(); + if (numgrid > 1) { + Long vmin = std::numeric_limits::max(); + Long vmax = -1; + int lmax = -1; + int smin = std::numeric_limits::max(); int imax = std::numeric_limits::lowest(); int imin = std::numeric_limits::lowest(); #ifdef AMREX_USE_OMP #pragma omp parallel -#endif - { - Long vmin_this = std::numeric_limits::max(); - Long vmax_this = -1; - int lmax_this = -1; - int smin_this = std::numeric_limits::max(); +#endif + { + Long vmin_this = std::numeric_limits::max(); + Long vmax_this = -1; + int lmax_this = -1; + int smin_this = std::numeric_limits::max(); int imax_this = std::numeric_limits::lowest(); int imin_this = std::numeric_limits::lowest(); #ifdef AMREX_USE_OMP #pragma omp for -#endif - for (int k = 0; k < numgrid; k++) { - const Box& bx = bs[k]; - Long v = bx.volume(); - int ss = bx.shortside(); - int ls = bx.longside(); - if (v < vmin_this || (v == vmin_this && ss < smin_this)) { - vmin_this = v; - smin_this = ss; - imin_this = k; - } - if (v > vmax_this || (v == vmax_this && ls > lmax_this)) { - vmax_this = v; - lmax_this = ls; - imax_this = k; - } - } +#endif + for (int k = 0; k < numgrid; k++) { + const Box& bx = bs[k]; + Long v = bx.volume(); + int ss = bx.shortside(); + int ls = bx.longside(); + if (v < vmin_this || (v == vmin_this && ss < smin_this)) { + vmin_this = v; + smin_this = ss; + imin_this = k; + } + if (v > vmax_this || (v == vmax_this && ls > lmax_this)) { + vmax_this = v; + lmax_this = ls; + imax_this = k; + } + } #ifdef AMREX_USE_OMP #pragma omp critical (amr_prtgs) -#endif - { - if (vmin_this < vmin || (vmin_this == vmin && smin_this < smin)) { - vmin = vmin_this; - smin = smin_this; - imin = imin_this; - } - if (vmax_this > vmax || (vmax_this == vmax && lmax_this > lmax)) { - vmax = vmax_this; - lmax = lmax_this; - imax = imax_this; - } - } - } - const Box& bmin = bs[imin]; - const Box& bmax = bs[imax]; - os << " " - << " smallest grid: " - AMREX_D_TERM(<< bmin.length(0), - << " x " << bmin.length(1), - << " x " << bmin.length(2)) - << " biggest grid: " - AMREX_D_TERM(<< bmax.length(0), - << " x " << bmax.length(1), - << " x " << bmax.length(2)) - << '\n'; - } +#endif + { + if (vmin_this < vmin || (vmin_this == vmin && smin_this < smin)) { + vmin = vmin_this; + smin = smin_this; + imin = imin_this; + } + if (vmax_this > vmax || (vmax_this == vmax && lmax_this > lmax)) { + vmax = vmax_this; + lmax = lmax_this; + imax = imax_this; + } + } + } + const Box& bmin = bs[imin]; + const Box& bmax = bs[imax]; + os << " " + << " smallest grid: " + AMREX_D_TERM(<< bmin.length(0), + << " x " << bmin.length(1), + << " x " << bmin.length(2)) + << " biggest grid: " + AMREX_D_TERM(<< bmax.length(0), + << " x " << bmax.length(1), + << " x " << bmax.length(2)) + << '\n'; + } } os << std::endl; // Make sure we flush! diff --git a/Src/AmrCore/AMReX_AmrMesh.cpp b/Src/AmrCore/AMReX_AmrMesh.cpp index 7805f6d91ab..e5fa2318fcc 100644 --- a/Src/AmrCore/AMReX_AmrMesh.cpp +++ b/Src/AmrCore/AMReX_AmrMesh.cpp @@ -327,31 +327,31 @@ AmrMesh::InitAmrMesh (int max_level_in, const Vector& n_cell_in, // Read computational domain and set geometry. { - Vector n_cell(AMREX_SPACEDIM); - if (n_cell_in[0] == -1) - { - pp.getarr("n_cell",n_cell,0,AMREX_SPACEDIM); - } - else - { - for (int i = 0; i < AMREX_SPACEDIM; i++) n_cell[i] = n_cell_in[i]; - } - - IntVect lo(IntVect::TheZeroVector()), hi(n_cell); - hi -= IntVect::TheUnitVector(); - Box index_domain(lo,hi); - for (int i = 0; i <= max_level; i++) - { - geom[i].define(index_domain, rb, coord, is_per); - if (i < max_level) { + Vector n_cell(AMREX_SPACEDIM); + if (n_cell_in[0] == -1) + { + pp.getarr("n_cell",n_cell,0,AMREX_SPACEDIM); + } + else + { + for (int i = 0; i < AMREX_SPACEDIM; i++) n_cell[i] = n_cell_in[i]; + } + + IntVect lo(IntVect::TheZeroVector()), hi(n_cell); + hi -= IntVect::TheUnitVector(); + Box index_domain(lo,hi); + for (int i = 0; i <= max_level; i++) + { + geom[i].define(index_domain, rb, coord, is_per); + if (i < max_level) { index_domain.refine(ref_ratio[i]); } - } + } } { - // chop up grids to have more grids than the number of procs - pp.query("refine_grid_layout", refine_grid_layout); + // chop up grids to have more grids than the number of procs + pp.query("refine_grid_layout", refine_grid_layout); } pp.query("check_input", check_input); @@ -425,15 +425,15 @@ AmrMesh::ChopGrids (int lev, BoxArray& ba, int target_size) const { IntVect chunk = max_grid_size[lev] / cnt; - for (int j = AMREX_SPACEDIM-1; j >= 0 ; j--) - { - chunk[j] /= 2; + for (int j = AMREX_SPACEDIM-1; j >= 0 ; j--) + { + chunk[j] /= 2; - if ( (ba.size() < target_size) && (chunk[j]%blocking_factor[lev][j] == 0) ) - { - ba.maxSize(chunk); - } - } + if ( (ba.size() < target_size) && (chunk[j]%blocking_factor[lev][j] == 0) ) + { + ba.maxSize(chunk); + } + } } } @@ -495,7 +495,7 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& rr_lev[i][n] = (ref_ratio[i][n]*bf_lev[i][n])/bf_lev[i+1][n]; } for (int i = lbase; i <= max_crse; i++) { - pc_domain[i] = amrex::coarsen(Geom(i).Domain(),bf_lev[i]); + pc_domain[i] = amrex::coarsen(Geom(i).Domain(),bf_lev[i]); } // // Construct proper nesting domains. @@ -526,9 +526,9 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& p_n_comp[i].refine(rr_lev[i-1]); p_n_comp[i].accrete(n_proper); - if (geom[i].isAnyPeriodic()) { - ProjPeriodic(p_n_comp[i], pc_domain[i], geom[i].isPeriodic()); - } + if (geom[i].isAnyPeriodic()) { + ProjPeriodic(p_n_comp[i], pc_domain[i], geom[i].isPeriodic()); + } p_n[i].parallelComplementIn(pc_domain[i],p_n_comp[i]); p_n[i].simplify(); @@ -571,8 +571,8 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& // if ( ! (useFixedCoarseGrids() && levc < useFixedUpToLevel()) ) { - ErrorEst(levc, tags, time, ngrow); - } + ErrorEst(levc, tags, time, ngrow); + } // // If new grids have been constructed above this level, project @@ -597,7 +597,7 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& BoxList bl_tagged = new_grids[levf+1].simplified_list(); bl_tagged.coarsen(ref_ratio[levf]); // - // This grows the boxes by n_error_buf[levf][idir] if they touch the edge + // This grows the boxes by n_error_buf[levf][idir] if they touch the edge // of the domain in preparation for them being shrunk by n_error_buf[levf][idir] later. // We want the net effect to be that grids are NOT shrunk away // from the edges of the domain. @@ -650,14 +650,14 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& if (useFixedCoarseGrids()) { - if (levc>=useFixedUpToLevel()) - { - tags.setVal(GetAreaNotToTag(levc), TagBox::CLEAR); - } - else - { - new_finest = std::max(new_finest,levf); - } + if (levc>=useFixedUpToLevel()) + { + tags.setVal(GetAreaNotToTag(levc), TagBox::CLEAR); + } + else + { + new_finest = std::max(new_finest,levf); + } } // @@ -676,7 +676,7 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& // Remove or add tagged points which violate/satisfy additional // user-specified criteria. // - ManualTagsPlacement(levc, tags, bf_lev); + ManualTagsPlacement(levc, tags, bf_lev); // // Map tagged points through periodic boundaries, if any. // @@ -691,8 +691,8 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& // // Create initial cluster containing all tagged points. // - Vector tagvec; - tags.collate(tagvec); + Vector tagvec; + tags.collate(tagvec); tags.clear(); if (tagvec.size() > 0) @@ -743,7 +743,7 @@ AmrMesh::MakeNewGrids (int lbase, Real time, int& new_finest, Vector& BL_ASSERT(new_bx.isDisjoint()); new_grids[levf] = BoxArray(std::move(new_bx), max_grid_size[levf]); - } + } } } @@ -787,14 +787,14 @@ AmrMesh::MakeNewGrids (Real time) { // define coarse level BoxArray and DistributionMap { - finest_level = 0; + finest_level = 0; - const BoxArray& ba = MakeBaseGrids(); - DistributionMapping dm(ba); + const BoxArray& ba = MakeBaseGrids(); + DistributionMapping dm(ba); const auto old_num_setdm = num_setdm; const auto old_num_setba = num_setba; - MakeNewLevelFromScratch(0, time, ba, dm); + MakeNewLevelFromScratch(0, time, ba, dm); if (old_num_setba == num_setba) { SetBoxArray(0, ba); @@ -806,63 +806,63 @@ AmrMesh::MakeNewGrids (Real time) if (max_level > 0) // build fine levels { - Vector new_grids(max_level+1); - new_grids[0] = grids[0]; - do - { - int new_finest; + Vector new_grids(max_level+1); + new_grids[0] = grids[0]; + do + { + int new_finest; - // Add (at most) one level at a time. - MakeNewGrids(finest_level,time,new_finest,new_grids); + // Add (at most) one level at a time. + MakeNewGrids(finest_level,time,new_finest,new_grids); - if (new_finest <= finest_level) break; - finest_level = new_finest; + if (new_finest <= finest_level) break; + finest_level = new_finest; - DistributionMapping dm(new_grids[new_finest]); + DistributionMapping dm(new_grids[new_finest]); const auto old_num_setdm = num_setdm; MakeNewLevelFromScratch(new_finest, time, new_grids[finest_level], dm); - SetBoxArray(new_finest, new_grids[new_finest]); + SetBoxArray(new_finest, new_grids[new_finest]); if (old_num_setdm == num_setdm) { SetDistributionMap(new_finest, dm); } - } - while (finest_level < max_level); + } + while (finest_level < max_level); - // Iterate grids to ensure fine grids encompass all interesting junk. + // Iterate grids to ensure fine grids encompass all interesting junk. if (iterate_on_new_grids) - { - for (int it=0; it<4; ++it) // try at most 4 times - { - for (int i = 1; i <= finest_level; ++i) { - new_grids[i] = grids[i]; - } - - int new_finest; - MakeNewGrids(0, time, new_finest, new_grids); - - if (new_finest < finest_level) break; - finest_level = new_finest; - - bool grids_the_same = true; - for (int lev = 1; lev <= new_finest; ++lev) { - if (new_grids[lev] != grids[lev]) { - grids_the_same = false; - DistributionMapping dm(new_grids[lev]); + { + for (int it=0; it<4; ++it) // try at most 4 times + { + for (int i = 1; i <= finest_level; ++i) { + new_grids[i] = grids[i]; + } + + int new_finest; + MakeNewGrids(0, time, new_finest, new_grids); + + if (new_finest < finest_level) break; + finest_level = new_finest; + + bool grids_the_same = true; + for (int lev = 1; lev <= new_finest; ++lev) { + if (new_grids[lev] != grids[lev]) { + grids_the_same = false; + DistributionMapping dm(new_grids[lev]); const auto old_num_setdm = num_setdm; MakeNewLevelFromScratch(lev, time, new_grids[lev], dm); - SetBoxArray(lev, new_grids[lev]); + SetBoxArray(lev, new_grids[lev]); if (old_num_setdm == num_setdm) { SetDistributionMap(lev, dm); } - } - } - if (grids_the_same) break; - } - } + } + } + if (grids_the_same) break; + } + } } } @@ -978,7 +978,7 @@ AmrMesh::checkInput () if (blocking_factor[i][idim] <= max_grid_size[i][idim]) if (max_grid_size[i][idim]%blocking_factor[i][idim] != 0) { { - amrex::Print() << "max_grid_size in direction " << idim + amrex::Print() << "max_grid_size in direction " << idim << " is " << max_grid_size[i][idim] << std::endl; amrex::Print() << "blocking_factor is " << blocking_factor[i][idim] << std::endl; amrex::Error("max_grid_size not divisible by blocking_factor"); diff --git a/Src/AmrCore/AMReX_AmrParGDB.H b/Src/AmrCore/AMReX_AmrParGDB.H index 423e164e426..35e6d666db8 100644 --- a/Src/AmrCore/AMReX_AmrParGDB.H +++ b/Src/AmrCore/AMReX_AmrParGDB.H @@ -13,12 +13,12 @@ class AmrParGDB public: explicit AmrParGDB (AmrCore* amr) noexcept - : m_amrcore(amr), + : m_amrcore(amr), m_geom(amr->maxLevel()+1), m_has_geom(amr->maxLevel()+1, 0), - m_dmap(amr->maxLevel()+1), - m_ba(amr->maxLevel()+1) - { } + m_dmap(amr->maxLevel()+1), + m_ba(amr->maxLevel()+1) + { } virtual ~AmrParGDB () {;} @@ -41,7 +41,7 @@ public: virtual const Vector& boxArray () const override; virtual void SetParticleBoxArray (int level, const BoxArray& new_ba) override; - virtual void SetParticleDistributionMap (int level, const DistributionMapping& new_dm) override; + virtual void SetParticleDistributionMap (int level, const DistributionMapping& new_dm) override; virtual void SetParticleGeometry (int level, const Geometry& new_geom) override; virtual void ClearParticleBoxArray (int level) override; @@ -107,9 +107,9 @@ const DistributionMapping& AmrParGDB::ParticleDistributionMap (int level) const { if (m_dmap[level].empty()) { - return m_amrcore->DistributionMap(level); + return m_amrcore->DistributionMap(level); } else { - return m_dmap[level]; + return m_dmap[level]; } } @@ -125,9 +125,9 @@ const Vector& AmrParGDB::ParticleDistributionMap () const { if (m_dmap[0].empty()) { - return m_amrcore->DistributionMap(); + return m_amrcore->DistributionMap(); } else { - return m_dmap; + return m_dmap; } } @@ -143,9 +143,9 @@ const BoxArray& AmrParGDB::ParticleBoxArray (int level) const { if (m_ba[level].empty()) { - return m_amrcore->boxArray(level); + return m_amrcore->boxArray(level); } else { - return m_ba[level]; + return m_ba[level]; } } @@ -161,9 +161,9 @@ const Vector& AmrParGDB::ParticleBoxArray () const { if (m_ba[0].empty()) { - return m_amrcore->boxArray(); + return m_amrcore->boxArray(); } else { - return m_ba; + return m_ba; } } diff --git a/Src/AmrCore/AMReX_AmrParticles.H b/Src/AmrCore/AMReX_AmrParticles.H index 01d8e550974..10aa7b1bff1 100644 --- a/Src/AmrCore/AMReX_AmrParticles.H +++ b/Src/AmrCore/AMReX_AmrParticles.H @@ -164,7 +164,7 @@ public: { } - AmrParticleContainer (const Vector & geom, + AmrParticleContainer (const Vector & geom, const Vector & dmap, const Vector & ba, const Vector & rr) @@ -182,7 +182,7 @@ class AmrTracerParticleContainer public: AmrTracerParticleContainer (AmrCore* amr_core) - : TracerParticleContainer(amr_core->GetParGDB()) + : TracerParticleContainer(amr_core->GetParGDB()) { } diff --git a/Src/AmrCore/AMReX_Cluster.cpp b/Src/AmrCore/AMReX_Cluster.cpp index a5662673c7b..8cc950aa694 100644 --- a/Src/AmrCore/AMReX_Cluster.cpp +++ b/Src/AmrCore/AMReX_Cluster.cpp @@ -47,7 +47,7 @@ class InBox } Cluster::Cluster (Cluster& c, - const Box& b) + const Box& b) : m_ar(0), m_len(0) @@ -108,7 +108,7 @@ Cluster::distribute (ClusterList& clst, BL_ASSERT(ok()); BL_ASSERT(bd.ok()); BL_ASSERT(clst.length() == 0); - + for (BoxDomain::const_iterator bdi = bd.begin(), End = bd.end(); bdi != End && ok(); ++bdi) @@ -158,7 +158,7 @@ Cluster::minBox () noexcept // static -int +int FindCut (const int* hist, int lo, int hi, @@ -399,7 +399,7 @@ Cluster::new_chop () } } BL_ASSERT(dir >= 0 && dir < AMREX_SPACEDIM); - + int nlo = 0; for (int i = lo[dir]; i < cut[dir]; i++) { nlo += hist[dir][i-lo[dir]]; @@ -425,7 +425,7 @@ Cluster::new_chop () // Replace the current box by the part of the box "below" the cut m_len = nlo; minBox(); - + if ( (eff() > oldeff) || (neweff > oldeff) || n_try > 0) { return newbox.release(); @@ -477,7 +477,7 @@ ClusterList::boxArray () const ba.set(i,(*cli)->box()); } - return ba; + return ba; } void @@ -508,7 +508,7 @@ ClusterList::boxList() const { blst.push_back((*cli)->box()); } - return blst; + return blst; } void @@ -575,7 +575,7 @@ ClusterList::intersect (const BoxDomain& dom) { Cluster* c = *cli; - bool assume_disjoint_ba = true; + bool assume_disjoint_ba = true; if (domba.contains(c->box(),assume_disjoint_ba)) { ++cli; diff --git a/Src/AmrCore/AMReX_ErrorList.H b/Src/AmrCore/AMReX_ErrorList.H index 06cbe7033cc..6777687ff9b 100644 --- a/Src/AmrCore/AMReX_ErrorList.H +++ b/Src/AmrCore/AMReX_ErrorList.H @@ -83,7 +83,7 @@ extern "C" */ typedef void (*ErrorFunc3DDefault)(int* tag, const int* tlo, const int* thi, const int* tagval, const int* clearval, - amrex::Real* data, const int* data_lo, const int* data_hi, + amrex::Real* data, const int* data_lo, const int* data_hi, const int* lo, const int * hi, const int* nvar, const int* domain_lo, const int* domain_hi, const amrex::Real* dx, const amrex::Real* xlo, diff --git a/Src/AmrCore/AMReX_ErrorList.cpp b/Src/AmrCore/AMReX_ErrorList.cpp index 7bb1742ea82..752633fd043 100644 --- a/Src/AmrCore/AMReX_ErrorList.cpp +++ b/Src/AmrCore/AMReX_ErrorList.cpp @@ -33,11 +33,11 @@ ErrorRec::ErrorFunc::clone () const ErrorRec::ErrorFunc::~ErrorFunc () {} void -ErrorRec::ErrorFunc::operator () (int* tag, AMREX_D_DECL(const int&tlo0,const int&tlo1,const int&tlo2), - AMREX_D_DECL(const int&thi0,const int&thi1,const int&thi2), +ErrorRec::ErrorFunc::operator () (int* tag, AMREX_D_DECL(const int&tlo0,const int&tlo1,const int&tlo2), + AMREX_D_DECL(const int&thi0,const int&thi1,const int&thi2), const int* tagval, const int* clearval, - Real* data, AMREX_D_DECL(const int&dlo0,const int&dlo1,const int&dlo2), - AMREX_D_DECL(const int&dhi0,const int&dhi1,const int&dhi2), + Real* data, AMREX_D_DECL(const int&dlo0,const int&dlo1,const int&dlo2), + AMREX_D_DECL(const int&dhi0,const int&dhi1,const int&dhi2), const int* lo, const int * hi, const int* nvar, const int* domain_lo, const int* domain_hi, const Real* dx, const Real* xlo, @@ -52,7 +52,7 @@ ErrorRec::ErrorFunc::operator () (int* tag, AMREX_D_DECL(const int&tlo0,const in } void -ErrorRec::ErrorFunc::operator () (int* tag, const int* tlo, const int* thi, +ErrorRec::ErrorFunc::operator () (int* tag, const int* tlo, const int* thi, const int* tagval, const int* clearval, Real* data, const int* dlo, const int* dhi, const int* lo, const int * hi, const int* nvar, @@ -68,7 +68,7 @@ ErrorRec::ErrorFunc::operator () (int* tag, const int* tlo, const int* thi, AMREX_ARLIM_3D(lo),AMREX_ARLIM_3D(hi),nvar, AMREX_ARLIM_3D(domain_lo),AMREX_ARLIM_3D(domain_hi), AMREX_ZFILL(dx),AMREX_ZFILL(xlo),AMREX_ZFILL(prob_lo),time,level); -} +} ErrorRec::ErrorFunc2::ErrorFunc2 () @@ -91,11 +91,11 @@ ErrorRec::ErrorFunc2::~ErrorFunc2 () {} void -ErrorRec::ErrorFunc2::operator () (int* tag, AMREX_D_DECL(const int&tlo0,const int&tlo1,const int&tlo2), - AMREX_D_DECL(const int&thi0,const int&thi1,const int&thi2), +ErrorRec::ErrorFunc2::operator () (int* tag, AMREX_D_DECL(const int&tlo0,const int&tlo1,const int&tlo2), + AMREX_D_DECL(const int&thi0,const int&thi1,const int&thi2), const int* tagval, const int* clearval, - Real* data, AMREX_D_DECL(const int&dlo0,const int&dlo1,const int&dlo2), - AMREX_D_DECL(const int&dhi0,const int&dhi1,const int&dhi2), + Real* data, AMREX_D_DECL(const int&dlo0,const int&dlo1,const int&dlo2), + AMREX_D_DECL(const int&dhi0,const int&dhi1,const int&dhi2), const int* lo, const int * hi, const int* nvar, const int* domain_lo, const int* domain_hi, const Real* dx, const int* level, const Real* avg) const @@ -176,7 +176,7 @@ ErrorList::size () const noexcept void ErrorList::add (const std::string& name, - int nextra, + int nextra, ErrorRec::ErrorType typ, const ErrorRec::ErrorFunc& func) { @@ -264,7 +264,7 @@ operator << (std::ostream& os, static std::map ng = { {GRAD,1}, {LESS,0}, {GREATER,0}, {VORT,0}, {BOX,0} }; return ng[m_test]; } - + inline void AMRErrorTag_LESS(const Box& bx, diff --git a/Src/AmrCore/AMReX_FillPatchUtil.H b/Src/AmrCore/AMReX_FillPatchUtil.H index 27a497e1abc..816f87d3cf4 100644 --- a/Src/AmrCore/AMReX_FillPatchUtil.H +++ b/Src/AmrCore/AMReX_FillPatchUtil.H @@ -34,7 +34,7 @@ namespace amrex template bool ProperlyNested (const IntVect& ratio, const IntVect& blocking_factor, int ngrow, - const IndexType& boxType, Interp* mapper); + const IndexType& boxType, Interp* mapper); template EnableIf_t::value> @@ -89,7 +89,7 @@ namespace amrex template , typename PostInterpHook=NullInterpHook > - EnableIf_t::value> + EnableIf_t::value> FillPatchTwoLevels (Array const& mf, IntVect const& nghost, Real time, const Vector >& cmf, const Vector& ct, const Vector >& fmf, const Vector& ft, diff --git a/Src/AmrCore/AMReX_FillPatchUtil_2d.F90 b/Src/AmrCore/AMReX_FillPatchUtil_2d.F90 index df4819d5c0f..ed4510e9b6c 100644 --- a/Src/AmrCore/AMReX_FillPatchUtil_2d.F90 +++ b/Src/AmrCore/AMReX_FillPatchUtil_2d.F90 @@ -14,7 +14,7 @@ subroutine amrex_interp_div_free_bfield (lo, hi, bx, bxlo, bxhi, by, bylo, byhi, real(amrex_real), intent(in ) :: cy(cylo(1):cyhi(1),cylo(2):cyhi(2)) integer :: i,j, ir, jr, ii, jj, clo(2), chi(2) - + real(amrex_real) :: dxinv(2), c1, c2, c3, x, y, rrinv real(amrex_real), dimension(:,:), contiguous, pointer :: dCxdy, dCydx real(amrex_real) :: coe_a0, coe_ax, coe_ay, coe_axy, coe_axx @@ -146,7 +146,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, & do j = lo(2), hi(2)+1 if (use_limiter .eq. 0) then do ic = clo(1),chi(1) - dc = 0.5d0*(tmpx(ic+1,j) - tmpx(ic-1,j)) + dc = 0.5d0*(tmpx(ic+1,j) - tmpx(ic-1,j)) ex(ic*2 ,j) = tmpx(ic,j) - 0.25d0*dc ex(ic*2+1,j) = tmpx(ic,j) + 0.25d0*dc end do @@ -157,7 +157,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, & c3 = theta*(tmpx(ic+1,j) - tmpx(ic ,j)) dc = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + ex(ic*2 ,j) = tmpx(ic,j) - 0.25d0*dc ex(ic*2+1,j) = tmpx(ic,j) + 0.25d0*dc end do @@ -177,7 +177,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, & do jc = clo(2), chi(2) if (use_limiter .eq. 0) then do i = lo(1), hi(1)+1 - dc = 0.5d0*(tmpy(i,jc+1) - tmpy(i,jc-1)) + dc = 0.5d0*(tmpy(i,jc+1) - tmpy(i,jc-1)) ey(i,jc*2 ) = tmpy(i,jc) - 0.25d0*dc ey(i,jc*2+1) = tmpy(i,jc) + 0.25d0*dc end do @@ -188,7 +188,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, & c3 = theta*(tmpy(i,jc+1) - tmpy(i,jc )) dc = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + ey(i,jc*2 ) = tmpy(i,jc) - 0.25d0*dc ey(i,jc*2+1) = tmpy(i,jc) + 0.25d0*dc end do @@ -215,7 +215,7 @@ subroutine amrex_interp_cc_bfield (lo, hi, by, bylo, byhi, cy, cylo, cyhi, rr, u real(amrex_real) :: c1, c2, c3, dc real(amrex_real), parameter :: theta = 2.d0 ! 1: minmod, 2: MC real(amrex_real), parameter :: one = 1._amrex_real - + clo = amrex_coarsen_intvect(2,lo,rr); chi = amrex_coarsen_intvect(2,hi,rr); @@ -274,7 +274,7 @@ subroutine amrex_interp_nd_efield (lo, hi, ey, eylo, eyhi, cy, cylo, cyhi, rr) b real(amrex_real), intent(in ) :: cy(cylo(1):cyhi(1),cylo(2):cyhi(2)) integer :: i,j,clo(2),chi(2) - + clo = amrex_coarsen_intvect(2,lo,rr); chi = amrex_coarsen_intvect(2,hi,rr); @@ -284,7 +284,7 @@ subroutine amrex_interp_nd_efield (lo, hi, ey, eylo, eyhi, cy, cylo, cyhi, rr) b ey(i*2+1,j*2) = 0.5d0*(cy(i,j)+cy(i+1,j)) end do i = chi(1)+1 - ey(i*2,j*2) = cy(i,j) + ey(i*2,j*2) = cy(i,j) if (j .le. chi(2)) then do i = clo(1), chi(1) ey(i*2 ,j*2+1) = 0.5d0*(cy(i,j)+cy(i,j+1)) diff --git a/Src/AmrCore/AMReX_FillPatchUtil_3d.F90 b/Src/AmrCore/AMReX_FillPatchUtil_3d.F90 index f5e6996aa58..a72fe18b93a 100644 --- a/Src/AmrCore/AMReX_FillPatchUtil_3d.F90 +++ b/Src/AmrCore/AMReX_FillPatchUtil_3d.F90 @@ -23,7 +23,7 @@ subroutine amrex_interp_div_free_bfield (lo, hi, bx, bxlo, bxhi, by, bylo, byhi, real(amrex_real) :: coe_c0, coe_cx, coe_cy, coe_cz, coe_cxz, coe_cyz, coe_czz real(amrex_real), parameter :: theta = 2.d0 ! 1: minmod, 2: MC real(amrex_real), parameter :: one = 1.0_amrex_real - + clo = amrex_coarsen_intvect(3,lo,rr); chi = amrex_coarsen_intvect(3,hi,rr); @@ -51,7 +51,7 @@ subroutine amrex_interp_div_free_bfield (lo, hi, bx, bxlo, bxhi, by, bylo, byhi, c3 = theta*(cx(i,j+1,k)-cx(i,j ,k)) dCxdy(i,j,k) = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + c1 = theta*(cx(i,j,k )-cx(i,j,k-1)) c2 = 0.5d0*(cx(i,j,k+1)-cx(i,j,k-1)) c3 = theta*(cx(i,j,k+1)-cx(i,j,k )) @@ -76,7 +76,7 @@ subroutine amrex_interp_div_free_bfield (lo, hi, bx, bxlo, bxhi, by, bylo, byhi, c3 = theta*(cy(i+1,j,k)-cy(i ,j,k)) dCydx(i,j,k) = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + c1 = theta*(cy(i,j,k )-cy(i,j,k-1)) c2 = 0.5d0*(cy(i,j,k+1)-cy(i,j,k-1)) c3 = theta*(cy(i,j,k+1)-cy(i,j,k )) @@ -101,7 +101,7 @@ subroutine amrex_interp_div_free_bfield (lo, hi, bx, bxlo, bxhi, by, bylo, byhi, c3 = theta*(cz(i+1,j,k)-cz(i ,j,k)) dCzdx(i,j,k) = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + c1 = theta*(cz(i,j ,k)-cz(i,j-1,k)) c2 = 0.5d0*(cz(i,j+1,k)-cz(i,j-1,k)) c3 = theta*(cz(i,j+1,k)-cz(i,j ,k)) @@ -248,12 +248,12 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo end do end do end do - + do k = lo(3), hi(3)+1 do j = lo(2), hi(2)+1 if (use_limiter .eq. 0) then do ic = clo(1),chi(1) - dc = 0.5d0*(tmpx(ic+1,j,k) - tmpx(ic-1,j,k)) + dc = 0.5d0*(tmpx(ic+1,j,k) - tmpx(ic-1,j,k)) ex(ic*2 ,j,k) = tmpx(ic,j,k) - 0.25d0*dc ex(ic*2+1,j,k) = tmpx(ic,j,k) + 0.25d0*dc end do @@ -264,7 +264,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo c3 = theta*(tmpx(ic+1,j,k) - tmpx(ic ,j,k)) dc = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + ex(ic*2 ,j,k) = tmpx(ic,j,k) - 0.25d0*dc ex(ic*2+1,j,k) = tmpx(ic,j,k) + 0.25d0*dc end do @@ -280,7 +280,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo tmpy(ic*2+1,jc,kc*2) = 0.5d0*(cy(ic,jc,kc)+cy(ic+1,jc,kc)) end do ic = chi(1)+1 - tmpy(ic*2,jc,kc*2) = cy(ic,jc,kc) + tmpy(ic*2,jc,kc*2) = cy(ic,jc,kc) end do end do @@ -291,12 +291,12 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo end do end do end do - + do k = lo(3), hi(3)+1 do jc = clo(2), chi(2) if (use_limiter .eq. 0) then do i = lo(1), hi(1)+1 - dc = 0.5d0*(tmpy(i,jc+1,k) - tmpy(i,jc-1,k)) + dc = 0.5d0*(tmpy(i,jc+1,k) - tmpy(i,jc-1,k)) ey(i,jc*2 ,k) = tmpy(i,jc,k) - 0.25d0*dc ey(i,jc*2+1,k) = tmpy(i,jc,k) + 0.25d0*dc end do @@ -307,7 +307,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo c3 = theta*(tmpy(i,jc+1,k) - tmpy(i,jc ,k)) dc = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + ey(i,jc*2 ,k) = tmpy(i,jc,k) - 0.25d0*dc ey(i,jc*2+1,k) = tmpy(i,jc,k) + 0.25d0*dc end do @@ -315,7 +315,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo end do end do - + ! ez do kc = clo(3)-1, chi(3)+1 do jc = clo(2) , chi(2)+1 @@ -338,7 +338,7 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo do j = lo(2), hi(2)+1 if (use_limiter .eq. 0) then do i = lo(1), hi(1)+1 - dc = 0.5d0*(tmpz(i,j,kc+1) - tmpz(i,j,kc-1)) + dc = 0.5d0*(tmpz(i,j,kc+1) - tmpz(i,j,kc-1)) ez(i,j,kc*2 ) = tmpz(i,j,kc) - 0.25d0*dc ez(i,j,kc*2+1) = tmpz(i,j,kc) + 0.25d0*dc end do @@ -349,16 +349,16 @@ subroutine amrex_interp_efield (lo, hi, ex, exlo, exhi, ey, eylo, eyhi, ez, ezlo c3 = theta*(tmpz(i,j,kc+1) - tmpz(i,j,kc )) dc = 0.25d0*(sign(one,c1)+sign(one,c2))*(sign(one,c1)+sign(one,c3)) & *min(abs(c1),abs(c2),abs(c3)) - + ez(i,j,kc*2 ) = tmpz(i,j,kc) - 0.25d0*dc ez(i,j,kc*2+1) = tmpz(i,j,kc) + 0.25d0*dc end do end if end do end do - + call bl_deallocate(tmpx) call bl_deallocate(tmpy) call bl_deallocate(tmpz) - + end subroutine amrex_interp_efield diff --git a/Src/AmrCore/AMReX_FillPatchUtil_F.H b/Src/AmrCore/AMReX_FillPatchUtil_F.H index fa919f8c3cb..a1cdf95d1a8 100644 --- a/Src/AmrCore/AMReX_FillPatchUtil_F.H +++ b/Src/AmrCore/AMReX_FillPatchUtil_F.H @@ -7,7 +7,7 @@ #ifdef __cplusplus extern "C" { #endif - + void amrex_interp_div_free_bfield (const int* lo, const int* hi , amrex_real* bx, const int* bxlo, const int* bxhi #if (AMREX_SPACEDIM > 1) diff --git a/Src/AmrCore/AMReX_FillPatchUtil_I.H b/Src/AmrCore/AMReX_FillPatchUtil_I.H index 8ddeb86305f..3386cad2a6b 100644 --- a/Src/AmrCore/AMReX_FillPatchUtil_I.H +++ b/Src/AmrCore/AMReX_FillPatchUtil_I.H @@ -181,7 +181,7 @@ namespace { int>::type = 0> MF make_mf_crse_patch (FabArrayBase::FPinfo const& fpc, int ncomp, IndexType idx_type) { - MF mf_crse_patch(amrex::convert(fpc.ba_crse_patch, idx_type), fpc.dm_patch, + MF mf_crse_patch(amrex::convert(fpc.ba_crse_patch, idx_type), fpc.dm_patch, ncomp, 0, MFInfo(), *fpc.fact_crse_patch); return mf_crse_patch; } @@ -358,7 +358,7 @@ namespace { FAB& sfab = mf_crse_patch[mfi]; FAB& dfab = mf_fine_patch[mfi]; const Box& dbx = dfab.box(); - const Box& sbx = sfab.box(); + const Box& sbx = sfab.box(); amrex::setBC(sbx,cdomain,bcscomp,0,ncomp,bcs,bcr); @@ -372,8 +372,8 @@ namespace { } mf.ParallelCopy(mf_fine_patch, 0, dcomp, ncomp, IntVect{0}, nghost); - } - } + } + } FillPatchSingleLevel(mf, nghost, time, fmf, ft, scomp, dcomp, ncomp, fgeom, fbc, fbccomp); @@ -395,7 +395,7 @@ namespace { const PostInterpHook& post_interp, EB2::IndexSpace const* index_space) { - BL_PROFILE("FillPatchTwoLevels (Array)"); + BL_PROFILE("FillPatchTwoLevels (Array)"); using FAB = typename MF::FABType::value_type; using iFAB = typename iMultiFab::FABType::value_type; @@ -417,9 +417,9 @@ namespace { // Test all of them? - if (nghost.max() > 0 || mf[0]->getBDKey() != fmf[0][0]->getBDKey()) - { - const InterpolaterBoxCoarsener& coarsener = mapper->BoxCoarsener(ratio); + if (nghost.max() > 0 || mf[0]->getBDKey() != fmf[0][0]->getBDKey()) + { + const InterpolaterBoxCoarsener& coarsener = mapper->BoxCoarsener(ratio); // Convert to cell-centered MF meta-data for FPInfo. MF mf_cc_dummy( amrex::convert(mf[0]->boxArray(), IntVect::TheZeroVector()), @@ -427,15 +427,15 @@ namespace { MF fmf_cc_dummy( amrex::convert(fmf[0][0]->boxArray(), IntVect::TheZeroVector()), fmf[0][0]->DistributionMap(), ncomp, nghost, MFInfo().SetAlloc(false) ); - const FabArrayBase::FPinfo& fpc = FabArrayBase::TheFPinfo(fmf_cc_dummy, mf_cc_dummy, + const FabArrayBase::FPinfo& fpc = FabArrayBase::TheFPinfo(fmf_cc_dummy, mf_cc_dummy, nghost, coarsener, fgeom, cgeom, index_space); - if ( !fpc.ba_crse_patch.empty() ) - { + if ( !fpc.ba_crse_patch.empty() ) + { Array mf_crse_patch; Array mf_refined_patch; Array solve_mask; @@ -524,8 +524,8 @@ namespace { { mf[d]->ParallelCopy(mf_refined_patch[d], 0, dcomp, ncomp, IntVect{0}, nghost); } - } - } + } + } for (int d=0; d const& mf, IntVect const& ngho Vector bcr_d(ncomp); amrex::setBC(sfab[d]->box(), - amrex::convert(cgeom.Domain(), sfab[d]->box().ixType()), + amrex::convert(cgeom.Domain(), sfab[d]->box().ixType()), bcscomp,0,ncomp,bcs[d],bcr_d); for (int n=0; nnComp(); const Box& domain = geom.Domain(); - + for (int dir = 0; dir < AMREX_SPACEDIM; dir++) { - Orientation lo(dir, Orientation::low); - Orientation hi(dir, Orientation::high); - - FabSet& frlo = (*this)[lo]; - FabSet& frhi = (*this)[hi]; - - const BoxArray& balo = frlo.boxArray(); - const BoxArray& bahi = frhi.boxArray(); - + Orientation lo(dir, Orientation::low); + Orientation hi(dir, Orientation::high); + + FabSet& frlo = (*this)[lo]; + FabSet& frhi = (*this)[hi]; + + const BoxArray& balo = frlo.boxArray(); + const BoxArray& bahi = frhi.boxArray(); + #ifdef AMREX_USE_OMP #pragma omp parallel #endif { - for (FabSetIter fsi(frlo); fsi.isValid(); ++fsi) { - const Box& bx = fsi.validbox(); - const std::vector< std::pair >& isects = bahi.intersections(bx); - for (int ii = 0; ii < static_cast(isects.size()); ++ii) { - frlo[fsi].setVal(0.0, isects[ii].second, 0, nc); - } - if (geom.isPeriodic(dir)) { - if (bx.smallEnd(dir) == domain.smallEnd(dir)) { - const Box& sbx = amrex::shift(bx, dir, domain.length(dir)); - const std::vector >& isects2 = bahi.intersections(sbx); - for (int ii = 0; ii < static_cast(isects2.size()); ++ii) { - const Box& bx2 = amrex::shift(isects2[ii].second, dir, -domain.length(dir)); - frlo[fsi].setVal(0.0, bx2, 0, nc); - } - } + for (FabSetIter fsi(frlo); fsi.isValid(); ++fsi) { + const Box& bx = fsi.validbox(); + const std::vector< std::pair >& isects = bahi.intersections(bx); + for (int ii = 0; ii < static_cast(isects.size()); ++ii) { + frlo[fsi].setVal(0.0, isects[ii].second, 0, nc); + } + if (geom.isPeriodic(dir)) { + if (bx.smallEnd(dir) == domain.smallEnd(dir)) { + const Box& sbx = amrex::shift(bx, dir, domain.length(dir)); + const std::vector >& isects2 = bahi.intersections(sbx); + for (int ii = 0; ii < static_cast(isects2.size()); ++ii) { + const Box& bx2 = amrex::shift(isects2[ii].second, dir, -domain.length(dir)); + frlo[fsi].setVal(0.0, bx2, 0, nc); + } + } + } + } + + for (FabSetIter fsi(frhi); fsi.isValid(); ++fsi) { + const Box& bx = fsi.validbox(); + const std::vector< std::pair >& isects = balo.intersections(bx); + for (int ii = 0; ii < static_cast(isects.size()); ++ii) { + frhi[fsi].setVal(0.0, isects[ii].second, 0, nc); } - } - - for (FabSetIter fsi(frhi); fsi.isValid(); ++fsi) { - const Box& bx = fsi.validbox(); - const std::vector< std::pair >& isects = balo.intersections(bx); - for (int ii = 0; ii < static_cast(isects.size()); ++ii) { - frhi[fsi].setVal(0.0, isects[ii].second, 0, nc); - } - if (geom.isPeriodic(dir)) { - if (bx.bigEnd(dir) == domain.bigEnd(dir)) { - const Box& sbx = amrex::shift(bx, dir, -domain.length(dir)); - const std::vector >& isects2 = balo.intersections(sbx); - for (int ii = 0; ii < static_cast(isects2.size()); ++ii) { - const Box& bx2 = amrex::shift(isects2[ii].second, dir, domain.length(dir)); - frhi[fsi].setVal(0.0, bx2, 0, nc); - } - } - } - } - } + if (geom.isPeriodic(dir)) { + if (bx.bigEnd(dir) == domain.bigEnd(dir)) { + const Box& sbx = amrex::shift(bx, dir, -domain.length(dir)); + const std::vector >& isects2 = balo.intersections(sbx); + for (int ii = 0; ii < static_cast(isects2.size()); ++ii) { + const Box& bx2 = amrex::shift(isects2[ii].second, dir, domain.length(dir)); + frhi[fsi].setVal(0.0, bx2, 0, nc); + } + } + } + } + } } } @@ -691,7 +691,7 @@ FluxRegister::OverwriteFlux (Array const& crse_fluxes, Orientation lo_face(idim, Orientation::low); Orientation hi_face(idim, Orientation::high); - + bndry[lo_face].copyTo(fine_flux, 0, srccomp, 0, numcomp, cperiod); bndry[hi_face].plusTo(fine_flux, 0, srccomp, 0, numcomp, cperiod); @@ -731,7 +731,7 @@ void FluxRegister::read (const std::string& name, std::istream& is) { if (ncomp < 0) { - amrex::Abort("FluxRegister::read: FluxRegister not defined"); + amrex::Abort("FluxRegister::read: FluxRegister not defined"); } IntVect ratio_in; @@ -743,7 +743,7 @@ FluxRegister::read (const std::string& name, std::istream& is) is >> ncomp_in; if (ratio_in != ratio || fine_level_in != fine_level || ncomp_in != ncomp) { - amrex::Abort("FluxRegister::read: predefined FluxRegister does not match the one in istream"); + amrex::Abort("FluxRegister::read: predefined FluxRegister does not match the one in istream"); } BndryRegister* br = this; diff --git a/Src/AmrCore/AMReX_INTERP_1D.F90 b/Src/AmrCore/AMReX_INTERP_1D.F90 index b17c7d59e15..6ed6f3a7820 100644 --- a/Src/AmrCore/AMReX_INTERP_1D.F90 +++ b/Src/AmrCore/AMReX_INTERP_1D.F90 @@ -16,30 +16,30 @@ module amrex_interp_module #undef SLY #undef SLXY -! ::: +! ::: ! ::: -------------------------------------------------------------- ! ::: cbinterp: cell centered bilinear interpolation -! ::: +! ::: ! ::: NOTE: it is assumed that the coarse grid array is ! ::: large enough to define interpolated values ! ::: in the region fblo:fbhi on the fine grid -! ::: +! ::: ! ::: Inputs/Outputs ! ::: fine <=> (modify) fine grid array ! ::: fine_l1,fine_h1 => (const) index limits of fine grid ! ::: fb_l1,fb_h1 => (const) subregion of fine grid to get values -! ::: -! ::: crse => (const) coarse grid data +! ::: +! ::: crse => (const) coarse grid data ! ::: crse_l1,crse_h1 => (const) index limits of coarse grid -! ::: +! ::: ! ::: lratio => (const) refinement ratio between levels ! ::: nvar => (const) number of components in array -! ::: +! ::: ! ::: TEMPORARY ARRAYS ! ::: slx,sly,slxy => 1-D slope arrays ! ::: strip => 1-D temp array ! ::: -------------------------------------------------------------- -! ::: +! ::: subroutine AMREX_CBINTERP (crse, crse_l1,crse_h1, cb_l1,cb_h1, & fine, fine_l1,fine_h1, fb_l1,fb_h1, & lratio, nvar, & @@ -73,22 +73,22 @@ subroutine AMREX_CBINTERP (crse, crse_l1,crse_h1, cb_l1,cb_h1, & denom = one/real(2*lratio,amrex_real) hrat = lratio/2 - do 200 n = 1, nvar + do 200 n = 1, nvar ! first fill a strip that will fit do ic = cb_l1,cb_h1-1 sl(ic,SLX) = crse(ic+1,n)-crse(ic,n) - enddo + enddo do lx = 0, lratio-1 do ic = cb_l1, cb_h1-1 i = ic*lratio + lx x = denom*(two*lx + one) - strip(i) = crse(ic,n) + x*sl(ic,SLX) - enddo - enddo + strip(i) = crse(ic,n) + x*sl(ic,SLX) + enddo + enddo ! stuff into output array - do i = fb_l1, fb_h1 + do i = fb_l1, fb_h1 fine(i,n) = strip(i-hrat) enddo 200 continue @@ -99,29 +99,29 @@ end subroutine AMREX_CBINTERP #undef SLY #undef SLXY -! ::: +! ::: ! ::: -------------------------------------------------------------- ! ::: quartinterp: quartic conservative interpolation from coarse grid to ! ::: subregion of fine grid defined by (fblo,fbhi) -! ::: +! ::: ! ::: Inputs/Outputs ! ::: fine <=> (modify) fine grid array ! ::: flo,fhi => (const) index limits of fine grid ! ::: fblo,fbhi => (const) subregion of fine grid to get values ! ::: nvar => (const) number of variables in state vector ! ::: lratiox => (const) refinement ratio between levels -! ::: +! ::: ! ::: crse => (const) coarse grid data ! ::: clo,chi => (const) index limits of crse grid ! ::: cblo,cbhi => (const) coarse grid region containing fblo,fbhi and widen by 2 or 4 cells ! ::: ! ::: cb2lo,cb2hi => (const) coarse grid region containing fblo,fbhi ! ::: fb2lo,fb2hi => (const) fine version of cb2. It could be wider than fb -! ::: +! ::: ! ::: TEMPORARY ARRAYS ! ::: ftmp => 1-D temp array ! ::: -------------------------------------------------------------- -! ::: +! ::: subroutine AMREX_QUARTINTERP (fine, fine_l1,fine_h1, & fblo, fbhi, fb2lo, fb2hi, & crse, crse_l1,crse_h1, & @@ -152,7 +152,7 @@ subroutine AMREX_QUARTINTERP (fine, fine_l1,fine_h1, & 0.01171875D0 / !$$$ data cR/ 0.01171875D0, -0.0859375D0, 0.5d0, 0.0859375D0, & !$$$ -0.01171875D0 / - + if (lratiox .eq. 2) then do n = 1, nvar do i = cb2lo(1), cb2hi(1) diff --git a/Src/AmrCore/AMReX_INTERP_2D.F90 b/Src/AmrCore/AMReX_INTERP_2D.F90 index 6cb59913b88..8e93e495f70 100644 --- a/Src/AmrCore/AMReX_INTERP_2D.F90 +++ b/Src/AmrCore/AMReX_INTERP_2D.F90 @@ -12,30 +12,30 @@ module amrex_interp_module #define IX_PROJ(A,B) (A+B*iabs(A))/B-iabs(A) -! ::: +! ::: ! ::: -------------------------------------------------------------- ! ::: cbinterp: cell centered bilinear interpolation -! ::: +! ::: ! ::: NOTE: it is assumed that the coarse grid array is ! ::: large enough to define interpolated values ! ::: in the region fblo:fbhi on the fine grid -! ::: +! ::: ! ::: Inputs/Outputs ! ::: fine <=> (modify) fine grid array ! ::: fine_l1,fine_l2,fine_h1,fine_h2 => (const) index limits of fine grid ! ::: fb_l1,fb_l2,fb_h1,fb_h2 => (const) subregion of fine grid to get values -! ::: -! ::: crse => (const) coarse grid data +! ::: +! ::: crse => (const) coarse grid data ! ::: crse_l1,crse_l2,crse_h1,crse_h2 => (const) index limits of coarse grid -! ::: +! ::: ! ::: lratio(2) => (const) refinement ratio between levels ! ::: nvar => (const) number of components in array -! ::: +! ::: ! ::: TEMPORARY ARRAYS ! ::: slx,sly,slxy => 1-D slope arrays ! ::: strip => 1-D temp array ! ::: -------------------------------------------------------------- -! ::: +! ::: subroutine AMREX_CBINTERP (crse, crse_l1,crse_l2,crse_h1,crse_h2, cb_l1,cb_l2,cb_h1,cb_h2, & fine, fine_l1,fine_l2,fine_h1,fine_h2, fb_l1,fb_l2,fb_h1,fb_h2, & lratiox, lratioy, nvar, & @@ -68,8 +68,8 @@ subroutine AMREX_CBINTERP (crse, crse_l1,crse_l2,crse_h1,crse_h2, cb_l1,cb_l2,cb hraty = lratioy/2 do n = 1, nvar - do jc = cb_l2, cb_h2-1 - do ly = 0, lratioy-1 + do jc = cb_l2, cb_h2-1 + do ly = 0, lratioy-1 jfn = jc*lratioy + ly jfc = jfn + hraty if (jfc .ge. fb_l2 .and. jfc .le. fb_h2) then @@ -83,7 +83,7 @@ subroutine AMREX_CBINTERP (crse, crse_l1,crse_l2,crse_h1,crse_h2, cb_l1,cb_l2,cb strip(i) = XB + y*(XT - XB) end do end do - do i = fb_l1, fb_h1 + do i = fb_l1, fb_h1 fine(i,jfc,n) = strip(i-hratx) end do end if @@ -93,15 +93,15 @@ subroutine AMREX_CBINTERP (crse, crse_l1,crse_l2,crse_h1,crse_h2, cb_l1,cb_l2,cb end subroutine AMREX_CBINTERP -! ::: +! ::: ! ::: -------------------------------------------------------------- ! ::: cqinterp: cell-centered quadratic interpolation -! ::: +! ::: ! ::: NOTE: it is assumed that the coarse grid array is ! ::: large enough to define interpolated values ! ::: in the region fblo:fbhi on the fine grid -! ::: -! ::: +! ::: +! ::: subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & fb_l1, fb_l2, fb_h1, fb_h2, & @@ -159,23 +159,23 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & jst = ncsx jslo = cb_l2-1 - do i = fb_l1, fb_h1 + do i = fb_l1, fb_h1 fn = i-fslo(1)+1 ic = IX_PROJ(i,lratiox) fcen = half*(fvcx(i)+fvcx(i+1)) ccen = half*(cvcx(ic)+cvcx(ic+1)) voff(fn) = (fcen-ccen)/(cvcx(ic+1)-cvcx(ic)) - end do + end do - do n = 1, nvar - do i = clo, chi + do n = 1, nvar + do i = clo, chi crse(i,n) = merge(crse(i,n),zero,abs(crse(i,n)).gt.1.0d-50) end do end do - do 290 n = 1, nvar + do 290 n = 1, nvar - do i = 1, clen + do i = 1, clen cen = half*(crse(i+ist,n)-crse(i-ist,n)) diffxy = fourth*(crse(i+ist+jst,n)+crse(i-ist-jst,n) & -crse(i-ist+jst,n)-crse(i+ist-jst,n)) @@ -186,7 +186,7 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end do if (xok) then if (bclo(1,n) .eq. amrex_bc_ext_dir .or. bclo(1,n).eq.amrex_bc_hoextrap) then - do i = 1, clen, jst + do i = 1, clen, jst cen = -sixteen/fifteen*crse(i-ist,n) + half*crse(i,n) & + two3rd*crse(i+ist,n) - tenth*crse(i+2*ist,n) cslope(i,1)=cen @@ -195,7 +195,7 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end do end if if (bchi(1,n) .eq. amrex_bc_ext_dir .or. bchi(1,n).eq.amrex_bc_hoextrap) then - do i = ncbx, clen, jst + do i = ncbx, clen, jst cen = sixteen/fifteen*crse(i+ist,n) - half*crse(i,n) & - two3rd*crse(i-ist,n) + tenth*crse(i-2*ist,n) cslope(i,1)=cen @@ -205,7 +205,7 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end if end if - do i = 1, clen + do i = 1, clen cen = half*(crse(i+jst,n)-crse(i-jst,n)) diffyy = crse(i+jst,n)-two*crse(i,n)+crse(i-jst,n) cslope(i,2)=cen @@ -213,7 +213,7 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end do if (yok) then if (bclo(2,n) .eq. amrex_bc_ext_dir .or. bclo(2,n).eq.amrex_bc_hoextrap) then - do i = 1, ncbx + do i = 1, ncbx cen = -sixteen/fifteen*crse(i-jst,n) + half*crse(i,n) & + two3rd*crse(i+jst,n) - tenth*crse(i+2*jst,n) cslope(i,2)=cen @@ -222,7 +222,7 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end do end if if (bchi(2,n) .eq. amrex_bc_ext_dir .or. bchi(2,n).eq.amrex_bc_hoextrap) then - do i = clen-ncbx,clen + do i = clen-ncbx,clen cen = sixteen/fifteen*crse(i+jst,n) - half*crse(i,n) & - two3rd*crse(i-jst,n) + tenth*crse(i-2*jst,n) cslope(i,2)=cen @@ -232,12 +232,12 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end if end if - do 360 jc = cb_l2, cb_h2 - do 370 ioff = 1, lratiox + do 360 jc = cb_l2, cb_h2 + do 370 ioff = 1, lratiox icc = clo + ist + jst*(jc-jslo) istart = ioff iend = ioff + (ncbx-1)*lratiox - do 380 fn = istart, iend, lratiox + do 380 fn = istart, iend, lratiox fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -248,14 +248,14 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & 380 continue 370 continue - do 390 joff = 0, lratioy-1 + do 390 joff = 0, lratioy-1 j = lratioy*jc + joff if ((j.lt.fb_l2).or.(j.gt.fb_h2)) goto 390 fcen = half*(fvcy(j)+fvcy(j+1)) ccen = half*(cvcy(jc)+cvcy(jc+1)) yoff = (fcen-ccen)/(cvcy(jc+1)-cvcy(jc)) - do 400 i = fb_l1, fb_h1 + do 400 i = fb_l1, fb_h1 fn = i-fslo(1)+1 fine(i,j,n) = fdat(fn) & + voff(fn) *fslope(fn,1) & @@ -271,12 +271,12 @@ subroutine AMREX_CQINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & end subroutine AMREX_CQINTERP -! ::: +! ::: ! ::: -------------------------------------------------------------- ! ::: protect_interp: redo interpolation if the result of linccinterp ! ::: generates under- or overshoots. -! ::: -! ::: +! ::: +! ::: ! ::: Inputs/Outputs ! ::: fine <=> (modify) fine grid array ! ::: flo,fhi => (const) index limits of fine grid @@ -284,12 +284,12 @@ end subroutine AMREX_CQINTERP ! ::: cblo,cbhi => (const) coarse equivalent of fblo,fbhi ! ::: nvar => (const) number of variables in state vector ! ::: lratio(3) => (const) refinement ratio between levels -! ::: +! ::: ! ::: crse => (const) coarse grid data widended by 1 zone ! ::: clo,chi => (const) index limits of crse grid ! ::: ! ::: -------------------------------------------------------------- -! ::: +! ::: subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fbhi, & crse, crse_l1,crse_l2,crse_h1,crse_h2, cblo, cbhi, & @@ -356,7 +356,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb ! **************************************************************************************** ! -! If all the fine values are non-negative after the original interpolated +! If all the fine values are non-negative after the original interpolated ! correction, then we do nothing here. ! ! If any of the fine values are negative after the original interpolated @@ -364,7 +364,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb ! ! Special cases: ! -! 1) Coarse correction > 0, and fine_state has some cells with +! 1) Coarse correction > 0, and fine_state has some cells with ! negative values which will be filled before adding to the other cells. ! Use the correction to bring negative cells to zero, then ! distribute the remaining positive proportionally. @@ -380,7 +380,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb ! same negative value as possible. ! ! 4) Coarse correction < 0, fine_state has enough -! positive state to absorb it without making any fine +! positive state to absorb it without making any fine ! cells negative, BUT fine_state+fine is currently negative ! in at least one fine cell. Here just take a constant percentage ! away from each positive and don't touch the negatives. @@ -461,7 +461,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb enddo endif - + endif if (crseTot .gt. 0.d0 .and. crseTot .lt. abs(sumN)) then @@ -476,7 +476,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb do i = ilo,ihi if (fine_state(i,j,n) .lt. 0.d0) then fine(i,j,n) = alpha * abs(fine_state(i,j,n)) - else + else fine(i,j,n) = 0.d0 endif enddo @@ -504,7 +504,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb .and. (sumP+sumN+crseTot) .gt. 0.d0) then ! Here we have enough positive states to absorb all the ! negative correction *and* redistribute to make negative cells -! positive. +! positive. icase = 4 alpha = (crseTot + sumN) / sumP @@ -515,7 +515,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb fine(i,j,n) = -fine_state(i,j,n) else fine(i,j,n) = alpha * fine_state(i,j,n) - endif + endif enddo enddo @@ -524,8 +524,8 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb if (crseTot .lt. 0.d0 .and. abs(crseTot) .lt. sumP & .and. (sumP+sumN+crseTot) .le. 0.d0) then ! Here we have enough positive states to absorb all the -! negative correction, but not to fix the states already negative. -! We bring all the positive states to zero, and use whatever +! negative correction, but not to fix the states already negative. +! We bring all the positive states to zero, and use whatever ! remaining positiveness from the states to help the negative states. icase = 5 @@ -535,7 +535,7 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb do i = ilo,ihi if (fine_state(i,j,n) .gt. 0.d0) then fine(i,j,n) = -fine_state(i,j,n) - else + else fine(i,j,n) = alpha * fine_state(i,j,n) endif enddo @@ -609,30 +609,30 @@ subroutine AMREX_PROTECT_INTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, fblo, fb end subroutine AMREX_PROTECT_INTERP -! ::: +! ::: ! ::: -------------------------------------------------------------- ! ::: quartinterp: quartic conservative interpolation from coarse grid to ! ::: subregion of fine grid defined by (fblo,fbhi) -! ::: +! ::: ! ::: Inputs/Outputs ! ::: fine <=> (modify) fine grid array ! ::: flo,fhi => (const) index limits of fine grid ! ::: fblo,fbhi => (const) subregion of fine grid to get values ! ::: nvar => (const) number of variables in state vector ! ::: lratio[xy] => (const) refinement ratio between levels -! ::: +! ::: ! ::: crse => (const) coarse grid data ! ::: clo,chi => (const) index limits of crse grid ! ::: cblo,cbhi => (const) coarse grid region containing fblo,fbhi and widen by 2 or 4 cells ! ::: ! ::: cb2lo,cb2hi => (const) coarse grid region containing fblo,fbhi ! ::: fb2lo,fb2hi => (const) fine version of cb2. It could be wider than fb -! ::: +! ::: ! ::: TEMPORARY ARRAYS ! ::: ftmp => 1-D temp array ! ::: ctmp => 2-D temp array ! ::: -------------------------------------------------------------- -! ::: +! ::: subroutine AMREX_QUARTINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & fblo, fbhi, fb2lo, fb2hi, & crse, crse_l1,crse_l2,crse_h1,crse_h2, & @@ -664,7 +664,7 @@ subroutine AMREX_QUARTINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & 0.01171875D0 / ! data cR/ 0.01171875D0, -0.0859375D0, 0.5d0, 0.0859375D0, & ! -0.01171875D0 / - + if (lratiox.eq.2 .and. lratioy.eq.2) then do n = 1, nvar do j = cb2lo(2), cb2hi(2) @@ -675,7 +675,7 @@ subroutine AMREX_QUARTINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & + cL( 1)*crse(i,j+1,n) & + cL( 2)*crse(i,j+2,n)) ctmp(i,1) = 2.d0*crse(i,j,n) - ctmp(i,0) -!$$$ ctmp(i,1) = 2.d0*(cR(-2)*crse(i,j-2,n) & +!$$$ ctmp(i,1) = 2.d0*(cR(-2)*crse(i,j-2,n) & !$$$ + cR(-1)*crse(i,j-1,n) & !$$$ + cR( 0)*crse(i,j ,n) & !$$$ + cR( 1)*crse(i,j+1,n) & @@ -692,7 +692,7 @@ subroutine AMREX_QUARTINTERP (fine, fine_l1,fine_l2,fine_h1,fine_h2, & + cL( 1)*ctmp(i+1,iry) & + cL( 2)*ctmp(i+2,iry)) ftmp(ii+1) = 2.d0*ctmp(i,iry) - ftmp(ii) -!$$$ ftmp(ii+1) = 2.d0*(cR(-2)*ctmp(i-2,iry) & +!$$$ ftmp(ii+1) = 2.d0*(cR(-2)*ctmp(i-2,iry) & !$$$ + cR(-1)*ctmp(i-1,iry) & !$$$ + cR( 0)*ctmp(i ,iry) & !$$$ + cR( 1)*ctmp(i+1,iry) & diff --git a/Src/AmrCore/AMReX_INTERP_F.H b/Src/AmrCore/AMReX_INTERP_F.H index be0d1a25f4b..6bf66f4d641 100644 --- a/Src/AmrCore/AMReX_INTERP_F.H +++ b/Src/AmrCore/AMReX_INTERP_F.H @@ -9,54 +9,54 @@ extern "C" { #endif void amrex_cbinterp (const amrex_real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - AMREX_ARLIM_P(cblo), AMREX_ARLIM_P(cbhi), - amrex_real* fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), - AMREX_ARLIM_P(fblo), AMREX_ARLIM_P(fbhi), - AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), - const int* nvar, amrex_real* slope, const int* num_slope, - amrex_real* strip, const int* strip_lo, const int* strip_hi, - const int* actual_comp, const int* actual_state); + AMREX_ARLIM_P(cblo), AMREX_ARLIM_P(cbhi), + amrex_real* fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), + AMREX_ARLIM_P(fblo), AMREX_ARLIM_P(fbhi), + AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), + const int* nvar, amrex_real* slope, const int* num_slope, + amrex_real* strip, const int* strip_lo, const int* strip_hi, + const int* actual_comp, const int* actual_state); void amrex_cqinterp (amrex_real* fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), - AMREX_ARLIM_P(fblo), AMREX_ARLIM_P(fbhi), - const int* nvar, - AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), - const amrex_real* crse, const int* clo, const int* chi, - AMREX_ARLIM_P(cblo), AMREX_ARLIM_P(cbhi), - const int* fslo, const int* fshi, - amrex_real* cslope, const int* clen, - amrex_real* fslope, amrex_real* fdat, const int* flen, - amrex_real* voff, - const int* bc, const int* lim_slope, - AMREX_D_DECL(const amrex_real* fvcx,const amrex_real* fvcy, const amrex_real* fvcz), - AMREX_D_DECL(const amrex_real* cvcx,const amrex_real* cvcy, const amrex_real* cvcz), - const int* actual_comp, const int* actual_state); + AMREX_ARLIM_P(fblo), AMREX_ARLIM_P(fbhi), + const int* nvar, + AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), + const amrex_real* crse, const int* clo, const int* chi, + AMREX_ARLIM_P(cblo), AMREX_ARLIM_P(cbhi), + const int* fslo, const int* fshi, + amrex_real* cslope, const int* clen, + amrex_real* fslope, amrex_real* fdat, const int* flen, + amrex_real* voff, + const int* bc, const int* lim_slope, + AMREX_D_DECL(const amrex_real* fvcx,const amrex_real* fvcy, const amrex_real* fvcz), + AMREX_D_DECL(const amrex_real* cvcx,const amrex_real* cvcy, const amrex_real* cvcz), + const int* actual_comp, const int* actual_state); - void amrex_protect_interp ( amrex_real* fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), - const int* fblo, const int* fbhi, - const amrex_real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - const int* cblo, const int* cbhi, + void amrex_protect_interp (amrex_real* fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), + const int* fblo, const int* fbhi, + const amrex_real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), + const int* cblo, const int* cbhi, #if (AMREX_SPACEDIM == 2) - const amrex_real* fvcx, const amrex_real* fvcy, - AMREX_ARLIM_P(fblo), AMREX_ARLIM_P(fbhi), - const amrex_real* cvcx, const amrex_real* cvcy, - AMREX_ARLIM_P(cblo), AMREX_ARLIM_P(cbhi), + const amrex_real* fvcx, const amrex_real* fvcy, + AMREX_ARLIM_P(fblo), AMREX_ARLIM_P(fbhi), + const amrex_real* cvcx, const amrex_real* cvcy, + AMREX_ARLIM_P(cblo), AMREX_ARLIM_P(cbhi), #endif - amrex_real* state, AMREX_ARLIM_P(slo), AMREX_ARLIM_P(shi), - const int* nvar, - AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), - const int* bc); + amrex_real* state, AMREX_ARLIM_P(slo), AMREX_ARLIM_P(shi), + const int* nvar, + AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), + const int* bc); void amrex_quartinterp (amrex_real* fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), - const int* fblo, const int* fbhi, - const int* fb2lo, const int* fb2hi, - const amrex_real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - const int* cblo, const int* cbhi, - const int* cb2lo, const int* cb2hi, - const int* nvar, - AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), - AMREX_D_DECL(amrex_real* ftmp, amrex_real* ctmp, amrex_real* ctmp2), - const int* bc, const int* actual_comp, const int* actual_state); + const int* fblo, const int* fbhi, + const int* fb2lo, const int* fb2hi, + const amrex_real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), + const int* cblo, const int* cbhi, + const int* cb2lo, const int* cb2hi, + const int* nvar, + AMREX_D_DECL(const int* lrx,const int* lry,const int* lrz), + AMREX_D_DECL(amrex_real* ftmp, amrex_real* ctmp, amrex_real* ctmp2), + const int* bc, const int* actual_comp, const int* actual_state); #ifdef __cplusplus } #endif diff --git a/Src/AmrCore/AMReX_Interp_1D_C.H b/Src/AmrCore/AMReX_Interp_1D_C.H index c07f568b657..e2dfe7e015c 100644 --- a/Src/AmrCore/AMReX_Interp_1D_C.H +++ b/Src/AmrCore/AMReX_Interp_1D_C.H @@ -319,7 +319,7 @@ facediv_int (int /*ci*/, int /*cj*/, int /*ck*/, int /*nf*/, IntVect const& /*ratio*/, GpuArray const& /*cellSize*/) noexcept { - amrex::Abort("No 1D version of FaceDiv exists.\n"); + amrex::Abort("No 1D version of FaceDiv exists.\n"); } template diff --git a/Src/AmrCore/AMReX_Interp_2D_C.H b/Src/AmrCore/AMReX_Interp_2D_C.H index 091cf098fce..bf3ebde59aa 100644 --- a/Src/AmrCore/AMReX_Interp_2D_C.H +++ b/Src/AmrCore/AMReX_Interp_2D_C.H @@ -471,7 +471,7 @@ facediv_face_interp (int ci, int cj, int /*ck*/, } } } - + template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void facediv_int (int ci, int cj, int /*ck*/, int nf, diff --git a/Src/AmrCore/AMReX_Interp_3D_C.H b/Src/AmrCore/AMReX_Interp_3D_C.H index 6f045b7689e..a98b1654f01 100644 --- a/Src/AmrCore/AMReX_Interp_3D_C.H +++ b/Src/AmrCore/AMReX_Interp_3D_C.H @@ -21,7 +21,7 @@ ccinterp_compute_voff (Box const& cbx, IntVect const& ratio, Geometry const& /*c const auto& fhi = amrex::ubound(fbx); const int nxyz = flen.x + flen.y + flen.z; Vector voff(nxyz); - + const Real xrinv = Real(1.)/ratio[0]; const Real yrinv = Real(1.)/ratio[1]; const Real zrinv = Real(1.)/ratio[2]; @@ -104,7 +104,7 @@ compute_slopes (const Dim3& lo, const Dim3& hi, slopes(i,j,k,ns) = Real(16./15.)*u(i+1,j,k,nu) - Real(0.5)*u(i,j,k,nu) - Real(2./3.)*u(i-1,j,k,nu) + Real(0.1)*u(i-2,j,k,nu); } - } + } } else { for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { diff --git a/Src/AmrCore/AMReX_Interpolater.H b/Src/AmrCore/AMReX_Interpolater.H index 28c727dea95..292991b1f7e 100644 --- a/Src/AmrCore/AMReX_Interpolater.H +++ b/Src/AmrCore/AMReX_Interpolater.H @@ -690,7 +690,7 @@ public: * Divergence-free interpolation on face centered data. */ -class FaceDivFree +class FaceDivFree : public Interpolater { @@ -907,7 +907,7 @@ class InterpolaterBoxCoarsener { public: InterpolaterBoxCoarsener (Interpolater* mapper_, const IntVect& ratio_) - : mapper (mapper_), ratio(ratio_) { ; } + : mapper (mapper_), ratio(ratio_) { ; } /** * diff --git a/Src/AmrCore/AMReX_Interpolater.cpp b/Src/AmrCore/AMReX_Interpolater.cpp index 9d1c889b123..d20ed38c2da 100644 --- a/Src/AmrCore/AMReX_Interpolater.cpp +++ b/Src/AmrCore/AMReX_Interpolater.cpp @@ -868,7 +868,7 @@ CellConservativeQuartic::~CellConservativeQuartic () {} Box CellConservativeQuartic::CoarseBox (const Box& fine, - int ratio) + int ratio) { Box crse(amrex::coarsen(fine,ratio)); crse.grow(2); @@ -877,7 +877,7 @@ CellConservativeQuartic::CoarseBox (const Box& fine, Box CellConservativeQuartic::CoarseBox (const Box& fine, - const IntVect& ratio) + const IntVect& ratio) { Box crse = amrex::coarsen(fine,ratio); crse.grow(2); @@ -886,17 +886,17 @@ CellConservativeQuartic::CoarseBox (const Box& fine, void CellConservativeQuartic::interp (const FArrayBox& crse, - int crse_comp, - FArrayBox& fine, - int fine_comp, - int ncomp, - const Box& fine_region, - const IntVect& ratio, - const Geometry& /* crse_geom */, - const Geometry& /* fine_geom */, - Vector const& bcr, - int actual_comp, - int actual_state, + int crse_comp, + FArrayBox& fine, + int fine_comp, + int ncomp, + const Box& fine_region, + const IntVect& ratio, + const Geometry& /* crse_geom */, + const Geometry& /* fine_geom */, + Vector const& bcr, + int actual_comp, + int actual_state, RunOn /*runon*/) { BL_PROFILE("CellConservativeQuartic::interp()"); @@ -955,13 +955,13 @@ CellConservativeQuartic::interp (const FArrayBox& crse, #endif amrex_quartinterp (fdat,AMREX_ARLIM(flo),AMREX_ARLIM(fhi), - fblo, fbhi, fb2lo, fb2hi, - cdat,AMREX_ARLIM(clo),AMREX_ARLIM(chi), - cblo, cbhi, cb2lo, cb2hi, - &ncomp, - AMREX_D_DECL(&ratioV[0],&ratioV[1],&ratioV[2]), - AMREX_D_DECL(ftmp.dataPtr(), ctmp.dataPtr(), ctmp2.dataPtr()), - bc.dataPtr(),&actual_comp,&actual_state); + fblo, fbhi, fb2lo, fb2hi, + cdat,AMREX_ARLIM(clo),AMREX_ARLIM(chi), + cblo, cbhi, cb2lo, cb2hi, + &ncomp, + AMREX_D_DECL(&ratioV[0],&ratioV[1],&ratioV[2]), + AMREX_D_DECL(ftmp.dataPtr(), ctmp.dataPtr(), ctmp2.dataPtr()), + bc.dataPtr(),&actual_comp,&actual_state); } #endif diff --git a/Src/AmrCore/AMReX_TagBox.cpp b/Src/AmrCore/AMReX_TagBox.cpp index 863261edc32..98c63b5bbc7 100644 --- a/Src/AmrCore/AMReX_TagBox.cpp +++ b/Src/AmrCore/AMReX_TagBox.cpp @@ -157,9 +157,9 @@ TagBox::tags_and_untags (const Vector& ar) noexcept // DEPRECATED // Since a TagBox is a BaseFab, we can use this utility -// function to allocate an integer array to have the same number +// function to allocate an integer array to have the same number // of elements as cells in tilebx -void +void TagBox::get_itags(Vector& ar, const Box& tilebx) const noexcept { auto dlen = length(); @@ -167,7 +167,7 @@ TagBox::get_itags(Vector& ar, const Box& tilebx) const noexcept for (int idim=0; idim& ar, const Box& tilebx) const noexcept Ntb *= Ltb[idim]; stb += stride[idim] * (tilebx.smallEnd(idim) - domain.smallEnd(idim)); } - + if (ar.size() < Ntb) ar.resize(Ntb); - + const TagType* const p0 = dataPtr() + stb; // +stb to the lower corner of tilebox int* iptr = ar.dataPtr(); @@ -201,7 +201,7 @@ TagBox::get_itags(Vector& ar, const Box& tilebx) const noexcept // DEPRECATED // Set values as specified by the array -- this only tags. // only changes values in the tilebx region -void +void TagBox::tags (const Vector& ar, const Box& tilebx) noexcept { auto dlen = length(); @@ -209,7 +209,7 @@ TagBox::tags (const Vector& ar, const Box& tilebx) noexcept for (int idim=0; idim& ar, const Box& tilebx) noexcept Ltb[idim] = tilebx.length(idim); stb += stride[idim] * (tilebx.smallEnd(idim) - domain.smallEnd(idim)); } - + TagType* const p0 = dataPtr() + stb; // +stb to the lower corner of tilebox const int* iptr = ar.dataPtr(); @@ -235,7 +235,7 @@ TagBox::tags (const Vector& ar, const Box& tilebx) noexcept // DEPRECATED // Set values as specified by the array -- this tags and untags. // only changes values in the tilebx region -void +void TagBox::tags_and_untags (const Vector& ar, const Box& tilebx) noexcept { auto dlen = length(); @@ -243,7 +243,7 @@ TagBox::tags_and_untags (const Vector& ar, const Box& tilebx) noexcept for (int idim=0; idim& ar, const Box& tilebx) noexcept Ltb[idim] = tilebx.length(idim); stb += stride[idim] * (tilebx.smallEnd(idim) - domain.smallEnd(idim)); } - + TagType* const p0 = dataPtr() + stb; // +stb to the lower corner of tilebox const int* iptr = ar.dataPtr(); @@ -267,7 +267,7 @@ TagBox::tags_and_untags (const Vector& ar, const Box& tilebx) noexcept } TagBoxArray::TagBoxArray (const BoxArray& ba, - const DistributionMapping& dm, + const DistributionMapping& dm, int _ngrow) : FabArray(ba,dm,1,_ngrow,MFInfo(),DefaultFabFactory()) @@ -276,7 +276,7 @@ TagBoxArray::TagBoxArray (const BoxArray& ba, } TagBoxArray::TagBoxArray (const BoxArray& ba, - const DistributionMapping& dm, + const DistributionMapping& dm, const IntVect& _ngrow) : FabArray(ba,dm,1,_ngrow,MFInfo(),DefaultFabFactory()) @@ -284,7 +284,7 @@ TagBoxArray::TagBoxArray (const BoxArray& ba, setVal(TagBox::CLEAR); } -void +void TagBoxArray::buffer (const IntVect& nbuf) { AMREX_ASSERT(nbuf.allLE(n_grow)); @@ -625,8 +625,8 @@ TagBoxArray::collate (Vector& TheGlobalCollateSpace) const std::vector offset(countvec.size(),0); if (ParallelDescriptor::IOProcessor()) { for (int i = 1, N = offset.size(); i < N; i++) { - offset[i] = offset[i-1] + countvec[i-1]; - } + offset[i] = offset[i-1] + countvec[i-1]; + } } // // Gather all the tags to IOProcNumber into TheGlobalCollateSpace. diff --git a/Src/Base/AMReX.H b/Src/Base/AMReX.H index c24d09d3c66..28305feb966 100644 --- a/Src/Base/AMReX.H +++ b/Src/Base/AMReX.H @@ -48,7 +48,7 @@ namespace amrex #endif extern std::string exename; - + extern int verbose; extern int signal_handling; @@ -59,7 +59,7 @@ namespace amrex extern std::ostream* osout; extern std::ostream* oserr; - + extern ErrorHandler error_handler; extern int abort_on_unused_inputs; } diff --git a/Src/Base/AMReX.cpp b/Src/Base/AMReX.cpp index 417c0a5664f..1a0e7be846a 100644 --- a/Src/Base/AMReX.cpp +++ b/Src/Base/AMReX.cpp @@ -155,12 +155,12 @@ amrex::write_to_stderr_without_buffering (const char* str) if (str) { - std::ostringstream procall; - procall << ParallelDescriptor::MyProc() << "::"; + std::ostringstream procall; + procall << ParallelDescriptor::MyProc() << "::"; auto tmp = procall.str(); - const char *cprocall = tmp.c_str(); + const char *cprocall = tmp.c_str(); const char * const end = " !!!\n"; - fwrite(cprocall, strlen(cprocall), 1, stderr); + fwrite(cprocall, strlen(cprocall), 1, stderr); fwrite(str, strlen(str), 1, stderr); fwrite(end, strlen(end), 1, stderr); } @@ -173,10 +173,10 @@ write_lib_id(const char* msg) fflush(0); const char* const s = "amrex::"; fwrite(s, strlen(s), 1, stderr); - if ( msg ) + if ( msg ) { - fwrite(msg, strlen(msg), 1, stderr); - fwrite("::", 2, 1, stderr); + fwrite(msg, strlen(msg), 1, stderr); + fwrite("::", 2, 1, stderr); } } } @@ -232,7 +232,7 @@ void amrex::Warning_host (const char * msg) { if (msg) { - amrex::Print(Print::AllProcs,amrex::ErrorStream()) << msg << '!' << '\n'; + amrex::Print(Print::AllProcs,amrex::ErrorStream()) << msg << '!' << '\n'; } } @@ -517,15 +517,15 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse, #endif #elif defined(__APPLE__) - prev_fpe_mask = _MM_GET_EXCEPTION_MASK(); - curr_fpe_excepts = 0u; - if (invalid) curr_fpe_excepts |= _MM_MASK_INVALID; - if (divbyzero) curr_fpe_excepts |= _MM_MASK_DIV_ZERO; - if (overflow) curr_fpe_excepts |= _MM_MASK_OVERFLOW; - if (curr_fpe_excepts != 0u) { + prev_fpe_mask = _MM_GET_EXCEPTION_MASK(); + curr_fpe_excepts = 0u; + if (invalid) curr_fpe_excepts |= _MM_MASK_INVALID; + if (divbyzero) curr_fpe_excepts |= _MM_MASK_DIV_ZERO; + if (overflow) curr_fpe_excepts |= _MM_MASK_OVERFLOW; + if (curr_fpe_excepts != 0u) { _MM_SET_EXCEPTION_MASK(prev_fpe_mask & ~curr_fpe_excepts); prev_handler_sigfpe = signal(SIGFPE, BLBackTrace::handler); - } + } #endif } @@ -591,7 +591,7 @@ amrex::Initialize (int& argc, char**& argv, bool build_parm_parse, MPI_Query_thread(&provided); amrex::Print() << "MPI initialized with thread support level " << provided << std::endl; #endif - + #ifdef AMREX_USE_OMP // static_assert(_OPENMP >= 201107, "OpenMP >= 3.1 is required."); amrex::Print() << "OMP initialized with " @@ -655,28 +655,28 @@ amrex::Finalize (amrex::AMReX* pamrex) #ifndef BL_AMRPROF if (amrex::system::verbose > 1) { - int mp_min, mp_max, mp_tot; - amrex_mempool_get_stats(mp_min, mp_max, mp_tot); // in MB - if (ParallelDescriptor::NProcs() == 1) { - if (mp_tot > 0) { - amrex::Print() << "MemPool: " + int mp_min, mp_max, mp_tot; + amrex_mempool_get_stats(mp_min, mp_max, mp_tot); // in MB + if (ParallelDescriptor::NProcs() == 1) { + if (mp_tot > 0) { + amrex::Print() << "MemPool: " #ifdef AMREX_USE_OMP << "min used in a thread: " << mp_min << " MB, " << "max used in a thread: " << mp_max << " MB, " #endif << "tot used: " << mp_tot << " MB." << std::endl; - } - } else { - int global_max = mp_tot; - int global_min = mp_tot; - ParallelDescriptor::ReduceIntMax(global_max); - if (global_max > 0) { - ParallelDescriptor::ReduceIntMin(global_min); - amrex::Print() << "MemPool: " - << "min used in a rank: " << global_min << " MB, " - << "max used in a rank: " << global_max << " MB.\n"; - } - } + } + } else { + int global_max = mp_tot; + int global_min = mp_tot; + ParallelDescriptor::ReduceIntMax(global_max); + if (global_max > 0) { + ParallelDescriptor::ReduceIntMin(global_min); + amrex::Print() << "MemPool: " + << "min used in a rank: " << global_min << " MB, " + << "max used in a rank: " << global_max << " MB.\n"; + } + } } #endif @@ -704,9 +704,9 @@ amrex::Finalize (amrex::AMReX* pamrex) } #endif #elif defined(__APPLE__) - if (curr_fpe_excepts != 0u) { + if (curr_fpe_excepts != 0u) { _MM_SET_EXCEPTION_MASK(prev_fpe_mask); - } + } #endif } #endif diff --git a/Src/Base/AMReX_BCRec.cpp b/Src/Base/AMReX_BCRec.cpp index 714c0c8dddb..46ddd02ed18 100644 --- a/Src/Base/AMReX_BCRec.cpp +++ b/Src/Base/AMReX_BCRec.cpp @@ -30,7 +30,7 @@ setBC (const Box& bx, ? bc_dom[sc].hi(dir) : BCType::int_dir )); } } -} +} std::ostream& operator<< (std::ostream& os, diff --git a/Src/Base/AMReX_BLBackTrace.cpp b/Src/Base/AMReX_BLBackTrace.cpp index e714cb6d644..4cf82d0da49 100644 --- a/Src/Base/AMReX_BLBackTrace.cpp +++ b/Src/Base/AMReX_BLBackTrace.cpp @@ -50,55 +50,55 @@ BLBackTrace::handler(int s) switch (s) { case SIGSEGV: - amrex::ErrorStream() << "Segfault\n"; - break; + amrex::ErrorStream() << "Segfault\n"; + break; case SIGFPE: - amrex::ErrorStream() << "Erroneous arithmetic operation\n"; - break; + amrex::ErrorStream() << "Erroneous arithmetic operation\n"; + break; case SIGTERM: - amrex::ErrorStream() << "SIGTERM\n"; - break; + amrex::ErrorStream() << "SIGTERM\n"; + break; case SIGINT: - amrex::ErrorStream() << "SIGINT\n"; - break; + amrex::ErrorStream() << "SIGINT\n"; + break; case SIGABRT: - amrex::ErrorStream() << "SIGABRT\n"; - break; + amrex::ErrorStream() << "SIGABRT\n"; + break; } #if defined(AMREX_BACKTRACE_SUPPORTED) || defined(AMREX_TINY_PROFILING) std::string errfilename; { - std::ostringstream ss; - ss << "Backtrace." << ParallelDescriptor::MyProc(); + std::ostringstream ss; + ss << "Backtrace." << ParallelDescriptor::MyProc(); #ifdef AMREX_USE_OMP - ss << "." << omp_get_thread_num(); + ss << "." << omp_get_thread_num(); #endif - errfilename = ss.str(); + errfilename = ss.str(); } if (FILE* p = fopen(errfilename.c_str(), "w")) { #if defined(AMREX_BACKTRACE_SUPPORTED) - BLBackTrace::print_backtrace_info(p); + BLBackTrace::print_backtrace_info(p); #endif - fclose(p); + fclose(p); } - + amrex::ErrorStream() << "See " << errfilename << " file for details" << std::endl; if (!bt_stack.empty()) { - std::ofstream errfile; - errfile.open(errfilename.c_str(), std::ofstream::out | std::ofstream::app); - if (errfile.is_open()) { - errfile << std::endl; - while (!bt_stack.empty()) { - errfile << "== BACKTRACE == " << bt_stack.top().first - <<", " << bt_stack.top().second << "\n"; - bt_stack.pop(); - } - errfile << std::endl; - } + std::ofstream errfile; + errfile.open(errfilename.c_str(), std::ofstream::out | std::ofstream::app); + if (errfile.is_open()) { + errfile << std::endl; + while (!bt_stack.empty()) { + errfile << "== BACKTRACE == " << bt_stack.top().first + <<", " << bt_stack.top().second << "\n"; + bt_stack.pop(); + } + errfile << std::endl; + } } #ifdef AMREX_TINY_PROFILING @@ -109,12 +109,12 @@ BLBackTrace::handler(int s) errfile << std::endl; TinyProfiler::PrintCallStack(errfile); errfile << std::endl; - } + } } #endif if (ParallelDescriptor::NProcs() > 1) { - amrex::Sleep(3); + amrex::Sleep(3); } #endif @@ -132,7 +132,7 @@ BLBackTrace::print_backtrace_info (const std::string& filename) } else { - amrex::Print() << "Warning @ BLBackTrace::print_backtrace_info: " + amrex::Print() << "Warning @ BLBackTrace::print_backtrace_info: " << filename << " is not a valid output file." << std::endl; } @@ -178,7 +178,7 @@ BLBackTrace::print_backtrace_info (FILE* f) char **strings = backtrace_symbols(bt_buffer, nentries); if (strings != NULL) { - int have_eu_addr2line = 0; + int have_eu_addr2line = 0; int have_addr2line = 0; std::string cmd; { @@ -197,17 +197,17 @@ BLBackTrace::print_backtrace_info (FILE* f) } } - fprintf(f, "=== If no file names and line numbers are shown below, one can run\n"); - fprintf(f, " addr2line -Cpfie my_exefile my_line_address\n"); - fprintf(f, " to convert `my_line_address` (e.g., 0x4a6b) into file name and line number.\n"); + fprintf(f, "=== If no file names and line numbers are shown below, one can run\n"); + fprintf(f, " addr2line -Cpfie my_exefile my_line_address\n"); + fprintf(f, " to convert `my_line_address` (e.g., 0x4a6b) into file name and line number.\n"); fprintf(f, " Or one can use amrex/Tools/Backtrace/parse_bt.py.\n\n"); - fprintf(f, "=== Please note that the line number reported by addr2line may not be accurate.\n"); - fprintf(f, " One can use\n"); - fprintf(f, " readelf -wl my_exefile | grep my_line_address'\n"); - fprintf(f, " to find out the offset for that line.\n\n"); + fprintf(f, "=== Please note that the line number reported by addr2line may not be accurate.\n"); + fprintf(f, " One can use\n"); + fprintf(f, " readelf -wl my_exefile | grep my_line_address'\n"); + fprintf(f, " to find out the offset for that line.\n\n"); - for (int i = 0; i < nentries; ++i) + for (int i = 0; i < nentries; ++i) { fprintf(f, "%2d: %s\n", i, strings[i]); @@ -258,7 +258,7 @@ BLBackTrace::print_backtrace_info (FILE* f) } #endif fprintf(f, "\n"); - } + } std::free(strings); } @@ -304,44 +304,44 @@ BLBTer::BLBTer(const std::string& s, const char* file, int line) std::ostringstream ss; ss << "Line " << line << ", File " << file; line_file = ss.str(); - + #ifdef AMREX_USE_OMP if (omp_in_parallel()) { - std::ostringstream ss0; - ss0 << "Proc. " << ParallelDescriptor::MyProc() - << ", Thread " << omp_get_thread_num() - << ": \"" << s << "\""; - BLBackTrace::bt_stack.push(std::make_pair(ss0.str(), line_file)); + std::ostringstream ss0; + ss0 << "Proc. " << ParallelDescriptor::MyProc() + << ", Thread " << omp_get_thread_num() + << ": \"" << s << "\""; + BLBackTrace::bt_stack.push(std::make_pair(ss0.str(), line_file)); } else { #pragma omp parallel - { - std::ostringstream ss0; - ss0 << "Proc. " << ParallelDescriptor::MyProc() - << ", Master Thread" - << ": \"" << s << "\""; - BLBackTrace::bt_stack.push(std::make_pair(ss0.str(), line_file)); - } + { + std::ostringstream ss0; + ss0 << "Proc. " << ParallelDescriptor::MyProc() + << ", Master Thread" + << ": \"" << s << "\""; + BLBackTrace::bt_stack.push(std::make_pair(ss0.str(), line_file)); + } } #else std::ostringstream ss0; ss0 << "Proc. " << ParallelDescriptor::MyProc() - << ": \"" << s << "\""; + << ": \"" << s << "\""; BLBackTrace::bt_stack.push(std::make_pair(ss0.str(), line_file)); -#endif +#endif } BLBTer::~BLBTer() { #ifdef AMREX_USE_OMP if (omp_in_parallel()) { - pop_bt_stack(); + pop_bt_stack(); } else { #pragma omp parallel - { - pop_bt_stack(); - } + { + pop_bt_stack(); + } } #else pop_bt_stack(); @@ -352,9 +352,9 @@ void BLBTer::pop_bt_stack() { if (!BLBackTrace::bt_stack.empty()) { - if (BLBackTrace::bt_stack.top().second.compare(line_file) == 0) { - BLBackTrace::bt_stack.pop(); - } + if (BLBackTrace::bt_stack.top().second.compare(line_file) == 0) { + BLBackTrace::bt_stack.pop(); + } } } diff --git a/Src/Base/AMReX_BLProfiler.H b/Src/Base/AMReX_BLProfiler.H index 13032783f9f..abf8640eb31 100644 --- a/Src/Base/AMReX_BLProfiler.H +++ b/Src/Base/AMReX_BLProfiler.H @@ -40,16 +40,16 @@ class BLProfiler struct CallStats { CallStats() : callStackDepth(-2), csFNameNumber(-2), nCSCalls(0), totalTime(0.0), - stackTime(0.0), callTime(0.0) { } + stackTime(0.0), callTime(0.0) { } CallStats(int depth, int fnamenumber) : callStackDepth(depth), csFNameNumber(fnamenumber), nCSCalls(0), totalTime(0.0), - stackTime(0.0), callTime(0.0) { } + stackTime(0.0), callTime(0.0) { } CallStats(int depth, int fnamenumber, int ncalls, Real totaltime, Real stacktime, Real calltime) : callStackDepth(depth), csFNameNumber(fnamenumber), nCSCalls(ncalls), totalTime(totaltime), - stackTime(stacktime), callTime(calltime) { } + stackTime(stacktime), callTime(calltime) { } int callStackDepth, csFNameNumber; Long nCSCalls; Real totalTime, stackTime, callTime; @@ -184,7 +184,7 @@ class BLProfiler static void AddCommStat(const CommFuncType cft, const int size, const int pid, const int tag); static void AddWait(const CommFuncType cft, const MPI_Request &reqs, - const MPI_Status &status, const bool bc); + const MPI_Status &status, const bool bc); static void AddWaitsome(const CommFuncType cft, const Vector &reqs, const int completed, const Vector &status, const bool bc); @@ -322,14 +322,14 @@ namespace BLProfilerUtils { void WriteRow(std::ostream &os, const std::string &fname, const BLProfiler::ProfStats &pstats, const Real percent, - const int colWidth, const Real maxlen, - const bool bwriteavg); + const int colWidth, const Real maxlen, + const bool bwriteavg); void WriteStats(std::ostream &os, const std::map &mpStats, - const std::map &fnameNumbers, - const Vector &callTraces, - bool bwriteavg = false, bool bwriteinclusivetimes = false); + const std::map &fnameNumbers, + const Vector &callTraces, + bool bwriteavg = false, bool bwriteinclusivetimes = false); } diff --git a/Src/Base/AMReX_BLProfiler.cpp b/Src/Base/AMReX_BLProfiler.cpp index 80d89c57a51..fba68f34243 100644 --- a/Src/Base/AMReX_BLProfiler.cpp +++ b/Src/Base/AMReX_BLProfiler.cpp @@ -245,8 +245,8 @@ void BLProfiler::Initialize() { std::string cft; cfex >> cft; if( ! cfex.eof()) { - vEx.push_back(CommStats::StringToCFT(cft)); - } + vEx.push_back(CommStats::StringToCFT(cft)); + } } for(int i(0); i < vEx.size(); ++i) { CommStats::cftExclude.insert(vEx[i]); @@ -452,7 +452,7 @@ void BLProfiler::RegionStop(const std::string &rname) { std::map::iterator it = BLProfiler::mRegionNameNumbers.find(rname); if(it == BLProfiler::mRegionNameNumbers.end()) { // ---- error // amrex::Print() << "-------- error in RegionStop: region " << rname -// << " never started.\n"; +// << " never started.\n"; rnameNumber = BLProfiler::mRegionNameNumbers.size(); BLProfiler::mRegionNameNumbers.insert(std::pair(rname, rnameNumber)); } else { @@ -543,8 +543,8 @@ void WriteHeader(std::ostream &ios, const int colWidth, void WriteRow(std::ostream &ios, const std::string &fname, const BLProfiler::ProfStats &pstats, const Real percent, - const int colWidth, const Real maxlen, - const bool bwriteavg) + const int colWidth, const Real maxlen, + const bool bwriteavg) { int maxlenI = int(maxlen); int numPrec(4), pctPrec(2); @@ -561,33 +561,33 @@ void WriteRow(std::ostream &ios, const std::string &fname, ios << std::setw(maxlenI + 2) << fname << " " << std::setw(colWidth) << pstats.nCalls << " " << std::setprecision(numPrec) << std::fixed << std::setw(colWidth) - << pstats.minTime << " " + << pstats.minTime << " " << std::setprecision(numPrec) << std::fixed << std::setw(colWidth) - << pstats.avgTime << " " + << pstats.avgTime << " " << std::setprecision(numPrec) << std::fixed << std::setw(colWidth) - << pstats.maxTime << " " + << pstats.maxTime << " " << std::setprecision(numPrec) << std::fixed << std::setw(colWidth) - << stdDev << " " + << stdDev << " " << std::setprecision(numPrec) << std::fixed << std::setw(colWidth) - << coeffVariation << " " + << coeffVariation << " " << std::setprecision(pctPrec) << std::fixed << std::setw(colWidth) - << percent << " %" << '\n'; + << percent << " %" << '\n'; } else { ios << std::setw(maxlenI + 2) << fname << " " << std::setw(colWidth) << pstats.nCalls << " " << std::setprecision(numPrec) << std::fixed << std::setw(colWidth) - << pstats.totalTime << " " + << pstats.totalTime << " " << std::setprecision(pctPrec) << std::fixed << std::setw(colWidth) - << percent << " %" << '\n'; + << percent << " %" << '\n'; } } void WriteStats(std::ostream &ios, const std::map &mpStats, - const std::map &fnameNumbers, - const Vector &callTraces, - bool bwriteavg, bool bwriteinclusivetimes) + const std::map &fnameNumbers, + const Vector &callTraces, + bool bwriteavg, bool bwriteinclusivetimes) { const int myProc(ParallelDescriptor::MyProc()); const int colWidth(10); @@ -727,7 +727,7 @@ void WriteStats(std::ostream &ios, bool recursiveCall(false); for(int d(0); d < depth; ++d) { if(cs.csFNameNumber == callStack[d]) { - recursiveFuncs.insert(cs.csFNameNumber); + recursiveFuncs.insert(cs.csFNameNumber); recursiveCall = true; } } @@ -947,8 +947,8 @@ void BLProfiler::WriteBaseProfile(bool bFlushing, bool memCheck) { // ---- wri std::string dFileName(phFilePrefix + "_D_"); for(int p(0); p < nProcs; ++p) { std::string dFullName(NFilesIter::FileName(nOutFiles, dFileName, p, groupSets)); - phHeaderFile << "BLProfProc " << p << " datafile " << dFullName - << " seekpos " << seekPosOut[p] << '\n'; + phHeaderFile << "BLProfProc " << p << " datafile " << dFullName + << " seekpos " << seekPosOut[p] << '\n'; } phHeaderFile << "calcEndTime " << std::setprecision(16) << amrex::second() - startTime << '\n'; @@ -972,12 +972,12 @@ void BLProfiler::WriteCallTrace(bool bFlushing, bool memCheck) { // ---- write ParallelDescriptor::ReduceIntMax(nCT); bool doFlush(nCT > traceFlushSize); if(doFlush) { - amrex::Print() << "Flushing call traces: nCT traceFlushSize = " << nCT - << " " << traceFlushSize << "\n"; + amrex::Print() << "Flushing call traces: nCT traceFlushSize = " << nCT + << " " << traceFlushSize << "\n"; } else { - amrex::Print() << "Bypassing call trace flush, nCT < traceFlushSize: " << nCT - << " " << traceFlushSize << "\n"; - return; + amrex::Print() << "Bypassing call trace flush, nCT < traceFlushSize: " << nCT + << " " << traceFlushSize << "\n"; + return; } } @@ -1005,8 +1005,8 @@ void BLProfiler::WriteCallTrace(bool bFlushing, bool memCheck) { // ---- write amrex::SyncStrings(localStrings, syncedStrings, alreadySynced); if( ! alreadySynced) { // ---- need to remap names and numbers - amrex::Print() << "**** Warning: region names not synced: unsupported.\n"; - // unsupported for now + amrex::Print() << "**** Warning: region names not synced: unsupported.\n"; + // unsupported for now } if(ParallelDescriptor::IOProcessor()) { @@ -1025,7 +1025,7 @@ void BLProfiler::WriteCallTrace(bool bFlushing, bool memCheck) { // ---- write it != mRegionNameNumbers.end(); ++it) { csGlobalHeaderFile << "RegionName " << '"' << it->first << '"' - << ' ' << it->second << '\n'; + << ' ' << it->second << '\n'; } if(NFilesIter::CheckNFiles(nProcs, nOutFiles, groupSets)) { for(int i(0); i < nOutFiles; ++i) { @@ -1078,26 +1078,26 @@ void BLProfiler::WriteCallTrace(bool bFlushing, bool memCheck) { // ---- write { std::string localDFileName(NFilesIter::FileName(nOutFiles, shortDFileNamePrefix, myProc, groupSets)); nfiHeader.Stream() << "CallStatsProc " << myProc - << " nRSS " << rStartStop.size() - << " nTraceStats " << vCallTrace.size() - << " datafile " << localDFileName - << " seekpos " << nfiDatafile.SeekPos() // ---- data file seek position - << '\n'; + << " nRSS " << rStartStop.size() + << " nTraceStats " << vCallTrace.size() + << " datafile " << localDFileName + << " seekpos " << nfiDatafile.SeekPos() // ---- data file seek position + << '\n'; for(std::map::iterator it = mFNameNumbers.begin(); - it != mFNameNumbers.end(); ++it) + it != mFNameNumbers.end(); ++it) { - nfiHeader.Stream() << "fName " << '"' << it->first << '"' - << ' ' << it->second << '\n'; + nfiHeader.Stream() << "fName " << '"' << it->first << '"' + << ' ' << it->second << '\n'; } #ifdef BL_TRACE_PROFILING nfiHeader.Stream() << std::setprecision(16) << "timeMinMax " - << CallStats::minCallTime << ' ' - << CallStats::maxCallTime << '\n'; + << CallStats::minCallTime << ' ' + << CallStats::maxCallTime << '\n'; #endif if(rStartStop.size() > 0) { - nfiDatafile.Stream().write((char *) rStartStop.dataPtr(), - rStartStop.size() * sizeof(RStartStop)); + nfiDatafile.Stream().write((char *) rStartStop.dataPtr(), + rStartStop.size() * sizeof(RStartStop)); } if(vCallTrace.size() > 0) { baseSeekPos = nfiDatafile.SeekPos(); @@ -1109,46 +1109,46 @@ void BLProfiler::WriteCallTrace(bool bFlushing, bool memCheck) { // ---- write if(bFlushing) { // ---- save stacked CallStats for(int ci(0); ci < callIndexStack.size(); ++ci) { - CallStatsStack &csStack = callIndexStack[ci]; - if( ! csStack.bFlushed) { - if(baseSeekPos < 0) { - amrex::Print(Print::AllProcs) << "**** Error: baseSeekPos = " << baseSeekPos << "\n"; - break; - } - Long spos(baseSeekPos + csStack.index * sizeof(CallStats)); - callIndexPatch.push_back(CallStatsPatch(spos, vCallTrace[csStack.index], longDFileName)); - csStack.bFlushed = true; - csStack.index = callIndexPatch.size() - 1; - } + CallStatsStack &csStack = callIndexStack[ci]; + if( ! csStack.bFlushed) { + if(baseSeekPos < 0) { + amrex::Print(Print::AllProcs) << "**** Error: baseSeekPos = " << baseSeekPos << "\n"; + break; + } + Long spos(baseSeekPos + csStack.index * sizeof(CallStats)); + callIndexPatch.push_back(CallStatsPatch(spos, vCallTrace[csStack.index], longDFileName)); + csStack.bFlushed = true; + csStack.index = callIndexPatch.size() - 1; + } } } else { // ---- patch the incomplete CallStats on disk // ---- probably should throttle these for large nprocs for(int ci(0); ci < callIndexPatch.size(); ++ci) { - CallStatsPatch &csPatch = callIndexPatch[ci]; + CallStatsPatch &csPatch = callIndexPatch[ci]; std::fstream csDFile; CallStats csOnDisk; csDFile.open(csPatch.fileName.c_str(), std::ios::in | std::ios::out | - std::ios::binary); + std::ios::binary); if( ! csDFile.good()) { amrex::FileOpenFailed(csPatch.fileName); } csDFile.seekg(csPatch.seekPos, std::ios::beg); csDFile.read((char *) &csOnDisk, sizeof(CallStats)); - bool bReportPatches(false); - if(bReportPatches) { - amrex::Print(Print::AllProcs) - << myProc << "::PATCH: csOnDisk.st tt = " << csOnDisk.stackTime - << " " << csOnDisk.totalTime << '\n' - << myProc << "::PATCH: csPatch.st tt = " << csPatch.callStats.stackTime - << " " << csPatch.callStats.totalTime << " :::: " << csPatch.fileName << "\n"; - } - csOnDisk.totalTime = csPatch.callStats.totalTime; - csOnDisk.stackTime = csPatch.callStats.stackTime; + bool bReportPatches(false); + if(bReportPatches) { + amrex::Print(Print::AllProcs) + << myProc << "::PATCH: csOnDisk.st tt = " << csOnDisk.stackTime + << " " << csOnDisk.totalTime << '\n' + << myProc << "::PATCH: csPatch.st tt = " << csPatch.callStats.stackTime + << " " << csPatch.callStats.totalTime << " :::: " << csPatch.fileName << "\n"; + } + csOnDisk.totalTime = csPatch.callStats.totalTime; + csOnDisk.stackTime = csPatch.callStats.stackTime; csDFile.seekp(csPatch.seekPos, std::ios::beg); csDFile.write((char *) &csOnDisk, sizeof(CallStats)); - csDFile.flush(); - csDFile.close(); + csDFile.flush(); + csDFile.close(); } callIndexPatch.clear(); } @@ -1182,11 +1182,11 @@ void BLProfiler::WriteCommStats(bool bFlushing, bool memCheck) CommStats::cftExclude.erase(AllCFTypes); } amrex::Print() << "Bypassing comm stats flush, nCS < csFlushSize: " << nCS - << " " << csFlushSize << "\n"; + << " " << csFlushSize << "\n"; return; } else { amrex::Print() << "Flushing commstats: nCSmax csFlushSize = " << nCS - << " " << csFlushSize << "\n"; + << " " << csFlushSize << "\n"; } } @@ -1284,8 +1284,8 @@ void BLProfiler::WriteCommStats(bool bFlushing, bool memCheck) nfiHeader.Stream() << "CommProfProc " << myProc << " nCommStats " << vCommStats.size() << " datafile " << localDFileName - << " seekpos " << nfiDatafile.SeekPos() // ---- data file seek posotion - << " " << procName << '\n'; + << " seekpos " << nfiDatafile.SeekPos() // ---- data file seek posotion + << " " << procName << '\n'; for(int ib(0); ib < CommStats::barrierNames.size(); ++ib) { int seekindex(CommStats::barrierNames[ib].second); CommStats &cs = vCommStats[seekindex]; @@ -1336,7 +1336,7 @@ void BLProfiler::WriteCommStats(bool bFlushing, bool memCheck) ParallelDescriptor::Barrier("BLProfiler::WriteCommStats::end"); amrex::Print() << "BLProfiler::WriteCommStats(): time: " - << amrex::second() - wcsStart << "\n"; + << amrex::second() - wcsStart << "\n"; } @@ -1349,7 +1349,7 @@ void BLProfiler::WriteFortProfErrors() { it != BLProfiler::mFortProfs.end(); ++it) { amrex::Print() << "FFFF function not stopped: fname ptr = " << it->first - << " ---->" << it->second << "<----" << std::endl; + << " ---->" << it->second << "<----" << std::endl; } amrex::Print() << "FFFFFFFF -------- END FORTRAN PROFILING UNSTOPPED ERRORS" << std::endl; } @@ -1463,7 +1463,7 @@ void BLProfiler::AddAllReduce(const CommFuncType cft, const int size, } void BLProfiler::AddWait(const CommFuncType cft, const MPI_Request &req, - const MPI_Status &status, const bool beforecall) + const MPI_Status &status, const bool beforecall) { amrex::ignore_unused(req); #ifdef BL_USE_MPI diff --git a/Src/Base/AMReX_BLutil_F.F90 b/Src/Base/AMReX_BLutil_F.F90 index f822cf72693..1e5c745e321 100644 --- a/Src/Base/AMReX_BLutil_F.F90 +++ b/Src/Base/AMReX_BLutil_F.F90 @@ -8,7 +8,7 @@ ! which we set to -1 ! bl_str2int converts a FORTRAN string to an integer array, ! bl_int2str converts an integer array to a FORTRAN string. -! +! !----------------------------------------------------------------------- subroutine bl_str2int(iarr, n, str) diff --git a/Src/Base/AMReX_BaseFab.H b/Src/Base/AMReX_BaseFab.H index 67ca0a3594b..6fbae4fab11 100644 --- a/Src/Base/AMReX_BaseFab.H +++ b/Src/Base/AMReX_BaseFab.H @@ -224,7 +224,7 @@ public: //! Make BaseFab with desired domain (box) and number of components. explicit BaseFab (const Box& bx, int n = 1, bool alloc = true, - bool shared = false, Arena* ar = nullptr); + bool shared = false, Arena* ar = nullptr); BaseFab (const BaseFab& rhs, MakeType make_type, int scomp, int ncomp); @@ -1275,7 +1275,7 @@ public: template #endif T dot (const Box& xbx, int xcomp, const BaseFab& y, const Box& ybx, int ycomp, - int numcomp = 1) const noexcept; + int numcomp = 1) const noexcept; #if defined(AMREX_USE_GPU) template @@ -1950,7 +1950,7 @@ BaseFab::BaseFab (const Box& bx, int n, bool alloc, bool shared, Arena* ar) template BaseFab::BaseFab (const BaseFab& rhs, MakeType make_type, int scomp, int ncomp) - : DataAllocator{rhs.arena()}, + : DataAllocator{rhs.arena()}, dptr(const_cast(rhs.dataPtr(scomp))), domain(rhs.domain), nvar(ncomp), truesize(ncomp*rhs.domain.numPts()) @@ -2052,16 +2052,16 @@ BaseFab::resize (const Box& b, int n, Arena* ar) } else if (this->dptr == 0 || !this->ptr_owner) { - if (this->shared_memory) - amrex::Abort("BaseFab::resize: BaseFab in shared memory cannot increase size"); + if (this->shared_memory) + amrex::Abort("BaseFab::resize: BaseFab in shared memory cannot increase size"); this->dptr = nullptr; define(); } else if (this->nvar*this->domain.numPts() > this->truesize) { - if (this->shared_memory) - amrex::Abort("BaseFab::resize: BaseFab in shared memory cannot increase size"); + if (this->shared_memory) + amrex::Abort("BaseFab::resize: BaseFab in shared memory cannot increase size"); clear(); @@ -2100,26 +2100,26 @@ BaseFab::clear () noexcept // // Call T::~T() on the to-be-destroyed memory. // - if (this->ptr_owner) - { - if (this->shared_memory) - { - amrex::Abort("BaseFab::clear: BaseFab cannot be owner of shared memory"); - } + if (this->ptr_owner) + { + if (this->shared_memory) + { + amrex::Abort("BaseFab::clear: BaseFab cannot be owner of shared memory"); + } placementDelete(this->dptr, this->truesize); - this->free(this->dptr); + this->free(this->dptr); - if (this->nvar > 1) { - amrex::update_fab_stats(-this->truesize/this->nvar, -this->truesize, sizeof(T)); - } else { - amrex::update_fab_stats(0, -this->truesize, sizeof(T)); - } - } + if (this->nvar > 1) { + amrex::update_fab_stats(-this->truesize/this->nvar, -this->truesize, sizeof(T)); + } else { + amrex::update_fab_stats(0, -this->truesize, sizeof(T)); + } + } - this->dptr = 0; - this->truesize = 0; + this->dptr = 0; + this->truesize = 0; } } @@ -3002,7 +3002,7 @@ BaseFab::linComb (const BaseFab& f1, const Box& b1, int comp1, const auto slo1 = amrex::lbound(b1); const auto slo2 = amrex::lbound(b2); const Dim3 off1{slo1.x-dlo.x,slo1.y-dlo.y,slo1.z-dlo.z}; - const Dim3 off2{slo2.x-dlo.x,slo2.y-dlo.y,slo2.z-dlo.z}; + const Dim3 off2{slo2.x-dlo.x,slo2.y-dlo.y,slo2.z-dlo.z}; AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FLAG(run_on, b, numcomp, i, j, k, n, { diff --git a/Src/Base/AMReX_BaseFab.cpp b/Src/Base/AMReX_BaseFab.cpp index 57a8861c6be..4ae44944906 100644 --- a/Src/Base/AMReX_BaseFab.cpp +++ b/Src/Base/AMReX_BaseFab.cpp @@ -128,7 +128,7 @@ TotalCellsAllocatedInFabsHWM () noexcept #endif } -void +void ResetTotalBytesAllocatedInFabsHWM () noexcept { #ifdef AMREX_USE_OMP diff --git a/Src/Base/AMReX_BaseUmap_f.H b/Src/Base/AMReX_BaseUmap_f.H index 43db5f9e06c..ec33c762012 100644 --- a/Src/Base/AMReX_BaseUmap_f.H +++ b/Src/Base/AMReX_BaseUmap_f.H @@ -9,10 +9,10 @@ extern "C" { #endif - + amrex_real amrex_fort_umap_norm (const int* lo, const int* hi, const amrex_real* src, const int* src_sz, - const int* kt, const int* ktlo, const int* kthi, + const int* kt, const int* ktlo, const int* kthi, const int* max_mv, const int* ncomp, const int* p); diff --git a/Src/Base/AMReX_BlockMutex.H b/Src/Base/AMReX_BlockMutex.H index 15718b6b9d0..f1e6feea578 100644 --- a/Src/Base/AMReX_BlockMutex.H +++ b/Src/Base/AMReX_BlockMutex.H @@ -7,7 +7,7 @@ namespace amrex { #ifdef AMREX_USE_GPU - + struct BlockMutex { union state_t @@ -24,7 +24,7 @@ struct BlockMutex static void init_states (state_t* state, int N) noexcept; explicit BlockMutex (int N) noexcept; - + ~BlockMutex (); void operator= (BlockMutex const&) = delete; @@ -84,6 +84,6 @@ private: int m_nstates; state_t* m_state; }; -#endif +#endif } #endif diff --git a/Src/Base/AMReX_BlockMutex.cpp b/Src/Base/AMReX_BlockMutex.cpp index a01daa11072..32e6d8face1 100644 --- a/Src/Base/AMReX_BlockMutex.cpp +++ b/Src/Base/AMReX_BlockMutex.cpp @@ -36,6 +36,6 @@ BlockMutex::~BlockMutex () { } #endif - + } diff --git a/Src/Base/AMReX_Box.H b/Src/Base/AMReX_Box.H index 21e42831653..85f39d719e8 100644 --- a/Src/Base/AMReX_Box.H +++ b/Src/Base/AMReX_Box.H @@ -241,8 +241,8 @@ public: AMREX_GPU_HOST_DEVICE bool contains (const Box& b) const noexcept { - BL_ASSERT(sameType(b)); - return b.smallend.allGE(smallend) && b.bigend.allLE(bigend); + BL_ASSERT(sameType(b)); + return b.smallend.allGE(smallend) && b.bigend.allLE(bigend); } //! Returns true if argument is strictly contained within Box. @@ -256,8 +256,8 @@ public: AMREX_GPU_HOST_DEVICE bool strictly_contains (const Box& b) const noexcept { - BL_ASSERT(sameType(b)); - return b.smallend.allGT(smallend) && b.bigend.allLT(bigend); + BL_ASSERT(sameType(b)); + return b.smallend.allGT(smallend) && b.bigend.allLT(bigend); } //! Returns true if argument is strictly contained within Box. @@ -310,7 +310,7 @@ public: { return btype < rhs.btype || ((btype == rhs.btype) && - ( (smallend < rhs.smallend) || + ( (smallend < rhs.smallend) || ((smallend == rhs.smallend) && (bigend < rhs.bigend)) )); } AMREX_GPU_HOST_DEVICE @@ -975,14 +975,14 @@ Box::next (IntVect& p) const noexcept #if (AMREX_SPACEDIM >= 2) if (p[0] > bigend[0]) { - p[0] = smallend[0]; - ++p[1]; + p[0] = smallend[0]; + ++p[1]; #if (AMREX_SPACEDIM == 3) - if (p[1] > bigend[1]) - { - p[1] = smallend[1]; - ++p[2]; - } + if (p[1] > bigend[1]) + { + p[1] = smallend[1]; + ++p[2]; + } #endif } #endif @@ -1088,7 +1088,7 @@ public: int* data () && = delete; Box make_box () const noexcept { - return Box(IntVect(p), IntVect(p+AMREX_SPACEDIM), IntVect(p+2*AMREX_SPACEDIM)); + return Box(IntVect(p), IntVect(p+AMREX_SPACEDIM), IntVect(p+2*AMREX_SPACEDIM)); } static int size () noexcept { return 3*AMREX_SPACEDIM; } @@ -1414,7 +1414,7 @@ Box bdryNode (const Box& b, Orientation face, int len=1) noexcept if (face.isLow()) { int sm = low[dir]; - low.setVal(dir,sm-len+1); + low.setVal(dir,sm-len+1); hi.setVal(dir,sm); } else diff --git a/Src/Base/AMReX_Box.cpp b/Src/Base/AMReX_Box.cpp index b0d6423e032..436362c467e 100644 --- a/Src/Base/AMReX_Box.cpp +++ b/Src/Base/AMReX_Box.cpp @@ -48,26 +48,26 @@ operator>> (std::istream& is, if (c == '(') { is >> lo >> hi; - is >> c; - // Read an optional IndexType - is.putback(c); - if ( c == '(' ) - { - is >> typ; - } + is >> c; + // Read an optional IndexType + is.putback(c); + if ( c == '(' ) + { + is >> typ; + } is.ignore(BL_IGNORE_MAX,')'); } else if (c == '<') { - is.putback(c); + is.putback(c); is >> lo >> hi; - is >> c; - // Read an optional IndexType - is.putback(c); - if ( c == '<' ) - { - is >> typ; - } + is >> c; + // Read an optional IndexType + is.putback(c); + if ( c == '<' ) + { + is >> typ; + } //is.ignore(BL_IGNORE_MAX,'>'); } else @@ -87,20 +87,20 @@ BoxCommHelper::BoxCommHelper (const Box& bx, int* p_) : p(p_) { if (p == 0) { - v.resize(3*AMREX_SPACEDIM); - p = &v[0]; + v.resize(3*AMREX_SPACEDIM); + p = &v[0]; } AMREX_D_EXPR(p[0] = bx.smallend[0], - p[1] = bx.smallend[1], - p[2] = bx.smallend[2]); + p[1] = bx.smallend[1], + p[2] = bx.smallend[2]); AMREX_D_EXPR(p[0+AMREX_SPACEDIM] = bx.bigend[0], - p[1+AMREX_SPACEDIM] = bx.bigend[1], - p[2+AMREX_SPACEDIM] = bx.bigend[2]); + p[1+AMREX_SPACEDIM] = bx.bigend[1], + p[2+AMREX_SPACEDIM] = bx.bigend[2]); const IntVect& typ = bx.btype.ixType(); AMREX_D_EXPR(p[0+AMREX_SPACEDIM*2] = typ[0], - p[1+AMREX_SPACEDIM*2] = typ[1], - p[2+AMREX_SPACEDIM*2] = typ[2]); + p[1+AMREX_SPACEDIM*2] = typ[1], + p[2+AMREX_SPACEDIM*2] = typ[2]); } void diff --git a/Src/Base/AMReX_BoxArray.H b/Src/Base/AMReX_BoxArray.H index 630ad9dd3de..e265f476f5a 100644 --- a/Src/Base/AMReX_BoxArray.H +++ b/Src/Base/AMReX_BoxArray.H @@ -558,7 +558,7 @@ public: BoxArray (const BoxArray& rhs, const BATransformer& trans); BoxArray (BoxList&& bl, IntVect const& max_grid_size); - + /** * \brief Initialize the BoxArray from a single box. * It is an error if the BoxArray has already been initialized. @@ -758,10 +758,10 @@ public: //! intersect Box and BoxArray(+ghostcells), then store the result in isects void intersections (const Box& bx, std::vector< std::pair >& isects, - bool first_only, int ng) const; + bool first_only, int ng) const; void intersections (const Box& bx, std::vector< std::pair >& isects, - bool first_only, const IntVect& ng) const; + bool first_only, const IntVect& ng) const; //! Return box - boxarray BoxList complementIn (const Box& b) const; diff --git a/Src/Base/AMReX_BoxArray.cpp b/Src/Base/AMReX_BoxArray.cpp index 3e5ed261138..9633638983d 100644 --- a/Src/Base/AMReX_BoxArray.cpp +++ b/Src/Base/AMReX_BoxArray.cpp @@ -30,54 +30,54 @@ namespace { const int bl_ignore_max = 100000; } -BARef::BARef () -{ +BARef::BARef () +{ #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(1); -#endif +#endif } -BARef::BARef (size_t size) - : m_abox(size) -{ +BARef::BARef (size_t size) + : m_abox(size) +{ #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(1); -#endif +#endif } - + BARef::BARef (const Box& b) -{ - define(b); +{ + define(b); } BARef::BARef (const BoxList& bl) : m_abox(bl.data()) -{ +{ #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(1); -#endif +#endif } BARef::BARef (BoxList&& bl) noexcept : m_abox(std::move(bl.data())) -{ +{ #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(1); -#endif +#endif } BARef::BARef (std::istream& is) -{ +{ int ndims; - define(is, ndims); + define(is, ndims); } -BARef::BARef (const BARef& rhs) +BARef::BARef (const BARef& rhs) : m_abox(rhs.m_abox) // don't copy hash { #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(1); -#endif +#endif } BARef::~BARef () @@ -85,7 +85,7 @@ BARef::~BARef () #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(-1); updateMemoryUsage_hash(-1); -#endif +#endif } void @@ -171,7 +171,7 @@ BARef::define (BoxList&& bl) noexcept #endif } -void +void BARef::resize (Long n) { #ifdef AMREX_MEM_PROFILING updateMemoryUsage_box(-1); @@ -190,16 +190,16 @@ void BARef::updateMemoryUsage_box (int s) { if (m_abox.size() > 1) { - Long b = amrex::bytesOf(m_abox); - if (s > 0) { - total_box_bytes += b; - total_box_bytes_hwm = std::max(total_box_bytes_hwm, total_box_bytes); - ++numboxarrays; - numboxarrays_hwm = std::max(numboxarrays_hwm, numboxarrays); - } else { - total_box_bytes -= b; - --numboxarrays; - } + Long b = amrex::bytesOf(m_abox); + if (s > 0) { + total_box_bytes += b; + total_box_bytes_hwm = std::max(total_box_bytes_hwm, total_box_bytes); + ++numboxarrays; + numboxarrays_hwm = std::max(numboxarrays_hwm, numboxarrays); + } else { + total_box_bytes -= b; + --numboxarrays; + } } } @@ -207,17 +207,17 @@ void BARef::updateMemoryUsage_hash (int s) { if (hash.size() > 0) { - Long b = sizeof(hash); - for (const auto& x: hash) { - b += amrex::gcc_map_node_extra_bytes - + sizeof(IntVect) + amrex::bytesOf(x.second); - } - if (s > 0) { - total_hash_bytes += b; - total_hash_bytes_hwm = std::max(total_hash_bytes_hwm, total_hash_bytes); - } else { - total_hash_bytes -= b; - } + Long b = sizeof(hash); + for (const auto& x: hash) { + b += amrex::gcc_map_node_extra_bytes + + sizeof(IntVect) + amrex::bytesOf(x.second); + } + if (s > 0) { + total_hash_bytes += b; + total_hash_bytes_hwm = std::max(total_hash_bytes_hwm, total_hash_bytes); + } else { + total_hash_bytes -= b; + } } } #endif @@ -226,20 +226,20 @@ void BARef::Initialize () { if (!initialized) { - initialized = true; + initialized = true; #ifdef AMREX_MEM_PROFILING - MemProfiler::add("BoxArray", std::function - ([] () -> MemProfiler::MemInfo { - return {total_box_bytes, total_box_bytes_hwm}; - })); - MemProfiler::add("BoxArrayHash", std::function - ([] () -> MemProfiler::MemInfo { - return {total_hash_bytes, total_hash_bytes_hwm}; - })); - MemProfiler::add("BoxArray Innard", std::function - ([] () -> MemProfiler::NBuildsInfo { - return {numboxarrays, numboxarrays_hwm}; - })); + MemProfiler::add("BoxArray", std::function + ([] () -> MemProfiler::MemInfo { + return {total_box_bytes, total_box_bytes_hwm}; + })); + MemProfiler::add("BoxArrayHash", std::function + ([] () -> MemProfiler::MemInfo { + return {total_hash_bytes, total_hash_bytes_hwm}; + })); + MemProfiler::add("BoxArray Innard", std::function + ([] () -> MemProfiler::NBuildsInfo { + return {numboxarrays, numboxarrays_hwm}; + })); #endif } @@ -256,8 +256,8 @@ void BoxArray::Initialize () { if (!initialized) { - initialized = true; - BARef::Initialize(); + initialized = true; + BARef::Initialize(); } amrex::ExecOnFinalize(BoxArray::Finalize); @@ -584,7 +584,7 @@ BoxArray::refine (const IntVect& iv) #pragma omp parallel for #endif for (int i = 0; i < N; i++) { - BL_ASSERT(m_ref->m_abox[i].ok()); + BL_ASSERT(m_ref->m_abox[i].ok()); m_ref->m_abox[i].refine(iv); } return *this; @@ -756,7 +756,7 @@ BoxArray::growHi (int dir, BoxArray& BoxArray::surroundingNodes () { - + return this->convert(IndexType::TheNodeType()); } @@ -927,7 +927,7 @@ BoxArray::boxList () const BoxList newb; newb.data().reserve(N); if (N > 0) { - newb.set(ixType()); + newb.set(ixType()); auto const& bxs = this->m_ref->m_abox; if (m_bat.is_null()) { for (int i = 0; i < N; ++i) { @@ -952,9 +952,9 @@ bool BoxArray::contains (const IntVect& iv) const { if (size() > 0) { - return intersects(Box(iv,iv,ixType())); + return intersects(Box(iv,iv,ixType())); } else { - return false; + return false; } } @@ -973,14 +973,14 @@ BoxArray::contains (const Box& b, bool assume_disjoint_ba) const if (isects.size() > 0) { - if (assume_disjoint_ba) { + if (assume_disjoint_ba) { BL_ASSERT(isDisjoint()); - Long nbx = b.numPts(), nisects = 0L; - for (int i = 0, N = isects.size(); i < N; i++) { - nisects += isects[i].second.numPts(); - } - result = nbx == nisects; - } else { + Long nbx = b.numPts(), nisects = 0L; + for (int i = 0, N = isects.size(); i < N; i++) { + nisects += isects[i].second.numPts(); + } + result = nbx == nisects; + } else { Vector vflag(b.numPts(), 1); BaseFab fabflag(b, 1, vflag.data()); for (int i = 0, N = isects.size(); i < N; i++) { @@ -990,7 +990,7 @@ BoxArray::contains (const Box& b, bool assume_disjoint_ba) const if (x == 1) return false; } result = true; - } + } } } @@ -1007,7 +1007,7 @@ BoxArray::contains (const BoxArray& ba, bool assume_disjoint_ba) const for (int i = 0, N = ba.size(); i < N; ++i) { if (!contains(ba[i],assume_disjoint_ba)) { return false; - } + } } return true; @@ -1022,39 +1022,39 @@ BoxArray::minimalBox () const if (N > 0) { #ifdef AMREX_USE_OMP - bool use_single_thread = omp_in_parallel(); - const int nthreads = use_single_thread ? 1 : omp_get_max_threads(); + bool use_single_thread = omp_in_parallel(); + const int nthreads = use_single_thread ? 1 : omp_get_max_threads(); #else - bool use_single_thread = true; - const int nthreads = 1; + bool use_single_thread = true; + const int nthreads = 1; #endif - if (use_single_thread) - { - minbox = m_ref->m_abox[0]; - for (int i = 1; i < N; ++i) { - minbox.minBox(m_ref->m_abox[i]); - } - } - else - { - Vector bxs(nthreads, m_ref->m_abox[0]); + if (use_single_thread) + { + minbox = m_ref->m_abox[0]; + for (int i = 1; i < N; ++i) { + minbox.minBox(m_ref->m_abox[i]); + } + } + else + { + Vector bxs(nthreads, m_ref->m_abox[0]); #ifdef AMREX_USE_OMP #pragma omp parallel #endif - { + { int tid = OpenMP::get_thread_num(); #ifdef AMREX_USE_OMP #pragma omp for #endif - for (int i = 0; i < N; ++i) { - bxs[tid].minBox(m_ref->m_abox[i]); - } - } - minbox = bxs[0]; - for (int i = 1; i < nthreads; ++i) { - minbox.minBox(bxs[i]); - } - } + for (int i = 0; i < N; ++i) { + bxs[tid].minBox(m_ref->m_abox[i]); + } + } + minbox = bxs[0]; + for (int i = 1; i < nthreads; ++i) { + minbox.minBox(bxs[i]); + } + } } minbox.coarsen(crseRatio()).convert(ixType()); return minbox; @@ -1166,8 +1166,8 @@ BoxArray::intersections (const Box& bx, void BoxArray::intersections (const Box& bx, std::vector< std::pair >& isects, - bool first_only, - int ng) const + bool first_only, + int ng) const { intersections(bx,isects,first_only,IntVect(ng)); } @@ -1175,8 +1175,8 @@ BoxArray::intersections (const Box& bx, void BoxArray::intersections (const Box& bx, std::vector< std::pair >& isects, - bool first_only, - const IntVect& ng) const + bool first_only, + const IntVect& ng) const { // This is called too many times BL_PROFILE("BoxArray::intersections()"); @@ -1188,25 +1188,25 @@ BoxArray::intersections (const Box& bx, { BL_ASSERT(bx.ixType() == ixType()); - Box gbx = amrex::grow(bx,ng); + Box gbx = amrex::grow(bx,ng); - IntVect glo = gbx.smallEnd(); - IntVect ghi = gbx.bigEnd(); - const IntVect& doilo = getDoiLo(); - const IntVect& doihi = getDoiHi(); + IntVect glo = gbx.smallEnd(); + IntVect ghi = gbx.bigEnd(); + const IntVect& doilo = getDoiLo(); + const IntVect& doihi = getDoiHi(); - gbx.setSmall(glo - doihi).setBig(ghi + doilo); + gbx.setSmall(glo - doihi).setBig(ghi + doilo); gbx.refine(crseRatio()).coarsen(m_ref->crsn); - + const IntVect& sm = amrex::max(gbx.smallEnd()-1, m_ref->bbox.smallEnd()); const IntVect& bg = amrex::min(gbx.bigEnd(), m_ref->bbox.bigEnd()); Box cbx(sm,bg); cbx.normalize(); - if (!cbx.intersects(m_ref->bbox)) return; + if (!cbx.intersects(m_ref->bbox)) return; - auto TheEnd = BoxHashMap.cend(); + auto TheEnd = BoxHashMap.cend(); auto& abox = m_ref->m_abox; @@ -1365,7 +1365,7 @@ BoxArray::clear_hash_bin () const if (!m_ref->hash.empty()) { #ifdef AMREX_MEM_PROFILING - m_ref->updateMemoryUsage_hash(-1); + m_ref->updateMemoryUsage_hash(-1); #endif m_ref->hash.clear(); m_ref->has_hashmap = false; @@ -1440,7 +1440,7 @@ BoxArray::removeOverlap (bool simplify) bl.push_back(b); } } - + if (simplify) { bl.simplify(); } @@ -1461,12 +1461,12 @@ BoxArray::type_update () { if (!empty()) { - if (! ixType().cellCentered()) - { + if (! ixType().cellCentered()) + { for (auto& bx : m_ref->m_abox) { - bx.enclosedCells(); - } - } + bx.enclosedCells(); + } + } } } @@ -1498,11 +1498,11 @@ BoxArray::getHashMap () const // // Calculate the bounding box & maximum extent of the boxes. // - IntVect maxext = IntVect::TheUnitVector(); + IntVect maxext = IntVect::TheUnitVector(); Box boundingbox = m_ref->m_abox[0]; - const int N = size(); - for (int i = 0; i < N; ++i) + const int N = size(); + for (int i = 0; i < N; ++i) { Box bx = m_ref->m_abox[i]; bx.normalize(); @@ -1512,8 +1512,8 @@ BoxArray::getHashMap () const for (int i = 0; i < N; i++) { - const IntVect& crsnsmlend - = amrex::coarsen(m_ref->m_abox[i].smallEnd(),maxext); + const IntVect& crsnsmlend + = amrex::coarsen(m_ref->m_abox[i].smallEnd(),maxext); BoxHashMap[crsnsmlend].push_back(i); } @@ -1522,7 +1522,7 @@ BoxArray::getHashMap () const m_ref->bbox.normalize(); #ifdef AMREX_MEM_PROFILING - m_ref->updateMemoryUsage_hash(1); + m_ref->updateMemoryUsage_hash(1); #endif #ifdef AMREX_USE_OMP @@ -1542,8 +1542,8 @@ BoxArray::uniqify () if (m_ref.use_count() == 1) { clear_hash_bin(); } else { - auto p = std::make_shared(*m_ref); - std::swap(m_ref,p); + auto p = std::make_shared(*m_ref); + std::swap(m_ref,p); } IntVect cr = crseRatio(); if (cr != IntVect::TheUnitVector()) { @@ -1759,7 +1759,7 @@ GetBndryCells (const BoxArray& ba, BoxList bl_diff(btype); for (int i = 0, N = tba.size(); i < N; ++i) { - const Box& bx = tba[i]; + const Box& bx = tba[i]; amrex::boxDiff(bl_diff, amrex::grow(bx,ngrow), bx); gcells.join(bl_diff); } @@ -1830,13 +1830,13 @@ readBoxArray (BoxArray& ba, bool match (const BoxArray& x, const BoxArray& y) { if (x == y) { - return true; + return true; } else { - bool m = (x.size() == y.size()) && (x.ixType() == y.ixType()); - for (int i = 0, N = x.size(); i < N && m; ++i) { - m = x[i] == y[i]; - } - return m; + bool m = (x.size() == y.size()) && (x.ixType() == y.ixType()); + for (int i = 0, N = x.size(); i < N && m; ++i) { + m = x[i] == y[i]; + } + return m; } } diff --git a/Src/Base/AMReX_BoxDomain.cpp b/Src/Base/AMReX_BoxDomain.cpp index 8c4a7c4a7d3..37a04174e23 100644 --- a/Src/Base/AMReX_BoxDomain.cpp +++ b/Src/Base/AMReX_BoxDomain.cpp @@ -190,9 +190,9 @@ BoxDomain::ok () const { if (bli->intersects(*blii)) { -// amrex::Print(Print::AllProcs) << "Invalid DOMAIN, boxes overlap" << '\n' -// << "b1 = " << *bli << '\n' -// << "b2 = " << *blii << '\n'; +// amrex::Print(Print::AllProcs) << "Invalid DOMAIN, boxes overlap" << '\n' +// << "b1 = " << *bli << '\n' +// << "b2 = " << *blii << '\n'; status = false; } } diff --git a/Src/Base/AMReX_BoxList.H b/Src/Base/AMReX_BoxList.H index 603b8e2e78b..b8e5f6ed037 100644 --- a/Src/Base/AMReX_BoxList.H +++ b/Src/Base/AMReX_BoxList.H @@ -211,8 +211,8 @@ public: const Vector& data() const noexcept { return m_lbox; } void swap (BoxList& rhs) { - std::swap(m_lbox, rhs.m_lbox); - std::swap(btype, rhs.btype); + std::swap(m_lbox, rhs.m_lbox); + std::swap(btype, rhs.btype); } void Bcast (); diff --git a/Src/Base/AMReX_BoxList.cpp b/Src/Base/AMReX_BoxList.cpp index 6c9fa284538..f0f3bfd9e61 100644 --- a/Src/Base/AMReX_BoxList.cpp +++ b/Src/Base/AMReX_BoxList.cpp @@ -152,7 +152,7 @@ BoxList::BoxList () {} BoxList::BoxList (const Box& bx) - : + : m_lbox(1,bx), btype(bx.ixType()) { @@ -186,30 +186,30 @@ BoxList::BoxList(const Box& bx, const IntVect& tilesize) int ntiles = 1; IntVect nt; for (int d=0; d const& area, else if (coord == 1) { const Real pi = 3.1415926535897932; - AMREX_PRAGMA_SIMD + AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real ri = offset[0] + dx[0]*i; Real a = (2.*pi)*ri; @@ -81,7 +81,7 @@ amrex_setarea (Box const& bx, Array4 const& area, else { const Real pi = 3.1415926535897932; - AMREX_PRAGMA_SIMD + AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real ri = offset[0] + dx[0]*i; Real a = (4.0*pi)*ri*ri; @@ -109,7 +109,7 @@ amrex_setdloga (Box const& bx, Array4 const& dloga, } else if (coord == 1) { - AMREX_PRAGMA_SIMD + AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real rc = offset[0] + dx[0]*(i+0.5); dloga(i,0,0) = 1.0/rc; @@ -117,7 +117,7 @@ amrex_setdloga (Box const& bx, Array4 const& dloga, } else { - AMREX_PRAGMA_SIMD + AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real ri = offset[0] + dx[0]*(i+0.5); dloga(i,0,0) = 2.0/ri; diff --git a/Src/Base/AMReX_COORDSYS_2D_C.H b/Src/Base/AMReX_COORDSYS_2D_C.H index ce2e2b15916..6d2944775cd 100644 --- a/Src/Base/AMReX_COORDSYS_2D_C.H +++ b/Src/Base/AMReX_COORDSYS_2D_C.H @@ -115,7 +115,7 @@ amrex_setarea (Box const& bx, Array4 const& area, Real ti = offset[1] + dx[1]*(j); Real to = ti + dx[1]; Real tmp = (Real(2.)*pi)*(std::cos(ti)-std::cos(to)); - AMREX_PRAGMA_SIMD + AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real ri = offset[0] + dx[0]*(i); Real a = tmp*ri*ri; diff --git a/Src/Base/AMReX_CoordSys.cpp b/Src/Base/AMReX_CoordSys.cpp index e9372f7a458..97605e73141 100644 --- a/Src/Base/AMReX_CoordSys.cpp +++ b/Src/Base/AMReX_CoordSys.cpp @@ -37,10 +37,10 @@ CoordSys::CoordSys () noexcept // for (int k = 0; k < AMREX_SPACEDIM; k++) // { // dx[k] = cell_dx[k]; -// inv_dx[k] = 1.0/dx[k]; +// inv_dx[k] = 1.0/dx[k]; // } // } -// +// // CoordSys::CoordSys (const Real* cell_dx) // { // define(cell_dx); @@ -192,7 +192,7 @@ CoordSys::UpperIndex(const Real* point) const noexcept void CoordSys::GetVolume (FArrayBox& vol, - const Box& region) const + const Box& region) const { vol.resize(region,1); SetVolume(vol,region); @@ -200,7 +200,7 @@ CoordSys::GetVolume (FArrayBox& vol, void CoordSys::SetVolume (FArrayBox& a_volfab, - const Box& region) const + const Box& region) const { AMREX_ASSERT(ok); AMREX_ASSERT(region.cellCentered()); @@ -264,7 +264,7 @@ CoordSys::SetDLogA (FArrayBox& a_dlogafab, } void -CoordSys::GetFaceArea (FArrayBox& area, +CoordSys::GetFaceArea (FArrayBox& area, const Box& region, int dir) const { @@ -275,7 +275,7 @@ CoordSys::GetFaceArea (FArrayBox& area, } void -CoordSys::SetFaceArea (FArrayBox& a_areafab, +CoordSys::SetFaceArea (FArrayBox& a_areafab, const Box& region, int dir) const { @@ -302,7 +302,7 @@ CoordSys::SetFaceArea (FArrayBox& a_areafab, } void -CoordSys::GetEdgeLoc (Vector& loc, +CoordSys::GetEdgeLoc (Vector& loc, const Box& region, int dir) const { @@ -321,7 +321,7 @@ CoordSys::GetEdgeLoc (Vector& loc, } void -CoordSys::GetCellLoc (Vector& loc, +CoordSys::GetCellLoc (Vector& loc, const Box& region, int dir) const { @@ -373,7 +373,7 @@ CoordSys::GetEdgeVolCoord (Vector& vc, vc[i] = static_cast(FOURPI/3.)*r*r*r; } } -#endif +#endif } void @@ -409,7 +409,7 @@ CoordSys::GetCellVolCoord (Vector& vc, vc[i] = static_cast(FOURPI/3.)*r*r*r; } } -#endif +#endif } std::ostream& @@ -470,8 +470,8 @@ CoordSys::Volume (const IntVect& point) const return Volume(xlo,xhi); } -Real -CoordSys::Volume (const Real xlo[AMREX_SPACEDIM], +Real +CoordSys::Volume (const Real xlo[AMREX_SPACEDIM], const Real xhi[AMREX_SPACEDIM]) const { switch (c_sys) @@ -488,7 +488,7 @@ CoordSys::Volume (const Real xlo[AMREX_SPACEDIM], AMREX_ASSERT(0); } return 0; -} +} Real CoordSys::AreaLo (const IntVect& point, int dir) const noexcept diff --git a/Src/Base/AMReX_CudaGraph.H b/Src/Base/AMReX_CudaGraph.H index 1cb5ff00eb5..a03870b68b5 100644 --- a/Src/Base/AMReX_CudaGraph.H +++ b/Src/Base/AMReX_CudaGraph.H @@ -90,7 +90,7 @@ struct CudaGraph } m_parms_d = static_cast( The_Arena()->alloc(sizeof(T)*m_parms.size()) ); } - void setGraph(cudaGraphExec_t const& graph) { + void setGraph(cudaGraphExec_t const& graph) { m_graph = graph; graph_is_ready = true; } diff --git a/Src/Base/AMReX_CuptiTrace.H b/Src/Base/AMReX_CuptiTrace.H index beb13fb6069..78ff1f8e60d 100644 --- a/Src/Base/AMReX_CuptiTrace.H +++ b/Src/Base/AMReX_CuptiTrace.H @@ -31,7 +31,7 @@ private: unsigned long long timeElapsed_; int streamID_; std::string name_; - + public: void setUintID (unsigned) noexcept; void setCharID (std::string) noexcept; diff --git a/Src/Base/AMReX_CuptiTrace.cpp b/Src/Base/AMReX_CuptiTrace.cpp index 61bb9d99a08..c091bb63f6a 100644 --- a/Src/Base/AMReX_CuptiTrace.cpp +++ b/Src/Base/AMReX_CuptiTrace.cpp @@ -57,10 +57,10 @@ bfrRequestCallback (uint8_t* *bfr, size_t* size, size_t* maxNumRecords) noexcept void CUPTIAPI bfrCompleteCallback (CUcontext ctx, uint32_t streamId, uint8_t* bfr, size_t size, size_t validSize) noexcept -{ +{ CUptiResult status; CUpti_Activity* record = NULL; - + if (validSize > 0) { do { status = cuptiActivityGetNextRecord(bfr, validSize, &record); @@ -68,7 +68,7 @@ bfrCompleteCallback (CUcontext ctx, uint32_t streamId, uint8_t* bfr, std::unique_ptr recordUserData; recordUserData.reset(new CUpti_Activity_Userdata()); CUpti_ActivityKernel4* kernel = (CUpti_ActivityKernel4*) record; - + // Save record data recordUserData->setStartTime(kernel->start); recordUserData->setEndTime(kernel->end); @@ -112,12 +112,12 @@ initCuptiTrace () noexcept amrex::Print() << "CUPTI initialized\n"; } } - + void cuptiTraceStart () noexcept { cudaDeviceSynchronize(); - cuptiActivityFlushAll(0); + cuptiActivityFlushAll(0); activityRecordUserdata.clear(); } @@ -146,9 +146,9 @@ computeElapsedTimeUserdata (const std::vector streamIDToElapsedTimeMap; - + // Initialize tally of unique streams for (auto& record : activityRecordUserdata) { if (streamIDToElapsedTimeMap.find(record->getStreamID()) @@ -169,14 +169,14 @@ computeElapsedTimeUserdata (const std::vector&& pmap) noexcept; //! Build mapping out of BoxArray over nprocs processors. explicit DistributionMapping (const BoxArray& boxes, - int nprocs = ParallelDescriptor::NProcs()); + int nprocs = ParallelDescriptor::NProcs()); /** * \brief This is a very specialized distribution map. * Do NOT use it unless you really understand what it does. @@ -160,7 +160,7 @@ class DistributionMapping static bool SameRefs (const DistributionMapping& lhs, const DistributionMapping& rhs) - { return lhs.m_ref == rhs.m_ref; } + { return lhs.m_ref == rhs.m_ref; } static DistributionMapping makeKnapSack (const MultiFab& weight, int nmax=std::numeric_limits::max()); @@ -174,17 +174,17 @@ class DistributionMapping /** \brief Computes a new distribution mapping by distributing input costs * according to the `knapsack` algorithm. - * @param[in] rcost_local LayoutData of costs; contains, e.g., costs for the + * @param[in] rcost_local LayoutData of costs; contains, e.g., costs for the * local boxes in the FAB array, corresponding indices in the global * indices in the FAB array, and the distribution mapping * @param[in,out] currentEfficiency writes the efficiency (i.e., mean cost over * all MPI ranks, normalized to the max cost) given the current * distribution mapping - * @param[in,out] proposedEfficiency writes the efficiency for the proposed + * @param[in,out] proposedEfficiency writes the efficiency for the proposed * distribution mapping - * @param[in] nmax the maximum number of boxes that can be assigned to any + * @param[in] nmax the maximum number of boxes that can be assigned to any * MPI rank by the knapsack algorithm - * @param[in] broadcastToAll controls whether to transmit the proposed + * @param[in] broadcastToAll controls whether to transmit the proposed * distribution mapping to all other processes; setting this to * false allows to, e.g., test whether the proposed distribution * mapping is an improvement relative to the current distribution @@ -210,15 +210,15 @@ class DistributionMapping /** \brief Computes a new distribution mapping by distributing input costs * according to a `space filling curve` (SFC) algorithm. - * @param[in] rcost_local LayoutData of costs; contains, e.g., costs for the + * @param[in] rcost_local LayoutData of costs; contains, e.g., costs for the * local boxes in the FAB array, corresponding indices in the global * indices in the FAB array, and the distribution mapping * @param[in,out] currentEfficiency writes the efficiency (i.e., mean cost over * all MPI ranks, normalized to the max cost) given the current * distribution mapping - * @param[in,out] proposedEfficiency writes the efficiency for the proposed + * @param[in,out] proposedEfficiency writes the efficiency for the proposed * distribution mapping - * @param[in] broadcastToAll controls whether to transmit the proposed + * @param[in] broadcastToAll controls whether to transmit the proposed * distribution mapping to all other processes; setting this to * false allows to, e.g., test whether the proposed distribution * mapping is an improvement relative to the current distribution @@ -237,7 +237,7 @@ class DistributionMapping * if use_box_vol is true, weight boxes by their volume in Distribute * otherwise, all boxes will be treated with equal weight */ - static std::vector > makeSFC (const BoxArray& ba, + static std::vector > makeSFC (const BoxArray& ba, bool use_box_vol=true, const int nprocs=ParallelContext::NProcsSub() ); @@ -251,7 +251,7 @@ class DistributionMapping static void ComputeDistributionMappingEfficiency (const DistributionMapping& dm, const Vector& cost, Real* efficiency); - + private: const Vector& getIndexArray (); @@ -338,7 +338,7 @@ private: explicit Ref (Vector&& pmap) noexcept : m_pmap(std::move(pmap)) {} - //! dtor, copy-ctor, copy-op=, move-ctor, and move-op= are compiler generated. + //! dtor, copy-ctor, copy-op=, move-ctor, and move-op= are compiler generated. void clear () { m_pmap.clear(); m_index_array.clear(); m_ownership.clear(); } @@ -357,8 +357,8 @@ public: bool operator< (const RefID& rhs) const noexcept { return std::less()(data,rhs.data); } bool operator== (const RefID& rhs) const noexcept { return data == rhs.data; } bool operator!= (const RefID& rhs) const noexcept { return data != rhs.data; } - const Ref *dataPtr() const noexcept { return data; } - void PrintPtr(std::ostream &os) const { os << data << '\n'; } + const Ref *dataPtr() const noexcept { return data; } + void PrintPtr(std::ostream &os) const { os << data << '\n'; } friend std::ostream& operator<< (std::ostream& os, const RefID& id); private: Ref* data; diff --git a/Src/Base/AMReX_DistributionMapping.cpp b/Src/Base/AMReX_DistributionMapping.cpp index 3992d60c02a..b74b0e44188 100644 --- a/Src/Base/AMReX_DistributionMapping.cpp +++ b/Src/Base/AMReX_DistributionMapping.cpp @@ -177,12 +177,12 @@ DistributionMapping::Sort (std::vector& vec, { if (vec.size() > 1) { - if (reverse) { - std::stable_sort(vec.begin(), vec.end(), LIpairGT()); - } - else { - std::stable_sort(vec.begin(), vec.end(), LIpairLT()); - } + if (reverse) { + std::stable_sort(vec.begin(), vec.end(), LIpairGT()); + } + else { + std::stable_sort(vec.begin(), vec.end(), LIpairLT()); + } } } @@ -235,9 +235,9 @@ DistributionMapping::LeastUsedCPUs (int nprocs, void DistributionMapping::LeastUsedTeams (Vector & rteam, - Vector >& rworker, - int nteams, - int nworkers) + Vector >& rworker, + int nteams, + int nworkers) { #ifdef BL_USE_MPI BL_PROFILE("DistributionMapping::LeastUsedTeams()"); @@ -259,24 +259,24 @@ DistributionMapping::LeastUsedTeams (Vector & rteam, for (int i(0); i < nteams; ++i) { - rworker[i].resize(nworkers); + rworker[i].resize(nworkers); - Long teambytes = 0; - int offset = i*nworkers; - for (int j = 0; j < nworkers; ++j) - { - int globalrank = offset+j; - Long b = bytes[globalrank]; - teambytes += b; - LIworker[j] = LIpair(b,j); - } + Long teambytes = 0; + int offset = i*nworkers; + for (int j = 0; j < nworkers; ++j) + { + int globalrank = offset+j; + Long b = bytes[globalrank]; + teambytes += b; + LIworker[j] = LIpair(b,j); + } - Sort(LIworker, false); + Sort(LIworker, false); - for (int j = 0; j < nworkers; ++j) - { - rworker[i][j] = LIworker[j].second; - } + for (int j = 0; j < nworkers; ++j) + { + rworker[i][j] = LIworker[j].second; + } LIpairV.push_back(LIpair(teambytes,i)); } @@ -317,7 +317,7 @@ DistributionMapping::DistributionMapping (Vector&& pmap) noexcept } DistributionMapping::DistributionMapping (const BoxArray& boxes, - int nprocs) + int nprocs) : m_ref(std::make_shared(boxes.size())) { @@ -336,7 +336,7 @@ DistributionMapping::DistributionMapping (const DistributionMapping& d1, void DistributionMapping::define (const BoxArray& boxes, - int nprocs) + int nprocs) { m_ref->clear(); m_ref->m_pmap.resize(boxes.size()); @@ -392,11 +392,11 @@ DistributionMapping::RoundRobinDoIt (int nboxes, ord.resize(nprocs); std::iota(ord.begin(), ord.end(), 0); } - wrkerord.resize(nprocs); - for (int i = 0; i < nprocs; ++i) { - wrkerord[i].resize(1); - wrkerord[i][0] = 0; - } + wrkerord.resize(nprocs); + for (int i = 0; i < nprocs; ++i) { + wrkerord[i].resize(1); + wrkerord[i][0] = 0; + } } else { if (sort) { LeastUsedTeams(ord,wrkerord,nteams,nworkers); @@ -415,32 +415,32 @@ DistributionMapping::RoundRobinDoIt (int nboxes, if (LIpairV) { - BL_ASSERT(static_cast(LIpairV->size()) == nboxes); - - for (int i = 0; i < nboxes; ++i) - { - int tid = ord[i%nteams]; - int wid = (w[tid]++) % nworkers; - int rank = tid*nworkers + wrkerord[tid][wid]; - m_ref->m_pmap[(*LIpairV)[i].second] = ParallelContext::local_to_global_rank(rank); + BL_ASSERT(static_cast(LIpairV->size()) == nboxes); + + for (int i = 0; i < nboxes; ++i) + { + int tid = ord[i%nteams]; + int wid = (w[tid]++) % nworkers; + int rank = tid*nworkers + wrkerord[tid][wid]; + m_ref->m_pmap[(*LIpairV)[i].second] = ParallelContext::local_to_global_rank(rank); if (flag_verbose_mapper) { Print() << " Mapping box " << (*LIpairV)[i].second << " of size " << (*LIpairV)[i].first << " to rank " << rank << std::endl; } - } + } } else { - for (int i = 0; i < nboxes; ++i) - { - int tid = ord[i%nteams]; - int wid = (w[tid]++) % nworkers; - int rank = tid*nworkers + wrkerord[tid][wid]; - m_ref->m_pmap[i] = ParallelContext::local_to_global_rank(rank); + for (int i = 0; i < nboxes; ++i) + { + int tid = ord[i%nteams]; + int wid = (w[tid]++) % nworkers; + int rank = tid*nworkers + wrkerord[tid][wid]; + m_ref->m_pmap[i] = ParallelContext::local_to_global_rank(rank); if (flag_verbose_mapper) { Print() << " Mapping box " << i << " to rank " << rank << std::endl; } - } + } } } @@ -633,14 +633,14 @@ knapsack (const std::vector& wgts, while (!wblq.empty()) { - WeightedBoxList wbl = wblq.top(); + WeightedBoxList wbl = wblq.top(); wblq.pop(); - if (wbl.size() > 0) { - Real wgt = wbl.weight(); - sum_weight += wgt; - max_weight = std::max(wgt, max_weight); - wblv.push_back(wbl); - } + if (wbl.size() > 0) { + Real wgt = wbl.weight(); + sum_weight += wgt; + max_weight = std::max(wgt, max_weight); + wblv.push_back(wbl); + } } efficiency = sum_weight/(nprocs*max_weight); @@ -809,34 +809,34 @@ DistributionMapping::KnapSackDoIt (const std::vector& wgts, const int idx = LIpairV[i].second; const int tid = ord[i]; - const std::vector& vi = vec[idx]; - const int N = vi.size(); + const std::vector& vi = vec[idx]; + const int N = vi.size(); if (flag_verbose_mapper) { Print() << " Mapping bucket " << idx << " to rank " << tid << std::endl; } - if (nteams == nprocs) { - for (int j = 0; j < N; ++j) - { - m_ref->m_pmap[vi[j]] = ParallelContext::local_to_global_rank(tid); - } - } else { + if (nteams == nprocs) { + for (int j = 0; j < N; ++j) + { + m_ref->m_pmap[vi[j]] = ParallelContext::local_to_global_rank(tid); + } + } else { #ifdef BL_USE_TEAM - int leadrank = tid * nworkers; - for (int w = 0; w < nworkers; ++w) - { - ParallelDescriptor::team_for(0, N, w, [&] (int j) { + int leadrank = tid * nworkers; + for (int w = 0; w < nworkers; ++w) + { + ParallelDescriptor::team_for(0, N, w, [&] (int j) { m_ref->m_pmap[vi[j]] = leadrank + wrkerord[i][w]; - }); - } + }); + } #endif - } + } } if (verbose) { - amrex::Print() << "KNAPSACK efficiency: " << efficiency << '\n'; + amrex::Print() << "KNAPSACK efficiency: " << efficiency << '\n'; } } @@ -870,7 +870,7 @@ DistributionMapping::KnapSackProcessorMap (const std::vector& wgts, void DistributionMapping::KnapSackProcessorMap (const BoxArray& boxes, - int nprocs) + int nprocs) { BL_ASSERT(boxes.size() > 0); @@ -1157,12 +1157,12 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray& boxes, nworkers = ParallelDescriptor::TeamSize(); #else if (node_size > 0) { - nteams = nprocs/node_size; - nworkers = node_size; - if (nworkers*nteams != nprocs) { - nteams = nprocs; - nworkers = 1; - } + nteams = nprocs/node_size; + nworkers = node_size; + if (nworkers*nteams != nprocs) { + nteams = nprocs; + nworkers = 1; + } } #endif @@ -1258,60 +1258,60 @@ DistributionMapping::SFCProcessorMapDoIt (const BoxArray& boxes, const int tid = ord[i]; // tid is team id const int ivec = LIpairV[i].second; // index into vec const std::vector& vi = vec[ivec]; // this vector contains boxes assigned to this team - const int Nbx = vi.size(); // # of boxes assigned to this team + const int Nbx = vi.size(); // # of boxes assigned to this team if (flag_verbose_mapper) { Print() << "Mapping bucket " << LIpairV[i].second << " to rank " << ord[i] << std::endl; } - if (nteams == nprocs) { // In this case, team id is process id. - for (int j = 0; j < Nbx; ++j) - { - m_ref->m_pmap[vi[j]] = ParallelContext::local_to_global_rank(tid); - } - } - else // We would like to do knapsack within the team workers - { - std::vector local_wgts; - for (int j = 0; j < Nbx; ++j) { - local_wgts.push_back(wgts[vi[j]]); - } - - std::vector > kpres; - Real kpeff; - knapsack(local_wgts, nworkers, kpres, kpeff, true, N); - - // kpres has a size of nworkers. kpres[] contains a vector of indices into vi. - - // sort the knapsacked chunks - std::vector ww; - for (int w = 0; w < nworkers; ++w) { - Long wgt = 0; - for (std::vector::const_iterator it = kpres[w].begin(); - it != kpres[w].end(); ++it) - { - wgt += local_wgts[*it]; - } - ww.push_back(LIpair(wgt,w)); - } - Sort(ww,true); - - // ww is a sorted vector of pair whose first is the weight and second is a index - // into kpres. - - const Vector& sorted_workers = wrkerord[i]; - - const int leadrank = tid * nworkers; - - for (int w = 0; w < nworkers; ++w) - { - const int cpu = leadrank + sorted_workers[w]; - int ikp = ww[w].second; - const std::vector& js = kpres[ikp]; - for (std::vector::const_iterator it = js.begin(); it!=js.end(); ++it) - m_ref->m_pmap[vi[*it]] = cpu; - } - } + if (nteams == nprocs) { // In this case, team id is process id. + for (int j = 0; j < Nbx; ++j) + { + m_ref->m_pmap[vi[j]] = ParallelContext::local_to_global_rank(tid); + } + } + else // We would like to do knapsack within the team workers + { + std::vector local_wgts; + for (int j = 0; j < Nbx; ++j) { + local_wgts.push_back(wgts[vi[j]]); + } + + std::vector > kpres; + Real kpeff; + knapsack(local_wgts, nworkers, kpres, kpeff, true, N); + + // kpres has a size of nworkers. kpres[] contains a vector of indices into vi. + + // sort the knapsacked chunks + std::vector ww; + for (int w = 0; w < nworkers; ++w) { + Long wgt = 0; + for (std::vector::const_iterator it = kpres[w].begin(); + it != kpres[w].end(); ++it) + { + wgt += local_wgts[*it]; + } + ww.push_back(LIpair(wgt,w)); + } + Sort(ww,true); + + // ww is a sorted vector of pair whose first is the weight and second is a index + // into kpres. + + const Vector& sorted_workers = wrkerord[i]; + + const int leadrank = tid * nworkers; + + for (int w = 0; w < nworkers; ++w) + { + const int cpu = leadrank + sorted_workers[w]; + int ikp = ww[w].second; + const std::vector& js = kpres[ikp]; + for (std::vector::const_iterator it = js.begin(); it!=js.end(); ++it) + m_ref->m_pmap[vi[*it]] = cpu; + } + } } if (eff || verbose) @@ -1408,7 +1408,7 @@ DistributionMapping::SFCProcessorMap (const BoxArray& boxes, void DistributionMapping::RRSFCDoIt (const BoxArray& boxes, - int nprocs) + int nprocs) { BL_PROFILE("DistributionMapping::RRSFCDoIt()"); @@ -1435,7 +1435,7 @@ DistributionMapping::RRSFCDoIt (const BoxArray& boxes, // Distribute boxes using roundrobin for (int i = 0; i < nboxes; ++i) { - m_ref->m_pmap[i] = ParallelContext::local_to_global_rank(ord[i%nprocs]); + m_ref->m_pmap[i] = ParallelContext::local_to_global_rank(ord[i%nprocs]); } } @@ -1512,7 +1512,7 @@ DistributionMapping::makeKnapSack (const LayoutData& rcost_local, // 1. collect from rcost_local into the global cost vector rcost; then rcost is // complete (only) on root // 2. (optional; default true) Broadcast processor map of the new dm to others - + Vector rcost(rcost_local.size()); ParallelDescriptor::GatherLayoutDataToVector(rcost_local, rcost, root); // rcost is now filled out on root @@ -1549,7 +1549,7 @@ DistributionMapping::makeKnapSack (const LayoutData& rcost_local, { pmap = r.ProcessorMap(); } - + // Broadcast vector from which to construct new distribution mapping ParallelDescriptor::Bcast(&pmap[0], pmap.size(), root); if (ParallelDescriptor::MyProc() != root) @@ -1560,7 +1560,7 @@ DistributionMapping::makeKnapSack (const LayoutData& rcost_local, #else amrex::ignore_unused(broadcastToAll); #endif - + return r; } @@ -1570,7 +1570,7 @@ DistributionMapping::ComputeDistributionMappingEfficiency (const DistributionMap Real* efficiency) { const int nprocs = ParallelDescriptor::NProcs(); - + // This will store mapping from processor to the costs of FABs it controls, // (proc) --> ([cost_FAB_1, cost_FAB_2, ... ]), // for each proc @@ -1582,12 +1582,12 @@ DistributionMapping::ComputeDistributionMappingEfficiency (const DistributionMap { ++cnt[dm[i]]; } - + for (int i=0; i& rcost, const BoxArray& ba, boo DistributionMapping r; Vector cost(rcost.size()); - + Real wmax = *std::max_element(rcost.begin(), rcost.end()); Real scale = (wmax == 0) ? 1.e9_rt : 1.e9_rt/wmax; @@ -1727,7 +1727,7 @@ DistributionMapping::makeSFC (const Vector& rcost, const BoxArray& ba, Rea DistributionMapping r; Vector cost(rcost.size()); - + Real wmax = *std::max_element(rcost.begin(), rcost.end()); Real scale = (wmax == 0) ? 1.e9_rt : 1.e9_rt/wmax; @@ -1756,7 +1756,7 @@ DistributionMapping::makeSFC (const LayoutData& rcost_local, // 1. collect from rcost_local into the global cost vector rcost; then rcost is // complete (only) on root // 2. (optional; default true) Broadcast processor map of the new dm to others - + Vector rcost(rcost_local.size()); ParallelDescriptor::GatherLayoutDataToVector(rcost_local, rcost, root); // rcost is now filled out on root; @@ -1807,7 +1807,7 @@ DistributionMapping::makeSFC (const LayoutData& rcost_local, return r; } - + std::vector > DistributionMapping::makeSFC (const BoxArray& ba, bool use_box_vol, const int nprocs) { diff --git a/Src/Base/AMReX_Extension.H b/Src/Base/AMReX_Extension.H index 96253c1437b..b8373b1aa8d 100644 --- a/Src/Base/AMReX_Extension.H +++ b/Src/Base/AMReX_Extension.H @@ -41,13 +41,13 @@ // simd #if defined(AMREX_DEBUG) -#define AMREX_PRAGMA_SIMD +#define AMREX_PRAGMA_SIMD #elif defined(__CUDA_ARCH__) -#define AMREX_PRAGMA_SIMD +#define AMREX_PRAGMA_SIMD #elif defined(__HIP_DEVICE_COMPILE__) -#define AMREX_PRAGMA_SIMD +#define AMREX_PRAGMA_SIMD //#elif defined(AMREX_USE_OMP) && defined(_OPENMP) && (_OPENMP >= 201307) && !defined(__PGI) //#define AMREX_PRAGMA_SIMD _Pragma("omp simd") @@ -62,7 +62,7 @@ #define AMREX_PRAGMA_SIMD _Pragma("loop ivdep") #elif defined(__NEC__) -#define AMREX_PRAGMA_SIMD +#define AMREX_PRAGMA_SIMD #elif defined(__ibmxl__) #define AMREX_PRAGMA_SIMD _Pragma("ibm independent_loop") @@ -74,7 +74,7 @@ #define AMREX_PRAGMA_SIMD _Pragma("GCC ivdep") #else -#define AMREX_PRAGMA_SIMD +#define AMREX_PRAGMA_SIMD #endif /* simd */ @@ -130,7 +130,7 @@ // __attribute__((weak)) #if defined(AMREX_TYPECHECK) -#define AMREX_ATTRIBUTE_WEAK +#define AMREX_ATTRIBUTE_WEAK #elif defined(_WIN32) #define AMREX_ATTRIBUTE_WEAK #else diff --git a/Src/Base/AMReX_FACopyDescriptor.H b/Src/Base/AMReX_FACopyDescriptor.H index 6bfbb05959b..74d8966eee9 100644 --- a/Src/Base/AMReX_FACopyDescriptor.H +++ b/Src/Base/AMReX_FACopyDescriptor.H @@ -270,12 +270,12 @@ FabArrayCopyDescriptor::AddBoxDoIt (FabArrayId fabarrayid, FabCopyDescriptor* fcd = new FabCopyDescriptor; int remoteProc = fabArray->DistributionMap()[faindex]; - if(remoteProc >= ParallelDescriptor::NProcs()) { + if(remoteProc >= ParallelDescriptor::NProcs()) { std::ostringstream ss; ss << ParallelDescriptor::MyProc() << ":: _in AddBoxDoIt: nProcs remoteProc = " << ParallelDescriptor::NProcs() << " " << remoteProc << "\n"; - amrex::Abort("Bad remoteProc: "+ss.str()); - } + amrex::Abort("Bad remoteProc: "+ss.str()); + } fcd->fillBoxId = nextFillBoxId; fcd->subBox = intersect; fcd->myProc = myProc; @@ -520,14 +520,14 @@ FabArrayCopyDescriptor::CollectData () Vector SndsArray(NProcs,0), RcvsArray(NProcs,0); for (std::map::const_iterator it = Rcvs.begin(), End = Rcvs.end(); it != End; ++it) - { + { RcvsArray[it->first] = it->second; - } + } { BL_PROFILE_VAR("CollectData_Alltoall()", blpvCDATA); - BL_COMM_PROFILE(BLProfiler::Alltoall, sizeof(int), ParallelDescriptor::MyProc(), - BLProfiler::BeforeCall()); + BL_COMM_PROFILE(BLProfiler::Alltoall, sizeof(int), ParallelDescriptor::MyProc(), + BLProfiler::BeforeCall()); BL_MPI_REQUIRE( MPI_Alltoall(RcvsArray.dataPtr(), 1, @@ -537,8 +537,8 @@ FabArrayCopyDescriptor::CollectData () ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()) ); - BL_COMM_PROFILE(BLProfiler::Alltoall, sizeof(int), ParallelDescriptor::MyProc(), - BLProfiler::AfterCall()); + BL_COMM_PROFILE(BLProfiler::Alltoall, sizeof(int), ParallelDescriptor::MyProc(), + BLProfiler::AfterCall()); BL_PROFILE_VAR_STOP(blpvCDATA); } @@ -576,214 +576,214 @@ FabArrayCopyDescriptor::CollectData () if (N_snds > 0) { - // Recv meta-data + // Recv meta-data - int N = 0; + int N = 0; for (std::map::const_iterator it = Snds.begin(), End = Snds.end(); it != End; ++it) { - md_sender.push_back(it->first); - md_bcnts.push_back(it->second); - int cnt = it->second * Nints; - md_icnts.push_back(cnt); + md_sender.push_back(it->first); + md_bcnts.push_back(it->second); + int cnt = it->second * Nints; + md_icnts.push_back(cnt); md_offset.push_back(N); N += cnt; } - md_recv_data = static_cast(amrex::The_Arena()->alloc(N*sizeof(int))); + md_recv_data = static_cast(amrex::The_Arena()->alloc(N*sizeof(int))); - for (int i = 0; i < N_snds; ++i) - { - md_recv_reqs.push_back(ParallelDescriptor::Arecv(&md_recv_data[md_offset[i]], - md_icnts[i], md_sender[i], - SeqNum_md).req()); - } + for (int i = 0; i < N_snds; ++i) + { + md_recv_reqs.push_back(ParallelDescriptor::Arecv(&md_recv_data[md_offset[i]], + md_icnts[i], md_sender[i], + SeqNum_md).req()); + } } if (N_rcvs > 0) { - // Send meta-data + // Send meta-data for (std::map::const_iterator it = Rcvs.begin(), End = Rcvs.end(); it != End; ++it) { - int rank = it->first; - int Nmds = it->second; - int cnt = Nmds * Nints; - - int* p = static_cast(amrex::The_Arena()->alloc(cnt*sizeof(int))); - md_send_data.push_back(p); - - const FabComTagIterContainer& tags = RcvTags[rank]; - - // initialized the data - int * md = p; - for (int i = 0; i < Nmds; ++i, md += Nints) - { - md[0] = tags[i]->fabArrayId; - md[1] = tags[i]->fabIndex; - md[2] = tags[i]->srcComp; - md[3] = tags[i]->nComp; - const int* lo = tags[i]->box.loVect(); - const int* hi = tags[i]->box.hiVect(); - const IntVect& bxtyp = tags[i]->box.type(); - const int* tp = bxtyp.getVect(); - AMREX_D_EXPR(md[4] = lo[0], - md[5] = lo[1], - md[6] = lo[2]); - AMREX_D_EXPR(md[4+ AMREX_SPACEDIM] = hi[0], - md[5+ AMREX_SPACEDIM] = hi[1], - md[6+ AMREX_SPACEDIM] = hi[2]); - AMREX_D_EXPR(md[4+2*AMREX_SPACEDIM] = tp[0], - md[5+2*AMREX_SPACEDIM] = tp[1], - md[6+2*AMREX_SPACEDIM] = tp[2]); - } - - md_send_reqs.push_back(ParallelDescriptor::Asend(p,cnt,rank,SeqNum_md).req()); - } + int rank = it->first; + int Nmds = it->second; + int cnt = Nmds * Nints; + + int* p = static_cast(amrex::The_Arena()->alloc(cnt*sizeof(int))); + md_send_data.push_back(p); + + const FabComTagIterContainer& tags = RcvTags[rank]; + + // initialized the data + int * md = p; + for (int i = 0; i < Nmds; ++i, md += Nints) + { + md[0] = tags[i]->fabArrayId; + md[1] = tags[i]->fabIndex; + md[2] = tags[i]->srcComp; + md[3] = tags[i]->nComp; + const int* lo = tags[i]->box.loVect(); + const int* hi = tags[i]->box.hiVect(); + const IntVect& bxtyp = tags[i]->box.type(); + const int* tp = bxtyp.getVect(); + AMREX_D_EXPR(md[4] = lo[0], + md[5] = lo[1], + md[6] = lo[2]); + AMREX_D_EXPR(md[4+ AMREX_SPACEDIM] = hi[0], + md[5+ AMREX_SPACEDIM] = hi[1], + md[6+ AMREX_SPACEDIM] = hi[2]); + AMREX_D_EXPR(md[4+2*AMREX_SPACEDIM] = tp[0], + md[5+2*AMREX_SPACEDIM] = tp[1], + md[6+2*AMREX_SPACEDIM] = tp[2]); + } + + md_send_reqs.push_back(ParallelDescriptor::Asend(p,cnt,rank,SeqNum_md).req()); + } } if (N_rcvs > 0) { - recv_data = static_cast(amrex::The_Arena()->alloc(Total_Rcvs_Size*sizeof(value_type))); - - // Post receives for data - int Idx = 0; - for (std::map::const_iterator it = Npts.begin(); it != Npts.end(); ++it) - { - int Who = it->first; - int Cnt = it->second; - BL_ASSERT(Cnt > 0); - BL_ASSERT(Cnt < std::numeric_limits::max()); - data_sender.push_back(Who); - data_recv_reqs.push_back(ParallelDescriptor::Arecv(&recv_data[Idx], - Cnt,Who,SeqNum_data).req()); - data_offset.push_back(Idx); - Idx += Cnt; - } + recv_data = static_cast(amrex::The_Arena()->alloc(Total_Rcvs_Size*sizeof(value_type))); + + // Post receives for data + int Idx = 0; + for (std::map::const_iterator it = Npts.begin(); it != Npts.end(); ++it) + { + int Who = it->first; + int Cnt = it->second; + BL_ASSERT(Cnt > 0); + BL_ASSERT(Cnt < std::numeric_limits::max()); + data_sender.push_back(Who); + data_recv_reqs.push_back(ParallelDescriptor::Arecv(&recv_data[Idx], + Cnt,Who,SeqNum_data).req()); + data_offset.push_back(Idx); + Idx += Cnt; + } } // Wait on meta-data and do send if (N_snds > 0) { - int send_counter = 0; - while (send_counter++ < N_snds) - { - MPI_Status status; - int index; + int send_counter = 0; + while (send_counter++ < N_snds) + { + MPI_Status status; + int index; ParallelDescriptor::Waitany(md_recv_reqs, index, status); - int rank = status.MPI_SOURCE; - BL_ASSERT(status.MPI_TAG == SeqNum_md); - BL_ASSERT(rank == md_sender[index]); - - const int* p = &md_recv_data[md_offset[index]]; - int numboxes = md_bcnts[index]; - Vector faid(numboxes); - Vector fidx(numboxes); - Vector scomp(numboxes); - Vector ncomp(numboxes); - Vector npts(numboxes); - Vector bxs; - int N = 0; - const int * md = p; - for (int i = 0; i < numboxes; ++i, md += Nints) - { - faid[i] = md[0]; - fidx[i] = md[1]; - scomp[i] = md[2]; - ncomp[i] = md[3]; - bxs.push_back(Box(IntVect(&md[4]), - IntVect(&md[4+AMREX_SPACEDIM]), - IntVect(&md[4+AMREX_SPACEDIM*2]))); - npts[i] = bxs.back().numPts()*ncomp[i]; - N += npts[i]; - } - - BL_ASSERT(N < std::numeric_limits::max()); - - value_type* data = static_cast(amrex::The_Arena()->alloc(N*sizeof(value_type))); - value_type* dptr = data; - send_data.push_back(data); - - for (int i = 0; i < numboxes; ++i) - { - (*fabArrays[faid[i]])[fidx[i]].template copyToMem(bxs[i],scomp[i],ncomp[i],dptr); - dptr += npts[i]; - } - - data_send_reqs.push_back(ParallelDescriptor::Asend(data,N,rank,SeqNum_data).req()); - } - - amrex::The_Arena()->free(md_recv_data); + int rank = status.MPI_SOURCE; + BL_ASSERT(status.MPI_TAG == SeqNum_md); + BL_ASSERT(rank == md_sender[index]); + + const int* p = &md_recv_data[md_offset[index]]; + int numboxes = md_bcnts[index]; + Vector faid(numboxes); + Vector fidx(numboxes); + Vector scomp(numboxes); + Vector ncomp(numboxes); + Vector npts(numboxes); + Vector bxs; + int N = 0; + const int * md = p; + for (int i = 0; i < numboxes; ++i, md += Nints) + { + faid[i] = md[0]; + fidx[i] = md[1]; + scomp[i] = md[2]; + ncomp[i] = md[3]; + bxs.push_back(Box(IntVect(&md[4]), + IntVect(&md[4+AMREX_SPACEDIM]), + IntVect(&md[4+AMREX_SPACEDIM*2]))); + npts[i] = bxs.back().numPts()*ncomp[i]; + N += npts[i]; + } + + BL_ASSERT(N < std::numeric_limits::max()); + + value_type* data = static_cast(amrex::The_Arena()->alloc(N*sizeof(value_type))); + value_type* dptr = data; + send_data.push_back(data); + + for (int i = 0; i < numboxes; ++i) + { + (*fabArrays[faid[i]])[fidx[i]].template copyToMem(bxs[i],scomp[i],ncomp[i],dptr); + dptr += npts[i]; + } + + data_send_reqs.push_back(ParallelDescriptor::Asend(data,N,rank,SeqNum_data).req()); + } + + amrex::The_Arena()->free(md_recv_data); } // Wait and upack data if (N_rcvs > 0) { - Vector stats(N_rcvs); + Vector stats(N_rcvs); ParallelDescriptor::Waitall(md_send_reqs, stats); - for (int i = 0; i < N_rcvs; ++i) { + for (int i = 0; i < N_rcvs; ++i) { amrex::The_Arena()->free(md_send_data[i]); - } + } ParallelDescriptor::Waitall(data_recv_reqs, stats); - std::pair match; - std::map< int,FabComTagIterContainer >::const_iterator found; + std::pair match; + std::map< int,FabComTagIterContainer >::const_iterator found; - for (int k = 0; k < N_rcvs; k++) - { - const int Who = data_sender[k]; - const value_type* dptr = &recv_data[data_offset[k]]; + for (int k = 0; k < N_rcvs; k++) + { + const int Who = data_sender[k]; + const value_type* dptr = &recv_data[data_offset[k]]; - BL_ASSERT(dptr != 0); + BL_ASSERT(dptr != 0); - found = RcvTags.find(Who); + found = RcvTags.find(Who); - BL_ASSERT(found != RcvTags.end()); + BL_ASSERT(found != RcvTags.end()); - const FabComTagIterContainer& tags = found->second; + const FabComTagIterContainer& tags = found->second; - for (FabComTagIterContainer::const_iterator it = tags.begin(), End = tags.end(); - it != End; - ++it) - { - const FabArrayBase::FabComTag& tag = **it; + for (FabComTagIterContainer::const_iterator it = tags.begin(), End = tags.end(); + it != End; + ++it) + { + const FabArrayBase::FabComTag& tag = **it; - BL_ASSERT(tag.procThatHasData == Who); + BL_ASSERT(tag.procThatHasData == Who); - match = fabCopyDescList[tag.fabArrayId].equal_range(tag.fillBoxId); + match = fabCopyDescList[tag.fabArrayId].equal_range(tag.fillBoxId); - for (FCDMapIter fmi = match.first; fmi != match.second; ++fmi) - { - FabCopyDescriptor* fcdp = (*fmi).second; + for (FCDMapIter fmi = match.first; fmi != match.second; ++fmi) + { + FabCopyDescriptor* fcdp = (*fmi).second; - BL_ASSERT(fcdp->fillBoxId == tag.fillBoxId); + BL_ASSERT(fcdp->fillBoxId == tag.fillBoxId); - if (fcdp->subBox == tag.box) - { - BL_ASSERT(fcdp->localFabSource->dataPtr() != 0); - BL_ASSERT(fcdp->localFabSource->box() == tag.box); - const int Cnt = tag.box.numPts()*tag.nComp; - fcdp->localFabSource->template copyFromMem(tag.box,0,tag.nComp,dptr); - dptr += Cnt; - break; - } - } - } - } + if (fcdp->subBox == tag.box) + { + BL_ASSERT(fcdp->localFabSource->dataPtr() != 0); + BL_ASSERT(fcdp->localFabSource->box() == tag.box); + const int Cnt = tag.box.numPts()*tag.nComp; + fcdp->localFabSource->template copyFromMem(tag.box,0,tag.nComp,dptr); + dptr += Cnt; + break; + } + } + } + } - amrex::The_Arena()->free(recv_data); + amrex::The_Arena()->free(recv_data); } // Finished send if (N_snds > 0) { - Vector stats(N_snds); + Vector stats(N_snds); ParallelDescriptor::Waitall(data_send_reqs, stats); - for (int i = 0; i < N_snds; ++i) { + for (int i = 0; i < N_snds; ++i) { amrex::The_Arena()->free(send_data[i]); - } + } } #endif /*BL_USE_MPI*/ diff --git a/Src/Base/AMReX_FArrayBox.H b/Src/Base/AMReX_FArrayBox.H index d9547dac1af..ed6badd8af8 100644 --- a/Src/Base/AMReX_FArrayBox.H +++ b/Src/Base/AMReX_FArrayBox.H @@ -127,7 +127,7 @@ public: virtual void skip (std::istream& is, FArrayBox& f, - int nCompToSkip) const = 0; + int nCompToSkip) const = 0; /** * \brief Write out a header describing FArrayBox f that contains * nvar components. It must be the case that nvar <= f.nComp(). @@ -156,8 +156,8 @@ public: */ static FABio* read_header (std::istream& is, FArrayBox& f, - int compIndex, - int& nCompAvailable); + int compIndex, + int& nCompAvailable); }; // @@ -183,7 +183,7 @@ public: virtual void skip (std::istream& is, FArrayBox& f, - int nCompToSkip) const override; + int nCompToSkip) const override; private: virtual void write_header (std::ostream& os, @@ -250,8 +250,8 @@ public: */ explicit FArrayBox (const Box& b, int ncomp=1, - bool alloc=true, - bool shared=false, + bool alloc=true, + bool shared=false, Arena* ar = nullptr); FArrayBox (const FArrayBox& rhs, MakeType make_type, int scomp, int ncomp); diff --git a/Src/Base/AMReX_FArrayBox.cpp b/Src/Base/AMReX_FArrayBox.cpp index 13cbeba243c..64ef6eb6345 100644 --- a/Src/Base/AMReX_FArrayBox.cpp +++ b/Src/Base/AMReX_FArrayBox.cpp @@ -60,7 +60,7 @@ class FABio_8bit virtual void skip (std::istream& is, FArrayBox& f, - int nCompToSkip) const override; + int nCompToSkip) const override; private: virtual void write_header (std::ostream& os, const FArrayBox& f, @@ -88,7 +88,7 @@ class FABio_ascii virtual void skip (std::istream& is, FArrayBox& f, - int nCompToSkip) const override; + int nCompToSkip) const override; private: virtual void write_header (std::ostream& os, const FArrayBox& f, @@ -418,8 +418,8 @@ FArrayBox::Initialize () } initval = std::numeric_limits::has_quiet_NaN - ? std::numeric_limits::quiet_NaN() - : std::numeric_limits::max(); + ? std::numeric_limits::quiet_NaN() + : std::numeric_limits::max(); pp.query("initval", initval); pp.query("do_initval", do_initval); @@ -523,8 +523,8 @@ FABio::read_header (std::istream& is, FABio* FABio::read_header (std::istream& is, FArrayBox& f, - int /*compIndex*/, - int& nCompAvailable) + int /*compIndex*/, + int& nCompAvailable) { // BL_PROFILE("FArrayBox::read_header_is_i"); int nvar; @@ -550,8 +550,8 @@ FABio::read_header (std::istream& is, is >> machine; is >> bx; is >> nvar; - nCompAvailable = nvar; - nvar = 1; // make a single component fab + nCompAvailable = nvar; + nvar = 1; // make a single component fab // // Set the FArrayBox to the appropriate size. // @@ -581,8 +581,8 @@ FABio::read_header (std::istream& is, is >> *rd; is >> bx; is >> nvar; - nCompAvailable = nvar; - nvar = 1; // make a single component fab + nCompAvailable = nvar; + nvar = 1; // make a single component fab // // Set the FArrayBox to the appropriate size. // @@ -667,7 +667,7 @@ FABio_ascii::write (std::ostream& os, os << p; for(int k(0); k < num_comp; ++k) { os << " " << f(p,k+comp); - } + } os << '\n'; } os << '\n'; @@ -698,7 +698,7 @@ FABio_ascii::read (std::istream& is, } for(int k(0); k < f.nComp(); ++k) { is >> f(p, k); - } + } } if(is.fail()) { @@ -716,7 +716,7 @@ FABio_ascii::skip (std::istream& is, void FABio_ascii::skip (std::istream& /*is*/, FArrayBox& /*f*/, - int /*nCompToSkip*/) const + int /*nCompToSkip*/) const { amrex::Error("FABio_ascii::skip(..., int nCompToSkip) not implemented"); } @@ -784,7 +784,7 @@ FABio_8bit::read (std::istream& is, BL_ASSERT(nbytes == siz); while (is.get() != '\n') { ; // ---- do nothing - } + } is.read((char*)c,siz); Real* dat = f.dataPtr(k); const Real rng = (mx-mn)/255.0_rt; @@ -814,19 +814,19 @@ FABio_8bit::skip (std::istream& is, BL_ASSERT(nbytes == siz); while(is.get() != '\n') { ; // ---- do nothing - } + } is.seekg(siz, std::ios::cur); } if(is.fail()) { amrex::Error("FABio_8bit::skip() failed"); - } + } } void FABio_8bit::skip (std::istream& is, FArrayBox& f, - int nCompToSkip) const + int nCompToSkip) const { const Box& bx = f.box(); Long siz = bx.numPts(); @@ -836,7 +836,7 @@ FABio_8bit::skip (std::istream& is, BL_ASSERT(nbytes == siz); while(is.get() != '\n') { ; // ---- do nothing - } + } is.seekg(siz, std::ios::cur); } @@ -920,7 +920,7 @@ FABio_binary::skip (std::istream& is, void FABio_binary::skip (std::istream& is, FArrayBox& f, - int nCompToSkip) const + int nCompToSkip) const { const Box& bx = f.box(); Long base_siz = bx.numPts(); diff --git a/Src/Base/AMReX_FBI.H b/Src/Base/AMReX_FBI.H index 58ec9d42b99..bf5fc1a166d 100644 --- a/Src/Base/AMReX_FBI.H +++ b/Src/Base/AMReX_FBI.H @@ -293,7 +293,7 @@ fab_to_fab (Vector > const& copy_tags, int scomp, int dcomp, in #ifdef AMREX_USE_CUDA if (__all_sync(0xffffffff, msk == 0)) { // 0 means lock acquired #elif defined(AMREX_USE_HIP) - if (__all(msk == 0)) { + if (__all(msk == 0)) { #endif break; // all threads have acquired. } else { @@ -566,7 +566,7 @@ FabArray::FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp if ( !(TheFB.m_localCopy.ready()) ) { const_cast(TheFB).m_localCopy.resize(N_locs); - + int idx = 0; // Record the graph. for (MFIter mfi(*this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) @@ -575,7 +575,7 @@ FabArray::FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp const_cast(TheFB).m_localCopy.getHostPtr(0), (TheFB).m_localCopy.getDevicePtr(0), std::size_t(sizeof(CopyMemory)*N_locs) ); - + const auto& tags = loc_copy_tags[mfi]; for (auto const & tag : tags) { @@ -591,7 +591,7 @@ FabArray::FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp } }); } - + bool last_iter = mfi.LocalIndex() == (this->local_size()-1); cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter); if (last_iter) { const_cast(TheFB).m_localCopy.setGraph( graphExec ); } @@ -600,7 +600,7 @@ FabArray::FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp // Setup Launch Parameters // This is perfectly threadable, right? - // Additional optimization -> Check to see whether values need to be reset? + // Additional optimization -> Check to see whether values need to be reset? // Can then remove this setup and memcpy from CudaGraph::executeGraph. int idx = 0; for (MFIter mfi(*this); mfi.isValid(); ++mfi) @@ -614,7 +614,7 @@ FabArray::FB_local_copy_cuda_graph_1 (const FB& TheFB, int scomp, int ncomp scomp, ncomp)); } } - + // Launch Graph TheFB.m_localCopy.executeGraph(); } @@ -634,7 +634,7 @@ FabArray::FB_local_copy_cuda_graph_n (const FB& TheFB, int scomp, int ncomp BL_ASSERT(ParallelDescriptor::sameTeam(distributionMap[tag.dstIndex])); BL_ASSERT(ParallelDescriptor::sameTeam(distributionMap[tag.srcIndex])); - + if (distributionMap[tag.dstIndex] == ParallelDescriptor::MyProc()) { loc_copy_tags[tag.dstIndex].push_back @@ -642,13 +642,13 @@ FabArray::FB_local_copy_cuda_graph_n (const FB& TheFB, int scomp, int ncomp launches++; } } - + FillBoundary_test(); if ( !(TheFB.m_localCopy.ready()) ) { const_cast(TheFB).m_localCopy.resize(launches); - + int idx = 0; int cuda_stream = 0; for (MFIter mfi(*this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) @@ -664,7 +664,7 @@ FabArray::FB_local_copy_cuda_graph_n (const FB& TheFB, int scomp, int ncomp const auto& tag = tags[t]; const Dim3 offset = tag.offset.dim3(); - + CopyMemory* cmem = TheFB.m_localCopy.getDevicePtr(idx++); AMREX_HOST_DEVICE_FOR_3D(tag.dbox, i, j, k, { @@ -681,22 +681,22 @@ FabArray::FB_local_copy_cuda_graph_n (const FB& TheFB, int scomp, int ncomp } } } - + // Setup Launch Parameters // This is perfectly threadable, right? int idx = 0; for (MFIter mfi(*this); mfi.isValid(); ++mfi) { const auto& dst_array = this->array(mfi); - const auto& tags = loc_copy_tags[mfi]; + const auto& tags = loc_copy_tags[mfi]; for (auto const & tag : tags) { const_cast(TheFB).m_localCopy.setParams(idx++, makeCopyMemory(tag.sfab->array(), dst_array, - scomp, ncomp)); + scomp, ncomp)); } } - + // Launch Graph without synch. Local work is entirely independent. TheFB.m_localCopy.executeGraph(false); } @@ -733,7 +733,7 @@ FabArray::FB_pack_send_buffer_cuda_graph (const FB& TheFB, int scomp, int n } } const_cast(TheFB).m_copyToBuffer.resize(launches); - + // Record the graph. int idx = 0; for (Gpu::StreamIter sit(N_snds,Gpu::StreamItInfo().DisableDeviceSync()); @@ -786,14 +786,14 @@ FabArray::FB_pack_send_buffer_cuda_graph (const FB& TheFB, int scomp, int n tag.sbox, ncomp), scomp, ncomp)); - + dptr += (tag.sbox.numPts() * ncomp * sizeof(value_type)); } amrex::ignore_unused(send_size); BL_ASSERT(dptr <= send_data[j] + send_size[j]); } } - + // Launch Graph synched, so copyToBuffer is complete prior to posting sends. TheFB.m_copyToBuffer.executeGraph(); } @@ -827,11 +827,11 @@ FabArray::FB_unpack_recv_buffer_cuda_graph (const FB& TheFB, int dcomp, int BL_ASSERT(dptr <= recv_data[k] + recv_size[k]); } } - + if ( !(TheFB.m_copyFromBuffer.ready()) ) { const_cast(TheFB).m_copyFromBuffer.resize(launches); - + int idx = 0; for (MFIter mfi(*this, MFItInfo().DisableDeviceSync()); mfi.isValid(); ++mfi) { @@ -839,7 +839,7 @@ FabArray::FB_unpack_recv_buffer_cuda_graph (const FB& TheFB, int dcomp, int const_cast(TheFB).m_copyFromBuffer.getHostPtr(0), (TheFB).m_copyFromBuffer.getDevicePtr(0), std::size_t(sizeof(CopyMemory)*launches) ); - + const auto& tags = recv_copy_tags[mfi]; for (auto const & tag : tags) { @@ -854,14 +854,14 @@ FabArray::FB_unpack_recv_buffer_cuda_graph (const FB& TheFB, int dcomp, int } }); } - + bool last_iter = mfi.LocalIndex() == (this->local_size()-1); cudaGraphExec_t graphExec = amrex::Gpu::Device::stopGraphRecording(last_iter); if (last_iter) { const_cast(TheFB).m_copyFromBuffer.setGraph( graphExec ); } } } - // Setup graph. + // Setup graph. int idx = 0; for (MFIter mfi(*this); mfi.isValid(); ++mfi) { @@ -876,7 +876,7 @@ FabArray::FB_unpack_recv_buffer_cuda_graph (const FB& TheFB, int dcomp, int dcomp, ncomp)); } } - + // Launch Graph - synced because next action is freeing recv buffer. TheFB.m_copyFromBuffer.executeGraph(); } diff --git a/Src/Base/AMReX_FILCC_1D.F90 b/Src/Base/AMReX_FILCC_1D.F90 index d2993f9cb7b..45b7f6c1da1 100644 --- a/Src/Base/AMReX_FILCC_1D.F90 +++ b/Src/Base/AMReX_FILCC_1D.F90 @@ -10,8 +10,8 @@ !! \param domlo,domhi => index extent of problem domain !! \param dx => cell spacing !! \param xlo => physical location of lower left hand -!! corner of q array -!! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) +!! corner of q array +!! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) !! !! NOTE: all corner as well as edge data is filled if not EXT_DIR ! ----------------------------------------------------------- diff --git a/Src/Base/AMReX_FILCC_2D.F90 b/Src/Base/AMReX_FILCC_2D.F90 index f94edf92106..0cf06563328 100644 --- a/Src/Base/AMReX_FILCC_2D.F90 +++ b/Src/Base/AMReX_FILCC_2D.F90 @@ -12,8 +12,8 @@ !! \param domlo,domhi => index extent of problem domain !! \param dx => cell spacing !! \param xlo => physical location of lower left hand -!! corner of q array -!! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) +!! corner of q array +!! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) !! !! NOTE: all corner as well as edge data is filled if not EXT_DIR ! ----------------------------------------------------------- diff --git a/Src/Base/AMReX_FILCC_3D.F90 b/Src/Base/AMReX_FILCC_3D.F90 index 130fee9e55a..4e7f3256451 100644 --- a/Src/Base/AMReX_FILCC_3D.F90 +++ b/Src/Base/AMReX_FILCC_3D.F90 @@ -12,8 +12,8 @@ !! \param domlo,hi => index extent of problem domain !! \param dx => cell spacing !! \param xlo => physical location of lower left hand -!! corner of q array -!! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) +!! corner of q array +!! \param bc => array of boundary flags bc(SPACEDIM,lo:hi) !! !! NOTE: corner data not used in computing soln but must have !! reasonable values for arithmetic to live diff --git a/Src/Base/AMReX_FPC.cpp b/Src/Base/AMReX_FPC.cpp index 97b6ee4637f..f7131a45927 100644 --- a/Src/Base/AMReX_FPC.cpp +++ b/Src/Base/AMReX_FPC.cpp @@ -144,7 +144,7 @@ FPC::Native64RealDescriptor () return n64rd; } - + const RealDescriptor& FPC::Ieee32NormalRealDescriptor () diff --git a/Src/Base/AMReX_FabArray.H b/Src/Base/AMReX_FabArray.H index e91c89df059..53552446714 100644 --- a/Src/Base/AMReX_FabArray.H +++ b/Src/Base/AMReX_FabArray.H @@ -176,10 +176,10 @@ public: int nvar, int ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory); #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = DefaultFabFactory()); #endif @@ -188,10 +188,10 @@ public: int nvar, const IntVect& ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory); #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = DefaultFabFactory()); #endif @@ -213,26 +213,26 @@ public: * the default constructor. */ virtual void define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - int ngrow, + const DistributionMapping& dm, + int nvar, + int ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory); #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = DefaultFabFactory()); #endif virtual void define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - const IntVect& ngrow, + const DistributionMapping& dm, + int nvar, + const IntVect& ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory); #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = DefaultFabFactory()); #endif @@ -478,7 +478,7 @@ public: CpOp op = FabArrayBase::COPY) { ParallelCopy(fa,0,0,nComp(),0,0,period,op); } void copy (const FabArray& fa, - const Periodicity& period = Periodicity::NonPeriodic(), + const Periodicity& period = Periodicity::NonPeriodic(), CpOp op = FabArrayBase::COPY) { ParallelCopy(fa,period,op); } @@ -515,7 +515,7 @@ public: int src_comp, int dest_comp, int num_comp, - const Periodicity& period = Periodicity::NonPeriodic(), + const Periodicity& period = Periodicity::NonPeriodic(), CpOp op = FabArrayBase::COPY) { ParallelCopy(src,src_comp,dest_comp,num_comp, period, op); } @@ -615,18 +615,18 @@ public: int src_comp, int dest_comp, int num_comp, - int src_nghost, - int dst_nghost, - const Periodicity& period = Periodicity::NonPeriodic(), + int src_nghost, + int dst_nghost, + const Periodicity& period = Periodicity::NonPeriodic(), CpOp op = FabArrayBase::COPY) { ParallelCopy(src,src_comp,dest_comp,num_comp,IntVect(src_nghost),IntVect(dst_nghost),period,op); } void copy (const FabArray& src, int src_comp, int dest_comp, int num_comp, - const IntVect& src_nghost, - const IntVect& dst_nghost, - const Periodicity& period = Periodicity::NonPeriodic(), + const IntVect& src_nghost, + const IntVect& dst_nghost, + const Periodicity& period = Periodicity::NonPeriodic(), CpOp op = FabArrayBase::COPY) { ParallelCopy(src,src_comp,dest_comp,num_comp,src_nghost,dst_nghost,period,op); } @@ -646,7 +646,7 @@ public: * valid + nghost region of this FabArray with the FAB dest into dest. */ void copyTo (FAB& dest, - int nghost = 0) const; + int nghost = 0) const; /** * \brief Copy the values contained in the intersection of the @@ -655,7 +655,7 @@ public: */ void copyTo (FAB& dest, const Box& subbox, - int nghost = 0) const; + int nghost = 0) const; /** * \brief Copy the values contained in the intersection of the @@ -664,10 +664,10 @@ public: * component dest_comp in dest. */ void copyTo (FAB& dest, - int src_comp, - int dest_comp, - int num_comp, - int nghost = 0) const; + int src_comp, + int dest_comp, + int num_comp, + int nghost = 0) const; /** * \brief Copy the values contained in the intersection of the @@ -676,11 +676,11 @@ public: * dest, starting at component dest_comp in dest. */ void copyTo (FAB& dest, - const Box& subbox, - int src_comp, - int dest_comp, - int num_comp, - int nghost = 0) const; + const Box& subbox, + int src_comp, + int dest_comp, + int num_comp, + int nghost = 0) const; //! Shift the boxarray by vector v void shift (const IntVect& v); @@ -736,8 +736,8 @@ public: // interior : interior cells (i.e., valid cells) template ::value,int>::type = 0> void BuildMask (const Box& phys_domain, const Periodicity& period, - value_type covered, value_type notcovered, - value_type physbnd, value_type interior); + value_type covered, value_type notcovered, + value_type physbnd, value_type interior); // The following are private functions. But we have to make them public for cuda. @@ -832,33 +832,33 @@ protected: //! for shared memory struct ShMem { - ShMem () noexcept : alloc(false), n_values(0), n_points(0) + ShMem () noexcept : alloc(false), n_values(0), n_points(0) #if defined(BL_USE_MPI3) - , win(MPI_WIN_NULL) + , win(MPI_WIN_NULL) #endif - { } - ~ShMem () { + { } + ~ShMem () { #if defined(BL_USE_MPI3) - if (win != MPI_WIN_NULL) MPI_Win_free(&win); + if (win != MPI_WIN_NULL) MPI_Win_free(&win); #endif #ifdef BL_USE_TEAM - if (alloc) { - amrex::update_fab_stats(-n_points, -n_values, sizeof(value_type)); + if (alloc) { + amrex::update_fab_stats(-n_points, -n_values, sizeof(value_type)); } #endif - } - ShMem (ShMem&& rhs) noexcept + } + ShMem (ShMem&& rhs) noexcept : alloc(rhs.alloc), n_values(rhs.n_values), n_points(rhs.n_points) #if defined(BL_USE_MPI3) - , win(rhs.win) + , win(rhs.win) #endif - { - rhs.alloc = false; + { + rhs.alloc = false; #if defined(BL_USE_MPI3) - rhs.win = MPI_WIN_NULL; + rhs.win = MPI_WIN_NULL; #endif - } - ShMem& operator= (ShMem&& rhs) noexcept { + } + ShMem& operator= (ShMem&& rhs) noexcept { if (&rhs != this) { alloc = rhs.alloc; n_values = rhs.n_values; @@ -871,13 +871,13 @@ protected: } return *this; } - ShMem (const ShMem&) = delete; - ShMem& operator= (const ShMem&) = delete; - bool alloc; - Long n_values; - Long n_points; + ShMem (const ShMem&) = delete; + ShMem& operator= (const ShMem&) = delete; + bool alloc; + Long n_values; + Long n_points; #if defined(BL_USE_MPI3) - MPI_Win win; + MPI_Win win; #endif }; ShMem shmem; @@ -904,7 +904,7 @@ public: int ncomp, int SeqNum) const; - + AMREX_NODISCARD TheFaArenaPointer PostRcvs (const MapOfCopyComTagContainers& RcvTags, Vector& recv_data, Vector& recv_size, @@ -987,10 +987,10 @@ FabArray::defined (int K) const noexcept { int li = localindex(K); if (li >= 0 && li < static_cast(m_fabs_v.size()) && m_fabs_v[li] != 0) { - return true; + return true; } else { - return false; + return false; } } @@ -1000,10 +1000,10 @@ FabArray::defined (const MFIter& mfi) const noexcept { int li = mfi.LocalIndex(); if (li < static_cast(m_fabs_v.size()) && m_fabs_v[li] != 0) { - return true; + return true; } else { - return false; + return false; } } @@ -1239,7 +1239,7 @@ FabArray::FabArray (const BoxArray& bxs, const DistributionMapping& dm, int nvar, int ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) : FabArray(bxs,dm,nvar,IntVect(ngrow),info,factory) {} @@ -1249,7 +1249,7 @@ FabArray::FabArray (const BoxArray& bxs, const DistributionMapping& dm, int nvar, const IntVect& ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) : m_factory(factory.clone()), shmem() @@ -1359,10 +1359,10 @@ FabArray::ok () const template void FabArray::define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - int ngrow, - const MFInfo& info, + const DistributionMapping& dm, + int nvar, + int ngrow, + const MFInfo& info, const FabFactory& a_factory) { define(bxs,dm,nvar,IntVect(ngrow),info,a_factory); @@ -1371,10 +1371,10 @@ FabArray::define (const BoxArray& bxs, template void FabArray::define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - const IntVect& ngrow, - const MFInfo& info, + const DistributionMapping& dm, + int nvar, + const IntVect& ngrow, + const MFInfo& info, const FabFactory& a_factory) { std::unique_ptr > factory(a_factory.clone()); @@ -1638,7 +1638,7 @@ FabArray::setDomainBndry (value_type val, template void FabArray::copyTo (FAB& dest, - int nghost) const + int nghost) const { copyTo(dest, dest.box(), 0, 0, dest.nComp(), nghost); } @@ -1646,8 +1646,8 @@ FabArray::copyTo (FAB& dest, template void FabArray::copyTo (FAB& dest, - const Box& subbox, - int nghost) const + const Box& subbox, + int nghost) const { copyTo(dest, subbox, 0, 0, dest.nComp(), nghost); } @@ -1655,10 +1655,10 @@ FabArray::copyTo (FAB& dest, template void FabArray::copyTo (FAB& dest, - int scomp, - int dcomp, - int ncomp, - int nghost) const + int scomp, + int dcomp, + int ncomp, + int nghost) const { copyTo(dest, dest.box(), scomp, dcomp, ncomp, nghost); } @@ -1708,7 +1708,7 @@ FabArray::setVal (value_type val, #endif for (MFIter fai(*this,TilingIfNotGPU()); fai.isValid(); ++fai) { - const Box& bx = fai.growntilebox(nghost); + const Box& bx = fai.growntilebox(nghost); auto fab = this->array(fai); AMREX_HOST_DEVICE_PARALLEL_FOR_4D_FUSIBLE ( bx, ncomp, i, j, k, n, { @@ -1935,8 +1935,8 @@ FabArray::FillBoundary (bool cross) { BL_PROFILE("FabArray::FillBoundary()"); if ( n_grow.max() > 0 ) { - FillBoundary_nowait(0, nComp(), n_grow, Periodicity::NonPeriodic(), cross); - FillBoundary_finish(); + FillBoundary_nowait(0, nComp(), n_grow, Periodicity::NonPeriodic(), cross); + FillBoundary_finish(); } } @@ -1946,8 +1946,8 @@ FabArray::FillBoundary (const Periodicity& period, bool cross) { BL_PROFILE("FabArray::FillBoundary()"); if ( n_grow.max() > 0 ) { - FillBoundary_nowait(0, nComp(), n_grow, period, cross); - FillBoundary_finish(); + FillBoundary_nowait(0, nComp(), n_grow, period, cross); + FillBoundary_finish(); } } @@ -1959,8 +1959,8 @@ FabArray::FillBoundary (const IntVect& nghost, const Periodicity& period, b AMREX_ALWAYS_ASSERT_WITH_MESSAGE(nghost.allLE(nGrowVect()), "FillBoundary: asked to fill more ghost cells than we have"); if ( nghost.max() > 0 ) { - FillBoundary_nowait(0, nComp(), nghost, period, cross); - FillBoundary_finish(); + FillBoundary_nowait(0, nComp(), nghost, period, cross); + FillBoundary_finish(); } } @@ -1970,8 +1970,8 @@ FabArray::FillBoundary (int scomp, int ncomp, bool cross) { BL_PROFILE("FabArray::FillBoundary()"); if ( n_grow.max() > 0 ) { - FillBoundary_nowait(scomp, ncomp, n_grow, Periodicity::NonPeriodic(), cross); - FillBoundary_finish(); + FillBoundary_nowait(scomp, ncomp, n_grow, Periodicity::NonPeriodic(), cross); + FillBoundary_finish(); } } @@ -1981,8 +1981,8 @@ FabArray::FillBoundary (int scomp, int ncomp, const Periodicity& period, bo { BL_PROFILE("FabArray::FillBoundary()"); if ( n_grow.max() > 0 ) { - FillBoundary_nowait(scomp, ncomp, n_grow, period, cross); - FillBoundary_finish(); + FillBoundary_nowait(scomp, ncomp, n_grow, period, cross); + FillBoundary_finish(); } } @@ -1995,8 +1995,8 @@ FabArray::FillBoundary (int scomp, int ncomp, const IntVect& nghost, AMREX_ALWAYS_ASSERT_WITH_MESSAGE(nghost.allLE(nGrowVect()), "FillBoundary: asked to fill more ghost cells than we have"); if ( nghost.max() > 0 ) { - FillBoundary_nowait(scomp, ncomp, nghost, period, cross); - FillBoundary_finish(); + FillBoundary_nowait(scomp, ncomp, nghost, period, cross); + FillBoundary_finish(); } } @@ -2027,8 +2027,8 @@ FabArray::EnforcePeriodicity (const Periodicity& period) { BL_PROFILE("FabArray::EnforcePeriodicity"); if (period.isAnyPeriodic()) { - FBEP_nowait(0, nComp(), nGrowVect(), period, false, true); - FillBoundary_finish(); // unsafe unless isAnyPeriodic() + FBEP_nowait(0, nComp(), nGrowVect(), period, false, true); + FillBoundary_finish(); // unsafe unless isAnyPeriodic() } } @@ -2038,8 +2038,8 @@ FabArray::EnforcePeriodicity (int scomp, int ncomp, const Periodicity& peri { BL_PROFILE("FabArray::EnforcePeriodicity"); if (period.isAnyPeriodic()) { - FBEP_nowait(scomp, ncomp, nGrowVect(), period, false, true); - FillBoundary_finish(); // unsafe unless isAnyPeriodic() + FBEP_nowait(scomp, ncomp, nGrowVect(), period, false, true); + FillBoundary_finish(); // unsafe unless isAnyPeriodic() } } @@ -2050,8 +2050,8 @@ FabArray::EnforcePeriodicity (int scomp, int ncomp, const IntVect& nghost, { BL_PROFILE("FabArray::EnforcePeriodicity"); if (period.isAnyPeriodic()) { - FBEP_nowait(scomp, ncomp, nghost, period, false, true); - FillBoundary_finish(); // unsafe unless isAnyPeriodic() + FBEP_nowait(scomp, ncomp, nghost, period, false, true); + FillBoundary_finish(); // unsafe unless isAnyPeriodic() } } @@ -2076,17 +2076,17 @@ template template ::value,int>::type Z> void FabArray::BuildMask (const Box& phys_domain, const Periodicity& period, - value_type covered, value_type notcovered, - value_type physbnd, value_type interior) + value_type covered, value_type notcovered, + value_type physbnd, value_type interior) { int ncomp = this->nComp(); const IntVect& ngrow = this->nGrowVect(); Box domain = amrex::convert(phys_domain, boxArray().ixType()); for (int i = 0; i < AMREX_SPACEDIM; ++i) { - if (period.isPeriodic(i)) { - domain.grow(i, ngrow[i]); - } + if (period.isPeriodic(i)) { + domain.grow(i, ngrow[i]); + } } #ifdef AMREX_USE_OMP diff --git a/Src/Base/AMReX_FabArrayBase.H b/Src/Base/AMReX_FabArrayBase.H index 818bdac0983..c2bc042e064 100644 --- a/Src/Base/AMReX_FabArrayBase.H +++ b/Src/Base/AMReX_FabArrayBase.H @@ -145,57 +145,57 @@ public: // struct CacheStats { - int size; //!< current size: nbuild - nerase - int maxsize; //!< highest water mark of size - Long maxuse; //!< max # of uses of a cached item - Long nuse; //!< # of uses of the whole cache - Long nbuild; //!< # of build operations - Long nerase; //!< # of erase operations - Long bytes; - Long bytes_hwm; - std::string name; //!< name of the cache - explicit CacheStats (const std::string& name_) - : size(0),maxsize(0),maxuse(0),nuse(0),nbuild(0),nerase(0), - bytes(0L),bytes_hwm(0L),name(name_) {;} - void recordBuild () noexcept { - ++size; - ++nbuild; - maxsize = std::max(maxsize, size); - } - void recordErase (Long n) noexcept { - // n: how many times the item to be deleted has been used. - --size; - ++nerase; - maxuse = std::max(maxuse, n); - } - void recordUse () noexcept { ++nuse; } - void print () { - amrex::Print(Print::AllProcs) << "### " << name << " ###\n" - << " tot # of builds : " << nbuild << "\n" - << " tot # of erasures: " << nerase << "\n" - << " tot # of uses : " << nuse << "\n" - << " max cache size : " << maxsize << "\n" - << " max # of uses : " << maxuse << "\n"; - } + int size; //!< current size: nbuild - nerase + int maxsize; //!< highest water mark of size + Long maxuse; //!< max # of uses of a cached item + Long nuse; //!< # of uses of the whole cache + Long nbuild; //!< # of build operations + Long nerase; //!< # of erase operations + Long bytes; + Long bytes_hwm; + std::string name; //!< name of the cache + explicit CacheStats (const std::string& name_) + : size(0),maxsize(0),maxuse(0),nuse(0),nbuild(0),nerase(0), + bytes(0L),bytes_hwm(0L),name(name_) {;} + void recordBuild () noexcept { + ++size; + ++nbuild; + maxsize = std::max(maxsize, size); + } + void recordErase (Long n) noexcept { + // n: how many times the item to be deleted has been used. + --size; + ++nerase; + maxuse = std::max(maxuse, n); + } + void recordUse () noexcept { ++nuse; } + void print () { + amrex::Print(Print::AllProcs) << "### " << name << " ###\n" + << " tot # of builds : " << nbuild << "\n" + << " tot # of erasures: " << nerase << "\n" + << " tot # of uses : " << nuse << "\n" + << " max cache size : " << maxsize << "\n" + << " max # of uses : " << maxuse << "\n"; + } }; // //! Used by a bunch of routines when communicating via MPI. struct CopyComTag { - Box dbox; + Box dbox; Box sbox; int dstIndex; int srcIndex; - CopyComTag () noexcept {} - CopyComTag (const Box& db, const Box& sb, int didx, int sidx) noexcept - : dbox(db), sbox(sb), dstIndex(didx), srcIndex(sidx) {} - bool operator< (const CopyComTag& rhs) const noexcept { - return (srcIndex < rhs.srcIndex) || ((srcIndex == rhs.srcIndex) && ( + CopyComTag () noexcept {} + CopyComTag (const Box& db, const Box& sb, int didx, int sidx) noexcept + : dbox(db), sbox(sb), dstIndex(didx), srcIndex(sidx) {} + bool operator< (const CopyComTag& rhs) const noexcept { + return (srcIndex < rhs.srcIndex) || ((srcIndex == rhs.srcIndex) && ( (sbox.smallEnd() < rhs.sbox.smallEnd() - || ((sbox.smallEnd() == rhs.sbox.smallEnd()) && ( + || ((sbox.smallEnd() == rhs.sbox.smallEnd()) && ( (dstIndex < rhs.dstIndex) || ((dstIndex == rhs.dstIndex) && ( (dbox.smallEnd() < rhs.dbox.smallEnd())))))))); - } + } // // Some typedefs & helper functions used throughout the code. // @@ -237,7 +237,7 @@ public: }; BDKey getBDKey () const noexcept { - return {boxarray.getRefID(), distributionMap.getRefID()}; + return {boxarray.getRefID(), distributionMap.getRefID()}; } void updateBDKey (); @@ -469,14 +469,14 @@ public: void buildTileArray (const IntVect& tilesize, TileArray& ta) const; // void flushTileArray (const IntVect& tilesize = IntVect::TheZeroVector(), - bool no_assertion=false) const; + bool no_assertion=false) const; static void flushTileArrayCache (); //!< This flushes the entire cache. struct CommMetaData { // The cache of local and send/recv per FillBoundary() or ParallelCopy(). - bool m_threadsafe_loc = false; - bool m_threadsafe_rcv = false; + bool m_threadsafe_loc = false; + bool m_threadsafe_rcv = false; std::unique_ptr m_LocTags; std::unique_ptr m_SndTags; std::unique_ptr m_RcvTags; @@ -489,7 +489,7 @@ public: { FB (const FabArrayBase& fa, const IntVect& nghost, bool cross, const Periodicity& period, - bool enforce_periodicity_only, bool multi_ghost = false); + bool enforce_periodicity_only, bool multi_ghost = false); ~FB (); IndexType m_typ; @@ -690,7 +690,7 @@ public: #ifdef BL_USE_MPI static bool CheckRcvStats(Vector& recv_stats, - const Vector& recv_size, + const Vector& recv_size, int tag); #endif diff --git a/Src/Base/AMReX_FabArrayBase.cpp b/Src/Base/AMReX_FabArrayBase.cpp index f7818fe0587..c12c336b26b 100644 --- a/Src/Base/AMReX_FabArrayBase.cpp +++ b/Src/Base/AMReX_FabArrayBase.cpp @@ -101,12 +101,12 @@ FabArrayBase::Initialize () if (pp.queryarr("mfiter_tile_size", tilesize, 0, AMREX_SPACEDIM)) { - for (int i=0; i - ([] () -> MemProfiler::MemInfo { - return {m_TAC_stats.bytes, m_TAC_stats.bytes_hwm}; - })); + ([] () -> MemProfiler::MemInfo { + return {m_TAC_stats.bytes, m_TAC_stats.bytes_hwm}; + })); MemProfiler::add(m_FBC_stats.name, std::function - ([] () -> MemProfiler::MemInfo { - return {m_FBC_stats.bytes, m_FBC_stats.bytes_hwm}; - })); + ([] () -> MemProfiler::MemInfo { + return {m_FBC_stats.bytes, m_FBC_stats.bytes_hwm}; + })); MemProfiler::add(m_CPC_stats.name, std::function - ([] () -> MemProfiler::MemInfo { - return {m_CPC_stats.bytes, m_CPC_stats.bytes_hwm}; - })); + ([] () -> MemProfiler::MemInfo { + return {m_CPC_stats.bytes, m_CPC_stats.bytes_hwm}; + })); MemProfiler::add(m_FPinfo_stats.name, std::function - ([] () -> MemProfiler::MemInfo { - return {m_FPinfo_stats.bytes, m_FPinfo_stats.bytes_hwm}; - })); + ([] () -> MemProfiler::MemInfo { + return {m_FPinfo_stats.bytes, m_FPinfo_stats.bytes_hwm}; + })); MemProfiler::add(m_CFinfo_stats.name, std::function - ([] () -> MemProfiler::MemInfo { - return {m_CFinfo_stats.bytes, m_CFinfo_stats.bytes_hwm}; - })); + ([] () -> MemProfiler::MemInfo { + return {m_CFinfo_stats.bytes, m_CFinfo_stats.bytes_hwm}; + })); #endif } @@ -206,14 +206,14 @@ FabArrayBase::define (const BoxArray& bxs, n_grow = ngrow; n_comp = nvar; n_filled = IntVect(0); - + boxarray = bxs; - + BL_ASSERT(dm.ProcessorMap().size() == bxs.size()); distributionMap = dm; indexArray = distributionMap.getIndexArray(); - ownership = distributionMap.getOwnerShip(); + ownership = distributionMap.getOwnerShip(); } void @@ -237,8 +237,8 @@ FabArrayBase::bytesOfMapOfCopyComTagContainers (const FabArrayBase::MapOfCopyCom { Long r = sizeof(MapOfCopyComTagContainers); for (MapOfCopyComTagContainers::const_iterator it = m.begin(); it != m.end(); ++it) { - r += sizeof(it->first) + amrex::bytesOf(it->second) - + amrex::gcc_map_node_extra_bytes; + r += sizeof(it->first) + amrex::bytesOf(it->second) + + amrex::gcc_map_node_extra_bytes; } return r; } @@ -249,13 +249,13 @@ FabArrayBase::CPC::bytes () const Long cnt = sizeof(FabArrayBase::CPC); if (m_LocTags) - cnt += amrex::bytesOf(*m_LocTags); + cnt += amrex::bytesOf(*m_LocTags); if (m_SndTags) - cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_SndTags); + cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_SndTags); if (m_RcvTags) - cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_RcvTags); + cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_RcvTags); return cnt; } @@ -266,13 +266,13 @@ FabArrayBase::FB::bytes () const int cnt = sizeof(FabArrayBase::FB); if (m_LocTags) - cnt += amrex::bytesOf(*m_LocTags); + cnt += amrex::bytesOf(*m_LocTags); if (m_SndTags) - cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_SndTags); + cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_SndTags); if (m_RcvTags) - cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_RcvTags); + cnt += FabArrayBase::bytesOfMapOfCopyComTagContainers(*m_RcvTags); return cnt; } @@ -280,12 +280,12 @@ FabArrayBase::FB::bytes () const Long FabArrayBase::TileArray::bytes () const { - return sizeof(*this) - + (amrex::bytesOf(this->numLocalTiles) - sizeof(this->numLocalTiles)) - + (amrex::bytesOf(this->indexMap) - sizeof(this->indexMap)) - + (amrex::bytesOf(this->localIndexMap) - sizeof(this->localIndexMap)) - + (amrex::bytesOf(this->localTileIndexMap) - sizeof(this->localTileIndexMap)) - + (amrex::bytesOf(this->tileArray) - sizeof(this->tileArray)); + return sizeof(*this) + + (amrex::bytesOf(this->numLocalTiles) - sizeof(this->numLocalTiles)) + + (amrex::bytesOf(this->indexMap) - sizeof(this->indexMap)) + + (amrex::bytesOf(this->localIndexMap) - sizeof(this->localIndexMap)) + + (amrex::bytesOf(this->localTileIndexMap) - sizeof(this->localTileIndexMap)) + + (amrex::bytesOf(this->tileArray) - sizeof(this->tileArray)); } // @@ -293,32 +293,32 @@ FabArrayBase::TileArray::bytes () const // FabArrayBase::CPC::CPC (const FabArrayBase& dstfa, const IntVect& dstng, - const FabArrayBase& srcfa, const IntVect& srcng, - const Periodicity& period) - : m_srcbdk(srcfa.getBDKey()), - m_dstbdk(dstfa.getBDKey()), - m_srcng(srcng), - m_dstng(dstng), + const FabArrayBase& srcfa, const IntVect& srcng, + const Periodicity& period) + : m_srcbdk(srcfa.getBDKey()), + m_dstbdk(dstfa.getBDKey()), + m_srcng(srcng), + m_dstng(dstng), m_period(period), - m_srcba(srcfa.boxArray()), + m_srcba(srcfa.boxArray()), m_dstba(dstfa.boxArray()), m_nuse(0) { - this->define(m_dstba, dstfa.DistributionMap(), dstfa.IndexArray(), - m_srcba, srcfa.DistributionMap(), srcfa.IndexArray()); + this->define(m_dstba, dstfa.DistributionMap(), dstfa.IndexArray(), + m_srcba, srcfa.DistributionMap(), srcfa.IndexArray()); } -FabArrayBase::CPC::CPC (const BoxArray& dstba, const DistributionMapping& dstdm, - const Vector& dstidx, const IntVect& dstng, - const BoxArray& srcba, const DistributionMapping& srcdm, - const Vector& srcidx, const IntVect& srcng, - const Periodicity& period, int myproc) - : m_srcbdk(), - m_dstbdk(), - m_srcng(srcng), - m_dstng(dstng), +FabArrayBase::CPC::CPC (const BoxArray& dstba, const DistributionMapping& dstdm, + const Vector& dstidx, const IntVect& dstng, + const BoxArray& srcba, const DistributionMapping& srcdm, + const Vector& srcidx, const IntVect& srcng, + const Periodicity& period, int myproc) + : m_srcbdk(), + m_dstbdk(), + m_srcng(srcng), + m_dstng(dstng), m_period(period), - m_srcba(srcba), + m_srcba(srcba), m_dstba(dstba), m_nuse(0) { @@ -330,155 +330,155 @@ FabArrayBase::CPC::~CPC () void FabArrayBase::CPC::define (const BoxArray& ba_dst, const DistributionMapping& dm_dst, - const Vector& imap_dst, - const BoxArray& ba_src, const DistributionMapping& dm_src, - const Vector& imap_src, - int MyProc) + const Vector& imap_dst, + const BoxArray& ba_src, const DistributionMapping& dm_src, + const Vector& imap_src, + int MyProc) { BL_PROFILE("FabArrayBase::CPC::define()"); BL_ASSERT(ba_dst.size() > 0 && ba_src.size() > 0); BL_ASSERT(ba_dst.ixType() == ba_src.ixType()); - + m_LocTags.reset(new CopyComTag::CopyComTagsContainer); m_SndTags.reset(new CopyComTag::MapOfCopyComTagContainers); m_RcvTags.reset(new CopyComTag::MapOfCopyComTagContainers); - if (!(imap_dst.empty() && imap_src.empty())) + if (!(imap_dst.empty() && imap_src.empty())) { - const int nlocal_src = imap_src.size(); - const IntVect& ng_src = m_srcng; - const int nlocal_dst = imap_dst.size(); - const IntVect& ng_dst = m_dstng; - - std::vector< std::pair > isects; - - const std::vector& pshifts = m_period.shiftIntVect(); - - auto& send_tags = *m_SndTags; - - for (int i = 0; i < nlocal_src; ++i) - { - const int k_src = imap_src[i]; - const Box& bx_src = amrex::grow(ba_src[k_src], ng_src); - - for (std::vector::const_iterator pit=pshifts.begin(); pit!=pshifts.end(); ++pit) - { - ba_dst.intersections(bx_src+(*pit), isects, false, ng_dst); - - for (int j = 0, M = isects.size(); j < M; ++j) - { - const int k_dst = isects[j].first; - const Box& bx = isects[j].second; - const int dst_owner = dm_dst[k_dst]; - - if (ParallelDescriptor::sameTeam(dst_owner)) { - continue; // local copy will be dealt with later - } else if (MyProc == dm_src[k_src]) { - send_tags[dst_owner].push_back(CopyComTag(bx, bx-(*pit), k_dst, k_src)); - } - } - } - } - - auto& recv_tags = *m_RcvTags; - - BaseFab localtouch(The_Cpu_Arena()), remotetouch(The_Cpu_Arena()); - bool check_local = false, check_remote = false; + const int nlocal_src = imap_src.size(); + const IntVect& ng_src = m_srcng; + const int nlocal_dst = imap_dst.size(); + const IntVect& ng_dst = m_dstng; + + std::vector< std::pair > isects; + + const std::vector& pshifts = m_period.shiftIntVect(); + + auto& send_tags = *m_SndTags; + + for (int i = 0; i < nlocal_src; ++i) + { + const int k_src = imap_src[i]; + const Box& bx_src = amrex::grow(ba_src[k_src], ng_src); + + for (std::vector::const_iterator pit=pshifts.begin(); pit!=pshifts.end(); ++pit) + { + ba_dst.intersections(bx_src+(*pit), isects, false, ng_dst); + + for (int j = 0, M = isects.size(); j < M; ++j) + { + const int k_dst = isects[j].first; + const Box& bx = isects[j].second; + const int dst_owner = dm_dst[k_dst]; + + if (ParallelDescriptor::sameTeam(dst_owner)) { + continue; // local copy will be dealt with later + } else if (MyProc == dm_src[k_src]) { + send_tags[dst_owner].push_back(CopyComTag(bx, bx-(*pit), k_dst, k_src)); + } + } + } + } + + auto& recv_tags = *m_RcvTags; + + BaseFab localtouch(The_Cpu_Arena()), remotetouch(The_Cpu_Arena()); + bool check_local = false, check_remote = false; #if defined(AMREX_USE_GPU) check_local = true; check_remote = true; #elif defined(AMREX_USE_OMP) - if (omp_get_max_threads() > 1) { - check_local = true; - check_remote = true; - } + if (omp_get_max_threads() > 1) { + check_local = true; + check_remote = true; + } #endif - if (ParallelDescriptor::TeamSize() > 1) { - check_local = true; - } + if (ParallelDescriptor::TeamSize() > 1) { + check_local = true; + } m_threadsafe_loc = ! check_local; m_threadsafe_rcv = ! check_remote; - for (int i = 0; i < nlocal_dst; ++i) - { - const int k_dst = imap_dst[i]; - const Box& bx_dst = amrex::grow(ba_dst[k_dst], ng_dst); - - if (check_local) { - localtouch.resize(bx_dst); - localtouch.setVal(0); - } - - if (check_remote) { - remotetouch.resize(bx_dst); - remotetouch.setVal(0); - } - - for (std::vector::const_iterator pit=pshifts.begin(); pit!=pshifts.end(); ++pit) - { - ba_src.intersections(bx_dst+(*pit), isects, false, ng_src); - - for (int j = 0, M = isects.size(); j < M; ++j) - { - const int k_src = isects[j].first; - const Box& bx = isects[j].second - *pit; - const int src_owner = dm_src[k_src]; - - if (ParallelDescriptor::sameTeam(src_owner, MyProc)) { // local copy - const BoxList tilelist(bx, FabArrayBase::comm_tile_size); - for (BoxList::const_iterator - it_tile = tilelist.begin(), - End_tile = tilelist.end(); it_tile != End_tile; ++it_tile) - { - m_LocTags->push_back(CopyComTag(*it_tile, (*it_tile)+(*pit), k_dst, k_src)); - } - if (check_local) { - localtouch.plus(1, bx); - } - } else if (MyProc == dm_dst[k_dst]) { - recv_tags[src_owner].push_back(CopyComTag(bx, bx+(*pit), k_dst, k_src)); - if (check_remote) { - remotetouch.plus(1, bx); - } - } - } - } - - if (check_local) { - // safe if a cell is touched no more than once - // keep checking thread safety if it is safe so far - check_local = m_threadsafe_loc = localtouch.max() <= 1; - } - - if (check_remote) { - check_remote = m_threadsafe_rcv = remotetouch.max() <= 1; - } - } - - for (int ipass = 0; ipass < 2; ++ipass) // pass 0: send; pass 1: recv - { - CopyComTag::MapOfCopyComTagContainers & Tags = (ipass == 0) ? *m_SndTags : *m_RcvTags; + for (int i = 0; i < nlocal_dst; ++i) + { + const int k_dst = imap_dst[i]; + const Box& bx_dst = amrex::grow(ba_dst[k_dst], ng_dst); + + if (check_local) { + localtouch.resize(bx_dst); + localtouch.setVal(0); + } + + if (check_remote) { + remotetouch.resize(bx_dst); + remotetouch.setVal(0); + } + + for (std::vector::const_iterator pit=pshifts.begin(); pit!=pshifts.end(); ++pit) + { + ba_src.intersections(bx_dst+(*pit), isects, false, ng_src); + + for (int j = 0, M = isects.size(); j < M; ++j) + { + const int k_src = isects[j].first; + const Box& bx = isects[j].second - *pit; + const int src_owner = dm_src[k_src]; + + if (ParallelDescriptor::sameTeam(src_owner, MyProc)) { // local copy + const BoxList tilelist(bx, FabArrayBase::comm_tile_size); + for (BoxList::const_iterator + it_tile = tilelist.begin(), + End_tile = tilelist.end(); it_tile != End_tile; ++it_tile) + { + m_LocTags->push_back(CopyComTag(*it_tile, (*it_tile)+(*pit), k_dst, k_src)); + } + if (check_local) { + localtouch.plus(1, bx); + } + } else if (MyProc == dm_dst[k_dst]) { + recv_tags[src_owner].push_back(CopyComTag(bx, bx+(*pit), k_dst, k_src)); + if (check_remote) { + remotetouch.plus(1, bx); + } + } + } + } + + if (check_local) { + // safe if a cell is touched no more than once + // keep checking thread safety if it is safe so far + check_local = m_threadsafe_loc = localtouch.max() <= 1; + } + + if (check_remote) { + check_remote = m_threadsafe_rcv = remotetouch.max() <= 1; + } + } + + for (int ipass = 0; ipass < 2; ++ipass) // pass 0: send; pass 1: recv + { + CopyComTag::MapOfCopyComTagContainers & Tags = (ipass == 0) ? *m_SndTags : *m_RcvTags; for (auto& kv : Tags) - { - std::vector& cctv = kv.second; - // We need to fix the order so that the send and recv processes match. - std::sort(cctv.begin(), cctv.end()); - } - } + { + std::vector& cctv = kv.second; + // We need to fix the order so that the send and recv processes match. + std::sort(cctv.begin(), cctv.end()); + } + } } } FabArrayBase::CPC::CPC (const BoxArray& ba, const IntVect& ng, const DistributionMapping& dstdm, const DistributionMapping& srcdm) - : m_srcbdk(), - m_dstbdk(), - m_srcng(ng), - m_dstng(ng), + : m_srcbdk(), + m_dstbdk(), + m_srcng(ng), + m_dstng(ng), m_period(), - m_srcba(ba), + m_srcba(ba), m_dstba(ba), m_nuse(0) { @@ -526,37 +526,37 @@ FabArrayBase::flushCPC (bool no_assertion) const for (CPCacheIter it = er_it.first; it != er_it.second; ++it) { - const BDKey& srckey = it->second->m_srcbdk; - const BDKey& dstkey = it->second->m_dstbdk; + const BDKey& srckey = it->second->m_srcbdk; + const BDKey& dstkey = it->second->m_dstbdk; - BL_ASSERT((srckey==dstkey && srckey==m_bdkey) || - (m_bdkey==srckey) || (m_bdkey==dstkey)); + BL_ASSERT((srckey==dstkey && srckey==m_bdkey) || + (m_bdkey==srckey) || (m_bdkey==dstkey)); - if (srckey != dstkey) { - const BDKey& otherkey = (m_bdkey == srckey) ? dstkey : srckey; - std::pair o_er_it = m_TheCPCache.equal_range(otherkey); + if (srckey != dstkey) { + const BDKey& otherkey = (m_bdkey == srckey) ? dstkey : srckey; + std::pair o_er_it = m_TheCPCache.equal_range(otherkey); - for (CPCacheIter oit = o_er_it.first; oit != o_er_it.second; ++oit) - { - if (it->second == oit->second) - others.push_back(oit); - } - } + for (CPCacheIter oit = o_er_it.first; oit != o_er_it.second; ++oit) + { + if (it->second == oit->second) + others.push_back(oit); + } + } #ifdef AMREX_MEM_PROFILING - m_CPC_stats.bytes -= it->second->bytes(); + m_CPC_stats.bytes -= it->second->bytes(); #endif - m_CPC_stats.recordErase(it->second->m_nuse); - delete it->second; + m_CPC_stats.recordErase(it->second->m_nuse); + delete it->second; } m_TheCPCache.erase(er_it.first, er_it.second); for (std::vector::iterator it = others.begin(), - End = others.end(); it != End; ++it) + End = others.end(); it != End; ++it) { - m_TheCPCache.erase(*it); - } + m_TheCPCache.erase(*it); + } } void @@ -564,10 +564,10 @@ FabArrayBase::flushCPCache () { for (CPCacheIter it = m_TheCPCache.begin(); it != m_TheCPCache.end(); ++it) { - if (it->first == it->second->m_srcbdk) { - m_CPC_stats.recordErase(it->second->m_nuse); - delete it->second; - } + if (it->first == it->second->m_srcbdk) { + m_CPC_stats.recordErase(it->second->m_nuse); + delete it->second; + } } m_TheCPCache.clear(); #ifdef AMREX_MEM_PROFILING @@ -591,27 +591,27 @@ FabArrayBase::getCPC (const IntVect& dstng, const FabArrayBase& src, const IntVe for (CPCacheIter it = er_it.first; it != er_it.second; ++it) { - if (it->second->m_srcng == srcng && - it->second->m_dstng == dstng && - it->second->m_srcbdk == srckey && - it->second->m_dstbdk == dstkey && - it->second->m_period == period && - it->second->m_srcba == src.boxArray() && - it->second->m_dstba == boxArray()) - { - ++(it->second->m_nuse); - m_CPC_stats.recordUse(); - return *(it->second); - } - } - + if (it->second->m_srcng == srcng && + it->second->m_dstng == dstng && + it->second->m_srcbdk == srckey && + it->second->m_dstbdk == dstkey && + it->second->m_period == period && + it->second->m_srcba == src.boxArray() && + it->second->m_dstba == boxArray()) + { + ++(it->second->m_nuse); + m_CPC_stats.recordUse(); + return *(it->second); + } + } + // Have to build a new one CPC* new_cpc = new CPC(*this, dstng, src, srcng, period); #ifdef AMREX_MEM_PROFILING m_CPC_stats.bytes += new_cpc->bytes(); m_CPC_stats.bytes_hwm = std::max(m_CPC_stats.bytes_hwm, m_CPC_stats.bytes); -#endif +#endif new_cpc->m_nuse = 1; m_CPC_stats.recordBuild(); @@ -619,7 +619,7 @@ FabArrayBase::getCPC (const IntVect& dstng, const FabArrayBase& src, const IntVe m_TheCPCache.insert(er_it.second, CPCache::value_type(dstkey,new_cpc)); if (srckey != dstkey) - m_TheCPCache.insert( CPCache::value_type(srckey,new_cpc)); + m_TheCPCache.insert( CPCache::value_type(srckey,new_cpc)); return *new_cpc; } @@ -629,7 +629,7 @@ FabArrayBase::getCPC (const IntVect& dstng, const FabArrayBase& src, const IntVe // FabArrayBase::FB::FB (const FabArrayBase& fa, const IntVect& nghost, - bool cross, const Periodicity& period, + bool cross, const Periodicity& period, bool enforce_periodicity_only, bool multi_ghost) : m_typ(fa.boxArray().ixType()), m_crse_ratio(fa.boxArray().crseRatio()), @@ -644,12 +644,12 @@ FabArrayBase::FB::FB (const FabArrayBase& fa, const IntVect& nghost, m_RcvTags.reset(new CopyComTag::MapOfCopyComTagContainers); if (!fa.IndexArray().empty()) { - if (enforce_periodicity_only) { - BL_ASSERT(m_cross==false); - define_epo(fa); - } else { - define_fb(fa); - } + if (enforce_periodicity_only) { + BL_ASSERT(m_cross==false); + define_epo(fa); + } else { + define_fb(fa); + } } } @@ -665,16 +665,16 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa) // For local copy, all workers in the same team will have the identical copy of tags // so that they can share work. But for remote communication, they are all different. - + const int nlocal = imap.size(); const IntVect& ng = m_ngrow; const IntVect ng_ng = m_ngrow - 1; std::vector< std::pair > isects; - + const std::vector& pshifts = m_period.shiftIntVect(); - + auto& send_tags = *m_SndTags; - + for (int i = 0; i < nlocal; ++i) { const int ksnd = imap[i]; @@ -779,9 +779,9 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa) } bl.simplify(); } - for (BoxList::const_iterator lit = bl.begin(); lit != bl.end(); ++lit) - { - const Box& blbx = *lit; + for (BoxList::const_iterator lit = bl.begin(); lit != bl.end(); ++lit) + { + const Box& blbx = *lit; if (ParallelDescriptor::sameTeam(src_owner)) { // local copy const BoxList tilelist(blbx, FabArrayBase::comm_tile_size); @@ -865,7 +865,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa) { if (m_cross) { - cctv_tags_cross.push_back(CopyComTag(cross_box, cross_box+d2s, + cctv_tags_cross.push_back(CopyComTag(cross_box, cross_box+d2s, tag.dstIndex, tag.srcIndex)); } } @@ -889,50 +889,50 @@ FabArrayBase::FB::define_epo (const FabArrayBase& fa) // For local copy, all workers in the same team will have the identical copy of tags // so that they can share work. But for remote communication, they are all different. - + const int nlocal = imap.size(); const IntVect& ng = m_ngrow; const IndexType& typ = ba.ixType(); std::vector< std::pair > isects; - + const std::vector& pshifts = m_period.shiftIntVect(); - + auto& send_tags = *m_SndTags; Box pdomain = m_period.Domain(); pdomain.convert(typ); - + for (int i = 0; i < nlocal; ++i) { - const int ksnd = imap[i]; - Box bxsnd = amrex::grow(ba[ksnd],ng); - bxsnd &= pdomain; // source must be inside the periodic domain. - - if (!bxsnd.ok()) continue; - - for (auto pit=pshifts.cbegin(); pit!=pshifts.cend(); ++pit) - { - if (*pit != IntVect::TheZeroVector()) - { - ba.intersections(bxsnd+(*pit), isects, false, ng); - - for (int j = 0, M = isects.size(); j < M; ++j) - { - const int krcv = isects[j].first; - const Box& bx = isects[j].second; - const int dst_owner = dm[krcv]; - - if (ParallelDescriptor::sameTeam(dst_owner)) { - continue; // local copy will be dealt with later - } else if (MyProc == dm[ksnd]) { - const BoxList& bl = amrex::boxDiff(bx, pdomain); - for (BoxList::const_iterator lit = bl.begin(); lit != bl.end(); ++lit) { - send_tags[dst_owner].push_back(CopyComTag(*lit, (*lit)-(*pit), krcv, ksnd)); - } - } - } - } - } + const int ksnd = imap[i]; + Box bxsnd = amrex::grow(ba[ksnd],ng); + bxsnd &= pdomain; // source must be inside the periodic domain. + + if (!bxsnd.ok()) continue; + + for (auto pit=pshifts.cbegin(); pit!=pshifts.cend(); ++pit) + { + if (*pit != IntVect::TheZeroVector()) + { + ba.intersections(bxsnd+(*pit), isects, false, ng); + + for (int j = 0, M = isects.size(); j < M; ++j) + { + const int krcv = isects[j].first; + const Box& bx = isects[j].second; + const int dst_owner = dm[krcv]; + + if (ParallelDescriptor::sameTeam(dst_owner)) { + continue; // local copy will be dealt with later + } else if (MyProc == dm[ksnd]) { + const BoxList& bl = amrex::boxDiff(bx, pdomain); + for (BoxList::const_iterator lit = bl.begin(); lit != bl.end(); ++lit) { + send_tags[dst_owner].push_back(CopyComTag(*lit, (*lit)-(*pit), krcv, ksnd)); + } + } + } + } + } } auto& recv_tags = *m_RcvTags; @@ -941,8 +941,8 @@ FabArrayBase::FB::define_epo (const FabArrayBase& fa) bool check_local = false, check_remote = false; #if defined(AMREX_USE_OMP) if (omp_get_max_threads() > 1) { - check_local = true; - check_remote = true; + check_local = true; + check_remote = true; } #elif defined(AMREX_USE_GPU) check_local = true; @@ -950,7 +950,7 @@ FabArrayBase::FB::define_epo (const FabArrayBase& fa) #endif if (ParallelDescriptor::TeamSize() > 1) { - check_local = true; + check_local = true; } m_threadsafe_loc = ! check_local; @@ -958,86 +958,86 @@ FabArrayBase::FB::define_epo (const FabArrayBase& fa) for (int i = 0; i < nlocal; ++i) { - const int krcv = imap[i]; - const Box& vbx = ba[krcv]; - const Box& bxrcv = amrex::grow(vbx, ng); - - if (pdomain.contains(bxrcv)) continue; - - if (check_local) { - localtouch.resize(bxrcv); - localtouch.setVal(0); - } - - if (check_remote) { - remotetouch.resize(bxrcv); - remotetouch.setVal(0); - } - - for (std::vector::const_iterator pit=pshifts.begin(); pit!=pshifts.end(); ++pit) - { - if (*pit != IntVect::TheZeroVector()) - { - ba.intersections(bxrcv+(*pit), isects, false, ng); - - for (int j = 0, M = isects.size(); j < M; ++j) - { - const int ksnd = isects[j].first; - const Box& dst_bx = isects[j].second - *pit; - const int src_owner = dm[ksnd]; - - const BoxList& bl = amrex::boxDiff(dst_bx, pdomain); - - for (BoxList::const_iterator lit = bl.begin(); lit != bl.end(); ++lit) - { - Box sbx = (*lit) + (*pit); - sbx &= pdomain; // source must be inside the periodic domain. - - if (sbx.ok()) { - Box dbx = sbx - (*pit); - if (ParallelDescriptor::sameTeam(src_owner)) { // local copy - const BoxList tilelist(dbx, FabArrayBase::comm_tile_size); - for (BoxList::const_iterator - it_tile = tilelist.begin(), - End_tile = tilelist.end(); it_tile != End_tile; ++it_tile) - { - m_LocTags->push_back(CopyComTag(*it_tile, (*it_tile)+(*pit), krcv, ksnd)); - } - if (check_local) { - localtouch.plus(1, dbx); - } - } else if (MyProc == dm[krcv]) { - recv_tags[src_owner].push_back(CopyComTag(dbx, sbx, krcv, ksnd)); - if (check_remote) { - remotetouch.plus(1, dbx); - } - } - } - } - } - } - } - - if (check_local) { - // safe if a cell is touched no more than once - // keep checking thread safety if it is safe so far - check_local = m_threadsafe_loc = localtouch.max() <= 1; - } - - if (check_remote) { - check_remote = m_threadsafe_rcv = remotetouch.max() <= 1; - } + const int krcv = imap[i]; + const Box& vbx = ba[krcv]; + const Box& bxrcv = amrex::grow(vbx, ng); + + if (pdomain.contains(bxrcv)) continue; + + if (check_local) { + localtouch.resize(bxrcv); + localtouch.setVal(0); + } + + if (check_remote) { + remotetouch.resize(bxrcv); + remotetouch.setVal(0); + } + + for (std::vector::const_iterator pit=pshifts.begin(); pit!=pshifts.end(); ++pit) + { + if (*pit != IntVect::TheZeroVector()) + { + ba.intersections(bxrcv+(*pit), isects, false, ng); + + for (int j = 0, M = isects.size(); j < M; ++j) + { + const int ksnd = isects[j].first; + const Box& dst_bx = isects[j].second - *pit; + const int src_owner = dm[ksnd]; + + const BoxList& bl = amrex::boxDiff(dst_bx, pdomain); + + for (BoxList::const_iterator lit = bl.begin(); lit != bl.end(); ++lit) + { + Box sbx = (*lit) + (*pit); + sbx &= pdomain; // source must be inside the periodic domain. + + if (sbx.ok()) { + Box dbx = sbx - (*pit); + if (ParallelDescriptor::sameTeam(src_owner)) { // local copy + const BoxList tilelist(dbx, FabArrayBase::comm_tile_size); + for (BoxList::const_iterator + it_tile = tilelist.begin(), + End_tile = tilelist.end(); it_tile != End_tile; ++it_tile) + { + m_LocTags->push_back(CopyComTag(*it_tile, (*it_tile)+(*pit), krcv, ksnd)); + } + if (check_local) { + localtouch.plus(1, dbx); + } + } else if (MyProc == dm[krcv]) { + recv_tags[src_owner].push_back(CopyComTag(dbx, sbx, krcv, ksnd)); + if (check_remote) { + remotetouch.plus(1, dbx); + } + } + } + } + } + } + } + + if (check_local) { + // safe if a cell is touched no more than once + // keep checking thread safety if it is safe so far + check_local = m_threadsafe_loc = localtouch.max() <= 1; + } + + if (check_remote) { + check_remote = m_threadsafe_rcv = remotetouch.max() <= 1; + } } for (int ipass = 0; ipass < 2; ++ipass) // pass 0: send; pass 1: recv { - CopyComTag::MapOfCopyComTagContainers & Tags = (ipass == 0) ? *m_SndTags : *m_RcvTags; + CopyComTag::MapOfCopyComTagContainers & Tags = (ipass == 0) ? *m_SndTags : *m_RcvTags; for (auto& kv : Tags) - { - std::vector& cctv = kv.second; - // We need to fix the order so that the send and recv processes match. - std::sort(cctv.begin(), cctv.end()); - } + { + std::vector& cctv = kv.second; + // We need to fix the order so that the send and recv processes match. + std::sort(cctv.begin(), cctv.end()); + } } } @@ -1053,10 +1053,10 @@ FabArrayBase::flushFB (bool no_assertion) const for (FBCacheIter it = er_it.first; it != er_it.second; ++it) { #ifdef AMREX_MEM_PROFILING - m_FBC_stats.bytes -= it->second->bytes(); + m_FBC_stats.bytes -= it->second->bytes(); #endif - m_FBC_stats.recordErase(it->second->m_nuse); - delete it->second; + m_FBC_stats.recordErase(it->second->m_nuse); + delete it->second; } m_TheFBCache.erase(er_it.first, er_it.second); } @@ -1066,8 +1066,8 @@ FabArrayBase::flushFBCache () { for (FBCacheIter it = m_TheFBCache.begin(); it != m_TheFBCache.end(); ++it) { - m_FBC_stats.recordErase(it->second->m_nuse); - delete it->second; + m_FBC_stats.recordErase(it->second->m_nuse); + delete it->second; } m_TheFBCache.clear(); #ifdef AMREX_MEM_PROFILING @@ -1085,18 +1085,18 @@ FabArrayBase::getFB (const IntVect& nghost, const Periodicity& period, std::pair er_it = m_TheFBCache.equal_range(m_bdkey); for (FBCacheIter it = er_it.first; it != er_it.second; ++it) { - if (it->second->m_typ == boxArray().ixType() && + if (it->second->m_typ == boxArray().ixType() && it->second->m_crse_ratio == boxArray().crseRatio() && - it->second->m_ngrow == nghost && - it->second->m_cross == cross && - it->second->m_multi_ghost== m_multi_ghost && - it->second->m_epo == enforce_periodicity_only && - it->second->m_period == period ) - { - ++(it->second->m_nuse); - m_FBC_stats.recordUse(); - return *(it->second); - } + it->second->m_ngrow == nghost && + it->second->m_cross == cross && + it->second->m_multi_ghost== m_multi_ghost && + it->second->m_epo == enforce_periodicity_only && + it->second->m_period == period ) + { + ++(it->second->m_nuse); + m_FBC_stats.recordUse(); + return *(it->second); + } } // Have to build a new one @@ -1856,17 +1856,17 @@ FabArrayBase::TheFPinfo (const FabArrayBase& srcfa, for (FPinfoCacheIter it = er_it.first; it != er_it.second; ++it) { - if (it->second->m_srcbdk == srckey && - it->second->m_dstbdk == dstkey && - it->second->m_dstdomain == dstdomain && - it->second->m_dstng == dstng && - it->second->m_dstdomain.ixType() == dstdomain.ixType() && - it->second->m_coarsener->doit(it->second->m_dstdomain) == coarsener.doit(dstdomain)) - { - ++(it->second->m_nuse); - m_FPinfo_stats.recordUse(); - return *(it->second); - } + if (it->second->m_srcbdk == srckey && + it->second->m_dstbdk == dstkey && + it->second->m_dstdomain == dstdomain && + it->second->m_dstng == dstng && + it->second->m_dstdomain.ixType() == dstdomain.ixType() && + it->second->m_coarsener->doit(it->second->m_dstdomain) == coarsener.doit(dstdomain)) + { + ++(it->second->m_nuse); + m_FPinfo_stats.recordUse(); + return *(it->second); + } } // Have to build a new one @@ -1877,14 +1877,14 @@ FabArrayBase::TheFPinfo (const FabArrayBase& srcfa, m_FPinfo_stats.bytes += new_fpc->bytes(); m_FPinfo_stats.bytes_hwm = std::max(m_FPinfo_stats.bytes_hwm, m_FPinfo_stats.bytes); #endif - + new_fpc->m_nuse = 1; m_FPinfo_stats.recordBuild(); m_FPinfo_stats.recordUse(); m_TheFillPatchCache.insert(er_it.second, FPinfoCache::value_type(dstkey,new_fpc)); if (srckey != dstkey) - m_TheFillPatchCache.insert( FPinfoCache::value_type(srckey,new_fpc)); + m_TheFillPatchCache.insert( FPinfoCache::value_type(srckey,new_fpc)); return *new_fpc; } @@ -1901,36 +1901,36 @@ FabArrayBase::flushFPinfo (bool no_assertion) for (FPinfoCacheIter it = er_it.first; it != er_it.second; ++it) { - const BDKey& srckey = it->second->m_srcbdk; - const BDKey& dstkey = it->second->m_dstbdk; + const BDKey& srckey = it->second->m_srcbdk; + const BDKey& dstkey = it->second->m_dstbdk; - BL_ASSERT((srckey==dstkey && srckey==m_bdkey) || - (m_bdkey==srckey) || (m_bdkey==dstkey)); + BL_ASSERT((srckey==dstkey && srckey==m_bdkey) || + (m_bdkey==srckey) || (m_bdkey==dstkey)); - if (srckey != dstkey) { - const BDKey& otherkey = (m_bdkey == srckey) ? dstkey : srckey; - std::pair o_er_it = m_TheFillPatchCache.equal_range(otherkey); + if (srckey != dstkey) { + const BDKey& otherkey = (m_bdkey == srckey) ? dstkey : srckey; + std::pair o_er_it = m_TheFillPatchCache.equal_range(otherkey); - for (FPinfoCacheIter oit = o_er_it.first; oit != o_er_it.second; ++oit) - { - if (it->second == oit->second) - others.push_back(oit); - } - } + for (FPinfoCacheIter oit = o_er_it.first; oit != o_er_it.second; ++oit) + { + if (it->second == oit->second) + others.push_back(oit); + } + } #ifdef AMREX_MEM_PROFILING - m_FPinfo_stats.bytes -= it->second->bytes(); + m_FPinfo_stats.bytes -= it->second->bytes(); #endif - m_FPinfo_stats.recordErase(it->second->m_nuse); - delete it->second; + m_FPinfo_stats.recordErase(it->second->m_nuse); + delete it->second; } - + m_TheFillPatchCache.erase(er_it.first, er_it.second); for (std::vector::iterator it = others.begin(), - End = others.end(); it != End; ++it) + End = others.end(); it != End; ++it) { - m_TheFillPatchCache.erase(*it); + m_TheFillPatchCache.erase(*it); } } @@ -1946,7 +1946,7 @@ FabArrayBase::CFinfo::CFinfo (const FabArrayBase& finefa, m_nuse (0) { BL_PROFILE("CFinfo::CFinfo()"); - + m_fine_domain = Domain(finegm, ng, include_periodic, include_physbndry); const BoxArray& fba = amrex::convert(finefa.boxArray(), IndexType::TheCellType()); @@ -2077,12 +2077,12 @@ FabArrayBase::Finalize () FabArrayBase::flushTileArrayCache(); if (ParallelDescriptor::IOProcessor() && amrex::system::verbose > 1) { - m_FA_stats.print(); - m_TAC_stats.print(); - m_FBC_stats.print(); - m_CPC_stats.print(); - m_FPinfo_stats.print(); - m_CFinfo_stats.print(); + m_FA_stats.print(); + m_TAC_stats.print(); + m_FBC_stats.print(); + m_CPC_stats.print(); + m_FPinfo_stats.print(); + m_CFinfo_stats.print(); } if (amrex::system::verbose > 1) { @@ -2097,7 +2097,7 @@ FabArrayBase::Finalize () m_CFinfo_stats = CacheStats("CrseFineCache"); m_BD_count.clear(); - + m_FA_stats = FabArrayStats(); the_fa_arena = nullptr; @@ -2105,7 +2105,7 @@ FabArrayBase::Finalize () initialized = false; } -const FabArrayBase::TileArray* +const FabArrayBase::TileArray* FabArrayBase::getTileArray (const IntVect& tilesize) const { TileArray* p; @@ -2117,23 +2117,23 @@ FabArrayBase::getTileArray (const IntVect& tilesize) const BL_ASSERT(getBDKey() == m_bdkey); const IntVect& crse_ratio = boxArray().crseRatio(); - p = &FabArrayBase::m_TheTileArrayCache[m_bdkey][std::pair(tilesize,crse_ratio)]; - if (p->nuse == -1) { - buildTileArray(tilesize, *p); - p->nuse = 0; - m_TAC_stats.recordBuild(); + p = &FabArrayBase::m_TheTileArrayCache[m_bdkey][std::pair(tilesize,crse_ratio)]; + if (p->nuse == -1) { + buildTileArray(tilesize, *p); + p->nuse = 0; + m_TAC_stats.recordBuild(); #ifdef AMREX_MEM_PROFILING - m_TAC_stats.bytes += p->bytes(); - m_TAC_stats.bytes_hwm = std::max(m_TAC_stats.bytes_hwm, - m_TAC_stats.bytes); + m_TAC_stats.bytes += p->bytes(); + m_TAC_stats.bytes_hwm = std::max(m_TAC_stats.bytes_hwm, + m_TAC_stats.bytes); #endif - } + } #ifdef AMREX_USE_OMP #pragma omp master #endif - { - ++(p->nuse); - m_TAC_stats.recordUse(); + { + ++(p->nuse); + m_TAC_stats.recordUse(); } } @@ -2148,88 +2148,88 @@ FabArrayBase::buildTileArray (const IntVect& tileSize, TileArray& ta) const if (tileSize == IntVect::TheZeroVector()) { - for (int i = 0; i < N; ++i) - { - if (isOwner(i)) - { - const int K = indexArray[i]; - const Box& bx = boxarray.getCellCenteredBox(K); - ta.indexMap.push_back(K); - ta.localIndexMap.push_back(i); - ta.localTileIndexMap.push_back(0); - ta.numLocalTiles.push_back(1); - ta.tileArray.push_back(bx); - } - } + for (int i = 0; i < N; ++i) + { + if (isOwner(i)) + { + const int K = indexArray[i]; + const Box& bx = boxarray.getCellCenteredBox(K); + ta.indexMap.push_back(K); + ta.localIndexMap.push_back(i); + ta.localTileIndexMap.push_back(0); + ta.numLocalTiles.push_back(1); + ta.tileArray.push_back(bx); + } + } } else { - std::vector local_idxs(N); - std::iota(std::begin(local_idxs), std::end(local_idxs), 0); + std::vector local_idxs(N); + std::iota(std::begin(local_idxs), std::end(local_idxs), 0); #if defined(BL_USE_TEAM) - const int nworkers = ParallelDescriptor::TeamSize(); - if (nworkers > 1) { - // reorder it so that each worker will be more likely to work on their own fabs - std::stable_sort(local_idxs.begin(), local_idxs.end(), [this](int i, int j) - { return this->distributionMap[this->indexArray[i]] - < this->distributionMap[this->indexArray[j]]; }); - } -#endif - - for (std::vector::const_iterator it = local_idxs.begin(); it != local_idxs.end(); ++it) - { - const int i = *it; // local index - const int K = indexArray[i]; // global index - const Box& bx = boxarray.getCellCenteredBox(K); + const int nworkers = ParallelDescriptor::TeamSize(); + if (nworkers > 1) { + // reorder it so that each worker will be more likely to work on their own fabs + std::stable_sort(local_idxs.begin(), local_idxs.end(), [this](int i, int j) + { return this->distributionMap[this->indexArray[i]] + < this->distributionMap[this->indexArray[j]]; }); + } +#endif + + for (std::vector::const_iterator it = local_idxs.begin(); it != local_idxs.end(); ++it) + { + const int i = *it; // local index + const int K = indexArray[i]; // global index + const Box& bx = boxarray.getCellCenteredBox(K); // // This must be consistent with ParticleContainer::getTileIndex function!!! // - - IntVect nt_in_fab, tsize, nleft; - int ntiles = 1; - for (int d=0; dsecond.begin(); - tai_it != tao_it->second.end(); ++tai_it) - { + if (tileSize == IntVect::TheZeroVector()) + { + for (TAMap::const_iterator tai_it = tao_it->second.begin(); + tai_it != tao_it->second.end(); ++tai_it) + { #ifdef AMREX_MEM_PROFILING - m_TAC_stats.bytes -= tai_it->second.bytes(); -#endif - m_TAC_stats.recordErase(tai_it->second.nuse); - } - tao.erase(tao_it); - } - else - { - TAMap& tai = tao_it->second; + m_TAC_stats.bytes -= tai_it->second.bytes(); +#endif + m_TAC_stats.recordErase(tai_it->second.nuse); + } + tao.erase(tao_it); + } + else + { + TAMap& tai = tao_it->second; const IntVect& crse_ratio = boxArray().crseRatio(); - TAMap::iterator tai_it = tai.find(std::pair(tileSize,crse_ratio)); - if (tai_it != tai.end()) { + TAMap::iterator tai_it = tai.find(std::pair(tileSize,crse_ratio)); + if (tai_it != tai.end()) { #ifdef AMREX_MEM_PROFILING - m_TAC_stats.bytes -= tai_it->second.bytes(); -#endif - m_TAC_stats.recordErase(tai_it->second.nuse); - tai.erase(tai_it); - } - } + m_TAC_stats.bytes -= tai_it->second.bytes(); +#endif + m_TAC_stats.recordErase(tai_it->second.nuse); + tai.erase(tai_it); + } + } } } @@ -2275,13 +2275,13 @@ void FabArrayBase::flushTileArrayCache () { for (TACache::const_iterator tao_it = m_TheTileArrayCache.begin(); - tao_it != m_TheTileArrayCache.end(); ++tao_it) + tao_it != m_TheTileArrayCache.end(); ++tao_it) { - for (TAMap::const_iterator tai_it = tao_it->second.begin(); - tai_it != tao_it->second.end(); ++tai_it) - { - m_TAC_stats.recordErase(tai_it->second.nuse); - } + for (TAMap::const_iterator tai_it = tao_it->second.begin(); + tai_it != tao_it->second.end(); ++tai_it) + { + m_TAC_stats.recordErase(tai_it->second.nuse); + } } m_TheTileArrayCache.clear(); #ifdef AMREX_MEM_PROFILING @@ -2295,14 +2295,14 @@ FabArrayBase::clearThisBD (bool no_assertion) BL_ASSERT(boxarray.empty() || no_assertion || getBDKey() == m_bdkey); std::map::iterator cnt_it = m_BD_count.find(m_bdkey); - if (cnt_it != m_BD_count.end()) + if (cnt_it != m_BD_count.end()) { --(cnt_it->second); - if (cnt_it->second == 0) + if (cnt_it->second == 0) { m_BD_count.erase(cnt_it); - - // Since this is the last one built with these BoxArray + + // Since this is the last one built with these BoxArray // and DistributionMapping, erase it from caches. flushTileArray(IntVect::TheZeroVector(), no_assertion); flushFPinfo(no_assertion); @@ -2322,9 +2322,9 @@ FabArrayBase::addThisBD () m_bdkey = getBDKey(); int cnt = ++(m_BD_count[m_bdkey]); if (cnt == 1) { // new one - m_FA_stats.recordMaxNumBoxArrays(m_BD_count.size()); + m_FA_stats.recordMaxNumBoxArrays(m_BD_count.size()); } else { - m_FA_stats.recordMaxNumBAUse(cnt); + m_FA_stats.recordMaxNumBAUse(cnt); } } @@ -2332,8 +2332,8 @@ void FabArrayBase::updateBDKey () { if (getBDKey() != m_bdkey) { - clearThisBD(true); - addThisBD(); + clearThisBD(true); + addThisBD(); } } @@ -2341,12 +2341,12 @@ FabArrayBase::updateBDKey () bool FabArrayBase::CheckRcvStats(Vector& recv_stats, - const Vector& recv_size, - int tag) + const Vector& recv_size, + int tag) { for (int i = 0, n = recv_size.size(); i < n; ++i) { - if (recv_size[i] > 0) { - std::size_t count = 0; + if (recv_size[i] > 0) { + std::size_t count = 0; int tmp_count = 0; const int comm_data_type = ParallelDescriptor::select_comm_data_type(recv_size[i]); @@ -2369,7 +2369,7 @@ FabArrayBase::CheckRcvStats(Vector& recv_stats, amrex::Abort("TODO: message size is too big"); } - if (count != recv_size[i]) { + if (count != recv_size[i]) { if (amrex::Verbose()) { amrex::AllPrint() << "ERROR: Proc. " << ParallelContext::MyProcSub() << " received " << count << " bytes of data from Proc. " @@ -2380,8 +2380,8 @@ FabArrayBase::CheckRcvStats(Vector& recv_stats, << " with tag " << tag << "\n"; } return false; - } - } + } + } } return true; } diff --git a/Src/Base/AMReX_FabArrayCommI.H b/Src/Base/AMReX_FabArrayCommI.H index 8b3a8325674..70e49162913 100644 --- a/Src/Base/AMReX_FabArrayCommI.H +++ b/Src/Base/AMReX_FabArrayCommI.H @@ -7,7 +7,7 @@ template ::value,int>::type Z> void FabArray::FBEP_nowait (int scomp, int ncomp, const IntVect& nghost, const Periodicity& period, bool cross, - bool enforce_periodicity_only) + bool enforce_periodicity_only) { fb_cross = cross; fb_epo = enforce_periodicity_only; @@ -20,9 +20,9 @@ FabArray::FBEP_nowait (int scomp, int ncomp, const IntVect& nghost, bool work_to_do; if (enforce_periodicity_only) { - work_to_do = period.isAnyPeriodic(); + work_to_do = period.isAnyPeriodic(); } else { - work_to_do = nghost.max() > 0; + work_to_do = nghost.max() > 0; } if (!work_to_do) return; @@ -33,7 +33,7 @@ FabArray::FBEP_nowait (int scomp, int ncomp, const IntVect& nghost, // // There can only be local work to do. // - int N_locs = (*TheFB.m_LocTags).size(); + int N_locs = (*TheFB.m_LocTags).size(); if (N_locs == 0) return; #ifdef AMREX_USE_GPU if (Gpu::inLaunchRegion()) @@ -149,7 +149,7 @@ FabArray::FBEP_nowait (int scomp, int ncomp, const IntVect& nghost, #endif { FB_local_copy_cpu(TheFB, scomp, ncomp); - } + } FillBoundary_test(); } @@ -177,7 +177,7 @@ FabArray::FillBoundary_finish () if (N_rcvs > 0) { Vector recv_cctc(N_rcvs,nullptr); - for (int k = 0; k < N_rcvs; k++) + for (int k = 0; k < N_rcvs; k++) { if (fb_recv_size[k] > 0) { @@ -297,7 +297,7 @@ FabArray::ParallelCopy_nowait (const FabArray& src, if ((src.boxArray().ixType().cellCentered() || op == FabArrayBase::COPY) && (boxarray == src.boxarray && distributionMap == src.distributionMap) && - snghost == IntVect::TheZeroVector() && + snghost == IntVect::TheZeroVector() && dnghost == IntVect::TheZeroVector() && !period.isAnyPeriodic()) { @@ -314,21 +314,21 @@ FabArray::ParallelCopy_nowait (const FabArray& src, const Box& bx = fai.tilebox(); // avoid self copy or plus - if (this != &src) { + if (this != &src) { auto const sfab = src.array(fai); auto dfab = this->array(fai); - if (op == FabArrayBase::COPY) { + if (op == FabArrayBase::COPY) { AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, ncomp, i, j, k, n, { dfab(i,j,k,dcomp+n) = sfab(i,j,k,scomp+n); }); - } else { + } else { AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, ncomp, i, j, k, n, { dfab(i,j,k,dcomp+n) += sfab(i,j,k,scomp+n); }); } - } + } } pc_finished = true; @@ -343,7 +343,7 @@ FabArray::ParallelCopy_nowait (const FabArray& src, // // There can only be local work to do. // - int N_locs = (*thecpc.m_LocTags).size(); + int N_locs = (*thecpc.m_LocTags).size(); if (N_locs == 0) return; #ifdef AMREX_USE_GPU if (Gpu::inLaunchRegion()) @@ -402,22 +402,22 @@ FabArray::ParallelCopy_nowait (const FabArray& src, pc_the_recv_data = nullptr; pc_actual_n_rcvs = 0; - if (N_rcvs > 0) { + if (N_rcvs > 0) { PostRcvs(*thecpc.m_RcvTags, pc_the_recv_data, pc_recv_data, pc_recv_size, pc_recv_from, pc_recv_reqs, pc_NC, pc_tag); pc_actual_n_rcvs = N_rcvs - std::count(pc_recv_size.begin(), pc_recv_size.end(), 0); - } - - // - // Post send's - // - Vector send_data; - Vector send_size; - Vector send_rank; - Vector send_cctc; - - if (N_snds > 0) - { + } + + // + // Post send's + // + Vector send_data; + Vector send_size; + Vector send_rank; + Vector send_cctc; + + if (N_snds > 0) + { src.PrepareSendBuffers(*thecpc.m_SndTags, pc_the_send_data, send_data, send_size, send_rank, pc_send_reqs, send_cctc, pc_NC); @@ -434,13 +434,13 @@ FabArray::ParallelCopy_nowait (const FabArray& src, AMREX_ASSERT(pc_send_reqs.size() == N_snds); FabArray::PostSnds(send_data, send_size, send_rank, pc_send_reqs, pc_tag); - } + } // // Do the local work. Hope for a bit of communication/computation overlap. // if (N_locs > 0) - { + { #ifdef AMREX_USE_GPU if (Gpu::inLaunchRegion()) { @@ -534,7 +534,7 @@ FabArray::ParallelCopy_finish () if (! thecpc.m_SndTags->empty()) { Vector stats(pc_send_reqs.size()); ParallelDescriptor::Waitall(pc_send_reqs, stats); - } + } amrex::The_FA_Arena()->free(pc_the_send_data); pc_the_send_data = nullptr; } @@ -547,11 +547,11 @@ FabArray::ParallelCopy_finish () template void FabArray::copyTo (FAB& dest, - const Box& subbox, - int scomp, - int dcomp, - int ncomp, - int nghost) const + const Box& subbox, + int scomp, + int dcomp, + int ncomp, + int nghost) const { BL_PROFILE("FabArray::copy(fab)"); @@ -562,9 +562,9 @@ FabArray::copyTo (FAB& dest, { for (int j = 0, N = size(); j < N; ++j) { - const Box& bx = amrex::grow(boxarray[j],nghost); - const Box& destbox = bx & subbox; - if (destbox.ok()) + const Box& bx = amrex::grow(boxarray[j],nghost); + const Box& destbox = bx & subbox; + if (destbox.ok()) { dest.template copy(get(j),destbox,scomp,destbox,dcomp,ncomp); } @@ -578,10 +578,10 @@ FabArray::copyTo (FAB& dest, // #ifdef AMREX_DEBUG { - BoxCommHelper bch(subbox); - ParallelDescriptor::Bcast(bch.data(), bch.size(), 0, ParallelContext::CommunicatorSub()); - const Box& bx0 = bch.make_box(); - BL_ASSERT(subbox == bx0); + BoxCommHelper bch(subbox); + ParallelDescriptor::Bcast(bch.data(), bch.size(), 0, ParallelContext::CommunicatorSub()); + const Box& bx0 = bch.make_box(); + BL_ASSERT(subbox == bx0); } #endif @@ -592,23 +592,23 @@ FabArray::copyTo (FAB& dest, for (int j = 0, M = isects.size(); j < M; ++j) { - const int k = isects[j].first; - const Box& bx = isects[j].second; + const int k = isects[j].first; + const Box& bx = isects[j].second; - ovlp.resize(bx,ncomp); + ovlp.resize(bx,ncomp); - if (ParallelDescriptor::MyProc() == distributionMap[k]) - { - ovlp.template copy(get(k),bx,scomp,bx,0,ncomp); - } + if (ParallelDescriptor::MyProc() == distributionMap[k]) + { + ovlp.template copy(get(k),bx,scomp,bx,0,ncomp); + } - const int N = bx.numPts()*ncomp; + const int N = bx.numPts()*ncomp; - ParallelDescriptor::Bcast(ovlp.dataPtr(),N, + ParallelDescriptor::Bcast(ovlp.dataPtr(),N, ParallelContext::global_to_local_rank(distributionMap[k]), ParallelContext::CommunicatorSub()); - dest.template copy(ovlp,bx,0,bx,dcomp,ncomp); + dest.template copy(ovlp,bx,0,bx,dcomp,ncomp); } } diff --git a/Src/Base/AMReX_FabArrayUtility.H b/Src/Base/AMReX_FabArrayUtility.H index 643d2177707..fffa116acc3 100644 --- a/Src/Base/AMReX_FabArrayUtility.H +++ b/Src/Base/AMReX_FabArrayUtility.H @@ -1427,21 +1427,21 @@ printCell (FabArray const& mf, const IntVect& cell, int comp = -1, { const Box& bx = amrex::grow(mfi.validbox(), ng); if (bx.contains(cell)) { - if (comp >= 0) { + if (comp >= 0) { amrex::AllPrint().SetPrecision(17) << " At cell " << cell << " in Box " << bx << ": " << mf[mfi](cell, comp) << std::endl; - } else { + } else { std::ostringstream ss; ss.precision(17); const int ncomp = mf.nComp(); for (int i = 0; i < ncomp-1; ++i) - { + { ss << mf[mfi](cell,i) << ", "; - } + } ss << mf[mfi](cell,ncomp-1); amrex::AllPrint() << " At cell " << cell << " in Box " << bx << ": " << ss.str() << std::endl; - } + } } } } @@ -1674,7 +1674,7 @@ OverrideSync (FabArray & fa, FabArray const& msk, const Periodicity& BL_PROFILE("OverrideSync()"); if (fa.ixType().cellCentered()) return; - + const int ncomp = fa.nComp(); #ifdef AMREX_USE_OMP @@ -1690,7 +1690,7 @@ OverrideSync (FabArray & fa, FabArray const& msk, const Periodicity& if (!ifab(i,j,k)) fab(i,j,k,n) = 0; }); } - + FabArray tmpmf(fa.boxArray(), fa.DistributionMap(), ncomp, 0, MFInfo(), fa.Factory()); tmpmf.setVal(0); diff --git a/Src/Base/AMReX_FabConv.cpp b/Src/Base/AMReX_FabConv.cpp index 0f750aa6b28..b59dc15e037 100644 --- a/Src/Base/AMReX_FabConv.cpp +++ b/Src/Base/AMReX_FabConv.cpp @@ -523,11 +523,11 @@ permute_real_word_order (void* out, const void* in, Long nitems, const int* outord, - const int* inord, + const int* inord, int REALSIZE) { // BL_PROFILE("permute_real_word_order"); - + char* pin = (char*) in; char* pout = (char*) out; @@ -972,7 +972,7 @@ RealDescriptor::convertToNativeFormat (Real* out, if(bAlwaysFixDenormals) { PD_fixdenormals(out, nitems, FPC::NativeRealDescriptor().format(), - FPC::NativeRealDescriptor().order()); + FPC::NativeRealDescriptor().order()); } } @@ -1005,7 +1005,7 @@ RealDescriptor::convertToNativeFormat (Real* out, if(bAlwaysFixDenormals) { PD_fixdenormals(out, get, FPC::NativeRealDescriptor().format(), - FPC::NativeRealDescriptor().order()); + FPC::NativeRealDescriptor().order()); } nitems -= get; out += get; @@ -1193,7 +1193,7 @@ RealDescriptor::convertToNativeFloatFormat (float* out, if(bAlwaysFixDenormals) { PD_fixdenormals(out, get, FPC::Native32RealDescriptor().format(), - FPC::Native32RealDescriptor().order()); + FPC::Native32RealDescriptor().order()); } nitems -= get; out += get; @@ -1235,7 +1235,7 @@ RealDescriptor::convertToNativeDoubleFormat (double* out, if(bAlwaysFixDenormals) { PD_fixdenormals(out, get, FPC::Native64RealDescriptor().format(), - FPC::Native64RealDescriptor().order()); + FPC::Native64RealDescriptor().order()); } nitems -= get; out += get; diff --git a/Src/Base/AMReX_FileSystem.cpp b/Src/Base/AMReX_FileSystem.cpp index 2d47456137c..5caf8de26c5 100644 --- a/Src/Base/AMReX_FileSystem.cpp +++ b/Src/Base/AMReX_FileSystem.cpp @@ -20,7 +20,7 @@ CreateDirectories (std::string const& p, mode_t /*mode*/, bool verbose) amrex::AllPrint() << "amrex::UtilCreateDirectory failed to create " << p << ": " << ec.message() << std::endl; } - return !ec; + return !ec; } bool diff --git a/Src/Base/AMReX_Geometry.H b/Src/Base/AMReX_Geometry.H index 64e24d50d14..ecb8d0bc520 100644 --- a/Src/Base/AMReX_Geometry.H +++ b/Src/Base/AMReX_Geometry.H @@ -75,9 +75,9 @@ public: Geometry () noexcept; //! Constructor taking the rectangular domain. explicit Geometry (const Box& dom, - const RealBox* rb = nullptr, - int coord = -1, - int const* is_per = nullptr) noexcept; + const RealBox* rb = nullptr, + int coord = -1, + int const* is_per = nullptr) noexcept; Geometry (const Box& dom, const RealBox& rb, int coord, Array const& is_per) noexcept; @@ -172,7 +172,7 @@ public: */ void GetFaceArea (MultiFab& area, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, int dir, int grow) const; //! Fill the pre-built multifab with area @@ -210,14 +210,14 @@ public: int period (int dir) const noexcept { BL_ASSERT(is_periodic[dir]); return domain.length(dir); } Periodicity periodicity () const noexcept { - return Periodicity(IntVect(AMREX_D_DECL(domain.length(0) * is_periodic[0], - domain.length(1) * is_periodic[1], - domain.length(2) * is_periodic[2]))); + return Periodicity(IntVect(AMREX_D_DECL(domain.length(0) * is_periodic[0], + domain.length(1) * is_periodic[1], + domain.length(2) * is_periodic[2]))); } Periodicity periodicity (const Box& b) const noexcept { AMREX_ASSERT(b.cellCentered()); - return Periodicity(IntVect(AMREX_D_DECL(b.length(0) * is_periodic[0], + return Periodicity(IntVect(AMREX_D_DECL(b.length(0) * is_periodic[0], b.length(1) * is_periodic[1], b.length(2) * is_periodic[2]))); } diff --git a/Src/Base/AMReX_Geometry.cpp b/Src/Base/AMReX_Geometry.cpp index 0254a94f4d3..eca4c396089 100644 --- a/Src/Base/AMReX_Geometry.cpp +++ b/Src/Base/AMReX_Geometry.cpp @@ -118,11 +118,11 @@ Geometry::Setup (const RealBox* rb, int coord, int const* isper) noexcept ParmParse pp("geometry"); if (coord >=0 && coord <= 2) { - gg->SetCoord( (CoordType) coord ); + gg->SetCoord( (CoordType) coord ); } else { coord = 0; // default is Cartesian coordinates pp.query("coord_sys",coord); - gg->SetCoord( (CoordType) coord ); + gg->SetCoord( (CoordType) coord ); } if (rb == nullptr) { @@ -189,7 +189,7 @@ Geometry::ResetDefaultCoord (int coord) noexcept void Geometry::GetVolume (MultiFab& vol, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, int ngrow) const { vol.define(grds,dm,1,ngrow,MFInfo(),FArrayBoxFactory()); @@ -204,7 +204,7 @@ Geometry::GetVolume (MultiFab& vol) const #endif for (MFIter mfi(vol,TilingIfNotGPU()); mfi.isValid(); ++mfi) { - CoordSys::SetVolume(vol[mfi], mfi.growntilebox()); + CoordSys::SetVolume(vol[mfi], mfi.growntilebox()); } } @@ -220,7 +220,7 @@ Geometry::GetVolume (FArrayBox& vol, #if (AMREX_SPACEDIM <= 2) void Geometry::GetDLogA (MultiFab& dloga, - const BoxArray& grds, + const BoxArray& grds, const DistributionMapping& dm, int dir, int ngrow) const @@ -231,7 +231,7 @@ Geometry::GetDLogA (MultiFab& dloga, #endif for (MFIter mfi(dloga,TilingIfNotGPU()); mfi.isValid(); ++mfi) { - CoordSys::SetDLogA(dloga[mfi], mfi.growntilebox(), dir); + CoordSys::SetDLogA(dloga[mfi], mfi.growntilebox(), dir); } } #endif @@ -239,7 +239,7 @@ Geometry::GetDLogA (MultiFab& dloga, void Geometry::GetFaceArea (MultiFab& area, const BoxArray& grds, - const DistributionMapping& dm, + const DistributionMapping& dm, int dir, int ngrow) const { @@ -259,7 +259,7 @@ Geometry::GetFaceArea (MultiFab& area, #endif for (MFIter mfi(area,TilingIfNotGPU()); mfi.isValid(); ++mfi) { - CoordSys::SetFaceArea(area[mfi],mfi.growntilebox(),dir); + CoordSys::SetFaceArea(area[mfi],mfi.growntilebox(),dir); } } @@ -275,7 +275,7 @@ Geometry::GetFaceArea (FArrayBox& area, void Geometry::periodicShift (const Box& target, - const Box& src, + const Box& src, Vector& out) const noexcept { out.resize(0); diff --git a/Src/Base/AMReX_GpuAllocators.H b/Src/Base/AMReX_GpuAllocators.H index 55f2dbbd0a3..1ea7ff41868 100644 --- a/Src/Base/AMReX_GpuAllocators.H +++ b/Src/Base/AMReX_GpuAllocators.H @@ -40,9 +40,9 @@ namespace amrex { inline value_type* allocate(std::size_t n) { - value_type* result = nullptr; + value_type* result = nullptr; result = (value_type*) The_Arena()->alloc(n * sizeof(T)); - return result; + return result; } inline void deallocate(value_type* ptr, std::size_t) @@ -61,9 +61,9 @@ namespace amrex { inline value_type* allocate(std::size_t n) { - value_type* result = nullptr; + value_type* result = nullptr; result = (value_type*) The_Device_Arena()->alloc(n * sizeof(T)); - return result; + return result; } inline void deallocate(value_type* ptr, std::size_t) @@ -82,9 +82,9 @@ namespace amrex { inline value_type* allocate(std::size_t n) { - value_type* result = nullptr; + value_type* result = nullptr; result = (value_type*) The_Pinned_Arena()->alloc(n * sizeof(T)); - return result; + return result; } inline void deallocate(value_type* ptr, std::size_t) @@ -103,9 +103,9 @@ namespace amrex { inline value_type* allocate(std::size_t n) { - value_type* result = nullptr; + value_type* result = nullptr; result = (value_type*) The_Managed_Arena()->alloc(n * sizeof(T)); - return result; + return result; } inline void deallocate(value_type* ptr, std::size_t) @@ -125,7 +125,7 @@ namespace amrex { inline value_type* allocate(std::size_t n) { - value_type* result = nullptr; + value_type* result = nullptr; if (m_use_gpu_aware_mpi) { result = (value_type*) The_Arena()->alloc(n * sizeof(T)); @@ -134,7 +134,7 @@ namespace amrex { { result = (value_type*) The_Pinned_Arena()->alloc(n * sizeof(T)); } - return result; + return result; } inline void deallocate(value_type* ptr, std::size_t) diff --git a/Src/Base/AMReX_GpuComplex.H b/Src/Base/AMReX_GpuComplex.H index fa59226b259..022cf1f580f 100644 --- a/Src/Base/AMReX_GpuComplex.H +++ b/Src/Base/AMReX_GpuComplex.H @@ -10,8 +10,8 @@ namespace amrex { template struct GpuComplex; -template -AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +template +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T norm (const GpuComplex& a_z) noexcept; /** @@ -19,9 +19,9 @@ T norm (const GpuComplex& a_z) noexcept; * work in device code with Cuda yet. * * Should be bit-wise compatible with std::complex. - */ + */ template -struct GpuComplex +struct GpuComplex { using value_type = T; @@ -29,26 +29,26 @@ struct GpuComplex /** * \brief Construct a complex number given the real and imaginary part. - */ - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE constexpr GpuComplex (const T& a_r = T(), const T& a_i = T()) noexcept : m_real(a_r), m_imag(a_i) {} /** * \brief Return the real part. - */ + */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE constexpr T real () const noexcept { return m_real; } /** * \brief Return the imaginary part. - */ + */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE constexpr T imag () const noexcept { return m_imag; } /** * \brief Add a real number to this complex number. - */ + */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator+= (const T& a_t) noexcept { @@ -58,7 +58,7 @@ struct GpuComplex /** * \brief Subtract a real number from this complex number. - */ + */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator-= (const T& a_t) noexcept { @@ -68,7 +68,7 @@ struct GpuComplex /** * \brief Multiply this complex number by a real. - */ + */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator*= (const T& a_t) noexcept { @@ -79,7 +79,7 @@ struct GpuComplex /** * \brief Divide this complex number by a real. - */ + */ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator/= (const T& a_t) noexcept { @@ -90,7 +90,7 @@ struct GpuComplex /** * \brief Add another complex number to this one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator+= (const GpuComplex& a_z) noexcept @@ -102,7 +102,7 @@ struct GpuComplex /** * \brief Subtract another complex number from this one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator-= (const GpuComplex& a_z) noexcept @@ -114,7 +114,7 @@ struct GpuComplex /** * \brief Multiply this complex number by another one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator*= (const GpuComplex& a_z) noexcept @@ -127,7 +127,7 @@ struct GpuComplex /** * \brief Divide this complex number by another one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex& operator/= (const GpuComplex& a_z) noexcept @@ -141,7 +141,7 @@ struct GpuComplex /** * \brief Print this complex number to an ostream. - */ + */ template friend std::ostream & operator<< (std::ostream &out, const GpuComplex& c); }; @@ -152,25 +152,25 @@ std::ostream& operator<< (std::ostream& out, const GpuComplex& c) out << c.m_real; out << " + " << c.m_imag << "i"; return out; -} +} /** * \brief Identity operation on a complex number. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator+ (const GpuComplex& a_x) { return a_x; } /** * \brief Negate a complex number. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator- (const GpuComplex& a_x) { return GpuComplex(-a_x.real(), -a_x.imag()); } /** * \brief Subtract two complex numbers. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator- (const GpuComplex& a_x, const GpuComplex& a_y) noexcept @@ -182,7 +182,7 @@ GpuComplex operator- (const GpuComplex& a_x, const GpuComplex& a_y) noe /** * \brief Subtract a real number from a complex one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator- (const GpuComplex& a_x, const T& a_y) noexcept @@ -194,7 +194,7 @@ GpuComplex operator- (const GpuComplex& a_x, const T& a_y) noexcept /** * \brief Subtract a complex number from a real one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator- (const T& a_x, const GpuComplex& a_y) noexcept @@ -206,7 +206,7 @@ GpuComplex operator- (const T& a_x, const GpuComplex& a_y) noexcept /** * \brief Add two complex numbers. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator+ (const GpuComplex& a_x, const GpuComplex& a_y) noexcept @@ -218,7 +218,7 @@ GpuComplex operator+ (const GpuComplex& a_x, const GpuComplex& a_y) noe /** * \brief Add a real number to a complex one - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator+ (const GpuComplex& a_x, const T& a_y) noexcept @@ -230,7 +230,7 @@ GpuComplex operator+ (const GpuComplex& a_x, const T& a_y) noexcept /** * \brief Add a complex number to a real one - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator+ (const T& a_x, const GpuComplex& a_y) noexcept @@ -242,7 +242,7 @@ GpuComplex operator+ (const T& a_x, const GpuComplex& a_y) noexcept /** * \brief Multiply two complex numbers. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator* (const GpuComplex& a_x, const GpuComplex& a_y) noexcept @@ -254,7 +254,7 @@ GpuComplex operator* (const GpuComplex& a_x, const GpuComplex& a_y) noe /** * \brief Multiply a complex number by a real one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator* (const GpuComplex& a_x, const T& a_y) noexcept @@ -266,7 +266,7 @@ GpuComplex operator* (const GpuComplex& a_x, const T& a_y) noexcept /** * \brief Multiply a real number by a complex one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator* (const T& a_x, const GpuComplex& a_y) noexcept @@ -278,7 +278,7 @@ GpuComplex operator* (const T& a_x, const GpuComplex& a_y) noexcept /** * \brief Divide a complex number by another one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator/ (const GpuComplex& a_x, const GpuComplex& a_y) noexcept @@ -290,7 +290,7 @@ GpuComplex operator/ (const GpuComplex& a_x, const GpuComplex& a_y) noe /** * \brief Divide a complex number by a real. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator/ (const GpuComplex& a_x, const T& a_y) noexcept @@ -302,7 +302,7 @@ GpuComplex operator/ (const GpuComplex& a_x, const T& a_y) noexcept /** * \brief Divide a real number by a complex one. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex operator/ (const T& a_x, const GpuComplex& a_y) noexcept @@ -314,27 +314,27 @@ GpuComplex operator/ (const T& a_x, const GpuComplex& a_y) noexcept /** * \brief Return a complex number given its polar representation. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex polar (const T& a_r, const T& a_theta) noexcept { return GpuComplex(a_r * std::cos(a_theta), a_r * std::sin(a_theta)); -} +} /** * \brief Complex expotential function. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex exp (const GpuComplex& a_z) noexcept { return polar(std::exp(a_z.real()), a_z.imag()); -} +} /** * \brief Return the norm (magnitude squared) of a complex number - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T norm (const GpuComplex& a_z) noexcept @@ -342,11 +342,11 @@ T norm (const GpuComplex& a_z) noexcept const T x = a_z.real(); const T y = a_z.imag(); return x * x + y * y; -} +} /** * \brief Return the absolute value of a complex number - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T abs (const GpuComplex& a_z) noexcept @@ -364,7 +364,7 @@ T abs (const GpuComplex& a_z) noexcept /** * \brief Return the square root of a complex number - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex sqrt (const GpuComplex& a_z) noexcept @@ -389,7 +389,7 @@ GpuComplex sqrt (const GpuComplex& a_z) noexcept /** * \brief Return the angle of a complex number's polar representation. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE T arg (const GpuComplex& a_z) noexcept @@ -399,7 +399,7 @@ T arg (const GpuComplex& a_z) noexcept /** * \brief Complex natural logarithm function. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex log (const GpuComplex& a_z) noexcept @@ -409,14 +409,14 @@ GpuComplex log (const GpuComplex& a_z) noexcept /** * \brief Raise a complex number to a (real) power. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex pow (const GpuComplex& a_z, const T& a_y) noexcept { if (a_z.imag() == T() && a_z.real() > T()) return std::pow(a_z.real(), a_y); - + GpuComplex t = amrex::log(a_z); return amrex::polar(std::exp(a_y * t.real()), a_y * t.imag()); } @@ -428,21 +428,21 @@ namespace detail GpuComplex complex_pow_unsigned (GpuComplex a_z, unsigned a_n) { GpuComplex y = a_n % 2 ? a_z : GpuComplex(1); - + while (a_n >>= 1) { a_z *= a_z; if (a_n % 2) y *= a_z; } - + return y; } } /** * \brief Raise a complex number to an integer power. - */ + */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE GpuComplex pow (const GpuComplex& a_z, int a_n) noexcept @@ -451,7 +451,7 @@ GpuComplex pow (const GpuComplex& a_z, int a_n) noexcept ? GpuComplex(1) / detail::complex_pow_unsigned(a_z, (unsigned)(-a_n)) : detail::complex_pow_unsigned(a_z, a_n); } - + } #endif diff --git a/Src/Base/AMReX_GpuContainers.H b/Src/Base/AMReX_GpuContainers.H index 01fc7c10956..3227967a0c0 100644 --- a/Src/Base/AMReX_GpuContainers.H +++ b/Src/Base/AMReX_GpuContainers.H @@ -58,15 +58,15 @@ namespace Gpu { using PolymorphicVector = PODVector >; /** - * \brief This is identical to ManagedVector. The ManagedDeviceVector + * \brief This is identical to ManagedVector. The ManagedDeviceVector * form is deprecated and will be removed in a future release. * */ template using ManagedDeviceVector = PODVector >; - + #else - //! When Cuda is off, all these containers revert to amrex::Vector. + //! When Cuda is off, all these containers revert to amrex::Vector. template using DeviceVector = PODVector; @@ -108,7 +108,7 @@ namespace Gpu { * \param result Where in the output to start writing * * Example usage: - * + * * Gpu::copy(Gpu::hostToDevice, a.begin(), a.end(), b.begin()); */ template @@ -141,7 +141,7 @@ namespace Gpu { * \param result Where in the output to start writing * * Example usage: - * + * * Gpu::copy(Gpu::deviceToHost, a.begin(), a.end(), b.begin()); */ template @@ -152,7 +152,7 @@ namespace Gpu { "Can only copy trivially copyable types"); auto size = std::distance(begin, end); if (size == 0) return; -#ifdef AMREX_USE_GPU +#ifdef AMREX_USE_GPU dtoh_memcpy(&(*result), &(*begin), size*sizeof(value_type)); #else std::memcpy(&(*result), &(*begin), size*sizeof(value_type)); @@ -174,7 +174,7 @@ namespace Gpu { * \param result Where in the output to start writing * * Example usage: - * + * * Gpu::copy(Gpu::deviceToDevice, a.begin(), a.end(), b.begin()); */ template @@ -197,7 +197,7 @@ namespace Gpu { * memcpy, so it assumes contiguous storage. The amrex-provided containers * like Gpu::HostVector, Gpu::DeviceVector, etc. meet this requirement. * - * This version is asynchronous - CPU execution will continue, whether or not the + * This version is asynchronous - CPU execution will continue, whether or not the * copy is finished. * * \tparam InIter The input iterator type @@ -208,9 +208,9 @@ namespace Gpu { * \param result Where in the output to start writing * * Example usage: - * + * * Gpu::copy(Gpu::hostToDevice, a.begin(), a.end(), b.begin()); - */ + */ template void copyAsync (HostToDevice, InIter begin, InIter end, OutIter result) noexcept { @@ -223,7 +223,7 @@ namespace Gpu { htod_memcpy_async(&(*result), &(*begin), size*sizeof(value_type)); #else std::memcpy(&(*result), &(*begin), size*sizeof(value_type)); -#endif +#endif } /** @@ -231,7 +231,7 @@ namespace Gpu { * memcpy, so it assumes contiguous storage. The amrex-provided containers * like Gpu::HostVector, Gpu::DeviceVector, etc. meet this requirement. * - * This version is asynchronous - CPU execution will continue, whether or not the + * This version is asynchronous - CPU execution will continue, whether or not the * copy is finished. * * \tparam InIter The input iterator type @@ -242,7 +242,7 @@ namespace Gpu { * \param result Where in the output to start writing * * Example usage: - * + * * Gpu::copy(Gpu::deviceToHost, a.begin(), a.end(), b.begin()); */ template @@ -253,11 +253,11 @@ namespace Gpu { "Can only copy trivially copyable types"); auto size = std::distance(begin, end); if (size == 0) return; -#ifdef AMREX_USE_GPU +#ifdef AMREX_USE_GPU dtoh_memcpy_async(&(*result), &(*begin), size*sizeof(value_type)); #else std::memcpy(&(*result), &(*begin), size*sizeof(value_type)); -#endif +#endif } /** @@ -265,7 +265,7 @@ namespace Gpu { * memcpy, so it assumes contiguous storage. The amrex-provided containers * like Gpu::HostVector, Gpu::DeviceVector, etc. meet this requirement. * - * This version is asynchronous - CPU execution will continue, whether or not the + * This version is asynchronous - CPU execution will continue, whether or not the * copy is finished. * * \tparam InIter The input iterator type @@ -276,7 +276,7 @@ namespace Gpu { * \param result Where in the output to start writing * * Example usage: - * + * * Gpu::copy(Gpu::deviceToDevice, a.begin(), a.end(), b.begin()); */ template @@ -291,7 +291,7 @@ namespace Gpu { dtod_memcpy_async(&(*result), &(*begin), size*sizeof(value_type)); #else std::memcpy(&(*result), &(*begin), size*sizeof(value_type)); -#endif +#endif } }} diff --git a/Src/Base/AMReX_GpuDevice.cpp b/Src/Base/AMReX_GpuDevice.cpp index ac5c299c3ae..11d713fc862 100644 --- a/Src/Base/AMReX_GpuDevice.cpp +++ b/Src/Base/AMReX_GpuDevice.cpp @@ -318,19 +318,19 @@ Device::Initialize () amrex::Print() << "CUDA initialized with 1 GPU per MPI rank; " << num_devices_used << " GPU(s) used in total\n"; } - else + else { amrex::Print() << "CUDA initialized with " << num_devices_used << " GPU(s) and " << ParallelDescriptor::NProcs() << " ranks.\n"; } #else // Should always be using NVCC >= 10 now, so not going to bother with other combinations. - amrex::Print() << "CUDA initialized with 1 GPU\n"; + amrex::Print() << "CUDA initialized with 1 GPU\n"; #endif // AMREX_USE_MPI && NVCC >= 10 } cudaProfilerStart(); -#elif defined(AMREX_USE_HIP) +#elif defined(AMREX_USE_HIP) if (amrex::Verbose()) { if (ParallelDescriptor::NProcs() > 1) { #ifdef BL_USE_MPI @@ -642,7 +642,7 @@ Device::startGraphRecording(bool first_iter, void* h_ptr, void* d_ptr, size_t sz { if ((first_iter) && inLaunchRegion() && inGraphRegion()) { - // Uses passed information to do initial async memcpy in graph and + // Uses passed information to do initial async memcpy in graph and // links dependency to all streams using cudaEvents. setStreamIndex(0); @@ -652,7 +652,7 @@ Device::startGraphRecording(bool first_iter, void* h_ptr, void* d_ptr, size_t sz #if (__CUDACC_VER_MAJOR__ == 10) && (__CUDACC_VER_MINOR__ == 0) AMREX_CUDA_SAFE_CALL(cudaStreamBeginCapture(graph_stream)); -#else +#else AMREX_CUDA_SAFE_CALL(cudaStreamBeginCapture(graph_stream, cudaStreamCaptureModeGlobal)); #endif @@ -711,14 +711,14 @@ Device::instantiateGraph(cudaGraph_t graph) { cudaGraphExec_t graphExec; -#ifdef AMREX_DEBUG +#ifdef AMREX_DEBUG // Implementes cudaGraphInstantiate error logging feature. -// Upon error, delays abort until message is output. +// Upon error, delays abort until message is output. constexpr int log_size = 1028; char graph_log[log_size]; graph_log[0]='\0'; - cudaGraphInstantiate(&graphExec, graph, NULL, &(graph_log[0]), log_size); + cudaGraphInstantiate(&graphExec, graph, NULL, &(graph_log[0]), log_size); if (graph_log[0] != '\0') { @@ -727,7 +727,7 @@ Device::instantiateGraph(cudaGraph_t graph) } #else - AMREX_CUDA_SAFE_CALL(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0)); + AMREX_CUDA_SAFE_CALL(cudaGraphInstantiate(&graphExec, graph, NULL, NULL, 0)); #endif diff --git a/Src/Base/AMReX_GpuLaunch.H b/Src/Base/AMReX_GpuLaunch.H index 5d8801c0199..c6a05b28916 100644 --- a/Src/Base/AMReX_GpuLaunch.H +++ b/Src/Base/AMReX_GpuLaunch.H @@ -33,7 +33,7 @@ #ifdef AMREX_USE_CUDA # define AMREX_LAUNCH_KERNEL(blocks, threads, sharedMem, stream, ... ) \ amrex::launch_global<<>>(__VA_ARGS__); -#elif defined(AMREX_USE_HIP) +#elif defined(AMREX_USE_HIP) # define AMREX_LAUNCH_KERNEL(blocks, threads, sharedMem, stream, ... ) \ hipLaunchKernelGGL(launch_global, blocks, threads, sharedMem, stream, __VA_ARGS__); #endif @@ -51,19 +51,19 @@ namespace amrex { template AMREX_GPU_DEVICE void call_device (L&& f0) noexcept { f0(); } - + template AMREX_GPU_DEVICE void call_device (L&& f0, Lambdas&&... fs) noexcept { f0(); call_device(std::forward(fs)...); } #endif - + // CPU variation template void launch_host (L&& f0) noexcept { f0(); } - + template void launch_host (L&& f0, Lambdas&&... fs) noexcept { f0(); @@ -147,7 +147,7 @@ namespace Gpu { } ExecutionConfig (dim3 nb, dim3 nt, std::size_t sm=0) noexcept : numBlocks(nb), numThreads(nt), sharedMem(sm) {} - + dim3 numBlocks; dim3 numThreads; std::size_t sharedMem = 0; diff --git a/Src/Base/AMReX_GpuMemory.H b/Src/Base/AMReX_GpuMemory.H index a4007f6aaa4..0cfc490310e 100644 --- a/Src/Base/AMReX_GpuMemory.H +++ b/Src/Base/AMReX_GpuMemory.H @@ -20,7 +20,7 @@ struct Managed { { return The_Managed_Arena()->alloc(len); } - + void operator delete (void *ptr) { if (ptr) The_Managed_Arena()->free(ptr); @@ -30,19 +30,19 @@ struct Managed { }; struct Pinned { - + #ifdef AMREX_USE_GPU void *operator new (std::size_t len) { return The_Pinned_Arena()->alloc(len); } - + void operator delete (void *ptr) { if (ptr) The_Pinned_Arena()->free(ptr); } - + #endif }; diff --git a/Src/Base/AMReX_GpuUtility.H b/Src/Base/AMReX_GpuUtility.H index 35b3f56667b..938bd757925 100644 --- a/Src/Base/AMReX_GpuUtility.H +++ b/Src/Base/AMReX_GpuUtility.H @@ -164,7 +164,7 @@ namespace Gpu { struct StreamItInfo { bool device_sync; - StreamItInfo () noexcept + StreamItInfo () noexcept : device_sync(true) {} StreamItInfo& DisableDeviceSync () noexcept { device_sync = false; diff --git a/Src/Base/AMReX_IArrayBox.H b/Src/Base/AMReX_IArrayBox.H index 458f2470717..a22b92fdd6c 100644 --- a/Src/Base/AMReX_IArrayBox.H +++ b/Src/Base/AMReX_IArrayBox.H @@ -50,8 +50,8 @@ public: */ explicit IArrayBox (const Box& b, int ncomp=1, - bool alloc=true, - bool shared=false, + bool alloc=true, + bool shared=false, Arena* ar = nullptr); IArrayBox (const IArrayBox& rhs, MakeType make_type, int scomp, int ncomp); diff --git a/Src/Base/AMReX_IArrayBox.cpp b/Src/Base/AMReX_IArrayBox.cpp index 83dc476b56a..1fe2264da18 100644 --- a/Src/Base/AMReX_IArrayBox.cpp +++ b/Src/Base/AMReX_IArrayBox.cpp @@ -55,7 +55,7 @@ IArrayBox::IArrayBox (const Box& b, int n, Arena* ar) #ifndef AMREX_USE_GPU // For debugging purposes if ( do_initval ) { - setVal(std::numeric_limits::max()); + setVal(std::numeric_limits::max()); } #endif } @@ -66,7 +66,7 @@ IArrayBox::IArrayBox (const Box& b, int n, bool alloc, bool shared, Arena* ar) #ifndef AMREX_USE_GPU // For debugging purposes if ( alloc && do_initval ) { - setVal(std::numeric_limits::max()); + setVal(std::numeric_limits::max()); } #endif } diff --git a/Src/Base/AMReX_INT.H b/Src/Base/AMReX_INT.H index 5d5a70ce4d3..c4c5f7481fd 100644 --- a/Src/Base/AMReX_INT.H +++ b/Src/Base/AMReX_INT.H @@ -7,9 +7,9 @@ #include #else #ifdef AMREX_TYPECHECK -#define __attribute__(x) -#define __extension__ -#define __restrict +#define __attribute__(x) +#define __extension__ +#define __restrict #endif #include #endif diff --git a/Src/Base/AMReX_IntConv.H b/Src/Base/AMReX_IntConv.H index 64b0a3159e7..99d9c792489 100644 --- a/Src/Base/AMReX_IntConv.H +++ b/Src/Base/AMReX_IntConv.H @@ -31,7 +31,7 @@ namespace amrex { os.write((char*) &value, sizeof(To)); } } - + template void readIntData(To* data, std::size_t size, std::istream& is, const amrex::IntDescriptor& id) @@ -43,7 +43,7 @@ namespace amrex { if (swapEndian) value = swapBytes(value); data[j] = value; } - } + } } #endif diff --git a/Src/Base/AMReX_IntConv.cpp b/Src/Base/AMReX_IntConv.cpp index 8969d02b3b0..6854d61a66b 100644 --- a/Src/Base/AMReX_IntConv.cpp +++ b/Src/Base/AMReX_IntConv.cpp @@ -9,7 +9,7 @@ std::int16_t swapBytes(std::int16_t val) std::int32_t swapBytes(std::int32_t val) { - val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF ); + val = ((val << 8) & 0xFF00FF00) | ((val >> 8) & 0xFF00FF ); return (val << 16) | ((val >> 16) & 0xFFFF); } @@ -27,7 +27,7 @@ std::uint16_t swapBytes(std::uint16_t val) std::uint32_t swapBytes(std::uint32_t val) { - val = ((val << 8) & 0xFF00FF00 ) | ((val >> 8) & 0xFF00FF ); + val = ((val << 8) & 0xFF00FF00 ) | ((val >> 8) & 0xFF00FF ); return (val << 16) | (val >> 16); } diff --git a/Src/Base/AMReX_IntVect.H b/Src/Base/AMReX_IntVect.H index 081da4f4211..1486cfb0755 100644 --- a/Src/Base/AMReX_IntVect.H +++ b/Src/Base/AMReX_IntVect.H @@ -50,20 +50,20 @@ public: class shift_hasher { private: - static constexpr unsigned shift1 = sizeof(size_t)>=8 ? 20 : 10; - static constexpr unsigned shift2 = sizeof(size_t)>=8 ? 40 : 20; + static constexpr unsigned shift1 = sizeof(size_t)>=8 ? 20 : 10; + static constexpr unsigned shift2 = sizeof(size_t)>=8 ? 40 : 20; public: - std::size_t operator()(const IntVect& vec) const noexcept + std::size_t operator()(const IntVect& vec) const noexcept { - AMREX_D_DECL(std::size_t ret0 = vec[0], ret1 = vec[1], ret2 = vec[2]); + AMREX_D_DECL(std::size_t ret0 = vec[0], ret1 = vec[1], ret2 = vec[2]); #if AMREX_SPACEDIM == 1 - return ret0; + return ret0; #elif AMREX_SPACEDIM == 2 - return ret0 ^ (ret1 << shift1); + return ret0 ^ (ret1 << shift1); #else - return ret0 ^ (ret1 << shift1) ^ (ret2 << shift2); + return ret0 ^ (ret1 << shift1) ^ (ret2 << shift2); #endif - } + } }; using hasher = shift_hasher; @@ -274,19 +274,19 @@ public: AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator<= (const IntVect& rhs) const noexcept { - return !(rhs < *this); + return !(rhs < *this); } //! Retrun true if this is lexicographically greater than rhs. AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator> (const IntVect& rhs) const noexcept { - return rhs < *this; + return rhs < *this; } //! Retrun true if this is lexicographically greater than or equal to rhs. AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE bool operator>= (const IntVect& rhs) const noexcept { - return !(*this < rhs); + return !(*this < rhs); } /** * \brief Returns true if this is less than argument for all components. diff --git a/Src/Base/AMReX_Lazy.H b/Src/Base/AMReX_Lazy.H index 81645e76b02..5e244410b7c 100644 --- a/Src/Base/AMReX_Lazy.H +++ b/Src/Base/AMReX_Lazy.H @@ -16,7 +16,7 @@ namespace Lazy void QueueReduction (Func); void EvalReduction (); - + void Finalize (); } } diff --git a/Src/Base/AMReX_Lazy.cpp b/Src/Base/AMReX_Lazy.cpp index 5d01c2f2787..3e3b3ced8c5 100644 --- a/Src/Base/AMReX_Lazy.cpp +++ b/Src/Base/AMReX_Lazy.cpp @@ -9,24 +9,24 @@ namespace Lazy void QueueReduction (Func f) { #ifdef BL_USE_MPI - reduction_queue.push_back(f); - const int max_queue_size = 64; - if (reduction_queue.size() >= max_queue_size) - EvalReduction(); + reduction_queue.push_back(f); + const int max_queue_size = 64; + if (reduction_queue.size() >= max_queue_size) + EvalReduction(); #else - f(); + f(); #endif } void EvalReduction () { #ifdef BL_USE_MPI - static int count = 0; - ++count; - if (count == 1) { - for (auto&& f : reduction_queue) - f(); - reduction_queue.clear(); + static int count = 0; + ++count; + if (count == 1) { + for (auto&& f : reduction_queue) + f(); + reduction_queue.clear(); count = 0; } #endif @@ -34,7 +34,7 @@ namespace Lazy void Finalize () { - EvalReduction(); + EvalReduction(); } } diff --git a/Src/Base/AMReX_MFCopyDescriptor.cpp b/Src/Base/AMReX_MFCopyDescriptor.cpp index f56ba0169bc..67df5394d2d 100644 --- a/Src/Base/AMReX_MFCopyDescriptor.cpp +++ b/Src/Base/AMReX_MFCopyDescriptor.cpp @@ -11,18 +11,18 @@ MultiFabCopyDescriptor::~MultiFabCopyDescriptor () {} void InterpAddBox (MultiFabCopyDescriptor& fabCopyDesc, - BoxList* returnUnfilledBoxes, - Vector& returnedFillBoxIds, - const Box& subbox, - MultiFabId faid1, - MultiFabId faid2, - Real t1, - Real t2, - Real t, - int src_comp, - int dest_comp, - int num_comp, - bool extrap) + BoxList* returnUnfilledBoxes, + Vector& returnedFillBoxIds, + const Box& subbox, + MultiFabId faid1, + MultiFabId faid2, + Real t1, + Real t2, + Real t, + int src_comp, + int dest_comp, + int num_comp, + bool extrap) { amrex::ignore_unused(extrap); @@ -75,17 +75,17 @@ InterpAddBox (MultiFabCopyDescriptor& fabCopyDesc, void InterpFillFab (MultiFabCopyDescriptor& fabCopyDesc, - const Vector& fillBoxIds, - MultiFabId faid1, - MultiFabId faid2, - FArrayBox& dest, - Real t1, - Real t2, - Real t, - int src_comp, // these comps need to be removed - int dest_comp, // from this routine - int num_comp, - bool extrap) + const Vector& fillBoxIds, + MultiFabId faid1, + MultiFabId faid2, + FArrayBox& dest, + Real t1, + Real t2, + Real t, + int src_comp, // these comps need to be removed + int dest_comp, // from this routine + int num_comp, + bool extrap) { amrex::ignore_unused(extrap); @@ -106,9 +106,9 @@ InterpFillFab (MultiFabCopyDescriptor& fabCopyDesc, BL_ASSERT(dest_comp + num_comp <= dest.nComp()); FArrayBox dest1(dest.box(), dest.nComp()); - dest1.setVal(std::numeric_limits::quiet_NaN()); + dest1.setVal(std::numeric_limits::quiet_NaN()); FArrayBox dest2(dest.box(), dest.nComp()); - dest2.setVal(std::numeric_limits::quiet_NaN()); + dest2.setVal(std::numeric_limits::quiet_NaN()); fabCopyDesc.FillFab(faid1, fillBoxIds[0], dest1); fabCopyDesc.FillFab(faid2, fillBoxIds[1], dest2); dest.linInterp diff --git a/Src/Base/AMReX_MFIter.H b/Src/Base/AMReX_MFIter.H index 01f5f0554f1..ae381f096fd 100644 --- a/Src/Base/AMReX_MFIter.H +++ b/Src/Base/AMReX_MFIter.H @@ -79,7 +79,7 @@ public: //! NoTeamBarrier: This option is for Team only. If on, there is no barrier in MFIter dtor. NoTeamBarrier = 0x04, //! SkipInit: Used by MFGhostIter - SkipInit = 0x08 + SkipInit = 0x08 }; /** @@ -88,23 +88,23 @@ public: * The tile size in this case is defined by FabArrayBase::mfiter_tile_size */ explicit MFIter (const FabArrayBase& fabarray, - unsigned char flags_=0); + unsigned char flags_=0); //! Enable tiling with the default tile size, which is defined by FabArrayBase::mfiter_tile_size MFIter (const FabArrayBase& fabarray, - bool do_tiling); + bool do_tiling); //! Enable tiling with explicit tile size and flags (See type Flags for more information) MFIter (const FabArrayBase& fabarray, - const IntVect& tilesize, - unsigned char flags_=0); + const IntVect& tilesize, + unsigned char flags_=0); MFIter (const BoxArray& ba, const DistributionMapping& dm, unsigned char flags_=0); MFIter (const BoxArray& ba, const DistributionMapping& dm, bool do_tiling); MFIter (const BoxArray& ba, const DistributionMapping& dm, - const IntVect& tilesize, unsigned char flags_=0); + const IntVect& tilesize, unsigned char flags_=0); MFIter (const FabArrayBase& fabarray, const MFItInfo& info); diff --git a/Src/Base/AMReX_MFIter.cpp b/Src/Base/AMReX_MFIter.cpp index 8881adb4e9f..8c40523f6c4 100644 --- a/Src/Base/AMReX_MFIter.cpp +++ b/Src/Base/AMReX_MFIter.cpp @@ -17,8 +17,8 @@ MFIter::allowMultipleMFIters (int allow) return allow; } -MFIter::MFIter (const FabArrayBase& fabarray_, - unsigned char flags_) +MFIter::MFIter (const FabArrayBase& fabarray_, + unsigned char flags_) : fabArray(fabarray_), tile_size((flags_ & Tiling) ? FabArrayBase::mfiter_tile_size : IntVect::TheZeroVector()), @@ -35,8 +35,8 @@ MFIter::MFIter (const FabArrayBase& fabarray_, Initialize(); } -MFIter::MFIter (const FabArrayBase& fabarray_, - bool do_tiling_) +MFIter::MFIter (const FabArrayBase& fabarray_, + bool do_tiling_) : fabArray(fabarray_), tile_size((do_tiling_) ? FabArrayBase::mfiter_tile_size : IntVect::TheZeroVector()), @@ -53,9 +53,9 @@ MFIter::MFIter (const FabArrayBase& fabarray_, Initialize(); } -MFIter::MFIter (const FabArrayBase& fabarray_, - const IntVect& tilesize_, - unsigned char flags_) +MFIter::MFIter (const FabArrayBase& fabarray_, + const IntVect& tilesize_, + unsigned char flags_) : fabArray(fabarray_), tile_size(tilesize_), @@ -218,7 +218,7 @@ MFIter::~MFIter () #ifdef BL_USE_TEAM if ( ! (flags & NoTeamBarrier) ) - ParallelDescriptor::MyTeam().MemoryBarrier(); + ParallelDescriptor::MyTeam().MemoryBarrier(); #endif #ifdef AMREX_USE_GPU @@ -255,67 +255,67 @@ MFIter::Initialize () } if (flags & SkipInit) { - return; + return; } else if (flags & AllBoxes) // a very special case { - index_map = &(fabArray.IndexArray()); - currentIndex = 0; - beginIndex = 0; - endIndex = index_map->size(); + index_map = &(fabArray.IndexArray()); + currentIndex = 0; + beginIndex = 0; + endIndex = index_map->size(); } else { - const FabArrayBase::TileArray* pta = fabArray.getTileArray(tile_size); - - index_map = &(pta->indexMap); - local_index_map = &(pta->localIndexMap); - tile_array = &(pta->tileArray); - local_tile_index_map = &(pta->localTileIndexMap); - num_local_tiles = &(pta->numLocalTiles); - - { - int rit = 0; - int nworkers = 1; + const FabArrayBase::TileArray* pta = fabArray.getTileArray(tile_size); + + index_map = &(pta->indexMap); + local_index_map = &(pta->localIndexMap); + tile_array = &(pta->tileArray); + local_tile_index_map = &(pta->localTileIndexMap); + num_local_tiles = &(pta->numLocalTiles); + + { + int rit = 0; + int nworkers = 1; #ifdef BL_USE_TEAM - if (ParallelDescriptor::TeamSize() > 1) { - if ( tile_size == IntVect::TheZeroVector() ) { - // In this case the TileArray contains only boxes owned by this worker. - // So there is no sharing going on. - rit = 0; - nworkers = 1; - } else { - rit = ParallelDescriptor::MyRankInTeam(); - nworkers = ParallelDescriptor::TeamSize(); - } - } + if (ParallelDescriptor::TeamSize() > 1) { + if ( tile_size == IntVect::TheZeroVector() ) { + // In this case the TileArray contains only boxes owned by this worker. + // So there is no sharing going on. + rit = 0; + nworkers = 1; + } else { + rit = ParallelDescriptor::MyRankInTeam(); + nworkers = ParallelDescriptor::TeamSize(); + } + } #endif - int ntot = index_map->size(); - - if (nworkers == 1) - { - beginIndex = 0; - endIndex = ntot; - } - else - { - int nr = ntot / nworkers; - int nlft = ntot - nr * nworkers; - if (rit < nlft) { // get nr+1 items - beginIndex = rit * (nr + 1); - endIndex = beginIndex + nr + 1; - } else { // get nr items - beginIndex = rit * nr + nlft; - endIndex = beginIndex + nr; - } - } - } - + int ntot = index_map->size(); + + if (nworkers == 1) + { + beginIndex = 0; + endIndex = ntot; + } + else + { + int nr = ntot / nworkers; + int nlft = ntot - nr * nworkers; + if (rit < nlft) { // get nr+1 items + beginIndex = rit * (nr + 1); + endIndex = beginIndex + nr + 1; + } else { // get nr items + beginIndex = rit * nr + nlft; + endIndex = beginIndex + nr; + } + } + } + #ifdef AMREX_USE_OMP - int nthreads = omp_get_num_threads(); - if (nthreads > 1) - { + int nthreads = omp_get_num_threads(); + if (nthreads > 1) + { if (dynamic) { beginIndex = omp_get_thread_num(); @@ -334,13 +334,13 @@ MFIter::Initialize () endIndex = beginIndex + nr; } } - } + } #endif - currentIndex = beginIndex; + currentIndex = beginIndex; #ifdef AMREX_USE_GPU - Gpu::Device::setStreamIndex((streams > 0) ? currentIndex%streams : -1); + Gpu::Device::setStreamIndex((streams > 0) ? currentIndex%streams : -1); if (!OpenMP::in_parallel()) { if (index_map->size() >= Gpu::getFuseNumKernelsThreshold()) { gpu_fsg.reset(new Gpu::FuseSafeGuard(true)); @@ -348,27 +348,27 @@ MFIter::Initialize () } #endif - typ = fabArray.boxArray().ixType(); + typ = fabArray.boxArray().ixType(); } } -Box +Box MFIter::tilebox () const noexcept -{ +{ BL_ASSERT(tile_array != 0); Box bx((*tile_array)[currentIndex]); if (! typ.cellCentered()) { - bx.convert(typ); - const Box& vbx = validbox(); - const IntVect& Big = vbx.bigEnd(); - for (int d=0; d= vccbx.bigEnd(d)) { - bx.growHi(d, ngrow[d]); - } + if (bx.smallEnd(d) == vccbx.smallEnd(d)) { + bx.growLo(d, ngrow[d]); + } + if (bx.bigEnd(d) >= vccbx.bigEnd(d)) { + bx.growHi(d, ngrow[d]); + } } return bx; } Box MFIter::nodaltilebox (int dir) const noexcept -{ +{ BL_ASSERT(dir < AMREX_SPACEDIM); BL_ASSERT(tile_array != 0); Box bx((*tile_array)[currentIndex]); @@ -422,24 +422,24 @@ MFIter::nodaltilebox (int dir) const noexcept const IntVect& Big = vbx.bigEnd(); int d0, d1; if (dir < 0) { - d0 = 0; - d1 = AMREX_SPACEDIM-1; + d0 = 0; + d1 = AMREX_SPACEDIM-1; } else { - d0 = d1 = dir; + d0 = d1 = dir; } for (int d=d0; d<=d1; ++d) { - if (typ.cellCentered(d)) { // validbox should also be cell-centered in d-direction. - bx.surroundingNodes(d); - if (bx.bigEnd(d) <= Big[d]) { - bx.growHi(d,-1); - } - } + if (typ.cellCentered(d)) { // validbox should also be cell-centered in d-direction. + bx.surroundingNodes(d); + if (bx.bigEnd(d) <= Big[d]) { + bx.growHi(d,-1); + } + } } return bx; } // Note that a small negative ng is supported. -Box +Box MFIter::growntilebox (int a_ng) const noexcept { Box bx = tilebox(); @@ -447,12 +447,12 @@ MFIter::growntilebox (int a_ng) const noexcept if (a_ng < -100) ngv = fabArray.nGrowVect(); const Box& vbx = validbox(); for (int d=0; d 1) { - rit = ParallelDescriptor::MyRankInTeam(); - nworkers = ParallelDescriptor::TeamSize(); + rit = ParallelDescriptor::MyRankInTeam(); + nworkers = ParallelDescriptor::TeamSize(); } #endif @@ -545,21 +545,21 @@ MFGhostIter::Initialize () Vector alllocalindex; for (int i=0; i < fabArray.IndexArray().size(); ++i) { - int K = fabArray.IndexArray()[i]; - const Box& vbx = fabArray.box(K); - const Box& fbx = fabArray.fabbox(K); - - const BoxList& diff = amrex::boxDiff(fbx, vbx); - - for (BoxList::const_iterator bli = diff.begin(); bli != diff.end(); ++bli) { - BoxList tiles(*bli, FabArrayBase::mfghostiter_tile_size); - int nt = tiles.size(); - for (int it=0; it - ([] () -> MemProfiler::MemInfo { - int MB_min, MB_max, MB_tot; - amrex_mempool_get_stats(MB_min, MB_max, MB_tot); - Long b = MB_tot * (1024L*1024L); - return {b, b}; - })); + MemProfiler::add("MemPool", std::function + ([] () -> MemProfiler::MemInfo { + int MB_min, MB_max, MB_tot; + amrex_mempool_get_stats(MB_min, MB_max, MB_tot); + Long b = MB_tot * (1024L*1024L); + return {b, b}; + })); #endif } } @@ -89,7 +89,7 @@ void* amrex_mempool_alloc (size_t nbytes) return the_memory_pool[tid]->alloc(nbytes); } -void amrex_mempool_free (void* p) +void amrex_mempool_free (void* p) { int tid = OpenMP::get_thread_num(); the_memory_pool[tid]->free(p); diff --git a/Src/Base/AMReX_MemProfiler.H b/Src/Base/AMReX_MemProfiler.H index e0b27701ebd..00610adf3f9 100644 --- a/Src/Base/AMReX_MemProfiler.H +++ b/Src/Base/AMReX_MemProfiler.H @@ -49,14 +49,14 @@ private: Long mn; Long mx; }; - friend std::ostream& operator<< (std::ostream& os, + friend std::ostream& operator<< (std::ostream& os, const MemProfiler::Bytes& bytes); struct Builds { int mn; int mx; }; - friend std::ostream& operator<< (std::ostream& os, + friend std::ostream& operator<< (std::ostream& os, const MemProfiler::Builds& builds); static MemProfiler& getInstance (); diff --git a/Src/Base/AMReX_MemProfiler.cpp b/Src/Base/AMReX_MemProfiler.cpp index d79aa4492d8..170a2fdc356 100644 --- a/Src/Base/AMReX_MemProfiler.cpp +++ b/Src/Base/AMReX_MemProfiler.cpp @@ -20,7 +20,7 @@ namespace amrex { std::unique_ptr MemProfiler::the_instance = nullptr; -void +void MemProfiler::add (const std::string& name, std::function&& f) { MemProfiler& mprofiler = getInstance(); @@ -33,7 +33,7 @@ MemProfiler::add (const std::string& name, std::function&& f) mprofiler.the_funcs.push_back(std::move(f)); } -void +void MemProfiler::add (const std::string& name, std::function&& f) { MemProfiler& mprofiler = getInstance(); @@ -46,7 +46,7 @@ MemProfiler::add (const std::string& name, std::function&& f) mprofiler.the_funcs_builds.push_back(std::move(f)); } -MemProfiler& +MemProfiler& MemProfiler::getInstance () { if (the_instance == nullptr) { @@ -66,10 +66,10 @@ MemProfiler::report (const std::string& prefix) { static std::string memory_log_name; if (memory_log_name.empty()) { - ParmParse pp("amrex"); - pp.query("memory_log", memory_log_name); - if (memory_log_name.empty()) - memory_log_name = "memlog"; + ParmParse pp("amrex"); + pp.query("memory_log", memory_log_name); + if (memory_log_name.empty()) + memory_log_name = "memlog"; } getInstance().report_(prefix, memory_log_name); @@ -81,9 +81,9 @@ MemProfiler::report_ (const std::string& prefix, const std::string& memory_log_n std::vector cur_min; std::vector hwm_min; for (auto&& f: the_funcs) { - const MemInfo& minfo = f(); - cur_min.push_back(minfo.current_bytes); - hwm_min.push_back(minfo.hwm_bytes); + const MemInfo& minfo = f(); + cur_min.push_back(minfo.current_bytes); + hwm_min.push_back(minfo.hwm_bytes); } std::vector cur_max = cur_min; std::vector hwm_max = hwm_min; @@ -91,9 +91,9 @@ MemProfiler::report_ (const std::string& prefix, const std::string& memory_log_n std::vector num_builds_min; std::vector hwm_builds_min; for (auto&& f: the_funcs_builds) { - const NBuildsInfo& binfo = f(); - num_builds_min.push_back(binfo.current_builds); - hwm_builds_min.push_back(binfo.hwm_builds); + const NBuildsInfo& binfo = f(); + num_builds_min.push_back(binfo.current_builds); + hwm_builds_min.push_back(binfo.hwm_builds); } std::vector num_builds_max = num_builds_min; std::vector hwm_builds_max = hwm_builds_min; @@ -114,53 +114,53 @@ MemProfiler::report_ (const std::string& prefix, const std::string& memory_log_n const int ipstat = 1; const int npstat = 4; { - static pid_t pid = getpid(); - std::string fname = "/proc/"+std::to_string(pid) + "/status"; - std::ifstream ifs(fname.c_str()); - std::string token, unit; - Long n; - int nfound = 0; - while (ifs >> token) { - if (token == "VmPeak:") { - nfound++; - ifs >> n >> unit; - if (unit == "kB") { - mymin[ipstat+0] = mymax[ipstat+0] = n * 1024L; - } else { - ierr_proc_status = -1; - break; - } - } else if (token == "VmSize:") { - nfound++; - ifs >> n >> unit; - if (unit == "kB") { - mymin[ipstat+1] = mymax[ipstat+1] = n * 1024L; - } else { - ierr_proc_status = -1; - break; - } - } else if (token == "VmHWM:") { - nfound++; - ifs >> n >> unit; - if (unit == "kB") { - mymin[ipstat+2] = mymax[ipstat+2] = n * 1024L; - } else { - ierr_proc_status = -1; - break; - } - } else if (token == "VmRSS:") { - nfound++; - ifs >> n >> unit; - if (unit == "kB") { - mymin[ipstat+3] = mymax[ipstat+3] = n * 1024L; - } else { - ierr_proc_status = -1; - break; - } - } - ifs.ignore(std::numeric_limits::max(), '\n'); - if (nfound == npstat) break; - } + static pid_t pid = getpid(); + std::string fname = "/proc/"+std::to_string(pid) + "/status"; + std::ifstream ifs(fname.c_str()); + std::string token, unit; + Long n; + int nfound = 0; + while (ifs >> token) { + if (token == "VmPeak:") { + nfound++; + ifs >> n >> unit; + if (unit == "kB") { + mymin[ipstat+0] = mymax[ipstat+0] = n * 1024L; + } else { + ierr_proc_status = -1; + break; + } + } else if (token == "VmSize:") { + nfound++; + ifs >> n >> unit; + if (unit == "kB") { + mymin[ipstat+1] = mymax[ipstat+1] = n * 1024L; + } else { + ierr_proc_status = -1; + break; + } + } else if (token == "VmHWM:") { + nfound++; + ifs >> n >> unit; + if (unit == "kB") { + mymin[ipstat+2] = mymax[ipstat+2] = n * 1024L; + } else { + ierr_proc_status = -1; + break; + } + } else if (token == "VmRSS:") { + nfound++; + ifs >> n >> unit; + if (unit == "kB") { + mymin[ipstat+3] = mymax[ipstat+3] = n * 1024L; + } else { + ierr_proc_status = -1; + break; + } + } + ifs.ignore(std::numeric_limits::max(), '\n'); + if (nfound == npstat) break; + } } int ierr_sysinfo; @@ -168,35 +168,35 @@ MemProfiler::report_ (const std::string& prefix, const std::string& memory_log_n const int isinfo = ipstat + npstat; const int nsinfo = 4; { - struct sysinfo info; - ierr_sysinfo = sysinfo(&info); - if (ierr_sysinfo == 0) { - mymin[isinfo+0] = mymax[isinfo+0] = info.mem_unit * info.totalram; - mymin[isinfo+1] = mymax[isinfo+1] = info.mem_unit * info.freeram; - mymin[isinfo+2] = mymax[isinfo+2] = info.mem_unit * (info.bufferram + info.freeram); - mymin[isinfo+3] = mymax[isinfo+3] = info.mem_unit * info.sharedram; - } - - // Have to read /proc/meminfo to get Cached memory. - { - std::ifstream ifs("/proc/meminfo"); - std::string token, unit; - Long cached; - while (ifs >> token) { - if (token == "Cached:") { - ifs >> cached >> unit; - break; - } - ifs.ignore(std::numeric_limits::max(), '\n'); - } - if (unit == "kB") { - ierr_cached = 0; - mymin[isinfo+2] += cached*1024L; - mymax[isinfo+2] = mymin[isinfo+2]; - } else { - ierr_cached = -1; - } - } + struct sysinfo info; + ierr_sysinfo = sysinfo(&info); + if (ierr_sysinfo == 0) { + mymin[isinfo+0] = mymax[isinfo+0] = info.mem_unit * info.totalram; + mymin[isinfo+1] = mymax[isinfo+1] = info.mem_unit * info.freeram; + mymin[isinfo+2] = mymax[isinfo+2] = info.mem_unit * (info.bufferram + info.freeram); + mymin[isinfo+3] = mymax[isinfo+3] = info.mem_unit * info.sharedram; + } + + // Have to read /proc/meminfo to get Cached memory. + { + std::ifstream ifs("/proc/meminfo"); + std::string token, unit; + Long cached; + while (ifs >> token) { + if (token == "Cached:") { + ifs >> cached >> unit; + break; + } + ifs.ignore(std::numeric_limits::max(), '\n'); + } + if (unit == "kB") { + ierr_cached = 0; + mymin[isinfo+2] += cached*1024L; + mymax[isinfo+2] = mymin[isinfo+2]; + } else { + ierr_cached = -1; + } + } } #endif @@ -215,119 +215,119 @@ MemProfiler::report_ (const std::string& prefix, const std::string& memory_log_n if (ParallelDescriptor::IOProcessor()) { - std::ofstream memlog(memory_log_name.c_str(), - std::ofstream::out|std::ofstream::app); - if (!memlog.good()) return; - - static int width_name = 0; - if (width_name == 0) { - for (auto& x: the_names) - width_name = std::max(width_name, int(x.size())); - for (auto& x: the_names_builds) - width_name = std::max(width_name, int(x.size())); - } - const int width_bytes = 18; - - const std::string dash_name(width_name,'-'); - const std::string dash_bytes(width_bytes,'-'); - const std::string ident(6,' '); - - if (!prefix.empty()) - memlog << prefix << " "; - - memlog << "Memory Profile Report Across Processes:\n"; - - memlog << std::setfill(' '); - memlog << ident; - memlog << "| " << std::setw(width_name) << std::left << "Name" << " | " - << std::setw(width_bytes) << std::right << "Current " << " | " - << std::setw(width_bytes) << "High Water Mark " << " |\n"; - std::setw(0); - - memlog << ident; - memlog << "|-" << dash_name << "-+-" << dash_bytes << "-+-" << dash_bytes << "-|\n"; - - std::vector idxs(the_names.size()); - std::iota(idxs.begin(), idxs.end(), 0); - std::sort(idxs.begin(), idxs.end(), [&](int i, int j) - { return hwm_max[i] > hwm_max[j]; }); - - for (int ii = 0; ii < idxs.size(); ++ii) { - int i = idxs[ii]; - if (hwm_max[i] > 0) { - memlog << ident; - memlog << "| " << std::setw(width_name) << std::left << the_names[i] << " | "; - memlog << Bytes{cur_min[i],cur_max[i]} << " | "; - memlog << Bytes{hwm_min[i],hwm_max[i]} << " |\n"; - } - } - - memlog << ident; - memlog << "|-" << dash_name << "-+-" << dash_bytes << "-+-" << dash_bytes << "-|\n"; - - memlog << ident; - memlog << "| " << std::setw(width_name) << std::left << "Total" << " | "; - memlog << Bytes{mymin[0],mymax[0]} << " | " << std::setw(width_bytes) << " " << " |\n"; - memlog << std::setw(0); - - // Number of builds - { - memlog << "\n"; - memlog << ident; - memlog << "| " << std::setw(width_name) << std::left << "Name" << " | " - << std::setw(width_bytes) << std::right << "Current # " << " | " - << std::setw(width_bytes) << "High Water Mark #" << " |\n"; - std::setw(0); - - memlog << ident; - memlog << "|-" << dash_name << "-+-" << dash_bytes << "-+-" << dash_bytes << "-|\n"; - - for (int i = 0; i < the_names_builds.size(); ++i) { - if (hwm_builds_max[i] > 0) { - memlog << ident; - memlog << "| " << std::setw(width_name) << std::left << the_names_builds[i] << " | "; - memlog << Builds{num_builds_min[i],num_builds_max[i]} << " | "; - memlog << Builds{hwm_builds_min[i],hwm_builds_max[i]} << " |\n"; - } - } - } + std::ofstream memlog(memory_log_name.c_str(), + std::ofstream::out|std::ofstream::app); + if (!memlog.good()) return; + + static int width_name = 0; + if (width_name == 0) { + for (auto& x: the_names) + width_name = std::max(width_name, int(x.size())); + for (auto& x: the_names_builds) + width_name = std::max(width_name, int(x.size())); + } + const int width_bytes = 18; + + const std::string dash_name(width_name,'-'); + const std::string dash_bytes(width_bytes,'-'); + const std::string ident(6,' '); + + if (!prefix.empty()) + memlog << prefix << " "; + + memlog << "Memory Profile Report Across Processes:\n"; + + memlog << std::setfill(' '); + memlog << ident; + memlog << "| " << std::setw(width_name) << std::left << "Name" << " | " + << std::setw(width_bytes) << std::right << "Current " << " | " + << std::setw(width_bytes) << "High Water Mark " << " |\n"; + std::setw(0); + + memlog << ident; + memlog << "|-" << dash_name << "-+-" << dash_bytes << "-+-" << dash_bytes << "-|\n"; + + std::vector idxs(the_names.size()); + std::iota(idxs.begin(), idxs.end(), 0); + std::sort(idxs.begin(), idxs.end(), [&](int i, int j) + { return hwm_max[i] > hwm_max[j]; }); + + for (int ii = 0; ii < idxs.size(); ++ii) { + int i = idxs[ii]; + if (hwm_max[i] > 0) { + memlog << ident; + memlog << "| " << std::setw(width_name) << std::left << the_names[i] << " | "; + memlog << Bytes{cur_min[i],cur_max[i]} << " | "; + memlog << Bytes{hwm_min[i],hwm_max[i]} << " |\n"; + } + } + + memlog << ident; + memlog << "|-" << dash_name << "-+-" << dash_bytes << "-+-" << dash_bytes << "-|\n"; + + memlog << ident; + memlog << "| " << std::setw(width_name) << std::left << "Total" << " | "; + memlog << Bytes{mymin[0],mymax[0]} << " | " << std::setw(width_bytes) << " " << " |\n"; + memlog << std::setw(0); + + // Number of builds + { + memlog << "\n"; + memlog << ident; + memlog << "| " << std::setw(width_name) << std::left << "Name" << " | " + << std::setw(width_bytes) << std::right << "Current # " << " | " + << std::setw(width_bytes) << "High Water Mark #" << " |\n"; + std::setw(0); + + memlog << ident; + memlog << "|-" << dash_name << "-+-" << dash_bytes << "-+-" << dash_bytes << "-|\n"; + + for (int i = 0; i < the_names_builds.size(); ++i) { + if (hwm_builds_max[i] > 0) { + memlog << ident; + memlog << "| " << std::setw(width_name) << std::left << the_names_builds[i] << " | "; + memlog << Builds{num_builds_min[i],num_builds_max[i]} << " | "; + memlog << Builds{hwm_builds_min[i],hwm_builds_max[i]} << " |\n"; + } + } + } #ifdef __linux__ - if (ierr_proc_status == 0) { - memlog << "\n"; - memlog << " * " << std::setw(width_bytes) << std::left << "Proc VmPeak" - << " " << std::setw(width_bytes) << "VmSize" - << " " << std::setw(width_bytes) << "VmHWM" - << " " << std::setw(width_bytes) << "VmRSS" << "\n"; - memlog << " "; - for (int i = 0; i < npstat; ++i) - memlog << " [" << Bytes{mymin[ipstat+i], mymax[ipstat+i]} << "]"; - memlog << "\n"; - } - - if (ierr_sysinfo == 0) { - memlog << "\n"; - memlog << " * " << std::setw(width_bytes) << std::left << "Node total" - << " " << std::setw(width_bytes) << "free"; - if (ierr_cached == 0) - memlog << " " << std::setw(width_bytes) << "free+buffers+cached" << " "; - else - memlog << " " << std::setw(width_bytes) << "free+buffers" << " "; - memlog << std::setw(width_bytes) << "shared" << "\n"; - memlog << " "; - for (int i = 0; i < nsinfo; ++i) - memlog << " [" << Bytes{mymin[isinfo+i], mymax[isinfo+i]} << "]"; - memlog << "\n"; - } + if (ierr_proc_status == 0) { + memlog << "\n"; + memlog << " * " << std::setw(width_bytes) << std::left << "Proc VmPeak" + << " " << std::setw(width_bytes) << "VmSize" + << " " << std::setw(width_bytes) << "VmHWM" + << " " << std::setw(width_bytes) << "VmRSS" << "\n"; + memlog << " "; + for (int i = 0; i < npstat; ++i) + memlog << " [" << Bytes{mymin[ipstat+i], mymax[ipstat+i]} << "]"; + memlog << "\n"; + } + + if (ierr_sysinfo == 0) { + memlog << "\n"; + memlog << " * " << std::setw(width_bytes) << std::left << "Node total" + << " " << std::setw(width_bytes) << "free"; + if (ierr_cached == 0) + memlog << " " << std::setw(width_bytes) << "free+buffers+cached" << " "; + else + memlog << " " << std::setw(width_bytes) << "free+buffers" << " "; + memlog << std::setw(width_bytes) << "shared" << "\n"; + memlog << " "; + for (int i = 0; i < nsinfo; ++i) + memlog << " [" << Bytes{mymin[isinfo+i], mymax[isinfo+i]} << "]"; + memlog << "\n"; + } #endif - memlog << std::endl; + memlog << std::endl; - memlog.close(); + memlog.close(); } } -std::ostream& +std::ostream& operator<< (std::ostream& os, const MemProfiler::Bytes& bytes) { constexpr Long G = 1024L*1024L*1024L; @@ -337,31 +337,31 @@ operator<< (std::ostream& os, const MemProfiler::Bytes& bytes) Long fac; std::string unit; if (bytes.mn >= 10L*G) { - fac = G; - unit = "GB"; + fac = G; + unit = "GB"; } else if (bytes.mn >= 10L*M) { - fac = M; - unit = "MB"; + fac = M; + unit = "MB"; } else if (bytes.mn >= 10L*K) { - fac = K; - unit = "KB"; + fac = K; + unit = "KB"; } else { - fac = 1L; - unit = "B"; + fac = 1L; + unit = "B"; } os << std::setw(5) << std::right << bytes.mn/fac << " ... " - << std::setw(5) << std::left << bytes.mx/fac << " " + << std::setw(5) << std::left << bytes.mx/fac << " " << std::setw(2) << std::right << unit; os << std::setw(0); return os; } -std::ostream& +std::ostream& operator<< (std::ostream& os, const MemProfiler::Builds& builds) { os << std::setw(6) << std::right << builds.mn << " ... " - << std::setw(7) << std::left << builds.mx; + << std::setw(7) << std::left << builds.mx; os << std::setw(0); return os; } diff --git a/Src/Base/AMReX_MultiFab.H b/Src/Base/AMReX_MultiFab.H index f0066d187b3..44bc2fca725 100644 --- a/Src/Base/AMReX_MultiFab.H +++ b/Src/Base/AMReX_MultiFab.H @@ -63,10 +63,10 @@ public: int ncomp, int ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory); #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = FArrayBoxFactory()); #endif @@ -75,10 +75,10 @@ public: int ncomp, const IntVect& ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory); #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = FArrayBoxFactory()); #endif @@ -99,26 +99,26 @@ public: MultiFab& operator= (const MultiFab& rhs) = delete; virtual void define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - int ngrow, + const DistributionMapping& dm, + int nvar, + int ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) override; #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = FArrayBoxFactory()) override; #endif virtual void define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - const IntVect& ngrow, + const DistributionMapping& dm, + int nvar, + const IntVect& ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) override; #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = FArrayBoxFactory()) override; #endif @@ -132,7 +132,7 @@ public: */ Real min (int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Identical to min() function, but confines its * search to intersection of Box b and the MultiFab. @@ -140,7 +140,7 @@ public: Real min (const Box& b, int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Returns the maximum value contained in component comp of the * MultiFab. The parameter nghost determines the number of @@ -149,7 +149,7 @@ public: */ Real max (int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Identical to the previous max() function, but confines its * search to intersection of Box b and the MultiFab. @@ -157,7 +157,7 @@ public: Real max (const Box& b, int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Returns the maximum *absolute* value contained in * component comp of the MultiFab. @@ -415,36 +415,36 @@ public: * against divide by zero. */ void divide (const MultiFab& mf, - int strt_comp, - int num_comp, - int nghost); + int strt_comp, + int num_comp, + int nghost); /** * \brief Returns the dot product of two MultiFabs. */ static Real Dot (const MultiFab& x, int xcomp, - const MultiFab& y, int ycomp, - int num_comp, int nghost, bool local = false); + const MultiFab& y, int ycomp, + int num_comp, int nghost, bool local = false); /** * \brief Returns the dot product of a MultiFab with itself. */ static Real Dot (const MultiFab& x, int xcomp, - int num_comp, int nghost, bool local = false); + int num_comp, int nghost, bool local = false); static Real Dot (const iMultiFab& mask, const MultiFab& x, int xcomp, - const MultiFab& y, int ycomp, - int num_comp, int nghost, bool local = false); + const MultiFab& y, int ycomp, + int num_comp, int nghost, bool local = false); /** * \brief Add src to dst including nghost ghost cells. * The two MultiFabs MUST have the same underlying BoxArray. */ static void Add (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); static void Add (MultiFab& dst, const MultiFab& src, @@ -495,134 +495,134 @@ public: * The two MultiFabs MUST have the same underlying BoxArray. */ static void Subtract (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); static void Subtract (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - const IntVect& nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief Multiply dst by src including nghost ghost cells. * The two MultiFabs MUST have the same underlying BoxArray. */ static void Multiply (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); static void Multiply (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - const IntVect& nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief Divide dst by src including nghost ghost cells. * The two MultiFabs MUST have the same underlying BoxArray. */ static void Divide (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); static void Divide (MultiFab& dst, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - const IntVect& nghost); + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief dst += a*src */ static void Saxpy (MultiFab& dst, - Real a, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + Real a, + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); static void Saxpy (MultiFab& dst, - Real a, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - const IntVect& nghost); + Real a, + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief dst = src + a*dst */ static void Xpay (MultiFab& dst, - Real a, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + Real a, + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); static void Xpay (MultiFab& dst, - Real a, - const MultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - const IntVect& nghost); + Real a, + const MultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief dst = a*x + b*y */ static void LinComb (MultiFab& dst, - Real a, - const MultiFab& x, - int xcomp, - Real b, - const MultiFab& y, - int ycomp, - int dstcomp, - int numcomp, - int nghost); + Real a, + const MultiFab& x, + int xcomp, + Real b, + const MultiFab& y, + int ycomp, + int dstcomp, + int numcomp, + int nghost); static void LinComb (MultiFab& dst, - Real a, - const MultiFab& x, - int xcomp, - Real b, - const MultiFab& y, - int ycomp, - int dstcomp, - int numcomp, - const IntVect& nghost); + Real a, + const MultiFab& x, + int xcomp, + Real b, + const MultiFab& y, + int ycomp, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief dst += src1*src2 */ static void AddProduct (MultiFab& dst, - const MultiFab& src1, - int comp1, - const MultiFab& src2, - int comp2, - int dstcomp, - int numcomp, - int nghost); + const MultiFab& src1, + int comp1, + const MultiFab& src2, + int comp2, + int dstcomp, + int numcomp, + int nghost); static void AddProduct (MultiFab& dst, - const MultiFab& src1, - int comp1, - const MultiFab& src2, - int comp2, - int dstcomp, - int numcomp, - const IntVect& nghost); + const MultiFab& src1, + int comp1, + const MultiFab& src2, + int comp2, + int dstcomp, + int numcomp, + const IntVect& nghost); /** * \brief Are there any NaNs in the MF? diff --git a/Src/Base/AMReX_MultiFab.cpp b/Src/Base/AMReX_MultiFab.cpp index 7334c3dd514..155873d8a62 100644 --- a/Src/Base/AMReX_MultiFab.cpp +++ b/Src/Base/AMReX_MultiFab.cpp @@ -34,8 +34,8 @@ namespace Real MultiFab::Dot (const MultiFab& x, int xcomp, - const MultiFab& y, int ycomp, - int numcomp, int nghost, bool local) + const MultiFab& y, int ycomp, + int numcomp, int nghost, bool local) { BL_ASSERT(x.boxArray() == y.boxArray()); BL_ASSERT(x.DistributionMap() == y.DistributionMap()); @@ -82,7 +82,7 @@ MultiFab::Dot (const MultiFab& x, int xcomp, Real MultiFab::Dot (const MultiFab& x, int xcomp, int numcomp, int nghost, bool local) { - BL_ASSERT(x.nGrow() >= nghost); + BL_ASSERT(x.nGrow() >= nghost); Real sm = amrex::ReduceSum(x, nghost, [=] AMREX_GPU_HOST_DEVICE (Box const& bx, Array4 const& xfab) -> Real @@ -105,8 +105,8 @@ MultiFab::Dot (const MultiFab& x, int xcomp, int numcomp, int nghost, bool local Real MultiFab::Dot (const iMultiFab& mask, const MultiFab& x, int xcomp, - const MultiFab& y, int ycomp, - int numcomp, int nghost, bool local) + const MultiFab& y, int ycomp, + int numcomp, int nghost, bool local) { BL_ASSERT(x.boxArray() == y.boxArray()); BL_ASSERT(x.boxArray() == mask.boxArray()); @@ -171,7 +171,7 @@ MultiFab::Copy (MultiFab& dst, const MultiFab& src, BL_ASSERT(dst.nGrowVect().allGE(nghost)); BL_PROFILE("MultiFab::Copy()"); - + amrex::Copy(dst,src,srccomp,dstcomp,numcomp,nghost); } @@ -240,7 +240,7 @@ MultiFab::Subtract (MultiFab& dst, const MultiFab& src, void MultiFab::Subtract (MultiFab& dst, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, const IntVect& nghost) + int srccomp, int dstcomp, int numcomp, const IntVect& nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -253,14 +253,14 @@ MultiFab::Subtract (MultiFab& dst, const MultiFab& src, void MultiFab::Multiply (MultiFab& dst, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, int nghost) + int srccomp, int dstcomp, int numcomp, int nghost) { Multiply(dst,src,srccomp,dstcomp,numcomp,IntVect(nghost)); } void MultiFab::Multiply (MultiFab& dst, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, const IntVect& nghost) + int srccomp, int dstcomp, int numcomp, const IntVect& nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -273,14 +273,14 @@ MultiFab::Multiply (MultiFab& dst, const MultiFab& src, void MultiFab::Divide (MultiFab& dst, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, int nghost) + int srccomp, int dstcomp, int numcomp, int nghost) { Divide(dst,src,srccomp,dstcomp,numcomp,IntVect(nghost)); } void MultiFab::Divide (MultiFab& dst, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, const IntVect& nghost) + int srccomp, int dstcomp, int numcomp, const IntVect& nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -293,14 +293,14 @@ MultiFab::Divide (MultiFab& dst, const MultiFab& src, void MultiFab::Saxpy (MultiFab& dst, Real a, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, int nghost) + int srccomp, int dstcomp, int numcomp, int nghost) { Saxpy(dst,a,src,srccomp,dstcomp,numcomp,IntVect(nghost)); } void MultiFab::Saxpy (MultiFab& dst, Real a, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, const IntVect& nghost) + int srccomp, int dstcomp, int numcomp, const IntVect& nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -328,14 +328,14 @@ MultiFab::Saxpy (MultiFab& dst, Real a, const MultiFab& src, void MultiFab::Xpay (MultiFab& dst, Real a, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, int nghost) + int srccomp, int dstcomp, int numcomp, int nghost) { Xpay(dst,a,src,srccomp,dstcomp,numcomp,IntVect(nghost)); } void MultiFab::Xpay (MultiFab& dst, Real a, const MultiFab& src, - int srccomp, int dstcomp, int numcomp, const IntVect& nghost) + int srccomp, int dstcomp, int numcomp, const IntVect& nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -389,7 +389,7 @@ MultiFab::LinComb (MultiFab& dst, for (MFIter mfi(dst,TilingIfNotGPU()); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(nghost); - + if (bx.ok()) { auto const xfab = x.array(mfi); auto const yfab = y.array(mfi); @@ -501,9 +501,9 @@ MultiFab::Initialize () #ifdef AMREX_MEM_PROFILING MemProfiler::add("MultiFab", std::function - ([] () -> MemProfiler::NBuildsInfo { - return {num_multifabs, num_multifabs_hwm}; - })); + ([] () -> MemProfiler::NBuildsInfo { + return {num_multifabs, num_multifabs_hwm}; + })); #endif } @@ -525,7 +525,7 @@ MultiFab::MultiFab (const BoxArray& bxs, const DistributionMapping& dm, int ncomp, int ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) : MultiFab(bxs,dm,ncomp,IntVect(ngrow),info,factory) {} @@ -534,7 +534,7 @@ MultiFab::MultiFab (const BoxArray& bxs, const DistributionMapping& dm, int ncomp, const IntVect& ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) : FabArray(bxs,dm,ncomp,ngrow,info,factory) @@ -583,7 +583,7 @@ MultiFab::define (const BoxArray& bxs, const DistributionMapping& dm, int nvar, int ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) { define(bxs, dm, nvar, IntVect(ngrow), info, factory); @@ -595,7 +595,7 @@ MultiFab::define (const BoxArray& bxs, const DistributionMapping& dm, int nvar, const IntVect& ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) { this->FabArray::define(bxs,dm,nvar,ngrow,info,factory); @@ -611,11 +611,11 @@ MultiFab::initVal () for (MFIter mfi(*this); mfi.isValid(); ++mfi) { FArrayBox& fab = (*this)[mfi]; - fab.initVal(); + fab.initVal(); } } -bool +bool MultiFab::contains_nan (int scomp, int ncomp, int ngrow, @@ -624,7 +624,7 @@ MultiFab::contains_nan (int scomp, return contains_nan(scomp, ncomp, IntVect(ngrow), local); } -bool +bool MultiFab::contains_nan (int scomp, int ncomp, const IntVect& ngrow, @@ -652,7 +652,7 @@ MultiFab::contains_nan (int scomp, return r; } -bool +bool MultiFab::contains_nan (bool local) const { return contains_nan(0,nComp(),nGrowVect(),local); @@ -677,18 +677,18 @@ MultiFab::contains_inf (int scomp, int ncomp, IntVect const& ngrow, bool local) }); if (!local) - ParallelAllReduce::Or(r, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Or(r, ParallelContext::CommunicatorSub()); return r; } -bool +bool MultiFab::contains_inf (int scomp, int ncomp, int ngrow, bool local) const { return contains_inf(scomp,ncomp,IntVect(ngrow),local); } -bool +bool MultiFab::contains_inf (bool local) const { return contains_inf(0,nComp(),nGrow(),local); @@ -734,7 +734,7 @@ MultiFab::min (int comp, int nghost, bool local) const } if (!local) - ParallelAllReduce::Min(mn, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Min(mn, ParallelContext::CommunicatorSub()); return mn; } @@ -757,7 +757,7 @@ MultiFab::min (const Box& region, int comp, int nghost, bool local) const }); if (!local) - ParallelAllReduce::Min(mn, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Min(mn, ParallelContext::CommunicatorSub()); return mn; @@ -803,7 +803,7 @@ MultiFab::max (int comp, int nghost, bool local) const } if (!local) - ParallelAllReduce::Max(mx, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Max(mx, ParallelContext::CommunicatorSub()); return mx; } @@ -826,7 +826,7 @@ MultiFab::max (const Box& region, int comp, int nghost, bool local) const }); if (!local) - ParallelAllReduce::Max(mx, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Max(mx, ParallelContext::CommunicatorSub()); return mx; } @@ -894,7 +894,7 @@ MultiFab::norm0 (const iMultiFab& mask, int comp, int nghost, bool local) const return r; }); - if (!local) ParallelAllReduce::Max(nm0, ParallelContext::CommunicatorSub()); + if (!local) ParallelAllReduce::Max(nm0, ParallelContext::CommunicatorSub()); return nm0; } @@ -939,7 +939,7 @@ MultiFab::norm0 (int comp, int nghost, bool local, bool ignore_covered ) const } if (!local) - ParallelAllReduce::Max(nm0, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Max(nm0, ParallelContext::CommunicatorSub()); return nm0; } @@ -956,7 +956,7 @@ MultiFab::norm0 (const Vector& comps, int nghost, bool local, bool ignore_c } if (!local) - ParallelAllReduce::Max(nm0.dataPtr(), n, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Max(nm0.dataPtr(), n, ParallelContext::CommunicatorSub()); return nm0; } @@ -1045,7 +1045,7 @@ MultiFab::norm1 (int comp, int ngrow, bool local) const }); if (!local) - ParallelAllReduce::Sum(nm1, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Sum(nm1, ParallelContext::CommunicatorSub()); return nm1; } @@ -1064,7 +1064,7 @@ MultiFab::norm1 (const Vector& comps, int ngrow, bool local) const } if (!local) - ParallelAllReduce::Sum(nm1.dataPtr(), n, ParallelContext::CommunicatorSub()); + ParallelAllReduce::Sum(nm1.dataPtr(), n, ParallelContext::CommunicatorSub()); return nm1; } @@ -1232,7 +1232,7 @@ MultiFab::OverlapMask (const Periodicity& period) const #endif { std::vector< std::pair > isects; - + for (MFIter mfi(*p); mfi.isValid(); ++mfi) { const Box& bx = (*p)[mfi].box(); @@ -1297,13 +1297,13 @@ MultiFab::WeightedSync (const MultiFab& wgt, const Periodicity& period) BL_PROFILE("MultiFab::WeightedSync()"); if (ixType().cellCentered()) return; - + const int ncomp = nComp(); for (int comp = 0; comp < ncomp; ++comp) { MultiFab::Multiply(*this, wgt, 0, comp, 1, 0); } - + MultiFab tmpmf(boxArray(), DistributionMap(), ncomp, 0, MFInfo(), Factory()); tmpmf.setVal(0.0); tmpmf.ParallelCopy(*this, period, FabArrayBase::ADD); diff --git a/Src/Base/AMReX_MultiFabUtil.cpp b/Src/Base/AMReX_MultiFabUtil.cpp index da3f0c08ecf..5838ee4014b 100644 --- a/Src/Base/AMReX_MultiFabUtil.cpp +++ b/Src/Base/AMReX_MultiFabUtil.cpp @@ -107,7 +107,7 @@ namespace amrex } void average_face_to_cellcenter (MultiFab& cc, const Vector& fc, - const Geometry& geom) + const Geometry& geom) { average_face_to_cellcenter(cc, Array @@ -148,10 +148,10 @@ namespace amrex void average_face_to_cellcenter (MultiFab& cc, const Array& fc, - const Geometry& geom) + const Geometry& geom) { - AMREX_ASSERT(cc.nComp() >= AMREX_SPACEDIM); - AMREX_ASSERT(fc[0]->nComp() == 1); // We only expect fc to have the gradient perpendicular to the face + AMREX_ASSERT(cc.nComp() >= AMREX_SPACEDIM); + AMREX_ASSERT(fc[0]->nComp() == 1); // We only expect fc to have the gradient perpendicular to the face const GeometryData gd = geom.data(); amrex::ignore_unused(gd); @@ -182,7 +182,7 @@ namespace amrex } void average_cellcenter_to_face (const Vector& fc, const MultiFab& cc, - const Geometry& geom, int ncomp, bool use_harmonic_averaging) + const Geometry& geom, int ncomp, bool use_harmonic_averaging) { average_cellcenter_to_face(Array{{AMREX_D_DECL(fc[0],fc[1],fc[2])}}, cc, geom, ncomp, use_harmonic_averaging); @@ -192,21 +192,21 @@ namespace amrex void average_cellcenter_to_face (const Array& fc, const MultiFab& cc, const Geometry& geom, int ncomp, bool use_harmonic_averaging) { - AMREX_ASSERT(cc.nComp() == ncomp); - AMREX_ASSERT(cc.nGrow() >= 1); - AMREX_ASSERT(fc[0]->nComp() == ncomp); // We only expect fc to have the gradient perpendicular to the face + AMREX_ASSERT(cc.nComp() == ncomp); + AMREX_ASSERT(cc.nGrow() >= 1); + AMREX_ASSERT(fc[0]->nComp() == ncomp); // We only expect fc to have the gradient perpendicular to the face #if (AMREX_SPACEDIM >= 2) - AMREX_ASSERT(fc[1]->nComp() == ncomp); // We only expect fc to have the gradient perpendicular to the face + AMREX_ASSERT(fc[1]->nComp() == ncomp); // We only expect fc to have the gradient perpendicular to the face #endif #if (AMREX_SPACEDIM == 3) - AMREX_ASSERT(fc[2]->nComp() == ncomp); // We only expect fc to have the gradient perpendicular to the face + AMREX_ASSERT(fc[2]->nComp() == ncomp); // We only expect fc to have the gradient perpendicular to the face #endif #if (AMREX_SPACEDIM == 1) const GeometryData& gd = geom.data(); - if (use_harmonic_averaging) - AMREX_ASSERT(gd.Coord() == 0); + if (use_harmonic_averaging) + AMREX_ASSERT(gd.Coord() == 0); #else amrex::ignore_unused(geom); #endif @@ -214,8 +214,8 @@ namespace amrex #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(cc,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { + for (MFIter mfi(cc,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { AMREX_D_TERM(const Box& xbx = mfi.nodaltilebox(0);, const Box& ybx = mfi.nodaltilebox(1);, const Box& zbx = mfi.nodaltilebox(2);); @@ -229,18 +229,18 @@ namespace amrex #if (AMREX_SPACEDIM == 1) AMREX_LAUNCH_HOST_DEVICE_FUSIBLE_LAMBDA (index_bounds, tbx, { - amrex_avg_cc_to_fc(tbx, xbx, fxarr, ccarr, gd, ncomp, + amrex_avg_cc_to_fc(tbx, xbx, fxarr, ccarr, gd, ncomp, use_harmonic_averaging); }); #else AMREX_LAUNCH_HOST_DEVICE_FUSIBLE_LAMBDA (index_bounds, tbx, { amrex_avg_cc_to_fc(tbx, AMREX_D_DECL(xbx,ybx,zbx), - AMREX_D_DECL(fxarr,fyarr,fzarr), ccarr, ncomp, + AMREX_D_DECL(fxarr,fyarr,fzarr), ccarr, ncomp, use_harmonic_averaging); }); #endif - } + } } // ************************************************************************************************************* @@ -249,14 +249,14 @@ namespace amrex // We do NOT assume that the coarse layout is a coarsened version of the fine layout. // This version DOES use volume-weighting. void average_down (const MultiFab& S_fine, MultiFab& S_crse, - const Geometry& fgeom, const Geometry& cgeom, + const Geometry& fgeom, const Geometry& cgeom, int scomp, int ncomp, int rr) { average_down(S_fine,S_crse,fgeom,cgeom,scomp,ncomp,rr*IntVect::TheUnitVector()); } void average_down (const MultiFab& S_fine, MultiFab& S_crse, - const Geometry& fgeom, const Geometry& cgeom, + const Geometry& fgeom, const Geometry& cgeom, int scomp, int ncomp, const IntVect& ratio) { amrex::ignore_unused(fgeom,cgeom); @@ -269,8 +269,8 @@ namespace amrex } #if (AMREX_SPACEDIM == 3) - amrex::average_down(S_fine, S_crse, scomp, ncomp, ratio); - return; + amrex::average_down(S_fine, S_crse, scomp, ncomp, ratio); + return; #else AMREX_ASSERT(S_crse.nComp() == S_fine.nComp()); @@ -278,15 +278,15 @@ namespace amrex // // Coarsen() the fine stuff on processors owning the fine data. // - const BoxArray& fine_BA = S_fine.boxArray(); - const DistributionMapping& fine_dm = S_fine.DistributionMap(); + const BoxArray& fine_BA = S_fine.boxArray(); + const DistributionMapping& fine_dm = S_fine.DistributionMap(); BoxArray crse_S_fine_BA = fine_BA; - crse_S_fine_BA.coarsen(ratio); + crse_S_fine_BA.coarsen(ratio); MultiFab crse_S_fine(crse_S_fine_BA,fine_dm,ncomp,0,MFInfo(),FArrayBoxFactory()); - MultiFab fvolume; - fgeom.GetVolume(fvolume, fine_BA, fine_dm, 0); + MultiFab fvolume; + fgeom.GetVolume(fvolume, fine_BA, fine_dm, 0); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) @@ -304,7 +304,7 @@ namespace amrex amrex_avgdown_with_vol(tbx,crsearr,finearr,finevolarr, 0,scomp,ncomp,ratio); }); - } + } S_crse.copy(crse_S_fine,0,scomp,ncomp); #endif @@ -462,7 +462,7 @@ namespace amrex // Average fine face-based MultiFab onto crse face-based MultiFab. void average_down_faces (const Array& fine, const Array& crse, - const IntVect& ratio, int ngcrse) + const IntVect& ratio, int ngcrse) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -473,7 +473,7 @@ namespace amrex void average_down_faces (const MultiFab& fine, MultiFab& crse, const IntVect& ratio, int ngcrse) { - AMREX_ASSERT(crse.nComp() == fine.nComp()); + AMREX_ASSERT(crse.nComp() == fine.nComp()); AMREX_ASSERT(fine.ixType() == crse.ixType()); const auto type = fine.ixType(); int dir; @@ -555,7 +555,7 @@ namespace amrex void average_down_edges (const MultiFab& fine, MultiFab& crse, const IntVect& ratio, int ngcrse) { - AMREX_ASSERT(crse.nComp() == fine.nComp()); + AMREX_ASSERT(crse.nComp() == fine.nComp()); AMREX_ASSERT(fine.ixType() == crse.ixType()); const auto type = fine.ixType(); int dir; diff --git a/Src/Base/AMReX_MultiFabUtil_1D_C.H b/Src/Base/AMReX_MultiFabUtil_1D_C.H index 69c953eaa30..c994459c9f9 100644 --- a/Src/Base/AMReX_MultiFabUtil_1D_C.H +++ b/Src/Base/AMReX_MultiFabUtil_1D_C.H @@ -126,7 +126,7 @@ void amrex_avg_cc_to_fc (Box const& ndbx, Box const& xbx, fx(i,0,0,n) = Real(0.0); else { - fx(i,0,0,n) = Real(2.0) / ( Real(1.0) / cc(i-1,0,0,n) + Real(1.0) / cc(i,0,0,n) ); + fx(i,0,0,n) = Real(2.0) / ( Real(1.0) / cc(i-1,0,0,n) + Real(1.0) / cc(i,0,0,n) ); } }} } else { @@ -142,7 +142,7 @@ void amrex_avg_cc_to_fc (Box const& ndbx, Box const& xbx, { const Real problo = gd.ProbLo(0); const Real dx = gd.CellSize(0); - + for (int n = 0; n < ncomp; ++n) { AMREX_PRAGMA_SIMD for (int i = xlo.x; i <= xhi.x; ++i) { @@ -157,7 +157,7 @@ void amrex_avg_cc_to_fc (Box const& ndbx, Box const& xbx, { const Real problo = gd.ProbLo(0); const Real dx = gd.CellSize(0); - + for (int n = 0; n < ncomp; ++n) { AMREX_PRAGMA_SIMD for (int i = xlo.x; i <= xhi.x; ++i) { diff --git a/Src/Base/AMReX_NFiles.H b/Src/Base/AMReX_NFiles.H index adbba94749d..d2f832ce621 100644 --- a/Src/Base/AMReX_NFiles.H +++ b/Src/Base/AMReX_NFiles.H @@ -148,7 +148,7 @@ class NFilesIter return(whichProc % anf); } else { - int nProcs(ParallelDescriptor::NProcs()); + int nProcs(ParallelDescriptor::NProcs()); return(whichProc / LengthOfSet(nProcs, anf)); } } @@ -158,13 +158,13 @@ class NFilesIter static std::string FileName(int nOutFiles, const std::string &filePrefix, - int whichProc, bool groupSets) + int whichProc, bool groupSets) { BL_ASSERT(whichProc >= 0 && whichProc < ParallelDescriptor::NProcs()); return ( amrex::Concatenate(filePrefix, FileNumber(ActualNFiles(nOutFiles), - whichProc, groupSets), minDigits ) ); + whichProc, groupSets), minDigits ) ); } static std::string FileName(int fileNumber, diff --git a/Src/Base/AMReX_NFiles.cpp b/Src/Base/AMReX_NFiles.cpp index 0709dbc3141..9118e8572d6 100644 --- a/Src/Base/AMReX_NFiles.cpp +++ b/Src/Base/AMReX_NFiles.cpp @@ -162,7 +162,7 @@ void NFilesIter::SetSparseFPP(const Vector &ranksToWrite) NFilesIter::NFilesIter(const std::string &filename, - const Vector &readranks, + const Vector &readranks, bool setBuf) { isReading = true; @@ -368,7 +368,7 @@ NFilesIter &NFilesIter::operator++() { if(mySparseFileNumber != -1) { fileStream.flush(); fileStream.close(); - } + } finishedWriting = true; } else { // ---- the general static set selection @@ -412,11 +412,11 @@ NFilesIter &NFilesIter::operator++() { for(int i(0); i < nProcs; ++i) { int procSet(WhichSetPosition(i, nProcs, nOutFiles, groupSets)); int whichFileNumber(NFilesIter::FileNumber(nOutFiles, i, groupSets)); - // ---- procSet == 0 have already written their data - if(procSet == 0) { - fileNumbersWriteOrder[whichFileNumber].push_back(i); + // ---- procSet == 0 have already written their data + if(procSet == 0) { + fileNumbersWriteOrder[whichFileNumber].push_back(i); --remainingWriters; - } + } if(procSet != 0) { procsToWrite[whichFileNumber].push_back(i); } @@ -444,21 +444,21 @@ NFilesIter &NFilesIter::operator++() { break; // ---- found one } } - if(nextProcToWrite == -1) { + if(nextProcToWrite == -1) { --remainingWriters; -// amrex::Print() << myProc << "::IOIOIOIO: nptw == -1 rW = " << remainingWriters << std::endl; - } else { +// amrex::Print() << myProc << "::IOIOIOIO: nptw == -1 rW = " << remainingWriters << std::endl; + } else { - fileNumbersWriteOrder[nextFileNumberToWrite].push_back(nextProcToWrite); + fileNumbersWriteOrder[nextFileNumberToWrite].push_back(nextProcToWrite); ParallelDescriptor::Send(&nextFileNumberToWrite, 1, nextProcToWrite, writeTag); - + rmess = ParallelDescriptor::Recv(&nextFileNumberAvailable, 1, MPI_ANY_SOURCE, doneTag); availableFileNumbers.insert(nextFileNumberAvailable); --remainingWriters; - } + } } - unreadMessages.push_back(std::make_pair(doneTag, setZeroProcs.size() - 1)); + unreadMessages.push_back(std::make_pair(doneTag, setZeroProcs.size() - 1)); } else { // ---- tell the coordinatorProc we are done writing ParallelDescriptor::Send(&fileNumber, 1, coordinatorProc, doneTag); @@ -537,7 +537,7 @@ Vector NFilesIter::FileNumbersWritten() amrex::AllPrint() << "**** Error in NFilesIter::FileNumbersWritten(): " << " coordinatorProc nProcs total procSet.size() = " << coordinatorProc << " " << nProcs << " " - << total << " " << procSet.size() << std::endl; + << total << " " << procSet.size() << std::endl; } #endif diff --git a/Src/Base/AMReX_NonLocalBC.H b/Src/Base/AMReX_NonLocalBC.H index b58dcb351ed..c86b897386f 100644 --- a/Src/Base/AMReX_NonLocalBC.H +++ b/Src/Base/AMReX_NonLocalBC.H @@ -22,7 +22,7 @@ struct HasInverseMemFn : IsDetectedExact {}; //! coordinate systems. template struct IsIndexMapping - : Conjunction, + : Conjunction, HasInverseMemFn> {}; //////////////////////////////////////////////////////////////////////////////////// @@ -97,8 +97,8 @@ struct MultiBlockIndexMapping { //! //! \return Returns IntVect{dtos(Dim3{iv})} template -EnableIf_t::value, IntVect> -Apply(DTOS dtos, const IntVect& iv) +EnableIf_t::value, IntVect> +Apply(DTOS dtos, const IntVect& iv) { Dim3 i = dtos(iv.dim3()); return IntVect{AMREX_D_DECL(i.x, i.y, i.z)}; @@ -113,8 +113,8 @@ Apply(DTOS dtos, const IntVect& iv) //! \return Returns the smallest Box in the source index space that contains images of //! Apply(dtos, box.smallEnd()) and Apply(dtos, box.bigEnd()). template -EnableIf_t::value && !IsCallableR::value, Box> -Image(DTOS dtos, const Box& box) +EnableIf_t::value && !IsCallableR::value, Box> +Image(DTOS dtos, const Box& box) { IntVect mapped_smallEnd = Apply(dtos, box.smallEnd()); IntVect mapped_bigEnd = Apply(dtos, box.bigEnd()); @@ -136,8 +136,8 @@ Image(DTOS dtos, const Box& box) //! \return Returns the smallest Box in the source index space that contains images of //! Apply(dtos, box.smallEnd()) and Apply(dtos, box.bigEnd()). template -EnableIf_t::value && IsCallableR::value, Box> -Image(DTOS dtos, const Box& box) +EnableIf_t::value && IsCallableR::value, Box> +Image(DTOS dtos, const Box& box) { // "Forget" the index type mapping and invoke Image without changing the index type. Box srcbox = Image([&dtos](Dim3 i) { return dtos(i); }, box); @@ -158,8 +158,8 @@ Image(DTOS dtos, const Box& box) //! //! \return Returns IntVect{dtos.Inverse(Dim3{iv})} template -EnableIf_t::value, IntVect> -ApplyInverse(DTOS dtos, const IntVect& iv) +EnableIf_t::value, IntVect> +ApplyInverse(DTOS dtos, const IntVect& iv) { return Apply([&dtos](Dim3 i) { return dtos.Inverse(i); }, iv); } @@ -173,8 +173,8 @@ ApplyInverse(DTOS dtos, const IntVect& iv) //! \return Returns the smallest Box in the destination index space that contains images of //! ApplyInverse(box.smallEnd()) and ApplyInverse(box.bigEnd()). template -EnableIf_t::value && !IsCallableR::value, Box> -InverseImage(DTOS dtos, const Box& box) +EnableIf_t::value && !IsCallableR::value, Box> +InverseImage(DTOS dtos, const Box& box) { return Image([&dtos](Dim3 i) { return dtos.Inverse(i); }, box); } @@ -188,8 +188,8 @@ InverseImage(DTOS dtos, const Box& box) //! \return Returns the smallest Box in the destination index space that contains images of //! ApplyInverse(box.smallEnd()) and ApplyInverse(box.bigEnd()). template -EnableIf_t::value && IsCallableR::value, Box> -InverseImage(DTOS dtos, const Box& box) +EnableIf_t::value && IsCallableR::value, Box> +InverseImage(DTOS dtos, const Box& box) { return Image([&dtos](auto&& i) { return dtos.Inverse(i); }, box); } @@ -212,7 +212,7 @@ struct MultiBlockCommMetaData : FabArrayBase::CommMetaData { //! \brief Build global meta data by calling the define() member function. //! //! \see MultiBlockCommMetaData::define - template ::value>> MultiBlockCommMetaData(const FabArrayBase& dst, const Box& dstbox, const FabArrayBase& src, const IntVect& ngrow, DTOS dtos); @@ -220,7 +220,7 @@ struct MultiBlockCommMetaData : FabArrayBase::CommMetaData { //! \brief Build global meta data by calling the define() member function. //! //! \see MultiBlockCommMetaData::define - template ::value>> MultiBlockCommMetaData(const BoxArray& dstba, const DistributionMapping& dstdm, const Box& dstbox, const BoxArray& srcba, @@ -238,7 +238,7 @@ struct MultiBlockCommMetaData : FabArrayBase::CommMetaData { //! \tparam DTOS This parameter needs to satisfy IsIndexMapping and HasInverseImage. //! //! \param[in] dstba The destination box array. - //! + //! //! \param[in] dstdm The destination distribution mapping. //! //! \param[in] dstbox The box that will be filled and that lives in the destination space. @@ -266,7 +266,7 @@ struct MultiBlockCommMetaData : FabArrayBase::CommMetaData { //! \brief This type trait tests if a type P is a projetion for FAB. template struct IsFabProjection - : IsCallableR, Dim3, int> + : IsCallableR, Dim3, int> {}; //////////////////////////////////////////////////////////////////////////////////// @@ -344,7 +344,7 @@ static_assert(IsFabProjection, FArrayBox>(), ""); //! This class is used with MapComponents to define fab projections that swap components on FABs, //! e.g. for velocity components. The template arguments I, J should be greater or equal to -1. The //! value -1 indicates run-time values and increase the object size of this function object. -template struct SwapComponents { +template struct SwapComponents { static_assert(I >= 0 && J >= 0, "I >= 0 && J >= 0"); //! \brief Swaps indices I and J. @@ -368,7 +368,7 @@ template struct SwapComponents { template struct SwapComponents<-1, J> { static_assert(J >= 0, "J >= 0"); - + int I; constexpr int operator()(int i) const noexcept { const int map[2] = {I, J}; @@ -415,7 +415,7 @@ static_assert(DynamicSwapComponents{0, 1}(2) == 2, ""); //! \brief This class holds data buffers for either immediate MPI send or recv calls. //! //! The data buffers need to be kept alive until all MPI_Request have been completed. -//! All member variables are resized a corresponding call to PrepareCommBuffers and +//! All member variables are resized a corresponding call to PrepareCommBuffers and //! the data transactions are initiated either by calls to PostRecvs or PostSends. struct CommData { #ifdef AMREX_USE_MPI @@ -433,7 +433,7 @@ struct CommData { Vector request{}; //! \brief For each request the corresponding MPI_status, used for debugging. Vector stats{}; - //! \brief For each requst the copy comm tags for the corresponding data FABs. + //! \brief For each requst the copy comm tags for the corresponding data FABs. Vector cctc{}; #endif }; @@ -443,10 +443,10 @@ struct CommData { void PrepareCommBuffers(CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc, int n_components, std::size_t object_size, std::size_t align); -//! \brief Initiate all recvieves with MPI_Irecv calls associated with tag mpi_tag. +//! \brief Initiate all recvieves with MPI_Irecv calls associated with tag mpi_tag. void PostRecvs(CommData& comm, int mpi_tag); -//! \brief Initiate all sends with MPI_Isend calls associated with tag mpi_tag. +//! \brief Initiate all sends with MPI_Isend calls associated with tag mpi_tag. void PostSends(CommData& comm, int mpi_tag); #endif @@ -482,7 +482,7 @@ using LocalCopy_t = decltype(LocalCopy(std::declval()...)); //! \brief This type trait tests if a given type DP satisfies the DataPacking concept for type FAB. template -struct IsDataPacking : +struct IsDataPacking : Conjunction< IsDetected&, const FabArray&, const FabArrayBase::CopyComTagsContainer&> #ifdef AMREX_USE_MPI @@ -497,7 +497,7 @@ struct IsDataPacking : // [DataPacking.PackComponents] // // PackComponents is the simplest data packing policy. -// This provides us with sane default behaviour that we can use when defining new +// This provides us with sane default behaviour that we can use when defining new // data packing policies. //! \brief Contains information about which components take part of the data transaction. @@ -538,7 +538,7 @@ PrepareSendBuffers (const PackComponents& components, FabArray& dest, const //! \brief Calls PrepareComBuffers. template amrex::EnableIf_t::value> -PrepareRecvBuffers (const PackComponents& components, FabArray& dest, const FabArray& src, +PrepareRecvBuffers (const PackComponents& components, FabArray& dest, const FabArray& src, CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc) { using T = typename FAB::value_type; ignore_unused(dest, src); @@ -586,7 +586,7 @@ static_assert(IsDataPacking(), // //! \brief This class specializes behaviour on local copies and unpacking recieve buffers. //! -//! It takes a DTOS and FabProjection to apply them on the reciever side. +//! It takes a DTOS and FabProjection to apply them on the reciever side. template struct ApplyDtosAndProjectionOnReciever : PackComponents { constexpr ApplyDtosAndProjectionOnReciever() = default; @@ -669,11 +669,11 @@ static constexpr struct DoLocalCopy {} do_local_copy{}; //! //! \return Returns a CommHandler object that owns context and memory buffers for the whole life //! time of the MPI transaction. -template ::value>, typename = EnableIf_t::value>> #ifdef AMREX_USE_MPI -AMREX_NODISCARD CommHandler +AMREX_NODISCARD CommHandler ParallelCopy_nowait (NoLocalCopy, FabArray& dest, const FabArray& src, const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing) { CommHandler handler{}; @@ -728,13 +728,13 @@ CommHandler ParallelCopy_nowait (NoLocalCopy, FabArray&, const FabArray::value>, typename = EnableIf_t::value>> #ifdef AMREX_USE_MPI -AMREX_NODISCARD +AMREX_NODISCARD #endif -CommHandler +CommHandler ParallelCopy_nowait (FabArray& dest, const FabArray& src, const FabArrayBase::CommMetaData& cmd, const DataPacking& data_packing) { CommHandler comm = ParallelCopy_nowait(no_local_copy, dest, src, cmd, data_packing); @@ -900,12 +900,12 @@ ParallelCopy (FabArray& dest, const FabArray& src, const FabArrayBase: //! \brief Call ParallelCopy_nowait followed by ParallelCopy_finish, strong typed version. //! -//! This function constructs a new MultiCommMetaData from the given DTOS, destbox and ngrow. +//! This function constructs a new MultiCommMetaData from the given DTOS, destbox and ngrow. //! //! \param[out] dest The Multifab that is going to be filled with recieved data. //! -//! \param[in] destbox The index box in the destination space that will be filled by data from -//! src. The source box that describes the dependencies will be computed by +//! \param[in] destbox The index box in the destination space that will be filled by data from +//! src. The source box that describes the dependencies will be computed by //! the specified DTOS. //! //! \param[in] src The Multifab that is used to fill the send buffers. @@ -917,9 +917,9 @@ ParallelCopy (FabArray& dest, const FabArray& src, const FabArrayBase: //! \param[in] ncomp The number of successive components that will be copied. //! //! \param[in] ngrow The amount of ghost cells that will be taking into consideration. -//! Note, even if destbox contains indices outside the domain we need to +//! Note, even if destbox contains indices outside the domain we need to //! specify an appropiate ngrow that covers the amound of ghost cells that -//! we want to copy. +//! we want to copy. //! //! \param[in] dtos An index mapping that maps indices from destination space to source space //! and from source space to destination space. @@ -939,12 +939,12 @@ ParallelCopy (FabArray& dest, const Box& destbox, const FabArray& src, //! \brief Call ParallelCopy_nowait followed by ParallelCopy_finish. //! -//! This function constructs a new MultiCommMetaData from the given DTOS, destbox and ngrow. +//! This function constructs a new MultiCommMetaData from the given DTOS, destbox and ngrow. //! //! \param[out] dest The Multifab that is going to be filled with recieved data. //! -//! \param[in] destbox The index box in the destination space that will be filled by data from -//! src. The source box that describes the dependencies will be computed by +//! \param[in] destbox The index box in the destination space that will be filled by data from +//! src. The source box that describes the dependencies will be computed by //! the specified DTOS. //! //! \param[in] src The Multifab that is used to fill the send buffers. @@ -956,9 +956,9 @@ ParallelCopy (FabArray& dest, const Box& destbox, const FabArray& src, //! \param[in] ncomp The number of successive components that will be copied. //! //! \param[in] ngrow The amount of ghost cells that will be taking into consideration. -//! Note, even if destbox contains indices outside the domain we need to +//! Note, even if destbox contains indices outside the domain we need to //! specify an appropiate ngrow that covers the amound of ghost cells that -//! we want to copy. +//! we want to copy. //! //! \param[in] dtos An index mapping that maps indices from destination space to source space //! and from source space to destination space. diff --git a/Src/Base/AMReX_NonLocalBC.cpp b/Src/Base/AMReX_NonLocalBC.cpp index 78d97bca584..9b2a6b38c5b 100644 --- a/Src/Base/AMReX_NonLocalBC.cpp +++ b/Src/Base/AMReX_NonLocalBC.cpp @@ -4,7 +4,7 @@ namespace amrex { namespace NonLocalBC { #ifdef AMREX_USE_MPI // Note, this is copied and modified from PrepareSendBuffers and PostRcvs -void PrepareCommBuffers(CommData& comm, +void PrepareCommBuffers(CommData& comm, const FabArrayBase::MapOfCopyComTagContainers& cctc, int n_components, std::size_t object_size, std::size_t align) { @@ -34,7 +34,7 @@ void PrepareCommBuffers(CommData& comm, std::size_t nbytes = 0; for (auto const& cct : kv.second) { - // Note: Does this hold for all FAB types? + // Note: Does this hold for all FAB types? // This nBytes() implementation is currently also assumed in unpack_recv_buffers nbytes += cct.sbox.numPts() * object_size * n_components; } diff --git a/Src/Base/AMReX_NonLocalBCImpl.H b/Src/Base/AMReX_NonLocalBCImpl.H index 36869ceba58..52707edd2ce 100644 --- a/Src/Base/AMReX_NonLocalBCImpl.H +++ b/Src/Base/AMReX_NonLocalBCImpl.H @@ -353,7 +353,7 @@ MultiBlockCommMetaData::MultiBlockCommMetaData (const BoxArray& dstba, const Dis template EnableIf_t::value> MultiBlockCommMetaData::define (const BoxArray& dstba, const DistributionMapping& dstdm, const Box& dstbox, - const BoxArray& srcba, const DistributionMapping& srcdm, const IntVect& ngrow, + const BoxArray& srcba, const DistributionMapping& srcdm, const IntVect& ngrow, DTOS dtos) { m_LocTags = std::make_unique(); m_SndTags = std::make_unique(); diff --git a/Src/Base/AMReX_PODVector.H b/Src/Base/AMReX_PODVector.H index c423bca3dd7..a23d4e2862b 100644 --- a/Src/Base/AMReX_PODVector.H +++ b/Src/Base/AMReX_PODVector.H @@ -96,36 +96,36 @@ namespace amrex for (const auto& value : list) { *(data++) = value; } } } - + template > class PODVector : public Allocator - { + { // static_assert(std::is_standard_layout(), "PODVector can only hold standard layout types"); static_assert(AMREX_IS_TRIVIALLY_COPYABLE(T), "PODVector can only hold trivially copyable types"); // static_assert(std::is_trivially_default_constructible(), "PODVector can only hold trivial dc types"); - + using Allocator::allocate; using Allocator::deallocate; - - public: + + public: typedef T value_type; typedef Allocator allocator_type; typedef std::size_t size_t; typedef std::size_t size_type; typedef std::ptrdiff_t difference_type; - + typedef T& reference; typedef T* pointer; typedef T* iterator; typedef std::reverse_iterator reverse_iterator; - + typedef const T& const_reference; typedef const T* const_pointer; typedef const T* const_iterator; typedef std::reverse_iterator const_reverse_iterator; - - private: + + private: pointer m_data; size_type m_size, m_capacity; @@ -133,37 +133,37 @@ namespace amrex explicit PODVector () noexcept : Allocator(), m_data(nullptr), m_size(0), m_capacity(0) {} - + explicit PODVector (const allocator_type& a_allocator) noexcept : Allocator(a_allocator), m_data(nullptr), m_size(0), m_capacity(0) {} - + explicit PODVector (size_type a_size) noexcept - : m_data(nullptr), m_size(0), m_capacity(0) + : m_data(nullptr), m_size(0), m_capacity(0) { AllocateBuffer(GetNewCapacity(a_size)); m_size = a_size; } - + PODVector (size_type a_size, const value_type& a_value, const allocator_type& a_allocator = Allocator()) noexcept - : Allocator(a_allocator), m_data(nullptr), m_size(0), m_capacity(0) + : Allocator(a_allocator), m_data(nullptr), m_size(0), m_capacity(0) { AllocateBuffer(GetNewCapacity(a_size)); m_size = a_size; detail::uninitializedFillNImpl(m_data, a_size, a_value); } - + PODVector (std::initializer_list a_initializer_list, const allocator_type& a_allocator = Allocator()) noexcept - : Allocator(a_allocator), m_data(nullptr), m_size(0), m_capacity(0) + : Allocator(a_allocator), m_data(nullptr), m_size(0), m_capacity(0) { size_type size = a_initializer_list.size(); AllocateBuffer(GetNewCapacity(size)); - m_size = size; + m_size = size; detail::initFromListImpl(m_data, a_initializer_list); } - + PODVector (const PODVector& a_vector) noexcept : Allocator(a_vector), m_data(nullptr), m_size(0), m_capacity(0) { @@ -172,7 +172,7 @@ namespace amrex m_size = a_vector.size(); memCopyImpl(m_data, a_vector.m_data, a_vector.size() * sizeof(T)); } - + PODVector (PODVector&& a_vector) noexcept : Allocator(std::move(a_vector)), m_data(a_vector.m_data), @@ -183,7 +183,7 @@ namespace amrex a_vector.m_size = 0; a_vector.m_capacity = 0; } - + PODVector (PODVector&& a_vector, const allocator_type& a_allocator) noexcept : Allocator(a_allocator), m_data(a_vector.m_data), @@ -194,29 +194,29 @@ namespace amrex a_vector.m_size = 0; a_vector.m_capacity = 0; } - + ~PODVector () noexcept { deallocate(m_data, capacity()); } - + PODVector& operator= (const PODVector& a_vector) noexcept { return CopyAssignmentDoIt(a_vector, typename std::allocator_traits:: propagate_on_container_copy_assignment()); } - + PODVector& operator= (PODVector&& a_vector) noexcept (std::allocator_traits::propagate_on_container_move_assignment::value) { return MoveAssignmentDoIt(std::move(a_vector), typename std::allocator_traits:: propagate_on_container_move_assignment()); } - + iterator erase (const_iterator a_pos) noexcept { --m_size; detail::memMoveImpl(const_cast(a_pos), a_pos+1, (end() - a_pos)*sizeof(T), *this); return const_cast(a_pos); } - + iterator erase (const_iterator a_first, const_iterator a_last) noexcept { size_type num_to_erase = a_last - a_first; @@ -225,7 +225,7 @@ namespace amrex detail::memMoveImpl(const_cast(a_first), a_first+num_to_erase, (end() - a_first)*sizeof(T), *this); return const_cast(a_first); } - + iterator insert (const_iterator a_pos, const T& a_item) noexcept { if(m_size == m_capacity) @@ -240,12 +240,12 @@ namespace amrex detail::memMoveImpl(const_cast(a_pos)+1, a_pos, (end() - a_pos) * sizeof(T), *this); ++m_size; } - + *const_cast(a_pos) = a_item; - + return const_cast(a_pos); } - + iterator insert (const_iterator a_pos, size_type a_count, const T& a_value) noexcept { if (a_count == 0) return const_cast(a_pos); @@ -263,10 +263,10 @@ namespace amrex } detail::uninitializedFillNImpl(const_cast(a_pos), a_count, a_value); - + return const_cast(a_pos); } - + iterator insert (const_iterator a_pos, T&& a_item) noexcept { if(m_size == m_capacity) @@ -281,12 +281,12 @@ namespace amrex detail::memMoveImpl(const_cast(a_pos)+1, a_pos, (end() - a_pos) * sizeof(T), *this); ++m_size; } - + *const_cast(a_pos) = a_item; - + return const_cast(a_pos); } - + iterator insert (const_iterator a_pos, std::initializer_list a_initializer_list) noexcept { size_type count = a_initializer_list.size(); @@ -303,10 +303,10 @@ namespace amrex m_size += count; } // copy the initializer list pointer dst = const_cast(a_pos); - for (const auto& value : a_initializer_list) { *(dst++) = value; } + for (const auto& value : a_initializer_list) { *(dst++) = value; } return const_cast(a_pos); - } - + } + template ::difference_type> iterator insert (const_iterator a_pos, InputIt a_first, InputIt a_last) noexcept { @@ -327,14 +327,14 @@ namespace amrex while(a_first != a_last) { *(dst++) = *(a_first++); } return const_cast(a_pos); } - + void assign (size_type a_count, const T& a_value) noexcept { if ( a_count > capacity() ) AllocateBuffer(GetNewCapacity(a_count)); m_size = a_count; detail::uninitializedFillNImpl(m_data, a_count, a_value); - } - + } + void assign (std::initializer_list a_initializer_list) noexcept { if(a_initializer_list.size() > capacity()) @@ -342,8 +342,8 @@ namespace amrex m_size = a_initializer_list.size(); // copy the initializer list pointer dst = const_cast(m_data); for (const auto& value : a_initializer_list) { *(dst++) = value; } - } - + } + template ::difference_type> void assign (InputIt a_first, InputIt a_last) noexcept { @@ -352,86 +352,86 @@ namespace amrex m_size = count; pointer dst = const_cast(m_data); while(a_first != a_last) { *(dst++) = *(a_first++); } } - - // don't have the emplace methods, but not sure how often we use those. + + // don't have the emplace methods, but not sure how often we use those. allocator_type get_allocator () const noexcept { return *this; } - + void push_back (const T& a_value) noexcept { if (m_size == m_capacity) AllocateBuffer(GetNewCapacity(1)); m_data[m_size] = a_value; ++m_size; } - + void push_back (T&& a_value) noexcept { if (m_size == m_capacity) AllocateBuffer(GetNewCapacity(1)); m_data[m_size] = std::move(a_value); ++m_size; } - + void pop_back () noexcept { --m_size; } - + void clear () noexcept { m_size = 0; } - + size_type size () const noexcept { return m_size; } - + size_type max_size () const noexcept { return Allocator::max_size(); } - + size_type capacity () const noexcept { return m_capacity; } - + bool empty () const noexcept { return m_size == 0; } - + T& operator[] (size_type a_index) noexcept { return m_data[a_index]; } - + const T& operator[] (size_type a_index) const noexcept { return m_data[a_index]; } - + T& front () noexcept { return *m_data; } - + const T& front () const noexcept { return *m_data; } - + T& back () noexcept { return *(m_data + m_size - 1); } - + const T& back () const noexcept { return *(m_data + m_size - 1); } - + T* data () noexcept { return m_data; } - + const T* data () const noexcept { return m_data; } - + T* dataPtr () noexcept { return m_data; } - + const T* dataPtr () const noexcept { return m_data; } - + iterator begin () noexcept { return m_data; } - + const_iterator begin () const noexcept { return m_data; } - + iterator end () noexcept { return m_data + m_size; } - + const_iterator end () const noexcept { return m_data + m_size; } - + reverse_iterator rbegin () noexcept { return reverse_iterator(end()); } - + const_reverse_iterator rbegin () const noexcept { return const_reverse_iterator(end()); } - + reverse_iterator rend () noexcept { return reverse_iterator(begin()); } - + const_reverse_iterator rend () const noexcept { return const_reverse_iterator(begin()); } - + const_iterator cbegin () const noexcept { return m_data; } - + const_iterator cend () const noexcept { return m_data + m_size; } - + const_reverse_iterator crbegin () const noexcept { return const_reverse_iterator(end()); } - + const_reverse_iterator crend () const noexcept { return const_reverse_iterator(begin()); } - + void resize (size_type a_new_size) noexcept { if (capacity() < a_new_size) AllocateBuffer(GetNewCapacity(a_new_size - capacity())); m_size = a_new_size; } - + void resize (size_type a_new_size, const T& a_val) noexcept { size_type old_size = size(); @@ -443,12 +443,12 @@ namespace amrex a_val); } } - - void reserve (size_type a_capacity) noexcept + + void reserve (size_type a_capacity) noexcept { if(capacity() < a_capacity) AllocateBuffer(a_capacity); } - + void shrink_to_fit () noexcept { const size_type current_size = size(); @@ -464,7 +464,7 @@ namespace amrex AllocateBuffer(current_size); } } - + void swap (PODVector& a_vector) noexcept { std::swap(m_data, a_vector.m_data); @@ -472,27 +472,27 @@ namespace amrex std::swap(m_capacity, a_vector.m_capacity); std::swap(static_cast(a_vector), static_cast(*this)); } - + private: - + // this is where we would change the growth strategy size_type GetNewCapacity (size_type a_num_to_be_added) const noexcept { size_type new_capacity = capacity(); - + if (capacity() == 0) { new_capacity = std::max(64 / sizeof(T), size_type(1)); } - + while (new_capacity < (capacity() + a_num_to_be_added)) { new_capacity = (3 * new_capacity + 1)/2; } - + return new_capacity; } - + // this is where we would play games with the allocator void AllocateBuffer (size_type a_capacity) noexcept { @@ -502,7 +502,7 @@ namespace amrex m_data = new_data; m_capacity = a_capacity; } - + // need to have this version too void AllocateBufferForInsert(size_type a_capacity, size_type a_index, size_type a_count) noexcept { @@ -519,16 +519,16 @@ namespace amrex m_size = size() + a_count; m_capacity = a_capacity; } - + PODVector& CopyAssignmentDoIt (const PODVector& a_vector, std::true_type) noexcept { - const size_t other_size = a_vector.size(); - if ( other_size > capacity() ) { AllocateBuffer(other_size); } - m_size = other_size; + const size_t other_size = a_vector.size(); + if ( other_size > capacity() ) { AllocateBuffer(other_size); } + m_size = other_size; detail::memCopyImpl(m_data, a_vector.m_data, size() * sizeof(T)); return *this; } - + PODVector& CopyAssignmentDoIt (const PODVector& a_vector, std::false_type) noexcept { if (static_cast(a_vector) == static_cast(*this)) @@ -537,25 +537,25 @@ namespace amrex } else { - const size_t other_size = a_vector.size(); - if ( other_size > capacity() ) { AllocateBuffer(other_size); } - m_size = other_size; + const size_t other_size = a_vector.size(); + if ( other_size > capacity() ) { AllocateBuffer(other_size); } + m_size = other_size; detail::memCopyImpl(m_data, a_vector.m_data, size() * sizeof(T)); Allocator::operator=(static_cast(a_vector)); return *this; } } - + PODVector& MoveAssignmentDoIt (PODVector&& a_vector, std::false_type) noexcept { if(static_cast(a_vector) == static_cast(*this)) { deallocate(m_data, capacity()); - + m_data = a_vector.m_data; m_size = a_vector.m_size; m_capacity = a_vector.m_capacity; - + a_vector.m_data = nullptr; a_vector.m_size = 0; a_vector.m_capacity = 0; @@ -565,24 +565,24 @@ namespace amrex // if the allocators are not the same we give up and copy CopyAssignmentDoIt(a_vector, std::false_type()); } - + return *this; } - + PODVector& MoveAssignmentDoIt (PODVector&& a_vector, std::true_type) noexcept { deallocate(m_data, capacity()); - + Allocator::operator=(std::move(static_cast(a_vector))); - + m_data = a_vector.m_data; m_size = a_vector.m_size; m_capacity = a_vector.m_capacity; - + a_vector.m_data = nullptr; a_vector.m_size = 0; a_vector.m_capacity = 0; - + return *this; } }; diff --git a/Src/Base/AMReX_ParallelContext.cpp b/Src/Base/AMReX_ParallelContext.cpp index 2408430558d..6f369b08801 100644 --- a/Src/Base/AMReX_ParallelContext.cpp +++ b/Src/Base/AMReX_ParallelContext.cpp @@ -101,7 +101,7 @@ Frame::global_to_local_rank (int* local, const int* global, std::size_t n) const #else amrex::ignore_unused(global); for (std::size_t i = 0; i < n; ++i) local[i] = 0; -#endif +#endif } int diff --git a/Src/Base/AMReX_ParallelDescriptor.H b/Src/Base/AMReX_ParallelDescriptor.H index 4336c1982c4..8098cb6fdf4 100644 --- a/Src/Base/AMReX_ParallelDescriptor.H +++ b/Src/Base/AMReX_ParallelDescriptor.H @@ -58,46 +58,46 @@ namespace ParallelDescriptor { public: - Message () : + Message () : m_finished(true), m_type(MPI_DATATYPE_NULL), m_req(MPI_REQUEST_NULL) {} - Message (MPI_Request req_, MPI_Datatype type_) : + Message (MPI_Request req_, MPI_Datatype type_) : m_finished(false), m_type(type_), m_req(req_) {} - Message (MPI_Status stat_, MPI_Datatype type_) : + Message (MPI_Status stat_, MPI_Datatype type_) : m_finished(true), m_type(type_), m_req(MPI_REQUEST_NULL), m_stat(stat_) {} - void wait (); - bool test (); - size_t count () const; - int tag () const; - int pid () const; - MPI_Datatype type () const { return m_type; } - MPI_Request req () const { return m_req; } - MPI_Status stat () const { return m_stat; } + void wait (); + bool test (); + size_t count () const; + int tag () const; + int pid () const; + MPI_Datatype type () const { return m_type; } + MPI_Request req () const { return m_req; } + MPI_Status stat () const { return m_stat; } private: - bool m_finished; - MPI_Datatype m_type; - MPI_Request m_req; - mutable MPI_Status m_stat; + bool m_finished; + MPI_Datatype m_type; + MPI_Request m_req; + mutable MPI_Status m_stat; }; #ifdef BL_USE_MPI void MPI_Error(const char* file, int line, const char* msg, int rc); -#define BL_MPI_REQUIRE(x) \ -do \ -{ \ - if ( int l_status_ = (x) ) \ - { \ +#define BL_MPI_REQUIRE(x) \ +do \ +{ \ + if ( int l_status_ = (x) ) \ + { \ amrex::ParallelDescriptor::MPI_Error(__FILE__,__LINE__,#x, l_status_); \ - } \ -} \ + } \ +} \ while ( false ) #endif @@ -106,7 +106,7 @@ while ( false ) * first routine in this class called from within a program. */ void StartParallel (int* argc = 0, - char*** argv = 0, + char*** argv = 0, MPI_Comm mpi_comm = MPI_COMM_WORLD); void Initialize (); @@ -135,74 +135,74 @@ while ( false ) MyProc (MPI_Comm comm) noexcept { #ifdef BL_USE_MPI - int r; - MPI_Comm_rank(comm,&r); - return r; + int r; + MPI_Comm_rank(comm,&r); + return r; #else amrex::ignore_unused(comm); - return 0; + return 0; #endif } //! Provide functionalities needed to construct a team of processes to perform a particular job struct ProcessTeam { - typedef MPI_Comm team_t; + typedef MPI_Comm team_t; //! synchronize processes within the team - void Barrier () const { - if (m_size > 1) { + void Barrier () const { + if (m_size > 1) { #if defined(BL_USE_MPI3) - MPI_Barrier(m_team_comm); + MPI_Barrier(m_team_comm); #endif - } - } + } + } //! memory fence - void MemoryBarrier () const { - if (m_size > 1) { + void MemoryBarrier () const { + if (m_size > 1) { #ifdef AMREX_USE_OMP - if (omp_in_parallel()) { + if (omp_in_parallel()) { #pragma omp barrier - } + } #pragma omp master #endif - { + { #if defined(BL_USE_MPI3) - std::atomic_thread_fence(std::memory_order_release); - MPI_Barrier(m_team_comm); - std::atomic_thread_fence(std::memory_order_acquire); + std::atomic_thread_fence(std::memory_order_release); + MPI_Barrier(m_team_comm); + std::atomic_thread_fence(std::memory_order_acquire); #endif - } - } - } + } + } + } //! free a communicator - void clear () { + void clear () { #if defined(BL_USE_MPI3) - if (m_size > 1) { - MPI_Comm_free(&m_team_comm); - if (m_rankInTeam==0) MPI_Comm_free(&m_lead_comm); - } + if (m_size > 1) { + MPI_Comm_free(&m_team_comm); + if (m_rankInTeam==0) MPI_Comm_free(&m_lead_comm); + } #endif - } + } - const team_t& get() const { - return m_team_comm; - } + const team_t& get() const { + return m_team_comm; + } //! return the communicator - const MPI_Comm& get_team_comm() const { return m_team_comm; } - const MPI_Comm& get_lead_comm() const { return m_lead_comm; } - - int m_numTeams; - int m_size; - int m_color; - int m_lead; - int m_rankInTeam; - int m_do_team_reduce; - - MPI_Comm m_team_comm; - MPI_Comm m_lead_comm; + const MPI_Comm& get_team_comm() const { return m_team_comm; } + const MPI_Comm& get_lead_comm() const { return m_lead_comm; } + + int m_numTeams; + int m_size; + int m_color; + int m_lead; + int m_rankInTeam; + int m_do_team_reduce; + + MPI_Comm m_team_comm; + MPI_Comm m_lead_comm; }; extern ProcessTeam m_Team; @@ -269,130 +269,130 @@ while ( false ) inline int TeamSize () noexcept { - return m_Team.m_size; + return m_Team.m_size; } inline int NTeams () noexcept { - return m_Team.m_numTeams; + return m_Team.m_numTeams; } inline int MyTeamColor () noexcept { - return m_Team.m_color; + return m_Team.m_color; } inline int MyTeamLead () noexcept { - return m_Team.m_lead; + return m_Team.m_lead; } inline int MyRankInTeam () noexcept { - return m_Team.m_rankInTeam; + return m_Team.m_rankInTeam; } inline int TeamLead (int rank) noexcept { - return (rank >= 0) ? (rank - rank % m_Team.m_size) : MPI_PROC_NULL; + return (rank >= 0) ? (rank - rank % m_Team.m_size) : MPI_PROC_NULL; } inline bool isTeamLead () noexcept { - return MyRankInTeam() == 0; + return MyRankInTeam() == 0; } inline bool sameTeam (int rank) noexcept { - return MyTeamLead() == TeamLead(rank); + return MyTeamLead() == TeamLead(rank); } inline bool sameTeam (int rankA, int rankB) noexcept { - return TeamLead(rankA) == TeamLead(rankB); + return TeamLead(rankA) == TeamLead(rankB); } inline int RankInLeadComm (int rank) noexcept { - return (rank >= 0) ? (rank / m_Team.m_size) : MPI_PROC_NULL; + return (rank >= 0) ? (rank / m_Team.m_size) : MPI_PROC_NULL; } inline bool doTeamReduce () noexcept { - return m_Team.m_do_team_reduce; + return m_Team.m_do_team_reduce; } inline const ProcessTeam& MyTeam () noexcept { - return m_Team; + return m_Team; } inline std::pair team_range (int begin, int end, int rit = -1, int nworkers = 0) noexcept { - int rb, re; - { - if (rit < 0) rit = ParallelDescriptor::MyRankInTeam(); - if (nworkers == 0) nworkers = ParallelDescriptor::TeamSize(); - BL_ASSERT(rit 1) { - int tid = omp_get_thread_num(); - int ntot = re - rb; - int nr = ntot / nthreads; - int nlft = ntot - nr * nthreads; - if (tid < nlft) { // get nr+1 items - rb += tid * (nr + 1); - re = rb + nr + 1; - } else { // get nr items - rb += tid * nr + nlft; - re = rb + nr; - } - } + int nthreads = omp_get_num_threads(); + if (nthreads > 1) { + int tid = omp_get_thread_num(); + int ntot = re - rb; + int nr = ntot / nthreads; + int nlft = ntot - nr * nthreads; + if (tid < nlft) { // get nr+1 items + rb += tid * (nr + 1); + re = rb + nr + 1; + } else { // get nr items + rb += tid * nr + nlft; + re = rb + nr; + } + } #endif - return std::make_pair(rb,re); + return std::make_pair(rb,re); } template void team_for (int begin, int end, const F& f) { - const auto& range = team_range(begin, end); - for (int i = range.first; i < range.second; ++i) { - f(i); - } + const auto& range = team_range(begin, end); + for (int i = range.first; i < range.second; ++i) { + f(i); + } } template // rit: rank in team void team_for (int begin, int end, int rit, const F& f) { - const auto& range = team_range(begin, end, rit); - for (int i = range.first; i < range.second; ++i) { - f(i); - } + const auto& range = team_range(begin, end, rit); + for (int i = range.first; i < range.second; ++i) { + f(i); + } } template // rit: rank in team void team_for (int begin, int end, int rit, int nworkers, const F& f) { - const auto& range = team_range(begin, end, rit, nworkers); - for (int i = range.first; i < range.second; ++i) { - f(i); - } + const auto& range = team_range(begin, end, rit, nworkers); + for (int i = range.first; i < range.second; ++i) { + f(i); + } } void Barrier (const std::string& message = Unnamed); @@ -617,8 +617,8 @@ while ( false ) template std::vector Gather(const T&, int root); template void Gatherv (const T* send, int sc, - T* recv, const std::vector& rc, const std::vector& disp, - int root); + T* recv, const std::vector& rc, const std::vector& disp, + int root); //! Gather LayoutData values to a vector on root template void GatherLayoutDataToVector (const LayoutData& sendbuf, @@ -632,7 +632,7 @@ while ( false ) void ReadAndBcastFile(const std::string &filename, Vector &charBuf, bool bExitOnError = true, - const MPI_Comm &comm = Communicator() ); + const MPI_Comm &comm = Communicator() ); void IProbe(int src_pid, int tag, int &mflag, MPI_Status &status); void IProbe(int src_pid, int tag, MPI_Comm comm, int &mflag, MPI_Status &status); @@ -871,13 +871,13 @@ void ParallelDescriptor::Bcast (T* t, size_t n, int root, - const MPI_Comm &comm) + const MPI_Comm &comm) { #ifdef BL_LAZY int r; MPI_Comm_compare(comm, Communicator(), &r); if (r == MPI_IDENT) - Lazy::EvalReduction(); + Lazy::EvalReduction(); #endif BL_ASSERT(n < std::numeric_limits::max()); @@ -942,15 +942,15 @@ ParallelDescriptor::Gather (const T& t, int root) template void ParallelDescriptor::Gatherv (const T* send, int sc, - T* recv, const std::vector& rc, const std::vector& disp, - int root) + T* recv, const std::vector& rc, const std::vector& disp, + int root) { BL_PROFILE_T_S("ParallelDescriptor::Gatherv(Ti)", T); BL_COMM_PROFILE(BLProfiler::Gatherv, BLProfiler::BeforeCall(), root, BLProfiler::NoTag()); MPI_Gatherv(send, sc, ParallelDescriptor::Mpi_typemap::type(), - recv, &rc[0], &disp[0], ParallelDescriptor::Mpi_typemap::type(), - root, Communicator()); + recv, &rc[0], &disp[0], ParallelDescriptor::Mpi_typemap::type(), + root, Communicator()); BL_COMM_PROFILE(BLProfiler::Gatherv, std::accumulate(rc.begin(),rc.end(),0)*sizeof(T), root, BLProfiler::NoTag()); } @@ -961,12 +961,12 @@ ParallelDescriptor::GatherLayoutDataToVector (const LayoutData& sendbuf, Vector& recvbuf, int root) { BL_PROFILE_T_S("ParallelDescriptor::GatherLayoutData(Ti)", T); - + // Gather prelims Vector index_to_send = sendbuf.IndexArray(); Vector T_to_send; T_to_send.reserve(sendbuf.local_size()); - + for (int i : sendbuf.IndexArray()) { T_to_send.push_back(sendbuf[i]); @@ -990,7 +990,7 @@ ParallelDescriptor::GatherLayoutDataToVector (const LayoutData& sendbuf, { new_ind_to_old_ind[old_pmap[i]].push_back(i); } - + // Flatten Vector new_index_to_old_index; new_index_to_old_index.reserve(old_pmap.size()); @@ -1023,7 +1023,7 @@ ParallelDescriptor::GatherLayoutDataToVector (const LayoutData& sendbuf, { // Invert the map (new_index) --> (old_index) Vector old_index_to_new_index(sendbuf.size()); - + for (int i=0; i MPI_Datatype Mpi_typemap::type() if ( mpi_type_intvect == MPI_DATATYPE_NULL ) { - MPI_Datatype types[] = { MPI_INT }; - int blocklens[] = { AMREX_SPACEDIM }; - MPI_Aint disp[] = { 0 }; - BL_MPI_REQUIRE( MPI_Type_create_struct(1, blocklens, disp, types, &mpi_type_intvect) ); + MPI_Datatype types[] = { MPI_INT }; + int blocklens[] = { AMREX_SPACEDIM }; + MPI_Aint disp[] = { 0 }; + BL_MPI_REQUIRE( MPI_Type_create_struct(1, blocklens, disp, types, &mpi_type_intvect) ); MPI_Aint lb, extent; BL_MPI_REQUIRE( MPI_Type_get_extent(mpi_type_intvect, &lb, &extent) ); if (extent != sizeof(IntVect)) { @@ -1326,7 +1326,7 @@ template <> MPI_Datatype Mpi_typemap::type() BL_MPI_REQUIRE( MPI_Type_create_resized(tmp, 0, sizeof(IntVect), &mpi_type_intvect) ); BL_MPI_REQUIRE( MPI_Type_free(&tmp) ); } - BL_MPI_REQUIRE( MPI_Type_commit( &mpi_type_intvect ) ); + BL_MPI_REQUIRE( MPI_Type_commit( &mpi_type_intvect ) ); } return mpi_type_intvect; } @@ -1338,9 +1338,9 @@ template <> MPI_Datatype Mpi_typemap::type() if ( mpi_type_indextype == MPI_DATATYPE_NULL ) { - MPI_Datatype types[] = { MPI_UNSIGNED }; - int blocklens[] = { 1 }; - MPI_Aint disp[] = { 0 }; + MPI_Datatype types[] = { MPI_UNSIGNED }; + int blocklens[] = { 1 }; + MPI_Aint disp[] = { 0 }; BL_MPI_REQUIRE( MPI_Type_create_struct(1, blocklens, disp, types, &mpi_type_indextype) ); MPI_Aint lb, extent; BL_MPI_REQUIRE( MPI_Type_get_extent(mpi_type_indextype, &lb, &extent) ); @@ -1349,7 +1349,7 @@ template <> MPI_Datatype Mpi_typemap::type() BL_MPI_REQUIRE( MPI_Type_create_resized(tmp, 0, sizeof(IndexType), &mpi_type_indextype) ); BL_MPI_REQUIRE( MPI_Type_free(&tmp) ); } - BL_MPI_REQUIRE( MPI_Type_commit( &mpi_type_indextype ) ); + BL_MPI_REQUIRE( MPI_Type_commit( &mpi_type_indextype ) ); } return mpi_type_indextype; } @@ -1361,17 +1361,17 @@ template <> MPI_Datatype Mpi_typemap::type() if ( mpi_type_box == MPI_DATATYPE_NULL ) { - Box bx[2]; - MPI_Datatype types[] = { - Mpi_typemap::type(), - Mpi_typemap::type(), - Mpi_typemap::type(), + Box bx[2]; + MPI_Datatype types[] = { + Mpi_typemap::type(), + Mpi_typemap::type(), + Mpi_typemap::type(), }; - int blocklens[] = { 1, 1, 1 }; - MPI_Aint disp[3]; - BL_MPI_REQUIRE( MPI_Get_address(&bx[0].smallend, &disp[0]) ); - BL_MPI_REQUIRE( MPI_Get_address(&bx[0].bigend, &disp[1]) ); - BL_MPI_REQUIRE( MPI_Get_address(&bx[0].btype, &disp[2]) ); + int blocklens[] = { 1, 1, 1 }; + MPI_Aint disp[3]; + BL_MPI_REQUIRE( MPI_Get_address(&bx[0].smallend, &disp[0]) ); + BL_MPI_REQUIRE( MPI_Get_address(&bx[0].bigend, &disp[1]) ); + BL_MPI_REQUIRE( MPI_Get_address(&bx[0].btype, &disp[2]) ); disp[2] -= disp[0]; disp[1] -= disp[0]; disp[0] = 0; @@ -1383,7 +1383,7 @@ template <> MPI_Datatype Mpi_typemap::type() BL_MPI_REQUIRE( MPI_Type_create_resized(tmp, 0, sizeof(bx[0]), &mpi_type_box) ); BL_MPI_REQUIRE( MPI_Type_free(&tmp) ); } - BL_MPI_REQUIRE( MPI_Type_commit( &mpi_type_box ) ); + BL_MPI_REQUIRE( MPI_Type_commit( &mpi_type_box ) ); } return mpi_type_box; } @@ -1411,16 +1411,16 @@ ReadAndBcastFile (const std::string& filename, Vector& charBuf, iss.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); iss.open(filename.c_str(), std::ios::in); if ( ! iss.good()) { - if(bExitOnError) { + if(bExitOnError) { amrex::FileOpenFailed(filename); - } else { + } else { fileLength = -1; - } + } } else { iss.seekg(0, std::ios::end); fileLength = static_cast(iss.tellg()); iss.seekg(0, std::ios::beg); - } + } } ParallelDescriptor::Bcast(&fileLength, 1, ParallelDescriptor::IOProcessorNumber(), comm); @@ -1477,7 +1477,7 @@ StartTeams () int rank = ParallelDescriptor::MyProc(); if (nprocs % team_size != 0) - amrex::Abort("Number of processes not divisible by team size"); + amrex::Abort("Number of processes not divisible by team size"); m_Team.m_numTeams = nprocs / team_size; m_Team.m_size = team_size; @@ -1489,23 +1489,23 @@ StartTeams () #if defined(BL_USE_MPI3) { - MPI_Group grp, team_grp, lead_grp; - BL_MPI_REQUIRE( MPI_Comm_group(ParallelDescriptor::Communicator(), &grp) ); - int team_ranks[team_size]; - for (int i = 0; i < team_size; ++i) { - team_ranks[i] = MyTeamLead() + i; - } - BL_MPI_REQUIRE( MPI_Group_incl(grp, team_size, team_ranks, &team_grp) ); - BL_MPI_REQUIRE( MPI_Comm_create(ParallelDescriptor::Communicator(), - team_grp, &m_Team.m_team_comm) ); - - std::vectorlead_ranks(m_Team.m_numTeams); - for (int i = 0; i < lead_ranks.size(); ++i) { - lead_ranks[i] = i * team_size; - } - BL_MPI_REQUIRE( MPI_Group_incl(grp, lead_ranks.size(), &lead_ranks[0], &lead_grp) ); - BL_MPI_REQUIRE( MPI_Comm_create(ParallelDescriptor::Communicator(), - lead_grp, &m_Team.m_lead_comm) ); + MPI_Group grp, team_grp, lead_grp; + BL_MPI_REQUIRE( MPI_Comm_group(ParallelDescriptor::Communicator(), &grp) ); + int team_ranks[team_size]; + for (int i = 0; i < team_size; ++i) { + team_ranks[i] = MyTeamLead() + i; + } + BL_MPI_REQUIRE( MPI_Group_incl(grp, team_size, team_ranks, &team_grp) ); + BL_MPI_REQUIRE( MPI_Comm_create(ParallelDescriptor::Communicator(), + team_grp, &m_Team.m_team_comm) ); + + std::vectorlead_ranks(m_Team.m_numTeams); + for (int i = 0; i < lead_ranks.size(); ++i) { + lead_ranks[i] = i * team_size; + } + BL_MPI_REQUIRE( MPI_Group_incl(grp, lead_ranks.size(), &lead_ranks[0], &lead_grp) ); + BL_MPI_REQUIRE( MPI_Comm_create(ParallelDescriptor::Communicator(), + lead_grp, &m_Team.m_lead_comm) ); BL_MPI_REQUIRE( MPI_Group_free(&grp) ); BL_MPI_REQUIRE( MPI_Group_free(&team_grp) ); diff --git a/Src/Base/AMReX_ParallelDescriptor_F.F90 b/Src/Base/AMReX_ParallelDescriptor_F.F90 index a54143a12d2..bf52b345cfb 100644 --- a/Src/Base/AMReX_ParallelDescriptor_F.F90 +++ b/Src/Base/AMReX_ParallelDescriptor_F.F90 @@ -16,7 +16,7 @@ function amrex_fi_pd_nprocs () bind(c) implicit none integer(c_int) :: amrex_fi_pd_nprocs end function amrex_fi_pd_nprocs - + function amrex_fi_pd_ioprocessor () bind(c) import implicit none diff --git a/Src/Base/AMReX_ParmParse.H b/Src/Base/AMReX_ParmParse.H index c180071cd68..5e3c801dc05 100644 --- a/Src/Base/AMReX_ParmParse.H +++ b/Src/Base/AMReX_ParmParse.H @@ -919,8 +919,8 @@ public: * ParmParse object. */ static void Initialize(int argc, - char** argv, - const char* parfile); + char** argv, + const char* parfile); /** * \brief The destructor. The internal static table will only be deleted * if there are no other ParmParse objects in existence. @@ -977,7 +977,7 @@ struct ParmParse::PP_entry PP_entry (const std::string& name, const std::string& vals); PP_entry (const std::string& name, - const Table& table); + const Table& table); PP_entry (const PP_entry& pe); PP_entry& operator= (const PP_entry& pe); ~PP_entry (); diff --git a/Src/Base/AMReX_ParmParse.cpp b/Src/Base/AMReX_ParmParse.cpp index 386636cc0e0..c286ae8c5ba 100644 --- a/Src/Base/AMReX_ParmParse.cpp +++ b/Src/Base/AMReX_ParmParse.cpp @@ -71,7 +71,7 @@ ParmParse::PP_entry::PP_entry (const PP_entry& pe) { if ( pe.m_table ) { - m_table = new Table(*pe.m_table); + m_table = new Table(*pe.m_table); } } @@ -79,7 +79,7 @@ ParmParse::PP_entry::~PP_entry () { delete m_table; } - + ParmParse::PP_entry& ParmParse::PP_entry::operator= (const PP_entry& pe) { @@ -90,7 +90,7 @@ ParmParse::PP_entry::operator= (const PP_entry& pe) m_queried = pe.m_queried; if ( pe.m_table ) { - m_table = new Table(*pe.m_table); + m_table = new Table(*pe.m_table); } return *this; } @@ -102,8 +102,8 @@ ParmParse::PP_entry::print () const { int n = m_vals.size(); for ( int i = 0; i < n; i++) { - t << m_vals[i]; - if ( i < n-1 ) t << " "; + t << m_vals[i]; + if ( i < n-1 ) t << " "; } return t.str(); } @@ -115,8 +115,8 @@ operator<< (std::ostream& os, const ParmParse::PP_entry& pp) int n = pp.m_vals.size(); for ( int i = 0; i < n; i++ ) { - os << pp.m_vals[i]; - if ( i < n-1 ) os << ", "; + os << pp.m_vals[i]; + if ( i < n-1 ) os << ", "; } os << "]"; @@ -160,7 +160,7 @@ is (const std::string& str, std::string& val) return true; } -template <> +template <> bool is (const std::string& str, bool& val) { @@ -224,31 +224,31 @@ eat_garbage (const char*& str) { for (;;) { - if ( *str == 0 ) break; + if ( *str == 0 ) break; else if ( *str == '#' ) { while ( *str && *str != '\n' ) - { - str++; - } - continue; + { + str++; + } + continue; } else if ( isspace(*str) ) - { - str++; - } + { + str++; + } else - { + { break; - } + } } } PType getToken (const char*& str, - std::string& ostr) + std::string& ostr) { -#define ERROR_MESS \ +#define ERROR_MESS \ amrex::ErrorStream() << "ParmParse::getToken(): invalid string = " << ostr << '\n'; \ amrex::ErrorStream() << "STATE = " << state_name[state] \ << ", next char = " << ch << '\n'; \ @@ -275,7 +275,7 @@ getToken (const char*& str, char ch = *str; if ( ch == 0 ) { - amrex::Error("ParmParse::getToken: EOF while parsing"); + amrex::Error("ParmParse::getToken: EOF while parsing"); } switch (state) { @@ -290,21 +290,21 @@ getToken (const char*& str, str++; state = QUOTED_STRING; } - else if ( ch == '(' ) - { - ostr += ch; str++; pcnt = 1; - state = LIST; - } - else if ( ch == '{' ) - { - str++; - return pOpenBracket; - } - else if ( ch == '}' ) - { - str++; - return pCloseBracket; - } + else if ( ch == '(' ) + { + ostr += ch; str++; pcnt = 1; + state = LIST; + } + else if ( ch == '{' ) + { + str++; + return pOpenBracket; + } + else if ( ch == '}' ) + { + str++; + return pCloseBracket; + } else if ( isalpha(ch) ) { ostr += ch; str++; @@ -332,23 +332,23 @@ getToken (const char*& str, } break; case LIST: - if ( ch == '(' ) - { - ostr += ch; str++; pcnt++; - } - else if ( ch == ')' ) - { - ostr += ch; str++; pcnt--; - if ( pcnt == 0 ) + if ( ch == '(' ) + { + ostr += ch; str++; pcnt++; + } + else if ( ch == ')' ) + { + ostr += ch; str++; pcnt--; + if ( pcnt == 0 ) { - return pValue; + return pValue; } - } - else - { - ostr += ch; str++; - } - break; + } + else + { + ostr += ch; str++; + } + break; case STRING: if ( isspace(ch) || ch == '=' ) { @@ -385,8 +385,8 @@ getToken (const char*& str, static bool ppfound (const std::string& keyword, - const ParmParse::PP_entry& pe, - bool recordQ) + const ParmParse::PP_entry& pe, + bool recordQ) { return (recordQ == (pe.m_table!=0)) && (keyword == pe.m_name); } @@ -399,9 +399,9 @@ ppfound (const std::string& keyword, const ParmParse::PP_entry* ppindex (const ParmParse::Table& table, - int n, - const std::string& name, - bool recordQ) + int n, + const std::string& name, + bool recordQ) { const ParmParse::PP_entry* fnd = 0; @@ -427,15 +427,15 @@ ppindex (const ParmParse::Table& table, { fnd = &*li; if ( --n < 0 ) - { + { break; - } + } } } if ( n >= 0) - { + { fnd = 0; - } + } } if ( fnd ) @@ -444,12 +444,12 @@ ppindex (const ParmParse::Table& table, // Found an entry; mark all occurences of name as used. // for ( const_list_iterator li = table.begin(), End = table.end(); li != End; ++li ) - { + { if ( ppfound(name, *li, recordQ) ) - { + { li->m_queried = true; - } - } + } + } } return fnd; } @@ -459,16 +459,16 @@ bldTable (const char*& str, std::list& tab); static void read_file (const char* fname, - std::list& tab) + std::list& tab) { // // Space for input file if it exists. // if ( fname != 0 && fname[0] != 0 ) { - Vector fileCharPtr; - std::string filename = fname; - ParallelDescriptor::ReadAndBcastFile(filename, fileCharPtr); + Vector fileCharPtr; + std::string filename = fname; + ParallelDescriptor::ReadAndBcastFile(filename, fileCharPtr); std::istringstream is(fileCharPtr.data()); std::ostringstream os_cxx(std::ios_base::out); @@ -507,8 +507,8 @@ read_file (const char* fname, void addDefn (std::string& def, - std::list& val, - std::list& tab) + std::list& val, + std::list& tab) { static const std::string FileKeyword("FILE"); // @@ -548,8 +548,8 @@ addDefn (std::string& def, void addTable (std::string& def, - ParmParse::Table& val, - std::list& tab) + ParmParse::Table& val, + std::list& tab) { if ( def.empty() ) { @@ -562,7 +562,7 @@ addTable (std::string& def, if ( val.empty() ) { amrex::ErrorStream() << "ParmParse::addTable(): no values for Table " << def << "\n"; - amrex::Abort(); + amrex::Abort(); } tab.push_back(ParmParse::PP_entry(def, val)); val.clear(); @@ -571,7 +571,7 @@ addTable (std::string& def, void bldTable (const char*& str, - std::list& tab) + std::list& tab) { std::string cur_name; std::list cur_list; @@ -582,70 +582,70 @@ bldTable (const char*& str, { std::string tokname; - PType token = getToken(str,tokname); + PType token = getToken(str,tokname); - switch (token) - { - case pCloseBracket: - if ( !cur_name.empty() && cur_list.empty() ) - { - amrex::Abort("ParmParse::bldTable() defn with no list"); - } + switch (token) + { + case pCloseBracket: + if ( !cur_name.empty() && cur_list.empty() ) + { + amrex::Abort("ParmParse::bldTable() defn with no list"); + } AMREX_FALLTHROUGH; - case pEOF: - addDefn(cur_name,cur_list,tab); - return; - case pOpenBracket: - if ( cur_name.empty() ) - { - amrex::Abort("ParmParse::bldTabe() '{' with no blocknamne"); - } - if ( !cur_list.empty() ) - { - tmp_str = cur_list.back(); - cur_list.pop_back(); - addDefn(cur_name, cur_list, tab); - cur_name = tmp_str; - } - bldTable(str, cur_table); - addTable(cur_name, cur_table, tab); - break; - case pEQ_sign: - if ( cur_name.empty() ) - { - amrex::Abort("ParmParse::bldTable() EQ with no current defn"); - } - if ( !cur_list.empty() ) - { - tmp_str = cur_list.back(); - cur_list.pop_back(); - addDefn(cur_name,cur_list,tab); - cur_name = tmp_str; - } - // - // Read one too far, remove last name on list. - // - break; - case pDefn: - if ( cur_name.empty() ) - { - cur_name = tokname; - break; - } - // - // Otherwise, fall through, this may be a string. - // + case pEOF: + addDefn(cur_name,cur_list,tab); + return; + case pOpenBracket: + if ( cur_name.empty() ) + { + amrex::Abort("ParmParse::bldTabe() '{' with no blocknamne"); + } + if ( !cur_list.empty() ) + { + tmp_str = cur_list.back(); + cur_list.pop_back(); + addDefn(cur_name, cur_list, tab); + cur_name = tmp_str; + } + bldTable(str, cur_table); + addTable(cur_name, cur_table, tab); + break; + case pEQ_sign: + if ( cur_name.empty() ) + { + amrex::Abort("ParmParse::bldTable() EQ with no current defn"); + } + if ( !cur_list.empty() ) + { + tmp_str = cur_list.back(); + cur_list.pop_back(); + addDefn(cur_name,cur_list,tab); + cur_name = tmp_str; + } + // + // Read one too far, remove last name on list. + // + break; + case pDefn: + if ( cur_name.empty() ) + { + cur_name = tokname; + break; + } + // + // Otherwise, fall through, this may be a string. + // AMREX_FALLTHROUGH; - case pValue: - if ( cur_name.empty() ) - { - std::string msg("ParmParse::bldTable(): value with no defn: "); - msg += tokname; - amrex::Abort(msg.c_str()); - } - cur_list.push_back(tokname); - break; - } + case pValue: + if ( cur_name.empty() ) + { + std::string msg("ParmParse::bldTable(): value with no defn: "); + msg += tokname; + amrex::Abort(msg.c_str()); + } + cur_list.push_back(tokname); + break; + } } } @@ -654,10 +654,10 @@ namespace template bool squeryval (const ParmParse::Table& table, - const std::string& name, - T& ptr, - int ival, - int occurence) + const std::string& name, + T& ptr, + int ival, + int occurence) { // // Get last occurrance of name in table. @@ -675,13 +675,13 @@ squeryval (const ParmParse::Table& table, amrex::ErrorStream() << "ParmParse::queryval no value number" << ival << " for "; if ( occurence == ParmParse::LAST ) - { + { amrex::ErrorStream() << "last occurence of "; - } + } else - { + { amrex::ErrorStream() << " occurence " << occurence << " of "; - } + } amrex::ErrorStream() << def->m_name << '\n' << *def << '\n'; amrex::Abort(); } @@ -694,19 +694,19 @@ squeryval (const ParmParse::Table& table, amrex::ErrorStream() << "ParmParse::queryval type mismatch on value number " << ival << " of " << '\n'; if ( occurence == ParmParse::LAST ) - { + { amrex::ErrorStream() << " last occurence of "; - } + } else - { + { amrex::ErrorStream() << " occurence number " << occurence << " of "; - } + } amrex::ErrorStream() << def->m_name << '\n'; amrex::ErrorStream() << " Expected an \"" << tok_name(ptr) << "\" type which can't be parsed from the string \"" << valname << "\"\n" - << *def << '\n'; + << *def << '\n'; amrex::Abort(); } return true; @@ -715,20 +715,20 @@ squeryval (const ParmParse::Table& table, template void sgetval (const ParmParse::Table& table, - const std::string& name, - T& ptr, - int ival, - int occurence) + const std::string& name, + T& ptr, + int ival, + int occurence) { if ( squeryval(table, name,ptr,ival,occurence) == 0 ) { amrex::ErrorStream() << "ParmParse::getval "; if ( occurence >= 0 ) - { + { amrex::ErrorStream() << "occurence number " << occurence << " of "; - } + } amrex::ErrorStream() << "ParmParse::getval(): " << name @@ -742,11 +742,11 @@ sgetval (const ParmParse::Table& table, template bool squeryarr (const ParmParse::Table& table, - const std::string& name, - std::vector& ptr, - int start_ix, - int num_val, - int occurence) + const std::string& name, + std::vector& ptr, + int start_ix, + int num_val, + int occurence) { // // Get last occurrance of name in table. @@ -762,7 +762,7 @@ squeryarr (const ParmParse::Table& table, // if ( num_val == ParmParse::ALL ) { - num_val = def->m_vals.size(); + num_val = def->m_vals.size(); } if ( num_val == 0 ) return true; @@ -776,40 +776,40 @@ squeryarr (const ParmParse::Table& table, { amrex::ErrorStream() << "ParmParse::queryarr too many values requested for"; if ( occurence == ParmParse::LAST ) - { + { amrex::ErrorStream() << " last occurence of "; - } + } else - { + { amrex::ErrorStream() << " occurence " << occurence << " of "; - } + } amrex::ErrorStream() << def->m_name << '\n' << *def << '\n'; amrex::Abort(); } for ( int n = start_ix; n <= stop_ix; n++ ) { - const std::string& valname = def->m_vals[n]; - bool ok = is(valname, ptr[n]); - if ( !ok ) - { - amrex::ErrorStream() << "ParmParse::queryarr type mismatch on value number " - << n << " of "; - if ( occurence == ParmParse::LAST ) - { - amrex::ErrorStream() << " last occurence of "; - } - else - { - amrex::ErrorStream() << " occurence number " << occurence << " of "; - } - amrex::ErrorStream() << def->m_name << '\n'; - amrex::ErrorStream() << " Expected an \"" - << tok_name(ptr) - << "\" type which can't be parsed from the string \"" - << valname << "\"\n" - << *def << '\n'; - amrex::Abort(); - } + const std::string& valname = def->m_vals[n]; + bool ok = is(valname, ptr[n]); + if ( !ok ) + { + amrex::ErrorStream() << "ParmParse::queryarr type mismatch on value number " + << n << " of "; + if ( occurence == ParmParse::LAST ) + { + amrex::ErrorStream() << " last occurence of "; + } + else + { + amrex::ErrorStream() << " occurence number " << occurence << " of "; + } + amrex::ErrorStream() << def->m_name << '\n'; + amrex::ErrorStream() << " Expected an \"" + << tok_name(ptr) + << "\" type which can't be parsed from the string \"" + << valname << "\"\n" + << *def << '\n'; + amrex::Abort(); + } } return true; } @@ -817,19 +817,19 @@ squeryarr (const ParmParse::Table& table, template void sgetarr (const ParmParse::Table& table, - const std::string& name, - std::vector& ptr, - int start_ix, - int num_val, - int occurence) + const std::string& name, + std::vector& ptr, + int start_ix, + int num_val, + int occurence) { if ( squeryarr(table,name,ptr,start_ix,num_val,occurence) == 0 ) { amrex::ErrorStream() << "ParmParse::sgetarr "; if ( occurence >= 0 ) - { + { amrex::ErrorStream() << "occurence number " << occurence << " of "; - } + } amrex::ErrorStream() << "ParmParse::sgetarr(): " << name << " not found in table" @@ -883,7 +883,7 @@ ppinit (int argc, char** argv, const char* parfile, ParmParse::Table& table) { if ( parfile != 0 ) { - read_file(parfile, table); + read_file(parfile, table); } if ( argc > 0 ) @@ -896,7 +896,7 @@ ppinit (int argc, char** argv, const char* parfile, ParmParse::Table& table) argstr += SPACE; } std::list arg_table; - const char* b = argstr.c_str(); + const char* b = argstr.c_str(); bldTable(b, arg_table); // // Append arg_table to end of existing table. @@ -913,7 +913,7 @@ ParmParse::prefixedName (const std::string& str) const { if ( str.empty() ) { - amrex::Error("ParmParse::prefixedName: has empty name"); + amrex::Error("ParmParse::prefixedName: has empty name"); } if ( !m_pstack.top().empty()) { @@ -928,11 +928,11 @@ ParmParse::pushPrefix (const std::string& str) std::string s(str); if ( !s.empty() ) { - if ( !m_pstack.top().empty() ) - { - s = m_pstack.top() + "." + s; - } - m_pstack.push(s); + if ( !m_pstack.top().empty() ) + { + s = m_pstack.top() + "." + s; + } + m_pstack.push(s); } } @@ -941,7 +941,7 @@ ParmParse::popPrefix () { if ( m_pstack.size() <= 1 ) { - amrex::Error("ParmParse::popPrefix: stack underflow"); + amrex::Error("ParmParse::popPrefix: stack underflow"); } m_pstack.pop(); } @@ -978,7 +978,7 @@ ParmParse::Frame::~Frame () BL_ASSERT( m_np > 0 ); while ( m_np ) { - pop(); + pop(); } BL_ASSERT( m_np == 0 ); } @@ -1016,10 +1016,10 @@ unused_table_entries_q (const ParmParse::Table& table, const std::string& prefix { for ( const_list_iterator li = table.begin(), End = table.end(); li != End; ++li ) { - if ( li->m_table ) - { - if ( !li->m_queried ) - { + if ( li->m_table ) + { + if ( !li->m_queried ) + { if (prefix.empty()) { return true; } else { @@ -1027,14 +1027,14 @@ unused_table_entries_q (const ParmParse::Table& table, const std::string& prefix return true; } } - } - else - { - if (unused_table_entries_q(*li->m_table, prefix)) return true; - } - } - else if ( !li->m_queried ) - { + } + else + { + if (unused_table_entries_q(*li->m_table, prefix)) return true; + } + } + else if ( !li->m_queried ) + { if (prefix.empty()) { return true; } else { @@ -1042,7 +1042,7 @@ unused_table_entries_q (const ParmParse::Table& table, const std::string& prefix return true; } } - } + } } return false; } @@ -1053,36 +1053,36 @@ finalize_table (const std::string& pfx, const ParmParse::Table& table) { for ( const_list_iterator li = table.begin(), End = table.end(); li != End; ++li ) { - if ( li->m_table ) - { - if ( !li->m_queried ) - { + if ( li->m_table ) + { + if ( !li->m_queried ) + { if (finalize_verbose) { amrex::AllPrint() << "Record " << li->m_name << std::endl; } - } - else - { - finalize_table(pfx + "::" + li->m_name, *li->m_table); - } - } - else if ( !li->m_queried ) - { + } + else + { + finalize_table(pfx + "::" + li->m_name, *li->m_table); + } + } + else if ( !li->m_queried ) + { if (finalize_verbose) { amrex::AllPrint() << pfx << "::" << *li << std::endl; } - } + } } } void ParmParse::Initialize (int argc, - char** argv, - const char* parfile) + char** argv, + const char* parfile) { if ( initialized ) { - amrex::Error("ParmParse::Initialize(): already initialized!"); + amrex::Error("ParmParse::Initialize(): already initialized!"); } ppinit(argc, argv, parfile, g_table); @@ -1140,7 +1140,7 @@ ParmParse::getUnusedInputs (const std::string& prefix) get_unused_inputs(r, g_table, prefix); return r; } - + void ParmParse::Finalize () { @@ -1169,11 +1169,11 @@ ParmParse::dumpTable (std::ostream& os, bool prettyPrint) { for ( const_list_iterator li = g_table.begin(), End = g_table.end(); li != End; ++li ) { - if(prettyPrint && li->m_queried) { - os << li->print() << std::endl; - } - else - os << *li << std::endl; + if(prettyPrint && li->m_queried) { + os << li->print() << std::endl; + } + else + os << *li << std::endl; } } @@ -1944,10 +1944,10 @@ ParmParse::countname (const std::string& name) const int cnt = 0; for ( const_list_iterator li = m_table.begin(), End = m_table.end(); li != End; ++li ) { - if ( ppfound(prefixedName(name), *li, false) ) - { - cnt++; - } + if ( ppfound(prefixedName(name), *li, false) ) + { + cnt++; + } } return cnt; } @@ -1958,10 +1958,10 @@ ParmParse::countRecords (const std::string& name) const int cnt = 0; for ( const_list_iterator li = m_table.begin(), End = m_table.end(); li != End; ++li ) { - if ( ppfound(prefixedName(name), *li, true) ) - { - cnt++; - } + if ( ppfound(prefixedName(name), *li, true) ) + { + cnt++; + } } return cnt; } @@ -1981,12 +1981,12 @@ ParmParse::contains (const char* name) const // Found an entry; mark all occurences of name as used. // for ( const_list_iterator lli = m_table.begin(); lli != m_table.end(); ++lli ) - { + { if ( ppfound(prefixedName(name), *lli, false) ) - { + { lli->m_queried = true; - } - } + } + } return true; } } @@ -1999,8 +1999,8 @@ ParmParse::getRecord (const std::string& name, int n) const const PP_entry* pe = ppindex(m_table, n, prefixedName(name), true); if ( pe == 0 ) { - amrex::ErrorStream() << "ParmParse::getRecord: record " << name << " not found" << std::endl; - amrex::Abort(); + amrex::ErrorStream() << "ParmParse::getRecord: record " << name << " not found" << std::endl; + amrex::Abort(); } return Record(ParmParse(*pe->m_table)); } diff --git a/Src/Base/AMReX_Partition.H b/Src/Base/AMReX_Partition.H index cb8b9e372d0..73e4d980626 100644 --- a/Src/Base/AMReX_Partition.H +++ b/Src/Base/AMReX_Partition.H @@ -14,7 +14,7 @@ namespace amrex { namespace detail { - template + template int amrex_partition_helper (T const* AMREX_RESTRICT pv, T* AMREX_RESTRICT pv2, int n, F && f) { return Scan::PrefixSum (n, @@ -61,11 +61,11 @@ namespace detail * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * - * This version is not stable, if you want that behavior use + * This version is not stable, if you want that behavior use * amrex::StablePartition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -75,9 +75,9 @@ namespace detail * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template -int Partition (T* data, int beg, int end, F && f) +int Partition (T* data, int beg, int end, F && f) { int n = end - beg; Gpu::DeviceVector v2(n); @@ -91,11 +91,11 @@ int Partition (T* data, int beg, int end, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * - * This version is not stable, if you want that behavior use + * This version is not stable, if you want that behavior use * amrex::StablePartition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -104,7 +104,7 @@ int Partition (T* data, int beg, int end, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int Partition (T* data, int n, F && f) { @@ -116,11 +116,11 @@ int Partition (T* data, int n, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * - * This version is not stable, if you want that behavior use + * This version is not stable, if you want that behavior use * amrex::StablePartition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -144,13 +144,13 @@ int Partition (Gpu::DeviceVector& v, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * * This version is stable, meaning that, within each side of the resulting * array, order is maintained - if element i was before element j in the input, * then it will also be before j in the output. If you don't care about this property, * use amrex::Partition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -175,13 +175,13 @@ int StablePartition (T* data, int beg, int end, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * * This version is stable, meaning that, within each side of the resulting * array, order is maintained - if element i was before element j in the input, * then it will also be before j in the output. If you don't care about this property, * use amrex::Partition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -190,7 +190,7 @@ int StablePartition (T* data, int beg, int end, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int StablePartition (T* data, int n, F && f) { @@ -202,13 +202,13 @@ int StablePartition (T* data, int n, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * * This version is stable, meaning that, within each side of the resulting * array, order is maintained - if element i was before element j in the input, * then it will also be before j in the output. If you don't care about this property, * use amrex::Partition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -216,13 +216,13 @@ int StablePartition (T* data, int n, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int StablePartition (Gpu::DeviceVector& v, F && f) { int n = Partition(v, std::forward(f)); int n2 = static_cast(v.size()) - n; - detail::amrex_stable_partition_helper(v.dataPtr() + n, n2); + detail::amrex_stable_partition_helper(v.dataPtr() + n, n2); return n; } @@ -233,11 +233,11 @@ int StablePartition (Gpu::DeviceVector& v, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * - * This version is not stable, if you want that behavior use + * This version is not stable, if you want that behavior use * amrex::StablePartition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -249,10 +249,10 @@ int StablePartition (Gpu::DeviceVector& v, F && f) * Returns the index of the first element for which f is 0. */ template -int Partition (T* data, int beg, int end, F && f) +int Partition (T* data, int beg, int end, F && f) { auto it = std::partition(data + beg, data + end, f); - return static_cast(std::distance(data + beg, it)); + return static_cast(std::distance(data + beg, it)); } /** @@ -260,11 +260,11 @@ int Partition (T* data, int beg, int end, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * - * This version is not stable, if you want that behavior use + * This version is not stable, if you want that behavior use * amrex::StablePartition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -273,7 +273,7 @@ int Partition (T* data, int beg, int end, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int Partition (T* data, int n, F && f) { @@ -285,11 +285,11 @@ int Partition (T* data, int n, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * - * This version is not stable, if you want that behavior use + * This version is not stable, if you want that behavior use * amrex::StablePartition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -297,7 +297,7 @@ int Partition (T* data, int n, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int Partition (Gpu::DeviceVector& v, F && f) { @@ -310,13 +310,13 @@ int Partition (Gpu::DeviceVector& v, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * * This version is stable, meaning that, within each side of the resulting * array, order is maintained - if element i was before element j in the input, * then it will also be before j in the output. If you don't care about this property, * use amrex::Partition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -326,12 +326,12 @@ int Partition (Gpu::DeviceVector& v, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template -int StablePartition (T* data, int beg, int end, F && f) +int StablePartition (T* data, int beg, int end, F && f) { auto it = std::stable_partition(data + beg, data + end, f); - return static_cast(std::distance(data + beg, it)); + return static_cast(std::distance(data + beg, it)); } /** @@ -339,13 +339,13 @@ int StablePartition (T* data, int beg, int end, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * * This version is stable, meaning that, within each side of the resulting * array, order is maintained - if element i was before element j in the input, * then it will also be before j in the output. If you don't care about this property, * use amrex::Partition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -354,7 +354,7 @@ int StablePartition (T* data, int beg, int end, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int StablePartition (T* data, int n, F && f) { @@ -366,13 +366,13 @@ int StablePartition (T* data, int n, F && f) * * After calling this, all the items for which the predicate is true * will be before the items for which the predicate is false in the - * input array. + * input array. * * This version is stable, meaning that, within each side of the resulting * array, order is maintained - if element i was before element j in the input, * then it will also be before j in the output. If you don't care about this property, * use amrex::Partition instead. - * + * * \tparam T type of the data to be partitioned. * \tparam F type of the predicate function. * @@ -380,7 +380,7 @@ int StablePartition (T* data, int n, F && f) * \param f predicate function that returns 1 or 0 for each input * * Returns the index of the first element for which f is 0. - */ + */ template int StablePartition (Gpu::DeviceVector& v, F && f) { diff --git a/Src/Base/AMReX_Periodicity.H b/Src/Base/AMReX_Periodicity.H index 490f9712cc9..a57921f0f95 100644 --- a/Src/Base/AMReX_Periodicity.H +++ b/Src/Base/AMReX_Periodicity.H @@ -9,7 +9,7 @@ namespace amrex { /** -* \brief This provides length of period for periodic domains. 0 means it is +* \brief This provides length of period for periodic domains. 0 means it is * not periodic in that direction. * It is also assumed that the periodic domain starts with index 0. */ @@ -20,18 +20,18 @@ public: explicit Periodicity (const IntVect& v) noexcept : period(v) {} bool isAnyPeriodic () const noexcept - { return AMREX_D_TERM(period[0]>0, || period[1]>0, || period[2]>0); } + { return AMREX_D_TERM(period[0]>0, || period[1]>0, || period[2]>0); } bool isAllPeriodic () const noexcept - { return AMREX_D_TERM(period[0]>0, && period[1]>0, && period[2]>0); } + { return AMREX_D_TERM(period[0]>0, && period[1]>0, && period[2]>0); } bool isPeriodic (int dir) const noexcept - { return period[dir]>0; } + { return period[dir]>0; } bool operator==(const Periodicity& rhs) const noexcept - { return period == rhs.period; } + { return period == rhs.period; } //! Cell-centered domain Box "infinitely" long in non-periodic directions. Box Domain () const noexcept; - + std::vector shiftIntVect () const; static const Periodicity& NonPeriodic () noexcept; diff --git a/Src/Base/AMReX_Periodicity.cpp b/Src/Base/AMReX_Periodicity.cpp index 8e9cf5e4ffb..7ddd2036fea 100644 --- a/Src/Base/AMReX_Periodicity.cpp +++ b/Src/Base/AMReX_Periodicity.cpp @@ -13,15 +13,15 @@ Periodicity::shiftIntVect () const int jmp[3] = {1,1,1}; for (int i = 0; i < AMREX_SPACEDIM; ++i) { - if (isPeriodic(i)) { - per[i] = jmp[i] = period[i]; - } + if (isPeriodic(i)) { + per[i] = jmp[i] = period[i]; + } } for (int i = -per[0]; i <= per[0]; i += jmp[0]) { for (int j = -per[1]; j <= per[1]; j += jmp[1]) { for (int k = -per[2]; k <= per[2]; k += jmp[2]) { - r.push_back(IntVect(AMREX_D_DECL(i,j,k))); + r.push_back(IntVect(AMREX_D_DECL(i,j,k))); } } } @@ -34,13 +34,13 @@ Periodicity::Domain () const noexcept { Box pdomain; for (int i = 0; i < AMREX_SPACEDIM; ++i) { - if (isPeriodic(i)) { - pdomain.setSmall(i,0); - pdomain.setBig (i,period[i]-1); - } else { - pdomain.setSmall(i, std::numeric_limits::min()); - pdomain.setBig(i, std::numeric_limits::max()-1); // so that it can be nodalized. - } + if (isPeriodic(i)) { + pdomain.setSmall(i,0); + pdomain.setBig (i,period[i]-1); + } else { + pdomain.setSmall(i, std::numeric_limits::min()); + pdomain.setBig(i, std::numeric_limits::max()-1); // so that it can be nodalized. + } } return pdomain; } diff --git a/Src/Base/AMReX_PlotFileUtil.H b/Src/Base/AMReX_PlotFileUtil.H index 583a6f2bcb8..d49ac33ecd0 100644 --- a/Src/Base/AMReX_PlotFileUtil.H +++ b/Src/Base/AMReX_PlotFileUtil.H @@ -32,7 +32,7 @@ namespace amrex std::string MultiFabFileFullPrefix (int level, const std::string &plotfilename, const std::string &levelPrefix = "Level_", - const std::string &mfPrefix = "Cell"); + const std::string &mfPrefix = "Cell"); /** @@ -50,8 +50,8 @@ namespace amrex */ void PreBuildDirectorHierarchy (const std::string &dirName, const std::string &subDirPrefix, - int nSubDirs, - bool callBarrier); + int nSubDirs, + bool callBarrier); /** * write a generic plot file header to the file plotfilename/Header @@ -59,22 +59,22 @@ namespace amrex */ void WriteGenericPlotfileHeader (std::ostream &HeaderFile, int nlevels, - const Vector &bArray, - const Vector &varnames, - const Vector &geom, - Real time, - const Vector &level_steps, - const Vector &ref_ratio, - const std::string &versionName = "HyperCLaw-V1.1", - const std::string &levelPrefix = "Level_", - const std::string &mfPrefix = "Cell"); + const Vector &bArray, + const Vector &varnames, + const Vector &geom, + Real time, + const Vector &level_steps, + const Vector &ref_ratio, + const std::string &versionName = "HyperCLaw-V1.1", + const std::string &levelPrefix = "Level_", + const std::string &mfPrefix = "Cell"); void WriteSingleLevelPlotfile (const std::string &plotfilename, - const MultiFab &mf, - const Vector &varnames, - const Geometry &geom, - Real t, - int level_step, + const MultiFab &mf, + const Vector &varnames, + const Geometry &geom, + Real t, + int level_step, const std::string &versionName = "HyperCLaw-V1.1", const std::string &levelPrefix = "Level_", const std::string &mfPrefix = "Cell", @@ -82,12 +82,12 @@ namespace amrex void WriteMultiLevelPlotfile (const std::string &plotfilename, int nlevels, - const Vector &mf, - const Vector &varnames, - const Vector &geom, - Real time, - const Vector &level_steps, - const Vector &ref_ratio, + const Vector &mf, + const Vector &varnames, + const Vector &geom, + Real time, + const Vector &level_steps, + const Vector &ref_ratio, const std::string &versionName = "HyperCLaw-V1.1", const std::string &levelPrefix = "Level_", const std::string &mfPrefix = "Cell", @@ -96,22 +96,22 @@ namespace amrex #ifdef AMREX_USE_HDF5 void WriteGenericPlotfileHeaderHDF5 (hid_t fid, int nlevels, - const Vector &bArray, - const Vector &varnames, - const Vector &geom, - Real time, - const Vector &level_steps, - const Vector &ref_ratio, - const std::string &versionName = "HyperCLaw-V1.1", - const std::string &levelPrefix = "Level_", - const std::string &mfPrefix = "Cell"); + const Vector &bArray, + const Vector &varnames, + const Vector &geom, + Real time, + const Vector &level_steps, + const Vector &ref_ratio, + const std::string &versionName = "HyperCLaw-V1.1", + const std::string &levelPrefix = "Level_", + const std::string &mfPrefix = "Cell"); void WriteSingleLevelPlotfileHDF5 (const std::string &plotfilename, - const MultiFab &mf, - const Vector &varnames, - const Geometry &geom, - Real t, - int level_step, + const MultiFab &mf, + const Vector &varnames, + const Geometry &geom, + Real t, + int level_step, const std::string &versionName = "HyperCLaw-V1.1", const std::string &levelPrefix = "Level_", const std::string &mfPrefix = "Cell", @@ -119,12 +119,12 @@ namespace amrex void WriteMultiLevelPlotfileHDF5 (const std::string &plotfilename, int nlevels, - const Vector &mf, - const Vector &varnames, - const Vector &geom, - Real time, - const Vector &level_steps, - const Vector &ref_ratio, + const Vector &mf, + const Vector &varnames, + const Vector &geom, + Real time, + const Vector &level_steps, + const Vector &ref_ratio, const std::string &versionName = "HyperCLaw-V1.1", const std::string &levelPrefix = "Level_", const std::string &mfPrefix = "Cell", diff --git a/Src/Base/AMReX_PlotFileUtil.cpp b/Src/Base/AMReX_PlotFileUtil.cpp index 88b6b46db0d..9604be1340f 100644 --- a/Src/Base/AMReX_PlotFileUtil.cpp +++ b/Src/Base/AMReX_PlotFileUtil.cpp @@ -41,7 +41,7 @@ std::string LevelFullPath (int level, { std::string r(plotfilename); if ( ! r.empty() && r.back() != '/') { - r += '/'; + r += '/'; } r += LevelPath(level, levelPrefix); // e.g., plt00005/Level_5 return r; @@ -54,7 +54,7 @@ std::string MultiFabFileFullPrefix (int level, { std::string r(plotfilename); if ( ! r.empty() && r.back() != '/') { - r += '/'; + r += '/'; } r += MultiFabHeaderPath(level, levelPrefix, mfPrefix); return r; @@ -100,51 +100,51 @@ WriteGenericPlotfileHeader (std::ostream &HeaderFile, int finest_level(nlevels - 1); - HeaderFile.precision(17); + HeaderFile.precision(17); - // ---- this is the generic plot file type name + // ---- this is the generic plot file type name HeaderFile << versionName << '\n'; HeaderFile << varnames.size() << '\n'; for (int ivar = 0; ivar < varnames.size(); ++ivar) { - HeaderFile << varnames[ivar] << "\n"; + HeaderFile << varnames[ivar] << "\n"; } HeaderFile << AMREX_SPACEDIM << '\n'; HeaderFile << time << '\n'; HeaderFile << finest_level << '\n'; for (int i = 0; i < AMREX_SPACEDIM; ++i) { HeaderFile << geom[0].ProbLo(i) << ' '; - } + } HeaderFile << '\n'; for (int i = 0; i < AMREX_SPACEDIM; ++i) { HeaderFile << geom[0].ProbHi(i) << ' '; - } + } HeaderFile << '\n'; for (int i = 0; i < finest_level; ++i) { HeaderFile << ref_ratio[i][0] << ' '; - } + } HeaderFile << '\n'; - for (int i = 0; i <= finest_level; ++i) { - HeaderFile << geom[i].Domain() << ' '; - } + for (int i = 0; i <= finest_level; ++i) { + HeaderFile << geom[i].Domain() << ' '; + } HeaderFile << '\n'; for (int i = 0; i <= finest_level; ++i) { HeaderFile << level_steps[i] << ' '; - } + } HeaderFile << '\n'; for (int i = 0; i <= finest_level; ++i) { for (int k = 0; k < AMREX_SPACEDIM; ++k) { HeaderFile << geom[i].CellSize()[k] << ' '; - } + } HeaderFile << '\n'; } HeaderFile << (int) geom[0].Coord() << '\n'; HeaderFile << "0\n"; - for (int level = 0; level <= finest_level; ++level) { - HeaderFile << level << ' ' << bArray[level].size() << ' ' << time << '\n'; - HeaderFile << level_steps[level] << '\n'; + for (int level = 0; level <= finest_level; ++level) { + HeaderFile << level << ' ' << bArray[level].size() << ' ' << time << '\n'; + HeaderFile << level_steps[level] << '\n'; const IntVect& domain_lo = geom[level].Domain().smallEnd(); for (int i = 0; i < bArray[level].size(); ++i) @@ -159,8 +159,8 @@ WriteGenericPlotfileHeader (std::ostream &HeaderFile, } } - HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n'; - } + HeaderFile << MultiFabHeaderPath(level, levelPrefix, mfPrefix) << '\n'; + } } @@ -251,7 +251,7 @@ WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels, // -plotfile name // -vector of MultiFabs // -vector of Geometrys -// variable names are written as "Var0", "Var1", etc. +// variable names are written as "Var0", "Var1", etc. // refinement ratio is computed from the Geometry vector // "time" and "level_steps" are set to zero void WriteMLMF (const std::string &plotfilename, @@ -282,11 +282,11 @@ void WriteMLMF (const std::string &plotfilename, // set time to zero Real time = 0.; - + WriteMultiLevelPlotfile(plotfilename, nlevs, mf, varnames, - geom, time, step_array, ref_ratio); - -} + geom, time, step_array, ref_ratio); + +} void @@ -424,7 +424,7 @@ EB_WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels, std::ofstream HeaderFile; HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | - std::ofstream::trunc | + std::ofstream::trunc | std::ofstream::binary); if( ! HeaderFile.good()) { FileOpenFailed(HeaderFileName); @@ -456,7 +456,7 @@ EB_WriteMultiLevelPlotfile (const std::string& plotfilename, int nlevels, MultiFab::Copy(mf_tmp, *mf[level], 0, 0, nc, 0); auto const& factory = dynamic_cast(mf[level]->Factory()); MultiFab::Copy(mf_tmp, factory.getVolFrac(), 0, nc, 1, 0); - VisMF::Write(mf_tmp, MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix)); + VisMF::Write(mf_tmp, MultiFabFileFullPrefix(level, plotfilename, levelPrefix, mfPrefix)); } // VisMF::SetNOutFiles(saveNFiles); @@ -612,7 +612,7 @@ WriteGenericPlotfileHeaderHDF5 (hid_t fid, const Vector &ref_ratio, const std::string &versionName, const std::string &levelPrefix, - const std::string &mfPrefix, + const std::string &mfPrefix, const Vector& extra_dirs) { BL_PROFILE("WriteGenericPlotfileHeaderHDF5()"); @@ -688,9 +688,9 @@ WriteGenericPlotfileHeaderHDF5 (hid_t fid, } int ratio = 1; - if (ref_ratio.size() > 0) + if (ref_ratio.size() > 0) ratio = ref_ratio[level][0]; - + if (level == finest_level) { ratio = 1; } @@ -761,18 +761,18 @@ WriteGenericPlotfileHeaderHDF5 (hid_t fid, H5Tclose(comp_dtype); } -void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, - int nlevels, - const Vector& mf, - const Vector& varnames, - const Vector& geom, - Real time, - const Vector& level_steps, - const Vector& ref_ratio, - const std::string &versionName, - const std::string &levelPrefix, - const std::string &mfPrefix, - const Vector& extra_dirs) +void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, + int nlevels, + const Vector& mf, + const Vector& varnames, + const Vector& geom, + Real time, + const Vector& level_steps, + const Vector& ref_ratio, + const std::string &versionName, + const std::string &levelPrefix, + const std::string &mfPrefix, + const Vector& extra_dirs) { BL_PROFILE("WriteMultiLevelPlotfileHDF5"); @@ -784,7 +784,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, int myProc(ParallelDescriptor::MyProc()); int nProcs(ParallelDescriptor::NProcs()); - + herr_t ret; int finest_level = nlevels-1; @@ -807,7 +807,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, // Create the HDF5 file fid = H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - if (fid < 0) + if (fid < 0) FileOpenFailed(filename.c_str()); H5Pclose(fapl); @@ -827,38 +827,38 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, hid_t babox_id; babox_id = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int)); if (1 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT); } else if (2 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT); } else if (3 == AMREX_SPACEDIM) { - H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (babox_id, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT); } - + hid_t center_id = H5Tcreate (H5T_COMPOUND, AMREX_SPACEDIM * sizeof(int)); if (1 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); } else if (2 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); } else if (3 == AMREX_SPACEDIM) { - H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); - H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "i", 0 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "j", 1 * sizeof(int), H5T_NATIVE_INT); + H5Tinsert (center_id, "k", 2 * sizeof(int), H5T_NATIVE_INT); } - + fapl = H5Pcreate (H5P_FILE_ACCESS); dxpl = H5Pcreate(H5P_DATASET_XFER); @@ -879,7 +879,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, // All process open the file fid = H5Fopen(filename.c_str(), H5F_ACC_RDWR, fapl); - if (fid < 0) + if (fid < 0) FileOpenFailed(filename.c_str()); RealDescriptor *whichRD = nullptr; @@ -919,13 +919,13 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, std::string dataname("data:datatype=0"); hsize_t flatdims[1]; flatdims[0] = grids.size(); - + flatdims[0] = grids.size(); boxdataspace = H5Screate_simple(1, flatdims, NULL); - - + + boxdataset = H5Dcreate(grp, bdsname.c_str(), babox_id, boxdataspace, H5P_DEFAULT, H5P_DEFAULT,H5P_DEFAULT); - + // Create a boxarray sorted by rank std::map > gridMap; for(int i(0); i < grids.size(); ++i) { @@ -945,7 +945,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, ++bIndex; } } - + hsize_t oflatdims[1]; oflatdims[0] = sortedGrids.size() + 1; offsetdataspace = H5Screate_simple(1, oflatdims, NULL); @@ -957,7 +957,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, centerdataspace = H5Screate_simple(1, centerdims, NULL); centerdataset = H5Dcreate(grp, centername.c_str(), center_id, centerdataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); if(centerdataset < 0) { std::cout << "Create center dataset failed! ret = " << centerdataset << std::endl; } - + Vector offsets(sortedGrids.size() + 1); unsigned long long currentOffset(0L); for(int b(0); b < sortedGrids.size(); ++b) { @@ -965,7 +965,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, currentOffset += sortedGrids[b].numPts() * ncomp; } offsets[sortedGrids.size()] = currentOffset; - + Vector procOffsets(nProcs); int posCount(0); Vector procBufferSize(nProcs); @@ -982,7 +982,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, } totalOffset += procBufferSize[proc]; } - + if(ParallelDescriptor::IOProcessor()) { int vbCount(0); Vector vbox(sortedGrids.size() * 2 * AMREX_SPACEDIM); @@ -995,7 +995,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, } ++vbCount; } - + #ifdef BL_USE_MPI H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_INDEPENDENT); #endif @@ -1010,21 +1010,21 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, ret = H5Dwrite(boxdataset, babox_id, H5S_ALL, H5S_ALL, dxpl, &(vbox[0])); if(ret < 0) { std::cout << "Write box dataset failed! ret = " << ret << std::endl; } } - + hsize_t hs_procsize[1], hs_allprocsize[1], ch_offset[1]; - + ch_offset[0] = procOffsets[myProc]; // ---- offset on this proc hs_procsize[0] = procBufferSize[myProc]; // ---- size of buffer on this proc hs_allprocsize[0] = offsets[sortedGrids.size()]; // ---- size of buffer on all procs - + hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL); hid_t memdataspace = H5Screate_simple(1, hs_procsize, NULL); hid_t dataset = H5Dcreate(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if(dataset < 0) + if(dataset < 0) std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl; - + H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL); - + Vector a_buffer(procBufferSize[myProc], -1.0); /* Long dataCount(0); */ /* for(MFIter mfi(*mf[level]); mfi.isValid(); ++mfi) { */ @@ -1055,7 +1055,7 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, RealDescriptor::convertFromNativeFormat(static_cast (a_buffer.dataPtr()+writeDataSize), writeDataItems, fab.dataPtr(), *whichRD); } else { // ---- copy from the fab - memcpy(static_cast (a_buffer.dataPtr()+writeDataSize), + memcpy(static_cast (a_buffer.dataPtr()+writeDataSize), fab.dataPtr(), writeDataItems * whichRDBytes); } writeDataSize += writeDataItems; @@ -1067,12 +1067,12 @@ void WriteMultiLevelPlotfileHDF5 (const std::string& plotfilename, ret = H5Pset_dxpl_mpio(dxpl, H5FD_MPIO_COLLECTIVE); #endif ret = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl, a_buffer.dataPtr()); - if(ret < 0) + if(ret < 0) std::cout << ParallelDescriptor::MyProc() << "Write data failed! ret = " << ret << std::endl; BL_PROFILE_VAR_STOP(h5dwg); - + H5Sclose(memdataspace); H5Sclose(dataspace); H5Dclose(dataset); diff --git a/Src/Base/AMReX_Print.H b/Src/Base/AMReX_Print.H index 93205d01a29..b906e6a53a9 100644 --- a/Src/Base/AMReX_Print.H +++ b/Src/Base/AMReX_Print.H @@ -34,40 +34,40 @@ namespace amrex { public: - static constexpr int AllProcs = -1; + static constexpr int AllProcs = -1; /** * \brief Print on I/O Processor of the default communicator - * Example: Print() << " x = " << x << std::endl; + * Example: Print() << " x = " << x << std::endl; */ - explicit Print (std::ostream& os_ = amrex::OutStream()) - : rank(ParallelContext::IOProcessorNumberSub()) - , comm(ParallelContext::CommunicatorSub()) - , os(os_) - { ss.precision(os.precision()); } + explicit Print (std::ostream& os_ = amrex::OutStream()) + : rank(ParallelContext::IOProcessorNumberSub()) + , comm(ParallelContext::CommunicatorSub()) + , os(os_) + { ss.precision(os.precision()); } /** * \brief Print on all processors of the default communicator * Example: Print(Print::AllProcs) << " x = " << x << std::endl; */ - Print (int rank_, std::ostream& os_ = amrex::OutStream()) - : rank(rank_) - , comm(ParallelContext::CommunicatorSub()) - , os(os_) - { ss.precision(os.precision()); } + Print (int rank_, std::ostream& os_ = amrex::OutStream()) + : rank(rank_) + , comm(ParallelContext::CommunicatorSub()) + , os(os_) + { ss.precision(os.precision()); } - /** + /** * \brief Print on process rank_ of communicator comm_ * Example: Print(rank_, comm_) << " x = " << x << std::endl; */ - Print (int rank_, MPI_Comm comm_, std::ostream& os_ = amrex::OutStream()) - : rank(rank_) - , comm(comm_) - , os(os_) - { ss.precision(os.precision()); } - - ~Print () { - if (rank == AllProcs || rank == ParallelContext::MyProcSub()) { + Print (int rank_, MPI_Comm comm_, std::ostream& os_ = amrex::OutStream()) + : rank(rank_) + , comm(comm_) + , os(os_) + { ss.precision(os.precision()); } + + ~Print () { + if (rank == AllProcs || rank == ParallelContext::MyProcSub()) { std::ostream * my_os = ParallelContext::OFSPtrSub(); if (my_os) { my_os->flush(); @@ -77,21 +77,21 @@ namespace amrex os.flush(); os << ss.str(); os.flush(); - } - } + } + } Print& SetPrecision(int p) { ss.precision(p); return *this; - } + } - template - Print& operator<< (const T& x) { + template + Print& operator<< (const T& x) { ss << x; - return *this; - } + return *this; + } - Print& operator<< ( std::basic_ostream >& + Print& operator<< ( std::basic_ostream >& (*func)(std::basic_ostream >&)) { ss << func; @@ -99,21 +99,21 @@ namespace amrex } private: - int rank; - MPI_Comm comm; + int rank; + MPI_Comm comm; std::ostream &os; - std::ostringstream ss; + std::ostringstream ss; }; //! Print on all processors of the default communicator class AllPrint - : public Print + : public Print { public: //! Example: AllPrint() << " x = " << x << std::endl; - explicit AllPrint (std::ostream& os_ = amrex::OutStream()) - : Print(Print::AllProcs, os_) - {} + explicit AllPrint (std::ostream& os_ = amrex::OutStream()) + : Print(Print::AllProcs, os_) + {} }; @@ -122,46 +122,46 @@ namespace amrex { public: - static constexpr int AllProcs = -1; + static constexpr int AllProcs = -1; - explicit PrintToFile (const std::string& file_name_) - : file_name(file_name_) + explicit PrintToFile (const std::string& file_name_) + : file_name(file_name_) , rank(ParallelContext::IOProcessorNumberSub()) - , comm(ParallelContext::CommunicatorSub()) + , comm(ParallelContext::CommunicatorSub()) { Initialize(); } - - PrintToFile (const std::string& file_name_, int rank_ ) - : file_name(file_name_) - , rank(rank_) - , comm(ParallelContext::CommunicatorSub()) + + PrintToFile (const std::string& file_name_, int rank_ ) + : file_name(file_name_) + , rank(rank_) + , comm(ParallelContext::CommunicatorSub()) { Initialize(); } - - PrintToFile (const std::string& file_name_, int rank_, MPI_Comm comm_) - : file_name(file_name_) - , rank(rank_) - , comm(comm_) + + PrintToFile (const std::string& file_name_, int rank_, MPI_Comm comm_) + : file_name(file_name_) + , rank(rank_) + , comm(comm_) { Initialize(); } - ~PrintToFile () { - if (rank == AllProcs || rank == ParallelContext::MyProcSub()) { - ofs.flush(); - ofs << ss.str(); - ofs.flush(); - } - } + ~PrintToFile () { + if (rank == AllProcs || rank == ParallelContext::MyProcSub()) { + ofs.flush(); + ofs << ss.str(); + ofs.flush(); + } + } PrintToFile& SetPrecision(int p) { ss.precision(p); return *this; - } + } - template - PrintToFile& operator<< (const T& x) { + template + PrintToFile& operator<< (const T& x) { ss << x; - return *this; - } + return *this; + } - PrintToFile& operator<< ( std::basic_ostream >& + PrintToFile& operator<< ( std::basic_ostream >& (*func)(std::basic_ostream >&)) { ss << func; @@ -187,21 +187,21 @@ namespace amrex } std::string file_name; - int rank; - MPI_Comm comm; + int rank; + MPI_Comm comm; std::ofstream ofs; - std::ostringstream ss; + std::ostringstream ss; }; //! Print on all processors of the default communicator class AllPrintToFile - : public PrintToFile + : public PrintToFile { public: //! Example: AllPrint() << " x = " << x << std::endl; - explicit AllPrintToFile (const std::string& file_name_) - : PrintToFile(file_name_, Print::AllProcs) - {} + explicit AllPrintToFile (const std::string& file_name_) + : PrintToFile(file_name_, Print::AllProcs) + {} }; } diff --git a/Src/Base/AMReX_REAL.H b/Src/Base/AMReX_REAL.H index 6d95b20f499..9fd8965ff74 100644 --- a/Src/Base/AMReX_REAL.H +++ b/Src/Base/AMReX_REAL.H @@ -33,7 +33,7 @@ #include #endif -/* +/* Real is a typedef specifying the precision of the floating-point calculations in C++ code. It will be either float or double depending upon which of the macros BL_USE_FLOAT or diff --git a/Src/Base/AMReX_RealBox.cpp b/Src/Base/AMReX_RealBox.cpp index 38ca8ba4ad0..2554e3c3918 100644 --- a/Src/Base/AMReX_RealBox.cpp +++ b/Src/Base/AMReX_RealBox.cpp @@ -18,7 +18,7 @@ RealBox::RealBox (const Box& bx, xlo[i] = base[i] + dx[i]*blo[i]; int shft = (bx.type(i) == IndexType::CELL ? 1 : 0); xhi[i] = base[i] + dx[i]*(bhi[i]+ shft); - } + } } RealBox::RealBox (const std::array& a_lo, diff --git a/Src/Base/AMReX_RealVect.H b/Src/Base/AMReX_RealVect.H index 94a93de6526..03a253a5147 100644 --- a/Src/Base/AMReX_RealVect.H +++ b/Src/Base/AMReX_RealVect.H @@ -119,7 +119,7 @@ public: AMREX_GPU_HOST_DEVICE inline Real* begin () noexcept { return vect; } - + /// /** Return pointer to the first component. @@ -127,7 +127,7 @@ public: AMREX_GPU_HOST_DEVICE inline const Real* begin () const noexcept { return vect; } - + /// /** Return pointer to the one past last component @@ -135,7 +135,7 @@ public: AMREX_GPU_HOST_DEVICE inline Real* end () noexcept { return vect + AMREX_SPACEDIM; } - + /// /** Return pointer to the one past last component @@ -143,7 +143,7 @@ public: AMREX_GPU_HOST_DEVICE inline const Real* end () const noexcept { return vect + AMREX_SPACEDIM; } - + /*@}*/ /** @@ -349,7 +349,7 @@ public: /// /** - Returns an IntVect whose components are the std::floor of the RealVect + Returns an IntVect whose components are the std::floor of the RealVect components. */ AMREX_GPU_HOST_DEVICE inline @@ -357,7 +357,7 @@ public: /// /** - Returns an IntVect whose components are the std::floor of the RealVect + Returns an IntVect whose components are the std::floor of the RealVect components. */ AMREX_GPU_HOST_DEVICE inline @@ -365,7 +365,7 @@ public: /// /** - Returns an IntVect whose components are the std::floor of the RealVect + Returns an IntVect whose components are the std::floor of the RealVect components. */ AMREX_GPU_HOST_DEVICE inline diff --git a/Src/Base/AMReX_Reduce.H b/Src/Base/AMReX_Reduce.H index b5997e1aa30..371f35951f4 100644 --- a/Src/Base/AMReX_Reduce.H +++ b/Src/Base/AMReX_Reduce.H @@ -62,7 +62,7 @@ namespace Reduce { namespace detail { { P().init(amrex::get(t)); } - + template void for_each_init (T& t) { diff --git a/Src/Base/AMReX_SPACE.H b/Src/Base/AMReX_SPACE.H index 37eaeb03903..d89db7bd95f 100644 --- a/Src/Base/AMReX_SPACE.H +++ b/Src/Base/AMReX_SPACE.H @@ -57,7 +57,7 @@ namespace amrex arguments are evaluated. This macro can be used to enhance the portability of code that works with structures whose size depends on AMREX_SPACEDIM. - + For example: AMREX_D_EXPR(vect[0] *= s, vect[1] *= s, vect[2] *= s); diff --git a/Src/Base/AMReX_Scan.H b/Src/Base/AMReX_Scan.H index 60e05b44aa8..67af283e6f2 100644 --- a/Src/Base/AMReX_Scan.H +++ b/Src/Base/AMReX_Scan.H @@ -282,7 +282,7 @@ T PrefixSum (N n, FIN && fin, FOUT && fout, Type type) gh.item->barrier(sycl::access::fence_space::local_space); // shared[0:nwarps) holds the inclusive sum of warp sums. - + // Also note x still holds the inclusive sum within the // warp. Given these two, we can compute the inclusive // sum within this chunk. @@ -523,7 +523,7 @@ T PrefixSum (N n, FIN && fin, FOUT && fout, Type type) __syncthreads(); // shared[0:nwarps) holds the inclusive sum of warp sums. - + // Also note x still holds the inclusive sum within the // warp. Given these two, we can compute the inclusive // sum within this chunk. @@ -715,10 +715,10 @@ namespace Gpu return result_end; #else if (begin == end) return result; - + typename std::iterator_traits::value_type sum = *begin; *result++ = sum - *begin; - + while (++begin != end) { sum = std::move(sum) + *begin; *result++ = sum - *begin; diff --git a/Src/Base/AMReX_Slopes_K.H b/Src/Base/AMReX_Slopes_K.H index 83727439065..8bc0f55d711 100644 --- a/Src/Base/AMReX_Slopes_K.H +++ b/Src/Base/AMReX_Slopes_K.H @@ -57,7 +57,7 @@ amrex::Real amrex_calc_xslope (int i, int j, int k, int n, int order, } AMREX_GPU_DEVICE AMREX_FORCE_INLINE -amrex::Real amrex_calc_xslope_extdir (int i, int j, int k, int n, int order, +amrex::Real amrex_calc_xslope_extdir (int i, int j, int k, int n, int order, amrex::Array4 const& q, bool edlo, bool edhi, int domlo, int domhi) noexcept { @@ -197,7 +197,7 @@ amrex::Real amrex_calc_yslope (int i, int j, int k, int n, int order, } AMREX_GPU_DEVICE AMREX_FORCE_INLINE -amrex::Real amrex_calc_yslope_extdir (int i, int j, int k, int n, int order, +amrex::Real amrex_calc_yslope_extdir (int i, int j, int k, int n, int order, amrex::Array4 const& q, bool edlo, bool edhi, int domlo, int domhi) noexcept { @@ -214,7 +214,7 @@ amrex::Real amrex_calc_yslope_extdir (int i, int j, int k, int n, int order, amrex::Real slope = amrex::min(amrex::Math::abs(dl),amrex::Math::abs(dc),amrex::Math::abs(dr)); slope = (dr*dl > 0.0) ? slope : 0.0; return (dc > 0.0) ? slope : -slope; - + } else if (order == 4) { amrex::Real dlft, drgt, dcen, dfm, dfp, dlim, dsgn, dtemp, dlimsh,dsgnsh; @@ -335,7 +335,7 @@ amrex::Real amrex_calc_zslope (int i, int j, int k, int n, int order, } AMREX_GPU_DEVICE AMREX_FORCE_INLINE -amrex::Real amrex_calc_zslope_extdir (int i, int j, int k, int n, int order, +amrex::Real amrex_calc_zslope_extdir (int i, int j, int k, int n, int order, amrex::Array4 const& q, bool edlo, bool edhi, int domlo, int domhi) noexcept { @@ -353,7 +353,7 @@ amrex::Real amrex_calc_zslope_extdir (int i, int j, int k, int n, int order, amrex::Real slope = amrex::min(amrex::Math::abs(dl),amrex::Math::abs(dc),amrex::Math::abs(dr)); slope = (dr*dl > 0.0) ? slope : 0.0; return (dc > 0.0) ? slope : -slope; - + } else if (order == 4) { amrex::Real dlft, drgt, dcen, dfm, dfp, dlim, dsgn, dtemp, dlimsh, dsgnsh; diff --git a/Src/Base/AMReX_ThirdPartyProfiling.H b/Src/Base/AMReX_ThirdPartyProfiling.H index 4ee6269d295..6c0158f9b24 100644 --- a/Src/Base/AMReX_ThirdPartyProfiling.H +++ b/Src/Base/AMReX_ThirdPartyProfiling.H @@ -4,6 +4,6 @@ #include #elif defined (AMREX_VTUNE) #include -#elif defined (AMREX_FORGE) +#elif defined (AMREX_FORGE) #include #endif diff --git a/Src/Base/AMReX_TinyProfiler.H b/Src/Base/AMReX_TinyProfiler.H index 994ff85420e..0829a4ac09f 100644 --- a/Src/Base/AMReX_TinyProfiler.H +++ b/Src/Base/AMReX_TinyProfiler.H @@ -56,7 +56,7 @@ private: bool usesCUPTI; //!< uses CUPTI Long nk; //!< number of kernel calls }; - + //! stats across processes struct ProcStats { @@ -73,7 +73,7 @@ private: bool usesCUPTI; std::string fname; static bool compex (const ProcStats& lhs, const ProcStats& rhs) { - return lhs.dtexmax > rhs.dtexmax; + return lhs.dtexmax > rhs.dtexmax; } static bool compin (const ProcStats& lhs, const ProcStats& rhs) { return lhs.dtinmax > rhs.dtinmax; diff --git a/Src/Base/AMReX_TinyProfiler.cpp b/Src/Base/AMReX_TinyProfiler.cpp index 76b60a63c26..3b961c05d6a 100644 --- a/Src/Base/AMReX_TinyProfiler.cpp +++ b/Src/Base/AMReX_TinyProfiler.cpp @@ -79,25 +79,25 @@ TinyProfiler::start () noexcept if (stats.empty() && !regionstack.empty()) { double t; - if (!uCUPTI) { - t = amrex::second(); - } else { + if (!uCUPTI) { + t = amrex::second(); + } else { #ifdef AMREX_USE_CUPTI - cudaDeviceSynchronize(); - cuptiActivityFlushAll(0); - activityRecordUserdata.clear(); - t = amrex::second(); + cudaDeviceSynchronize(); + cuptiActivityFlushAll(0); + activityRecordUserdata.clear(); + t = amrex::second(); #endif - } + } - ttstack.emplace_back(std::make_tuple(t, 0.0, &fname)); - global_depth = ttstack.size(); + ttstack.emplace_back(std::make_tuple(t, 0.0, &fname)); + global_depth = ttstack.size(); #ifdef AMREX_USE_CUDA if (device_synchronize_around_region) { amrex::Gpu::Device::synchronize(); } - nvtxRangePush(fname.c_str()); + nvtxRangePush(fname.c_str()); #endif for (auto const& region : regionstack) @@ -118,10 +118,10 @@ TinyProfiler::stop () noexcept #ifdef AMREX_USE_OMP #pragma omp master #endif - if (!stats.empty()) + if (!stats.empty()) { double t; - int nKernelCalls = 0; + int nKernelCalls = 0; #ifdef AMREX_USE_CUPTI if (uCUPTI) { cudaDeviceSynchronize(); @@ -131,60 +131,60 @@ TinyProfiler::stop () noexcept } else #endif { - t = amrex::second(); + t = amrex::second(); } - while (static_cast(ttstack.size()) > global_depth) { - ttstack.pop_back(); - }; - - if (static_cast(ttstack.size()) == global_depth) - { - const std::tuple& tt = ttstack.back(); - - // first: wall time when the pair is pushed into the stack - // second: accumulated dt of children - double dtin; - double dtex; - if (!uCUPTI) { - dtin = t - std::get<0>(tt); // elapsed time since start() is called. - dtex = dtin - std::get<1>(tt); - } else { - dtin = t; - dtex = dtin - std::get<1>(tt); - } - - for (Stats* st : stats) + while (static_cast(ttstack.size()) > global_depth) { + ttstack.pop_back(); + }; + + if (static_cast(ttstack.size()) == global_depth) { - --(st->depth); - ++(st->n); - if (st->depth == 0) { - st->dtin += dtin; + const std::tuple& tt = ttstack.back(); + + // first: wall time when the pair is pushed into the stack + // second: accumulated dt of children + double dtin; + double dtex; + if (!uCUPTI) { + dtin = t - std::get<0>(tt); // elapsed time since start() is called. + dtex = dtin - std::get<1>(tt); + } else { + dtin = t; + dtex = dtin - std::get<1>(tt); } - st->dtex += dtex; - st->usesCUPTI = uCUPTI; - if (uCUPTI) { - st->nk += nKernelCalls; + + for (Stats* st : stats) + { + --(st->depth); + ++(st->n); + if (st->depth == 0) { + st->dtin += dtin; + } + st->dtex += dtex; + st->usesCUPTI = uCUPTI; + if (uCUPTI) { + st->nk += nKernelCalls; + } } - } - ttstack.pop_back(); - if (!ttstack.empty()) { - std::tuple& parent = ttstack.back(); - std::get<1>(parent) += dtin; - } + ttstack.pop_back(); + if (!ttstack.empty()) { + std::tuple& parent = ttstack.back(); + std::get<1>(parent) += dtin; + } #ifdef AMREX_USE_CUDA - if (device_synchronize_around_region) { - amrex::Gpu::Device::synchronize(); - } - nvtxRangePop(); + if (device_synchronize_around_region) { + amrex::Gpu::Device::synchronize(); + } + nvtxRangePop(); #endif - } else { - improperly_nested_timers.insert(fname); - } + } else { + improperly_nested_timers.insert(fname); + } - stats.clear(); + stats.clear(); } if (verbose) { amrex::Print() << " TP: Leaving " << fname << std::endl; @@ -198,7 +198,7 @@ TinyProfiler::stop (unsigned boxUintID) noexcept #ifdef AMREX_USE_OMP #pragma omp master #endif - if (!stats.empty()) + if (!stats.empty()) { double t; cudaDeviceSynchronize(); @@ -206,12 +206,12 @@ TinyProfiler::stop (unsigned boxUintID) noexcept t = computeElapsedTimeUserdata(activityRecordUserdata); int nKernelCalls = activityRecordUserdata.size(); - for (auto& record : activityRecordUserdata) + for (auto& record : activityRecordUserdata) { record->setUintID(boxUintID); } - while (static_cast(ttstack.size()) > global_depth) + while (static_cast(ttstack.size()) > global_depth) { ttstack.pop_back(); }; @@ -226,23 +226,23 @@ TinyProfiler::stop (unsigned boxUintID) noexcept double dtex; dtin = t; - dtex = dtin - std::get<1>(tt); + dtex = dtin - std::get<1>(tt); for (Stats* st : stats) { --(st->depth); ++(st->n); - if (st->depth == 0) + if (st->depth == 0) { st->dtin += dtin; } st->dtex += dtex; st->usesCUPTI = uCUPTI; - st->nk += nKernelCalls; + st->nk += nKernelCalls; } ttstack.pop_back(); - if (!ttstack.empty()) + if (!ttstack.empty()) { std::tuple& parent = ttstack.back(); std::get<1>(parent) += dtin; @@ -254,7 +254,7 @@ TinyProfiler::stop (unsigned boxUintID) noexcept } nvtxRangePop(); #endif - } else + } else { improperly_nested_timers.insert(fname); } @@ -285,7 +285,7 @@ void TinyProfiler::Finalize (bool bFlushing) noexcept { static bool finalized = false; - if (!bFlushing) { // If flushing, don't make this the last time! + if (!bFlushing) { // If flushing, don't make this the last time! if (finalized) { return; } else { @@ -335,7 +335,7 @@ TinyProfiler::Finalize (bool bFlushing) noexcept { amrex::Print() << "\n\n"; amrex::Print().SetPrecision(4) - <<"TinyProfiler total time across processes [min...avg...max]: " + <<"TinyProfiler total time across processes [min...avg...max]: " << dt_min << " ... " << dt_avg << " ... " << dt_max << "\n"; } @@ -490,11 +490,11 @@ TinyProfiler::PrintStats (std::map& regstats, double dt_max) << std::setw(wt+2) << it->dtexmin << std::setw(wt+2) << it->dtexavg << std::setw(wt+2) << it->dtexmax - << std::setprecision(2) << std::setw(wp+1) << std::fixed + << std::setprecision(2) << std::setw(wp+1) << std::fixed << it->dtexmax*(100.0/dt_max) << "%"; amrex::OutStream().unsetf(std::ios_base::fixed); - amrex::OutStream() << "\n"; -#ifdef AMREX_USE_CUPTI + amrex::OutStream() << "\n"; +#ifdef AMREX_USE_CUPTI if (it->usesCUPTI) { amrex::OutStream() << std::setprecision(4) << std::left @@ -528,7 +528,7 @@ TinyProfiler::PrintStats (std::map& regstats, double dt_max) for (auto it = allprocstats.cbegin(); it != allprocstats.cend(); ++it) { #ifdef AMREX_USE_CUPTI - if (it->usesCUPTI) + if (it->usesCUPTI) { amrex::OutStream() << hlinehlf << "START CUPTI Trace Stats-" << hlinehlf << "\n"; } @@ -540,12 +540,12 @@ TinyProfiler::PrintStats (std::map& regstats, double dt_max) << std::setw(wt+2) << it->dtinmin << std::setw(wt+2) << it->dtinavg << std::setw(wt+2) << it->dtinmax - << std::setprecision(2) << std::setw(wp+1) << std::fixed + << std::setprecision(2) << std::setw(wp+1) << std::fixed << it->dtinmax*(100.0/dt_max) << "%"; amrex::OutStream().unsetf(std::ios_base::fixed); amrex::OutStream() << "\n"; #ifdef AMREX_USE_CUPTI - if (it->usesCUPTI) + if (it->usesCUPTI) { amrex::OutStream() << std::setprecision(4) << std::left << std::setw(maxfnamelen) // it->fname diff --git a/Src/Base/AMReX_TypeTraits.H b/Src/Base/AMReX_TypeTraits.H index 1b1468bb503..73d56684587 100644 --- a/Src/Base/AMReX_TypeTraits.H +++ b/Src/Base/AMReX_TypeTraits.H @@ -122,10 +122,10 @@ namespace amrex // // is_detected was proposed to C++ but is surpassed by concepts. // - // The implementation is taken from + // The implementation is taken from // // https://en.cppreference.com/w/cpp/experimental/is_detected - + namespace detail { template using Void_t = void; @@ -194,7 +194,7 @@ namespace amrex // [traits.Negation] - //! Logical traits let us combine multiple type requirements in one EnableIf_t clause. + //! Logical traits let us combine multiple type requirements in one EnableIf_t clause. #if defined(__cpp_lib_logical_traits) template using Conjunction = std::conjunction; template using Disjunction = std::disjunction; diff --git a/Src/Base/AMReX_Utility.H b/Src/Base/AMReX_Utility.H index 07da0ef3a44..62dba4c8dfb 100644 --- a/Src/Base/AMReX_Utility.H +++ b/Src/Base/AMReX_Utility.H @@ -42,7 +42,7 @@ namespace amrex const std::vector& Tokenize (const std::string& instr, const std::string& separators); - //! Converts all characters of the string into lower or uppercase based on std::locale + //! Converts all characters of the string into lower or uppercase based on std::locale std::string toLower (std::string s); std::string toUpper (std::string s); @@ -62,15 +62,15 @@ namespace amrex * if path is NULL or "/". mode is the mode passed to mkdir() for * any directories that must be created (for example: 0755). * verbose will print out the directory creation steps. - * + * * For example, if it is passed the string "/a/b/c/d/e/f/g", it * will return successfully when all the directories in the pathname * exist; i.e. when the full pathname is a valid directory. * * In a Windows environment, the path separator is a '\', so that if using - * the example given above you must pass the string + * the example given above you must pass the string * "\\a\\b\\c\\d\\e\\f\\g" (Note that you must escape the backslash in a - * character string), + * character string), * * Only the last mkdir return value is checked for success as errno may not * be set to EEXIST if a directory exists but mkdir has other reasons @@ -79,7 +79,7 @@ namespace amrex */ bool UtilCreateDirectory (const std::string& path, mode_t mode, - bool verbose = false); + bool verbose = false); //! Output a message and abort when couldn't create the directory. void CreateDirectoryFailed (const std::string& dir); //! Output a message and abort when couldn't open the file. @@ -165,7 +165,7 @@ namespace amrex * "p" MUST be in the open interval (0,1). * * Entry points for Fortran: - * + * * REAL_T val * call blinvnormdistbest(val) * @@ -179,44 +179,44 @@ namespace amrex /* \brief cumulative refinement ratio between levels */ int CRRBetweenLevels(int fromlevel, int tolevel, const Vector &refratios); - + class expect; std::istream& operator>>(std::istream&, const expect& exp); - + class expect { - friend std::istream& operator>>(std::istream&, const expect& exp); + friend std::istream& operator>>(std::istream&, const expect& exp); public: - explicit expect(const std::string& str_); - explicit expect(const char* istr_); - explicit expect(char c); - const std::string& the_string() const; + explicit expect(const std::string& str_); + explicit expect(const char* istr_); + explicit expect(char c); + const std::string& the_string() const; private: - std::string istr; + std::string istr; }; class StreamRetry { public: StreamRetry(std::ostream &os, const std::string &suffix, - const int maxtries); + const int maxtries); StreamRetry(const std::string &filename, - const bool abortonretryfailure, - const int maxtries); - bool TryOutput(); - bool TryFileOutput(); + const bool abortonretryfailure, + const int maxtries); + bool TryOutput(); + bool TryFileOutput(); static int NStreamErrors() { return nStreamErrors; } static void ClearStreamErrors() { nStreamErrors = 0; } private: int tries, maxTries; - bool abortOnRetryFailure; - std::string fileName; - std::ostream &sros; - std::ostream::pos_type spos; - std::string suffix; + bool abortOnRetryFailure; + std::string fileName; + std::ostream &sros; + std::ostream::pos_type spos; + std::string suffix; - static int nStreamErrors; + static int nStreamErrors; }; Vector SerializeStringArray(const Vector &stringArray); @@ -328,7 +328,7 @@ template void amrex::BroadcastArray(Vector &aT, int myLocalId, int r /* The macro BL_FORT_USE_LOWERCASE indicates that C++ code should call Fortran routines using lowercase letters for all the letters in the - routine. This is set automatically by the make subsystem. + routine. This is set automatically by the make subsystem. For example, if the Fortran routine is named abcxyx, then it will be called in C++ code as abcxyx. @@ -347,7 +347,7 @@ template void amrex::BroadcastArray(Vector &aT, int myLocalId, int r */ #define BL_IGNORE_MAX 100000 -template +template amrex::Long amrex::bytesOf (const std::vector& v) { diff --git a/Src/Base/AMReX_Utility.cpp b/Src/Base/AMReX_Utility.cpp index 2b7151591e9..a5c37d2a3ca 100644 --- a/Src/Base/AMReX_Utility.cpp +++ b/Src/Base/AMReX_Utility.cpp @@ -148,7 +148,7 @@ amrex::Concatenate (const std::string& root, bool amrex::UtilCreateDirectory (const std::string& path, - mode_t mode, bool verbose) + mode_t mode, bool verbose) { return FileSystem::CreateDirectories(path, mode, verbose); } @@ -210,9 +210,9 @@ amrex::UtilCreateCleanDirectory (const std::string &path, bool callbarrier) void amrex::UtilCreateDirectoryDestructive(const std::string &path, bool callbarrier) { - if(ParallelContext::IOProcessorSub()) + if(ParallelContext::IOProcessorSub()) { - if(amrex::FileExists(path)) + if(amrex::FileExists(path)) { if (amrex::Verbose() > 1) { amrex::Print() << "amrex::UtilCreateCleanDirectoryDestructive(): " << path @@ -220,12 +220,12 @@ amrex::UtilCreateDirectoryDestructive(const std::string &path, bool callbarrier) } FileSystem::RemoveAll(path); } - if( ! amrex::UtilCreateDirectory(path, 0755)) + if( ! amrex::UtilCreateDirectory(path, 0755)) { amrex::CreateDirectoryFailed(path); } } - if(callbarrier) + if(callbarrier) { // Force other processors to wait until directory is built. ParallelDescriptor::Barrier("amrex::UtilCreateCleanDirectoryDestructive"); @@ -403,7 +403,7 @@ amrex::InvNormDist (double p) // // Parameters: // -// Input, double P, the value of the cumulative probability +// Input, double P, the value of the cumulative probability // densitity function. 0 < P < 1. Fails if P is outside this range. // // Output, the normal deviate value with the property that the @@ -416,7 +416,7 @@ amrex::InvNormDistBest (double p) { static const double a[8] = - { + { 3.3871328727963666080, 1.3314166789178437745e+2, 1.9715909503065514427e+3, 1.3731693765509461125e+4, 4.5921953931549871457e+4, 6.7265770927008700853e+4, @@ -537,20 +537,20 @@ amrex::operator>>(std::istream& is, const expect& exp) int n = 0; while ( n < len ) { - char c; - is >> c; - if ( !is ) break; - if ( c != exp.istr[n++] ) - { - is.putback(c); - break; - } + char c; + is >> c; + if ( !is ) break; + if ( c != exp.istr[n++] ) + { + is.putback(c); + break; + } } if ( n != len ) { - is.clear(std::ios::badbit|is.rdstate()); - std::string msg = "expect fails to find \"" + exp.the_string() + "\""; - amrex::Error(msg.c_str()); + is.clear(std::ios::badbit|is.rdstate()); + std::string msg = "expect fails to find \"" + exp.the_string() + "\""; + amrex::Error(msg.c_str()); } return is; } @@ -590,7 +590,7 @@ amrex::StreamRetry::StreamRetry(std::ostream &a_os, const std::string &a_suffix, } amrex::StreamRetry::StreamRetry(const std::string &filename, - const bool abortonretryfailure, + const bool abortonretryfailure, const int maxtries) : tries(0), maxTries(maxtries), abortOnRetryFailure(abortonretryfailure), @@ -729,7 +729,7 @@ void amrex::SyncStrings(const Vector &localStrings, while( ! pfIn.eof()) { std::getline(pfIn, pfName, '\n'); if( ! pfIn.eof()) { - ioprocStrings.push_back(pfName); + ioprocStrings.push_back(pfName); } } // ---- now check if they match on non ioprocs @@ -749,7 +749,7 @@ void amrex::SyncStrings(const Vector &localStrings, bool matched(false); for(int i(0); i < ioprocStrings.size(); ++i) { if(localStringsCopy[n] == ioprocStrings[i]) { - matched = true; + matched = true; } } if( ! matched) { @@ -800,7 +800,7 @@ void amrex::SyncStrings(const Vector &localStrings, } BL_MPI_REQUIRE( MPI_Gatherv(sendCharArray.dataPtr(), - sendStringsSize, + sendStringsSize, ParallelDescriptor::Mpi_typemap::type(), recvStrings.dataPtr(), nChars.dataPtr(), @@ -816,7 +816,7 @@ void amrex::SyncStrings(const Vector &localStrings, while( ! pfIn.eof()) { std::getline(pfIn, pfName, '\n'); if( ! pfIn.eof()) { - syncedStrings.push_back(pfName); // ---- add the gathered strings + syncedStrings.push_back(pfName); // ---- add the gathered strings } } } @@ -844,7 +844,7 @@ void amrex::SyncStrings(const Vector &localStrings, while( ! syncedIn.eof()) { std::getline(syncedIn, syncedName, '\n'); if( ! syncedIn.eof()) { - syncedStrings.push_back(syncedName); + syncedStrings.push_back(syncedName); } } } diff --git a/Src/Base/AMReX_Vector.H b/Src/Base/AMReX_Vector.H index 13490128eb1..04a9a2ee5f3 100644 --- a/Src/Base/AMReX_Vector.H +++ b/Src/Base/AMReX_Vector.H @@ -33,13 +33,13 @@ public: T& operator[] (size_type i) noexcept { BL_ASSERT( i < (this->std::vector::size()) ); - return this->std::vector::operator[](i); + return this->std::vector::operator[](i); } const T& operator[] (size_type i) const noexcept { BL_ASSERT( i < (this->std::vector::size()) ); - return this->std::vector::operator[](i); + return this->std::vector::operator[](i); } //! get access to the underlying data pointer @@ -50,7 +50,7 @@ public: Long size () const noexcept {return static_cast(std::vector::size());} }; - + } namespace amrex @@ -161,12 +161,12 @@ namespace amrex IsBaseFab::value, int>::type = 0 > Vector > - GetVecOfArrOfPtrs(Vector >& a) + GetVecOfArrOfPtrs(Vector >& a) { - Vector > r; - r.reserve(a.size()); - for (auto &x: a) r.push_back(GetArrOfPtrs(x)); - return r; + Vector > r; + r.reserve(a.size()); + for (auto &x: a) r.push_back(GetArrOfPtrs(x)); + return r; } #endif @@ -186,7 +186,7 @@ namespace amrex ///////////////////////////////////////////////////////////// - template + template void RemoveDuplicates (Vector& vec) { std::sort(vec.begin(), vec.end()); auto it = std::unique(vec.begin(), vec.end()); diff --git a/Src/Base/AMReX_VectorIO.H b/Src/Base/AMReX_VectorIO.H index 3492fff0e94..4d79335b4e3 100644 --- a/Src/Base/AMReX_VectorIO.H +++ b/Src/Base/AMReX_VectorIO.H @@ -24,7 +24,7 @@ namespace amrex { //! Read int data from the istream. The arguments are a pointer to data buffer //! to read into, the size of that buffer, the istream, and an IntDescriptor - //! that describes the format of the data on disk. + //! that describes the format of the data on disk. //! The buffer is assumed to be large enough to store 'size' integers, and it is //! the user's reponsiblity to allocate this data. void readIntData(int* data, std::size_t size, std::istream& is, const IntDescriptor& id); @@ -39,7 +39,7 @@ namespace amrex { //! Read int data from the istream. The arguments are a pointer to data buffer //! to read into, the size of that buffer, the istream, and an IntDescriptor - //! that describes the format of the data on disk. + //! that describes the format of the data on disk. //! The buffer is assumed to be large enough to store 'size' longs, and it is //! the user's reponsiblity to allocate this data. void readLongData(Long* data, std::size_t size, std::istream& is, const IntDescriptor& id); @@ -48,7 +48,7 @@ namespace amrex { //! to write, the size of the data buffer, the ostream, and an optional //! RealDescriptor that describes the data format to use for writing. If no //! RealDescriptor is provided, the data will be written using the native - //! format for your machine. + //! format for your machine. void writeRealData(const Real* data, std::size_t size, std::ostream& os, const RealDescriptor& rd = FPC::NativeRealDescriptor()); @@ -133,4 +133,4 @@ namespace amrex { } } -#endif +#endif diff --git a/Src/Base/AMReX_VectorIO.cpp b/Src/Base/AMReX_VectorIO.cpp index 31b6e1dec10..90bb7a224e1 100644 --- a/Src/Base/AMReX_VectorIO.cpp +++ b/Src/Base/AMReX_VectorIO.cpp @@ -29,10 +29,10 @@ void amrex::writeIntData(const int* data, std::size_t size, std::ostream& os, void amrex::readIntData(int* data, std::size_t size, std::istream& is, const IntDescriptor& id) { - if (id == FPC::NativeIntDescriptor()) + if (id == FPC::NativeIntDescriptor()) { is.read((char*) data, size * id.numBytes()); - } + } else if (id.numBytes() == 2) { readIntData(data, size, is, id); @@ -77,10 +77,10 @@ void amrex::writeLongData(const Long* data, std::size_t size, std::ostream& os, void amrex::readLongData(Long* data, std::size_t size, std::istream& is, const IntDescriptor& id) { - if (id == FPC::NativeLongDescriptor()) + if (id == FPC::NativeLongDescriptor()) { is.read((char*) data, size * id.numBytes()); - } + } else if (id.numBytes() == 2) { readIntData(data, size, is, id); diff --git a/Src/Base/AMReX_VisMF.H b/Src/Base/AMReX_VisMF.H index fd6b04167a9..98ce8943cb6 100644 --- a/Src/Base/AMReX_VisMF.H +++ b/Src/Base/AMReX_VisMF.H @@ -84,7 +84,7 @@ public: Header (); //! Construct from a FabArray. Header (const FabArray& fafab, VisMF::How how, Version version = Version_v1, - bool calcMinMax = true, MPI_Comm = ParallelDescriptor::Communicator()); + bool calcMinMax = true, MPI_Comm = ParallelDescriptor::Communicator()); Header (Header&& rhs) noexcept = default; @@ -307,7 +307,7 @@ private: //! fileNumbers must be passed in for dynamic set selection [proc] static void FindOffsets (const FabArray &fafab, - const std::string &fafab_name, + const std::string &fafab_name, VisMF::Header &hdr, VisMF::Header::Version whichVersion, NFilesIter &nfi, diff --git a/Src/Base/AMReX_VisMF.cpp b/Src/Base/AMReX_VisMF.cpp index 242b4afcfd5..e48fda01b15 100644 --- a/Src/Base/AMReX_VisMF.cpp +++ b/Src/Base/AMReX_VisMF.cpp @@ -246,9 +246,9 @@ operator>> (std::istream& is, #else is >> ar[i][j] >> ch; #endif - if( ch != ',' ) { - amrex::Error("Expected a ',' got something else"); - } + if( ch != ',' ) { + amrex::Error("Expected a ',' got something else"); + } } } @@ -383,15 +383,15 @@ operator>> (std::istream &is, hd.m_famax.resize(hd.m_ncomp); for(int i(0); i < hd.m_famin.size(); ++i) { is >> hd.m_famin[i] >> ch; - if( ch != ',' ) { - amrex::Error("Expected a ',' when reading hd.m_famin"); - } + if( ch != ',' ) { + amrex::Error("Expected a ',' when reading hd.m_famin"); + } } for(int i(0); i < hd.m_famax.size(); ++i) { is >> hd.m_famax[i] >> ch; - if( ch != ',' ) { - amrex::Error("Expected a ',' when reading hd.m_famax"); - } + if( ch != ',' ) { + amrex::Error("Expected a ',' when reading hd.m_famax"); + } } } if(hd.m_vers == VisMF::Header::NoFabHeader_v1 || @@ -557,7 +557,7 @@ VisMF::readFAB (int idx, FArrayBox* VisMF::readFAB (int idx, - int ncomp) + int ncomp) { return VisMF::readFAB(idx, m_fafabname, m_hdr, ncomp); } @@ -647,8 +647,8 @@ VisMF::Header::Header () VisMF::Header::Header (const FabArray& mf, VisMF::How how, - Version version, - bool calcMinMax, + Version version, + bool calcMinMax, MPI_Comm comm) : m_vers(version), @@ -902,7 +902,7 @@ VisMF::WriteHeaderDoit (const std::string&mf_name, const VisMF::Header& hdr) Long VisMF::WriteHeader (const std::string &mf_name, VisMF::Header &hdr, - int procToWrite, MPI_Comm comm) + int procToWrite, MPI_Comm comm) { // BL_PROFILE("VisMF::WriteHeader"); Long bytesWritten(0); @@ -911,14 +911,14 @@ VisMF::WriteHeader (const std::string &mf_name, VisMF::Header &hdr, bytesWritten += WriteHeaderDoit(mf_name, hdr); - if(checkFilePositions) { + if(checkFilePositions) { std::stringstream hss; - hss << hdr; - if(static_cast(hss.tellp()) != bytesWritten) { + hss << hdr; + if(static_cast(hss.tellp()) != bytesWritten) { amrex::ErrorStream() << "**** tellp error: hss.tellp() != bytesWritten : " << hss.tellp() << " " << bytesWritten << std::endl; - } - } + } + } } return bytesWritten; @@ -1141,7 +1141,7 @@ VisMF::WriteOnlyHeader (const FabArray & mf, fod.m_head = -1; } - // Write header on the IOProcessorNumber + // Write header on the IOProcessorNumber int coordinatorProc(ParallelDescriptor::IOProcessorNumber()); bytesWritten += VisMF::WriteHeader(mf_name, hdr, coordinatorProc); @@ -1151,10 +1151,10 @@ VisMF::WriteOnlyHeader (const FabArray & mf, void VisMF::FindOffsets (const FabArray &mf, - const std::string &filePrefix, + const std::string &filePrefix, VisMF::Header &hdr, - VisMF::Header::Version /*whichVersion*/, - NFilesIter &nfi, MPI_Comm comm) + VisMF::Header::Version /*whichVersion*/, + NFilesIter &nfi, MPI_Comm comm) { // BL_PROFILE("VisMF::FindOffsets"); @@ -1248,67 +1248,67 @@ VisMF::FindOffsets (const FabArray &mf, int nComps(mf.nComp()); if(myProc == coordinatorProc) { // ---- calculate offsets - const BoxArray &mfBA = mf.boxArray(); - const DistributionMapping &mfDM = mf.DistributionMap(); - Vector fabHeaderBytes(mfBA.size(), 0); - int nFiles(NFilesIter::ActualNFiles(nOutFiles)); - int whichFileNumber(-1); - std::string whichFileName; - Vector currentOffset(nProcs, 0L); + const BoxArray &mfBA = mf.boxArray(); + const DistributionMapping &mfDM = mf.DistributionMap(); + Vector fabHeaderBytes(mfBA.size(), 0); + int nFiles(NFilesIter::ActualNFiles(nOutFiles)); + int whichFileNumber(-1); + std::string whichFileName; + Vector currentOffset(nProcs, 0L); if(hdr.m_vers == VisMF::Header::Version_v1) { - // ---- find the length of the fab header instead of asking the file system - for(int i(0); i < mfBA.size(); ++i) { + // ---- find the length of the fab header instead of asking the file system + for(int i(0); i < mfBA.size(); ++i) { std::stringstream hss; - FArrayBox tempFab(mf.fabbox(i), nComps, false); // ---- no alloc + FArrayBox tempFab(mf.fabbox(i), nComps, false); // ---- no alloc fio.write_header(hss, tempFab, tempFab.nComp()); - fabHeaderBytes[i] = static_cast(hss.tellp()); - } - } + fabHeaderBytes[i] = static_cast(hss.tellp()); + } + } - std::map > rankBoxOrder; // ---- [rank, boxarray index array] - for(int i(0); i < mfBA.size(); ++i) { - rankBoxOrder[mfDM[i]].push_back(i); - } + std::map > rankBoxOrder; // ---- [rank, boxarray index array] + for(int i(0); i < mfBA.size(); ++i) { + rankBoxOrder[mfDM[i]].push_back(i); + } - Vector fileNumbers; + Vector fileNumbers; if(nfi.GetDynamic()) { - fileNumbers = nfi.FileNumbersWritten(); + fileNumbers = nfi.FileNumbersWritten(); } else if(nfi.GetSparseFPP()) { // if sparse, write to (file number = rank) - fileNumbers.resize(nProcs); - for(int i(0); i < nProcs; ++i) { - fileNumbers[i] = i; + fileNumbers.resize(nProcs); + for(int i(0); i < nProcs; ++i) { + fileNumbers[i] = i; } } else { - fileNumbers.resize(nProcs); - for(int i(0); i < nProcs; ++i) { - fileNumbers[i] = NFilesIter::FileNumber(nFiles, i, groupSets); - } - } + fileNumbers.resize(nProcs); + for(int i(0); i < nProcs; ++i) { + fileNumbers[i] = NFilesIter::FileNumber(nFiles, i, groupSets); + } + } - const Vector< Vector > &fileNumbersWriteOrder = nfi.FileNumbersWriteOrder(); + const Vector< Vector > &fileNumbersWriteOrder = nfi.FileNumbersWriteOrder(); - for(int fn(0); fn < fileNumbersWriteOrder.size(); ++fn) { - for(int ri(0); ri < fileNumbersWriteOrder[fn].size(); ++ri) { - int rank(fileNumbersWriteOrder[fn][ri]); - std::map >::iterator rboIter = rankBoxOrder.find(rank); + for(int fn(0); fn < fileNumbersWriteOrder.size(); ++fn) { + for(int ri(0); ri < fileNumbersWriteOrder[fn].size(); ++ri) { + int rank(fileNumbersWriteOrder[fn][ri]); + std::map >::iterator rboIter = rankBoxOrder.find(rank); if(rboIter != rankBoxOrder.end()) { Vector &index = rboIter->second; - whichFileNumber = fileNumbers[rank]; - whichFileName = VisMF::BaseName(NFilesIter::FileName(whichFileNumber, filePrefix)); + whichFileNumber = fileNumbers[rank]; + whichFileName = VisMF::BaseName(NFilesIter::FileName(whichFileNumber, filePrefix)); - for(int i(0); i < index.size(); ++i) { + for(int i(0); i < index.size(); ++i) { hdr.m_fod[index[i]].m_name = whichFileName; hdr.m_fod[index[i]].m_head = currentOffset[whichFileNumber]; currentOffset[whichFileNumber] += mf.fabbox(index[i]).numPts() * nComps * whichRDBytes - + fabHeaderBytes[index[i]]; + + fabHeaderBytes[index[i]]; } } - } - } + } + } } delete whichRD; } @@ -1327,21 +1327,21 @@ VisMF::RemoveFiles(const std::string &mf_name, bool a_verbose) if(a_verbose) { if(retVal != 0) { amrex::Print() << "---- error removing: " << MFHdrFileName << " errno = " - << strerror(errno) << std::endl; + << strerror(errno) << std::endl; } } for(int ip(0); ip < nOutFiles; ++ip) { std::string fileName(NFilesIter::FileName(nOutFiles, mf_name + FabFileSuffix, ip, true)); if(a_verbose) { amrex::Print() << "---- removing: " << fileName << std::endl; - } + } int rv(std::remove(fileName.c_str())); if(a_verbose) { if(rv != 0) { amrex::Print() << "---- error removing: " << fileName << " errno = " - << strerror(errno) << std::endl; + << strerror(errno) << std::endl; } - } + } } } } @@ -1383,7 +1383,7 @@ FArrayBox* VisMF::readFAB (int idx, const std::string &mf_name, const VisMF::Header &hdr, - int whichComp) + int whichComp) { // BL_PROFILE("VisMF::readFAB_idx"); Box fab_box(hdr.m_ba[idx]); @@ -1407,24 +1407,24 @@ VisMF::readFAB (int idx, } } else { if(whichComp == -1) { // ---- read all components - if(hdr.m_writtenRD == FPC::NativeRealDescriptor()) { + if(hdr.m_writtenRD == FPC::NativeRealDescriptor()) { infs->read((char *) fab->dataPtr(), fab->nBytes()); - } else { + } else { Long readDataItems(fab->box().numPts() * fab->nComp()); RealDescriptor::convertToNativeFormat(fab->dataPtr(), readDataItems, - *infs, hdr.m_writtenRD); - } + *infs, hdr.m_writtenRD); + } } else { Long bytesPerComp(fab->box().numPts() * hdr.m_writtenRD.numBytes()); infs->seekg(bytesPerComp * whichComp, std::ios::cur); - if(hdr.m_writtenRD == FPC::NativeRealDescriptor()) { + if(hdr.m_writtenRD == FPC::NativeRealDescriptor()) { infs->read((char *) fab->dataPtr(), bytesPerComp); - } else { + } else { Long readDataItems(fab->box().numPts()); // ---- one component only RealDescriptor::convertToNativeFormat(fab->dataPtr(), readDataItems, - *infs, hdr.m_writtenRD); - } + *infs, hdr.m_writtenRD); + } } } @@ -1436,7 +1436,7 @@ VisMF::readFAB (int idx, void VisMF::readFAB (FabArray &mf, - int idx, + int idx, const std::string& mf_name, const VisMF::Header& hdr) { @@ -1455,7 +1455,7 @@ VisMF::readFAB (FabArray &mf, } else { Long readDataItems(fab.box().numPts() * fab.nComp()); RealDescriptor::convertToNativeFormat(fab.dataPtr(), readDataItems, - *infs, hdr.m_writtenRD); + *infs, hdr.m_writtenRD); } } else { fab.readFrom(*infs); @@ -1468,9 +1468,9 @@ VisMF::readFAB (FabArray &mf, void VisMF::Read (FabArray &mf, const std::string &mf_name, - const char *faHeader, - int coordinatorProc, - int allow_empty_mf) + const char *faHeader, + int coordinatorProc, + int allow_empty_mf) { BL_PROFILE("VisMF::Read()"); @@ -1490,13 +1490,13 @@ VisMF::Read (FabArray &mf, { hStartTime = amrex::second(); std::string fileCharPtrString; - if(faHeader == nullptr) { + if(faHeader == nullptr) { Vector fileCharPtr; ParallelDescriptor::ReadAndBcastFile(FullHdrFileName, fileCharPtr); fileCharPtrString = fileCharPtr.dataPtr(); - } else { + } else { fileCharPtrString = faHeader; - } + } std::istringstream infs(fileCharPtrString, std::istringstream::in); infs >> hdr; @@ -1507,9 +1507,9 @@ VisMF::Read (FabArray &mf, // This allows us to read in an empty MultiFab without an error -- but only if explicitly told to if (allow_empty_mf > 0) { - if (hdr.m_ba.size() == 0) return; + if (hdr.m_ba.size() == 0) return; } else { - if (hdr.m_ba.size() == 0) + if (hdr.m_ba.size() == 0) { amrex::Print() << "In trying to read " << mf_name << std::endl; amrex::Error("Empty box array"); @@ -1518,10 +1518,10 @@ VisMF::Read (FabArray &mf, if (mf.empty()) { - DistributionMapping dm(hdr.m_ba); - mf.define(hdr.m_ba, dm, hdr.m_ncomp, hdr.m_ngrow, MFInfo(), FArrayBoxFactory()); + DistributionMapping dm(hdr.m_ba); + mf.define(hdr.m_ba, dm, hdr.m_ncomp, hdr.m_ngrow, MFInfo(), FArrayBoxFactory()); } else { - BL_ASSERT(amrex::match(hdr.m_ba,mf.boxArray())); + BL_ASSERT(amrex::match(hdr.m_ba,mf.boxArray())); } #ifdef BL_USE_MPI @@ -1567,7 +1567,7 @@ VisMF::Read (FabArray &mf, Vector &frc = frcIter->second; // ---- sort by offset std::sort(frc.begin(), frc.end(), [] (const FabReadLink &a, const FabReadLink &b) - { return a.fileOffset < b.fileOffset; } ); + { return a.fileOffset < b.fileOffset; } ); Vector nBoxesPerRank(nRanksPerFile[currentFileIndex]); amrex::NItemsPerBin(frc.size(), nBoxesPerRank); @@ -1576,14 +1576,14 @@ VisMF::Read (FabArray &mf, for(int nbpr(0); nbpr < nBoxesPerRank.size(); ++nbpr) { for(int nb(0); nb < nBoxesPerRank[nbpr]; ++nb) { - baFileOrder.set(indexFileOrder, frc[frcIndex].box); - ranksFileOrder[indexFileOrder] = currentRank; - frc[frcIndex].rankToRead = currentRank; - frc[frcIndex].faIndex = indexFileOrder; - readFileRanks[fileName].insert(currentRank); + baFileOrder.set(indexFileOrder, frc[frcIndex].box); + ranksFileOrder[indexFileOrder] = currentRank; + frc[frcIndex].rankToRead = currentRank; + frc[frcIndex].faIndex = indexFileOrder; + readFileRanks[fileName].insert(currentRank); - ++frcIndex; - ++indexFileOrder; + ++frcIndex; + ++indexFileOrder; } ++currentRank; currentRank = std::min(currentRank, nProcs - 1); @@ -1625,11 +1625,11 @@ VisMF::Read (FabArray &mf, int sIndex(0), sCount(0); for(setIter = rfrSplitSet.begin(); setIter != rfrSplitSet.end(); ++setIter) { streamSets[sIndex].insert(*setIter); - if(++sCount >= ranksPerStream) { - sCount = 0; - ++sIndex; - sIndex = std::min(sIndex, streamSets.size() - 1); - } + if(++sCount >= ranksPerStream) { + sCount = 0; + ++sIndex; + sIndex = std::min(sIndex, streamSets.size() - 1); + } } for(int iSet(0); iSet < streamSets.size(); ++iSet) { @@ -1641,87 +1641,87 @@ VisMF::Read (FabArray &mf, if(rfrSet.find(myProc) != rfrSet.end()) { // ---- myProc needs to read this file const std::string &fileName = rfrIter->first; - std::string fullFileName(VisMF::DirName(mf_name) + fileName); - frcIter = FileReadChains.find(fileName); - BL_ASSERT(frcIter != FileReadChains.end()); + std::string fullFileName(VisMF::DirName(mf_name) + fileName); + frcIter = FileReadChains.find(fileName); + BL_ASSERT(frcIter != FileReadChains.end()); Vector &frc = frcIter->second; for(NFilesIter nfi(fullFileName, readRanks); nfi.ReadyToRead(); ++nfi) { - // ---- confirm the data is contiguous in the stream - Long firstOffset(-1); - for(int i(0); i < frc.size(); ++i) { - if(myProc == frc[i].rankToRead) { - firstOffset = frc[i].fileOffset; - break; - } - } - - bool dataIsContiguous(true); - Long currentOffset(firstOffset), bytesToRead(0); - int nFABs(0); - - for(int i(0); i < frc.size(); ++i) { - if(myProc == frc[i].rankToRead) { - if(currentOffset != frc[i].fileOffset) { + // ---- confirm the data is contiguous in the stream + Long firstOffset(-1); + for(int i(0); i < frc.size(); ++i) { + if(myProc == frc[i].rankToRead) { + firstOffset = frc[i].fileOffset; + break; + } + } + + bool dataIsContiguous(true); + Long currentOffset(firstOffset), bytesToRead(0); + int nFABs(0); + + for(int i(0); i < frc.size(); ++i) { + if(myProc == frc[i].rankToRead) { + if(currentOffset != frc[i].fileOffset) { dataIsContiguous = false; - } else { - FArrayBox &fab = whichFA[frc[i].faIndex]; - Long fabBytesToRead(fab.box().numPts() * fab.nComp() * hdr.m_writtenRD.numBytes()); + } else { + FArrayBox &fab = whichFA[frc[i].faIndex]; + Long fabBytesToRead(fab.box().numPts() * fab.nComp() * hdr.m_writtenRD.numBytes()); currentOffset += fabBytesToRead; bytesToRead += fabBytesToRead; - ++nFABs; - } - } - } - char *allFabData; - bool canCombineFABs(false); - if(nFABs > 1 && dataIsContiguous && VisMF::useSingleRead) { - allFabData = new(std::nothrow) char[bytesToRead]; - if(allFabData == nullptr) { - canCombineFABs = false; - } else { - canCombineFABs = true; - } - } - if(canCombineFABs) { + ++nFABs; + } + } + } + char *allFabData; + bool canCombineFABs(false); + if(nFABs > 1 && dataIsContiguous && VisMF::useSingleRead) { + allFabData = new(std::nothrow) char[bytesToRead]; + if(allFabData == nullptr) { + canCombineFABs = false; + } else { + canCombineFABs = true; + } + } + if(canCombineFABs) { nfi.Stream().seekp(firstOffset, std::ios::beg); nfi.Stream().read(allFabData, bytesToRead); - currentOffset = 0; // ---- this is now relative to allFabData - - for(int i(0); i < frc.size(); ++i) { - if(myProc == frc[i].rankToRead) { - char *afPtr = allFabData + currentOffset; - FArrayBox &fab = whichFA[frc[i].faIndex]; - Long readDataItems(fab.box().numPts() * fab.nComp()); - if(doConvert) { - RealDescriptor::convertToNativeFormat(fab.dataPtr(), readDataItems, - afPtr, hdr.m_writtenRD); - } else { + currentOffset = 0; // ---- this is now relative to allFabData + + for(int i(0); i < frc.size(); ++i) { + if(myProc == frc[i].rankToRead) { + char *afPtr = allFabData + currentOffset; + FArrayBox &fab = whichFA[frc[i].faIndex]; + Long readDataItems(fab.box().numPts() * fab.nComp()); + if(doConvert) { + RealDescriptor::convertToNativeFormat(fab.dataPtr(), readDataItems, + afPtr, hdr.m_writtenRD); + } else { memcpy(fab.dataPtr(), afPtr, fab.nBytes()); - } + } currentOffset += readDataItems * hdr.m_writtenRD.numBytes(); - } - } - delete [] allFabData; - - } else { // ---- cannot use one read - for(int i(0); i < frc.size(); ++i) { - if(myProc == frc[i].rankToRead) { - if(static_cast(nfi.SeekPos()) != frc[i].fileOffset) { + } + } + delete [] allFabData; + + } else { // ---- cannot use one read + for(int i(0); i < frc.size(); ++i) { + if(myProc == frc[i].rankToRead) { + if(static_cast(nfi.SeekPos()) != frc[i].fileOffset) { nfi.Stream().seekp(frc[i].fileOffset, std::ios::beg); - } - FArrayBox &fab = whichFA[frc[i].faIndex]; - Long readDataItems(fab.box().numPts() * fab.nComp()); - if(doConvert) { - RealDescriptor::convertToNativeFormat(fab.dataPtr(), readDataItems, - nfi.Stream(), hdr.m_writtenRD); - } else { + } + FArrayBox &fab = whichFA[frc[i].faIndex]; + Long readDataItems(fab.box().numPts() * fab.nComp()); + if(doConvert) { + RealDescriptor::convertToNativeFormat(fab.dataPtr(), readDataItems, + nfi.Stream(), hdr.m_writtenRD); + } else { nfi.Stream().read((char *) fab.dataPtr(), fab.nBytes()); - } - } - } - } + } + } + } + } } // ---- end NFilesIter } @@ -1757,10 +1757,10 @@ VisMF::Read (FabArray &mf, } if(myProc == coordinatorProc) { std::string fname(hdr.m_fod[i].m_name); - if(fileNames.insert(std::pair(fname,allReadsIndex)).second) - { - ++allReadsIndex; - } + if(fileNames.insert(std::pair(fname,allReadsIndex)).second) + { + ++allReadsIndex; + } } } @@ -1779,14 +1779,14 @@ VisMF::Read (FabArray &mf, whichProc = mf.DistributionMap()[i]; iSeekPos = hdr.m_fod[i].m_head; std::string fname(hdr.m_fod[i].m_name); - fileNamesIter = fileNames.find(fname); - if(fileNamesIter != fileNames.end()) { - int findex(fileNames.find(fname)->second); - allReads[findex][whichProc].insert(std::pair(iSeekPos, i)); - } else { + fileNamesIter = fileNames.find(fname); + if(fileNamesIter != fileNames.end()) { + int findex(fileNames.find(fname)->second); + allReads[findex][whichProc].insert(std::pair(iSeekPos, i)); + } else { amrex::ErrorStream() << "**** Error: filename not found = " << fname << std::endl; amrex::Abort("**** Error in VisMF::Read"); - } + } } } @@ -1799,82 +1799,82 @@ VisMF::Read (FabArray &mf, MPI_Status status; int doneFlag; while(totalIOReqs > 0) { - std::vector vReads; + std::vector vReads; std::multiset::iterator aFilesIter; aFilesIter = availableFiles.begin(); while(aFilesIter != availableFiles.end()) { // handle available files - int arIndex(*aFilesIter); - if(allReads[arIndex].empty()) { + int arIndex(*aFilesIter); + if(allReads[arIndex].empty()) { availableFiles.erase(arIndex); aFilesIter = availableFiles.begin(); - continue; - } + continue; + } std::map >::iterator whichRead; - for(whichRead = allReads[arIndex].begin(); - whichRead != allReads[arIndex].end(); ++whichRead) - { - int tryProc(whichRead->first); - if(busyProcs.find(tryProc) == busyProcs.end()) { // tryProc not busy - busyProcs.insert(tryProc); - int nReads(whichRead->second.size()); - int ir(0); - vReads.resize(nReads); + for(whichRead = allReads[arIndex].begin(); + whichRead != allReads[arIndex].end(); ++whichRead) + { + int tryProc(whichRead->first); + if(busyProcs.find(tryProc) == busyProcs.end()) { // tryProc not busy + busyProcs.insert(tryProc); + int nReads(whichRead->second.size()); + int ir(0); + vReads.resize(nReads); std::map::iterator imiter; - for(imiter = whichRead->second.begin(); - imiter != whichRead->second.end(); ++imiter) - { - vReads[ir] = imiter->second; // the mfindex - ++ir; - } - if(tryProc == ioProcNum) { - iopFileIndex = arIndex; - for(int irr(0); irr < nReads; ++irr) { - iopReads.push_back(vReads[irr]); - } - } else { - ParallelDescriptor::Send(vReads, tryProc, readTag); - ++messTotal; - ++reqsPending; - } + for(imiter = whichRead->second.begin(); + imiter != whichRead->second.end(); ++imiter) + { + vReads[ir] = imiter->second; // the mfindex + ++ir; + } + if(tryProc == ioProcNum) { + iopFileIndex = arIndex; + for(int irr(0); irr < nReads; ++irr) { + iopReads.push_back(vReads[irr]); + } + } else { + ParallelDescriptor::Send(vReads, tryProc, readTag); + ++messTotal; + ++reqsPending; + } availableFiles.erase(aFilesIter); aFilesIter = availableFiles.begin(); - break; - } - } - if(whichRead == allReads[arIndex].end()) { - ++aFilesIter; - } else { - allReads[arIndex].erase(whichRead); - } + break; + } + } + if(whichRead == allReads[arIndex].end()) { + ++aFilesIter; + } else { + allReads[arIndex].erase(whichRead); + } } // end while(aFilesIter...) - while( ! iopReads.empty()) { - int index(iopReads.front()); - VisMF::readFAB(mf,index, mf_name, hdr); - --totalIOReqs; - iopReads.pop_front(); - if(iopReads.empty()) { - availableFiles.insert(iopFileIndex); - busyProcs.erase(ioProcNum); - } - ParallelDescriptor::IProbe(MPI_ANY_SOURCE, doneTag, doneFlag, status); - if(doneFlag) { - break; - } - } - - if(reqsPending > 0) { + while( ! iopReads.empty()) { + int index(iopReads.front()); + VisMF::readFAB(mf,index, mf_name, hdr); + --totalIOReqs; + iopReads.pop_front(); + if(iopReads.empty()) { + availableFiles.insert(iopFileIndex); + busyProcs.erase(ioProcNum); + } + ParallelDescriptor::IProbe(MPI_ANY_SOURCE, doneTag, doneFlag, status); + if(doneFlag) { + break; + } + } + + if(reqsPending > 0) { rmess = ParallelDescriptor::Recv(iDone, MPI_ANY_SOURCE, doneTag); - int index(iDone[iDoneIndex]); - int procDone(rmess.pid()); - totalIOReqs -= iDone[iDoneCount]; - --reqsPending; - busyProcs.erase(procDone); + int index(iDone[iDoneIndex]); + int procDone(rmess.pid()); + totalIOReqs -= iDone[iDoneCount]; + --reqsPending; + busyProcs.erase(procDone); std::string fname(hdr.m_fod[index].m_name); - int fileIndex(fileNames.find(fname)->second); - availableFiles.insert(fileIndex); - } + int fileIndex(fileNames.find(fname)->second); + availableFiles.insert(fileIndex); + } } // end while(totalIOReqs...) @@ -1883,12 +1883,12 @@ VisMF::Read (FabArray &mf, while(nReqs > 0) { rmess = ParallelDescriptor::Recv(recReads, ioProcNum, readTag); for(int ir(0); ir < static_cast(rmess.count()); ++ir) { - int mfIndex(recReads[ir]); - VisMF::readFAB(mf,mfIndex, mf_name, hdr); - } + int mfIndex(recReads[ir]); + VisMF::readFAB(mf,mfIndex, mf_name, hdr); + } nReqs -= rmess.count(); - iDone[iDoneIndex] = recReads[0]; - iDone[iDoneCount] = rmess.count(); + iDone[iDoneIndex] = recReads[0]; + iDone[iDoneCount] = rmess.count(); ParallelDescriptor::Send(iDone, ioProcNum, doneTag); } } @@ -1942,7 +1942,7 @@ VisMF::Exist (const std::string& mf_name) void VisMF::ReadFAHeader (const std::string &fafabName, - Vector &faHeader) + Vector &faHeader) { // BL_PROFILE("VisMF::ReadFAHeader()"); @@ -2024,7 +2024,7 @@ VisMF::Check (const std::string& mf_name) badFab = true; } if(badFab) { - ++nBadFabs; + ++nBadFabs; if (verbose) { amrex::AllPrint() << "**** Error in file: " << FullName << " Bad Fab at index = " << i << " seekpos = " << fod.m_head << " box = " << hdr.m_ba[i] diff --git a/Src/Base/AMReX_bc_types_mod.F90 b/Src/Base/AMReX_bc_types_mod.F90 index 1a044898404..c326d49e419 100644 --- a/Src/Base/AMReX_bc_types_mod.F90 +++ b/Src/Base/AMReX_bc_types_mod.F90 @@ -4,7 +4,7 @@ module amrex_bc_types_module implicit none ! include 'AMReX_bc_types.fi' - + private integer, parameter, public :: amrex_bc_bogus = -666 diff --git a/Src/Base/AMReX_ccse-mpi.H b/Src/Base/AMReX_ccse-mpi.H index 5e27cf9ae0e..41a22bfbd73 100644 --- a/Src/Base/AMReX_ccse-mpi.H +++ b/Src/Base/AMReX_ccse-mpi.H @@ -69,7 +69,7 @@ namespace ParallelDescriptor //! Communication datatype (note: this structure also works without MPI) struct Mpi_typemap { - static MPI_Datatype type(); + static MPI_Datatype type(); }; } } diff --git a/Src/Base/AMReX_error_fi.cpp b/Src/Base/AMReX_error_fi.cpp index bb533fa3a9d..29ba25d4e5c 100644 --- a/Src/Base/AMReX_error_fi.cpp +++ b/Src/Base/AMReX_error_fi.cpp @@ -5,16 +5,16 @@ extern "C" { void amrex_fi_error (const char* message) { - amrex::Error(message); + amrex::Error(message); } void amrex_fi_abort (const char* message) { - amrex::Abort(message); + amrex::Abort(message); } void amrex_fi_warning (const char* message) { - amrex::Warning(message); + amrex::Warning(message); } } diff --git a/Src/Base/AMReX_error_mod.F90 b/Src/Base/AMReX_error_mod.F90 index e4dcd02bd81..5f2f62e9fcf 100644 --- a/Src/Base/AMReX_error_mod.F90 +++ b/Src/Base/AMReX_error_mod.F90 @@ -3,7 +3,7 @@ module amrex_error_module use iso_c_binding use amrex_string_module - + implicit none interface amrex_error @@ -26,7 +26,7 @@ end subroutine amrex_fi_error subroutine amrex_fi_abort (message) bind(c) import character(kind=c_char), intent(in) :: message(*) - end subroutine amrex_fi_abort + end subroutine amrex_fi_abort subroutine amrex_fi_warning (message) bind(c) import diff --git a/Src/Base/AMReX_filcc_mod.F90 b/Src/Base/AMReX_filcc_mod.F90 index fa09a908ec4..f46bdb70b2f 100644 --- a/Src/Base/AMReX_filcc_mod.F90 +++ b/Src/Base/AMReX_filcc_mod.F90 @@ -169,7 +169,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) ! Do nothing. else if (bc(1,1,n) .eq. amrex_bc_foextrap) then - + do k = lo(3), hi(3) do j = lo(2), hi(2) do i = imin, imax @@ -177,7 +177,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(1,1,n) .eq. amrex_bc_hoextrap) then do k = lo(3), hi(3) @@ -206,7 +206,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) q(i,j,k,n) = 2*q(ilo,j,k,n) - q(ilo+1,j,k,n) end do end do - end do + end do else if (bc(1,1,n) .eq. amrex_bc_reflect_even) then @@ -227,7 +227,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + end if end if @@ -249,7 +249,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(1,2,n) .eq. amrex_bc_hoextrap) then do k = lo(3), hi(3) @@ -289,7 +289,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(1,2,n) .eq. amrex_bc_reflect_odd) then do k = lo(3), hi(3) @@ -299,7 +299,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + end if end if @@ -323,7 +323,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(2,1,n) .eq. amrex_bc_hoextrap) then do k = lo(3), hi(3) @@ -373,7 +373,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + end if end if @@ -395,7 +395,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(2,2,n) .eq. amrex_bc_hoextrap) then do k = lo(3), hi(3) @@ -411,7 +411,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) q(i,j,k,n) = half * (3*q(i,jhi,k,n) - q(i,jhi-1,k,n)) end if end if - + end do end do end do @@ -421,7 +421,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) do k = lo(3), hi(3) do j = jmin, jmax do i = lo(1), hi(1) - q(i,j,k,n) = 2*q(i,jhi,k,n) - q(i,jhi-1,k,n) + q(i,j,k,n) = 2*q(i,jhi,k,n) - q(i,jhi-1,k,n) end do end do end do @@ -435,7 +435,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(2,2,n) .eq. amrex_bc_reflect_odd) then do k = lo(3), hi(3) @@ -445,7 +445,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + end if end if @@ -463,9 +463,9 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) if (bc(3,1,n) .eq. amrex_bc_ext_dir) then ! Do nothing. - + else if (bc(3,1,n) .eq. amrex_bc_foextrap) then - + do k = kmin, kmax do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -493,9 +493,9 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(3,1,n) .eq. amrex_bc_hoextrapcc) then - + do k = kmin, kmax do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -513,7 +513,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(3,1,n) .eq. amrex_bc_reflect_odd) then do k = kmin, kmax @@ -523,7 +523,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + end if end if @@ -535,9 +535,9 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) if (bc(3,2,n) .eq. amrex_bc_ext_dir) then ! Do nothing. - + else if (bc(3,2,n) .eq. amrex_bc_foextrap) then - + do k = kmin, kmax do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -565,9 +565,9 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(3,2,n) .eq. amrex_bc_hoextrapcc) then - + do k = kmin, kmax do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -585,7 +585,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end do end do end do - + else if (bc(3,2,n) .eq. amrex_bc_reflect_odd) then do k = kmin, kmax @@ -624,22 +624,22 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) if (i.ge.imin .and. i.le.imax .and. j.ge.jmin .and. j.le.jmax) then do k = lo(3), hi(3) - + if (jlo+2 <= je) then q(i,j,k,n) = half * eighth * (15*q(ilo-1,jlo,k,n) - 10*q(ilo-1,jlo+1,k,n) + 3*q(ilo-1,jlo+2,k,n)) else q(i,j,k,n) = half * half * (3*q(ilo-1,jlo,k,n) - q(ilo-1,jlo+1,k,n)) end if - + if (ilo+2 <= ie) then q(i,j,k,n) = q(ilo-1,jlo-1,k,n) + & half * eighth * (15*q(ilo,jlo-1,k,n) - 10*q(ilo+1,jlo-1,k,n) + 3*q(ilo+2,jlo-1,k,n)) else q(i,j,k,n) = q(ilo-1,jlo-1,k,n) + half * half * (3*q(ilo,jlo-1,k,n) - q(ilo+1,jlo-1,k,n)) end if - + #if AMREX_SPACEDIM == 3 - + if (k == klo-1 .and. bc(3,1,n) .eq. amrex_bc_hoextrap) then if (klo+2 <= ke) then q(i,j,k,n) = eighth * ( (15*q(ilo-1,jlo-1,klo,n) - 10*q(ilo-1,jlo-1,klo+1,n) + & @@ -648,7 +648,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) q(i,j,k,n) = half * (3*q(ilo-1,jlo-1,klo,n) - q(ilo-1,jlo-1,klo+1,n)) end if end if - + if (k == khi+1 .and. bc(3,2,n) .eq. amrex_bc_hoextrap) then if (khi-2 >= ks) then q(i,j,k,n) = eighth * ( (15*q(ilo-1,jlo-1,khi,n) - 10*q(ilo-1,jlo-1,khi-1,n) + & @@ -658,7 +658,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end if end if #endif - + end do end if end if @@ -688,7 +688,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(ilo-1,jhi,k,n) - q(ilo-1,jhi-1,k,n)) end if - + if (ilo+2 <= ie) then q(i,j,k,n) = q(ilo-1,jhi+1,k,n) + & half * eighth * (15*q(ilo,jhi+1,k,n) - 10*q(ilo+1,jhi+1,k,n) + 3*q(ilo+2,jhi+1,k,n)) @@ -705,7 +705,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) q(i,j,k,n) = half * (3*q(ilo-1,jhi+1,klo,n) - q(ilo-1,jhi+1,klo+1,n)) end if end if - + if (k == khi+1 .and. bc(3,2,n) .eq. amrex_bc_hoextrap) then if (khi-2 >= ks) then q(i,j,k,n) = eighth * ( (15*q(ilo-1,jhi+1,khi,n) - 10*q(ilo-1,jhi+1,khi-1,n) + & @@ -746,14 +746,14 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(ihi+1,jlo,k,n) - q(ihi+1,jlo+1,k,n)) end if - + if (ihi-2 >= is) then q(i,j,k,n) = q(ihi+1,jlo-1,k,n) + & half * eighth * (15*q(ihi,jlo-1,k,n) - 10*q(ihi-1,jlo-1,k,n) + 3*q(ihi-2,jlo-1,k,n)) else q(i,j,k,n) = q(ihi+1,jlo-1,k,n) + half * half * (3*q(ihi,jlo-1,k,n) - q(ihi-1,jlo-1,k,n)) end if - + #if (AMREX_SPACEDIM == 3) if (k == klo-1 .and. bc(3,1,n) .eq. amrex_bc_hoextrap) then if (klo+2 <= ke) then @@ -762,7 +762,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) q(i,j,k,n) = half * (3*q(ihi+1,jlo-1,klo,n) - q(ihi+1,jlo-1,klo+1,n)) end if end if - + if (k == khi+1 .and. bc(3,2,n) .eq. amrex_bc_hoextrap) then if (khi-2 >= ks) then q(i,j,k,n) = eighth * (15*q(ihi+1,jlo-1,khi,n) - 10*q(ihi+1,jlo-1,khi-1,n) + 3*q(ihi+1,jlo-1,khi-2,n)) @@ -795,20 +795,20 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) if (i.ge.imin .and. i.le.imax .and. j.ge.jmin .and. j.le.jmax) then do k = lo(3), hi(3) - + if (jhi-2 >= js) then q(i,j,k,n) = half * eighth * (15*q(ihi+1,jhi,k,n) - 10*q(ihi+1,jhi-1,k,n) + 3*q(ihi+1,jhi-2,k,n)) else q(i,j,k,n) = half * half * (3*q(ihi+1,jhi,k,n) - q(ihi+1,jhi-1,k,n)) end if - + if (ihi-2 >= is) then q(i,j,k,n) = q(ihi+1,jhi+1,k,n) + & half * eighth * (15*q(ihi,jhi+1,k,n) - 10*q(ihi-1,jhi+1,k,n) + 3*q(ihi-2,jhi+1,k,n)) else q(i,j,k,n) = q(ihi+1,jhi+1,k,n) + half * half * (3*q(ihi,jhi+1,k,n) - q(ihi-1,jhi+1,k,n)) end if - + #if (AMREX_SPACEDIM == 3) if (k == klo-1 .and. bc(3,1,n) .eq. amrex_bc_hoextrap) then if (klo+2 <= ke) then @@ -817,7 +817,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) q(i,j,k,n) = half * (3*q(ihi+1,jhi+1,klo,n) - q(ihi+1,jhi+1,klo+1,n)) end if end if - + if (k == khi+1 .and. bc(3,2,n) .eq. amrex_bc_hoextrap) then if (khi-2 >= ks) then q(i,j,k,n) = eighth * (15*q(ihi+1,jhi+1,khi,n) - 10*q(ihi+1,jhi+1,khi-1,n) + 3*q(ihi+1,jhi+1,khi-2,n)) @@ -826,7 +826,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) end if end if #endif - + end do end if end if @@ -850,7 +850,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) k = klo-1 if (i.ge.imin .and. i.le.imax .and. k.ge.kmin .and. k.le.kmax) then - + do j = lo(2), hi(2) if (klo+2 <= ke) then @@ -858,7 +858,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(ilo-1,j,klo,n) - q(ilo-1,j,klo+1,n)) end if - + if (ilo+2 <= ie) then q(i,j,k,n) = q(ilo-1,j,klo-1,n) + & half * eighth * (15*q(ilo,j,klo-1,n) - 10*q(ilo+1,j,klo-1,n) + 3*q(ilo+2,j,klo-1,n)) @@ -895,7 +895,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(ilo-1,j,khi,n) - q(ilo-1,j,khi-1,n)) end if - + if (ilo+2 <= ie) then q(i,j,k,n) = q(ilo-1,j,khi+1,n) + & half * eighth * (15*q(ilo,j,khi+1,n) - 10*q(ilo+1,j,khi+1,n) + 3*q(ilo+2,j,khi+1,n)) @@ -931,7 +931,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(ihi+1,j,klo,n) - q(ihi+1,j,klo+1,n)) end if - + if (ihi-2 >= is) then q(i,j,k,n) = q(ihi+1,j,klo-1,n) + & half * eighth * (15*q(ihi,j,klo-1,n) - 10*q(ihi-1,j,klo-1,n) + 3*q(ihi-2,j,klo-1,n)) @@ -948,7 +948,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) ! if (bc(1,2,n) .eq. amrex_bc_hoextrap .and. bc(3,2,n) .eq. amrex_bc_hoextrap) then - + if (hi(1) > ihi .and. hi(3) > khi) then imin = max(lo(1),ihi+1) imax = hi(1) @@ -967,14 +967,14 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(ihi+1,j,khi,n) - q(ihi+1,j,khi-1,n)) end if - + if (ihi-2 >= is) then q(i,j,k,n) = q(ihi+1,j,khi+1,n) + & half * eighth * (15*q(ihi,j,khi+1,n) - 10*q(ihi-1,j,khi+1,n) + 3*q(ihi-2,j,khi+1,n)) else q(i,j,k,n) = q(ihi+1,j,khi+1,n) + half * half * (3*q(ihi,j,khi+1,n) - q(ihi-1,j,khi+1,n)) end if - + end do end if end if @@ -1004,7 +1004,7 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(i,jlo-1,klo,n) - q(i,jlo-1,klo+1,n)) end if - + if (jlo+2 <= je) then q(i,j,k,n) = q(i,jlo-1,klo-1,n) + & half * eighth * (15*q(i,jlo,klo-1,n) - 10*q(i,jlo+1,klo-1,n) + 3*q(i,jlo+2,klo-1,n)) @@ -1041,14 +1041,14 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(i,jlo-1,khi,n) - q(i,jlo-1,khi-1,n)) end if - + if (jlo+2 <= je) then q(i,j,k,n) = q(i,jlo-1,khi+1,n) + & half * eighth * (15*q(i,jlo,khi+1,n) - 10*q(i,jlo+1,khi+1,n) + 3*q(i,jlo+2,khi+1,n)) else q(i,j,k,n) = q(i,jlo-1,khi+1,n) + half * half * (3*q(i,jlo,khi+1,n) - q(i,jlo+1,khi+1,n)) end if - + end do end if end if @@ -1078,14 +1078,14 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(i,jhi+1,klo,n) - q(i,jhi+1,klo+1,n)) end if - + if (jhi-2 >= js) then q(i,j,k,n) = q(i,jhi+1,klo-1,n) + & half * eighth * (15*q(i,jhi,klo-1,n) - 10*q(i,jhi-1,klo-1,n) + 3*q(i,jhi-2,klo-1,n)) else q(i,j,k,n) = q(i,jhi+1,klo-1,n) + half * half * (3*q(i,jhi,klo-1,n) - q(i,jhi-1,klo-1,n)) end if - + end do end if end if @@ -1115,14 +1115,14 @@ subroutine amrex_filccn(lo, hi, q, q_lo, q_hi, ncomp, domlo, domhi, dx, xlo, bc) else q(i,j,k,n) = half * half * (3*q(i,jhi+1,khi,n) - q(i,jhi+1,khi-1,n)) end if - + if (jhi-2 >= js) then q(i,j,k,n) = q(i,jhi+1,khi+1,n) + & half * eighth * (15*q(i,jhi,khi+1,n) - 10*q(i,jhi-1,khi+1,n) + 3*q(i,jhi-2,khi+1,n)) else q(i,j,k,n) = q(i,jhi+1,khi+1,n) + half * half * (3*q(i,jhi,khi+1,n) - q(i,jhi-1,khi+1,n)) end if - + end do end if end if @@ -1199,7 +1199,7 @@ subroutine amrex_hoextraptocc_3d(q,q_l1,q_l2,q_l3,q_h1,q_h2,q_h3,domlo,domhi,dx, q(ilo+2,j,k) end do end do - else + else do k = q_l3,q_h3 do j = q_l2,q_h2 q(ilo-1,j,k) = 2*q(ilo,j,k) - q(ilo+1,j,k) @@ -1394,7 +1394,7 @@ subroutine amrex_hoextraptocc_3d(q,q_l1,q_l2,q_l3,q_h1,q_h2,q_h3,domlo,domhi,dx, ! **************************************************************************** ! if ((nlft .gt. 0) .and. (ntop .gt. 0)) then - if (jhi-2 .ge. js) then + if (jhi-2 .ge. js) then do k = q_l3,q_h3 q(ilo-1,jhi+1,k) = half * & (3*q(ilo-1,jhi,k) - 3*q(ilo-1,jhi-1,k) + & @@ -1407,7 +1407,7 @@ subroutine amrex_hoextraptocc_3d(q,q_l1,q_l2,q_l3,q_h1,q_h2,q_h3,domlo,domhi,dx, end do end if - if (ilo+2 .le. ie) then + if (ilo+2 .le. ie) then do k = q_l3,q_h3 q(ilo-1,jhi+1,k) = q(ilo-1,jhi+1,k) + half * & (3*q(ilo,jhi+1,k) - 3*q(ilo+1,jhi+1,k) + & @@ -1447,7 +1447,7 @@ subroutine amrex_hoextraptocc_3d(q,q_l1,q_l2,q_l3,q_h1,q_h2,q_h3,domlo,domhi,dx, ! **************************************************************************** ! if ((nrgt .gt. 0) .and. (nbot .gt. 0)) then - if (jlo+2 .le. je) then + if (jlo+2 .le. je) then do k = q_l3,q_h3 q(ihi+1,jlo-1,k) = half * & (3*q(ihi+1,jlo,k) - 3*q(ihi+1,jlo+1,k) + & @@ -1460,7 +1460,7 @@ subroutine amrex_hoextraptocc_3d(q,q_l1,q_l2,q_l3,q_h1,q_h2,q_h3,domlo,domhi,dx, end do end if - if (ihi-2 .ge. is) then + if (ihi-2 .ge. is) then do k = q_l3,q_h3 q(ihi+1,jlo-1,k) = q(ihi+1,jlo-1,k) + half * & (3*q(ihi,jlo-1,k) - 3*q(ihi-1,jlo-1,k) + & @@ -1832,14 +1832,14 @@ subroutine amrex_hoextraptocc_2d(q,q_l1,q_l2,q_h1,q_h2,domlo,domhi,dx,xlo) ilo = domlo(1) do i = 2, nlft do j = q_l2, q_h2 - q(ilo-i,j) = q(ilo,j) + q(ilo-i,j) = q(ilo,j) end do end do - if (ilo+2 .le. ie) then + if (ilo+2 .le. ie) then do j = q_l2, q_h2 q(ilo-1,j) = 3*q(ilo,j) - 3*q(ilo+1,j) + q(ilo+2,j) end do - else + else do j = q_l2, q_h2 q(ilo-1,j) = 2*q(ilo,j) - q(ilo+1,j) end do @@ -1908,24 +1908,24 @@ subroutine amrex_hoextraptocc_2d(q,q_l1,q_l2,q_h1,q_h2,domlo,domhi,dx,xlo) q(ilo-1,jlo-1) = half * (2*q(ilo-1,jlo) - q(ilo-1,jlo+1)) end if - if (ilo+2 .le. ie) then + if (ilo+2 .le. ie) then q(ilo-1,jlo-1) = q(ilo-1,jlo-1) + half * & - (3*q(ilo,jlo-1) - 3*q(ilo+1,jlo-1) + q(ilo+2,jlo-1)) + (3*q(ilo,jlo-1) - 3*q(ilo+1,jlo-1) + q(ilo+2,jlo-1)) else q(ilo-1,jlo-1) = q(ilo-1,jlo-1) + half * & (2*q(ilo,jlo-1) - q(ilo+1,jlo-1)) end if end if - if ((nlft .gt. 0) .and. (ntop .gt. 0)) then - if (jhi-2 .ge. js) then + if ((nlft .gt. 0) .and. (ntop .gt. 0)) then + if (jhi-2 .ge. js) then q(ilo-1,jhi+1) = half * & (3*q(ilo-1,jhi) - 3*q(ilo-1,jhi-1) + q(ilo-1,jhi-2)) else q(ilo-1,jhi+1) = half * (2*q(ilo-1,jhi) - q(ilo-1,jhi-1)) end if - if (ilo+2 .le. ie) then + if (ilo+2 .le. ie) then q(ilo-1,jhi+1) = q(ilo-1,jhi+1) + half * & (3*q(ilo,jhi+1) - 3*q(ilo+1,jhi+1) + q(ilo+2,jhi+1)) else @@ -1934,15 +1934,15 @@ subroutine amrex_hoextraptocc_2d(q,q_l1,q_l2,q_h1,q_h2,domlo,domhi,dx,xlo) end if end if - if ((nrgt .gt. 0) .and. (nbot .gt. 0)) then - if (jlo+2 .le. je) then + if ((nrgt .gt. 0) .and. (nbot .gt. 0)) then + if (jlo+2 .le. je) then q(ihi+1,jlo-1) = half * & (3*q(ihi+1,jlo) - 3*q(ihi+1,jlo+1) + q(ihi+1,jlo+2)) else q(ihi+1,jlo-1) = half * (2*q(ihi+1,jlo) - q(ihi+1,jlo+1)) end if - if (ihi-2 .ge. is) then + if (ihi-2 .ge. is) then q(ihi+1,jlo-1) = q(ihi+1,jlo-1) + half * & (3*q(ihi,jlo-1) - 3*q(ihi-1,jlo-1) + q(ihi-2,jlo-1)) else @@ -1951,15 +1951,15 @@ subroutine amrex_hoextraptocc_2d(q,q_l1,q_l2,q_h1,q_h2,domlo,domhi,dx,xlo) end if end if - if ((nrgt .gt. 0) .and. (ntop .gt. 0)) then - if (jhi-2 .ge. js) then + if ((nrgt .gt. 0) .and. (ntop .gt. 0)) then + if (jhi-2 .ge. js) then q(ihi+1,jhi+1) = half * & (3*q(ihi+1,jhi) - 3*q(ihi+1,jhi-1) + q(ihi+1,jhi-2)) else q(ihi+1,jhi+1) = half * (2*q(ihi+1,jhi) - q(ihi+1,jhi-1)) end if - if (ihi-2 .ge. is) then + if (ihi-2 .ge. is) then q(ihi+1,jhi+1) = q(ihi+1,jhi+1) + half * & (3*q(ihi,jhi+1) - 3*q(ihi-1,jhi+1) + q(ihi-2,jhi+1)) else diff --git a/Src/Base/AMReX_iMultiFab.H b/Src/Base/AMReX_iMultiFab.H index 1bbcf783b4a..f73e75c5fc0 100644 --- a/Src/Base/AMReX_iMultiFab.H +++ b/Src/Base/AMReX_iMultiFab.H @@ -43,9 +43,9 @@ public: * a region of definition defined by the grow factor ngrow. */ iMultiFab (const BoxArray& bs, - const DistributionMapping& dm, - int ncomp, - int ngrow, + const DistributionMapping& dm, + int ncomp, + int ngrow, #ifdef AMREX_STRICT_MODE const MFInfo& info, const FabFactory& factory); @@ -55,9 +55,9 @@ public: #endif iMultiFab (const BoxArray& bs, - const DistributionMapping& dm, - int ncomp, - const IntVect& ngrow, + const DistributionMapping& dm, + int ncomp, + const IntVect& ngrow, #ifdef AMREX_STRICT_MODE const MFInfo& info, const FabFactory& factory); @@ -101,7 +101,7 @@ public: */ int min (int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Identical to the previous min() function, but confines its @@ -115,7 +115,7 @@ public: int min (const Box& b, int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Returns the maximum value contained in component comp of the @@ -128,8 +128,8 @@ public: * \param local */ int max (int comp, - int nghost = 0, - bool local = false) const; + int nghost = 0, + bool local = false) const; /** * \brief Identical to the previous max() function, but confines its @@ -143,7 +143,7 @@ public: int max (const Box& b, int comp, int nghost = 0, - bool local = false) const; + bool local = false) const; /** * \brief Returns the sum in component comp @@ -391,9 +391,9 @@ public: * \param nghost */ void divide (const iMultiFab& mf, - int strt_comp, - int num_comp, - int nghost); + int strt_comp, + int num_comp, + int nghost); /** * \brief Add src to dst including nghost ghost cells. @@ -407,11 +407,11 @@ public: * \param nghost */ static void Add (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); /** * \brief Copy from src to dst including nghost ghost cells. @@ -443,11 +443,11 @@ public: * \param nghost */ static void Subtract (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); /** * \brief Multiply dst by src including nghost ghost cells. @@ -461,11 +461,11 @@ public: * \param nghost */ static void Multiply (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); /** * \brief Divide dst by src including nghost ghost cells. @@ -479,33 +479,33 @@ public: * \param nghost */ static void Divide (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost); + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost); virtual void define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - const IntVect& ngrow, + const DistributionMapping& dm, + int nvar, + const IntVect& ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) override; #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = DefaultFabFactory()) override; #endif virtual void define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - int ngrow, + const DistributionMapping& dm, + int nvar, + int ngrow, #ifdef AMREX_STRICT_MODE - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) override; #else - const MFInfo& info = MFInfo(), + const MFInfo& info = MFInfo(), const FabFactory& factory = DefaultFabFactory()) override; #endif diff --git a/Src/Base/AMReX_iMultiFab.cpp b/Src/Base/AMReX_iMultiFab.cpp index 1e8801edd4d..d11aebb3c49 100644 --- a/Src/Base/AMReX_iMultiFab.cpp +++ b/Src/Base/AMReX_iMultiFab.cpp @@ -23,11 +23,11 @@ namespace void iMultiFab::Add (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost) + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -53,11 +53,11 @@ iMultiFab::Copy (iMultiFab& dst, void iMultiFab::Subtract (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost) + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -68,11 +68,11 @@ iMultiFab::Subtract (iMultiFab& dst, void iMultiFab::Multiply (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost) + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -83,11 +83,11 @@ iMultiFab::Multiply (iMultiFab& dst, void iMultiFab::Divide (iMultiFab& dst, - const iMultiFab& src, - int srccomp, - int dstcomp, - int numcomp, - int nghost) + const iMultiFab& src, + int srccomp, + int dstcomp, + int numcomp, + int nghost) { BL_ASSERT(dst.boxArray() == src.boxArray()); BL_ASSERT(dst.distributionMap == src.distributionMap); @@ -154,7 +154,7 @@ iMultiFab::iMultiFab (const BoxArray& bxs, const DistributionMapping& dm, int ncomp, int ngrow, - const MFInfo& info, + const MFInfo& info, const FabFactory& factory) : iMultiFab(bxs,dm,ncomp,IntVect(ngrow),info,factory) { @@ -189,10 +189,10 @@ iMultiFab::operator= (int r) void iMultiFab::define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - const IntVect& ngrow, - const MFInfo& info, + const DistributionMapping& dm, + int nvar, + const IntVect& ngrow, + const MFInfo& info, const FabFactory& factory) { this->FabArray::define(bxs,dm,nvar,ngrow,info, factory); @@ -200,10 +200,10 @@ iMultiFab::define (const BoxArray& bxs, void iMultiFab::define (const BoxArray& bxs, - const DistributionMapping& dm, - int nvar, - int ngrow, - const MFInfo& info, + const DistributionMapping& dm, + int nvar, + int ngrow, + const MFInfo& info, const FabFactory& factory) { this->FabArray::define(bxs,dm,nvar,ngrow,info, factory); @@ -227,10 +227,10 @@ iMultiFab::min (int comp, int nghost, bool local) const r = amrex::min(r, fab(i,j,k,comp)); }); return r; - }); + }); if (!local) - ParallelDescriptor::ReduceIntMin(mn); + ParallelDescriptor::ReduceIntMin(mn); return mn; } @@ -257,15 +257,15 @@ iMultiFab::min (const Box& region, int comp, int nghost, bool local) const }); if (!local) - ParallelDescriptor::ReduceIntMin(mn); + ParallelDescriptor::ReduceIntMin(mn); return mn; } int iMultiFab::max (int comp, - int nghost, - bool local) const + int nghost, + bool local) const { BL_ASSERT(nghost >= 0 && nghost <= n_grow.min()); @@ -282,10 +282,10 @@ iMultiFab::max (int comp, r = amrex::max(r, fab(i,j,k,comp)); }); return r; - }); + }); if (!local) - ParallelDescriptor::ReduceIntMax(mx); + ParallelDescriptor::ReduceIntMax(mx); return mx; } @@ -312,7 +312,7 @@ iMultiFab::max (const Box& region, int comp, int nghost, bool local) const }); if (!local) - ParallelDescriptor::ReduceIntMax(mx); + ParallelDescriptor::ReduceIntMax(mx); return mx; } @@ -430,9 +430,9 @@ iMultiFab::minus (const iMultiFab& mf, void iMultiFab::divide (const iMultiFab& mf, - int strt_comp, - int num_comp, - int nghost) + int strt_comp, + int num_comp, + int nghost) { iMultiFab::Divide(*this, mf, strt_comp, strt_comp, num_comp, nghost); } @@ -552,7 +552,7 @@ OwnerMask (FabArrayBase const& mf, const Periodicity& period, const IntVect& ngr #endif { std::vector< std::pair > isects; - + for (MFIter mfi(*p); mfi.isValid(); ++mfi) { const Box& bx = (*p)[mfi].box(); @@ -571,7 +571,7 @@ OwnerMask (FabArrayBase const& mf, const Periodicity& period, const IntVect& ngr { const int oi = is.first; const Box& obx = is.second-iv; - if ((oi < idx) || (oi == idx && iv < IntVect::TheZeroVector())) + if ((oi < idx) || (oi == idx && iv < IntVect::TheZeroVector())) { if (run_on_gpu) { tags.push_back({arr,obx}); diff --git a/Src/Base/AMReX_mempool_mod.F90 b/Src/Base/AMReX_mempool_mod.F90 index 607ebe49f16..9ba299c9df0 100644 --- a/Src/Base/AMReX_mempool_mod.F90 +++ b/Src/Base/AMReX_mempool_mod.F90 @@ -68,13 +68,13 @@ module amrex_mempool_module module procedure bl_deallocate_i3 end interface - interface + interface function amrex_mempool_alloc (nbytes) result(p) bind(c) use, intrinsic :: iso_c_binding type(c_ptr) :: p integer(kind=c_size_t), intent(in), value :: nbytes end function amrex_mempool_alloc - + subroutine amrex_mempool_free (p) bind(c) use, intrinsic :: iso_c_binding type(c_ptr), value :: p @@ -124,7 +124,7 @@ subroutine bl_allocate_r2(a, lo1, hi1, lo2, hi2) real(c_real), pointer :: fp(:,:) n1 = max(hi1-lo1+1, 1) n2 = max(hi2-lo2+1, 1) - sz = int(n1,c_size_t) * int(n2,c_size_t) + sz = int(n1,c_size_t) * int(n2,c_size_t) cp = amrex_mempool_alloc(szr*sz) call amrex_real_array_init(cp, sz) call c_f_pointer(cp, fp, shape=(/n1,n2/)) diff --git a/Src/Base/AMReX_parmparse_fi.cpp b/Src/Base/AMReX_parmparse_fi.cpp index d44e695e8a0..0816b1606e3 100644 --- a/Src/Base/AMReX_parmparse_fi.cpp +++ b/Src/Base/AMReX_parmparse_fi.cpp @@ -11,34 +11,34 @@ extern "C" { void amrex_new_parmparse (ParmParse*& pp, const char* name) { - pp = new ParmParse(std::string(name)); + pp = new ParmParse(std::string(name)); } void amrex_delete_parmparse (ParmParse* pp) { - delete pp; + delete pp; } int amrex_parmparse_get_counts (ParmParse* pp, const char* name) { - return pp->countval(name); + return pp->countval(name); } void amrex_parmparse_get_int (ParmParse* pp, const char* name, int* v) { - pp->get(name, *v); + pp->get(name, *v); } void amrex_parmparse_get_real (ParmParse* pp, const char* name, Real* v) { - pp->get(name, *v); + pp->get(name, *v); } void amrex_parmparse_get_bool (ParmParse* pp, const char* name, int* v) { - bool b; - pp->get(name, b); - *v = b; + bool b; + pp->get(name, b); + *v = b; } void amrex_parmparse_get_string (ParmParse* pp, const char* name, char*& v, int* len) @@ -59,20 +59,20 @@ extern "C" void amrex_parmparse_get_intarr (ParmParse* pp, const char* name, int v[], int len) { - Vector r; - pp->getarr(name, r); - for (int i = 0; i < len; ++i) { - v[i] = r[i]; - } + Vector r; + pp->getarr(name, r); + for (int i = 0; i < len; ++i) { + v[i] = r[i]; + } } void amrex_parmparse_get_realarr (ParmParse* pp, const char* name, Real v[], int len) { - Vector r; - pp->getarr(name, r); - for (int i = 0; i < len; ++i) { - v[i] = r[i]; - } + Vector r; + pp->getarr(name, r); + for (int i = 0; i < len; ++i) { + v[i] = r[i]; + } } void amrex_parmparse_get_stringarr (ParmParse* pp, const char* name, char** v, int* sv, int n) @@ -89,18 +89,18 @@ extern "C" int amrex_parmparse_query_int (ParmParse* pp, const char* name, int* v) { - return pp->query(name, *v); + return pp->query(name, *v); } int amrex_parmparse_query_real (ParmParse* pp, const char* name, Real* v) { - return pp->query(name, *v); + return pp->query(name, *v); } int amrex_parmparse_query_bool (ParmParse* pp, const char* name, int* v) { - bool b; - if (pp->query(name, b)) { + bool b; + if (pp->query(name, b)) { *v = b; return 1; } else { @@ -127,7 +127,7 @@ extern "C" { pp->add(name,v); } - + void amrex_parmparse_add_bool (ParmParse* pp, const char* name, int v) { pp->add(name,static_cast(v)); diff --git a/Src/Base/AMReX_parmparse_mod.F90 b/Src/Base/AMReX_parmparse_mod.F90 index 8f31f6af575..2d33b4da40d 100644 --- a/Src/Base/AMReX_parmparse_mod.F90 +++ b/Src/Base/AMReX_parmparse_mod.F90 @@ -77,7 +77,7 @@ function amrex_parmparse_get_counts (pp, name) bind(c) implicit none integer(c_int) :: amrex_parmparse_get_counts type(c_ptr), value :: pp - character(kind=c_char), intent(in) :: name(*) + character(kind=c_char), intent(in) :: name(*) end function amrex_parmparse_get_counts subroutine amrex_parmparse_get_int (pp, name, v) bind(c) @@ -386,7 +386,7 @@ subroutine query_real (this, name, v, flag) real(amrex_real) :: v integer :: iflag iflag = amrex_parmparse_query_real (this%p, amrex_string_f_to_c(name), v) - if (present(flag)) flag = iflag.ne.0 + if (present(flag)) flag = iflag.ne.0 end subroutine query_real subroutine query_logical (this, name, v, flag) @@ -399,7 +399,7 @@ subroutine query_logical (this, name, v, flag) if (iflag.eq.1) then v = i.eq.1 end if - if (present(flag)) flag = iflag.ne.0 + if (present(flag)) flag = iflag.ne.0 end subroutine query_logical subroutine query_string (this, name, v, flag) @@ -495,7 +495,7 @@ subroutine add_string (this, name, v) character(*), intent(in) :: v call amrex_parmparse_add_string(this%p, amrex_string_f_to_c(name), amrex_string_f_to_c(v)) end subroutine add_string - + subroutine add_intarr (this, name, v) class(amrex_parmparse), intent(inout) :: this character(*), intent(in) :: name diff --git a/Src/Boundary/AMReX_BndryData.H b/Src/Boundary/AMReX_BndryData.H index 5037592c0c3..0deee88c66c 100644 --- a/Src/Boundary/AMReX_BndryData.H +++ b/Src/Boundary/AMReX_BndryData.H @@ -22,7 +22,7 @@ namespace amrex { // // A BndryData stores and manipulates boundary data // information on each side of each box in a BoxArray. -// +// /** * \brief A BndryData contains a BndryRegister about each side of each grid in @@ -61,7 +61,7 @@ public: * domain (cell-centered) */ BndryData (const BoxArray& grids, - const DistributionMapping& dmap, + const DistributionMapping& dmap, int ncomp, const Geometry& geom); @@ -79,7 +79,7 @@ public: //! alocate bndry fabs along given face void define (const BoxArray& grids, - const DistributionMapping& dmap, + const DistributionMapping& dmap, int ncomp, const Geometry& geom); // diff --git a/Src/Boundary/AMReX_BndryData.cpp b/Src/Boundary/AMReX_BndryData.cpp index 81c6cac9e6f..969ae34c828 100644 --- a/Src/Boundary/AMReX_BndryData.cpp +++ b/Src/Boundary/AMReX_BndryData.cpp @@ -17,8 +17,8 @@ BndryData::BndryData () noexcept m_ncomp(-1), m_defined(false) {} BndryData::BndryData (const BoxArray& _grids, - const DistributionMapping& _dmap, - int _ncomp, + const DistributionMapping& _dmap, + int _ncomp, const Geometry& _geom) : geom(_geom), @@ -99,9 +99,9 @@ BndryData::init (const BndryData& src) masks.resize(2*AMREX_SPACEDIM); for (int i = 0; i < 2*AMREX_SPACEDIM; i++) { - const MultiMask& smasks = src.masks[i]; - masks[i].define(smasks.boxArray(), smasks.DistributionMap(), smasks.nComp()); - MultiMask::Copy(masks[i], smasks); + const MultiMask& smasks = src.masks[i]; + masks[i].define(smasks.boxArray(), smasks.DistributionMap(), smasks.nComp()); + MultiMask::Copy(masks[i], smasks); } } @@ -130,7 +130,7 @@ BndryData::~BndryData () void BndryData::define (const BoxArray& _grids, - const DistributionMapping& _dmap, + const DistributionMapping& _dmap, int _ncomp, const Geometry& _geom) { @@ -160,7 +160,7 @@ BndryData::define (const BoxArray& _grids, { Orientation face = fi(); BndryRegister::define(face,IndexType::TheCellType(),0,1,1,_ncomp,_dmap); - masks[face].define(grids, _dmap, geom, face, 0, 2, NTangHalfWidth, 1, true); + masks[face].define(grids, _dmap, geom, face, 0, 2, NTangHalfWidth, 1, true); } // // Define "bcond" and "bcloc". @@ -193,7 +193,7 @@ BndryData::define (const BoxArray& _grids, void BndryData::setValue (Orientation face, int n, Real val) noexcept { - auto& fab = bndry[face][n]; + auto& fab = bndry[face][n]; auto arr = fab.array(); const Box& bx = fab.box(); const int ncomp = m_ncomp; diff --git a/Src/Boundary/AMReX_BndryRegister.H b/Src/Boundary/AMReX_BndryRegister.H index f153baab068..b1692d4470d 100644 --- a/Src/Boundary/AMReX_BndryRegister.H +++ b/Src/Boundary/AMReX_BndryRegister.H @@ -41,7 +41,7 @@ public: //! The constructor, given number of cells in/out, extent and number of components (assumes cell-centered boxes, and allocates cell-centered FABs) BndryRegister (const BoxArray& grids_, - const DistributionMapping& dmap, + const DistributionMapping& dmap, int in_rad, int out_rad, int extent_rad, @@ -61,19 +61,19 @@ public: BndryRegister& operator= (BndryRegister&& rhs) = delete; void define (const BoxArray& grids_, - const DistributionMapping& dmap, - int in_rad, - int out_rad, - int extent_rad, - int ncomp); + const DistributionMapping& dmap, + int in_rad, + int out_rad, + int extent_rad, + int ncomp); //! Build FABs along given face, specifying the DistributionMapping. void define (Orientation face, - IndexType typ, - int in_rad, - int out_rad, - int extent_rad, - int ncomp, - const DistributionMapping& dm); + IndexType typ, + int in_rad, + int out_rad, + int extent_rad, + int ncomp, + const DistributionMapping& dm); void clear (); diff --git a/Src/Boundary/AMReX_BndryRegister.cpp b/Src/Boundary/AMReX_BndryRegister.cpp index 28b709ab42e..0f22aed3c94 100644 --- a/Src/Boundary/AMReX_BndryRegister.cpp +++ b/Src/Boundary/AMReX_BndryRegister.cpp @@ -15,7 +15,7 @@ BndryRegister::BndryRegister () noexcept {} BndryRegister::~BndryRegister () {} BndryRegister::BndryRegister (const BoxArray& grids_, - const DistributionMapping& dmap, + const DistributionMapping& dmap, int in_rad, int out_rad, int extent_rad, @@ -34,11 +34,11 @@ BndryRegister::BndryRegister (const BoxArray& grids_, void BndryRegister::define (const BoxArray& grids_, - const DistributionMapping& dmap, - int in_rad, - int out_rad, - int extent_rad, - int ncomp) + const DistributionMapping& dmap, + int in_rad, + int out_rad, + int extent_rad, + int ncomp) { grids = grids_; for (OrientationIter face; face; ++face) @@ -107,12 +107,12 @@ BndryRegister::operator= (const BndryRegister& src) void BndryRegister::define (Orientation _face, - IndexType _typ, - int _in_rad, - int _out_rad, - int _extent_rad, - int _ncomp, - const DistributionMapping& dmap) + IndexType _typ, + int _in_rad, + int _out_rad, + int _extent_rad, + int _ncomp, + const DistributionMapping& dmap) { BoxArray fsBA(grids, BATransformer(_face,_typ,_in_rad,_out_rad,_extent_rad)); @@ -121,7 +121,7 @@ BndryRegister::define (Orientation _face, BL_ASSERT(fabs.size() == 0); fabs.define(fsBA,dmap,_ncomp); - // + // // Go ahead and assign values to the boundary register fabs // since in some places APPLYBC (specifically in the tensor // operator) the boundary registers are used for a few calculations @@ -163,7 +163,7 @@ BndryRegister::operator+= (const BndryRegister& rhs) #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (FabSetIter bfsi(rhs[f]); bfsi.isValid(); ++bfsi) { + for (FabSetIter bfsi(rhs[f]); bfsi.isValid(); ++bfsi) { const Box& bx = bfsi.validbox(); auto const sfab = rhs[f].array(bfsi); auto dfab = bndry[f].array(bfsi); @@ -171,7 +171,7 @@ BndryRegister::operator+= (const BndryRegister& rhs) { dfab(i,j,k,n) += sfab(i,j,k,n); }); - } + } } return *this; } @@ -271,7 +271,7 @@ BndryRegister::read (const std::string& name, std::istream& is) grids_in.readFrom(is); if (!amrex::match(grids,grids_in)) { - amrex::Abort("BndryRegister::read: grids do not match"); + amrex::Abort("BndryRegister::read: grids do not match"); } for (OrientationIter face; face; ++face) @@ -291,12 +291,12 @@ BndryRegister::read (const std::string& name, std::istream& is) } // Local copy function -void +void BndryRegister::Copy (BndryRegister& dst, const BndryRegister& src) { for (OrientationIter face; face; ++face) { - FabSet::Copy(dst[face()], src[face()]); + FabSet::Copy(dst[face()], src[face()]); } } diff --git a/Src/Boundary/AMReX_FabSet.H b/Src/Boundary/AMReX_FabSet.H index 678e2a0f43e..4955f2d7839 100644 --- a/Src/Boundary/AMReX_FabSet.H +++ b/Src/Boundary/AMReX_FabSet.H @@ -78,7 +78,7 @@ public: const BoxArray& boxArray () const noexcept { return m_mf.boxArray(); } const DistributionMapping& DistributionMap () const noexcept - { return m_mf.DistributionMap(); } + { return m_mf.DistributionMap(); } int nComp () const noexcept { return m_mf.nComp(); } @@ -92,13 +92,13 @@ public: FabSet& plusFrom (const FabSet& src, int scomp, int dcomp, int ncomp); FabSet& plusFrom (const MultiFab& src, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period = Periodicity::NonPeriodic()); + const Periodicity& period = Periodicity::NonPeriodic()); void copyTo (MultiFab& dest, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period = Periodicity::NonPeriodic()) const; + const Periodicity& period = Periodicity::NonPeriodic()) const; void plusTo (MultiFab& dest, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period = Periodicity::NonPeriodic()) const; + const Periodicity& period = Periodicity::NonPeriodic()) const; void setVal (Real val); @@ -109,7 +109,7 @@ public: //!< Linear combination: this := a*mfa + b*mfb FabSet& linComb (Real a, const MultiFab& mfa, int a_comp, - Real b, const MultiFab& mfb, int b_comp, + Real b, const MultiFab& mfb, int b_comp, int dcomp, int ncomp, int ngrow); // @@ -131,7 +131,7 @@ class FabSetIter { public: explicit FabSetIter (const FabSet& fs) - : MFIter(fs.m_mf) { } + : MFIter(fs.m_mf) { } }; } diff --git a/Src/Boundary/AMReX_FabSet.cpp b/Src/Boundary/AMReX_FabSet.cpp index 10b67d78958..333bfa624bc 100644 --- a/Src/Boundary/AMReX_FabSet.cpp +++ b/Src/Boundary/AMReX_FabSet.cpp @@ -30,7 +30,7 @@ FabSet::copyFrom (const FabSet& src, int scomp, int dcomp, int ncomp) #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (FabSetIter fsi(*this); fsi.isValid(); ++fsi) { + for (FabSetIter fsi(*this); fsi.isValid(); ++fsi) { const Box& bx = fsi.validbox(); auto const srcfab = src.array(fsi); auto dstfab = this->array(fsi); @@ -38,16 +38,16 @@ FabSet::copyFrom (const FabSet& src, int scomp, int dcomp, int ncomp) { dstfab(i,j,k,n+dcomp) = srcfab(i,j,k,n+scomp); }); - } + } } else { - m_mf.copy(src.m_mf,scomp,dcomp,ncomp); + m_mf.copy(src.m_mf,scomp,dcomp,ncomp); } return *this; } FabSet& FabSet::copyFrom (const MultiFab& src, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period) + const Periodicity& period) { BL_ASSERT(boxArray() != src.boxArray()); m_mf.copy(src,scomp,dcomp,ncomp,ngrow,0,period); @@ -61,7 +61,7 @@ FabSet::plusFrom (const FabSet& src, int scomp, int dcomp, int ncomp) #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (FabSetIter fsi(*this); fsi.isValid(); ++fsi) { + for (FabSetIter fsi(*this); fsi.isValid(); ++fsi) { const Box& bx = fsi.validbox(); auto const srcfab = src.array(fsi); auto dstfab = this->array(fsi); @@ -69,16 +69,16 @@ FabSet::plusFrom (const FabSet& src, int scomp, int dcomp, int ncomp) { dstfab(i,j,k,n+dcomp) += srcfab(i,j,k,n+scomp); }); - } + } } else { - amrex::Abort("FabSet::plusFrom: parallel plusFrom not supported"); + amrex::Abort("FabSet::plusFrom: parallel plusFrom not supported"); } return *this; } FabSet& FabSet::plusFrom (const MultiFab& src, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period) + const Periodicity& period) { BL_ASSERT(boxArray() != src.boxArray()); m_mf.copy(src,scomp,dcomp,ncomp,ngrow,0,period,FabArrayBase::ADD); @@ -87,7 +87,7 @@ FabSet::plusFrom (const MultiFab& src, int ngrow, int scomp, int dcomp, int ncom void FabSet::copyTo (MultiFab& dest, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period) const + const Periodicity& period) const { BL_ASSERT(boxArray() != dest.boxArray()); dest.copy(m_mf,scomp,dcomp,ncomp,0,ngrow,period); @@ -95,7 +95,7 @@ FabSet::copyTo (MultiFab& dest, int ngrow, int scomp, int dcomp, int ncomp, void FabSet::plusTo (MultiFab& dest, int ngrow, int scomp, int dcomp, int ncomp, - const Periodicity& period) const + const Periodicity& period) const { BL_ASSERT(boxArray() != dest.boxArray()); dest.copy(m_mf,scomp,dcomp,ncomp,0,ngrow,period,FabArrayBase::ADD); @@ -158,11 +158,11 @@ FabSet::linComb (Real a, Real b, const FabSet& src, int scomp, int dcomp, int nc } // Linear combination: this := a*mfa + b*mfb -// CastroRadiation is the only code that uses this function. +// CastroRadiation is the only code that uses this function. FabSet& FabSet::linComb (Real a, const MultiFab& mfa, int a_comp, - Real b, const MultiFab& mfb, int b_comp, - int dcomp, int ncomp, int ngrow) + Real b, const MultiFab& mfb, int b_comp, + int dcomp, int ncomp, int ngrow) { BL_PROFILE("FabSet::linComb()"); BL_ASSERT(ngrow <= mfa.nGrow()); @@ -228,7 +228,7 @@ void FabSet::read(const std::string& name) { if (m_mf.empty()) { - amrex::Abort("FabSet::read: not predefined"); + amrex::Abort("FabSet::read: not predefined"); } VisMF::Read(m_mf,name); } diff --git a/Src/Boundary/AMReX_InterpBndryData.H b/Src/Boundary/AMReX_InterpBndryData.H index cd7eb0e92d2..609728e1d70 100644 --- a/Src/Boundary/AMReX_InterpBndryData.H +++ b/Src/Boundary/AMReX_InterpBndryData.H @@ -58,7 +58,7 @@ public: * \param geom */ InterpBndryData (const BoxArray& _grids, - const DistributionMapping& _dmap, + const DistributionMapping& _dmap, int _ncomp, const Geometry& geom); @@ -97,7 +97,7 @@ public: */ virtual void setBndryConds (const BCRec& /*phys_bc*/, const IntVect& /*ratio*/, - int /*comp*/ = 0) {} + int /*comp*/ = 0) {} /** * \brief set bndry values at coarse level (non interpolation performed) diff --git a/Src/Boundary/AMReX_InterpBndryData.cpp b/Src/Boundary/AMReX_InterpBndryData.cpp index 5be192534d0..45608164cba 100644 --- a/Src/Boundary/AMReX_InterpBndryData.cpp +++ b/Src/Boundary/AMReX_InterpBndryData.cpp @@ -31,7 +31,7 @@ InterpBndryData::operator= (const InterpBndryData& rhs) } InterpBndryData::InterpBndryData (const BoxArray& _grids, - const DistributionMapping& _dmap, + const DistributionMapping& _dmap, int _ncomp, const Geometry& _geom) : @@ -89,7 +89,7 @@ InterpBndryData::setBndryValues (const MultiFab& mf, BL_ASSERT(grids == mf.boxArray()); for (int n = bnd_start; n < bnd_start+num_comp; ++n) { - setBndryConds(bc, ref_ratio, n); + setBndryConds(bc, ref_ratio, n); } #ifdef AMREX_USE_OMP @@ -151,7 +151,7 @@ InterpBndryData::BndryValuesDoIt (BndryRegister& crse, int f_start, int bnd_start, int num_comp, - const IntVect& ratio, + const IntVect& ratio, const BCRec* bc, int max_order) { diff --git a/Src/Boundary/AMReX_MacBndry.H b/Src/Boundary/AMReX_MacBndry.H index 7f516646c01..9e9cce4a670 100644 --- a/Src/Boundary/AMReX_MacBndry.H +++ b/Src/Boundary/AMReX_MacBndry.H @@ -16,7 +16,7 @@ public: MacBndry (); MacBndry (const BoxArray& _grids, - const DistributionMapping& _dmap, + const DistributionMapping& _dmap, int _ncomp, const Geometry& _geom); diff --git a/Src/Boundary/AMReX_MacBndry.cpp b/Src/Boundary/AMReX_MacBndry.cpp index c8c1437a9b8..4ce15b86987 100644 --- a/Src/Boundary/AMReX_MacBndry.cpp +++ b/Src/Boundary/AMReX_MacBndry.cpp @@ -12,7 +12,7 @@ MacBndry::MacBndry () } MacBndry::MacBndry (const BoxArray& _grids, - const DistributionMapping& _dmap, + const DistributionMapping& _dmap, int _ncomp, const Geometry& _geom) : @@ -32,7 +32,7 @@ MacBndry::setBndryConds (const BCRec& phys_bc, void MacBndry::setBndryConds (const BCRec& phys_bc, const IntVect& ratio, - int comp) + int comp) { m_phys_bc = phys_bc; @@ -66,7 +66,7 @@ MacBndry::setBndryConds (const BCRec& phys_bc, // const int p_bc = (face.isLow() ? phys_bc.lo(dir) : phys_bc.hi(dir)); - bctag[face][comp] = (p_bc == PhysBCType::outflow) + bctag[face][comp] = (p_bc == PhysBCType::outflow) ? AMREX_LO_DIRICHLET : AMREX_LO_NEUMANN; bloc[face] = 0; } @@ -78,7 +78,7 @@ MacBndry::setBndryConds (const BCRec& phys_bc, const Real delta = dx[dir]*ratio[dir]; bctag[face][comp] = AMREX_LO_DIRICHLET; - bloc[face] = 0.5_rt*delta; + bloc[face] = 0.5_rt*delta; } } } diff --git a/Src/Boundary/AMReX_Mask.H b/Src/Boundary/AMReX_Mask.H index 99aad394355..d1d3fd7fba1 100644 --- a/Src/Boundary/AMReX_Mask.H +++ b/Src/Boundary/AMReX_Mask.H @@ -36,9 +36,9 @@ public: Mask (const Box& bx, int nc, Arena* ar); explicit Mask (const Box& bx, - int nc = 1, - bool alloc=true, - bool shared=false, + int nc = 1, + bool alloc=true, + bool shared=false, Arena* ar = nullptr); /** diff --git a/Src/Boundary/AMReX_MultiMask.H b/Src/Boundary/AMReX_MultiMask.H index 987f59ef38e..893ead9cf2c 100644 --- a/Src/Boundary/AMReX_MultiMask.H +++ b/Src/Boundary/AMReX_MultiMask.H @@ -20,7 +20,7 @@ public: MultiMask () { ; } MultiMask (const BoxArray& ba, const DistributionMapping& dm, int ncomp); MultiMask (const BoxArray& regba, const DistributionMapping& dm, const Geometry& geom, - Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval); + Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval); ~MultiMask () = default; @@ -32,7 +32,7 @@ public: void define (const BoxArray& ba, const DistributionMapping& dm, int ncomp); void define (const BoxArray& regba, const DistributionMapping& dm, const Geometry& geom, - Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval); + Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval); Mask& operator[] (const MFIter& mfi) noexcept { return m_fa[mfi]; } const Mask& operator[] (const MFIter& mfi) const noexcept { return m_fa[mfi]; } @@ -59,8 +59,8 @@ class MultiMaskIter : public MFIter { public: - explicit MultiMaskIter (const MultiMask& mm) - : MFIter(mm.m_fa) { } + explicit MultiMaskIter (const MultiMask& mm) + : MFIter(mm.m_fa) { } }; } diff --git a/Src/Boundary/AMReX_MultiMask.cpp b/Src/Boundary/AMReX_MultiMask.cpp index 7281c6fc21e..90984dac241 100644 --- a/Src/Boundary/AMReX_MultiMask.cpp +++ b/Src/Boundary/AMReX_MultiMask.cpp @@ -9,7 +9,7 @@ MultiMask::MultiMask (const BoxArray& ba, const DistributionMapping& dm, int nco { } MultiMask::MultiMask (const BoxArray& regba, const DistributionMapping& dm, const Geometry& geom, - Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval) + Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval) { define(regba, dm, geom, face, in_rad, out_rad, extent_rad, ncomp, initval); } @@ -23,7 +23,7 @@ MultiMask::define (const BoxArray& ba, const DistributionMapping& dm, int ncomp) void MultiMask::define (const BoxArray& regba, const DistributionMapping& dm, const Geometry& geom, - Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval) + Orientation face, int in_rad, int out_rad, int extent_rad, int ncomp, bool initval) { BL_ASSERT(m_fa.size() == 0); @@ -70,7 +70,7 @@ MultiMask::define (const BoxArray& regba, const DistributionMapping& dm, const G } } -void +void MultiMask::Copy (MultiMask& dst, const MultiMask& src) { BL_ASSERT(dst.nComp() == src.nComp()); diff --git a/Src/Boundary/AMReX_YAFluxRegister.cpp b/Src/Boundary/AMReX_YAFluxRegister.cpp index 0678080eb44..aa5d69585b1 100644 --- a/Src/Boundary/AMReX_YAFluxRegister.cpp +++ b/Src/Boundary/AMReX_YAFluxRegister.cpp @@ -73,7 +73,7 @@ YAFluxRegister::define (const BoxArray& fba, const BoxArray& cba, cfba.uniqify(); #ifdef AMREX_USE_OMP - + const int nthreads = omp_get_max_threads(); Vector bl_priv(nthreads, BoxList()); Vector > procmap_priv(nthreads); @@ -299,7 +299,7 @@ YAFluxRegister::FineAdd (const MFIter& mfi, flux[idim] = &ftmp[idim]; } } - + AMREX_ASSERT(bx.cellCentered()); for (int idim=0; idim < AMREX_SPACEDIM; ++idim) diff --git a/Src/EB/AMReX_EB2_3D_C.H b/Src/EB/AMReX_EB2_3D_C.H index c16d447e377..54f4bd53c39 100644 --- a/Src/EB/AMReX_EB2_3D_C.H +++ b/Src/EB/AMReX_EB2_3D_C.H @@ -222,13 +222,13 @@ void amrex_eb2_cover_multiple_cuts (Box const& tbx, Box const& bxg2, fx(i,j,k) = Type::covered; // Cover the cut edges containing any regular corner - if (s(i,j ,k ) < 0.0) + if (s(i,j ,k ) < 0.0) cover_cut_edges_given_corner(i,j ,k ,ex,ey,ez); - if (s(i,j+1,k ) < 0.0) + if (s(i,j+1,k ) < 0.0) cover_cut_edges_given_corner(i,j+1,k ,ex,ey,ez); - if (s(i,j ,k+1) < 0.0) + if (s(i,j ,k+1) < 0.0) cover_cut_edges_given_corner(i,j ,k+1,ex,ey,ez); - if (s(i,j+1,k+1) < 0.0) + if (s(i,j+1,k+1) < 0.0) cover_cut_edges_given_corner(i,j+1,k+1,ex,ey,ez); } }); @@ -252,13 +252,13 @@ void amrex_eb2_cover_multiple_cuts (Box const& tbx, Box const& bxg2, fy(i,j,k) = Type::covered; // Cover the cut edges containing any regular corner - if (s(i ,j,k ) < 0.0) + if (s(i ,j,k ) < 0.0) cover_cut_edges_given_corner(i ,j,k ,ex,ey,ez); - if (s(i+1,j,k ) < 0.0) + if (s(i+1,j,k ) < 0.0) cover_cut_edges_given_corner(i+1,j,k ,ex,ey,ez); - if (s(i ,j,k+1) < 0.0) + if (s(i ,j,k+1) < 0.0) cover_cut_edges_given_corner(i ,j,k+1,ex,ey,ez); - if (s(i+1,j,k+1) < 0.0) + if (s(i+1,j,k+1) < 0.0) cover_cut_edges_given_corner(i+1,j,k+1,ex,ey,ez); } }); @@ -281,13 +281,13 @@ void amrex_eb2_cover_multiple_cuts (Box const& tbx, Box const& bxg2, fz(i,j,k) = Type::covered; // Cover the cut edges containing any regular corner - if (s(i ,j ,k) < 0.0) + if (s(i ,j ,k) < 0.0) cover_cut_edges_given_corner(i ,j ,k,ex,ey,ez); - if (s(i+1,j ,k) < 0.0) + if (s(i+1,j ,k) < 0.0) cover_cut_edges_given_corner(i+1,j ,k,ex,ey,ez); - if (s(i ,j+1,k) < 0.0) + if (s(i ,j+1,k) < 0.0) cover_cut_edges_given_corner(i ,j+1,k,ex,ey,ez); - if (s(i+1,j+1,k) < 0.0) + if (s(i+1,j+1,k) < 0.0) cover_cut_edges_given_corner(i+1,j+1,k,ex,ey,ez); } }); diff --git a/Src/EB/AMReX_EB2_IF_Complement.H b/Src/EB/AMReX_EB2_IF_Complement.H index 7b52c51be66..a8a144f752d 100644 --- a/Src/EB/AMReX_EB2_IF_Complement.H +++ b/Src/EB/AMReX_EB2_IF_Complement.H @@ -18,7 +18,7 @@ public: ComplementIF (F&& a_f) : m_f(std::move(a_f)) {} ComplementIF (F const& a_f) : m_f(a_f) {} - + ComplementIF (const ComplementIF& rhs) = default; ComplementIF (ComplementIF&& rhs) = default; ComplementIF& operator= (const ComplementIF& rhs) = delete; diff --git a/Src/EB/AMReX_EB2_IF_Lathe.H b/Src/EB/AMReX_EB2_IF_Lathe.H index 54513f6a8d5..d59320ca925 100644 --- a/Src/EB/AMReX_EB2_IF_Lathe.H +++ b/Src/EB/AMReX_EB2_IF_Lathe.H @@ -19,7 +19,7 @@ public: LatheIF (F&& a_f) : m_f(std::move(a_f)) {} LatheIF (F const& a_f) : m_f(a_f) {} - + LatheIF (const LatheIF& rhs) = default; LatheIF (LatheIF&& rhs) = default; LatheIF& operator= (const LatheIF& rhs) = delete; @@ -44,7 +44,7 @@ public: return m_f(r,0.0); #else return m_f(r,z,0.0); -#endif +#endif } protected: diff --git a/Src/EB/AMReX_EB2_IF_Sphere.H b/Src/EB/AMReX_EB2_IF_Sphere.H index 63bdc5ba18a..d28fdbd4241 100644 --- a/Src/EB/AMReX_EB2_IF_Sphere.H +++ b/Src/EB/AMReX_EB2_IF_Sphere.H @@ -39,7 +39,7 @@ public: } protected: - + Real m_radius; XDim3 m_center; // diff --git a/Src/EB/AMReX_EB2_Level.cpp b/Src/EB/AMReX_EB2_Level.cpp index 3d45c0d4391..7ccead1e553 100644 --- a/Src/EB/AMReX_EB2_Level.cpp +++ b/Src/EB/AMReX_EB2_Level.cpp @@ -208,7 +208,7 @@ Level::coarsenFromFine (Level& fineLevel, bool fill_boundary) if (!fine_covered_grids.empty()) { const std::vector& pshifts = fine_period.shiftIntVect(); - + #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif diff --git a/Src/EB/AMReX_EBFluxRegister.cpp b/Src/EB/AMReX_EBFluxRegister.cpp index aaeb255d18f..6b6125733f5 100644 --- a/Src/EB/AMReX_EBFluxRegister.cpp +++ b/Src/EB/AMReX_EBFluxRegister.cpp @@ -255,9 +255,9 @@ EBFluxRegister::Reflux (MultiFab& crse_state, const amrex::MultiFab& crse_vfrac, m_ncomp, 1, MFInfo(), FArrayBoxFactory()); MultiFab::Copy(grown_crse_data, m_crse_data, 0, 0, m_ncomp, 0); grown_crse_data.FillBoundary(m_crse_geom.periodicity()); - + m_crse_data.setVal(0.0); - + auto const& factory = dynamic_cast(crse_state.Factory()); auto const& flags = factory.getMultiEBCellFlagFab(); @@ -321,9 +321,9 @@ EBFluxRegister::Reflux (MultiFab& crse_state, const amrex::MultiFab& crse_vfrac, { const Box& cbx = mfi.tilebox(); const Box& fbx = amrex::refine(cbx, m_ratio); - + const auto& ebflag = flags[mfi]; - + if (ebflag.getType(fbx) != FabType::covered) { Array4 const& d = fine_state.array(mfi); diff --git a/Src/EB/AMReX_EBInterpolater.cpp b/Src/EB/AMReX_EBInterpolater.cpp index c4853e824ce..4c0f720391e 100644 --- a/Src/EB/AMReX_EBInterpolater.cpp +++ b/Src/EB/AMReX_EBInterpolater.cpp @@ -46,12 +46,12 @@ EBCellConservativeLinear::interp (const FArrayBox& crse, { const EBFArrayBox& crse_eb = static_cast(crse); EBFArrayBox& fine_eb = static_cast(fine); - + const EBCellFlagFab& crse_flag = crse_eb.getEBCellFlagFab(); const EBCellFlagFab& fine_flag = fine_eb.getEBCellFlagFab(); - + const Box& crse_bx = CoarseBox(target_fine_region,ratio); - + const FabType ftype = fine_flag.getType(target_fine_region); const FabType ctype = crse_flag.getType(crse_bx); @@ -76,7 +76,7 @@ EBCellConservativeLinear::interp (const FArrayBox& crse, } }); } - } + } } } diff --git a/Src/EB/AMReX_EBMultiFabUtil.H b/Src/EB/AMReX_EBMultiFabUtil.H index 56ed8c878d3..0d78199cf89 100644 --- a/Src/EB/AMReX_EBMultiFabUtil.H +++ b/Src/EB/AMReX_EBMultiFabUtil.H @@ -9,7 +9,7 @@ namespace amrex { - void EB_set_covered (MultiFab& mf, Real val); + void EB_set_covered (MultiFab& mf, Real val); void EB_set_covered (MultiFab& mf, int icomp, int ncomp, int ngrow, Real val); void EB_set_covered (MultiFab& mf, int icomp, int ncomp, const Vector& vals); void EB_set_covered (MultiFab& mf, int icomp, int ncomp, int ngrow, const Vector& vals); @@ -48,10 +48,10 @@ namespace amrex // Cell faces to cell centers void EB_average_face_to_cellcenter (MultiFab& ccmf, int dcomp, const Array& fmf); - + // Cell centers to cell centroids void EB_interp_CC_to_Centroid (MultiFab& cntrd, const MultiFab& cc, int scomp, int dcomp, int ncomp, const Geometry& geom); - + // Cell centers to face centroids void EB_interp_CC_to_FaceCentroid (const MultiFab& cc, AMREX_D_DECL( MultiFab& fcx, diff --git a/Src/EB/AMReX_EBMultiFabUtil.cpp b/Src/EB/AMReX_EBMultiFabUtil.cpp index 3a38b581bd6..1dd3e71d7a8 100644 --- a/Src/EB/AMReX_EBMultiFabUtil.cpp +++ b/Src/EB/AMReX_EBMultiFabUtil.cpp @@ -686,7 +686,7 @@ void EB_computeDivergence (MultiFab& divu, const ArraynComp()); #endif - + if (!divu.hasEBFabFactory()) { amrex::computeDivergence(divu, umac, geom); @@ -855,7 +855,7 @@ EB_interp_CC_to_Centroid (MultiFab& cent, const MultiFab& cc, int scomp, int dco } cent.FillBoundary(dcomp,ncomp,geom.periodicity()); - + } void @@ -873,11 +873,11 @@ EB_interp_CC_to_FaceCentroid (const MultiFab& cc, const auto& fcent = factory.getFaceCent(); AMREX_ALWAYS_ASSERT(a_bcs.size() == ncomp ); - + Box domain(a_geom.Domain()); - + const int nghost(4); - + // Initialize edge state AMREX_D_TERM(fc_x.setVal(1e40,dcomp,ncomp);, fc_y.setVal(1e40,dcomp,ncomp);, @@ -897,7 +897,7 @@ EB_interp_CC_to_FaceCentroid (const MultiFab& cc, { d_bcs = a_bcs.dataPtr(); } - + MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.SetDynamic(true); #ifdef AMREX_USE_OMP @@ -918,7 +918,7 @@ EB_interp_CC_to_FaceCentroid (const MultiFab& cc, AMREX_D_TERM(Array4 const& edg_x = fc_x.array(mfi,dcomp);, Array4 const& edg_y = fc_y.array(mfi,dcomp);, Array4 const& edg_z = fc_z.array(mfi,dcomp)); - + if (fabtyp_ghost == FabType::regular ) { AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM @@ -963,7 +963,7 @@ EB_interp_CC_to_FaceCentroid (const MultiFab& cc, } } } - + fc_x.FillBoundary(a_geom.periodicity()); fc_y.FillBoundary(a_geom.periodicity()); #if ( AMREX_SPACEDIM == 3 ) @@ -977,20 +977,20 @@ void EB_interp_CellCentroid_to_FaceCentroid (const MultiFab& phi_centroid, const Array& phi_faces, int scomp, int dcomp, int nc, const Geometry& geom, - const amrex::Vector& a_bcs) + const amrex::Vector& a_bcs) { - EB_interp_CellCentroid_to_FaceCentroid (phi_centroid, AMREX_D_DECL( *phi_faces[0], *phi_faces[1], *phi_faces[2] ), + EB_interp_CellCentroid_to_FaceCentroid (phi_centroid, AMREX_D_DECL( *phi_faces[0], *phi_faces[1], *phi_faces[2] ), scomp, dcomp, nc, geom, a_bcs); } // Cell centroids to face centroids void EB_interp_CellCentroid_to_FaceCentroid (const MultiFab& phi_centroid, - const Vector& phi_faces, + const Vector& phi_faces, int scomp, int dcomp, int nc, const Geometry& geom, - const amrex::Vector& a_bcs) + const amrex::Vector& a_bcs) { - EB_interp_CellCentroid_to_FaceCentroid (phi_centroid, AMREX_D_DECL( *phi_faces[0], *phi_faces[1], *phi_faces[2] ), + EB_interp_CellCentroid_to_FaceCentroid (phi_centroid, AMREX_D_DECL( *phi_faces[0], *phi_faces[1], *phi_faces[2] ), scomp, dcomp, nc, geom, a_bcs); } @@ -1012,11 +1012,11 @@ EB_interp_CellCentroid_to_FaceCentroid (const MultiFab& phi_centroid, // We assume that we start from the first component of bcs ... we may need to generalize this AMREX_ALWAYS_ASSERT(a_bcs.size() >= ncomp ); - + Box domain(a_geom.Domain()); - + const int nghost(4); - + // Initialize edge state AMREX_D_TERM(phi_xface.setVal(1e40,dcomp,ncomp);, phi_yface.setVal(1e40,dcomp,ncomp);, @@ -1036,7 +1036,7 @@ EB_interp_CellCentroid_to_FaceCentroid (const MultiFab& phi_centroid, { d_bcs = a_bcs.dataPtr(); } - + MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.SetDynamic(true); #ifdef AMREX_USE_OMP @@ -1057,7 +1057,7 @@ EB_interp_CellCentroid_to_FaceCentroid (const MultiFab& phi_centroid, AMREX_D_TERM(Array4 const& phi_x = phi_xface.array(mfi,dcomp);, Array4 const& phi_y = phi_yface.array(mfi,dcomp);, Array4 const& phi_z = phi_zface.array(mfi,dcomp)); - + if (fabtyp_ghost == FabType::regular ) { AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM @@ -1106,7 +1106,7 @@ EB_interp_CellCentroid_to_FaceCentroid (const MultiFab& phi_centroid, } } } - + phi_xface.FillBoundary(a_geom.periodicity()); phi_yface.FillBoundary(a_geom.periodicity()); #if ( AMREX_SPACEDIM == 3 ) diff --git a/Src/EB/AMReX_EBMultiFabUtil_2D_C.H b/Src/EB/AMReX_EBMultiFabUtil_2D_C.H index fe2c51b4dd6..908392d05a8 100644 --- a/Src/EB/AMReX_EBMultiFabUtil_2D_C.H +++ b/Src/EB/AMReX_EBMultiFabUtil_2D_C.H @@ -411,7 +411,7 @@ void eb_interp_centroid2facecent_x (Box const& ubx, else if ( apx(i,j,k) == 1 && (cvol(i,j,k) == 1 && cvol(i-1,j,k) == 1) ) { edg_x(i,j,k,n) = 0.5 * ( phi(i,j,k,n) + phi(i-1,j,k,n) ); - } + } else if (apx(i,j,k) == 1 && (amrex::Math::abs(ccent(i,j,k,1)-ccent(i-1,j,k,1)) < 1.e-8) ) { Real d0 = 0.5 + ccent(i ,j,k,0); // distance in x-dir from centroid(i ,j) to face(i,j) @@ -419,8 +419,8 @@ void eb_interp_centroid2facecent_x (Box const& ubx, Real a0 = d1 / (d0 + d1); // coefficient of phi(i ,j,k,n) Real a1 = d0 / (d0 + d1); // coefficient of phi(i-1,j,k,n) edg_x(i,j,k,n) = a0*phi(i,j,k,n) + a1*phi(i-1,j,k,n); // interpolation of phi in x-direction - } - else + } + else { int ii = i - 1; int jj; @@ -452,8 +452,8 @@ void eb_interp_centroid2facecent_x (Box const& ubx, Real phi01_jj = a0_jj * phi(i,jj,k,n) + a1_jj * phi(ii,jj,k,n); // interpolation of phi in x-direction Real y01_jj = a0_jj * ccent(i,jj,k,1) + a1_jj * ccent(ii,jj,k,1); // interpolation of y-loc in x-direction - - edg_x(i,j,k,n) = ( ( fcx(i,j,k) - y01 ) * phi01_jj + + + edg_x(i,j,k,n) = ( ( fcx(i,j,k) - y01 ) * phi01_jj + (1. - fcx(i,j,k) + y01_jj) * phi01 ) / (1. - y01 + y01_jj); } }); @@ -498,8 +498,8 @@ void eb_interp_centroid2facecent_y (Box const& vbx, Real a0 = d1 / (d0 + d1); // coefficient of phi(i,j ,k,n) Real a1 = d0 / (d0 + d1); // coefficient of phi(i,j-1,k,n) edg_y(i,j,k,n) = a0*phi(i,j,k,n) + a1*phi(i,j-1,k,n); // interpolation of phi in y-direction - } - else + } + else { int jj = j - 1; int ii; @@ -533,8 +533,8 @@ void eb_interp_centroid2facecent_y (Box const& vbx, Real phi01_ii = a0_ii * phi(ii,j,k,n) + a1_ii * phi(ii,jj,k,n); // interpolation of phi in y-direction Real x01_ii = a0_ii * ccent(ii,j,k,0) + a1_ii * ccent(ii,jj,k,0); // interpolation of x-loc in y-direction - - edg_y(i,j,k,n) = ( ( fcy(i,j,k) - x01 ) * phi01_ii + + + edg_y(i,j,k,n) = ( ( fcy(i,j,k) - x01 ) * phi01_ii + (1. - fcy(i,j,k) + x01_ii) * phi01 ) / (1. - x01 + x01_ii); } }); @@ -563,7 +563,7 @@ void eb_interp_cc2face_x (Box const& ubx, else { edg_x(i,j,k,n) = 0.5 * ( phi(i,j,k,n) + phi(i-1,j,k,n) ); - } + } }); } diff --git a/Src/EB/AMReX_EBMultiFabUtil_3D_C.H b/Src/EB/AMReX_EBMultiFabUtil_3D_C.H index ba1b80f24a5..5df2174aaa5 100644 --- a/Src/EB/AMReX_EBMultiFabUtil_3D_C.H +++ b/Src/EB/AMReX_EBMultiFabUtil_3D_C.H @@ -13,11 +13,11 @@ EB_interp_in_quad(Real xint,Real yint,Real v0,Real v1,Real v2,Real v3, // // 2D Stencil // -// v3 @ (x3,y3) v2 @ (x2,y2) +// v3 @ (x3,y3) v2 @ (x2,y2) // // (xint,yint) // -// v0 @ (x0,y0) v1 @ (x1,y1) +// v0 @ (x0,y0) v1 @ (x1,y1) Real va = v3 - v0; Real vb = v2 - v0; @@ -42,7 +42,7 @@ EB_interp_in_quad(Real xint,Real yint,Real v0,Real v1,Real v2,Real v3, xyder = xyder / det; Real value = v0 + xder*(xint-x0)+yder*(yint-y0)+ xyder*(xint-x0)*(yint-y0); - + return value; } @@ -389,7 +389,7 @@ void eb_interp_cc2cent (Box const& box, phicent(i,j,k,n) = phicc(i,j,k,n); } else - { + { Real gx = cent(i,j,k,0); Real gy = cent(i,j,k,1); Real gz = cent(i,j,k,2); @@ -450,11 +450,11 @@ void eb_interp_cc2facecent_x (Box const& ubx, else { edg_x(i,j,k,n) = 0.5 * ( phi(i,j,k,n) + phi(i-1,j,k,n) ); - } + } } else { - Real gx = 0.5; + Real gx = 0.5; Real gy = fcx(i,j,k,0); Real gz = fcx(i,j,k,1); int ii = i - 1; @@ -475,7 +475,7 @@ void eb_interp_cc2facecent_x (Box const& ubx, + ( gxz - gxyz) * phi(ii,j ,kk,n) + ( gxy - gxyz) * phi(ii,jj,k ,n) + ( gxyz) * phi(ii,jj,kk,n); - } + } } }); } @@ -516,8 +516,8 @@ void eb_interp_cc2facecent_y (Box const& vbx, } } else - { - Real gx = fcy(i,j,k,0); + { + Real gx = fcy(i,j,k,0); Real gy = 0.5; Real gz = fcy(i,j,k,1); int ii = (gx < 0.0) ? i - 1 : i + 1; @@ -538,7 +538,7 @@ void eb_interp_cc2facecent_y (Box const& vbx, + ( gxz - gxyz) * phi(ii,j ,kk,n) + ( gxy - gxyz) * phi(ii,jj,k ,n) + ( gxyz) * phi(ii,jj,kk,n); - } + } } }); } @@ -580,7 +580,7 @@ void eb_interp_cc2facecent_z (Box const& wbx, } else { - Real gx = fcz(i,j,k,0); + Real gx = fcz(i,j,k,0); Real gy = fcz(i,j,k,1); Real gz = 0.5; int ii = (gx < 0.0) ? i - 1 : i + 1; @@ -601,7 +601,7 @@ void eb_interp_cc2facecent_z (Box const& wbx, + ( gxz - gxyz) * phi(ii,j ,kk,n) + ( gxy - gxyz) * phi(ii,jj,k ,n) + ( gxyz) * phi(ii,jj,kk,n); - } + } } }); } @@ -639,8 +639,8 @@ void eb_interp_centroid2facecent_x (Box const& ubx, else if (apx(i,j,k) == 1 && cvol(i,j,k) == 1 && cvol(i-1,j,k) == 1) { phi_x(i,j,k,n) = 0.5 * ( phi(i,j,k,n) + phi(i-1,j,k,n) ); - } - else if ( apx(i,j,k) == 1 && (amrex::Math::abs(ccent(i,j,k,1)-ccent(i-1,j,k,1)) < 1.e-8) && + } + else if ( apx(i,j,k) == 1 && (amrex::Math::abs(ccent(i,j,k,1)-ccent(i-1,j,k,1)) < 1.e-8) && (amrex::Math::abs(ccent(i,j,k,2)-ccent(i-1,j,k,2)) < 1.e-8) ) { Real d0 = 0.5 + ccent(i ,j,k,0); // distance in x-dir from centroid(i ,j) to face(i,j) @@ -649,7 +649,7 @@ void eb_interp_centroid2facecent_x (Box const& ubx, Real a1 = d0 / (d0 + d1); // coefficient of phi(i-1,j,k,n) phi_x(i,j,k,n) = a0*phi(i,j,k,n) + a1*phi(i-1,j,k,n); // interpolation of phi in x-direction } - else + else { // We must add additional tests to avoid the case where fcx is very close to zero, but j-1/j+1 or k-1/k+1 // might be covered cells -- this can happen when the EB is exactly aligned with the horizontal grid planes @@ -685,11 +685,11 @@ void eb_interp_centroid2facecent_x (Box const& ubx, Real test_zero_jkalt = cvol(i-1,jalt,k) * cvol(i-1,j,kalt) * cvol(i-1,jalt,kalt) * cvol(i ,jalt,k) * cvol(i ,j,kalt) * cvol(i ,jalt,kalt); - if (test_zero_jalt > 0.) + if (test_zero_jalt > 0.) jj = jalt; - else if (test_zero_kalt > 0.) + else if (test_zero_kalt > 0.) kk = kalt; - else if (test_zero_jkalt > 0.) + else if (test_zero_jkalt > 0.) { jj = jalt; kk = kalt; @@ -720,26 +720,26 @@ void eb_interp_centroid2facecent_x (Box const& ubx, Real a0_jj_kk = d1_jj_kk / (d0_jj_kk + d1_jj_kk); // coefficient of phi(i ,jj,kk,n) Real a1_jj_kk = 1.0 - a0_jj_kk; // coefficient of phi(i-1,jj,kk,n) - Real phi01 = a0 * phi(i, j, k,n) + a1 * phi(i-1, j, k,n); // interpolation in x-dir of phi + Real phi01 = a0 * phi(i, j, k,n) + a1 * phi(i-1, j, k,n); // interpolation in x-dir of phi Real y01 = a0 * ccent(i, j, k,1) + a1 * ccent(i-1, j, k,1); // interpolation in x-dir of y-loc Real z01 = a0 * ccent(i, j, k,2) + a1 * ccent(i-1, j, k,2); // interpolation in x-dir of z-loc - Real phi01_jj = a0_jj * phi(i,jj, k,n) + a1_jj * phi(i-1,jj, k,n); // interpolation in x-dir of phi + Real phi01_jj = a0_jj * phi(i,jj, k,n) + a1_jj * phi(i-1,jj, k,n); // interpolation in x-dir of phi Real y01_jj = a0_jj * ccent(i,jj, k,1) + a1_jj * ccent(i-1,jj, k,1); // interpolation in x-dir of y-loc Real z01_jj = a0_jj * ccent(i,jj, k,2) + a1_jj * ccent(i-1,jj, k,2); // interpolation in x-dir of z-loc - Real phi01_kk = a0_kk * phi(i, j,kk,n) + a1_kk * phi(i-1, j,kk,n); // interpolation in x-dir of phi + Real phi01_kk = a0_kk * phi(i, j,kk,n) + a1_kk * phi(i-1, j,kk,n); // interpolation in x-dir of phi Real y01_kk = a0_kk * ccent(i, j,kk,1) + a1_kk * ccent(i-1, j,kk,1); // interpolation in x-dir of y-loc Real z01_kk = a0_kk * ccent(i, j,kk,2) + a1_kk * ccent(i-1, j,kk,2); // interpolation in x-dir of z-loc - Real phi01_jj_kk = a0_jj_kk * phi(i,jj,kk,n) + a1_jj_kk * phi(i-1,jj,kk,n); // interpolation in x-dir of phi + Real phi01_jj_kk = a0_jj_kk * phi(i,jj,kk,n) + a1_jj_kk * phi(i-1,jj,kk,n); // interpolation in x-dir of phi Real y01_jj_kk = a0_jj_kk * ccent(i,jj,kk,1) + a1_jj_kk * ccent(i-1,jj,kk,1); // interpolation in x-dir of y-loc Real z01_jj_kk = a0_jj_kk * ccent(i,jj,kk,2) + a1_jj_kk * ccent(i-1,jj,kk,2); // interpolation in x-dir of z-loc - + // 2D interpolation on x-face from interpolated points at (y01,z01), (y01_jj,z01_jj), (y01_kk,z01_kk), (y01_jj_kk,z01_jj_kk) // to centroid of x-face(i,j,k) at (fcx(i,j,k,0), fcx(i,j,k,1)) - // We order these in order to form a set of four points on the x-face ... + // We order these in order to form a set of four points on the x-face ... // (x0,y0) : lower left // (x1,y1) : lower right // (x2,y2) : upper right @@ -759,7 +759,7 @@ void eb_interp_centroid2facecent_x (Box const& ubx, else z = 0.5 + fcx(i,j,k,1); // (j,k) is in upper half of stencil so z > 0 - if (j < jj && k < kk) // (j,k) is lower left, (j+1,k+1) is upper right + if (j < jj && k < kk) // (j,k) is lower left, (j+1,k+1) is upper right { Real y_ll = -0.5+y01; Real z_ll = -0.5+z01; @@ -772,7 +772,7 @@ void eb_interp_centroid2facecent_x (Box const& ubx, phi_x(i,j,k,n) = EB_interp_in_quad(y,z, phi01, phi01_jj, phi01_jj_kk, phi01_kk, y_ll, z_ll, y_hl, z_hl, y_hh, z_hh, y_lh, z_lh); } - else if (jj < j && k < kk) // (j-1,k) is lower left, (j,k+1) is upper right + else if (jj < j && k < kk) // (j-1,k) is lower left, (j,k+1) is upper right { Real y_ll = -0.5+y01_jj; Real z_ll = -0.5+z01_jj; @@ -785,7 +785,7 @@ void eb_interp_centroid2facecent_x (Box const& ubx, phi_x(i,j,k,n) = EB_interp_in_quad(y,z,phi01_jj,phi01,phi01_kk, phi01_jj_kk, y_ll, z_ll, y_hl, z_hl, y_hh, z_hh, y_lh, z_lh); } - else if (j < jj && kk < k) // (j,k-1) is lower left, (j+1,k) is upper right + else if (j < jj && kk < k) // (j,k-1) is lower left, (j+1,k) is upper right { Real y_ll = -0.5+y01_kk; Real z_ll = -0.5+z01_kk; @@ -798,7 +798,7 @@ void eb_interp_centroid2facecent_x (Box const& ubx, phi_x(i,j,k,n) = EB_interp_in_quad(y,z,phi01_kk,phi01_jj_kk,phi01_jj, phi01, y_ll, z_ll, y_hl, z_hl, y_hh, z_hh, y_lh, z_lh); } - else if (jj < j && kk < k) // (j-1,k-1) is lower left, (j,k) is upper right + else if (jj < j && kk < k) // (j-1,k-1) is lower left, (j,k) is upper right { Real y_ll = -0.5+y01_jj_kk; Real z_ll = -0.5+z01_jj_kk; @@ -811,14 +811,14 @@ void eb_interp_centroid2facecent_x (Box const& ubx, phi_x(i,j,k,n) = EB_interp_in_quad(y,z,phi01_jj_kk,phi01_kk,phi01, phi01_jj, y_ll, z_ll, y_hl, z_hl, y_hh, z_hh, y_lh, z_lh); } - else + else { amrex::Abort("Bad option in interpolation from cell centroid to x-face centroid!"); } } }); } - + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void eb_interp_centroid2facecent_y (Box const& vbx, Array4 const& phi, @@ -862,7 +862,7 @@ void eb_interp_centroid2facecent_y (Box const& vbx, Real a1 = d0 / (d0 + d1); // coefficient of phi(i,j-1,k,n) phi_y(i,j,k,n) = a0*phi(i,j,k,n) + a1*phi(i,j-1,k,n); // interpolation of phi in y-direction } - else + else { // We must add additional tests to avoid the case where fcy is very close to zero, but i-1/i+1 or k-1/k+1 @@ -896,12 +896,12 @@ void eb_interp_centroid2facecent_y (Box const& vbx, Real test_zero_kalt = cvol(ii ,j-1,k) * cvol(i,j-1,kalt) * cvol(ii ,j-1,kalt) * cvol(ii ,j ,k) * cvol(i,j ,kalt) * cvol(ii ,j ,kalt); - Real test_zero_ikalt = cvol(ialt,j-1,k) * cvol(i,j-1,kalt) * cvol(ialt,j-1,kalt) * + Real test_zero_ikalt = cvol(ialt,j-1,k) * cvol(i,j-1,kalt) * cvol(ialt,j-1,kalt) * cvol(ialt,j ,k) * cvol(i,j ,kalt) * cvol(ialt,j ,kalt); - if (test_zero_ialt > 0.) + if (test_zero_ialt > 0.) ii = ialt; - else if (test_zero_kalt > 0.) + else if (test_zero_kalt > 0.) kk = kalt; else if (test_zero_ikalt > 0.) { @@ -934,23 +934,23 @@ void eb_interp_centroid2facecent_y (Box const& vbx, Real a0_ii_kk = d1_ii_kk / (d0_ii_kk + d1_ii_kk); // coefficient of phi(ii,j ,kk,n) Real a1_ii_kk = d0_ii_kk / (d0_ii_kk + d1_ii_kk); // coefficient of phi(ii,j-1,kk,n) - Real phi01 = a0 * phi( i, j, k,n) + a1 * phi( i, j-1, k,n); // interpolation in y-dir of phi + Real phi01 = a0 * phi( i, j, k,n) + a1 * phi( i, j-1, k,n); // interpolation in y-dir of phi Real x01 = a0 * ccent( i, j, k,0) + a1 * ccent( i, j-1, k,0); // interpolation in y-dir of x-loc Real z01 = a0 * ccent( i, j, k,2) + a1 * ccent( i, j-1, k,2); // interpolation in y-dir of z-loc - Real phi01_ii = a0_ii * phi(ii, j, k,n) + a1_ii * phi(ii, j-1, k,n); // interpolation in y-dir of phi + Real phi01_ii = a0_ii * phi(ii, j, k,n) + a1_ii * phi(ii, j-1, k,n); // interpolation in y-dir of phi Real x01_ii = a0_ii * ccent(ii, j, k,0) + a1_ii * ccent(ii, j-1, k,0); // interpolation in y-dir of x-loc Real z01_ii = a0_ii * ccent(ii, j, k,2) + a1_ii * ccent(ii, j-1, k,2); // interpolation in y-dir of z-loc - Real phi01_kk = a0_kk * phi( i, j,kk,n) + a1_kk * phi( i, j-1,kk,n); // interpolation in y-dir of phi + Real phi01_kk = a0_kk * phi( i, j,kk,n) + a1_kk * phi( i, j-1,kk,n); // interpolation in y-dir of phi Real x01_kk = a0_kk * ccent( i, j,kk,0) + a1_kk * ccent( i, j-1,kk,0); // interpolation in y-dir of x-loc Real z01_kk = a0_kk * ccent( i, j,kk,2) + a1_kk * ccent( i, j-1,kk,2); // interpolation in y-dir of z-loc - Real phi01_ii_kk = a0_ii_kk * phi(ii, j,kk,n) + a1_ii_kk * phi(ii, j-1,kk,n); // interpolation in y-dir of phi + Real phi01_ii_kk = a0_ii_kk * phi(ii, j,kk,n) + a1_ii_kk * phi(ii, j-1,kk,n); // interpolation in y-dir of phi Real x01_ii_kk = a0_ii_kk * ccent(ii, j,kk,0) + a1_ii_kk * ccent(ii, j-1,kk,0); // interpolation in y-dir of x-loc Real z01_ii_kk = a0_ii_kk * ccent(ii, j,kk,2) + a1_ii_kk * ccent(ii, j-1,kk,2); // interpolation in y-dir of z-loc - // We order these in order to form a set of four points on the x-face ... + // We order these in order to form a set of four points on the x-face ... // (x0,y0) : lower left // (x1,y1) : lower right // (x2,y2) : upper right @@ -970,7 +970,7 @@ void eb_interp_centroid2facecent_y (Box const& vbx, else z = 0.5 + fcy(i,j,k,1); // (i,k) is in upper half of stencil so z > 0 - if (i < ii && k < kk) // (i,k) is lower left, (i+1,k+1) is upper right + if (i < ii && k < kk) // (i,k) is lower left, (i+1,k+1) is upper right { Real x_ll = -0.5+x01; Real z_ll = -0.5+z01; @@ -983,7 +983,7 @@ void eb_interp_centroid2facecent_y (Box const& vbx, phi_y(i,j,k,n) = EB_interp_in_quad(x,z, phi01, phi01_ii, phi01_ii_kk, phi01_kk, x_ll, z_ll, x_hl, z_hl, x_hh, z_hh, x_lh, z_lh); } - else if (ii < i && k < kk) // (i-1,k) is lower left, (i,k+1) is upper right + else if (ii < i && k < kk) // (i-1,k) is lower left, (i,k+1) is upper right { Real x_ll = -0.5+x01_ii; Real z_ll = -0.5+z01_ii; @@ -996,7 +996,7 @@ void eb_interp_centroid2facecent_y (Box const& vbx, phi_y(i,j,k,n) = EB_interp_in_quad(x,z,phi01_ii,phi01,phi01_kk, phi01_ii_kk, x_ll, z_ll, x_hl, z_hl, x_hh, z_hh, x_lh, z_lh); } - else if (i < ii && kk < k) // (i,k-1) is lower left, (i+1,k) is upper right + else if (i < ii && kk < k) // (i,k-1) is lower left, (i+1,k) is upper right { Real x_ll = -0.5+x01_kk; Real z_ll = -0.5+z01_kk; @@ -1009,7 +1009,7 @@ void eb_interp_centroid2facecent_y (Box const& vbx, phi_y(i,j,k,n) = EB_interp_in_quad(x,z,phi01_kk,phi01_ii_kk,phi01_ii, phi01, x_ll, z_ll, x_hl, z_hl, x_hh, z_hh, x_lh, z_lh); } - else if (ii < i && kk < k) // (i-1,k-1) is lower left, (i,k) is upper right + else if (ii < i && kk < k) // (i-1,k-1) is lower left, (i,k) is upper right { Real x_ll = -0.5+x01_ii_kk; Real z_ll = -0.5+z01_ii_kk; @@ -1022,7 +1022,7 @@ void eb_interp_centroid2facecent_y (Box const& vbx, phi_y(i,j,k,n) = EB_interp_in_quad(x,z,phi01_ii_kk,phi01_kk,phi01, phi01_ii, x_ll, z_ll, x_hl, z_hl, x_hh, z_hh, x_lh, z_lh); } - else + else { amrex::Abort("Bad option in interpolation from cell centroid to y-face centroid!"); } @@ -1073,7 +1073,7 @@ void eb_interp_centroid2facecent_z (Box const& wbx, Real a1 = d0 / (d0 + d1); // coefficient of phi(i,j,k-1,n) phi_z(i,j,k,n) = a0*phi(i,j,k,n) + a1*phi(i,j,k-1,n); // interpolation of phi in z-direction } - else + else { // We must add additional tests to avoid the case where fcz is very close to zero, but i-1/i+1 or j-1/j+1 // might be covered cells -- this can happen when the EB is exactly aligned with the grid planes @@ -1103,15 +1103,15 @@ void eb_interp_centroid2facecent_z (Box const& wbx, Real test_zero_ialt = cvol(ialt,j,k-1) * cvol(i,jj ,k-1) * cvol(ialt,jj ,k-1) * cvol(ialt,j,k ) * cvol(i,jj ,k ) * cvol(ialt,jj ,k ); - Real test_zero_jalt = cvol(ii ,j,k-1) * cvol(i,jalt,k-1) * cvol(ii ,jalt,k-1) * + Real test_zero_jalt = cvol(ii ,j,k-1) * cvol(i,jalt,k-1) * cvol(ii ,jalt,k-1) * cvol(ii ,j,k ) * cvol(i,jalt,k ) * cvol(ii ,jalt,k ); Real test_zero_ijalt = cvol(ialt,j,k-1) * cvol(i,jalt,k-1) * cvol(ialt,jalt,k-1) * cvol(ialt,j,k ) * cvol(i,jalt,k ) * cvol(ialt,jalt,k ); - if (test_zero_ialt > 0.) + if (test_zero_ialt > 0.) ii = ialt; - else if (test_zero_jalt > 0.) + else if (test_zero_jalt > 0.) jj = jalt; else if (test_zero_ijalt > 0.) { @@ -1160,11 +1160,11 @@ void eb_interp_centroid2facecent_z (Box const& wbx, Real x01_ii_jj = a0_ii_jj * ccent(ii,jj,k,0) + a1_ii_jj * ccent(ii,jj,k-1,0); // interpolation of x-loc in z-direction Real y01_ii_jj = a0_ii_jj * ccent(ii,jj,k,1) + a1_ii_jj * ccent(ii,jj,k-1,1); // interpolation of y-loc in z-direction - // We order these in order to form a set of four points on the x-face ... - // (x0,y0) : lower left + // We order these in order to form a set of four points on the x-face ... + // (x0,y0) : lower left // (x1,y1) : lower right // (x2,y2) : upper right - // (x3,y3) : upper left + // (x3,y3) : upper left Real x,y; @@ -1180,7 +1180,7 @@ void eb_interp_centroid2facecent_z (Box const& wbx, else y = 0.5 + fcz(i,j,k,1); // (i,k) is in upper half of stencil so z > 0 - if (i < ii && j < jj) // (i,j) is lower left, (i+1,j+1) is upper right + if (i < ii && j < jj) // (i,j) is lower left, (i+1,j+1) is upper right { Real x_ll = -0.5+x01; Real y_ll = -0.5+y01; @@ -1193,7 +1193,7 @@ void eb_interp_centroid2facecent_z (Box const& wbx, phi_z(i,j,k,n) = EB_interp_in_quad(x,y, phi01, phi01_ii, phi01_ii_jj, phi01_jj, x_ll, y_ll, x_hl, y_hl, x_hh, y_hh, x_lh, y_lh); } - else if (ii < i && j < jj) // (i-1,j) is lower left, (i,j+1) is upper right + else if (ii < i && j < jj) // (i-1,j) is lower left, (i,j+1) is upper right { Real x_ll = -0.5+x01_ii; Real y_ll = -0.5+y01_ii; @@ -1206,7 +1206,7 @@ void eb_interp_centroid2facecent_z (Box const& wbx, phi_z(i,j,k,n) = EB_interp_in_quad(x,y,phi01_ii,phi01,phi01_jj, phi01_ii_jj, x_ll, y_ll, x_hl, y_hl, x_hh, y_hh, x_lh, y_lh); } - else if (i < ii && jj < j) // (i,j-1) is lower left, (i+1,j) is upper right + else if (i < ii && jj < j) // (i,j-1) is lower left, (i+1,j) is upper right { Real x_ll = -0.5+x01_jj; Real y_ll = -0.5+y01_jj; @@ -1219,7 +1219,7 @@ void eb_interp_centroid2facecent_z (Box const& wbx, phi_z(i,j,k,n) = EB_interp_in_quad(x,y,phi01_jj,phi01_ii_jj,phi01_ii, phi01, x_ll, y_ll, x_hl, y_hl, x_hh, y_hh, x_lh, y_lh); } - else if (ii < i && jj < j) // (i-1,j-1) is lower left, (i,j) is upper right + else if (ii < i && jj < j) // (i-1,j-1) is lower left, (i,j) is upper right { Real x_ll = -0.5+x01_ii_jj; Real y_ll = -0.5+y01_ii_jj; @@ -1232,7 +1232,7 @@ void eb_interp_centroid2facecent_z (Box const& wbx, phi_z(i,j,k,n) = EB_interp_in_quad(x,y,phi01_ii_jj,phi01_jj,phi01, phi01_ii, x_ll, y_ll, x_hl, y_hl, x_hh, y_hh, x_lh, y_lh); } - else + else { amrex::Abort("Bad option in interpolation from cell centroid to z-face centroid!"); } @@ -1263,7 +1263,7 @@ void eb_interp_cc2face_x (Box const& ubx, else { edg_x(i,j,k,n) = 0.5 * ( phi(i,j,k,n) + phi(i-1,j,k,n) ); - } + } }); } diff --git a/Src/EB/AMReX_EBToPVD.H b/Src/EB/AMReX_EBToPVD.H index 073c9da7320..f135d16f8dd 100644 --- a/Src/EB/AMReX_EBToPVD.H +++ b/Src/EB/AMReX_EBToPVD.H @@ -16,15 +16,15 @@ class EBToPVD { public: EBToPVD(): m_grid(0) {} - void EBToPolygon(const Real* problo, const Real* dx, - const Box & bx, Array4 const& flag, - Array4 const& bcent, + void EBToPolygon(const Real* problo, const Real* dx, + const Box & bx, Array4 const& flag, + Array4 const& bcent, Array4 const& apx, Array4 const& apy, Array4 const& apz); void WriteEBVTP(const int myID) const; void WritePVTP(const int nProcs) const; - void EBGridCoverage(const int myID, const Real* problo, const Real* dx, + void EBGridCoverage(const int myID, const Real* problo, const Real* dx, const Box &bx, Array4 const& flag); private: @@ -42,7 +42,7 @@ private: const Real* dx) const; // Fills count and flags selecting the alphas which are in (0,1) - void calc_intersects(int& int_count, std::array& intersects_flags, + void calc_intersects(int& int_count, std::array& intersects_flags, const std::array& alpha) const; void print_points(std::ofstream& myfile) const; diff --git a/Src/EB/AMReX_EBToPVD.cpp b/Src/EB/AMReX_EBToPVD.cpp index 350cda42223..ee078480b18 100644 --- a/Src/EB/AMReX_EBToPVD.cpp +++ b/Src/EB/AMReX_EBToPVD.cpp @@ -10,7 +10,7 @@ #include namespace { -amrex::Real dot_product(const std::array& a, const std::array& b) +amrex::Real dot_product(const std::array& a, const std::array& b) { return a[0]*b[0] + a[1]*b[1] + a[2]*b[2]; } @@ -24,9 +24,9 @@ bool intersects(amrex::Real val) namespace amrex { -void EBToPVD::EBToPolygon(const Real* problo, const Real* dx, - const Box & bx, Array4 const& flag, - Array4 const& bcent, +void EBToPVD::EBToPolygon(const Real* problo, const Real* dx, + const Box & bx, Array4 const& flag, + Array4 const& bcent, Array4 const& apx, Array4 const& apy, Array4 const& apz) { const auto lo = lbound(bx); @@ -337,7 +337,7 @@ void EBToPVD::calc_hesse(Real& distance, std::array& n0, Real& p, void EBToPVD::calc_alpha(std::array& alpha, const std::array& n0, Real p, const std::array,8>& vertex, - const Real* dx) const + const Real* dx) const { // default (large) value std::fill(alpha.begin(), alpha.end(), 10.0); @@ -367,7 +367,7 @@ void EBToPVD::calc_alpha(std::array& alpha, } } -void EBToPVD::calc_intersects(int& int_count, std::array& intersects_flags, +void EBToPVD::calc_intersects(int& int_count, std::array& intersects_flags, const std::array& alpha) const { int_count = 0; @@ -419,7 +419,7 @@ void EBToPVD::print_connectivity(std::ofstream& myfile) const myfile << "\n"; } -void EBToPVD::EBGridCoverage(const int myID, const Real* problo, const Real* dx, +void EBToPVD::EBGridCoverage(const int myID, const Real* problo, const Real* dx, const Box &bx, Array4 const& flag) { int lc1 = 0; @@ -458,9 +458,9 @@ void EBToPVD::EBGridCoverage(const int myID, const Real* problo, const Real* dx, myfile.precision(6); myfile << "\n"; myfile << "\n"; - myfile << "\n"; - myfile << "\n"; + myfile << "\n"; myfile << "\n"; diff --git a/Src/EB/AMReX_EB_LeastSquares_2D_K.H b/Src/EB/AMReX_EB_LeastSquares_2D_K.H index 479d213e5ee..3280b78fa37 100644 --- a/Src/EB/AMReX_EB_LeastSquares_2D_K.H +++ b/Src/EB/AMReX_EB_LeastSquares_2D_K.H @@ -269,7 +269,7 @@ Real grad_x_of_phi_on_centroids(int i,int j,int k,int n, if (!flag(ii,jj,k).isRegular()) { int a_ind_eb = a_ind + 6; - + Amatrix(a_ind_eb,0) = 1.0; Amatrix(a_ind_eb,1) = x_off + bcent(ii,jj,k,0); Amatrix(a_ind_eb,2) = y_off + bcent(ii,jj,k,1) - yloc_on_xface; diff --git a/Src/EB/AMReX_EB_LeastSquares_3D_K.H b/Src/EB/AMReX_EB_LeastSquares_3D_K.H index d8d1359ce4a..976280eb893 100644 --- a/Src/EB/AMReX_EB_LeastSquares_3D_K.H +++ b/Src/EB/AMReX_EB_LeastSquares_3D_K.H @@ -27,10 +27,10 @@ void decomp_chol_np10(Array2D& aa) { sum1 = sum1 - aa(ii,kk)*aa(jj,kk); } - + if (ii == jj) { - if (sum1 <= 0.) + if (sum1 <= 0.) { p[ii] = 0.0; ising = 1; @@ -38,7 +38,7 @@ void decomp_chol_np10(Array2D& aa) p[ii] = std::sqrt(sum1); } } else { - if (ising == 0) + if (ising == 0) aa(jj,ii) = sum1 / p[ii]; else aa(jj,ii) = 0.0; @@ -53,7 +53,7 @@ void decomp_chol_np10(Array2D& aa) aa(ii,jj) = 0.0; // Zero upper triangle aa(ii,jj) = aa(jj,ii); // Zero upper triangle } - + aa(ii,ii) = p[ii]; } } @@ -314,11 +314,11 @@ void cholsol_for_eb(Array2D& Amatrix, Array1D& b) AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_x_of_phi_on_centroids(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, - Array4 const& bcent, + Array4 const& bcent, Real& yloc_on_xface, Real& zloc_on_xface, bool is_eb_dirichlet, bool is_eb_inhomog) { @@ -339,11 +339,11 @@ Real grad_x_of_phi_on_centroids(int i,int j,int k,int n, if (!flag(ii,jj,kk).isCovered()) { int a_ind = (jj-(j-1)) + 3*(kk-(k-1)) + 9*(ii-(i-1)); - + Real x_off = static_cast(ii-i) + 0.5; - Real y_off = static_cast(jj-j); - Real z_off = static_cast(kk-k); - + Real y_off = static_cast(jj-j); + Real z_off = static_cast(kk-k); + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0); Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1) - yloc_on_xface; @@ -383,23 +383,23 @@ Real grad_x_of_phi_on_centroids(int i,int j,int k,int n, rhs(irow) += Amatrix(a_ind,irow)* phi(ii,jj,kk,n); - if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog) + if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog) rhs(irow) += Amatrix(a_ind+18,irow)*phieb(ii,jj,kk,n); } } cholsol_np10(Amatrix, rhs); - return rhs(1); + return rhs(1); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_y_of_phi_on_centroids(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, - Array4 const& bcent, + Array4 const& bcent, Real& xloc_on_yface, Real& zloc_on_yface, bool is_eb_dirichlet, bool is_eb_inhomog) { @@ -421,10 +421,10 @@ Real grad_y_of_phi_on_centroids(int i,int j,int k,int n, { int a_ind = (ii-(i-1)) + 3*(kk-(k-1)) + 9*(jj-(j-1)); - Real x_off = static_cast(ii-i); + Real x_off = static_cast(ii-i); Real y_off = static_cast(jj-j) + 0.5; Real z_off = static_cast(kk-k); - + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0) - xloc_on_yface; Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1); @@ -464,23 +464,23 @@ Real grad_y_of_phi_on_centroids(int i,int j,int k,int n, rhs(irow) += Amatrix(a_ind,irow)* phi(ii,jj,kk,n); - if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog) + if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog) rhs(irow) += Amatrix(a_ind+18,irow)*phieb(ii,jj,kk,n); } } cholsol_np10(Amatrix, rhs); - return rhs(2); + return rhs(2); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_z_of_phi_on_centroids(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, - Array4 const& bcent, + Array4 const& bcent, Real& xloc_on_zface, Real& yloc_on_zface, bool is_eb_dirichlet, bool is_eb_inhomog) { @@ -504,10 +504,10 @@ Real grad_z_of_phi_on_centroids(int i,int j,int k,int n, { int a_ind = (ii-(i-1)) + 3*(jj-(j-1)) + 9*(kk-(k-1)); - Real x_off = static_cast(ii-i); - Real y_off = static_cast(jj-j); + Real x_off = static_cast(ii-i); + Real y_off = static_cast(jj-j); Real z_off = static_cast(kk-k) + 0.5; - + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0) - xloc_on_zface; Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1) - yloc_on_zface; @@ -556,13 +556,13 @@ Real grad_z_of_phi_on_centroids(int i,int j,int k,int n, cholsol_np10(Amatrix, rhs); - return rhs(3); + return rhs(3); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_eb_of_phi_on_centroids(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, Array4 const& bcent, @@ -590,7 +590,7 @@ Real grad_eb_of_phi_on_centroids(int i,int j,int k,int n, Real x_off = static_cast(ii-i) - bcent(i,j,k,0); Real y_off = static_cast(jj-j) - bcent(i,j,k,1); Real z_off = static_cast(kk-k) - bcent(i,j,k,2); - + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0); Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1); @@ -644,12 +644,12 @@ Real grad_eb_of_phi_on_centroids(int i,int j,int k,int n, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_x_of_phi_on_centroids_extdir(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, Array4 const& bcent, - Array4 const& vfrac, + Array4 const& vfrac, Real& yloc_on_xface, Real& zloc_on_xface, bool is_eb_dirichlet, bool is_eb_inhomog, const bool on_x_face, const int domlo_x, const int domhi_x, @@ -700,7 +700,7 @@ Real grad_x_of_phi_on_centroids_extdir(int i,int j,int k,int n, Real x_off = static_cast(ii-i) + 0.5; Real y_off = static_cast(jj-j); Real z_off = static_cast(kk-k); - + if(on_x_face){ if (ii < domlo_x && (vfrac(ii+1,jj,kk) != 1.0 || vfrac(ii+2,jj,kk) != 1.0) ) continue; @@ -721,7 +721,7 @@ Real grad_x_of_phi_on_centroids_extdir(int i,int j,int k,int n, if (kk > domhi_z && (vfrac(ii,jj,kk-1) != 1.0 || vfrac(ii,jj,kk-2) != 1.0) ) continue; } - + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0); Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1) - yloc_on_xface; @@ -768,7 +768,7 @@ Real grad_x_of_phi_on_centroids_extdir(int i,int j,int k,int n, rhs(irow) += Amatrix(a_ind,irow)* phi_val; - if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog && Amatrix(a_ind+27,irow) != 0.0) + if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog && Amatrix(a_ind+27,irow) != 0.0) rhs(irow) += Amatrix(a_ind+27,irow)*phieb(ii,jj,kk,n); } } @@ -780,12 +780,12 @@ Real grad_x_of_phi_on_centroids_extdir(int i,int j,int k,int n, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_y_of_phi_on_centroids_extdir(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, Array4 const& bcent, - Array4 const& vfrac, + Array4 const& vfrac, Real& xloc_on_yface, Real& zloc_on_yface, bool is_eb_dirichlet, bool is_eb_inhomog, const bool on_x_face, const int domlo_x, const int domhi_x, @@ -831,7 +831,7 @@ Real grad_y_of_phi_on_centroids_extdir(int i,int j,int k,int n, { int a_ind = (ii-(i-1)) + 3*(kk-(k-1)) + 9*(jj-(j-jm)); - Real x_off = static_cast(ii-i); + Real x_off = static_cast(ii-i); Real y_off = static_cast(jj-j) + 0.5; Real z_off = static_cast(kk-k); @@ -902,7 +902,7 @@ Real grad_y_of_phi_on_centroids_extdir(int i,int j,int k,int n, rhs(irow) += Amatrix(a_ind,irow)* phi_val; - if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog && Amatrix(a_ind+27,irow) != 0.0) + if (flag(ii,jj,kk).isSingleValued() && is_eb_inhomog && Amatrix(a_ind+27,irow) != 0.0) rhs(irow) += Amatrix(a_ind+27,irow)*phieb(ii,jj,kk,n); } } @@ -914,12 +914,12 @@ Real grad_y_of_phi_on_centroids_extdir(int i,int j,int k,int n, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_z_of_phi_on_centroids_extdir(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, Array4 const& bcent, - Array4 const& vfrac, + Array4 const& vfrac, Real& xloc_on_zface, Real& yloc_on_zface, bool is_eb_dirichlet, bool is_eb_inhomog, const bool on_x_face, const int domlo_x, const int domhi_x, @@ -966,8 +966,8 @@ Real grad_z_of_phi_on_centroids_extdir(int i,int j,int k,int n, { int a_ind = (ii-(i-1)) + 3*(jj-(j-1)) + 9*(kk-(k-km)); - Real x_off = static_cast(ii-i); - Real y_off = static_cast(jj-j); + Real x_off = static_cast(ii-i); + Real y_off = static_cast(jj-j); Real z_off = static_cast(kk-k) + 0.5; if(on_x_face){ @@ -990,7 +990,7 @@ Real grad_z_of_phi_on_centroids_extdir(int i,int j,int k,int n, if (kk > domhi_z && (vfrac(ii,jj,kk-1) != 1.0 || vfrac(ii,jj,kk-2) != 1.0) ) continue; } - + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0) - xloc_on_zface; Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1) - yloc_on_zface ; @@ -1050,8 +1050,8 @@ Real grad_z_of_phi_on_centroids_extdir(int i,int j,int k,int n, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real grad_eb_of_phi_on_centroids_extdir(int i,int j,int k,int n, - Array4 const& phi, - Array4 const& phieb, + Array4 const& phi, + Array4 const& phieb, Array4 const& flag, Array4 const& ccent, Array4 const& bcent, @@ -1122,13 +1122,13 @@ Real grad_eb_of_phi_on_centroids_extdir(int i,int j,int k,int n, if (kk > domhi_z && (vfrac(ii,jj,kk-1) != 1.0 || vfrac(ii,jj,kk-2) != 1.0) ) continue; } - + Amatrix(a_ind,0) = 1.0; Amatrix(a_ind,1) = x_off + ccent(ii,jj,kk,0); Amatrix(a_ind,2) = y_off + ccent(ii,jj,kk,1); Amatrix(a_ind,3) = z_off + ccent(ii,jj,kk,2); - if (flag(ii,jj,kk).isSingleValued() && + if (flag(ii,jj,kk).isSingleValued() && domlo_x <= ii && ii <= domhi_x && domlo_y <= jj && jj <= domhi_y && domlo_z <= kk && kk <= domhi_z) @@ -1163,7 +1163,7 @@ Real grad_eb_of_phi_on_centroids_extdir(int i,int j,int k,int n, if (!flag(ii,jj,kk).isCovered()) { int a_ind = (ii-(i-1)) + 3*(jj-(j-1)) + 9*(kk-(k-1)); - + Real phi_val = Amatrix(a_ind,0) * phi(ii,jj,kk,n); rhs(irow) += Amatrix(a_ind,irow)* phi_val; diff --git a/Src/EB/AMReX_EB_STL_utils.H b/Src/EB/AMReX_EB_STL_utils.H index 26df4cc62c2..886e7b57a84 100644 --- a/Src/EB/AMReX_EB_STL_utils.H +++ b/Src/EB/AMReX_EB_STL_utils.H @@ -25,11 +25,11 @@ namespace amrex Gpu::DeviceVector m_tri_pts_d; Gpu::DeviceVector m_tri_normals_d; - int m_num_tri=0; + int m_num_tri=0; int m_ndata_per_tri=9; //three points x 3 coordinates int m_ndata_per_normal=3; //three components int m_nlines_per_facet=7; //specific to ASCII STLs - Real m_inside = -1.0; + Real m_inside = -1.0; Real m_outside = 1.0; diff --git a/Src/EB/AMReX_EB_STL_utils.cpp b/Src/EB/AMReX_EB_STL_utils.cpp index 13f0f7fac8c..0afb97de8e1 100644 --- a/Src/EB/AMReX_EB_STL_utils.cpp +++ b/Src/EB/AMReX_EB_STL_utils.cpp @@ -8,7 +8,7 @@ namespace amrex { std::string tmpline,tmp1,tmp2; int nlines=0; - + Vector fileCharPtr; ParallelDescriptor::ReadAndBcastFile(fname, fileCharPtr); std::string fileCharPtrString(fileCharPtr.dataPtr()); @@ -65,7 +65,7 @@ namespace amrex std::getline(infile,tmpline); //vertex 2 std::istringstream vertex2(tmpline); - vertex2>>tmp1 + vertex2>>tmp1 >>m_tri_pts_h[i*m_ndata_per_tri+3] >>m_tri_pts_h[i*m_ndata_per_tri+4] >>m_tri_pts_h[i*m_ndata_per_tri+5]; @@ -81,13 +81,13 @@ namespace amrex std::getline(infile,tmpline); //end facet } - //device vectors + //device vectors m_tri_pts_d.resize(m_num_tri*m_ndata_per_tri); m_tri_normals_d.resize(m_num_tri*m_ndata_per_normal); Gpu::copy(Gpu::hostToDevice, m_tri_pts_h.begin(), m_tri_pts_h.end(), m_tri_pts_d.begin()); - Gpu::copy(Gpu::hostToDevice, + Gpu::copy(Gpu::hostToDevice, m_tri_normals_h.begin(), m_tri_normals_h.end(), m_tri_normals_d.begin()); } diff --git a/Src/EB/AMReX_EB_slopes_K.H b/Src/EB/AMReX_EB_slopes_K.H index a19d48e4bd6..389dc0fbae2 100644 --- a/Src/EB/AMReX_EB_slopes_K.H +++ b/Src/EB/AMReX_EB_slopes_K.H @@ -16,7 +16,7 @@ namespace { #if (AMREX_SPACEDIM > 1) #ifdef AMREX_USE_EB -// amrex_calc_slopes_eb calculates the slope in each coordinate direction using a +// amrex_calc_slopes_eb calculates the slope in each coordinate direction using a // least squares linear fit to the 9 in 2D / 27 in 3D nearest neighbors, with the function // going through the centroid of cell(i,j,k). This does not assume that the cell centroids, // where the data is assume to live, are the same as cell centers. Note that this routine @@ -26,7 +26,7 @@ namespace { // // All the slopes are returned in one call. -#if (AMREX_SPACEDIM == 2) +#if (AMREX_SPACEDIM == 2) AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::GpuArray amrex_calc_slopes_eb_given_A (int i, int j, int /*k*/, int n, @@ -85,7 +85,7 @@ amrex_calc_slopes_eb_given_A (int i, int j, int /*k*/, int n, amrex::Real detAtA_x = (Atb[0] *AtA[1][1]) - - (AtA[0][1]*Atb[1]); + (AtA[0][1]*Atb[1]); // Slope at centroid of (i,j,k) amrex::Real xs = detAtA_x / detAtA; @@ -115,14 +115,14 @@ amrex_calc_slopes_eb_given_A (int i, int j, int k, int n, { for(int jj(-1); jj<=1; jj++){ for(int ii(-1); ii<=1; ii++){ - + if( flag(i,j,k).isConnected(ii,jj,kk) && ! (ii==0 && jj==0 && kk==0)) { - + du[lc] = state(i+ii,j+jj,k+kk,n) - state(i,j,k,n); - + } else { - + du[lc] = 0.0; } lc++; @@ -193,7 +193,7 @@ amrex_calc_slopes_eb_given_A (int i, int j, int k, int n, } #endif -// amrex_calc_slopes_eb calculates the slope in each coordinate direction using a +// amrex_calc_slopes_eb calculates the slope in each coordinate direction using a // 1) standard 2nd order limited slope if all three cells in the stencil are regular cells // OR // 2) least squares linear fit to the at-most 9 in 2D / 27 in 3D nearest neighbors, with the function @@ -201,7 +201,7 @@ amrex_calc_slopes_eb_given_A (int i, int j, int k, int n, // where the data is assume to live, are the same as cell centers. Note that calc_slopes_eb // defines the matrix A and passes this A to amrex_calc_slopes_eb_given_A. // -// This routine assumes that there are no relevant hoextrap/extdir domain boundary conditions for this cell -- +// This routine assumes that there are no relevant hoextrap/extdir domain boundary conditions for this cell -- // it does not test for them so this should not be called if such boundaries might be present // // All the slopes are returned in one call. @@ -254,27 +254,27 @@ amrex_calc_slopes_eb (int i, int j, int k, int n, } } - // + // // These slopes use the EB stencil without testing whether it is actually needed - // + // const auto& slopes = amrex_calc_slopes_eb_given_A (i,j,k,n,A,state,flag); AMREX_D_TERM(amrex::Real xslope = slopes[0];, amrex::Real yslope = slopes[1];, amrex::Real zslope = slopes[2];); - // + // // Here we over-write -- if possible -- with a stencil not using the EB stencil - // + // // X direction - if (flag(i,j,k).isRegular() && flag(i-1,j,k).isRegular() && flag(i+1,j,k).isRegular()) + if (flag(i,j,k).isRegular() && flag(i-1,j,k).isRegular() && flag(i+1,j,k).isRegular()) { int order = 2; xslope = amrex_calc_xslope(i,j,k,n,order,state); } // Y direction - if (flag(i,j,k).isRegular() && flag(i,j-1,k).isRegular() && flag(i,j+1,k).isRegular()) + if (flag(i,j,k).isRegular() && flag(i,j-1,k).isRegular() && flag(i,j+1,k).isRegular()) { int order = 2; yslope = amrex_calc_yslope(i,j,k,n,order,state); @@ -282,17 +282,17 @@ amrex_calc_slopes_eb (int i, int j, int k, int n, #if (AMREX_SPACEDIM == 3) // Z direction - if (flag(i,j,k).isRegular() && flag(i,j,k-1).isRegular() && flag(i,j,k+1).isRegular()) + if (flag(i,j,k).isRegular() && flag(i,j,k-1).isRegular() && flag(i,j,k+1).isRegular()) { int order = 2; zslope = amrex_calc_zslope(i,j,k,n,order,state); } #endif - + return {AMREX_D_DECL(xslope,yslope,zslope)}; } -// amrex_calc_slopes_extdir_eb calculates the slope in each coordinate direction using a +// amrex_calc_slopes_extdir_eb calculates the slope in each coordinate direction using a // 1) standard limited slope if all three cells in the stencil are regular cells // (this stencil sees the extdir/hoextrap boundary condition if there is one) // OR @@ -310,12 +310,12 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, amrex::Array4 const& state, amrex::Array4 const& ccent, AMREX_D_DECL( - amrex::Array4 const& fcx, + amrex::Array4 const& fcx, amrex::Array4 const& fcy, amrex::Array4 const& fcz), amrex::Array4 const& flag, AMREX_D_DECL(bool edlo_x, bool edlo_y, bool edlo_z), - AMREX_D_DECL(bool edhi_x, bool edhi_y, bool edhi_z), + AMREX_D_DECL(bool edhi_x, bool edhi_y, bool edhi_z), AMREX_D_DECL(int domlo_x, int domlo_y, int domlo_z), AMREX_D_DECL(int domhi_x, int domhi_y, int domhi_z)) noexcept { @@ -333,7 +333,7 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, bool needs_bdry_stencil = (edlo_x && i <= domlo_x) || (edhi_x && i >= domhi_x) || (edlo_y && j <= domlo_y) || (edhi_y && j >= domhi_y); #if (AMREX_SPACEDIM == 3) - needs_bdry_stencil = needs_bdry_stencil || + needs_bdry_stencil = needs_bdry_stencil || (edlo_z && k <= domlo_z) || (edhi_z && k >= domhi_z); #endif @@ -343,7 +343,7 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, if (!needs_bdry_stencil) { // This returns slopes calculated with the regular 1-d approach if all cells in the stencil - // are regular. If not, it uses the EB-aware least squares approach to fit a linear profile + // are regular. If not, it uses the EB-aware least squares approach to fit a linear profile // using the neighboring un-covered cells. const auto& slopes = amrex_calc_slopes_eb (i,j,k,n,state,ccent,flag); return slopes; @@ -354,16 +354,16 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, int lc=0; #if (AMREX_SPACEDIM == 3) - for(int kk(-1); kk<=1; kk++) + for(int kk(-1); kk<=1; kk++) #else int kk = 0; #endif { - for(int jj(-1); jj<=1; jj++) + for(int jj(-1); jj<=1; jj++) for(int ii(-1); ii<=1; ii++) { if( flag(i,j,k).isConnected(ii,jj,kk) && - ! (ii==0 && jj==0 && kk==0)) + ! (ii==0 && jj==0 && kk==0)) { bool ilo_test = ( edlo_x && (i == domlo_x) && ii == -1); bool ihi_test = ( edhi_x && (i == domhi_x) && ii == 1); @@ -407,7 +407,7 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, } A[lc][0] = 0.5 ; } - + // Do corrections for entire y-face if (jlo_test) { @@ -421,7 +421,7 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, A[lc][1] = -0.5 ; } else if (jhi_test) { - + if (!ilo_test && !ihi_test && !klo_test && !khi_test) { A[lc][0] = ii + fcy(i+ii,j+jj,k+kk,0); @@ -450,7 +450,7 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, A[lc][1] = jj + fcz(i+ii,j+jj,k+kk,1); } A[lc][2] = 0.5 ; - } + } #endif AMREX_D_TERM(A[lc][0] -= ccent(i,j,k,0);, A[lc][1] -= ccent(i,j,k,1);, @@ -481,11 +481,11 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, if ( flag(i,j,k).isRegular() && flag(i-1,j,k).isRegular() && flag(i+1,j,k).isRegular() ) xslope = amrex_calc_xslope_extdir(i,j,k,n,order,state,edlo_x,edhi_x,domlo_x,domhi_x); - if ( flag(i,j,k).isRegular() && flag(i,j-1,k).isRegular() && flag(i,j+1,k).isRegular() ) + if ( flag(i,j,k).isRegular() && flag(i,j-1,k).isRegular() && flag(i,j+1,k).isRegular() ) yslope = amrex_calc_yslope_extdir(i,j,k,n,order,state,edlo_y,edhi_y,domlo_y,domhi_y); #if (AMREX_SPACEDIM == 3) - if ( flag(i,j,k).isRegular() && flag(i,j,k-1).isRegular() && flag(i,j,k+1).isRegular() ) + if ( flag(i,j,k).isRegular() && flag(i,j,k-1).isRegular() && flag(i,j,k+1).isRegular() ) zslope = amrex_calc_zslope_extdir(i,j,k,n,order,state,edlo_z,edhi_z,domlo_z,domhi_z); #endif } // end of needs_bndry_stencil @@ -494,19 +494,19 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, // TODO: is this the right thing to do at a HOEXTRAP boundary?? #if (AMREX_SPACEDIM == 2) if ( (edlo_x && i < domlo_x) || (edhi_x && i > domhi_x) || - (edlo_y && j < domlo_y) || (edhi_y && j > domhi_y) ) - { - AMREX_D_TERM(xslope = 0.;, - xslope = 0.; yslope = 0.;, + (edlo_y && j < domlo_y) || (edhi_y && j > domhi_y) ) + { + AMREX_D_TERM(xslope = 0.;, + xslope = 0.; yslope = 0.;, xslope = 0.; yslope = 0.; zslope = 0.;); - } + } #elif (AMREX_SPACEDIM == 3) if ( (edlo_x && i < domlo_x) || (edhi_x && i > domhi_x) || (edlo_y && j < domlo_y) || (edhi_y && j > domhi_y) || (edlo_z && k < domlo_z) || (edhi_z && k > domhi_z) ) { - AMREX_D_TERM(xslope = 0.;, - xslope = 0.; yslope = 0.;, + AMREX_D_TERM(xslope = 0.;, + xslope = 0.; yslope = 0.;, xslope = 0.; yslope = 0.; zslope = 0.;); } #endif @@ -515,10 +515,10 @@ amrex_calc_slopes_extdir_eb (int i, int j, int k, int n, } AMREX_GPU_DEVICE AMREX_FORCE_INLINE -amrex::Real +amrex::Real amrex_calc_alpha_stencil(amrex::Real q_hat, amrex::Real q_max, amrex::Real q_min, amrex::Real state, amrex::Real alpha) noexcept -{ +{ amrex::Real alpha_temp = 0.0; amrex::Real small = 1.0e-13; @@ -542,7 +542,7 @@ amrex_calc_alpha_limiter(int i, int j, int k, int n, amrex::Array4 const& fcy, amrex::Array4 const& fcz), amrex::Array4 const& ccent) noexcept -{ +{ amrex::Real alpha = 2.0; amrex::Real q_min = state(i,j,k,n); amrex::Real q_max = state(i,j,k,n); @@ -554,10 +554,10 @@ amrex_calc_alpha_limiter(int i, int j, int k, int n, //Compute max and min values in a 3x3 stencil and how many cut or regular faces do we have #if (AMREX_SPACEDIM == 3) for(int kk(-1); kk<=1; kk++) -#else +#else int kk = 0; -#endif - { +#endif + { for(int jj(-1); jj<=1; jj++){ for(int ii(-1); ii<=1; ii++){ if( flag(i,j,k).isConnected(ii,jj,kk) and @@ -708,7 +708,7 @@ amrex_calc_alpha_limiter(int i, int j, int k, int n, //is multiplied by a limiter based on the work of Barth-Jespersen. AMREX_GPU_DEVICE AMREX_FORCE_INLINE amrex::GpuArray -amrex_lim_slopes_eb (int i, int j, int k, int n, +amrex_lim_slopes_eb (int i, int j, int k, int n, amrex::Array4 const& state, amrex::Array4 const& ccent, AMREX_D_DECL(amrex::Array4 const& fcx, @@ -720,8 +720,8 @@ amrex_lim_slopes_eb (int i, int j, int k, int n, amrex::GpuArray slopes; amrex::GpuArray alpha_lim; - slopes = amrex_calc_slopes_eb(i,j,k,n,state,ccent,flag); - alpha_lim = amrex_calc_alpha_limiter(i,j,k,n,state,flag,slopes,AMREX_D_DECL(fcx,fcy,fcz),ccent); + slopes = amrex_calc_slopes_eb(i,j,k,n,state,ccent,flag); + alpha_lim = amrex_calc_alpha_limiter(i,j,k,n,state,flag,slopes,AMREX_D_DECL(fcx,fcy,fcz),ccent); // Setting limiter to 1 for stencils that just consists of non-EB cells because // amrex_calc_slopes_eb routine will call the slope routine for non-EB stencils that has already a limiter @@ -755,7 +755,7 @@ amrex_lim_slopes_extdir_eb (int i, int j, int k, int n, AMREX_D_DECL(bool edhi_x, bool edhi_y, bool edhi_z), AMREX_D_DECL(int domlo_x, int domlo_y, int domlo_z), AMREX_D_DECL(int domhi_x, int domhi_y, int domhi_z)) noexcept -{ +{ amrex::GpuArray slopes; amrex::GpuArray alpha_lim; diff --git a/Src/EB/AMReX_EB_triGeomOps_K.H b/Src/EB/AMReX_EB_triGeomOps_K.H index 3d83fa4fafc..f0a0e8663ed 100644 --- a/Src/EB/AMReX_EB_triGeomOps_K.H +++ b/Src/EB/AMReX_EB_triGeomOps_K.H @@ -22,11 +22,11 @@ namespace amrex //================================================================================ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE Real side_op(Real L1[6],Real L2[6]) { - return( L1[0]*L2[4] - + L1[1]*L2[5] - + L1[2]*L2[3] - + L1[3]*L2[2] - + L1[4]*L2[0] + return( L1[0]*L2[4] + + L1[1]*L2[5] + + L1[2]*L2[3] + + L1[3]*L2[2] + + L1[4]*L2[0] + L1[5]*L2[1] ); } //================================================================================ @@ -178,8 +178,8 @@ namespace amrex p1[1]=midp[1]; p1[2]=midp[2]; } - else //plane_eq_mid is 0 - //or error: p1,midp and p2 are on the same side + else //plane_eq_mid is 0 + //or error: p1,midp and p2 are on the same side //which is not what this function is meant for { if(plane_eq_mid*plane_eq1 > 0.0 && plane_eq_mid*plane_eq2 > 0.0) @@ -263,7 +263,7 @@ namespace amrex (Math::abs(S2) < eps && Math::abs(S3) < eps) ) //S2,S3=0 { - //Print()<<"vertex intersection type 1\n"; + //Print()<<"vertex intersection type 1\n"; //don't chose vertex 2 or 3 get_plucker_coords(v2,t1,L2); get_plucker_coords(t1,v1,L3); diff --git a/Src/EB/AMReX_MultiCutFab.H b/Src/EB/AMReX_MultiCutFab.H index 056c71927d6..f54d9b7537c 100644 --- a/Src/EB/AMReX_MultiCutFab.H +++ b/Src/EB/AMReX_MultiCutFab.H @@ -88,7 +88,7 @@ public: ~MultiCutFab (); MultiCutFab (MultiCutFab&& rhs) noexcept = default; - + MultiCutFab (const MultiCutFab& rhs) = delete; MultiCutFab& operator= (const MultiCutFab& rhs) = delete; MultiCutFab& operator= (MultiCutFab&& rhs) = delete; diff --git a/Src/EB/AMReX_WriteEBSurface.H b/Src/EB/AMReX_WriteEBSurface.H index b43931b12c0..749871211ac 100644 --- a/Src/EB/AMReX_WriteEBSurface.H +++ b/Src/EB/AMReX_WriteEBSurface.H @@ -10,7 +10,7 @@ namespace amrex { class EBFArrayBoxFactory; void WriteEBSurface (const amrex::BoxArray & ba, const amrex::DistributionMapping & dmap, const amrex::Geometry & geom, - const amrex::EBFArrayBoxFactory * ebf); + const amrex::EBFArrayBoxFactory * ebf); } diff --git a/Src/EB/AMReX_WriteEBSurface.cpp b/Src/EB/AMReX_WriteEBSurface.cpp index 9bb8220a890..c6cf071621c 100644 --- a/Src/EB/AMReX_WriteEBSurface.cpp +++ b/Src/EB/AMReX_WriteEBSurface.cpp @@ -37,11 +37,11 @@ void WriteEBSurface (const BoxArray & ba, const DistributionMapping & dmap, cons bndrycent = &(ebf->getBndryCent()); eb_to_pvd.EBToPolygon( - problo, dx, - bx, my_flag.const_array(), - bndrycent->const_array(mfi), - areafrac[0]->const_array(mfi), - areafrac[1]->const_array(mfi), + problo, dx, + bx, my_flag.const_array(), + bndrycent->const_array(mfi), + areafrac[0]->const_array(mfi), + areafrac[1]->const_array(mfi), areafrac[2]->const_array(mfi)); } diff --git a/Src/EB/AMReX_ebcellflag_mod.F90 b/Src/EB/AMReX_ebcellflag_mod.F90 index 268aece2dc6..b5c18daba7a 100644 --- a/Src/EB/AMReX_ebcellflag_mod.F90 +++ b/Src/EB/AMReX_ebcellflag_mod.F90 @@ -8,7 +8,7 @@ module amrex_ebcellflag_module set_regular_cell, set_covered_cell, set_single_valued_cell, set_neighbor, clear_neighbor, & clear_allneighbors, is_neighbor, get_neighbor_cells_int_single, & regular, single_valued, multi_valued, covered - + integer, parameter :: w_type = 2 integer, parameter :: w_numvofs = 3 integer, parameter :: pos_numvofs = 2; @@ -39,7 +39,7 @@ elemental function get_neighbor_cells_int_single (flag,i,j,k) get_neighbor_cells_int_single = 0 end if end function get_neighbor_cells_int_single - + elemental logical function is_regular_cell (flag) integer, intent(in) :: flag is_regular_cell = ibits(flag,0,w_type) .eq. regular diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H index f8fd48a080e..c238c843d0a 100644 --- a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H +++ b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H @@ -194,7 +194,7 @@ ParticleTileToBlueprint(const ParticleTile -void +void ParticleContainerToBlueprint(const ParticleContainer; // @@ -230,7 +230,7 @@ ParticleContainerToBlueprint(const ParticleContainer my_lvl_offsets; my_lvl_offsets.resize(num_levels); @@ -242,7 +242,7 @@ ParticleContainerToBlueprint(const ParticleContainer& beta); void setVerbose (int _verbose); void setIsMatrixSingular(bool flag) { is_matrix_singular = flag; } - virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, + virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, int max_iter, const BndryData& bndry, int max_bndry_order) = 0; static HYPRE_Int ispow2 (HYPRE_Int i) @@ -67,7 +67,7 @@ public: static constexpr HYPRE_Int eb_stencil_size = AMREX_D_TERM(3, *3, *3); protected: - + MPI_Comm comm = MPI_COMM_NULL; Geometry geom; @@ -85,7 +85,7 @@ protected: Real scalar_a, scalar_b; MultiFab diaginv; - + FabFactory const* m_factory = nullptr; BndryData const* m_bndry = nullptr; int m_maxorder = -1; diff --git a/Src/Extern/HYPRE/AMReX_Hypre.cpp b/Src/Extern/HYPRE/AMReX_Hypre.cpp index aed7d3b1b02..a904842112b 100644 --- a/Src/Extern/HYPRE/AMReX_Hypre.cpp +++ b/Src/Extern/HYPRE/AMReX_Hypre.cpp @@ -24,7 +24,7 @@ makeHypre (const BoxArray& grids, const DistributionMapping& dmap, return std::unique_ptr(new HypreABecLap2(grids, dmap, geom, comm_)); } else { return std::unique_ptr(new HypreABecLap3(grids, dmap, geom, comm_)); - } + } } Hypre::Hypre (const BoxArray& grids, const DistributionMapping& dmap, @@ -45,11 +45,11 @@ Hypre::Hypre (const BoxArray& grids, const DistributionMapping& dmap, int ngrow = 0; acoefs.define(grids, dmap, ncomp, ngrow); acoefs.setVal(0.0); - + #ifdef AMREX_USE_EB ngrow = 1; #endif - + for (int i = 0; i < AMREX_SPACEDIM; ++i) { BoxArray edge_boxes(grids); edge_boxes.surroundingNodes(i); diff --git a/Src/Extern/HYPRE/AMReX_HypreABecLap.H b/Src/Extern/HYPRE/AMReX_HypreABecLap.H index 7d0c91c9436..f95b085dbcb 100644 --- a/Src/Extern/HYPRE/AMReX_HypreABecLap.H +++ b/Src/Extern/HYPRE/AMReX_HypreABecLap.H @@ -26,10 +26,10 @@ public: HypreABecLap (const BoxArray& grids, const DistributionMapping& dmap, const Geometry& geom_, MPI_Comm comm_); virtual ~HypreABecLap (); - - virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, + + virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, int max_iter, const BndryData& bndry, int max_bndry_order) final; - + private: HYPRE_StructGrid grid = NULL; diff --git a/Src/Extern/HYPRE/AMReX_HypreABecLap.cpp b/Src/Extern/HYPRE/AMReX_HypreABecLap.cpp index 29a9eefecae..e492ff8fd70 100644 --- a/Src/Extern/HYPRE/AMReX_HypreABecLap.cpp +++ b/Src/Extern/HYPRE/AMReX_HypreABecLap.cpp @@ -54,8 +54,8 @@ HypreABecLap::solve(MultiFab& soln, const MultiFab& rhs, Real reltol, Real absto // loadVectors(soln, rhs); // - HYPRE_StructVectorAssemble(x); - HYPRE_StructVectorAssemble(b); + HYPRE_StructVectorAssemble(x); + HYPRE_StructVectorAssemble(b); HYPRE_StructPFMGSetMaxIter(solver, maxiter); HYPRE_StructPFMGSetTol(solver, reltol); @@ -83,7 +83,7 @@ HypreABecLap::solve(MultiFab& soln, const MultiFab& rhs, Real reltol, Real absto Real res; HYPRE_StructPFMGGetNumIterations(solver, &num_iterations); HYPRE_StructPFMGGetFinalRelativeResidualNorm(solver, &res); - + amrex::Print() << "\n" << num_iterations << " Hypre PFMG Iterations, Relative Residual " << res << std::endl; @@ -176,7 +176,7 @@ HypreABecLap::prepareSolver () HYPRE_StructMatrixCreate(comm, grid, stencil, &A); HYPRE_StructMatrixInitialize(A); - HYPRE_StructStencilDestroy(stencil); + HYPRE_StructStencilDestroy(stencil); HYPRE_StructVectorCreate(comm, grid, &b); HYPRE_StructVectorCreate(comm, grid, &x); diff --git a/Src/Extern/HYPRE/AMReX_HypreABecLap2.H b/Src/Extern/HYPRE/AMReX_HypreABecLap2.H index d0705b364b9..750d1529dd3 100644 --- a/Src/Extern/HYPRE/AMReX_HypreABecLap2.H +++ b/Src/Extern/HYPRE/AMReX_HypreABecLap2.H @@ -19,7 +19,7 @@ public: const Geometry& geom_, MPI_Comm comm_); virtual ~HypreABecLap2 (); - virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, + virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, int max_iter, const BndryData& bndry, int max_bndry_order) final; private : diff --git a/Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp b/Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp index a6af1d6ff68..ab6e26b2818 100644 --- a/Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp +++ b/Src/Extern/HYPRE/AMReX_HypreABecLap2.cpp @@ -35,7 +35,7 @@ HypreABecLap2::~HypreABecLap2 () } void -HypreABecLap2::solve (MultiFab& soln, const MultiFab& rhs, Real reltol, Real abstol, +HypreABecLap2::solve (MultiFab& soln, const MultiFab& rhs, Real reltol, Real abstol, int maxiter, const BndryData& bndry, int max_bndry_order) { if (solver == NULL || m_bndry != &bndry || m_maxorder != max_bndry_order) @@ -87,7 +87,7 @@ HypreABecLap2::solve (MultiFab& soln, const MultiFab& rhs, Real reltol, Real abs HYPRE_ParCSRMatrix par_A; HYPRE_ParVector par_b; HYPRE_ParVector par_x; - + HYPRE_SStructMatrixGetObject(A, (void**) &par_A); HYPRE_SStructVectorGetObject(b, (void**) &par_b); HYPRE_SStructVectorGetObject(x, (void**) &par_x); @@ -262,8 +262,8 @@ HypreABecLap2::prepareSolver () 0, regular_stencil_size, stencil_indices.data(), mat); Gpu::synchronize(); - } - HYPRE_SStructMatrixAssemble(A); + } + HYPRE_SStructMatrixAssemble(A); // create solver HYPRE_BoomerAMGCreate(&solver); diff --git a/Src/Extern/HYPRE/AMReX_HypreABecLap3.H b/Src/Extern/HYPRE/AMReX_HypreABecLap3.H index cc103d8dab9..a22955caadf 100644 --- a/Src/Extern/HYPRE/AMReX_HypreABecLap3.H +++ b/Src/Extern/HYPRE/AMReX_HypreABecLap3.H @@ -23,7 +23,7 @@ public: const iMultiFab* overset_mask = nullptr); virtual ~HypreABecLap3 (); - virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, + virtual void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, int max_iter, const BndryData& bndry, int max_bndry_order) final; #ifdef AMREX_USE_EB diff --git a/Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp b/Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp index d6dfd89cf25..18cc0f0dd7b 100644 --- a/Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp +++ b/Src/Extern/HYPRE/AMReX_HypreABecLap3.cpp @@ -83,19 +83,19 @@ HypreABecLap3::getSolution (MultiFab& a_soln) MultiFab::Copy(a_soln, tmp, 0, 0, 1, 0); } } - + void HypreABecLap3::prepareSolver () { BL_PROFILE("HypreABecLap3::prepareSolver()"); - + int num_procs, myid; MPI_Comm_size(comm, &num_procs); MPI_Comm_rank(comm, &myid); const BoxArray& ba = acoefs.boxArray(); const DistributionMapping& dm = acoefs.DistributionMap(); - + #if defined(AMREX_DEBUG) || defined(AMREX_TESTING) if (sizeof(HYPRE_Int) < sizeof(Long)) { Long ncells_grids = ba.numPts(); @@ -221,7 +221,7 @@ HypreABecLap3::prepareSolver () for (MFIter mfi(acoefs); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); - + #ifdef AMREX_USE_EB const auto fabtyp = (flags) ? (*flags)[mfi].getType(bx) : FabType::regular; #else @@ -407,7 +407,7 @@ void HypreABecLap3::loadVectors (MultiFab& soln, const MultiFab& rhs) { BL_PROFILE("HypreABecLap3::loadVectors()"); - + #ifdef AMREX_USE_EB auto ebfactory = dynamic_cast(m_factory); const FabArray* flags = (ebfactory) ? &(ebfactory->getMultiEBCellFlagFab()) : nullptr; @@ -487,5 +487,5 @@ HypreABecLap3::loadVectors (MultiFab& soln, const MultiFab& rhs) } } } - + } // namespace amrex diff --git a/Src/Extern/PETSc/AMReX_PETSc.H b/Src/Extern/PETSc/AMReX_PETSc.H index 95c575b9fb8..552e019d05f 100644 --- a/Src/Extern/PETSc/AMReX_PETSc.H +++ b/Src/Extern/PETSc/AMReX_PETSc.H @@ -12,11 +12,11 @@ namespace amrex { -class PETScABecLap +class PETScABecLap { public: - + PETScABecLap (const BoxArray& grids, const DistributionMapping& dmap, const Geometry& geom_, MPI_Comm comm_); @@ -26,7 +26,7 @@ public: void setACoeffs (const MultiFab& alpha); void setBCoeffs (const Array& beta); void setVerbose (int _verbose); - void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, + void solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real abs_tol, int max_iter, const BndryData& bndry, int max_bndry_order); #ifdef AMREX_USE_EB @@ -37,7 +37,7 @@ private: static constexpr PetscInt regular_stencil_size = 2*AMREX_SPACEDIM + 1; static constexpr PetscInt eb_stencil_size = AMREX_D_TERM(3, *3, *3); - + MPI_Comm comm = MPI_COMM_NULL; Geometry geom; @@ -48,7 +48,7 @@ private: Real scalar_a, scalar_b; MultiFab diaginv; - + FabFactory const* m_factory = nullptr; BndryData const* m_bndry = nullptr; int m_maxorder = -1; diff --git a/Src/Extern/PETSc/AMReX_PETSc.cpp b/Src/Extern/PETSc/AMReX_PETSc.cpp index 394055782a2..7448f808bb6 100644 --- a/Src/Extern/PETSc/AMReX_PETSc.cpp +++ b/Src/Extern/PETSc/AMReX_PETSc.cpp @@ -41,11 +41,11 @@ PETScABecLap::PETScABecLap (const BoxArray& grids, const DistributionMapping& dm int ngrow = 0; acoefs.define(grids, dmap, ncomp, ngrow); acoefs.setVal(0.0); - + #ifdef AMREX_USE_EB ngrow = 1; #endif - + for (int i = 0; i < AMREX_SPACEDIM; ++i) { BoxArray edge_boxes(grids); edge_boxes.surroundingNodes(i); @@ -76,7 +76,7 @@ PETScABecLap::~PETScABecLap () m_factory = nullptr; m_bndry = nullptr; m_maxorder = -1; - + PetscFinalize(); } @@ -128,11 +128,11 @@ PETScABecLap::solve (MultiFab& soln, const MultiFab& rhs, Real rel_tol, Real /*a loadVectors(soln, rhs); // - VecAssemblyBegin(x); - VecAssemblyEnd(x); + VecAssemblyBegin(x); + VecAssemblyEnd(x); // - VecAssemblyBegin(b); - VecAssemblyEnd(b); + VecAssemblyBegin(b); + VecAssemblyEnd(b); KSPSetTolerances(solver, rel_tol, PETSC_DEFAULT, PETSC_DEFAULT, max_iter); KSPSolve(solver, b, x); if (verbose >= 2) @@ -156,7 +156,7 @@ PETScABecLap::prepareSolver () const BoxArray& ba = acoefs.boxArray(); const DistributionMapping& dm = acoefs.DistributionMap(); - + #if defined(AMREX_DEBUG) || defined(AMREX_TESTING) if (sizeof(PetscInt) < sizeof(Long)) { Long ncells_grids = ba.numPts(); @@ -258,7 +258,7 @@ PETScABecLap::prepareSolver () cid_arr(i,j,k) += os; cid_vec(i,j,k) = cid_arr(i,j,k); }); - } + } cell_id.FillBoundary(geom.periodicity()); @@ -266,14 +266,14 @@ PETScABecLap::prepareSolver () PetscInt d_nz = (eb_stencil_size + regular_stencil_size) / 2; // estimated amount of block off diag elements PetscInt o_nz = d_nz / 2; - MatCreate(PETSC_COMM_WORLD, &A); + MatCreate(PETSC_COMM_WORLD, &A); MatSetType(A, MATMPIAIJ); - MatSetSizes(A, ncells_proc, ncells_proc, ncells_world, ncells_world); + MatSetSizes(A, ncells_proc, ncells_proc, ncells_world, ncells_world); MatMPIAIJSetPreallocation(A, d_nz, NULL, o_nz, NULL ); //Maybe an over estimate of the diag/off diag #of non-zero entries, so we turn off malloc warnings - MatSetUp(A); - MatSetOption(A, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE); - + MatSetUp(A); + MatSetOption(A, MAT_NEW_NONZERO_LOCATION_ERR, PETSC_FALSE); + // A.SetValues const auto dx = geom.CellSizeArray(); const int bho = (m_maxorder > 2) ? 1 : 0; @@ -285,7 +285,7 @@ PETScABecLap::prepareSolver () for (MFIter mfi(acoefs); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); - + #ifdef AMREX_USE_EB const auto fabtyp = (flags) ? (*flags)[mfi].getType(bx) : FabType::regular; #else @@ -321,7 +321,7 @@ PETScABecLap::prepareSolver () Real sa = scalar_a; Real sb = scalar_b; - + if (fabtyp == FabType::regular) { constexpr int stencil_size = 2*AMREX_SPACEDIM+1; @@ -440,8 +440,8 @@ PETScABecLap::prepareSolver () Gpu::synchronize(); - //Load in by row! - int matid = 0; + //Load in by row! + int matid = 0; for (int rit = 0; rit < nrows; ++rit) { MatSetValues(A, 1, &rows[rit], ncols[rit], &cols[matid], &mat[matid], INSERT_VALUES); @@ -463,10 +463,10 @@ PETScABecLap::prepareSolver () // Classic AMG PCSetType(pc, PCGAMG); PCGAMGSetType(pc, PCGAMGAGG); - PCGAMGSetNSmooths(pc,0); -// PCSetType(pc, PCJACOBI); + PCGAMGSetNSmooths(pc,0); +// PCSetType(pc, PCJACOBI); + - // we are not using command line options KSPSetFromOptions(solver); // create b & x VecCreateMPI(PETSC_COMM_WORLD, ncells_proc, ncells_world, &x); diff --git a/Src/Extern/ProfParser/AMReX_AVGDOWN_1D.F b/Src/Extern/ProfParser/AMReX_AVGDOWN_1D.F index bd72fa40fcc..2e91364a729 100644 --- a/Src/Extern/ProfParser/AMReX_AVGDOWN_1D.F +++ b/Src/Extern/ProfParser/AMReX_AVGDOWN_1D.F @@ -19,9 +19,9 @@ subroutine FORT_CV_AVGDOWN ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -61,9 +61,9 @@ subroutine FORT_CV_AVGDOWN ( do ic = lo(1), hi(1) crse(ic,n) = crse(ic,n) * vol_inv end do - + end do - + end subroutine FORT_CV_AVGDOWN_STAG ( @@ -74,9 +74,9 @@ subroutine FORT_CV_AVGDOWN_STAG ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -129,9 +129,9 @@ subroutine FORT_CV_AVGDOWN_STAG ( end do end if - + end do - + end subroutine FORT_AVGDOWN ( @@ -143,9 +143,9 @@ subroutine FORT_AVGDOWN ( c ---------------------------------------------------------- c Volume-weight average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c cv = coarse grid volume array c fv = fine grid volume array diff --git a/Src/Extern/ProfParser/AMReX_AVGDOWN_2D.F b/Src/Extern/ProfParser/AMReX_AVGDOWN_2D.F index 69938bcfbc8..7bf28b49bcd 100644 --- a/Src/Extern/ProfParser/AMReX_AVGDOWN_2D.F +++ b/Src/Extern/ProfParser/AMReX_AVGDOWN_2D.F @@ -19,9 +19,9 @@ subroutine FORT_MAXVAL_AVGDOWN ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -66,7 +66,7 @@ subroutine FORT_MAXVAL_AVGDOWN ( end do end do - + end @@ -77,9 +77,9 @@ subroutine FORT_CV_AVGDOWN ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -130,9 +130,9 @@ subroutine FORT_CV_AVGDOWN ( crse(ic,jc,n) = crse(ic,jc,n) * vol_inv end do end do - + end do - + end @@ -145,9 +145,9 @@ subroutine FORT_CV_AVGDOWN_STAG ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -214,9 +214,9 @@ subroutine FORT_CV_AVGDOWN_STAG ( end do end if - + end do - + end subroutine FORT_AVGDOWN ( @@ -228,9 +228,9 @@ subroutine FORT_AVGDOWN ( c ---------------------------------------------------------- c Volume-weight average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c cv = coarse grid volume array c fv = fine grid volume array diff --git a/Src/Extern/ProfParser/AMReX_AVGDOWN_3D.F b/Src/Extern/ProfParser/AMReX_AVGDOWN_3D.F index 111074f3674..babb3729799 100644 --- a/Src/Extern/ProfParser/AMReX_AVGDOWN_3D.F +++ b/Src/Extern/ProfParser/AMReX_AVGDOWN_3D.F @@ -20,9 +20,9 @@ subroutine FORT_CV_AVGDOWN ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -57,7 +57,7 @@ subroutine FORT_CV_AVGDOWN ( end do end do end do -c +c c sum fine data c do koff = 0, lratz-1 @@ -87,7 +87,7 @@ subroutine FORT_CV_AVGDOWN ( end do end do - + end subroutine FORT_CV_AVGDOWN_STAG ( & nodal_dir, @@ -98,9 +98,9 @@ subroutine FORT_CV_AVGDOWN_STAG ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -143,7 +143,7 @@ subroutine FORT_CV_AVGDOWN_STAG ( end do end do end do -c +c c sum fine data c @@ -195,7 +195,7 @@ subroutine FORT_CV_AVGDOWN_STAG ( end if end do - + end @@ -209,9 +209,9 @@ subroutine FORT_AVGDOWN ( c ---------------------------------------------------------- c Volume-weight average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c cv = coarse grid volume array c fv = fine grid volume array diff --git a/Src/Extern/ProfParser/AMReX_AVGDOWN_F.H b/Src/Extern/ProfParser/AMReX_AVGDOWN_F.H index 42836cf3955..ef3f3d6b761 100644 --- a/Src/Extern/ProfParser/AMReX_AVGDOWN_F.H +++ b/Src/Extern/ProfParser/AMReX_AVGDOWN_F.H @@ -25,31 +25,31 @@ using amrex::Real; extern "C" { - + void FORT_MAXVAL_AVGDOWN (const Real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - const int* nvar, - const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), - const int* lo, const int* hi, const int* ratio); + const int* nvar, + const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), + const int* lo, const int* hi, const int* ratio); void FORT_CV_AVGDOWN (const Real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - const int* nvar, - const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), - const int* lo, const int* hi, const int* ratio); - - void FORT_CV_AVGDOWN_STAG (const int* nodal_dir, - const Real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - const int* nvar, - const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), - const int* lo, const int* hi, const int* ratio); - + const int* nvar, + const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), + const int* lo, const int* hi, const int* ratio); + + void FORT_CV_AVGDOWN_STAG (const int* nodal_dir, + const Real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), + const int* nvar, + const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), + const int* lo, const int* hi, const int* ratio); + void FORT_AVGDOWN (const Real* crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), const int* nvar, const Real* fine, AMREX_ARLIM_P(fine_lo), AMREX_ARLIM_P(fine_hi), - const Real* cvol, AMREX_ARLIM_P(cv_lo), AMREX_ARLIM_P(cv_hi), + const Real* cvol, AMREX_ARLIM_P(cv_lo), AMREX_ARLIM_P(cv_hi), const Real* fvol, AMREX_ARLIM_P(fv_lo), AMREX_ARLIM_P(fv_hi), const int* lo, const int* hi, const int* ratio); - + } - + #endif #endif /*_AVGDOWN_F_H_*/ diff --git a/Src/Extern/ProfParser/AMReX_BLProfStats.H b/Src/Extern/ProfParser/AMReX_BLProfStats.H index 5113b981f39..7f4b9d4b25e 100644 --- a/Src/Extern/ProfParser/AMReX_BLProfStats.H +++ b/Src/Extern/ProfParser/AMReX_BLProfStats.H @@ -173,8 +173,8 @@ class BLProfStats { virtual void SetCSSize(const int /*css*/) { } virtual void InitCommDataBlock(const int /*proc*/, const long /*ncommstats*/, const std::string &/*filename*/, const long /*seekpos*/, - const std::string &/*nodename*/ = "", - const int /*nodenumber*/ = -1) { } + const std::string &/*nodename*/ = "", + const int /*nodenumber*/ = -1) { } virtual void AddBarrier(long /*bnum*/, const std::string &/*bname*/, long /*index*/) { } virtual void AddReduction(const long /*rnum*/, const long /*index*/) { } virtual void AddTimeMinMax(const double /*tmin*/, const double /*tmax*/) { } @@ -208,8 +208,8 @@ class BLProfStats { virtual void AddProbDomain(const int /*lev*/, const amrex::Box &/*pd*/) { } virtual TimeRange MakeRegionPlt(amrex::FArrayBox &/*rFab*/, int /*noregionnumber*/, int /*width*/, int /*height*/, - amrex::Vector> &/*regionBoxes*/) - { return TimeRange(); } + amrex::Vector> &/*regionBoxes*/) + { return TimeRange(); } static void OpenAllStreams(const std::string &dirname); static void CloseAllStreams(); diff --git a/Src/Extern/ProfParser/AMReX_BLProfStats.cpp b/Src/Extern/ProfParser/AMReX_BLProfStats.cpp index b8badf54d11..b2ca8fc4209 100644 --- a/Src/Extern/ProfParser/AMReX_BLProfStats.cpp +++ b/Src/Extern/ProfParser/AMReX_BLProfStats.cpp @@ -59,8 +59,8 @@ extern void PrintTimeRangeList(const std::list &trL extern long amrex::FileSize(const std::string &filename); extern void amrex::MakeFuncPctTimesMF(const Vector > &funcStats, const Vector &blpFNames, - const std::map &mProfStats, - Real runTime, int dataNProcs); + const std::map &mProfStats, + Real runTime, int dataNProcs); extern void amrex::CollectMProfStats(std::map &mProfStats, const Vector > &funcStats, const Vector &fNames, @@ -91,7 +91,7 @@ bool BLProfStats::AddPiece(std::list &addToHere, if(tRange.stopTime >= tRangeNext.startTime) { tRangeNext.startTime = tRange.startTime; tRangeNext.stopTime = std::max(tRange.stopTime, tRangeNext.stopTime); - overlaps.push_back(it); + overlaps.push_back(it); } } } @@ -127,7 +127,7 @@ std::list BLProfStats::RangeIntersection( bool BLProfStats::RemovePiece(std::list &removeFromHere, const TimeRange &pieceToRemove) { - + bool piecesRemoved(false); std::list::iterator it; @@ -169,7 +169,7 @@ bool BLProfStats::RemovePiece(std::list &removeFromHere, } else if(bothHigh) { // ---- do nothing } - + } std::list::iterator >::iterator eit; for(eit = eraseThese.begin(); eit != eraseThese.end(); ++eit) { @@ -295,7 +295,7 @@ void BLProfStats::InitFilterTimeRanges() { for(int p(0); p < regionTimeRanges.size(); ++p) { for(int r(0); r < regionTimeRanges[p][regNum].size(); ++r) { TimeRange &trange = regionTimeRanges[p][regNum][r]; - //cout << "_here include: " << trange << endl; + //cout << "_here include: " << trange << endl; AddPiece(filterTimeRanges[p], trange); } } @@ -306,7 +306,7 @@ void BLProfStats::InitFilterTimeRanges() { for(int p(0); p < regionTimeRanges.size(); ++p) { for(int r(0); r < regionTimeRanges[p][regNum].size(); ++r) { TimeRange &trange = regionTimeRanges[p][regNum][r]; - //cout << "_here exclude: " << trange << endl; + //cout << "_here exclude: " << trange << endl; RemovePiece(filterTimeRanges[p], trange); } } @@ -428,7 +428,7 @@ void BLProfStats::CheckData() { long diff(-1); std::map::iterator it; for(it = maxSeekPerFile.begin(); it != maxSeekPerFile.end(); ++it) { - diff = it->second + dataSize - FileSize(it->first); + diff = it->second + dataSize - FileSize(it->first); if(diff != 0) { bad = true; cout << "fName maxSeek fSize diff = " << it->first << " " << it->second << " " diff --git a/Src/Extern/ProfParser/AMReX_BLProfUtilities.H b/Src/Extern/ProfParser/AMReX_BLProfUtilities.H index 5fb3eb1dfb3..3fd72b02f04 100644 --- a/Src/Extern/ProfParser/AMReX_BLProfUtilities.H +++ b/Src/Extern/ProfParser/AMReX_BLProfUtilities.H @@ -40,17 +40,17 @@ namespace amrex{ // -------------------------------------------------------------------- -std::string SanitizeName(const std::string &sname); +std::string SanitizeName(const std::string &sname); void SimpleRemoveOverlap(BoxArray &ba); void avgDown_doit(const FArrayBox &fine_fab, FArrayBox &crse_fab, const Box &ovlp, int scomp, int dcomp, int ncomp, - Vector &ratio); + Vector &ratio); Box FixCoarseBoxSize(const Box &fineBox, int rr); void avgDown(MultiFab &S_crse, MultiFab &S_fine, int scomp, int dcomp, int ncomp, Vector &ratio); void PrintTimeRangeList(const std::list &trList); void RedistFiles(); -int NHops(const Box &tbox, const IntVect &ivfrom, const IntVect &ivto); +int NHops(const Box &tbox, const IntVect &ivfrom, const IntVect &ivto); void Write2DFab(const string &filenameprefix, const int xdim, const int ydim, const double *data); void Write2DText(const string &filenameprefix, const int xdim, const int ydim, @@ -62,16 +62,16 @@ void WriteFab(const string &filenameprefix, const int xdim, const int ydim, long FileSize(const std::string &filename); void MakeFuncPctTimesMF(const Vector > &funcStats, const Vector &blpFNames, - const std::map &mProfStats, - Real runTime, int dataNProcs); + const std::map &mProfStats, + Real runTime, int dataNProcs); void CollectMProfStats(std::map &mProfStats, const Vector > &funcStats, const Vector &fNames, - Real runTime, int whichProc); + Real runTime, int whichProc); void GraphTopPct(const std::map &mProfStats, const Vector > &funcStats, const Vector &fNames, - Real runTime, int dataNProcs, Real gPercent); + Real runTime, int dataNProcs, Real gPercent); } // ---------------------------------------------------------------------- // ---------------------------------------------------------------------- diff --git a/Src/Extern/ProfParser/AMReX_BLProfUtilities.cpp b/Src/Extern/ProfParser/AMReX_BLProfUtilities.cpp index ed9e0b0278d..1265f86f3b3 100644 --- a/Src/Extern/ProfParser/AMReX_BLProfUtilities.cpp +++ b/Src/Extern/ProfParser/AMReX_BLProfUtilities.cpp @@ -74,10 +74,10 @@ void amrex::SimpleRemoveOverlap(BoxArray &ba) { for(int n(0); n < ba.size(); ++n) { if(i != n) { const Box &bn = ba[n]; - if(b.bigEnd(dim) == bn.smallEnd(dim)) { - b.growHi(dim, - 1); - ba.set(i, b); - } + if(b.bigEnd(dim) == bn.smallEnd(dim)) { + b.growHi(dim, - 1); + ba.set(i, b); + } } } } @@ -88,7 +88,7 @@ void amrex::SimpleRemoveOverlap(BoxArray &ba) { // -------------------------------------------------------------------- void amrex::avgDown_doit(const FArrayBox &fine_fab, FArrayBox &crse_fab, const Box &ovlp, int scomp, int dcomp, int ncomp, - Vector &ratio) + Vector &ratio) { const int *ovlo = ovlp.loVect(); const int *ovhi = ovlp.hiVect(); @@ -148,7 +148,7 @@ void amrex::avgDown(MultiFab &S_crse, MultiFab &S_fine, int scomp, int dcomp, for(MFIter mfi(S_fine); mfi.isValid(); ++mfi) { const int i(mfi.index()); avgDown_doit(S_fine[i], crse_S_fine[i], crse_S_fine_BA[i], - scomp, 0, ncomp, ratio); + scomp, 0, ncomp, ratio); } S_crse.copy(crse_S_fine, 0, dcomp, ncomp); } @@ -348,8 +348,8 @@ long amrex::FileSize(const std::string &filename) { // ---------------------------------------------------------------------- void amrex::MakeFuncPctTimesMF(const Vector > &funcStats, const Vector &blpFNames, - const std::map &mProfStats, - Real runTime, int dataNProcs) + const std::map &mProfStats, + Real runTime, int dataNProcs) { #ifdef BL_TRACE_PROFILING #if (BL_SPACEDIM == 2) @@ -401,7 +401,7 @@ void amrex::MakeFuncPctTimesMF(const Vector > &fun void amrex::CollectMProfStats(std::map &mProfStats, const Vector > &funcStats, const Vector &fNames, - Real runTime, int whichProc) + Real runTime, int whichProc) { for(int fnum(0); fnum < funcStats.size(); ++fnum) { BLProfiler::ProfStats ps; @@ -443,7 +443,7 @@ void amrex::CollectMProfStats(std::map &mPro void amrex::GraphTopPct(const std::map &mProfStats, const Vector > &funcStats, const Vector &fNames, - Real runTime, int dataNProcs, Real gPercent) + Real runTime, int dataNProcs, Real gPercent) { for(int fnum(0); fnum < funcStats.size(); ++fnum) { const std::string &fName(fNames[fnum]); diff --git a/Src/Extern/ProfParser/AMReX_BLWritePlotFile.H b/Src/Extern/ProfParser/AMReX_BLWritePlotFile.H index 475721d4d86..55acf081fbb 100644 --- a/Src/Extern/ProfParser/AMReX_BLWritePlotFile.H +++ b/Src/Extern/ProfParser/AMReX_BLWritePlotFile.H @@ -28,9 +28,9 @@ void WritePlotfile(const std::string &pfversion, const std::string &oFile, const amrex::Vector &names, const bool verbose, - const bool isCartGrid = false, - const Real *vfeps = 0, - const int *levelSteps = 0); + const bool isCartGrid = false, + const Real *vfeps = 0, + const int *levelSteps = 0); // this function makes a 2d plotfile from a 3d one @@ -49,9 +49,9 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, const std::string &oFile, const amrex::Vector &names, const bool verbose, - const bool isCartGrid = false, - const Real *vfeps = 0, - const int *levelSteps = 0); + const bool isCartGrid = false, + const Real *vfeps = 0, + const int *levelSteps = 0); } //namespace amrex #endif diff --git a/Src/Extern/ProfParser/AMReX_BLWritePlotFile.cpp b/Src/Extern/ProfParser/AMReX_BLWritePlotFile.cpp index 206fcbf0b86..19c04f66b3c 100644 --- a/Src/Extern/ProfParser/AMReX_BLWritePlotFile.cpp +++ b/Src/Extern/ProfParser/AMReX_BLWritePlotFile.cpp @@ -28,7 +28,7 @@ void WritePlotfile(const std::string &pfversion, const bool verbose, const bool isCartGrid, const Real *vfeps, - const int *levelSteps) + const int *levelSteps) { if(ParallelDescriptor::IOProcessor()) { if( ! amrex::UtilCreateDirectory(oFile,0755)) { @@ -44,18 +44,18 @@ void WritePlotfile(const std::string &pfversion, const int finestLevel(data.size() - 1); if(ParallelDescriptor::IOProcessor()) { - + std::string oFileHeader(oFile); oFileHeader += "/Header"; - + //os.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - + if(verbose && ParallelDescriptor::IOProcessor()) { std::cout << "Opening file = " << oFileHeader << '\n'; } - + os.open(oFileHeader.c_str(), std::ios::out|std::ios::binary); - + if(os.fail()) { amrex::FileOpenFailed(oFileHeader); } @@ -109,15 +109,15 @@ void WritePlotfile(const std::string &pfversion, int nGrids = ba.size(); char buf[64]; sprintf(buf, "Level_%d", iLevel); - + if(ParallelDescriptor::IOProcessor()) { os << iLevel << ' ' << nGrids << ' ' << time << '\n'; if(levelSteps != 0) { os << levelSteps[iLevel] << '\n'; - } else { + } else { os << 0 << '\n'; - } - + } + for(int i(0); i < nGrids; ++i) { const Box &b = ba[i]; for(int n(0); n < BL_SPACEDIM; ++n) { @@ -130,22 +130,22 @@ void WritePlotfile(const std::string &pfversion, std::string Level(oFile); Level += '/'; Level += buf; - + if( ! amrex::UtilCreateDirectory(Level, 0755)) { amrex::CreateDirectoryFailed(Level); - } + } } // Force other processors to wait till directory is built. ParallelDescriptor::Barrier(); // Now build the full relative pathname of the MultiFab. static const std::string MultiFabBaseName("MultiFab"); - + std::string PathName(oFile); PathName += '/'; PathName += buf; PathName += '/'; PathName += MultiFabBaseName; - + if(ParallelDescriptor::IOProcessor()) { // // The full name relative to the Header file. @@ -157,7 +157,7 @@ void WritePlotfile(const std::string &pfversion, } VisMF::Write(data[iLevel], PathName); } - + if(ParallelDescriptor::IOProcessor()) { os.close(); } @@ -334,9 +334,9 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, const std::string &oFile, const Vector &names, const bool verbose, - const bool isCartGrid, - const Real *vfeps, - const int *levelSteps) + const bool isCartGrid, + const Real *vfeps, + const int *levelSteps) { int whichPlane; @@ -357,22 +357,22 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, } // Force other processors to wait till directory is built. ParallelDescriptor::Barrier(); - + std::string oFileHeader(oFile); oFileHeader += "/Header"; - + VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); - + std::ofstream os; - + //os.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - + if(verbose && ParallelDescriptor::IOProcessor()) { std::cout << "Opening file = " << oFileHeader << '\n'; } - + os.open(oFileHeader.c_str(), std::ios::out|std::ios::binary); - + if(os.fail()) { FileOpenFailed(oFileHeader); } @@ -419,7 +419,7 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, for(int k = 0; k < bl_spacedim; k++) { if(k != whichPlane) { os << dxLevel[i][k] << ' '; - } + } } os << '\n'; } @@ -442,15 +442,15 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, int nGrids = ba.size(); char buf[64]; sprintf(buf, "Level_%d", iLevel); - + if(ParallelDescriptor::IOProcessor()) { os << iLevel << ' ' << nGrids << ' ' << time << '\n'; if(levelSteps != 0) { os << levelSteps[iLevel] << '\n'; - } else { + } else { os << 0 << '\n'; - } - + } + for(int i(0); i < nGrids; ++i) { const Box &b = ba[i]; for(int n(0); n < bl_spacedim; ++n) { @@ -458,7 +458,7 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, Real glo = b.smallEnd()[n] * dxLevel[iLevel][n]; Real ghi = (b.bigEnd()[n]+1) * dxLevel[iLevel][n]; os << glo << ' ' << ghi << '\n'; - } + } } } // @@ -467,10 +467,10 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, std::string Level(oFile); Level += '/'; Level += buf; - + if( ! UtilCreateDirectory(Level, 0755)) { CreateDirectoryFailed(Level); - } + } } // // Force other processors to wait till directory is built. @@ -480,13 +480,13 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, // Now build the full relative pathname of the MultiFab. // static const std::string MultiFabBaseName("MultiFab"); - + std::string PathName(oFile); PathName += '/'; PathName += buf; PathName += '/'; PathName += MultiFabBaseName; - + if(ParallelDescriptor::IOProcessor()) { // The full name relative to the Header file. std::string RelativePathName(buf); @@ -531,7 +531,7 @@ void WritePlotfile2DFrom3D(const std::string &pfversion, // ====================================== } - + os.close(); } // -------------------------------------------------------------------- diff --git a/Src/Extern/ProfParser/AMReX_CommProfStats.H b/Src/Extern/ProfParser/AMReX_CommProfStats.H index dd2f8d1c72c..4b6cf693113 100644 --- a/Src/Extern/ProfParser/AMReX_CommProfStats.H +++ b/Src/Extern/ProfParser/AMReX_CommProfStats.H @@ -40,7 +40,7 @@ class CommProfStats : public BLProfStats { struct BarrierEntry { BarrierEntry() : number(-1), name("__uninitialized_Barrier_Entry_name__"), - seekIndex(-1), bNameNumber(-1) { } + seekIndex(-1), bNameNumber(-1) { } BarrierEntry(long n, const std::string &nm, long i, int bnn) : number(n), name(nm), seekIndex(i), bNameNumber(bnn) { } @@ -69,13 +69,13 @@ class CommProfStats : public BLProfStats { struct DataBlock { DataBlock() : proc(0), size(0), fileName(""), seekpos(0), readoffset(0), nodeName(""), - nodeNumber(-1), nTimerTimes(0), timerTime(0.0), streamIndex(-1) - { } + nodeNumber(-1), nTimerTimes(0), timerTime(0.0), streamIndex(-1) + { } DataBlock(const int p, const long s, const std::string &fn, const long sp, const std::string &pn, const int nn, const int si) : proc(p), size(s), fileName(fn), seekpos(sp), readoffset(0), nodeName(pn), - nodeNumber(nn), nTimerTimes(0), timerTime(0.0), streamIndex(si) - { } + nodeNumber(nn), nTimerTimes(0), timerTime(0.0), streamIndex(si) + { } int proc; long size; @@ -100,7 +100,7 @@ class CommProfStats : public BLProfStats { const int rp, const int ds, const int tg, const Real st, const Real rt, const Real tt) : fromProc(sp), toProc(rp), dataSize(ds), - tag(tg), sendTime(st), recvTime(rt), totalTime(tt) { } + tag(tg), sendTime(st), recvTime(rt), totalTime(tt) { } int fromProc, toProc, dataSize, tag; Real sendTime, recvTime, totalTime; @@ -108,35 +108,35 @@ class CommProfStats : public BLProfStats { bool Matches(int fromproc, int toproc, int size, int mtag) { return(fromproc == fromProc && toproc == toProc && - size == dataSize && mtag == tag); + size == dataSize && mtag == tag); } bool Matches(int fromproc, int toproc, int size) { return(fromproc == fromProc && toproc == toProc && - size == dataSize); + size == dataSize); } long HashLong() { std::stringstream hss; - hss << fromProc << toProc << dataSize << tag; - return atol(hss.str().c_str()); + hss << fromProc << toProc << dataSize << tag; + return atol(hss.str().c_str()); } inline static long HashLong(int fp, int tp, int t) { - return(fp + tp + 64*t); + return(fp + tp + 64*t); } inline static long HashLong(int fp, int tp, int /*ds*/, int t) { - //if(fp < 0 || tp < 0 || ds < 0 || t < 0) { - //cout << ":::: HashLong: negative values: fp tp ds t = " - //<< fp << " " << tp << " " << ds << " " << t << endl; - //} + //if(fp < 0 || tp < 0 || ds < 0 || t < 0) { + //cout << ":::: HashLong: negative values: fp tp ds t = " + //<< fp << " " << tp << " " << ds << " " << t << endl; + //} //std::stringstream hss; - //// hss.seekp(0); - //// hss << fp << tp << ds << t; - //// return atol(hss.str().c_str()); - //return(fp + 100*tp + 1000*ds + 10000*t); - //return(fp + tp + 10*ds + 100*t); - //return(fp + tp + ds + 10*t); - //return(fp + tp + 32*t); - return(fp + tp + 64*t); - //return(fp + tp + ds + t); + //// hss.seekp(0); + //// hss << fp << tp << ds << t; + //// return atol(hss.str().c_str()); + //return(fp + 100*tp + 1000*ds + 10000*t); + //return(fp + tp + 10*ds + 100*t); + //return(fp + tp + ds + 10*t); + //return(fp + tp + 32*t); + return(fp + tp + 64*t); + //return(fp + tp + ds + t); } }; @@ -145,7 +145,7 @@ class CommProfStats : public BLProfStats { const int rp, const int ds, const int tg, const Real st, const Real rt, const Real tt) : SendRecvPair(sp, rp, ds, tg, st, rt, tt), unmatchedCFType(cft) - { } + { } amrex::BLProfiler::CommFuncType unmatchedCFType; }; @@ -156,34 +156,34 @@ class CommProfStats : public BLProfStats { void CheckCommData(amrex::Vector &nBMin, amrex::Vector &nBMax, amrex::Vector &nRMin, amrex::Vector &nRMax); void FillSendFAB(long &totalSends, long &totalSentData, - amrex::Vector &totalSendsPerProc, - amrex::Vector &totalSentDataPerProc, - amrex::FArrayBox &sendFAB, bool proxmap); + amrex::Vector &totalSendsPerProc, + amrex::Vector &totalSentDataPerProc, + amrex::FArrayBox &sendFAB, bool proxmap); void ReportSyncPointDataSetup(long &nBMax, long &nRMax); void ReportSyncPointData(amrex::Vector > &barrierExitTimes, amrex::Vector > &barrierWaitTimes, amrex::Vector > &reductionWaitTimes, - bool bDoReductions = false); + bool bDoReductions = false); void ReportStats(long &totalSentData, long &totalNCommStats, amrex::Vector &totalFuncCalls, - int bytesPerSlot, amrex::Vector &msgSizes, - int &minMsgSize, int &maxMsgSize, + int bytesPerSlot, amrex::Vector &msgSizes, + int &minMsgSize, int &maxMsgSize, Real &timeMin, Real &timeMax, Real &timerTime, - amrex::Vector &rankNodeNumbers); + amrex::Vector &rankNodeNumbers); void WriteTopoFab(); void FindTimeRange(BLProfStats::TimeRange& tr); void TimelineFAB(amrex::FArrayBox &timelineFAB, const amrex::Box &probDomain, const BLProfStats::TimeRange tr, - const int rankMin, const int rankMax, - const int rankStride, - const Real ntnMultiplier, const amrex::Vector &ntnNumbers, - const Real bnMultiplier, const amrex::Vector &bnNumbers); + const int rankMin, const int rankMax, + const int rankStride, + const Real ntnMultiplier, const amrex::Vector &ntnNumbers, + const Real bnMultiplier, const amrex::Vector &bnNumbers); void SendRecvData(const std::string &filename, const double tlo, const double thi); void SendRecvList(std::multimap &srMMap); virtual void InitCommDataBlock(const int proc, const long ncommstats, const std::string &filename, const long seekpos, - const std::string &nodename = "", const int nodenumber = -1); + const std::string &nodename = "", const int nodenumber = -1); int AfterBarrier(const int proc, const double t); diff --git a/Src/Extern/ProfParser/AMReX_CommProfStats.cpp b/Src/Extern/ProfParser/AMReX_CommProfStats.cpp index 8292fcf1ef1..9a78442715d 100644 --- a/Src/Extern/ProfParser/AMReX_CommProfStats.cpp +++ b/Src/Extern/ProfParser/AMReX_CommProfStats.cpp @@ -134,9 +134,9 @@ void CommProfStats::InitDataFileNames(const Vector &hfn) { // ---------------------------------------------------------------------- void CommProfStats::InitCommDataBlock(const int proc, const long ncommstats, const std::string &filename, - const long seekpos, - const std::string &procname, - const int nodenumber) + const long seekpos, + const std::string &procname, + const int nodenumber) { int streamindex; std::map::iterator it = commDataFileNames.find(filename); @@ -244,9 +244,9 @@ void CommProfStats::AddGridLevel(const int /*level*/, const int /*ngrids*/) { // ---------------------------------------------------------------------- void CommProfStats::AddGrid3D(int /*level*/, int /*xlo*/, int /*ylo*/, int /*zlo*/, - int /*xhi*/, int /*yhi*/, int /*zhi*/, - int /*xc*/, int /*yc*/, int /*zc*/, - int xn, int yn, int zn, int proc) + int /*xhi*/, int /*yhi*/, int /*zhi*/, + int /*xc*/, int /*yc*/, int /*zc*/, + int xn, int yn, int zn, int proc) { long nPoints(xn * yn * zn); std::map::iterator it = glMap.find(proc); @@ -522,7 +522,7 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, if(verbose) { cout << myProc << ": " << "CommProfProc " << dBlock.proc << " nCommStats " << dBlock.size << " datafile " << dBlock.fileName << " seekpos " - << dBlock.seekpos << endl; + << dBlock.seekpos << endl; cout << myProc << ": " << "barriers.size() = " << dBlock.barriers.size() << " " << dBlock.proc << endl; } @@ -539,14 +539,14 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, if(bIndex > nCS-1 || bIndex+1 > nCS-1) { cerr << "**** Error: bad bIndex: " << bIndex << " " << nCS << endl; - continue; + continue; } BLProfiler::CommStats &cs = dBlock.vCommStats[bIndex]; BLProfiler::CommStats &csNext = dBlock.vCommStats[bIndex + 1]; if(cs.cfType == BLProfiler::Barrier && csNext.cfType == BLProfiler::Barrier && cs.tag == csNext.tag && // these are the barrier numbers - cs.tag == bNumber ) + cs.tag == bNumber ) { if(cs.commpid != BLProfiler::BeforeCall() || csNext.commpid != BLProfiler::AfterCall()) { cerr << "**** Error: bad Barrier before, after." << endl; @@ -555,8 +555,8 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, << (cs.commpid) << " " << (csNext.commpid) << " " << (cs.tag) << " " << (csNext.tag) << " " << (cs.timeStamp) << " " << (csNext.timeStamp) << " " - << endl; - } + << endl; + } } else { cerr << "**** Error: bad Barriers." << endl; cerr << BLProfiler::CommStats::CFTToString(cs.cfType) << " " @@ -564,8 +564,8 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, << (cs.commpid) << " " << (csNext.commpid) << " " << (cs.tag) << " " << (csNext.tag) << " " << (cs.timeStamp) << " " << (csNext.timeStamp) << " " - << bName << " " << bNumber - << endl; + << bName << " " << bNumber + << endl; } } @@ -578,7 +578,7 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, if(bIndex > nCS-1 || bIndex+1 > nCS-1) { cerr << "**** Error: bad reduction bIndex: " << bIndex << " " << nCS << endl; - continue; + continue; } BLProfiler::CommStats &cs = dBlock.vCommStats[bIndex]; @@ -595,8 +595,8 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, << (cs.commpid) << " " << (csNext.commpid) << " " << (cs.tag) << " " << (csNext.tag) << " " << (cs.timeStamp) << " " << (csNext.timeStamp) << " " - << endl; - } + << endl; + } } else { cerr << "**** Error: bad Reductions." << endl; cerr << BLProfiler::CommStats::CFTToString(cs.cfType) << " " @@ -605,7 +605,7 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, << (cs.commpid) << " " << (csNext.commpid) << " " << (cs.tag) << " " << (csNext.tag) << " " << (cs.timeStamp) << " " << (csNext.timeStamp) << " " - << endl; + << endl; } } @@ -640,9 +640,9 @@ void CommProfStats::CheckCommData(Vector &nBMin, Vector &nBMax, // ---------------------------------------------------------------------- void CommProfStats::FillSendFAB(long &totalSends, long &totalSentData, - Vector &totalSendsPerProc, - Vector &totalSentDataPerProc, - FArrayBox &sendFAB, bool proxmap) + Vector &totalSendsPerProc, + Vector &totalSentDataPerProc, + FArrayBox &sendFAB, bool proxmap) { BL_PROFILE("CommProfStats::FillSendFAB"); @@ -679,21 +679,21 @@ void CommProfStats::FillSendFAB(long &totalSends, long &totalSentData, for(int i(0); i < dBlock.vCommStats.size(); ++i) { // ---- find sends and sum BLProfiler::CommStats &cs = dBlock.vCommStats[i]; if(IsSend(cs.cfType)) { - if(cs.size != BLProfiler::AfterCall()) { - rankTo = cs.commpid; - proxTo = proxFromRank[rankTo]; - if(proxTo >= smallY && proxTo <= bigY) { // ---- within to proc range - if(InTimeRange(proc, cs.timeStamp)) { - ++totalSends; + if(cs.size != BLProfiler::AfterCall()) { + rankTo = cs.commpid; + proxTo = proxFromRank[rankTo]; + if(proxTo >= smallY && proxTo <= bigY) { // ---- within to proc range + if(InTimeRange(proc, cs.timeStamp)) { + ++totalSends; totalSentData += cs.size; - ++totalSendsPerProc[proxFrom]; - totalSentDataPerProc[proxFrom] += cs.size; - index = (offsetX) + (xlen * (proxTo - smallY)); - tsp[index] += 1.0; - tsdp[index] += cs.size; - } - } - } + ++totalSendsPerProc[proxFrom]; + totalSentDataPerProc[proxFrom] += cs.size; + index = (offsetX) + (xlen * (proxTo - smallY)); + tsp[index] += 1.0; + tsdp[index] += cs.size; + } + } + } } } ClearCommStats(dBlock); @@ -735,19 +735,19 @@ void CommProfStats::FillSendFAB(long &totalSends, long &totalSentData, for(int i(0); i < dBlock.vCommStats.size(); ++i) { // ---- find sends and sum BLProfiler::CommStats &cs = dBlock.vCommStats[i]; if(IsSend(cs.cfType)) { - if(cs.size != BLProfiler::AfterCall()) { - if(cs.commpid >= smallY && cs.commpid <= bigY) { // ---- within to proc range - if(InTimeRange(proc, cs.timeStamp)) { - ++totalSends; + if(cs.size != BLProfiler::AfterCall()) { + if(cs.commpid >= smallY && cs.commpid <= bigY) { // ---- within to proc range + if(InTimeRange(proc, cs.timeStamp)) { + ++totalSends; totalSentData += cs.size; - ++totalSendsPerProc[proc]; - totalSentDataPerProc[proc] += cs.size; - index = (offsetX) + (xlen * (cs.commpid - smallY)); - tsp[index] += 1.0; - tsdp[index] += cs.size; - } - } - } + ++totalSendsPerProc[proc]; + totalSentDataPerProc[proc] += cs.size; + index = (offsetX) + (xlen * (cs.commpid - smallY)); + tsp[index] += 1.0; + tsdp[index] += cs.size; + } + } + } } } ClearCommStats(dBlock); @@ -791,7 +791,7 @@ void CommProfStats::ReportSyncPointDataSetup(long &nBMax, long &nRMax) void CommProfStats::ReportSyncPointData(Vector > &barrierExitTimes, Vector > &barrierWaitTimes, Vector > &reductionWaitTimes, - bool bDoReductions) + bool bDoReductions) { procNodeNumber.resize(dataNProcs); @@ -863,10 +863,10 @@ void CommProfStats::ReportSyncPointData(Vector > &barrierExitTimes, // ---------------------------------------------------------------------- void CommProfStats::ReportStats(long &totalSentData, long &totalNCommStats, Vector &totalFuncCalls, - int bytesPerSlot, Vector &msgSizes, - int &minMsgSize, int &maxMsgSize, + int bytesPerSlot, Vector &msgSizes, + int &minMsgSize, int &maxMsgSize, Real &timeMin, Real &timeMax, Real &timerTime, - Vector &rankNodeNumbers) + Vector &rankNodeNumbers) { amrex::ignore_unused(timerTime); @@ -896,15 +896,15 @@ void CommProfStats::ReportStats(long &totalSentData, long &totalNCommStats, for(int i(0); i < dBlock.vCommStats.size(); ++i) { // ------- sum sent data BLProfiler::CommStats &cs = dBlock.vCommStats[i]; if(IsSend(cs.cfType)) { - if(cs.size != BLProfiler::AfterCall()) { - if(InTimeRange(dBlock.proc, cs.timeStamp)) { + if(cs.size != BLProfiler::AfterCall()) { + if(InTimeRange(dBlock.proc, cs.timeStamp)) { totalSentData += cs.size; - slot = std::min(cs.size/bytesPerSlot, highSlot); - ++msgSizes[slot]; - minMsgSize = std::min(cs.size, minMsgSize); - maxMsgSize = std::max(cs.size, maxMsgSize); - } - } + slot = std::min(cs.size/bytesPerSlot, highSlot); + ++msgSizes[slot]; + minMsgSize = std::min(cs.size, minMsgSize); + maxMsgSize = std::max(cs.size, maxMsgSize); + } + } } } @@ -913,14 +913,14 @@ void CommProfStats::ReportStats(long &totalSentData, long &totalNCommStats, if((cs.size > -1 && cs.cfType != BLProfiler::Waitsome) || (cs.size == BLProfiler::BeforeCall() && cs.cfType == BLProfiler::Waitsome)) { - if(InTimeRange(dBlock.proc, cs.timeStamp)) { - if(cs.cfType >= 0 && cs.cfType < totalFuncCalls.size()) { + if(InTimeRange(dBlock.proc, cs.timeStamp)) { + if(cs.cfType >= 0 && cs.cfType < totalFuncCalls.size()) { ++totalFuncCalls[cs.cfType]; - } else { - std::cout << "--------:: totalFuncCalls.size() cs.cfType = " << totalFuncCalls.size() + } else { + std::cout << "--------:: totalFuncCalls.size() cs.cfType = " << totalFuncCalls.size() << " " << cs.cfType << std::endl; - } - } + } + } } } @@ -953,9 +953,9 @@ void CommProfStats::FindTimeRange(BLProfStats::TimeRange& tr) { void CommProfStats::TimelineFAB(FArrayBox &timelineFAB, const Box &probDomain, const BLProfStats::TimeRange tr, const int rankMin, const int rankMax, - const int rankStride, - const Real ntnMultiplier, const Vector &ntnNumbers, - const Real bnMultiplier, const Vector &bnNumbers) + const int rankStride, + const Real ntnMultiplier, const Vector &ntnNumbers, + const Real bnMultiplier, const Vector &bnNumbers) { BL_PROFILE("CommProfStats::TimelineFAB()"); @@ -1003,99 +1003,99 @@ void CommProfStats::TimelineFAB(FArrayBox &timelineFAB, const Box &probDomain, Real ts(cs.timeStamp); if((ts <= fabTimeHi && ts >= fabTimeLo) && InTimeRange(dBlock.proc, ts)) { // within time range xi = long( nTimeSlotsFab * ((ts - fabTimeLo) * ooTimeRangeFab) ); - if(xi == nTimeSlotsFab) { - --xi; - } - if(xi < 0 || xi >= nTimeSlotsFab) { - SHOWVAL(xi) - SHOWVAL(nTimeSlotsFab) - SHOWVAL(timeRangeFab) - SHOWVAL(ts) - amrex::Abort("xi out of range."); - } + if(xi == nTimeSlotsFab) { + --xi; + } + if(xi < 0 || xi >= nTimeSlotsFab) { + SHOWVAL(xi) + SHOWVAL(nTimeSlotsFab) + SHOWVAL(timeRangeFab) + SHOWVAL(ts) + amrex::Abort("xi out of range."); + } index = (((proc - fabRankLo) / rankStride) * nTimeSlotsFab) + xi; - if(index < 0 || index >= timelineFAB.box().numPts()) { - SHOWVAL(index) - SHOWVAL(timelineFAB.box().size()) - amrex::Abort("index out of range."); - } + if(index < 0 || index >= timelineFAB.box().numPts()) { + SHOWVAL(index) + SHOWVAL(timelineFAB.box().size()) + amrex::Abort("index out of range."); + } timeline[index] = cs.cfType; mpiCount[index] += 1.0; - Real ntnMult(0.0), bnMult(0.0); - if(cs.cfType == BLProfiler::NameTag) { // ---- add encoded value for the name tag name - ntnMult = ntnNumbers[cs.tag]; + Real ntnMult(0.0), bnMult(0.0); + if(cs.cfType == BLProfiler::NameTag) { // ---- add encoded value for the name tag name + ntnMult = ntnNumbers[cs.tag]; timeline[index] += ntnMult; - } - if(cs.cfType == BLProfiler::Barrier) { // ---- add encoded value for the barrier name - if(cs.tag >= dBlock.barriers.size()) { - cout << "******** TimelineFAB::0" << endl; - SHOWVAL(i); - SHOWVAL(cs.tag); - SHOWVAL(dBlock.barriers.size()); - for(int ib(0); ib < dBlock.barriers.size(); ++ib) { - cout << "be[" << ib << "] = " << dBlock.barriers[ib] << endl; - } - amrex::Abort("--------- bad barrier."); - } - BarrierEntry &be = dBlock.barriers[cs.tag]; - bnMult = bnNumbers[be.bNameNumber]; + } + if(cs.cfType == BLProfiler::Barrier) { // ---- add encoded value for the barrier name + if(cs.tag >= dBlock.barriers.size()) { + cout << "******** TimelineFAB::0" << endl; + SHOWVAL(i); + SHOWVAL(cs.tag); + SHOWVAL(dBlock.barriers.size()); + for(int ib(0); ib < dBlock.barriers.size(); ++ib) { + cout << "be[" << ib << "] = " << dBlock.barriers[ib] << endl; + } + amrex::Abort("--------- bad barrier."); + } + BarrierEntry &be = dBlock.barriers[cs.tag]; + bnMult = bnNumbers[be.bNameNumber]; timeline[index] += bnMult; - } + } // now fill in gaps if(i > 0 && xi < nTimeSlotsFab) { BLProfiler::CommStats &csPrev = dBlock.vCommStats[i-1]; - if(cs.cfType == csPrev.cfType - && - ( - ( - csPrev.size == BLProfiler::BeforeCall() - || - cs.size == BLProfiler::AfterCall() - ) - || - ( - csPrev.tag == BLProfiler::BeforeCall() - && - cs.tag == BLProfiler::AfterCall() - ) - - ) - ) - { + if(cs.cfType == csPrev.cfType + && + ( + ( + csPrev.size == BLProfiler::BeforeCall() + || + cs.size == BLProfiler::AfterCall() + ) + || + ( + csPrev.tag == BLProfiler::BeforeCall() + && + cs.tag == BLProfiler::AfterCall() + ) + + ) + ) + { Real prevTs(csPrev.timeStamp); - if(prevTs < fabTimeLo) { - cout << ":::: prevTs fabTimeLo = " << prevTs << " " << fabTimeLo << endl; - prevTs = fabTimeLo; - } + if(prevTs < fabTimeLo) { + cout << ":::: prevTs fabTimeLo = " << prevTs << " " << fabTimeLo << endl; + prevTs = fabTimeLo; + } long prevXi = long(nTimeSlotsFab * ((prevTs - fabTimeLo) * ooTimeRangeFab)); prevIndex = (((proc - fabRankLo) / rankStride) * nTimeSlotsFab) + prevXi; - for(long idx(prevIndex); idx < index; ++idx) { - if(idx < 0 || idx >= timelineFAB.box().numPts() || idx > index) { - SHOWVAL(proc) - SHOWVAL(ts) - SHOWVAL(prevTs) - SHOWVAL(fabTimeLo) - SHOWVAL(prevXi) - SHOWVAL(idx) - SHOWVAL(index) - SHOWVAL(prevIndex) - SHOWVAL(timelineFAB.box().size()) - //amrex::Abort("idx out of range."); - amrex::Print() << "CommProfStats::TimelineFAB::idx out of range." << std::endl; - continue; - } + for(long idx(prevIndex); idx < index; ++idx) { + if(idx < 0 || idx >= timelineFAB.box().numPts() || idx > index) { + SHOWVAL(proc) + SHOWVAL(ts) + SHOWVAL(prevTs) + SHOWVAL(fabTimeLo) + SHOWVAL(prevXi) + SHOWVAL(idx) + SHOWVAL(index) + SHOWVAL(prevIndex) + SHOWVAL(timelineFAB.box().size()) + //amrex::Abort("idx out of range."); + amrex::Print() << "CommProfStats::TimelineFAB::idx out of range." << std::endl; + continue; + } timeline[idx] = cs.cfType; mpiCount[idx] += 1.0; - if(cs.cfType == BLProfiler::NameTag) { + if(cs.cfType == BLProfiler::NameTag) { timeline[idx] += ntnMult; - } - if(cs.cfType == BLProfiler::Barrier) { + } + if(cs.cfType == BLProfiler::Barrier) { timeline[idx] += bnMult; - } - } - } + } + } + } } prevIndex = index; } @@ -1111,8 +1111,8 @@ bool CommProfStats::IsSend(const BLProfiler::CommFuncType &cft) { return(cft == BLProfiler::AsendTsii || cft == BLProfiler::AsendTsiiM || cft == BLProfiler::AsendvTii || - cft == BLProfiler::SendTsii || - cft == BLProfiler::SendvTii); + cft == BLProfiler::SendTsii || + cft == BLProfiler::SendvTii); } @@ -1153,9 +1153,9 @@ void CommProfStats::SendRecvList(std::multimap &srMM if(cs.size != BLProfiler::AfterCall()) { if(InTimeRange(proc, cs.timeStamp)) { srMMap.insert(std::pair(cs.timeStamp, - SendRecvPairUnpaired(cs.cfType, proc, cs.commpid, + SendRecvPairUnpaired(cs.cfType, proc, cs.commpid, cs.size, cs.tag, cs.timeStamp, - -1000.0, 0.0))); + -1000.0, 0.0))); } } } @@ -1163,9 +1163,9 @@ void CommProfStats::SendRecvList(std::multimap &srMM if(cs.size != BLProfiler::AfterCall()) { if(InTimeRange(proc, cs.timeStamp)) { srMMap.insert(std::pair(cs.timeStamp, - SendRecvPairUnpaired(cs.cfType, cs.commpid, proc, + SendRecvPairUnpaired(cs.cfType, cs.commpid, proc, cs.size, cs.tag, -2000.0, - cs.timeStamp, 0.0))); + cs.timeStamp, 0.0))); } } } @@ -1315,8 +1315,8 @@ void CommProfStats::SendRecvData(const std::string &filenameprefix, if(dataLeft[idb]) { dataLeft[idb] = ReadCommStats(dBlock, readSize); if(dataLeft[idb] == false) { - --anyDataLeft; - } + --anyDataLeft; + } } else { continue; } @@ -1355,104 +1355,104 @@ void CommProfStats::SendRecvData(const std::string &filenameprefix, double st0 = amrex::ParallelDescriptor::second(); int fromProcTemp, toProcTemp; - if(IsSend(cs.cfType)) { - fromProcTemp = proc; - toProcTemp = cs.commpid; - } else { - fromProcTemp = cs.commpid; - toProcTemp = proc; - } - //long hlong(SendRecvPair::HashLong(fromProcTemp, toProcTemp, cs.size, cs.tag)); - long hlong = SendRecvPair::HashLong(fromProcTemp, toProcTemp, cs.tag); - - //bool foundMatch(false); - int nMatches(0); - - pair::iterator, - unordered_multimap::iterator> upmSRPERI; - Vector::iterator> upmSRPMatchSave; - - maxTag = std::max(maxTag, cs.tag); - - unordered_multimap &upm = unpairedMessages[myThread][cs.tag]; - //maxupmmapsize = std::max(static_cast(unpairedMessages[myThread].size()), maxupmmapsize); - upmSRPERI = upm.equal_range(hlong); - for(unordered_multimap::iterator upmsrit = upmSRPERI.first; - upmsrit != upmSRPERI.second; ++upmsrit) - { - SendRecvPairUnpaired &srp = upmsrit->second; - if(srp.Matches(fromProcTemp, toProcTemp, cs.size, cs.tag)) { - if((IsSend(srp.unmatchedCFType) && IsBlockingRecv(cs.cfType)) || - (IsSend(cs.cfType) && IsBlockingRecv(srp.unmatchedCFType))) - { - upmSRPMatchSave.push_back(upmsrit); - ++nMatches; - maxNMatches = std::max(maxNMatches, nMatches); - } - } - } + if(IsSend(cs.cfType)) { + fromProcTemp = proc; + toProcTemp = cs.commpid; + } else { + fromProcTemp = cs.commpid; + toProcTemp = proc; + } + //long hlong(SendRecvPair::HashLong(fromProcTemp, toProcTemp, cs.size, cs.tag)); + long hlong = SendRecvPair::HashLong(fromProcTemp, toProcTemp, cs.tag); + + //bool foundMatch(false); + int nMatches(0); + + pair::iterator, + unordered_multimap::iterator> upmSRPERI; + Vector::iterator> upmSRPMatchSave; + + maxTag = std::max(maxTag, cs.tag); + + unordered_multimap &upm = unpairedMessages[myThread][cs.tag]; + //maxupmmapsize = std::max(static_cast(unpairedMessages[myThread].size()), maxupmmapsize); + upmSRPERI = upm.equal_range(hlong); + for(unordered_multimap::iterator upmsrit = upmSRPERI.first; + upmsrit != upmSRPERI.second; ++upmsrit) + { + SendRecvPairUnpaired &srp = upmsrit->second; + if(srp.Matches(fromProcTemp, toProcTemp, cs.size, cs.tag)) { + if((IsSend(srp.unmatchedCFType) && IsBlockingRecv(cs.cfType)) || + (IsSend(cs.cfType) && IsBlockingRecv(srp.unmatchedCFType))) + { + upmSRPMatchSave.push_back(upmsrit); + ++nMatches; + maxNMatches = std::max(maxNMatches, nMatches); + } + } + } s0 += amrex::ParallelDescriptor::second() - st0; if(ics+1 >= dBlock.vCommStats.size()) { - continue; - } + continue; + } - if(nMatches == 0) { // ------------------------ unpaired + if(nMatches == 0) { // ------------------------ unpaired double st1 = amrex::ParallelDescriptor::second(); - if(IsSend(cs.cfType)) { - sttemp = cs.timeStamp; - tttemp = sttemp; - } else { - if(ics+1 >= dBlock.vCommStats.size()) { - cout << "!!!!!!!!!!!!!!!!!!!!!!! cs on end (unpaired)." << endl; - } + if(IsSend(cs.cfType)) { + sttemp = cs.timeStamp; + tttemp = sttemp; + } else { + if(ics+1 >= dBlock.vCommStats.size()) { + cout << "!!!!!!!!!!!!!!!!!!!!!!! cs on end (unpaired)." << endl; + } BLProfiler::CommStats &csNext = dBlock.vCommStats[ics+1]; - rttemp = csNext.timeStamp; - tttemp = rttemp; - } - upm.insert(std::pair(hlong, - SendRecvPairUnpaired(cs.cfType, fromProcTemp, - toProcTemp, cs.size, cs.tag, - sttemp, rttemp, tttemp))); - ++upmsize; - maxupmsize = std::max(upmsize, maxupmsize); + rttemp = csNext.timeStamp; + tttemp = rttemp; + } + upm.insert(std::pair(hlong, + SendRecvPairUnpaired(cs.cfType, fromProcTemp, + toProcTemp, cs.size, cs.tag, + sttemp, rttemp, tttemp))); + ++upmsize; + maxupmsize = std::max(upmsize, maxupmsize); s1 += amrex::ParallelDescriptor::second() - st1; - } else { // ------------------------ paired + } else { // ------------------------ paired double st2 = amrex::ParallelDescriptor::second(); - unordered_multimap::iterator - earliestMatchSave = upmSRPMatchSave[0]; - for(int i(1); i < upmSRPMatchSave.size(); ++i) { - if(upmSRPMatchSave[i]->second.totalTime < earliestMatchSave->second.totalTime) { - earliestMatchSave = upmSRPMatchSave[i]; - } - } - SendRecvPairUnpaired &srpup = earliestMatchSave->second; - sttemp = srpup.sendTime; - rttemp = srpup.recvTime; + unordered_multimap::iterator + earliestMatchSave = upmSRPMatchSave[0]; + for(int i(1); i < upmSRPMatchSave.size(); ++i) { + if(upmSRPMatchSave[i]->second.totalTime < earliestMatchSave->second.totalTime) { + earliestMatchSave = upmSRPMatchSave[i]; + } + } + SendRecvPairUnpaired &srpup = earliestMatchSave->second; + sttemp = srpup.sendTime; + rttemp = srpup.recvTime; if(IsSend(cs.cfType)) { - sttemp = cs.timeStamp; - } else { - if(ics+1 >= dBlock.vCommStats.size()) { - cout << "!!!!!!!!!!!!!!!!!!!!!!! cs on end (paired)." << endl; - } + sttemp = cs.timeStamp; + } else { + if(ics+1 >= dBlock.vCommStats.size()) { + cout << "!!!!!!!!!!!!!!!!!!!!!!! cs on end (paired)." << endl; + } BLProfiler::CommStats &csNext = dBlock.vCommStats[ics + 1]; - rttemp = csNext.timeStamp; - } - tttemp = rttemp - sttemp; + rttemp = csNext.timeStamp; + } + tttemp = rttemp - sttemp; if(tttemp < 0.0) { ++mNeg; - } - ++mTotal; - pairedMessages[myThread].push_back(SendRecvPair(srpup.fromProc, srpup.toProc, - srpup.dataSize, srpup.tag, - sttemp, rttemp, tttemp)); - upm.erase(earliestMatchSave); - --upmsize; + } + ++mTotal; + pairedMessages[myThread].push_back(SendRecvPair(srpup.fromProc, srpup.toProc, + srpup.dataSize, srpup.tag, + sttemp, rttemp, tttemp)); + upm.erase(earliestMatchSave); + --upmsize; s2 += amrex::ParallelDescriptor::second() - st2; - } // end else paired + } // end else paired ptime += amrex::ParallelDescriptor::second() - pstart; @@ -1546,9 +1546,9 @@ s2 += amrex::ParallelDescriptor::second() - st2; sdataNC[index] += 1.0; // count if(bTopoFabInited) { - IntVect ivfrom(TopIVFromProcNumber(procNodeNumber[srp.fromProc])); - IntVect ivto(TopIVFromProcNumber(procNodeNumber[srp.toProc])); - int nhops(NHops(topoBox, ivfrom, ivto)); + IntVect ivfrom(TopIVFromProcNumber(procNodeNumber[srp.fromProc])); + IntVect ivto(TopIVFromProcNumber(procNodeNumber[srp.toProc])); + int nhops(NHops(topoBox, ivfrom, ivto)); sdataNHops[index] += nhops; } @@ -1601,7 +1601,7 @@ s2 += amrex::ParallelDescriptor::second() - st2; SHOWVAL(ts); SHOWVAL(zi); SHOWVAL(index); - abort(); + abort(); } ttData[index] += srp.totalTime; bwData[index] += srp.dataSize; @@ -1658,9 +1658,9 @@ return; if(i < 32) { cout << "unskew: fP sT tP rT betS betR s-r = " << srp.fromProc << " " << srp.sendTime << " > " << srp.toProc << " " << srp.recvTime << " " - << barrierExitTimes[srp.fromProc][sB] - << " " << barrierExitTimes[srp.toProc][rB] << " " - << barrierExitTimes[srp.fromProc][sB] - barrierExitTimes[srp.toProc][rB] << endl; + << barrierExitTimes[srp.fromProc][sB] + << " " << barrierExitTimes[srp.toProc][rB] << " " + << barrierExitTimes[srp.fromProc][sB] - barrierExitTimes[srp.toProc][rB] << endl; } //srp.totalTime += barrierExitTimes[srp.fromProc][sB] - barrierExitTimes[srp.toProc][rB]; srp.totalTime = (srp.recvTime - barrierExitTimes[srp.toProc][rB]) - @@ -1831,7 +1831,7 @@ void CommProfStats::WriteEdisonTopoMF() { for(int iy(0); iy < edisonNodeXYZ[ix].size(); ++iy) { for(int iz(0); iz < edisonNodeXYZ[ix][iy].size(); ++iz) { cout << "XYZ pid = " << ix << " " << iy << " " << iz << " " - << edisonNodeXYZ[ix][iy][iz] << endl; + << edisonNodeXYZ[ix][iy][iz] << endl; pnnout << ip++ << " " << edisonNodeXYZ[ix][iy][iz] << '\n'; } } diff --git a/Src/Extern/ProfParser/AMReX_ProfParserBatch.cpp b/Src/Extern/ProfParser/AMReX_ProfParserBatch.cpp index b10407e7f6c..4c0e5e25413 100644 --- a/Src/Extern/ProfParser/AMReX_ProfParserBatch.cpp +++ b/Src/Extern/ProfParser/AMReX_ProfParserBatch.cpp @@ -108,11 +108,11 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, while(ia < argc-1) { if(bIOP) cout << "argv[" << ia << "] = " << argv[ia] << endl; if(strcmp(argv[ia], "-v") == 0) { - if(ia < argc-2) { - verbose = atoi(argv[ia+1]); - } + if(ia < argc-2) { + verbose = atoi(argv[ia+1]); + } if(bIOP) cout << "*** verbose = " << verbose << endl; - ++ia; + ++ia; } else if(strcmp(argv[ia], "-ws") == 0) { bWriteSummary = true; } else if(strcmp(argv[ia], "-check") == 0) { // ---- commprof options @@ -127,12 +127,12 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, runStats = true; } else if(strcmp(argv[ia], "-actpf") == 0) { bRunACTPF = true; - if(ia < argc-2) { + if(ia < argc-2) { actFNames.push_back(argv[ia+1]); - } + } if(bIOP) cout << "*** output a plotfile for all call times for function" - << actFNames[actFNames.size() - 1] << endl; - ++ia; + << actFNames[actFNames.size() - 1] << endl; + ++ia; } else if(strcmp(argv[ia], "-sr") == 0) { if(bIOP) cout << "*** send receive pairing." << endl; runSendRecv = true; @@ -155,36 +155,36 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, if(bIOP) cout << "*** redist." << endl; runRedist = true; } else if(strcmp(argv[ia], "-msil") == 0) { - if(ia < argc-2) { + if(ia < argc-2) { maxSmallImageLength = atoi(argv[ia+1]); - } + } if(bIOP) cout << "*** msil = " << maxSmallImageLength << endl; - ++ia; + ++ia; } else if(strcmp(argv[ia], "-rra") == 0) { - if(ia < argc-2) { + if(ia < argc-2) { refRatioAll = atoi(argv[ia+1]); - } + } if(bIOP) cout << "*** rra = " << refRatioAll << endl; - ++ia; + ++ia; } else if(strcmp(argv[ia], "-nts") == 0) { - if(ia < argc-2) { + if(ia < argc-2) { nTimeSlots = atoi(argv[ia+1]); - } + } if(bIOP) cout << "*** nts = " << nTimeSlots << endl; - ++ia; + ++ia; } else if(strcmp(argv[ia], "-proc") == 0) { - if(ia < argc-2) { + if(ia < argc-2) { whichProc = atoi(argv[ia+1]); - } + } if(bIOP) cout << "*** whichProc = " << whichProc << endl; - ++ia; + ++ia; } else if(strcmp(argv[ia], "-of") == 0) { - if(ia < argc-2) { + if(ia < argc-2) { outfileName = argv[ia+1]; - filenameSet = true; - } + filenameSet = true; + } if(bIOP) cout << "*** outfileName = " << outfileName << endl; - ++ia; + ++ia; } else if(strcmp(argv[ia], "-proxmap") == 0) { if(bIOP) cout << "*** proxmap." << endl; proxMap = true; @@ -206,16 +206,16 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, } else if(strcmp(argv[ia], "-ttrace") == 0) { bWriteTextTrace = true; } else if(strcmp(argv[ia], "-gpct") == 0) { - if(ia < argc-2) { + if(ia < argc-2) { Real gpct(atof(argv[ia+1])); - if(gpct >= 0.0 && gpct <= 100.0) { - RegionsProfStats::SetGPercent(gpct); + if(gpct >= 0.0 && gpct <= 100.0) { + RegionsProfStats::SetGPercent(gpct); if(bIOP) cout << "*** gpct = " << gpct << endl; - } else { + } else { if(bIOP) cout << "*** gpct must be in range [0.0, 100.0]" << endl; - } - } - ++ia; + } + } + ++ia; } else if(strcmp(argv[ia], "-nocomb") == 0) { simpleCombine = false; } else if(strcmp(argv[ia], "-prof") == 0) { @@ -257,8 +257,8 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, if(bIOP) { DataServices::Dispatch(DataServices::WriteSummaryRequest, &pdServices, (void *) &(cout), - &writeAverage, whichProc, &useTrace, - &graphTopPct); + &writeAverage, whichProc, &useTrace, + &graphTopPct); } } else { pdServices.WriteSummary(cout, writeAverage, whichProc, useTrace, @@ -275,8 +275,8 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, DataServices::Dispatch(DataServices::InitTimeRanges, &pdServices); DataServices::Dispatch(DataServices::WriteSummaryRequest, &pdServices, (void *) &(cout), - &writeAverage, whichProc, &useTrace, - &graphTopPct); + &writeAverage, whichProc, &useTrace, + &graphTopPct); } } else { pdServices.InitRegionTimeRanges(); @@ -308,7 +308,7 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, if(bIOP) { DataServices::Dispatch(DataServices::RunStatsRequest, &pdServices, (void *) &(mpiFuncNames), - &statsCollected); + &statsCollected); } } else { pdServices.RunStats(mpiFuncNames, statsCollected); @@ -325,9 +325,9 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, if(bIOP) { DataServices::Dispatch(DataServices::RunSendsPFRequest, &pdServices, (void *) &(plotfileName), - maxSmallImageLength, - &proxMap, - refRatioAll); + maxSmallImageLength, + &proxMap, + refRatioAll); } } else { pdServices.RunSendsPF(plotfileName, maxSmallImageLength, @@ -347,16 +347,16 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, /* DataServices::Dispatch(DataServices::RunTimelinePFRequest, &pdServices, (void *) &(mpiFuncNames), (void *) &(plotfileName), - maxSmallImageLength, - refRatioAll, - nTimeSlots, - &statsCollected);*/ + maxSmallImageLength, + refRatioAll, + nTimeSlots, + &statsCollected);*/ } } else { BLProfStats::TimeRange subTimeRange = pdServices.FindCalcTimeRange(); - pdServices.RunTimelinePF(mpiFuncNames, plotfileName, subTimeRange, - maxSmallImageLength, refRatioAll, - nTimeSlots, statsCollected); + pdServices.RunTimelinePF(mpiFuncNames, plotfileName, subTimeRange, + maxSmallImageLength, refRatioAll, + nTimeSlots, statsCollected); } } @@ -464,7 +464,7 @@ bool ProfParserBatchFunctions(int argc, char *argv[], bool runDefault, bool anyFunctionsRun = runCheck || runSendRecv || runSendRecvList || runSyncPointData || runSendsPF || runTimelinePF || tcEdisonOnly || runStats || - runRedist || bMakeFilterFile || bWriteSummary || bWriteTraceSummary || + runRedist || bMakeFilterFile || bWriteSummary || bWriteTraceSummary || bMakeRegionPlt || bWriteHTML || bWriteHTMLNC || bWriteTextTrace || glOnly || bRunACTPF; diff --git a/Src/Extern/ProfParser/AMReX_RegionsProfStats.H b/Src/Extern/ProfParser/AMReX_RegionsProfStats.H index ecbd537a97b..b22c883076d 100644 --- a/Src/Extern/ProfParser/AMReX_RegionsProfStats.H +++ b/Src/Extern/ProfParser/AMReX_RegionsProfStats.H @@ -80,7 +80,7 @@ class RegionsProfStats : public BLProfStats { TimeRange MakeRegionPlt(amrex::FArrayBox &rFab, int noregionnumber, int width, int height, - amrex::Vector> ®ionBoxes); + amrex::Vector> ®ionBoxes); virtual void CollectFuncStats(amrex::Vector > &funcStats); virtual void WriteSummary(std::ostream &os, bool bwriteavg = false, int whichProc = 0, bool graphTopPct = true); @@ -97,7 +97,7 @@ class RegionsProfStats : public BLProfStats { static bool InitDBlocks() { return bInitDataBlocks; } static void InitDataFileNames(const amrex::Vector &hfn); bool TraceDataValid() const { return mFNameNumbersPerProc.size() > 0; } - + void SyncFNamesAndNumbers(); bool InitRegionTimeRanges(const amrex::Box &procBox); diff --git a/Src/Extern/ProfParser/AMReX_RegionsProfStats.cpp b/Src/Extern/ProfParser/AMReX_RegionsProfStats.cpp index 3710f8e4310..4f3b2423c01 100644 --- a/Src/Extern/ProfParser/AMReX_RegionsProfStats.cpp +++ b/Src/Extern/ProfParser/AMReX_RegionsProfStats.cpp @@ -57,8 +57,8 @@ Vector RegionsProfStats::regDataStreams; extern std::string SanitizeName(const std::string &s); extern void amrex::MakeFuncPctTimesMF(const Vector > &funcStats, const Vector &blpFNames, - const std::map &mProfStats, - Real runTime, int dataNProcs); + const std::map &mProfStats, + Real runTime, int dataNProcs); extern void amrex::CollectMProfStats(std::map &mProfStats, const Vector > &funcStats, const Vector &fNames, @@ -72,15 +72,15 @@ extern void amrex::GraphTopPct(const std::map 0.0) ? (( 1.0 - (CS.stackTime / CS.totalTime)) * 100.0) : ( 0.0 )) \ - << " % :: " \ - << CS.nCSCalls << " :: " << CS.callStackDepth << " :: " \ - << CS.callTime + << " % :: " \ + << CS.nCSCalls << " :: " << CS.callStackDepth << " :: " \ + << CS.callTime #define PRINTCSNC(CS) CS.csFNameNumber << " :: " << fNumberNames[CS.csFNameNumber] << " :: " \ << CS.totalTime << " :: " << CS.stackTime << " :: " << \ ((CS.totalTime > 0.0) ? (( 1.0 - (CS.stackTime / CS.totalTime)) * 100.0) : ( 0.0 )) \ - << " % :: " \ - << CS.nCSCalls << " :: " << CS.callStackDepth + << " % :: " \ + << CS.nCSCalls << " :: " << CS.callStackDepth // ---------------------------------------------------------------------- @@ -142,21 +142,21 @@ void RegionsProfStats::SyncFNamesAndNumbers() { int localIndex(mfnnit->second); for(int n(0); n < syncedStrings.size(); ++n) { if(findName == syncedStrings[n]) { - fnameRemap[p][localIndex] = n; - foundStrings[n] = n; - //cout << " p fname localIndex n = " << p << " " << findName << " " - //<< localIndex << " " << n << endl; - } + fnameRemap[p][localIndex] = n; + foundStrings[n] = n; + //cout << " p fname localIndex n = " << p << " " << findName << " " + //<< localIndex << " " << n << endl; + } } } for(int n(0); n < foundStrings.size(); ++n) { // fill in unfound strings if(foundStrings[n] < 0) { for(int ii(0); ii < fnameRemap[p].size(); ++ii) { - if(fnameRemap[p][ii] < 0) { - fnameRemap[p][ii] = n; - break; - } - } + if(fnameRemap[p][ii] < 0) { + fnameRemap[p][ii] = n; + break; + } + } } } } @@ -202,7 +202,7 @@ void RegionsProfStats::AddTimeMinMax(double tmin, double tmax) { // ---------------------------------------------------------------------- BLProfStats::TimeRange RegionsProfStats::MakeRegionPlt(FArrayBox &rFab, int noregionnumber, int width, int height, - Vector> ®ionBoxes) + Vector> ®ionBoxes) { amrex::ignore_unused(noregionnumber); @@ -259,10 +259,10 @@ BLProfStats::TimeRange RegionsProfStats::MakeRegionPlt(FArrayBox &rFab, int nore rStartTime[rss.rssRNumber] = -1.0; int xStart = int(xLength * rtStart / timeMax); int xStop = int(xLength * rtStop / timeMax); - xStop = std::min(xStop, xLength - 1); + xStop = std::min(xStop, xLength - 1); int yLo(rss.rssRNumber * yHeight), yHi(((rss.rssRNumber + 1) * yHeight) - 1); Box rBox(IntVect(xStart, yLo), IntVect(xStop, yHi)); - regionBoxes[rss.rssRNumber].push_back(rBox); + regionBoxes[rss.rssRNumber].push_back(rBox); rFab.setVal(rss.rssRNumber, rBox, 0); } } @@ -302,7 +302,7 @@ void RegionsProfStats::FillRegionTimeRanges(Vector> &rtr, } else { // stop region if(rStartTime[rss.rssRNumber] < 0.0) { // not started yet, mismatched start/stop } else { // stopping - rtr[rss.rssRNumber].push_back(TimeRange(rStartTime[rss.rssRNumber], rss.rssTime)); + rtr[rss.rssRNumber].push_back(TimeRange(rStartTime[rss.rssRNumber], rss.rssTime)); rStartTime[rss.rssRNumber] = -1.0; } } @@ -376,13 +376,13 @@ bool RegionsProfStats::InitRegionTimeRanges(const Box &procBox) { for(int i(0); i < dBlock.rStartStop.size(); ++i) { BLProfiler::RStartStop &rss = dBlock.rStartStop[i]; if(rss.rssStart) { // start region - if(rss.rssRNumber >= 0) { + if(rss.rssRNumber >= 0) { regionTimeRanges[dBlock.proc][rss.rssRNumber].push_back(TimeRange(rss.rssTime, -1.0)); - } + } } else { // stop region - if(rss.rssRNumber >= 0) { + if(rss.rssRNumber >= 0) { regionTimeRanges[dBlock.proc][rss.rssRNumber].back().stopTime = rss.rssTime; - } + } } } ClearBlock(dBlock); @@ -414,9 +414,9 @@ bool RegionsProfStats::InitRegionTimeRanges(const Box &procBox) { for(int r(0); r < regionTimeRanges[p].size(); ++r) { if( ! (p >= smallY && p <= bigY)) { // ---- not within myproc range // if(regionTimeRanges[p][r].size() > 0) { -// amrex::Abort("regionTimeRanges size error 0"); -// } - regionTimeRanges[p][r].resize(nRanges[r]); +// amrex::Abort("regionTimeRanges size error 0"); +// } + regionTimeRanges[p][r].resize(nRanges[r]); } } } @@ -428,9 +428,9 @@ bool RegionsProfStats::InitRegionTimeRanges(const Box &procBox) { for(int r(0); r < regionTimeRanges[p].size(); ++r) { if(p >= smallY && p <= bigY) { // ---- within myproc range for(int t(0); t < regionTimeRanges[p][r].size(); ++t) { -/* int index((p * (maxRNumber + 1) * totalRanges * 2) + - (r * totalRanges * 2) + (t * 2)); */ - long index((r * totalRanges * 2) + (t * 2) + +/* int index((p * (maxRNumber + 1) * totalRanges * 2) + + (r * totalRanges * 2) + (t * 2)); */ + long index((r * totalRanges * 2) + (t * 2) + (p * (maxRNumber + 1) * totalRanges * 2)); gAllRanges[index] = regionTimeRanges[p][r][t].startTime; @@ -456,7 +456,7 @@ bool RegionsProfStats::InitRegionTimeRanges(const Box &procBox) { } MPI_Allgatherv(localGAllRanges.dataPtr(), sendCount, ParallelDescriptor::Mpi_typemap::type(), gAllRanges.dataPtr(), recvCounts.dataPtr(), recvDispl.dataPtr(), - ParallelDescriptor::Mpi_typemap::type(), + ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); #endif @@ -466,7 +466,7 @@ bool RegionsProfStats::InitRegionTimeRanges(const Box &procBox) { for(int t(0); t < regionTimeRanges[p][r].size(); ++t) { /* int index((p * (maxRNumber + 1) * totalRanges * 2) + (r * totalRanges * 2) + (t * 2)); */ - long index((r * totalRanges * 2) + (t * 2) + + long index((r * totalRanges * 2) + (t * 2) + (p * (maxRNumber + 1) * totalRanges * 2)); regionTimeRanges[p][r][t].startTime = gAllRanges[index]; regionTimeRanges[p][r][t].stopTime = gAllRanges[index + 1]; @@ -483,11 +483,11 @@ bool RegionsProfStats::InitRegionTimeRanges(const Box &procBox) { for(int r(0); r < regionTimeRanges[p].size(); ++r) { for(int t(0); t < regionTimeRanges[p][r].size(); ++t) { if(regionTimeRanges[p][r][t].startTime != checkRegionTimeRanges[p][r][t].startTime) { - amrex::Abort("Bad checkRegionTimeRanges startTime"); - } + amrex::Abort("Bad checkRegionTimeRanges startTime"); + } if(regionTimeRanges[p][r][t].stopTime != checkRegionTimeRanges[p][r][t].stopTime) { - amrex::Abort("Bad checkRegionTimeRanges stopTime"); - } + amrex::Abort("Bad checkRegionTimeRanges stopTime"); + } } } } @@ -593,7 +593,7 @@ bool RegionsProfStats::AllCallTimesFAB(FArrayBox &actFab, // ---------------------------------------------------------------------- void RegionsProfStats::FillAllCallTimes(Vector > &allCallTimes, const std::string whichFuncName, - int whichFuncNumber, const Box &procBox) + int whichFuncNumber, const Box &procBox) { BL_PROFILE("RegionsProfStats::FillAllCallTimes"); @@ -671,7 +671,7 @@ void RegionsProfStats::WriteSummary(std::ostream &ios, bool /*bwriteavg*/, Real timeMin(std::numeric_limits::max()); Real timeMax(-std::numeric_limits::max()); - + Vector fNames(numbersToFName.size()); for(int i(0); i < fNames.size(); ++i) { if(i >= 0) { @@ -690,21 +690,21 @@ void RegionsProfStats::WriteSummary(std::ostream &ios, bool /*bwriteavg*/, ReadBlock(dBlock); if(dBlock.proc == whichProc) { for(int i(0); i < dBlock.vCallStats.size(); ++i) { - // ---- here we have to add only the part of this - // ---- callstat that intersects the region time range + // ---- here we have to add only the part of this + // ---- callstat that intersects the region time range BLProfiler::CallStats &cs = dBlock.vCallStats[i]; - TimeRange tRangeFull(cs.callTime, cs.callTime + cs.totalTime); - std::list intersectList = - RegionsProfStats::RangeIntersection(filterTimeRanges[whichProc], tRangeFull); - std::list::iterator tri; - for(tri = intersectList.begin(); tri != intersectList.end(); ++tri) { + TimeRange tRangeFull(cs.callTime, cs.callTime + cs.totalTime); + std::list intersectList = + RegionsProfStats::RangeIntersection(filterTimeRanges[whichProc], tRangeFull); + std::list::iterator tri; + for(tri = intersectList.begin(); tri != intersectList.end(); ++tri) { BLProfiler::CallStats csis(dBlock.vCallStats[i]); - csis.callTime = tri->startTime; - csis.totalTime = tri->stopTime - tri->startTime; + csis.callTime = tri->startTime; + csis.totalTime = tri->stopTime - tri->startTime; if(InTimeRange(dBlock.proc, cs.callTime)) { vCallStatsAllOneProc.push_back(csis); } - } + } } } @@ -773,14 +773,14 @@ void RegionsProfStats::CheckRegionsData() for(int p(0); p < regionTimeRangesCount.size(); ++p) { regionTimeRangesCount[p].resize(dataNProcs, 0); } - + for(int idb(0); idb < dataBlocks.size(); ++idb) { DataBlock &dBlock = dataBlocks[idb]; if(verbose) { cout << myProc << ": " << "RegionsProfProc " << dBlock.proc << " nTraceStats " << dBlock.nTraceStats << " nRSS " << dBlock.nRSS << " fileName " << dBlock.fileName << " seekpos " - << dBlock.seekpos << endl; + << dBlock.seekpos << endl; } ReadBlock(dBlock, true, false); // dont need to read the trace data @@ -790,7 +790,7 @@ void RegionsProfStats::CheckRegionsData() if(bIOP) { cerr << "***RegionsProfStats::CheckRegionsData: region number is greater than max number: " - << rss.rssRNumber << " > " << maxRNumber + 1 << endl; + << rss.rssRNumber << " > " << maxRNumber + 1 << endl; } if(rss.rssStart) { // start region regionTimeRangesCount[rss.rssRNumber][dBlock.proc]++; @@ -804,7 +804,7 @@ void RegionsProfStats::CheckRegionsData() } cout << myProc << ": " << "---------------------- checking regions consistency." << endl; - + for (int r(0); rFunction call times " << "(function number :: function name :: inclusive time :: exclusive time :: 1-e/i % :: ncalls :: callstackdepth :: call time)" - << '\n'; + << '\n'; csHTMLFile << "
    " << '\n'; listEnds.push("
"); @@ -925,119 +925,119 @@ void RegionsProfStats::WriteHTML(std::ostream &csHTMLFile, vCallTrace[0] = lastFlushedCS; // copy to the unused cs } for(int iCT(0); iCT < vCallTrace.size(); ++iCT) { - ++nodeNumber; + ++nodeNumber; BLProfiler::CallStats &cs = vCallTrace[iCT]; if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { // ---- the unused cs - continue; + continue; } if(cs.nCSCalls > 1) { - static int count(0); - if(count++ < 8) { + static int count(0); + if(count++ < 8) { std::cout << "DDDDDDDDDD cs.nCalls = " << cs.nCSCalls << " " - << fNumberNames[cs.csFNameNumber] << std::endl; - } + << fNumberNames[cs.csFNameNumber] << std::endl; + } } if(iCT == vCallTrace.size() - 1) { - if(combCallStats != 0) { + if(combCallStats != 0) { IcsHTMLFile << "
  • " << PRINTCS((*combCallStats)) << "
    • " << '\n'; - delete combCallStats; - combCallStats = 0; - } - if(bLastBlock) { + delete combCallStats; + combCallStats = 0; + } + if(bLastBlock) { IcsHTMLFile << "
  • " << PRINTCS(cs) << "
    • " << '\n'; for(int n(0); n < cs.callStackDepth; ++n) { - if( ! listEnds.empty()) { + if( ! listEnds.empty()) { IIcsHTMLFile << listEnds.top() << '\n'; listEnds.pop(); - } else { - std::cout << "WriteHTML::0: listEnds.empty(): csd n = " - << cs.callStackDepth << " " << n << std::endl; - } - if( ! listEnds.empty()) { + } else { + std::cout << "WriteHTML::0: listEnds.empty(): csd n = " + << cs.callStackDepth << " " << n << std::endl; + } + if( ! listEnds.empty()) { IIcsHTMLFile << listEnds.top() << '\n'; listEnds.pop(); - } else { - std::cout << "WriteHTML::1: listEnds.empty(): csd n = " - << cs.callStackDepth << " " << n << std::endl; - } + } else { + std::cout << "WriteHTML::1: listEnds.empty(): csd n = " + << cs.callStackDepth << " " << n << std::endl; + } } - } else { // ---- save the last calltrace - lastFlushedCS = cs; - } + } else { // ---- save the last calltrace + lastFlushedCS = cs; + } } else { BLProfiler::CallStats &csNext = vCallTrace[iCT + 1]; if(csNext.callStackDepth > cs.callStackDepth) { - if(combCallStats != 0) { + if(combCallStats != 0) { IcsHTMLFile << "
  • " << PRINTCS((*combCallStats)) << "
    • " << '\n'; - delete combCallStats; - combCallStats = 0; - } + delete combCallStats; + combCallStats = 0; + } IcsHTMLFile << "
  • " << '\n'; listEnds.push("
    • "); - colorLinkTime = colorLinkTimeStack.top(); - colorLinkTimeStack.push(cs.totalTime); - if(cs.totalTime > colorLinkTime * colorLinkPct) { + colorLinkTime = colorLinkTimeStack.top(); + colorLinkTimeStack.push(cs.totalTime); + if(cs.totalTime > colorLinkTime * colorLinkPct) { IcsHTMLFile << "" << PRINTCS(cs) << "" << '\n'; - } else { + << nodeNumber << "')\">" << PRINTCS(cs) << "" << '\n'; + } else { IcsHTMLFile << "" << PRINTCS(cs) << "" << '\n'; - } - if(cs.totalTime > colorLinkTime * colorLinkPct) { // ---- expand link + << nodeNumber << "')\">" << PRINTCS(cs) << "" << '\n'; + } + if(cs.totalTime > colorLinkTime * colorLinkPct) { // ---- expand link IcsHTMLFile << "
      " << '\n'; } else { IcsHTMLFile << "
        " << '\n'; } listEnds.push("
      "); } else if(csNext.callStackDepth == cs.callStackDepth) { - if(simpleCombine) { - if(iCT < vCallTrace.size() - 2 && cs.csFNameNumber == csNext.csFNameNumber) { - if(combCallStats == 0) { - combCallStats = new BLProfiler::CallStats(cs); - } else { - combCallStats->nCSCalls += cs.nCSCalls; - combCallStats->totalTime += cs.totalTime; - combCallStats->stackTime += cs.stackTime; - } - } else { - if(combCallStats != 0) { - if(cs.csFNameNumber == combCallStats->csFNameNumber) { - combCallStats->nCSCalls += cs.nCSCalls; - combCallStats->totalTime += cs.totalTime; - combCallStats->stackTime += cs.stackTime; - } + if(simpleCombine) { + if(iCT < vCallTrace.size() - 2 && cs.csFNameNumber == csNext.csFNameNumber) { + if(combCallStats == 0) { + combCallStats = new BLProfiler::CallStats(cs); + } else { + combCallStats->nCSCalls += cs.nCSCalls; + combCallStats->totalTime += cs.totalTime; + combCallStats->stackTime += cs.stackTime; + } + } else { + if(combCallStats != 0) { + if(cs.csFNameNumber == combCallStats->csFNameNumber) { + combCallStats->nCSCalls += cs.nCSCalls; + combCallStats->totalTime += cs.totalTime; + combCallStats->stackTime += cs.stackTime; + } IcsHTMLFile << "
    • " << PRINTCS((*combCallStats)) << "
      • " << '\n'; - delete combCallStats; - combCallStats = 0; - } else { + delete combCallStats; + combCallStats = 0; + } else { IcsHTMLFile << "
    • " << PRINTCS(cs) << "
      • " << '\n'; - } - } - } else { + } + } + } else { IcsHTMLFile << "
    • " << PRINTCS(cs) << "
      • " << '\n'; - } + } } else { - if(combCallStats != 0) { - if(cs.csFNameNumber == combCallStats->csFNameNumber) { - combCallStats->nCSCalls += cs.nCSCalls; - combCallStats->totalTime += cs.totalTime; - combCallStats->stackTime += cs.stackTime; - } + if(combCallStats != 0) { + if(cs.csFNameNumber == combCallStats->csFNameNumber) { + combCallStats->nCSCalls += cs.nCSCalls; + combCallStats->totalTime += cs.totalTime; + combCallStats->stackTime += cs.stackTime; + } IcsHTMLFile << "
    • " << PRINTCS((*combCallStats)) << "
      • " << '\n'; - delete combCallStats; - combCallStats = 0; - } else { + delete combCallStats; + combCallStats = 0; + } else { IcsHTMLFile << "
    • " << PRINTCS(cs) << "
      • " << '\n'; - } + } for(int n(0); n < cs.callStackDepth - csNext.callStackDepth; ++n) { IIcsHTMLFile << listEnds.top() << '\n'; listEnds.pop(); IIcsHTMLFile << listEnds.top() << '\n'; listEnds.pop(); - colorLinkTimeStack.pop(); + colorLinkTimeStack.pop(); } - colorLinkTime = colorLinkTimeStack.top(); + colorLinkTime = colorLinkTimeStack.top(); } } } @@ -1085,7 +1085,7 @@ void RegionsProfStats::PrintCallTreeNode(CallTreeNode &callTree, std::cout << "PCTN: " << fNumberNames[callTree.fnameNumber] << " stackDepth = " << callTree.stackDepth << " nCalls = " << callTree.nCalls << " stackTime = " - << callTree.stackTime << std::endl; + << callTree.stackTime << std::endl; } for(miter = callTree.calledFunctions.begin(); miter != callTree.calledFunctions.end(); ++miter) { PrintCallTreeNode(miter->second, fNumberNames); @@ -1129,7 +1129,7 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) csHTMLFile << "

      Function calls " << "(function number :: function name :: inclusive time :: exclusive time :: 1-e/i % :: ncalls :: callstackdepth)

      " - << '\n'; + << '\n'; csHTMLFile << "
        " << '\n'; listEnds.push("
      "); @@ -1159,7 +1159,7 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) BLProfiler::CallStats lastFlushedCS; int totalCalls(0); - + for(int idb(0); idb < dataBlocks.size(); ++idb) { DataBlock &dBlock = dataBlocks[idb]; if(dBlock.proc == whichProc) { @@ -1171,59 +1171,59 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) std::cout << "vCallTrace.size() = " << vCallTrace.size() << std::endl; if( ! bFirstBlock) { - vCallTrace[0] = lastFlushedCS; // copy to the unused cs + vCallTrace[0] = lastFlushedCS; // copy to the unused cs } for(int iCT(0); iCT < vCallTrace.size(); ++iCT) { BLProfiler::CallStats &cs = vCallTrace[iCT]; if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { // ---- the unused cs - continue; + continue; + } + + if(bAddFirstNode) { + // ---- add the node + std::pair::iterator, bool> retval; + retval = currentCTN->calledFunctions.insert( + std::pair(cs.csFNameNumber, CallTreeNode())); + miter = retval.first; + if(retval.second) { // ---- value inserted + miter->second.fnameNumber = cs.csFNameNumber; + miter->second.stackDepth = cs.callStackDepth; + miter->second.nCalls = 1; + miter->second.totalTime = cs.totalTime; + miter->second.stackTime = cs.stackTime; + } else { // ---- value existed + miter->second.nCalls += 1; + miter->second.totalTime += cs.totalTime; + miter->second.stackTime += cs.stackTime; + } + ++totalCalls; } + bAddFirstNode = true; - if(bAddFirstNode) { - // ---- add the node - std::pair::iterator, bool> retval; - retval = currentCTN->calledFunctions.insert( - std::pair(cs.csFNameNumber, CallTreeNode())); - miter = retval.first; - if(retval.second) { // ---- value inserted - miter->second.fnameNumber = cs.csFNameNumber; - miter->second.stackDepth = cs.callStackDepth; - miter->second.nCalls = 1; - miter->second.totalTime = cs.totalTime; - miter->second.stackTime = cs.stackTime; - } else { // ---- value existed - miter->second.nCalls += 1; - miter->second.totalTime += cs.totalTime; - miter->second.stackTime += cs.stackTime; - } - ++totalCalls; - } - bAddFirstNode = true; - if(iCT == vCallTrace.size() - 1) { - if(bLastBlock) { - } else { - lastFlushedCS = cs; - bAddFirstNode = false; - } - } else { + if(bLastBlock) { + } else { + lastFlushedCS = cs; + bAddFirstNode = false; + } + } else { BLProfiler::CallStats &csNext = vCallTrace[iCT + 1]; if(csNext.callStackDepth > cs.callStackDepth) { - currentCTN = &(miter->second); - if(currentCTN != 0) { - ctStack.push(currentCTN); - } - } else if(csNext.callStackDepth == cs.callStackDepth) { - // ---- node added above - } else { - for(int s(0); s < cs.callStackDepth - csNext.callStackDepth; ++s) { - ctStack.pop(); - } - currentCTN = ctStack.top(); - } - } + currentCTN = &(miter->second); + if(currentCTN != 0) { + ctStack.push(currentCTN); + } + } else if(csNext.callStackDepth == cs.callStackDepth) { + // ---- node added above + } else { + for(int s(0); s < cs.callStackDepth - csNext.callStackDepth; ++s) { + ctStack.pop(); + } + currentCTN = ctStack.top(); + } + } } bFirstBlock = false; } @@ -1238,10 +1238,10 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) int nodeNumber(-1); for(int iCT(0); iCT < vCallTrace.size(); ++iCT) { - ++nodeNumber; + ++nodeNumber; BLProfiler::CallStats &cs = vCallTrace[iCT]; if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { // ---- the unused cs - continue; + continue; } if(iCT == vCallTrace.size() - 1) { IcsHTMLFile << "
    • " << PRINTCSNC(cs) << "
      • " << '\n'; @@ -1251,14 +1251,14 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) listEnds.pop(); } else { std::cout << "WriteHTMLNC::0: listEnds.empty(): csd n = " - << cs.callStackDepth << " " << n << std::endl; + << cs.callStackDepth << " " << n << std::endl; } if( ! listEnds.empty()) { IIcsHTMLFile << listEnds.top() << '\n'; listEnds.pop(); } else { std::cout << "WriteHTMLNC::1: listEnds.empty(): csd n = " - << cs.callStackDepth << " " << n << std::endl; + << cs.callStackDepth << " " << n << std::endl; } } } else { @@ -1267,7 +1267,7 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) IcsHTMLFile << "
    • " << '\n'; listEnds.push("
      • "); IcsHTMLFile << "" << PRINTCSNC(cs) << "" << '\n'; + << nodeNumber << "')\">" << PRINTCSNC(cs) << "" << '\n'; if(cs.callStackDepth < 100) { IcsHTMLFile << "
        " << '\n'; } else { @@ -1284,14 +1284,14 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) listEnds.pop(); } else { std::cout << "WriteHTMLNC::2: listEnds.empty(): csd n = " - << cs.callStackDepth << " " << n << std::endl; + << cs.callStackDepth << " " << n << std::endl; } if( ! listEnds.empty()) { IIcsHTMLFile << listEnds.top() << '\n'; listEnds.pop(); } else { std::cout << "WriteHTMLNC::3: listEnds.empty(): csd n = " - << cs.callStackDepth << " " << n << std::endl; + << cs.callStackDepth << " " << n << std::endl; } } } @@ -1305,7 +1305,7 @@ void RegionsProfStats::WriteHTMLNC(std::ostream &csHTMLFile, int whichProc) csHTMLFile << listEnds.top() << '\n'; listEnds.pop(); } - + csHTMLFile << "" << '\n'; csHTMLFile << "" << '\n'; @@ -1342,15 +1342,15 @@ void RegionsProfStats::WriteTextTrace(std::ostream &ios, bool simpleCombine, for(int i(0); i < vCallTrace.size(); ++i) { BLProfiler::CallStats &cs = vCallTrace[i]; - if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { // ---- the unused cs - continue; - } + if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { // ---- the unused cs + continue; + } - if(i < vCallTrace.size() - 2) { + if(i < vCallTrace.size() - 2) { BLProfiler::CallStats &csNext = vCallTrace[i + 1]; - if(csNext.callStackDepth == cs.callStackDepth && - cs.csFNameNumber == csNext.csFNameNumber) - { + if(csNext.callStackDepth == cs.callStackDepth && + cs.csFNameNumber == csNext.csFNameNumber) + { if(combCallStats == 0) { combCallStats = new BLProfiler::CallStats(cs); } else { @@ -1358,26 +1358,26 @@ void RegionsProfStats::WriteTextTrace(std::ostream &ios, bool simpleCombine, combCallStats->totalTime += cs.totalTime; combCallStats->stackTime += cs.stackTime; } - } else { + } else { if(combCallStats == 0) { - PRINTCSTT(ios, cs, delimString); - } else { + PRINTCSTT(ios, cs, delimString); + } else { combCallStats->nCSCalls += cs.nCSCalls; combCallStats->totalTime += cs.totalTime; combCallStats->stackTime += cs.stackTime; - PRINTCSTT(ios, (*combCallStats), delimString); + PRINTCSTT(ios, (*combCallStats), delimString); delete combCallStats; combCallStats = 0; } - } - } else { + } + } else { if(combCallStats != 0) { - PRINTCSTT(ios, (*combCallStats), delimString); + PRINTCSTT(ios, (*combCallStats), delimString); delete combCallStats; combCallStats = 0; } - PRINTCSTT(ios, cs, delimString); - } + PRINTCSTT(ios, cs, delimString); + } } ClearBlock(dBlock); } @@ -1392,10 +1392,10 @@ void RegionsProfStats::WriteTextTrace(std::ostream &ios, bool simpleCombine, for(int i(0); i < vCallTrace.size(); ++i) { BLProfiler::CallStats &cs = vCallTrace[i]; - if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { - continue; - } - PRINTCSTT(ios, cs, delimString); + if(cs.callStackDepth < 0 || cs.csFNameNumber < 0) { + continue; + } + PRINTCSTT(ios, cs, delimString); } ClearBlock(dBlock); } diff --git a/Src/Extern/SWFFT/AlignedAllocator.h b/Src/Extern/SWFFT/AlignedAllocator.h index eb9b92de38e..2d16a843750 100644 --- a/Src/Extern/SWFFT/AlignedAllocator.h +++ b/Src/Extern/SWFFT/AlignedAllocator.h @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -75,7 +75,7 @@ class AlignedAllocator template struct rebind { - typedef AlignedAllocator other; + typedef AlignedAllocator other; }; public: diff --git a/Src/Extern/SWFFT/CheckDecomposition.c b/Src/Extern/SWFFT/CheckDecomposition.c index 2f76f78a159..0bd4a978c8d 100644 --- a/Src/Extern/SWFFT/CheckDecomposition.c +++ b/Src/Extern/SWFFT/CheckDecomposition.c @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -106,10 +106,10 @@ int main(int argc, char *argv[]) { } if(np3d != nproc) { fprintf(stderr,"ERROR: %d * %d * %d = %d != %d\n", - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2], - np3d, nproc); + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2], + np3d, nproc); exit(-1); } } @@ -122,13 +122,13 @@ int main(int argc, char *argv[]) { d->process_topology_1.nproc[2] = 1; // don't distribute outer dimensions period[0] = period[1] = period[2] = 1; //process_topology_1.nproc is filled with number of processors in each dim - MPI_Dims_create(nproc, ndim, d->process_topology_1.nproc); + MPI_Dims_create(nproc, ndim, d->process_topology_1.nproc); if(self == 0) { printf("distribution 1D: [%d:%d:%d]\n", - d->process_topology_1.nproc[0], - d->process_topology_1.nproc[1], - d->process_topology_1.nproc[2]); + d->process_topology_1.nproc[0], + d->process_topology_1.nproc[1], + d->process_topology_1.nproc[2]); fflush(stdout); } @@ -136,7 +136,7 @@ int main(int argc, char *argv[]) { d->process_topology_1.n[0] = n[0] / d->process_topology_1.nproc[0]; d->process_topology_1.n[1] = n[1] / d->process_topology_1.nproc[1]; d->process_topology_1.n[2] = n[2] / d->process_topology_1.nproc[2]; - + // set up process grid with 3d decomposition (CUBE) @@ -150,16 +150,16 @@ int main(int argc, char *argv[]) { if(self == 0) { printf("distribution 3D: [%d:%d:%d]\n", - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); fflush(stdout); } - + assert(n[0]%d->process_topology_3.nproc[0] == 0); assert(n[0]%d->process_topology_3.nproc[1] == 0); assert(n[0]%d->process_topology_3.nproc[2] == 0); - + //set local dimensions d->process_topology_3.n[0] = n[0] / d->process_topology_3.nproc[0]; d->process_topology_3.n[1] = n[1] / d->process_topology_3.nproc[1]; @@ -170,160 +170,160 @@ int main(int argc, char *argv[]) { // set up process grid with 2d decomposition (z_PENCILs ) d->process_topology_2_z.nproc[0] = 0; d->process_topology_2_z.nproc[1] = 0; - d->process_topology_2_z.nproc[2] = 1; // don't distribute outer dimension + d->process_topology_2_z.nproc[2] = 1; // don't distribute outer dimension period[0] = period[1] = period[2] = 1; MPI_Dims_create(nproc, ndim, d->process_topology_2_z.nproc); d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; - //variable used to ensure that pencils created fit inside the cuboids, + //variable used to ensure that pencils created fit inside the cuboids, //if not the code will assert out. - bool check_z_dims=false; - if(d->process_topology_2_z.n[0] != 0 - && d->process_topology_2_z.n[1] != 0 + bool check_z_dims=false; + if(d->process_topology_2_z.n[0] != 0 + && d->process_topology_2_z.n[1] != 0 && d->process_topology_2_z.n[2] != 0) {// protects from dividing by zero. - check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) + check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); - + if(self==0 && debug && !check_z_dims) - fprintf(stderr,"Need to fix Z PENCILS z_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_z.ns(%d,%d,%d)\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_3.n[0], - d->process_topology_3.n[1], - d->process_topology_3.n[2], - d->process_topology_2_z.n[0], - d->process_topology_2_z.n[1], - d->process_topology_2_z.n[2]); - - //try swaping pencil dimensions if current setup pencil dimensions dont + fprintf(stderr,"Need to fix Z PENCILS z_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_z.ns(%d,%d,%d)\n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_3.n[0], + d->process_topology_3.n[1], + d->process_topology_3.n[2], + d->process_topology_2_z.n[0], + d->process_topology_2_z.n[1], + d->process_topology_2_z.n[2]); + + //try swaping pencil dimensions if current setup pencil dimensions dont //fit inside the cubes. - if(!(check_z_dims) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[1]) == 0) + if(!(check_z_dims) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[1]) == 0) && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[0]) == 0)) { if(self==0 && debug) - fprintf(stderr,"Swaping Z pencils in initialization (%d,%d,%d)\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2]); + fprintf(stderr,"Swaping Z pencils in initialization (%d,%d,%d)\n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2]); int temp=d->process_topology_2_z.nproc[0]; d->process_topology_2_z.nproc[0] = d->process_topology_2_z.nproc[1]; d->process_topology_2_z.nproc[1] = temp; d->process_topology_2_z.nproc[2] = d->process_topology_2_z.nproc[2]; - + d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; - check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); + check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); } } else { check_z_dims=false; } -// if that did not work, make a pencil that does if inside the 3d cuboids by -// taking the cuboids dimensions (np1,np2,np3) and making pencils -// (np1,np2*np3,1), or (np1*np3,np2,1) on the most evenly distributed +// if that did not work, make a pencil that does if inside the 3d cuboids by +// taking the cuboids dimensions (np1,np2,np3) and making pencils +// (np1,np2*np3,1), or (np1*np3,np2,1) on the most evenly distributed // dimensions if(!check_z_dims){ if(self==0 && debug) - fprintf(stderr,"MAKING Z PENCILS FIT zprocs(%d,%d,%d) z.ns(%d,%d,%d)\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_2_z.n[0], - d->process_topology_2_z.n[1], - d->process_topology_2_z.n[2]); - + fprintf(stderr,"MAKING Z PENCILS FIT zprocs(%d,%d,%d) z.ns(%d,%d,%d)\n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_2_z.n[0], + d->process_topology_2_z.n[1], + d->process_topology_2_z.n[2]); + d->process_topology_2_z.nproc[2]=1; if(d->process_topology_3.n[0]>d->process_topology_3.n[1]) { d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]; d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]; - if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) - || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) + if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) + || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) { - d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]*d->process_topology_3.nproc[2]; - d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]; + d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]*d->process_topology_3.nproc[2]; + d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]; } } else { d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]*d->process_topology_3.nproc[2]; d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]; - if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) - || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) + if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) + || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) { - d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]; - d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]; + d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]; + d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]; } } d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; if(self==0 && debug) - fprintf(stderr,"MAKING Z PENCILS FIT AFTER zprocs(%d,%d,%d) z.ns(%d,%d,%d)\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_2_z.n[0], - d->process_topology_2_z.n[1], - d->process_topology_2_z.n[2]); - if(d->process_topology_2_z.n[0] != 0 - && d->process_topology_2_z.n[1] != 0 + fprintf(stderr,"MAKING Z PENCILS FIT AFTER zprocs(%d,%d,%d) z.ns(%d,%d,%d)\n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_2_z.n[0], + d->process_topology_2_z.n[1], + d->process_topology_2_z.n[2]); + if(d->process_topology_2_z.n[0] != 0 + && d->process_topology_2_z.n[1] != 0 && d->process_topology_2_z.n[2] != 0) {// protects from dividing by zero. - check_z_dims=((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); + check_z_dims=((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); } else { check_z_dims=false; } } - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_z: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_2_z.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_2_z.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); - } + } if(!check_z_dims && debug && (self==0)){ FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"Z DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); } //assert(check_z_dims); if(!check_z_dims) fprintf(stderr,"assert(check_z_dims) would have failed.\n"); -// if this happens, it is because the dimensions were chosen incorrectly. -// Either to many processors for the number of points in one dimenison (could -// not do at least 1 point per processor), or the methods above could -// not make a distribution of pencils that fit in the cubiods, which would -// happen if the user gave numbers that wouldent work (we require the number -// of processors in each dimension of the cuboid must be modulo the number of +// if this happens, it is because the dimensions were chosen incorrectly. +// Either to many processors for the number of points in one dimenison (could +// not do at least 1 point per processor), or the methods above could +// not make a distribution of pencils that fit in the cubiods, which would +// happen if the user gave numbers that wouldent work (we require the number +// of processors in each dimension of the cuboid must be modulo the number of // points in that dimension, otherwise, this error will happen). if(self == 0) { printf("distribution 2z: [%d:%d:%d]\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2]); + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2]); fflush(stdout); } @@ -338,86 +338,86 @@ int main(int argc, char *argv[]) { d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; d->process_topology_2_x.n[1] = n[1] / d->process_topology_2_x.nproc[1]; d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; - //variable used to ensure that pencils created fit inside the cuboids, + //variable used to ensure that pencils created fit inside the cuboids, //if not the code will assert out. bool check_x_dims = false; - if(d->process_topology_2_x.n[0] != 0 - && d->process_topology_2_x.n[1] != 0 + if(d->process_topology_2_x.n[0] != 0 + && d->process_topology_2_x.n[1] != 0 && d->process_topology_2_x.n[2] != 0) {// protects from dividing by zero. - check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) + check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) && (n[0] % (d->process_topology_2_x.nproc[0]) == 0); if(self==0 && debug && !check_x_dims) - fprintf(stderr,"Need to fix X PENCILS x_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_x.ns(%d,%d,%d)...\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_3.n[0], - d->process_topology_3.n[1], - d->process_topology_3.n[2], - d->process_topology_2_x.n[0], - d->process_topology_2_x.n[1], - d->process_topology_2_x.n[2]); - - //try swaping pencil dimensions if current setup does not have pencils + fprintf(stderr,"Need to fix X PENCILS x_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_x.ns(%d,%d,%d)...\n", + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_3.n[0], + d->process_topology_3.n[1], + d->process_topology_3.n[2], + d->process_topology_2_x.n[0], + d->process_topology_2_x.n[1], + d->process_topology_2_x.n[2]); + + //try swaping pencil dimensions if current setup does not have pencils //that fit inside cubes. - if(!(check_x_dims) - && ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[1]) == 0) + if(!(check_x_dims) + && ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[1]) == 0) && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[2]) == 0)) { if(self==0 && debug) - fprintf(stderr,"Swaping X pencils in initialization .... \n"); + fprintf(stderr,"Swaping X pencils in initialization .... \n"); d->process_topology_2_x.nproc[0] = d->process_topology_2_x.nproc[0]; int temp = d->process_topology_2_x.nproc[1]; d->process_topology_2_x.nproc[1] = d->process_topology_2_x.nproc[2]; d->process_topology_2_x.nproc[2] = temp; - + d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; d->process_topology_2_x.n[1] = n[1] / d->process_topology_2_x.nproc[1]; d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; - check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[0]) == 0); - } + check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[0]) == 0); + } } else{ check_x_dims=false; } -// if that did not work, make a pencil that does by taking the cuboid -// (np1,np2,np3) and making pencils of the form (1,np2*np1,np3) or +// if that did not work, make a pencil that does by taking the cuboid +// (np1,np2,np3) and making pencils of the form (1,np2*np1,np3) or // (1,np2*np1,np3) depending on the most even distribution it can. if(!check_x_dims){ if(self==0 && debug) fprintf(stderr,"MAKING X PENCILS FIT xprocs(%d,%d,%d) x.ns(%d,%d,%d)...\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_2_x.n[0], - d->process_topology_2_x.n[1], - d->process_topology_2_x.n[2]); + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_2_x.n[0], + d->process_topology_2_x.n[1], + d->process_topology_2_x.n[2]); d->process_topology_2_x.nproc[0] = 1; if(d->process_topology_3.nproc[2] > d->process_topology_3.nproc[1]) { d->process_topology_2_x.nproc[1] = d->process_topology_3.nproc[1]*d->process_topology_3.nproc[0]; d->process_topology_2_x.nproc[2] = d->process_topology_3.nproc[2]; - if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) { - d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]*d->process_topology_3.nproc[0]; - d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]; + d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]*d->process_topology_3.nproc[0]; + d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]; } } else { d->process_topology_2_x.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[0]; d->process_topology_2_x.nproc[1] = d->process_topology_3.nproc[1]; - if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) { - d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[0]; - d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]; + d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[0]; + d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]; } } d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; @@ -425,31 +425,31 @@ int main(int argc, char *argv[]) { d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; if(self==0 && debug) fprintf(stderr,"MAKING X PENCILS FIT AFTER xprocs(%d,%d,%d) x.ns(%d,%d,%d)...\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_2_x.n[0], - d->process_topology_2_x.n[1], - d->process_topology_2_x.n[2]); - if(d->process_topology_2_x.n[0] != 0 - && d->process_topology_2_x.n[1] != 0 + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_2_x.n[0], + d->process_topology_2_x.n[1], + d->process_topology_2_x.n[2]); + if(d->process_topology_2_x.n[0] != 0 + && d->process_topology_2_x.n[1] != 0 && d->process_topology_2_x.n[2] != 0) {// protects from dividing by zero. - check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[0]) == 0); + check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[0]) == 0); } else { check_x_dims=false; - } + } } - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_x: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_2_x.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_2_x.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); } @@ -457,32 +457,32 @@ int main(int argc, char *argv[]) { FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"X DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); } //assert(check_x_dims); if(!check_x_dims) fprintf(stderr,"assert(check_x_dims) would have failed.\n"); -// if this happens, it is because the dimensions were chosen incorrectly. -// Either to many processors for the number of points in one dimenison (could -// not do at least 1 point per processor), or the methods above could not make -// a distribution of pencils that fit in the cubiods, which would happen if the -// user gave numbers that wouldent work (we require the number of processors in -// each dimension of the cuboid must be modulo the number of points in that +// if this happens, it is because the dimensions were chosen incorrectly. +// Either to many processors for the number of points in one dimenison (could +// not do at least 1 point per processor), or the methods above could not make +// a distribution of pencils that fit in the cubiods, which would happen if the +// user gave numbers that wouldent work (we require the number of processors in +// each dimension of the cuboid must be modulo the number of points in that // dimension, otherwise, this error will happen). if(self == 0) { printf("distribution 2x: [%d:%d:%d]\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2]); + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2]); fflush(stdout); } - + // set up process grid with 2d decomposition (y_PENCILs) @@ -494,117 +494,117 @@ int main(int argc, char *argv[]) { d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; - //variable used to ensure that pencils created fit inside the cuboids, + //variable used to ensure that pencils created fit inside the cuboids, //if not the code will assert out. bool check_y_dims=false; - if(d->process_topology_2_y.n[0] != 0 - && d->process_topology_2_y.n[1] != 0 + if(d->process_topology_2_y.n[0] != 0 + && d->process_topology_2_y.n[1] != 0 && d->process_topology_2_y.n[2] != 0) {// protects from dividing by zero. - check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); + check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); if(self==0 && debug && !check_y_dims) fprintf(stderr,"Need to fix Y PENCILS y_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_y.ns(%d,%d,%d)...\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_3.n[0], - d->process_topology_3.n[1], - d->process_topology_3.n[2], - d->process_topology_2_y.n[0], - d->process_topology_2_y.n[1], - d->process_topology_2_y.n[2]); - //try swaping pencil dimensions if the current dimension of the pencils + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_3.n[0], + d->process_topology_3.n[1], + d->process_topology_3.n[2], + d->process_topology_2_y.n[0], + d->process_topology_2_y.n[1], + d->process_topology_2_y.n[2]); + //try swaping pencil dimensions if the current dimension of the pencils //does not fit inside the cubes. - if(!(check_y_dims) - && ((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[0]) == 0) + if(!(check_y_dims) + && ((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[0]) == 0) && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[2]) == 0)) { if(self==0 && debug) - fprintf(stderr,"Swaping Y pencils in initialization .... \n"); - + fprintf(stderr,"Swaping Y pencils in initialization .... \n"); + int temp = d->process_topology_2_y.nproc[0]; d->process_topology_2_y.nproc[0] = d->process_topology_2_y.nproc[2]; d->process_topology_2_y.nproc[2] = temp; d->process_topology_2_y.nproc[1] = d->process_topology_2_y.nproc[1]; - + d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; - check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); + check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); } } else { check_y_dims = false; } -// if that did not work, make a pencil that does by taking the cuboid -// (np1,np2,np3) and making pencils of the form (np1,1,np3*np2) or +// if that did not work, make a pencil that does by taking the cuboid +// (np1,np2,np3) and making pencils of the form (np1,1,np3*np2) or // (np1*np2,1,np3) depending on the most even distribution it can. if(!check_y_dims){ if(self==0 && debug) - fprintf(stderr,"MAKING Y PENCILS FIT yprocs(%d,%d,%d) y.ns(%d,%d,%d)...\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_2_y.n[0], - d->process_topology_2_y.n[1], - d->process_topology_2_y.n[2]); - + fprintf(stderr,"MAKING Y PENCILS FIT yprocs(%d,%d,%d) y.ns(%d,%d,%d)...\n", + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_2_y.n[0], + d->process_topology_2_y.n[1], + d->process_topology_2_y.n[2]); + d->process_topology_2_y.nproc[1]=1; if(d->process_topology_3.nproc[2] > d->process_topology_3.nproc[0]) { d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]*d->process_topology_3.nproc[1]; d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]; - if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) { - d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]; - d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]; + d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]; + d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]; } } else { d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]; d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]; - if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) { - d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]*d->process_topology_3.nproc[1]; - d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]; + d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]*d->process_topology_3.nproc[1]; + d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]; } } - + d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; if(self==0 && debug) fprintf(stderr,"MAKING Y PENCILS FIT AFTER yprocs(%d,%d,%d) y.ns(%d,%d,%d)...\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_2_y.n[0], - d->process_topology_2_y.n[1], - d->process_topology_2_y.n[2]); - if(d->process_topology_2_y.n[0] != 0 && d->process_topology_2_y.n[1] != 0 + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_2_y.n[0], + d->process_topology_2_y.n[1], + d->process_topology_2_y.n[2]); + if(d->process_topology_2_y.n[0] != 0 && d->process_topology_2_y.n[1] != 0 && d->process_topology_2_y.n[2] != 0) {// protects from dividing by zero. - check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); + check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); } else { check_y_dims=false; } } - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_y: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_2_y.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_2_y.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); } @@ -612,30 +612,30 @@ int main(int argc, char *argv[]) { FILE * outfile; outfile = fopen("error.data","a"); fprintf(outfile,"Y DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); } //assert(check_y_dims); if(!check_y_dims) fprintf(stderr,"assert(check_y_dims) would have failed.\n"); -// if this happens, it is because the dimensions were chosen incorrectly. -// Either to many processors for the number of points in one dimenison (could -// not do at least 1 point per processor), or the methods above could -// not make a distribution of pencils that fit in the cubiods, which would -// happen if the user gave numbers that wouldent work (we require the number of -// processors in each dimension of the cuboid must be modulo the number of +// if this happens, it is because the dimensions were chosen incorrectly. +// Either to many processors for the number of points in one dimenison (could +// not do at least 1 point per processor), or the methods above could +// not make a distribution of pencils that fit in the cubiods, which would +// happen if the user gave numbers that wouldent work (we require the number of +// processors in each dimension of the cuboid must be modulo the number of // points in that dimension, otherwise, this error will happen). if(self == 0) { printf("distribution 2y: [%d:%d:%d]\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2]); + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2]); fflush(stdout); } diff --git a/Src/Extern/SWFFT/Dfft.H b/Src/Extern/SWFFT/Dfft.H index 4254ca1d11a..7f39376036d 100644 --- a/Src/Extern/SWFFT/Dfft.H +++ b/Src/Extern/SWFFT/Dfft.H @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -88,7 +88,7 @@ class Dfft { public: - // + // int global_ng(int i) const { return d.global_ng(i); } int const (& global_ng() const)[3] { return d.global_ng(); } size_t global_size() const { return d.global_size(); } @@ -96,7 +96,7 @@ public: - // + // size_t local_size() const { return d.local_size(); } @@ -256,7 +256,7 @@ public: // copy from overloaded in to m_fo const int *local_dim_r = local_ng_rspace(); int indexc = 0; - int indexf = ghost0*(ghost0 + local_dim_r[2] + ghost1)*(ghost0 + local_dim_r[1] + ghost1); + int indexf = ghost0*(ghost0 + local_dim_r[2] + ghost1)*(ghost0 + local_dim_r[1] + ghost1); for(int local_r0=0; local_r0 < local_dim_r[0]; ++local_r0) { indexf += ghost0*(ghost0 + local_dim_r[2] + ghost1); for(int local_r1=0; local_r1 < local_dim_r[1]; ++local_r1) { @@ -455,11 +455,11 @@ public: unsigned int flags = FFTW_MEASURE) : d(dist), PlansMade(false) { - makePlans(forward_output, - forward_scratch, - backward_input, - backward_scratch, - flags); + makePlans(forward_output, + forward_scratch, + backward_input, + backward_scratch, + flags); } @@ -476,14 +476,14 @@ public: // FFTW_MEASURE = 0 void makePlans(complex_t *forward_output, - complex_t *forward_scratch, - complex_t *backward_input, - complex_t *backward_scratch, - unsigned int flags = FFTW_MEASURE) + complex_t *forward_scratch, + complex_t *backward_input, + complex_t *backward_scratch, + unsigned int flags = FFTW_MEASURE) { - if(forward_output == forward_scratch) + if(forward_output == forward_scratch) Error() << "Dfft::setBuffers() forward_output == forward_scratch"; - if(backward_input == backward_scratch) + if(backward_input == backward_scratch) Error() << "Dfft::setBuffers() backward_input == backward_scratch"; m_fo = forward_output; @@ -497,8 +497,8 @@ public: // fs --> fo m_plan_f_x = fftw_plan_many_dft(1, // rank - &(d.m_d.process_topology_2_x.n[0]), // const int *n, - d.m_d.process_topology_2_x.n[1] * d.m_d.process_topology_2_x.n[2], // howmany + &(d.m_d.process_topology_2_x.n[0]), // const int *n, + d.m_d.process_topology_2_x.n[1] * d.m_d.process_topology_2_x.n[2], // howmany FFTW_ADDR(m_fs), NULL, // const int *inembed, 1, // int istride, diff --git a/Src/Extern/SWFFT/DfftC.cpp b/Src/Extern/SWFFT/DfftC.cpp index a5896eefca9..6576003e844 100644 --- a/Src/Extern/SWFFT/DfftC.cpp +++ b/Src/Extern/SWFFT/DfftC.cpp @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -53,7 +53,7 @@ */ /// -// Give C linkage to C++ Dfft class so that Fortran can access its functions. +// Give C linkage to C++ Dfft class so that Fortran can access its functions. /// #include "complex-type.h" @@ -66,7 +66,7 @@ extern "C" { return new hacc::Dfft(dist); } - void Dfft__makePlans(hacc::Dfft* This, complex_t *forward_output, complex_t *forward_scratch, + void Dfft__makePlans(hacc::Dfft* This, complex_t *forward_output, complex_t *forward_scratch, complex_t *backward_input, complex_t *backward_scratch, unsigned int flags) { This->makePlans(forward_output, forward_scratch, backward_input, backward_scratch, flags); } @@ -93,7 +93,7 @@ extern "C" { void Dfft__self_rspace(hacc::Dfft* This, int n[3]) { for(size_t i = 0; i < 3; ++i) n[i] = This->self_rspace(i); - } + } void Dfft__nproc_rspace(hacc::Dfft* This, int n[3]) { for(size_t i = 0; i < 3; ++i) n[i] = This->nproc_rspace(i); diff --git a/Src/Extern/SWFFT/Distribution.H b/Src/Extern/SWFFT/Distribution.H index d8fadfdf4c8..7048f11b2d3 100644 --- a/Src/Extern/SWFFT/Distribution.H +++ b/Src/Extern/SWFFT/Distribution.H @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -78,21 +78,21 @@ public: // // constructors // - + // standard setup - Distribution(MPI_Comm comm, - int const n[], - bool debug = false) + Distribution(MPI_Comm comm, + int const n[], + bool debug = false) : m_comm(comm), m_rmap(NULL), m_debug(debug) { int Ndims[3] = { 0, 0, 0 }; initialize(comm, n, Ndims); } - Distribution(MPI_Comm comm, - int ng, - bool debug = false) + Distribution(MPI_Comm comm, + int ng, + bool debug = false) : m_comm(comm), m_rmap(NULL), m_debug(debug) { int n[3] = { ng, ng, ng }; @@ -127,14 +127,14 @@ public: // destructor // - virtual ~Distribution() + virtual ~Distribution() { distribution_fini(&m_d); } // initialization - + void initialize(MPI_Comm comm, int const n[], int const Ndims[]) { int flag; MPI_Initialized(&flag); @@ -243,7 +243,7 @@ public: MPI_Comm parent_comm() const { return m_comm;} - // + // int rank_2d_x(int c[]) { int r; @@ -264,7 +264,7 @@ public: } - // + // void coords_2d_x(int r, int c[]) { Coord_x_pencils(r, c, &m_d);} void coords_2d_y(int r, int c[]) { Coord_y_pencils(r, c, &m_d);} diff --git a/Src/Extern/SWFFT/DistributionC.cpp b/Src/Extern/SWFFT/DistributionC.cpp index 0d3c8606a3c..e768f08f0ec 100644 --- a/Src/Extern/SWFFT/DistributionC.cpp +++ b/Src/Extern/SWFFT/DistributionC.cpp @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -55,7 +55,7 @@ /// // Distribution / partition / decomposition of data // -// Give C linkage to C++ Distribution class so that Fortran can access its functions. +// Give C linkage to C++ Distribution class so that Fortran can access its functions. /// #include "Distribution.hpp" @@ -69,7 +69,7 @@ extern "C" { MPI_Fint Distribution__Cart_3D(hacc::Distribution* This) { MPI_Comm comm = This->cart_3d(); - return MPI_Comm_c2f(comm); + return MPI_Comm_c2f(comm); } void Distribution__delete(hacc::Distribution* This) { diff --git a/Src/Extern/SWFFT/Error.h b/Src/Extern/SWFFT/Error.h index dbbc04cf701..e0607f7e246 100644 --- a/Src/Extern/SWFFT/Error.h +++ b/Src/Extern/SWFFT/Error.h @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -84,7 +84,7 @@ class Error { template Error &operator << (const T &Value) { SS << Value; - return *this; + return *this; } protected: diff --git a/Src/Extern/SWFFT/TimingStats.h b/Src/Extern/SWFFT/TimingStats.h index 645bb5f59a6..bc2b6c7c06c 100644 --- a/Src/Extern/SWFFT/TimingStats.h +++ b/Src/Extern/SWFFT/TimingStats.h @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -69,8 +69,8 @@ extern "C" { inline void printTimingStats(MPI_Comm comm, // comm for MPI_Allreduce() - const char *preamble, // text at beginning of line - double dt) // delta t in seconds + const char *preamble, // text at beginning of line + double dt) // delta t in seconds { int myrank, nranks; double max, min, sum, avg, var, stdev; @@ -91,7 +91,7 @@ void printTimingStats(MPI_Comm comm, // comm for MPI_Allreduce() if(myrank==0) { printf("%s max %.3es avg %.3es min %.3es dev %.3es\n", - preamble, max, avg, min, stdev); + preamble, max, avg, min, stdev); } MPI_Barrier(comm); diff --git a/Src/Extern/SWFFT/complex-type.h b/Src/Extern/SWFFT/complex-type.h index 0f5ce78feaa..069f2393d95 100644 --- a/Src/Extern/SWFFT/complex-type.h +++ b/Src/Extern/SWFFT/complex-type.h @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE diff --git a/Src/Extern/SWFFT/distribution.c b/Src/Extern/SWFFT/distribution.c index ad888c7d947..40f99cbf210 100644 --- a/Src/Extern/SWFFT/distribution.c +++ b/Src/Extern/SWFFT/distribution.c @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -83,11 +83,11 @@ static inline char const *separator(int i, int n) } -//Go from rank of processor to its cartesian coords, and vica versa. +//Go from rank of processor to its cartesian coords, and vica versa. //Assumes the ranks increment in the z dimension then y then x. -void Coord_cube(int myrank, - int coord[], - distribution_t *d) +void Coord_cube(int myrank, + int coord[], + distribution_t *d) { coord[0]=myrank/(d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]); coord[1]=(myrank%(d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]))/(d->process_topology_3.nproc[2]); @@ -95,9 +95,9 @@ void Coord_cube(int myrank, return; } -void Rank_cube(int * myrank, - int coord[], - distribution_t *d) +void Rank_cube(int * myrank, + int coord[], + distribution_t *d) { *myrank = coord[2] + (d->process_topology_3.nproc[2])*(coord[1] + d->process_topology_3.nproc[1]*coord[0]); return; @@ -105,16 +105,16 @@ void Rank_cube(int * myrank, /* - The subsequent member functions are used to look up and ranks of x,y, and z + The subsequent member functions are used to look up and ranks of x,y, and z pencils from their coordinates, and vica versa. - The ordering of the ranks is such that pencils will be going through cubes - with the same rank sequencially. (since the cubes ranks are already - determined and can not be changed, these routines figure out which ranks + The ordering of the ranks is such that pencils will be going through cubes + with the same rank sequencially. (since the cubes ranks are already + determined and can not be changed, these routines figure out which ranks the pencils should be assigned so that there is no communication hangs.) */ -void Coord_x_pencils(int myrank, - int coord[], - distribution_t *d) +void Coord_x_pencils(int myrank, + int coord[], + distribution_t *d) { // asserts only one processor in x_direction assert(d->process_topology_2_x.nproc[0] == 1); @@ -123,52 +123,52 @@ void Coord_x_pencils(int myrank, int num_pen_in_cube_col=d->process_topology_2_x.nproc[1]/d->process_topology_3.nproc[1]; int num_pen_in_cube_row=d->process_topology_2_x.nproc[2]/d->process_topology_3.nproc[2]; int num_cubes=(d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]); - + /* - the x_pencil ranks increment in each cube sequencially, after reaching the - last cube the second slot in the first cube is the next rank, and then the - process repeats. num_repeats, is the number of times this repetition had to + the x_pencil ranks increment in each cube sequencially, after reaching the + last cube the second slot in the first cube is the next rank, and then the + process repeats. num_repeats, is the number of times this repetition had to have occured to increment to the current rank. */ int num_repeats=myrank/(num_cubes); - - //now subtract the difference of how many repetitions, to find the lowest - //rank in the cube it resides. + + //now subtract the difference of how many repetitions, to find the lowest + //rank in the cube it resides. int low_rank=myrank-num_repeats*num_cubes; - - //find the y and z coords of the low_rank, then adjust coords for ranks + + //find the y and z coords of the low_rank, then adjust coords for ranks //that repeated around the cube. - coord[1] = (low_rank/d->process_topology_3.nproc[2])*num_pen_in_cube_col + coord[1] = (low_rank/d->process_topology_3.nproc[2])*num_pen_in_cube_col + num_repeats%num_pen_in_cube_col; coord[2] = (low_rank%d->process_topology_3.nproc[2])*num_pen_in_cube_row + num_repeats/num_pen_in_cube_col; - + return; } -void Rank_x_pencils(int * myrank, - int coord[], - distribution_t *d) +void Rank_x_pencils(int * myrank, + int coord[], + distribution_t *d) { int num_pen_in_cube_col=d->process_topology_2_x.nproc[1]/d->process_topology_3.nproc[1]; int num_pen_in_cube_row=d->process_topology_2_x.nproc[2]/d->process_topology_3.nproc[2]; if(num_pen_in_cube_col == 0) - fprintf(stderr,"num_cube_col%d ", - d->process_topology_2_x.nproc[1]/d->process_topology_3.nproc[1]); + fprintf(stderr,"num_cube_col%d ", + d->process_topology_2_x.nproc[1]/d->process_topology_3.nproc[1]); if(num_pen_in_cube_row ==0) fprintf(stderr,"num_cube_row%d ", d->process_topology_3.nproc[2]); assert(num_pen_in_cube_col !=0 && num_pen_in_cube_row !=0); int alpha = coord[1]%num_pen_in_cube_col; int num_cubes = (d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]); int beta = coord[2]%num_pen_in_cube_row; - *myrank = (alpha*num_cubes) - + ((coord[1]/num_pen_in_cube_col)*d->process_topology_3.nproc[2]) + *myrank = (alpha*num_cubes) + + ((coord[1]/num_pen_in_cube_col)*d->process_topology_3.nproc[2]) + (beta*(num_cubes)*num_pen_in_cube_col) + coord[2]/num_pen_in_cube_row; return; } -void Coord_y_pencils(int myrank, - int coord[], - distribution_t *d) +void Coord_y_pencils(int myrank, + int coord[], + distribution_t *d) { // asserts only one processor in y_direction assert(d->process_topology_2_y.nproc[1] == 1); @@ -177,16 +177,16 @@ void Coord_y_pencils(int myrank, int num_pen_in_cube_row = d->process_topology_2_y.nproc[2]/d->process_topology_3.nproc[2]; int alpha = myrank%(d->process_topology_2_y.nproc[2]); coord[0] = myrank/d->process_topology_2_y.nproc[2]; - - coord[2] = (alpha/d->process_topology_3.nproc[2]) + + coord[2] = (alpha/d->process_topology_3.nproc[2]) + (alpha%d->process_topology_3.nproc[2])*num_pen_in_cube_row; - + return; } -void Rank_y_pencils(int * myrank, - int coord[], - distribution_t *d) +void Rank_y_pencils(int * myrank, + int coord[], + distribution_t *d) { int num_pen_in_cube_col = d->process_topology_2_y.nproc[0]/d->process_topology_3.nproc[0]; int num_pen_in_cube_row = d->process_topology_2_y.nproc[2]/d->process_topology_3.nproc[2]; @@ -195,15 +195,15 @@ void Rank_y_pencils(int * myrank, //if(num_pen_in_cube_row ==0)fprintf(stderr,"num_cube_row%d ", d->process_topology_3.nproc[2]); assert(num_pen_in_cube_col !=0 && num_pen_in_cube_row !=0); int beta = coord[2]%num_pen_in_cube_row; - *myrank = coord[0]*d->process_topology_2_y.nproc[2] - + beta*d->process_topology_3.nproc[2] + *myrank = coord[0]*d->process_topology_2_y.nproc[2] + + beta*d->process_topology_3.nproc[2] + coord[2]/num_pen_in_cube_row; return; } -void Coord_z_pencils(int myrank, - int coord[], - distribution_t *d) +void Coord_z_pencils(int myrank, + int coord[], + distribution_t *d) { // asserts only one processor in z_direction assert(d->process_topology_2_z.nproc[2] == 1); @@ -215,68 +215,68 @@ void Coord_z_pencils(int myrank, int alpha = myrank/(d->process_topology_2_z.nproc[1]*num_pen_in_cube_row); coord[0] = alpha*num_pen_in_cube_row + (myrank%num_pen_in_cube)/num_pen_in_cube_col; coord[1] = ((myrank%(d->process_topology_2_z.nproc[1]*num_pen_in_cube_row))/num_pen_in_cube)*num_pen_in_cube_col + myrank%num_pen_in_cube_col; - + return; } -void Rank_z_pencils(int * myrank, - int coord[], - distribution_t *d) +void Rank_z_pencils(int * myrank, + int coord[], + distribution_t *d) { int num_pen_in_cube_col = d->process_topology_2_z.nproc[1]/d->process_topology_3.nproc[1]; int num_pen_in_cube_row = d->process_topology_2_z.nproc[0]/d->process_topology_3.nproc[0]; int num_pen_in_cube = d->process_topology_3.nproc[2]; if(num_pen_in_cube_col == 0) - fprintf(stderr,"num_cube_col%d ", - d->process_topology_2_z.nproc[1]/d->process_topology_3.nproc[1]); + fprintf(stderr,"num_cube_col%d ", + d->process_topology_2_z.nproc[1]/d->process_topology_3.nproc[1]); if(num_pen_in_cube_row == 0) fprintf(stderr,"num_cube_row%d ", d->process_topology_3.nproc[2]); assert(num_pen_in_cube_col !=0 && num_pen_in_cube_row !=0); int alpha = coord[1]%num_pen_in_cube_col; int beta = coord[0]%num_pen_in_cube_row; - *myrank = alpha - + ((coord[1]/num_pen_in_cube_col)*num_pen_in_cube) - + (beta*num_pen_in_cube_col) + *myrank = alpha + + ((coord[1]/num_pen_in_cube_col)*num_pen_in_cube) + + (beta*num_pen_in_cube_col) + (coord[0]/num_pen_in_cube_row)*d->process_topology_2_z.nproc[1]*num_pen_in_cube_row; return; } // create 1-, 2- and 3-d cartesian data distributions comm MPI Communicator -void distribution_init(MPI_Comm comm, - const int n[], - const int Ndims[], - distribution_t *d, +void distribution_init(MPI_Comm comm, + const int n[], + const int Ndims[], + distribution_t *d, const int* rmap, - bool debug) + bool debug) { -/* - As of 09/06/2011 The MPI function MPI_Dims_create is used to come up with - the most evenly distributed number of processors for the 3d distribution. +/* + As of 09/06/2011 The MPI function MPI_Dims_create is used to come up with + the most evenly distributed number of processors for the 3d distribution. Since this can actually vary between machines, we should later write our own - prime factorization function that does that for us. For the 2d distribution - pencils, Dims_create is also used, but the code then checks if the pencils - it outputs fits inside the 3d cuboids that were created. If it does not, it - tries swapping the dimensions of the pencils, and if they still do not fit, - it takes the 3d cubes dimensions of processors (np1,np2,np3) and (for - Z-pencils for example) makes pencils of the form (np1*np3,np2,1), or - (np1,np2*np3,1) which fit inside the cubes. However, even then, it is not - ensured that this will work since for example, if np1*np3 is bigger then - the number of points in one dimension (Ng) then there are not enough points + prime factorization function that does that for us. For the 2d distribution + pencils, Dims_create is also used, but the code then checks if the pencils + it outputs fits inside the 3d cuboids that were created. If it does not, it + tries swapping the dimensions of the pencils, and if they still do not fit, + it takes the 3d cubes dimensions of processors (np1,np2,np3) and (for + Z-pencils for example) makes pencils of the form (np1*np3,np2,1), or + (np1,np2*np3,1) which fit inside the cubes. However, even then, it is not + ensured that this will work since for example, if np1*np3 is bigger then + the number of points in one dimension (Ng) then there are not enough points for each processor to have at least one point in that dimension. So the code - checks this and asserts three variables check_x_dims check_y_dims, and - check_z_dims, which will assert if these kinda errors happen (as well as - checking errors coming from picking the total number of processors and Ng - in a way where the cubes will not fit for any orientation (like 100 procs - and Ng=101!)). Curretly the fix to these errors is to pick better values - for Ng and the total number of processors that work, however when we do - have our own prime factorization method, then that method could also make - pencils that fit inside the proper distribution (and we would not need so - many checks). In the mean time, to pick these "better" values for Ng, the - user should pick values such that: Ng % np1, Ng % np2, and Ng % np3 all + checks this and asserts three variables check_x_dims check_y_dims, and + check_z_dims, which will assert if these kinda errors happen (as well as + checking errors coming from picking the total number of processors and Ng + in a way where the cubes will not fit for any orientation (like 100 procs + and Ng=101!)). Curretly the fix to these errors is to pick better values + for Ng and the total number of processors that work, however when we do + have our own prime factorization method, then that method could also make + pencils that fit inside the proper distribution (and we would not need so + many checks). In the mean time, to pick these "better" values for Ng, the + user should pick values such that: Ng % np1, Ng % np2, and Ng % np3 all equal zero, and that np1*np2, np2*np3, and np3*np1 are all less then Ng. - (in other words, the cubes created fit inside the number of grid points, - and the number of pencils created is not more then the number of points + (in other words, the cubes created fit inside the number of grid points, + and the number of pencils created is not more then the number of points in a dimension (Ng)). */ d->parent = comm; @@ -289,7 +289,7 @@ void distribution_init(MPI_Comm comm, MPI_Comm_rank(comm, &self); MPI_Comm_size(comm, &nproc); - // Construct the rankmap[grid] --> rank map (converts grids to ranks) from the input + // Construct the rankmap[grid] --> rank map (converts grids to ranks) from the input // (if none is provided then we assume the trivial map {0, 1, ..., nproc}). d->rankmap = (int *) malloc(sizeof(int)*nproc); if(rmap) for(int i=0; irankmap[i] = rmap[i]; @@ -304,16 +304,16 @@ void distribution_init(MPI_Comm comm, } // Map this rank to the correct grid - self = d->gridmap[self]; + self = d->gridmap[self]; - if (!self) + if (!self) printf("Initializing redistribution using a %s layout on %d ranks.\n", #ifdef PENCIL - "pencil" + "pencil" #else - "slab" + "slab" #endif - ,nproc); + ,nproc); d->debug = debug; for (int i = 0; i < 3; ++i) @@ -325,13 +325,13 @@ void distribution_init(MPI_Comm comm, d->process_topology_1.nproc[2] = 1; // don't distribute outer dimensions period[0] = period[1] = period[2] = 1; //process_topology_1.nproc is filled with number of processors in each dim - MPI_Dims_create(nproc, ndim, d->process_topology_1.nproc); + MPI_Dims_create(nproc, ndim, d->process_topology_1.nproc); if(self == 0) { printf("distribution 1D: [%d:%d:%d]\n", - d->process_topology_1.nproc[0], - d->process_topology_1.nproc[1], - d->process_topology_1.nproc[2]); + d->process_topology_1.nproc[0], + d->process_topology_1.nproc[1], + d->process_topology_1.nproc[2]); fflush(stdout); } @@ -339,18 +339,18 @@ void distribution_init(MPI_Comm comm, fprintf(stderr, "Process grids:\n"); fprintf(stderr, " 1d: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_1.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_1.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); } //creates the new communicator - MPI_Cart_create(comm, ndim, d->process_topology_1.nproc, period, 0, - &d->process_topology_1.cart); + MPI_Cart_create(comm, ndim, d->process_topology_1.nproc, period, 0, + &d->process_topology_1.cart); //gets .self (is coordinate) - MPI_Cart_get(d->process_topology_1.cart, ndim, d->process_topology_1.nproc, - d->process_topology_1.period, d->process_topology_1.self); + MPI_Cart_get(d->process_topology_1.cart, ndim, d->process_topology_1.nproc, + d->process_topology_1.period, d->process_topology_1.self); //calculates the local dimensions (number of points in each dimension) d->process_topology_1.n[0] = n[0] / d->process_topology_1.nproc[0]; d->process_topology_1.n[1] = n[1] / d->process_topology_1.nproc[1]; @@ -367,51 +367,51 @@ void distribution_init(MPI_Comm comm, if(self == 0) { printf("distribution 3D: [%d:%d:%d]\n", - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); fflush(stdout); } - + if (d->debug && 0 == self) { fprintf(stderr, " 3d: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_3.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_3.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); } - MPI_Cart_create(comm, ndim, d->process_topology_3.nproc, period, 0, - &d->process_topology_3.cart); + MPI_Cart_create(comm, ndim, d->process_topology_3.nproc, period, 0, + &d->process_topology_3.cart); //finds cartesian coordinate of this current rank Coord_cube(self,d->process_topology_3.self,d); if(debug){ /* - this debug statment checks to see if the way coordinates found by - calculation matchs MPI's coord system (MPI might differ between machines - so this is why the code calculates the coord system itself, however with - debug on, can check if it matches MPI(even tho it is not enforced to match + this debug statment checks to see if the way coordinates found by + calculation matchs MPI's coord system (MPI might differ between machines + so this is why the code calculates the coord system itself, however with + debug on, can check if it matches MPI(even tho it is not enforced to match it.)). */ int prev_coord[3]; prev_coord[0]=d->process_topology_3.self[0]; prev_coord[1]=d->process_topology_3.self[1]; prev_coord[2]=d->process_topology_3.self[2]; - MPI_Cart_get(d->process_topology_3.cart, ndim, - d->process_topology_3.nproc, - d->process_topology_3.period, - d->process_topology_3.self); + MPI_Cart_get(d->process_topology_3.cart, ndim, + d->process_topology_3.nproc, + d->process_topology_3.period, + d->process_topology_3.self); for(int i=0; i < 3; i++) if(prev_coord[i] != d->process_topology_3.self[i]) - abort(); + abort(); } assert(n[0]%d->process_topology_3.nproc[0] == 0); assert(n[0]%d->process_topology_3.nproc[1] == 0); assert(n[0]%d->process_topology_3.nproc[2] == 0); - + //set local dimensions d->process_topology_3.n[0] = n[0] / d->process_topology_3.nproc[0]; d->process_topology_3.n[1] = n[1] / d->process_topology_3.nproc[1]; @@ -422,169 +422,169 @@ void distribution_init(MPI_Comm comm, // set up process grid with 2d decomposition (z_PENCILs ) d->process_topology_2_z.nproc[0] = 0; d->process_topology_2_z.nproc[1] = 0; - d->process_topology_2_z.nproc[2] = 1; // don't distribute outer dimension + d->process_topology_2_z.nproc[2] = 1; // don't distribute outer dimension period[0] = period[1] = period[2] = 1; MPI_Dims_create(nproc, ndim, d->process_topology_2_z.nproc); d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; - //variable used to ensure that pencils created fit inside the cuboids, + //variable used to ensure that pencils created fit inside the cuboids, //if not the code will assert out. - bool check_z_dims=false; - if(d->process_topology_2_z.n[0] != 0 - && d->process_topology_2_z.n[1] != 0 + bool check_z_dims=false; + if(d->process_topology_2_z.n[0] != 0 + && d->process_topology_2_z.n[1] != 0 && d->process_topology_2_z.n[2] != 0) {// protects from dividing by zero. - check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) + check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); - + if(self==0 && debug && !check_z_dims) - fprintf(stderr,"Need to fix Z PENCILS z_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_z.ns(%d,%d,%d).... \n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_3.n[0], - d->process_topology_3.n[1], - d->process_topology_3.n[2], - d->process_topology_2_z.n[0], - d->process_topology_2_z.n[1], - d->process_topology_2_z.n[2]); - - //try swaping pencil dimensions if current setup pencil dimensions dont + fprintf(stderr,"Need to fix Z PENCILS z_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_z.ns(%d,%d,%d).... \n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_3.n[0], + d->process_topology_3.n[1], + d->process_topology_3.n[2], + d->process_topology_2_z.n[0], + d->process_topology_2_z.n[1], + d->process_topology_2_z.n[2]); + + //try swaping pencil dimensions if current setup pencil dimensions dont //fit inside the cubes. - if(!(check_z_dims) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[1]) == 0) + if(!(check_z_dims) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[1]) == 0) && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[0]) == 0)) { if(self==0 && debug) - fprintf(stderr,"Swaping Z pencils in initialization (%d,%d,%d).... \n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2]); + fprintf(stderr,"Swaping Z pencils in initialization (%d,%d,%d).... \n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2]); int temp=d->process_topology_2_z.nproc[0]; d->process_topology_2_z.nproc[0] = d->process_topology_2_z.nproc[1]; d->process_topology_2_z.nproc[1] = temp; d->process_topology_2_z.nproc[2] = d->process_topology_2_z.nproc[2]; - + d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; - check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); + check_z_dims = ((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); } } else { check_z_dims=false; } /* - if that did not work, make a pencil that does if inside the 3d cuboids by - taking the cuboids dimensions (np1,np2,np3) and making pencils - (np1,np2*np3,1), or (np1*np3,np2,1) on the most evenly distributed + if that did not work, make a pencil that does if inside the 3d cuboids by + taking the cuboids dimensions (np1,np2,np3) and making pencils + (np1,np2*np3,1), or (np1*np3,np2,1) on the most evenly distributed dimensions */ if(!check_z_dims){ if(self==0 && debug) - fprintf(stderr,"MAKING Z PENCILS FIT zprocs(%d,%d,%d) z.ns(%d,%d,%d).... \n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_2_z.n[0], - d->process_topology_2_z.n[1], - d->process_topology_2_z.n[2]); - + fprintf(stderr,"MAKING Z PENCILS FIT zprocs(%d,%d,%d) z.ns(%d,%d,%d).... \n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_2_z.n[0], + d->process_topology_2_z.n[1], + d->process_topology_2_z.n[2]); + d->process_topology_2_z.nproc[2]=1; if(d->process_topology_3.n[0]>d->process_topology_3.n[1]) { d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]; d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]; - if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) - || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) + if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) + || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) { - d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]*d->process_topology_3.nproc[2]; - d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]; + d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]*d->process_topology_3.nproc[2]; + d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]; } } else { d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]*d->process_topology_3.nproc[2]; d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]; - if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) - || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) + if((n[0] % (d->process_topology_2_z.nproc[0]) != 0) + || (n[0] % (d->process_topology_2_z.nproc[1]) != 0)) { - d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]; - d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]; + d->process_topology_2_z.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[2]; + d->process_topology_2_z.nproc[0]=d->process_topology_3.nproc[0]; } } d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; if(self==0 && debug) - fprintf(stderr,"MAKING Z PENCILS FIT AFTER zprocs(%d,%d,%d) z.ns(%d,%d,%d)...\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_2_z.n[0], - d->process_topology_2_z.n[1], - d->process_topology_2_z.n[2]); - if(d->process_topology_2_z.n[0] != 0 - && d->process_topology_2_z.n[1] != 0 + fprintf(stderr,"MAKING Z PENCILS FIT AFTER zprocs(%d,%d,%d) z.ns(%d,%d,%d)...\n", + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_2_z.n[0], + d->process_topology_2_z.n[1], + d->process_topology_2_z.n[2]); + if(d->process_topology_2_z.n[0] != 0 + && d->process_topology_2_z.n[1] != 0 && d->process_topology_2_z.n[2] != 0) {// protects from dividing by zero. - check_z_dims=((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) - && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); + check_z_dims=((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) + && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); } else { check_z_dims=false; } } - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_z: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_2_z.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_2_z.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); - } + } if(!check_z_dims && debug && (self==0)){ FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"Z DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2], - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2], + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); } assert(check_z_dims); /* - if this happends, it is because the dimensions were chosen incorrectly. - Either to many processors for the number of points in one dimenison (could - not do at least 1 point per processor), or the methods above could - not make a distribution of pencils that fit in the cubiods, which would - happen if the user gave numbers that wouldent work (we require the number - of processors in each dimension of the cuboid must be modulo the number of + if this happends, it is because the dimensions were chosen incorrectly. + Either to many processors for the number of points in one dimenison (could + not do at least 1 point per processor), or the methods above could + not make a distribution of pencils that fit in the cubiods, which would + happen if the user gave numbers that wouldent work (we require the number + of processors in each dimension of the cuboid must be modulo the number of points in that dimension, otherwise, this error will happen). */ - MPI_Cart_create(comm, - ndim, - d->process_topology_2_z.nproc, - period, - 0, - &d->process_topology_2_z.cart); + MPI_Cart_create(comm, + ndim, + d->process_topology_2_z.nproc, + period, + 0, + &d->process_topology_2_z.cart); //find the cartesian coord of the current rank (for the z_pencil) Coord_z_pencils(self,d->process_topology_2_z.self,d); if(self == 0) { printf("distribution 2z: [%d:%d:%d]\n", - d->process_topology_2_z.nproc[0], - d->process_topology_2_z.nproc[1], - d->process_topology_2_z.nproc[2]); + d->process_topology_2_z.nproc[0], + d->process_topology_2_z.nproc[1], + d->process_topology_2_z.nproc[2]); fflush(stdout); } @@ -599,88 +599,88 @@ void distribution_init(MPI_Comm comm, d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; d->process_topology_2_x.n[1] = n[1] / d->process_topology_2_x.nproc[1]; d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; - //variable used to ensure that pencils created fit inside the cuboids, + //variable used to ensure that pencils created fit inside the cuboids, //if not the code will assert out. bool check_x_dims = false; - if(d->process_topology_2_x.n[0] != 0 - && d->process_topology_2_x.n[1] != 0 + if(d->process_topology_2_x.n[0] != 0 + && d->process_topology_2_x.n[1] != 0 && d->process_topology_2_x.n[2] != 0) {// protects from dividing by zero. - check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) + check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) && (n[0] % (d->process_topology_2_x.nproc[1]) == 0); if(self==0 && debug && !check_x_dims) - fprintf(stderr,"Need to fix X PENCILS x_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_x.ns(%d,%d,%d)...\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_3.n[0], - d->process_topology_3.n[1], - d->process_topology_3.n[2], - d->process_topology_2_x.n[0], - d->process_topology_2_x.n[1], - d->process_topology_2_x.n[2]); - - //try swaping pencil dimensions if current setup does not have pencils + fprintf(stderr,"Need to fix X PENCILS x_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_x.ns(%d,%d,%d)...\n", + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_3.n[0], + d->process_topology_3.n[1], + d->process_topology_3.n[2], + d->process_topology_2_x.n[0], + d->process_topology_2_x.n[1], + d->process_topology_2_x.n[2]); + + //try swaping pencil dimensions if current setup does not have pencils //that fit inside cubes. - if(!(check_x_dims) - && ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[1]) == 0) + if(!(check_x_dims) + && ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[1]) == 0) && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[2]) == 0)) { if(self==0 && debug) - fprintf(stderr,"Swaping X pencils in initialization .... \n"); + fprintf(stderr,"Swaping X pencils in initialization .... \n"); d->process_topology_2_x.nproc[0] = d->process_topology_2_x.nproc[0]; int temp = d->process_topology_2_x.nproc[1]; d->process_topology_2_x.nproc[1] = d->process_topology_2_x.nproc[2]; d->process_topology_2_x.nproc[2] = temp; - + d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; d->process_topology_2_x.n[1] = n[1] / d->process_topology_2_x.nproc[1]; d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; - check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[1]) == 0); - } + check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[1]) == 0); + } } else{ check_x_dims=false; } /* - if that did not work, make a pencil that does by taking the cuboid - (np1,np2,np3) and making pencils of the form (1,np2*np1,np3) or + if that did not work, make a pencil that does by taking the cuboid + (np1,np2,np3) and making pencils of the form (1,np2*np1,np3) or (1,np2*np1,np3) depending on the most even distribution it can. */ if(!check_x_dims){ if(self==0 && debug) fprintf(stderr,"MAKING X PENCILS FIT xprocs(%d,%d,%d) x.ns(%d,%d,%d)...\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_2_x.n[0], - d->process_topology_2_x.n[1], - d->process_topology_2_x.n[2]); + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_2_x.n[0], + d->process_topology_2_x.n[1], + d->process_topology_2_x.n[2]); d->process_topology_2_x.nproc[0] = 1; if(d->process_topology_3.nproc[2] > d->process_topology_3.nproc[1]) { d->process_topology_2_x.nproc[1] = d->process_topology_3.nproc[1]*d->process_topology_3.nproc[0]; d->process_topology_2_x.nproc[2] = d->process_topology_3.nproc[2]; - if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) { - d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]*d->process_topology_3.nproc[0]; - d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]; + d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]*d->process_topology_3.nproc[0]; + d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]; } } else { d->process_topology_2_x.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[0]; d->process_topology_2_x.nproc[1] = d->process_topology_3.nproc[1]; - if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_x.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_x.nproc[0]) != 0)) { - d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[0]; - d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]; + d->process_topology_2_x.nproc[1]=d->process_topology_3.nproc[1]*d->process_topology_3.nproc[0]; + d->process_topology_2_x.nproc[2]=d->process_topology_3.nproc[2]; } } d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; @@ -688,31 +688,31 @@ void distribution_init(MPI_Comm comm, d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; if(self==0 && debug) fprintf(stderr,"MAKING X PENCILS FIT AFTER xprocs(%d,%d,%d) x.ns(%d,%d,%d)...\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_2_x.n[0], - d->process_topology_2_x.n[1], - d->process_topology_2_x.n[2]); - if(d->process_topology_2_x.n[0] != 0 - && d->process_topology_2_x.n[1] != 0 + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_2_x.n[0], + d->process_topology_2_x.n[1], + d->process_topology_2_x.n[2]); + if(d->process_topology_2_x.n[0] != 0 + && d->process_topology_2_x.n[1] != 0 && d->process_topology_2_x.n[2] != 0) {// protects from dividing by zero. - check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) - && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_x.nproc[1]) == 0); + check_x_dims = ((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) + && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_x.nproc[1]) == 0); } else { check_x_dims=false; - } + } } - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_x: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_2_x.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_2_x.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); } @@ -720,39 +720,39 @@ void distribution_init(MPI_Comm comm, FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"X DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2], - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2], + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); } assert(check_x_dims); /* - if this happends, it is because the dimensions were chosen incorrectly. - Either to many processors for the number of points in one dimenison (could - not do at least 1 point per processor), or the methods above could not make - a distribution of pencils that fit in the cubiods, which would happen if the - user gave numbers that wouldent work (we require the number of processors in - each dimension of the cuboid must be modulo the number of points in that + if this happends, it is because the dimensions were chosen incorrectly. + Either to many processors for the number of points in one dimenison (could + not do at least 1 point per processor), or the methods above could not make + a distribution of pencils that fit in the cubiods, which would happen if the + user gave numbers that wouldent work (we require the number of processors in + each dimension of the cuboid must be modulo the number of points in that dimension, otherwise, this error will happen). */ - MPI_Cart_create(comm, - ndim, - d->process_topology_2_x.nproc, - period, - 0, - &d->process_topology_2_x.cart); + MPI_Cart_create(comm, + ndim, + d->process_topology_2_x.nproc, + period, + 0, + &d->process_topology_2_x.cart); Coord_x_pencils(self, d->process_topology_2_x.self, d); if(self == 0) { printf("distribution 2x: [%d:%d:%d]\n", - d->process_topology_2_x.nproc[0], - d->process_topology_2_x.nproc[1], - d->process_topology_2_x.nproc[2]); + d->process_topology_2_x.nproc[0], + d->process_topology_2_x.nproc[1], + d->process_topology_2_x.nproc[2]); fflush(stdout); } - + // set up process grid with 2d decomposition (y_PENCILs) @@ -764,119 +764,119 @@ void distribution_init(MPI_Comm comm, d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; - //variable used to ensure that pencils created fit inside the cuboids, + //variable used to ensure that pencils created fit inside the cuboids, //if not the code will assert out. bool check_y_dims=false; - if(d->process_topology_2_y.n[0] != 0 - && d->process_topology_2_y.n[1] != 0 + if(d->process_topology_2_y.n[0] != 0 + && d->process_topology_2_y.n[1] != 0 && d->process_topology_2_y.n[2] != 0) {// protects from dividing by zero. - check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); + check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); if(self==0 && debug && !check_y_dims) fprintf(stderr,"Need to fix Y PENCILS y_procs(%d,%d,%d) 3d.ns(%d,%d,%d) 2d_y.ns(%d,%d,%d)...\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_3.n[0], - d->process_topology_3.n[1], - d->process_topology_3.n[2], - d->process_topology_2_y.n[0], - d->process_topology_2_y.n[1], - d->process_topology_2_y.n[2]); - //try swaping pencil dimensions if the current dimension of the pencils + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_3.n[0], + d->process_topology_3.n[1], + d->process_topology_3.n[2], + d->process_topology_2_y.n[0], + d->process_topology_2_y.n[1], + d->process_topology_2_y.n[2]); + //try swaping pencil dimensions if the current dimension of the pencils //does not fit inside the cubes. - if(!(check_y_dims) - && ((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[0]) == 0) + if(!(check_y_dims) + && ((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[0]) == 0) && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[2]) == 0)) { if(self==0 && debug) - fprintf(stderr,"Swaping Y pencils in initialization .... \n"); - + fprintf(stderr,"Swaping Y pencils in initialization .... \n"); + int temp = d->process_topology_2_y.nproc[0]; d->process_topology_2_y.nproc[0] = d->process_topology_2_y.nproc[2]; d->process_topology_2_y.nproc[2] = temp; d->process_topology_2_y.nproc[1] = d->process_topology_2_y.nproc[1]; - + d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; - check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); + check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); } } else { check_y_dims = false; } /* - if that did not work, make a pencil that does by taking the cuboid - (np1,np2,np3) and making pencils of the form (np1,1,np3*np2) or + if that did not work, make a pencil that does by taking the cuboid + (np1,np2,np3) and making pencils of the form (np1,1,np3*np2) or (np1*np2,1,np3) depending on the most even distribution it can. */ if(!check_y_dims){ if(self==0 && debug) - fprintf(stderr,"MAKING Y PENCILS FIT yprocs(%d,%d,%d) y.ns(%d,%d,%d)...\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_2_y.n[0], - d->process_topology_2_y.n[1], - d->process_topology_2_y.n[2]); - + fprintf(stderr,"MAKING Y PENCILS FIT yprocs(%d,%d,%d) y.ns(%d,%d,%d)...\n", + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_2_y.n[0], + d->process_topology_2_y.n[1], + d->process_topology_2_y.n[2]); + d->process_topology_2_y.nproc[1]=1; if(d->process_topology_3.nproc[2] > d->process_topology_3.nproc[0]) { d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]*d->process_topology_3.nproc[1]; d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]; - if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) { - d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]; - d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]; + d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]; + d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]; } } else { d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]*d->process_topology_3.nproc[1]; d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]; - if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) - || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) + if((n[0] % (d->process_topology_2_y.nproc[2]) != 0) + || (n[0] % (d->process_topology_2_y.nproc[0]) != 0)) { - d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]*d->process_topology_3.nproc[1]; - d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]; + d->process_topology_2_y.nproc[0] = d->process_topology_3.nproc[0]*d->process_topology_3.nproc[1]; + d->process_topology_2_y.nproc[2] = d->process_topology_3.nproc[2]; } } - + d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; if(self==0 && debug) fprintf(stderr,"MAKING Y PENCILS FIT AFTER yprocs(%d,%d,%d) y.ns(%d,%d,%d)...\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_2_y.n[0], - d->process_topology_2_y.n[1], - d->process_topology_2_y.n[2]); - if(d->process_topology_2_y.n[0] != 0 && d->process_topology_2_y.n[1] != 0 + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_2_y.n[0], + d->process_topology_2_y.n[1], + d->process_topology_2_y.n[2]); + if(d->process_topology_2_y.n[0] != 0 && d->process_topology_2_y.n[1] != 0 && d->process_topology_2_y.n[2] != 0) {// protects from dividing by zero. - check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) - && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) - && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); + check_y_dims = (((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) + && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) + && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); } else { check_y_dims=false; } } - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_y: "); for (int i = 0; i < ndim; ++i) { - fprintf(stderr, "%d%s", - d->process_topology_2_y.nproc[i], - separator(i, ndim)); + fprintf(stderr, "%d%s", + d->process_topology_2_y.nproc[i], + separator(i, ndim)); } fprintf(stderr, "\n"); } @@ -884,42 +884,42 @@ void distribution_init(MPI_Comm comm, FILE * outfile; outfile = fopen("error.data","a"); fprintf(outfile,"Y DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2], - d->process_topology_3.nproc[0], - d->process_topology_3.nproc[1], - d->process_topology_3.nproc[2]); + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2], + d->process_topology_3.nproc[0], + d->process_topology_3.nproc[1], + d->process_topology_3.nproc[2]); } assert(check_y_dims); /* - if this happends, it is because the dimensions were chosen incorrectly. - Either to many processors for the number of points in one dimenison (could - not do at least 1 point per processor), or the methods above could - not make a distribution of pencils that fit in the cubiods, which would - happen if the user gave numbers that wouldent work (we require the number of - processors in each dimension of the cuboid must be modulo the number of + if this happends, it is because the dimensions were chosen incorrectly. + Either to many processors for the number of points in one dimenison (could + not do at least 1 point per processor), or the methods above could + not make a distribution of pencils that fit in the cubiods, which would + happen if the user gave numbers that wouldent work (we require the number of + processors in each dimension of the cuboid must be modulo the number of points in that dimension, otherwise, this error will happen). */ - MPI_Cart_create(comm, - ndim, - d->process_topology_2_y.nproc, - period, - 0, - &d->process_topology_2_y.cart); + MPI_Cart_create(comm, + ndim, + d->process_topology_2_y.nproc, + period, + 0, + &d->process_topology_2_y.cart); //find the cartesian coord of the current rank (for the y_pencil) Coord_y_pencils(self,d->process_topology_2_y.self,d); if(self == 0) { printf("distribution 2y: [%d:%d:%d]\n", - d->process_topology_2_y.nproc[0], - d->process_topology_2_y.nproc[1], - d->process_topology_2_y.nproc[2]); + d->process_topology_2_y.nproc[0], + d->process_topology_2_y.nproc[1], + d->process_topology_2_y.nproc[2]); fflush(stdout); } - + if (d->debug) { int myrank_cube; Rank_cube(&myrank_cube,d->process_topology_3.self,d); @@ -929,9 +929,9 @@ void distribution_init(MPI_Comm comm, Rank_y_pencils(&myrank_y,d->process_topology_2_y.self,d); int myrank_z; Rank_z_pencils(&myrank_z,d->process_topology_2_z.self,d); - if(myrank_z != self - || myrank_y != self - || myrank_x != self + if(myrank_z != self + || myrank_y != self + || myrank_x != self || myrank_cube != self) abort(); //means ranks were calculated wrong. if (0 == self) { @@ -940,32 +940,32 @@ void distribution_init(MPI_Comm comm, for (int p = 0; p < nproc; ++p) { MPI_Barrier(comm); if (p == self) { - fprintf(stderr, " %d: 1d = (%d, %d, %d), 2d_x = (%d, %d, %d) rank is= %d,2d_y = (%d, %d, %d) rank is= %d,2d_z = (%d, %d, %d) rank is= %d, 3d = (%d, %d, %d). rank is= %d\n", - self, - d->process_topology_1.self[0], - d->process_topology_1.self[1], - d->process_topology_1.self[2], - d->process_topology_2_x.self[0], - d->process_topology_2_x.self[1], - d->process_topology_2_x.self[2], - myrank_x, - d->process_topology_2_y.self[0], - d->process_topology_2_y.self[1], - d->process_topology_2_y.self[2], - myrank_y, - d->process_topology_2_z.self[0], - d->process_topology_2_z.self[1], - d->process_topology_2_z.self[2], - myrank_z, - d->process_topology_3.self[0], - d->process_topology_3.self[1], - d->process_topology_3.self[2], - myrank_cube); + fprintf(stderr, " %d: 1d = (%d, %d, %d), 2d_x = (%d, %d, %d) rank is= %d,2d_y = (%d, %d, %d) rank is= %d,2d_z = (%d, %d, %d) rank is= %d, 3d = (%d, %d, %d). rank is= %d\n", + self, + d->process_topology_1.self[0], + d->process_topology_1.self[1], + d->process_topology_1.self[2], + d->process_topology_2_x.self[0], + d->process_topology_2_x.self[1], + d->process_topology_2_x.self[2], + myrank_x, + d->process_topology_2_y.self[0], + d->process_topology_2_y.self[1], + d->process_topology_2_y.self[2], + myrank_y, + d->process_topology_2_z.self[0], + d->process_topology_2_z.self[1], + d->process_topology_2_z.self[2], + myrank_z, + d->process_topology_3.self[0], + d->process_topology_3.self[1], + d->process_topology_3.self[2], + myrank_cube); } } } - //allocate size of buffers used to hold pencil chunks of data in the + //allocate size of buffers used to hold pencil chunks of data in the //distribution routines for 3d to 1d and vica versa. int buff_z_chunk = d->process_topology_2_z.n[0]*d->process_topology_2_z.n[1]*d->process_topology_3.n[2]; int buff_y_chunk = d->process_topology_2_y.n[0]*d->process_topology_2_y.n[2]*d->process_topology_3.n[1]; @@ -978,7 +978,7 @@ void distribution_init(MPI_Comm comm, } if(buff_x_chunk > buff_size) buff_size = buff_x_chunk; - + d->d2_chunk=(complex_t *) malloc(sizeof(complex_t)*buff_size); d->d3_chunk=(complex_t *) malloc(sizeof(complex_t)*buff_size); } @@ -996,21 +996,21 @@ void Custom3D_Dims_create(const int Ndims[], int nproc, int ndims, int dims[]) } else { MPI_Dims_create(nproc, ndims, dims); - } + } } // create 1-, 2- and 3-d cartesian data distributions with explicitly // provided dimension lists -void distribution_init_explicit(MPI_Comm comm, - const int n[], +void distribution_init_explicit(MPI_Comm comm, + const int n[], int nproc_1d[], int nproc_2d_x[], int nproc_2d_y[], int nproc_2d_z[], int nproc_3d[], - distribution_t *d, - bool debug) + distribution_t *d, + bool debug) { d->parent = comm; @@ -1018,38 +1018,38 @@ void distribution_init_explicit(MPI_Comm comm, int self; int ndim = 3; int period[3]; - + MPI_Comm_rank(comm, &self); MPI_Comm_size(comm, &nproc); - + if (!self) printf("Initializing redistribution using a %s layout on %d ranks.\n", #ifdef PENCIL - "pencil" + "pencil" #else - "slab" + "slab" #endif - ,nproc); - + ,nproc); + d->debug = debug; for (int i = 0; i < 3; ++i) d->n[i] = n[i]; - + // check supplied dimension lists are valid assert(nproc_1d[0] == nproc); assert(nproc_1d[1] == 1); assert(nproc_1d[2] == 1); - + assert(nproc_2d_x[1] * nproc_2d_x[2] == nproc); assert(nproc_2d_x[0] == 1); - + assert(nproc_2d_y[0] * nproc_2d_y[2] == nproc); assert(nproc_2d_y[1] == 1); - + assert(nproc_2d_z[0] * nproc_2d_z[1] == nproc); assert(nproc_2d_z[2] == 1); - + assert(nproc_3d[0] * nproc_3d[1] * nproc_3d[2]== nproc); - + // set up process grid with 1d decomposition (SLABs) period[0] = period[1] = period[2] = 1; MPI_Cart_create(comm, ndim, nproc_1d, period, 0, &d->process_topology_1.cart); @@ -1065,7 +1065,7 @@ void distribution_init_explicit(MPI_Comm comm, } fprintf(stderr, "\n"); } - + // set up process grid with 3d decomposition (CUBE) period[0] = period[1] = period[2] = 1; MPI_Cart_create(comm, ndim, nproc_3d, period, 0, &d->process_topology_3.cart); @@ -1090,7 +1090,7 @@ void distribution_init_explicit(MPI_Comm comm, d->process_topology_3.n[0] = n[0] / d->process_topology_3.nproc[0]; d->process_topology_3.n[1] = n[1] / d->process_topology_3.nproc[1]; d->process_topology_3.n[2] = n[2] / d->process_topology_3.nproc[2]; - + // set up process grid with 2d_x decomposition (X dim Pencils) period[0] = period[1] = period[2] = 1; MPI_Cart_create(comm, ndim, nproc_2d_x, period, 0, &d->process_topology_2_x.cart); @@ -1100,18 +1100,18 @@ void distribution_init_explicit(MPI_Comm comm, d->process_topology_2_x.n[0] = n[0] / d->process_topology_2_x.nproc[0]; d->process_topology_2_x.n[1] = n[1] / d->process_topology_2_x.nproc[1]; d->process_topology_2_x.n[2] = n[2] / d->process_topology_2_x.nproc[2]; - + bool check_x_dims=((d->process_topology_3.n[2]) % (d->process_topology_2_x.n[2]) == 0) && ((d->process_topology_3.n[1]) % (d->process_topology_2_x.n[1]) == 0) && (n[0] % (d->process_topology_2_x.nproc[2]) == 0) && (n[0] % (d->process_topology_2_x.nproc[0]) == 0); if(!check_x_dims && debug && (self==0)){ FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"X DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n",d->process_topology_2_x.nproc[0],d->process_topology_2_x.nproc[1],d->process_topology_2_x.nproc[2], d->process_topology_3.nproc[0],d->process_topology_3.nproc[1],d->process_topology_3.nproc[2]); } - assert(check_x_dims);//if this happends, it is because the dimensions were chosen incorrectly. Either to many processors for the number of points in one dimenison (could not do at least 1 point per processor), or the methods above could - //not make a distribution of pencils that fit in the cubiods, which would happen if the user gave numbers that wouldent work (we require the number of processors in each dimension of the cuboid must be modulo the number of points + assert(check_x_dims);//if this happends, it is because the dimensions were chosen incorrectly. Either to many processors for the number of points in one dimenison (could not do at least 1 point per processor), or the methods above could + //not make a distribution of pencils that fit in the cubiods, which would happen if the user gave numbers that wouldent work (we require the number of processors in each dimension of the cuboid must be modulo the number of points //in that dimension, otherwise, this error will happen). Coord_x_pencils(self,d->process_topology_2_x.self,d); - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_x: "); for (int i = 0; i < ndim; ++i) { @@ -1119,7 +1119,7 @@ void distribution_init_explicit(MPI_Comm comm, } fprintf(stderr, "\n"); } - + // set up process grid with 2d_y decomposition (Y dim Pencils) period[0] = period[1] = period[2] = 1; MPI_Cart_create(comm, ndim, nproc_2d_y, period, 0, &d->process_topology_2_y.cart); @@ -1129,19 +1129,19 @@ void distribution_init_explicit(MPI_Comm comm, d->process_topology_2_y.n[0] = n[0] / d->process_topology_2_y.nproc[0]; d->process_topology_2_y.n[1] = n[1] / d->process_topology_2_y.nproc[1]; d->process_topology_2_y.n[2] = n[2] / d->process_topology_2_y.nproc[2]; - - + + bool check_y_dims=(((d->process_topology_3.n[2]) % (d->process_topology_2_y.n[2]) == 0) && ((d->process_topology_3.n[0]) % (d->process_topology_2_y.n[0]) == 0) && (n[0] % (d->process_topology_2_y.nproc[2]) == 0) && (n[0] % (d->process_topology_2_y.nproc[0]) == 0)); if(!check_y_dims && debug && (self==0)){ FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"Y DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n",d->process_topology_2_y.nproc[0],d->process_topology_2_y.nproc[1],d->process_topology_2_y.nproc[2], d->process_topology_3.nproc[0],d->process_topology_3.nproc[1],d->process_topology_3.nproc[2]); } - assert(check_y_dims);//if this happends, it is because the dimensions were chosen incorrectly. Either to many processors for the number of points in one dimenison (could not do at least 1 point per processor), or the methods above could - //not make a distribution of pencils that fit in the cubiods, which would happen if the user gave numbers that wouldent work (we require the number of processors in each dimension of the cuboid must be modulo the number of points + assert(check_y_dims);//if this happends, it is because the dimensions were chosen incorrectly. Either to many processors for the number of points in one dimenison (could not do at least 1 point per processor), or the methods above could + //not make a distribution of pencils that fit in the cubiods, which would happen if the user gave numbers that wouldent work (we require the number of processors in each dimension of the cuboid must be modulo the number of points //in that dimension, otherwise, this error will happen). Coord_y_pencils(self,d->process_topology_2_y.self,d); - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_y: "); for (int i = 0; i < ndim; ++i) { @@ -1149,7 +1149,7 @@ void distribution_init_explicit(MPI_Comm comm, } fprintf(stderr, "\n"); } - + // set up process grid with 2d_z decomposition (Z dim pencils) period[0] = period[1] = period[2] = 1; MPI_Cart_create(comm, ndim, nproc_2d_z, period, 0, &d->process_topology_2_z.cart); @@ -1159,19 +1159,19 @@ void distribution_init_explicit(MPI_Comm comm, d->process_topology_2_z.n[0] = n[0] / d->process_topology_2_z.nproc[0]; d->process_topology_2_z.n[1] = n[1] / d->process_topology_2_z.nproc[1]; d->process_topology_2_z.n[2] = n[2] / d->process_topology_2_z.nproc[2]; - - + + bool check_z_dims=((d->process_topology_3.n[0]) % (d->process_topology_2_z.n[0]) == 0) && ((d->process_topology_3.n[1]) % (d->process_topology_2_z.n[1]) == 0) && (n[0] % (d->process_topology_2_z.nproc[0]) == 0) && (n[0] % (d->process_topology_2_z.nproc[1]) == 0); if(!check_z_dims && debug && (self==0)){ FILE * outfile; outfile= fopen("error.data","a"); fprintf(outfile,"Z DIMS FAILS:(%d,%d,%d) (%d,%d,%d) \n",d->process_topology_2_z.nproc[0],d->process_topology_2_z.nproc[1],d->process_topology_2_z.nproc[2], d->process_topology_3.nproc[0],d->process_topology_3.nproc[1],d->process_topology_3.nproc[2]); } - assert(check_z_dims);//if this happends, it is because the dimensions were chosen incorrectly. Either to many processors for the number of points in one dimenison (could not do at least 1 point per processor), or the methods above could - //not make a distribution of pencils that fit in the cubiods, which would happen if the user gave numbers that wouldent work (we require the number of processors in each dimension of the cuboid must be modulo the number of points + assert(check_z_dims);//if this happends, it is because the dimensions were chosen incorrectly. Either to many processors for the number of points in one dimenison (could not do at least 1 point per processor), or the methods above could + //not make a distribution of pencils that fit in the cubiods, which would happen if the user gave numbers that wouldent work (we require the number of processors in each dimension of the cuboid must be modulo the number of points //in that dimension, otherwise, this error will happen). Coord_z_pencils(self,d->process_topology_2_z.self,d); - + if (d->debug && 0 == self) { fprintf(stderr, " 2d_z: "); for (int i = 0; i < ndim; ++i) { @@ -1180,7 +1180,7 @@ void distribution_init_explicit(MPI_Comm comm, fprintf(stderr, "\n"); } //assert that all pencils fit in the cuboid. - + if (d->debug) { int myrank_cube; Rank_cube(&myrank_cube,d->process_topology_3.self,d); @@ -1197,14 +1197,14 @@ void distribution_init_explicit(MPI_Comm comm, for (int p = 0; p < nproc; ++p) { MPI_Barrier(comm); if (p == self) { - fprintf(stderr, - " %d: 1d = (%d, %d, %d), 2d_x = (%d, %d, %d) rank (%d), 2d_y = (%d, %d, %d) rank (%d), 2d_z = (%d, %d, %d) rank (%d), 3d = (%d, %d, %d) rank (%d).\n", - self, - d->process_topology_1.self[0], d->process_topology_1.self[1], d->process_topology_1.self[2], - d->process_topology_2_x.self[0], d->process_topology_2_x.self[1], d->process_topology_2_x.self[2],myrank_x, - d->process_topology_2_y.self[0], d->process_topology_2_y.self[1], d->process_topology_2_y.self[2],myrank_y, - d->process_topology_2_z.self[0], d->process_topology_2_z.self[1], d->process_topology_2_z.self[2],myrank_z, - d->process_topology_3.self[0], d->process_topology_3.self[1], d->process_topology_3.self[2],myrank_cube); + fprintf(stderr, + " %d: 1d = (%d, %d, %d), 2d_x = (%d, %d, %d) rank (%d), 2d_y = (%d, %d, %d) rank (%d), 2d_z = (%d, %d, %d) rank (%d), 3d = (%d, %d, %d) rank (%d).\n", + self, + d->process_topology_1.self[0], d->process_topology_1.self[1], d->process_topology_1.self[2], + d->process_topology_2_x.self[0], d->process_topology_2_x.self[1], d->process_topology_2_x.self[2],myrank_x, + d->process_topology_2_y.self[0], d->process_topology_2_y.self[1], d->process_topology_2_y.self[2],myrank_y, + d->process_topology_2_z.self[0], d->process_topology_2_z.self[1], d->process_topology_2_z.self[2],myrank_z, + d->process_topology_3.self[0], d->process_topology_3.self[1], d->process_topology_3.self[2],myrank_cube); } } } @@ -1312,16 +1312,16 @@ static void redistribute(const complex_t *a, MPI_Comm subgrid_cart; int subgrid_self; int subgrid_nproc; - + // exchange data with processes in a 2-d slab of 3-d subdomains - + remaining_dim[0] = 0; remaining_dim[1] = 1; remaining_dim[2] = 1; MPI_Cart_sub(d->process_topology_3.cart, remaining_dim, &subgrid_cart); MPI_Comm_rank(subgrid_cart, &subgrid_self); MPI_Comm_size(subgrid_cart, &subgrid_nproc); - + for (int p = 0; p < subgrid_nproc; ++p) { int d1_peer = (subgrid_self + p) % subgrid_nproc; int d3_peer = (subgrid_self - p + subgrid_nproc) % subgrid_nproc; @@ -1331,17 +1331,17 @@ static void redistribute(const complex_t *a, int starts[3]; MPI_Datatype d1_type; MPI_Datatype d3_type; - + MPI_Cart_coords(subgrid_cart, d1_peer, 2, coord); if (0) { int self; MPI_Comm_rank(MPI_COMM_WORLD, &self); fprintf(stderr, "%d: d1_peer, d1_coord, d3_peer = %d, (%d, %d), %d\n", - self, d1_peer, coord[0], coord[1], d3_peer); + self, d1_peer, coord[0], coord[1], d3_peer); } - + // create dataypes representing a subarray in the 1- and 3-d distributions - + sizes[0] = d->process_topology_1.n[0]; sizes[1] = d->process_topology_1.n[1]; sizes[2] = d->process_topology_1.n[2]; @@ -1353,7 +1353,7 @@ static void redistribute(const complex_t *a, starts[2] = coord[1] * d->process_topology_3.n[2]; MPI_Type_create_subarray(3, sizes, subsizes, starts, MPI_ORDER_C, MPI_DOUBLE_COMPLEX, &d1_type); MPI_Type_commit(&d1_type); - + sizes[0] = d->process_topology_3.n[0]; sizes[1] = d->process_topology_3.n[1]; sizes[2] = d->process_topology_3.n[2]; @@ -1365,27 +1365,27 @@ static void redistribute(const complex_t *a, starts[2] = 0; MPI_Type_create_subarray(3, sizes, subsizes, starts, MPI_ORDER_C, MPI_DOUBLE_COMPLEX, &d3_type); MPI_Type_commit(&d3_type); - + // exchange data - + if (direction == REDISTRIBUTE_3_TO_1) { MPI_Sendrecv((char *) a, 1, d3_type, d3_peer, 0, - (char *) b, 1, d1_type, d1_peer, 0, - subgrid_cart, MPI_STATUS_IGNORE); + (char *) b, 1, d1_type, d1_peer, 0, + subgrid_cart, MPI_STATUS_IGNORE); } else if (direction == REDISTRIBUTE_1_TO_3) { MPI_Sendrecv((char *) a, 1, d1_type, d1_peer, 0, - (char *) b, 1, d3_type, d3_peer, 0, - subgrid_cart, MPI_STATUS_IGNORE); + (char *) b, 1, d3_type, d3_peer, 0, + subgrid_cart, MPI_STATUS_IGNORE); } else { abort(); } - + // free datatypes - + MPI_Type_free(&d1_type); MPI_Type_free(&d3_type); } - + MPI_Comm_free(&subgrid_cart); } @@ -1411,29 +1411,29 @@ static void redistribute_slab(const complex_t *a, int subgrid_nproc; ptrdiff_t d1_slice = d->process_topology_1.n[1] * d->process_topology_1.n[2] * sizeof(complex_t); ptrdiff_t d3_slice = d->process_topology_3.n[1] * d->process_topology_3.n[2] * sizeof(complex_t); - + // exchange data with processes in a 2-d slab of 3-d subdomains - + remaining_dim[0] = 0; remaining_dim[1] = 1; remaining_dim[2] = 1; MPI_Cart_sub(d->process_topology_3.cart, remaining_dim, &subgrid_cart); MPI_Comm_rank(subgrid_cart, &subgrid_self); MPI_Comm_size(subgrid_cart, &subgrid_nproc); - + for (int p = 0; p < subgrid_nproc; ++p) { int coord[2]; int d1_peer = (subgrid_self + p) % subgrid_nproc; int d3_peer = (subgrid_self - p + subgrid_nproc) % subgrid_nproc; - + MPI_Cart_coords(subgrid_cart, d1_peer, 2, coord); if (0) { int self; MPI_Comm_rank(MPI_COMM_WORLD, &self); fprintf(stderr, "%d: d1_peer, d1_coord, d3_peer = %d, (%d, %d), %d\n", - self, d1_peer, coord[0], coord[1], d3_peer); + self, d1_peer, coord[0], coord[1], d3_peer); } - + for (int slice = 0; slice < d->process_topology_1.n[0]; ++slice) { int sizes[2]; int subsizes[2]; @@ -1442,9 +1442,9 @@ static void redistribute_slab(const complex_t *a, MPI_Datatype d3_type; ptrdiff_t d1_offset = slice * d1_slice; ptrdiff_t d3_offset = (slice + d3_peer * d->process_topology_1.n[0]) * d3_slice; - + // create subarray dataypes representing the slice subarray in the 1- and 3-d distributions - + sizes[0] = d->process_topology_1.n[1]; sizes[1] = d->process_topology_1.n[2]; subsizes[0] = d->process_topology_3.n[1]; @@ -1453,33 +1453,33 @@ static void redistribute_slab(const complex_t *a, starts[1] = coord[1] * d->process_topology_3.n[2]; MPI_Type_create_subarray(2, sizes, subsizes, starts, MPI_ORDER_C, MPI_DOUBLE_COMPLEX, &d1_type); MPI_Type_commit(&d1_type); - + MPI_Type_contiguous(d->process_topology_3.n[1] * d->process_topology_3.n[2], - MPI_DOUBLE_COMPLEX, - &d3_type); + MPI_DOUBLE_COMPLEX, + &d3_type); MPI_Type_commit(&d3_type); - + // exchange data - + if (direction == REDISTRIBUTE_3_TO_1) { - MPI_Sendrecv((char *) a + d3_offset, 1, d3_type, d3_peer, 0, - (char *) b + d1_offset, 1, d1_type, d1_peer, 0, - subgrid_cart, MPI_STATUS_IGNORE); + MPI_Sendrecv((char *) a + d3_offset, 1, d3_type, d3_peer, 0, + (char *) b + d1_offset, 1, d1_type, d1_peer, 0, + subgrid_cart, MPI_STATUS_IGNORE); } else if (direction == REDISTRIBUTE_1_TO_3) { - MPI_Sendrecv((char *) a + d1_offset, 1, d1_type, d1_peer, 0, - (char *) b + d3_offset, 1, d3_type, d3_peer, 0, - subgrid_cart, MPI_STATUS_IGNORE); + MPI_Sendrecv((char *) a + d1_offset, 1, d1_type, d1_peer, 0, + (char *) b + d3_offset, 1, d3_type, d3_peer, 0, + subgrid_cart, MPI_STATUS_IGNORE); } else { - abort(); + abort(); } - + // free datatypes - + MPI_Type_free(&d1_type); MPI_Type_free(&d3_type); } } - + MPI_Comm_free(&subgrid_cart); } @@ -1525,7 +1525,7 @@ static void redistribute_2_and_3(const complex_t *a, complex_t *b, distribution_t *d, int direction, - int z_dim) + int z_dim) { int self = d->process_topology_1.self[0]; int npeers; @@ -1537,7 +1537,7 @@ static void redistribute_2_and_3(const complex_t *a, int x_dim=0,y_dim=0; //x_dim, y_dim and z_dim are the dimensions of the x,y,z axis of the pencil with respect to the original axis(where index 2 is into the grid, 1 is vertical translation and 0 is horizontal). switch(z_dim){ - case 0: x_dim=1; y_dim=2; + case 0: x_dim=1; y_dim=2; if((self == me) && print_me)fprintf(stderr, "DOING X PENCILS!...\n"); break; case 1: x_dim=2; y_dim=0; if((self == me && print_me))fprintf(stderr, "DOING Y PENCILS!...\n"); break; @@ -1545,64 +1545,64 @@ static void redistribute_2_and_3(const complex_t *a, if((self == me && print_me))fprintf(stderr, "DOING Z PENCILS!...\n"); break; default: assert("incorrect inputed dimension"); } - + // assuming dimensions are all commensurate, then the number of // peers to exchange with is the number of processes in the z_dimension // direction in the 3d distribution npeers = d->process_topology_3.nproc[z_dim]; //picked last direction (lets say into the grid) - + // book-keeping for the processor translation in the x-y plane int p0 = 0; int p1 = 0; int p1max = 0; - + MPI_Request req1=MPI_REQUEST_NULL; MPI_Request req2=MPI_REQUEST_NULL; - + int pencil_sizes[3]; int cube_sizes[3]; int subsizes[3]; - - + + cube_sizes[x_dim] = d->process_topology_3.n[x_dim]; - cube_sizes[y_dim] = d->process_topology_3.n[y_dim]; + cube_sizes[y_dim] = d->process_topology_3.n[y_dim]; cube_sizes[z_dim] = d->process_topology_3.n[z_dim]; - + //set varibles used to calculate the subarrays of each pencil and cube. switch(z_dim){ - case 0: - p1max = d->process_topology_2_x.nproc[x_dim] / d->process_topology_3.nproc[x_dim] - 1; + case 0: + p1max = d->process_topology_2_x.nproc[x_dim] / d->process_topology_3.nproc[x_dim] - 1; //find out the size of the chunk you need to use (stored in subsizes), and set sizes to the local size of the pencil. //The x and y dimensions of the subchunck will be the dimensions of the pencil (since the code asserts at the beginning that all pencils fit inside the 3d cuboid.) //The z dimension will be the dimension of the cuboid, since this will always be <= to the z_dim of the pencil. pencil_sizes[x_dim] = d->process_topology_2_x.n[x_dim]; - pencil_sizes[y_dim] = d->process_topology_2_x.n[y_dim]; - pencil_sizes[z_dim] = d->process_topology_2_x.n[z_dim]; + pencil_sizes[y_dim] = d->process_topology_2_x.n[y_dim]; + pencil_sizes[z_dim] = d->process_topology_2_x.n[z_dim]; subsizes[x_dim] = d->process_topology_2_x.n[x_dim]; - subsizes[y_dim] = d->process_topology_2_x.n[y_dim]; + subsizes[y_dim] = d->process_topology_2_x.n[y_dim]; break; - case 1: - p1max = d->process_topology_2_y.nproc[x_dim] / d->process_topology_3.nproc[x_dim] - 1; + case 1: + p1max = d->process_topology_2_y.nproc[x_dim] / d->process_topology_3.nproc[x_dim] - 1; pencil_sizes[x_dim] = d->process_topology_2_y.n[x_dim]; - pencil_sizes[y_dim] = d->process_topology_2_y.n[y_dim]; - pencil_sizes[z_dim] = d->process_topology_2_y.n[z_dim]; + pencil_sizes[y_dim] = d->process_topology_2_y.n[y_dim]; + pencil_sizes[z_dim] = d->process_topology_2_y.n[z_dim]; subsizes[x_dim] = d->process_topology_2_y.n[x_dim]; - subsizes[y_dim] = d->process_topology_2_y.n[y_dim]; + subsizes[y_dim] = d->process_topology_2_y.n[y_dim]; break; - case 2: - p1max = d->process_topology_2_z.nproc[y_dim] / d->process_topology_3.nproc[y_dim] - 1; + case 2: + p1max = d->process_topology_2_z.nproc[y_dim] / d->process_topology_3.nproc[y_dim] - 1; pencil_sizes[x_dim] = d->process_topology_2_z.n[x_dim]; - pencil_sizes[y_dim] = d->process_topology_2_z.n[y_dim]; - pencil_sizes[z_dim] = d->process_topology_2_z.n[z_dim]; + pencil_sizes[y_dim] = d->process_topology_2_z.n[y_dim]; + pencil_sizes[z_dim] = d->process_topology_2_z.n[z_dim]; subsizes[x_dim] = d->process_topology_2_z.n[x_dim]; - subsizes[y_dim] = d->process_topology_2_z.n[y_dim]; + subsizes[y_dim] = d->process_topology_2_z.n[y_dim]; break; } subsizes[z_dim] = d->process_topology_3.n[z_dim]; int chunk_size=subsizes[0]*subsizes[1]*subsizes[2];//size of data chunks that will be communicated between pencil and cube distributions. - + //set variables that will be used to find pencils chunks - int pencil_dims[3]={0,0,0};// size of entire pencil in its local coord system + int pencil_dims[3]={0,0,0};// size of entire pencil in its local coord system int local_sizes[3]={0,0,0}; //size of chunck in its local coord system. if(z_dim==2){ local_sizes[0]=subsizes[0]; @@ -1613,7 +1613,7 @@ static void redistribute_2_and_3(const complex_t *a, pencil_dims[2]=d->process_topology_2_z.n[2]; } else if(z_dim==1){ - + local_sizes[0]=subsizes[0]; local_sizes[1]=subsizes[2]; local_sizes[2]=subsizes[1]; @@ -1629,13 +1629,13 @@ static void redistribute_2_and_3(const complex_t *a, pencil_dims[1]=d->process_topology_2_x.n[1]; pencil_dims[2]=d->process_topology_2_x.n[0]; } - + if((self == me) && print_me)fprintf(stderr, "%d, %d, %d, %d Dimensions!...\n", x_dim,y_dim,z_dim, p1max); - + // communicate with our peers for (int p = 0; p < npeers; ++p) { if((self == me) && print_me)fprintf(stderr, "%d, %d, %d Made it beg-for!...\n", self,p, npeers); - + int d2_coord[3]; int d2_peer; int d2_peer_coord[3]; @@ -1649,56 +1649,56 @@ static void redistribute_2_and_3(const complex_t *a, //turn the processor coordinate into one specified by the number of data points in each dimension. for (int i = 0; i < 3; ++i) { switch(z_dim){ - case 0: d2_coord[i] = d->process_topology_2_x.self[i] * d->process_topology_2_x.n[i]; break; - case 1: d2_coord[i] = d->process_topology_2_y.self[i] * d->process_topology_2_y.n[i]; break; - case 2: d2_coord[i] = d->process_topology_2_z.self[i] * d->process_topology_2_z.n[i]; break; + case 0: d2_coord[i] = d->process_topology_2_x.self[i] * d->process_topology_2_x.n[i]; break; + case 1: d2_coord[i] = d->process_topology_2_y.self[i] * d->process_topology_2_y.n[i]; break; + case 2: d2_coord[i] = d->process_topology_2_z.self[i] * d->process_topology_2_z.n[i]; break; } } //over every iteration of the loop, transverse down the pencil (since it will be divided in chunks whose coordinates will only differ in the z_dimension. - d2_coord[z_dim] += p * d->process_topology_3.n[z_dim]; - - + d2_coord[z_dim] += p * d->process_topology_3.n[z_dim]; + + if((self == me) && print_me)fprintf(stderr, "%d, %d, %d Coord!...\n", d2_coord[0],d2_coord[1],d2_coord[2]); - - + + //d2_array_start is the starting index of the chunk in the pencils local coordinates. - d2_array_start[0] = d2_coord[x_dim] % pencil_sizes[x_dim]; - d2_array_start[1] = d2_coord[y_dim] % pencil_sizes[y_dim]; - d2_array_start[2] = d2_coord[z_dim] % pencil_sizes[z_dim]; - + d2_array_start[0] = d2_coord[x_dim] % pencil_sizes[x_dim]; + d2_array_start[1] = d2_coord[y_dim] % pencil_sizes[y_dim]; + d2_array_start[2] = d2_coord[z_dim] % pencil_sizes[z_dim]; + if (DEBUG_CONDITION || ((self== me) && print_me)) { fprintf(stderr, - "%d: pencil_sizes=(%d,%d,%d), cube_sizes=(%d,%d,%d), subsizes=(%d,%d,%d),d2_coord=(%d,%d,%d), d2_array_start=(%d,%d,%d) \n", - self, - pencil_sizes[0], pencil_sizes[1], pencil_sizes[2], - cube_sizes[0], cube_sizes[1], cube_sizes[2], - subsizes[0], subsizes[1], subsizes[2], - d2_coord[0], d2_coord[1], d2_coord[2], - d2_array_start[0],d2_array_start[1],d2_array_start[2]); + "%d: pencil_sizes=(%d,%d,%d), cube_sizes=(%d,%d,%d), subsizes=(%d,%d,%d),d2_coord=(%d,%d,%d), d2_array_start=(%d,%d,%d) \n", + self, + pencil_sizes[0], pencil_sizes[1], pencil_sizes[2], + cube_sizes[0], cube_sizes[1], cube_sizes[2], + subsizes[0], subsizes[1], subsizes[2], + d2_coord[0], d2_coord[1], d2_coord[2], + d2_array_start[0],d2_array_start[1],d2_array_start[2]); } - - + + //if making cuboids from pencils, right here we need to fill the d2_chunk array with the data that later needs to be sent to a cuboid. //The array is a chunk of the pencil and is why we needed to calculate the starting index for the array in the local coordinates. - if(direction == REDISTRIBUTE_2_TO_3){ + if(direction == REDISTRIBUTE_2_TO_3){ int64_t ch_indx=0; int dims_size=pencil_dims[0]*pencil_dims[1]*pencil_dims[2]; for(int i0=d2_array_start[0];i0= 0 && local_indx>=0 && local_indx < dims_size); - d->d2_chunk[ch_indx]=a[local_indx]; - ch_indx++; - } - } + for(int i1=d2_array_start[1];i1= 0 && local_indx>=0 && local_indx < dims_size); + d->d2_chunk[ch_indx]=a[local_indx]; + ch_indx++; + } + } } - + if((self == me) && print_me)fprintf(stderr, "%d, %d, %d, pencil_dims!...\n", pencil_dims[0],pencil_dims[1],pencil_dims[2]); } - - // what peer in the 3d distribution owns this subarray? + + // what peer in the 3d distribution owns this subarray? for (int i = 0; i < 3; ++i) { d3_peer_coord[i] = d2_coord[i] / d->process_topology_3.n[i]; } @@ -1711,42 +1711,42 @@ static void redistribute_2_and_3(const complex_t *a, } if((self == me) && print_me)fprintf(stderr, "%d, %d, Made it half way!...\n", self,p); if((self == me) && print_me)fprintf(stderr, "%d, %d, PEER!...\n", self,d3_peer); - + //By here in the for loop, we have broken the pencil into a chunk and found which cuboid it resides; over every iteration, the for-loop will break up the pencil in the z_dimension. //From here on we do the opposite. We divide the cuboid into chunks (that are the same size as the ones in the pencil), and determine which pencils own these chunks. - - + + // what is the coordinate of my pth subarray in the 3d distribution? for (int i = 0; i < 3; ++i) { switch(z_dim){ - case 0: d3_coord[i] = d->process_topology_3.self[i] * d->process_topology_3.n[i]; break; - case 1: d3_coord[i] = d->process_topology_3.self[i] * d->process_topology_3.n[i]; break; - case 2: d3_coord[i] = d->process_topology_3.self[i] * d->process_topology_3.n[i]; break; + case 0: d3_coord[i] = d->process_topology_3.self[i] * d->process_topology_3.n[i]; break; + case 1: d3_coord[i] = d->process_topology_3.self[i] * d->process_topology_3.n[i]; break; + case 2: d3_coord[i] = d->process_topology_3.self[i] * d->process_topology_3.n[i]; break; } } - + //now unlike above, we dont need to iterate in the z_dim, because for each processor its subarrays inward dimension is already set by the cubes z_dim. //Instead, each iteration of the for-loop will look at different subarrays whose locations in the cuboid differ by local x and y coords. - + switch(z_dim){ - //p1 is a place holder for the first translation . The outside for-loop will increment the coord in that direction, say x_dim, - //and keep doing so until all of the chunks in that dimension are calculated. Then it will increment p0 in the other dimension (in this example the y) + //p1 is a place holder for the first translation . The outside for-loop will increment the coord in that direction, say x_dim, + //and keep doing so until all of the chunks in that dimension are calculated. Then it will increment p0 in the other dimension (in this example the y) //and repeat until all of the subchunks in the x and y dimensions are calculated. - //are found. - //Note: p0 and p1 will increment different dimensions depending of whether it is using the x y or z pencils, this is because the set up of the coordinate system for each + //are found. + //Note: p0 and p1 will increment different dimensions depending of whether it is using the x y or z pencils, this is because the set up of the coordinate system for each //pencil is different and to ensure that no communications hang up later, the directions coded below are unique for each type of pencil. case 0: - d3_coord[y_dim] += p0 * d->process_topology_2_x.n[y_dim]; - d3_coord[x_dim] += p1 * d->process_topology_2_x.n[x_dim]; - break; + d3_coord[y_dim] += p0 * d->process_topology_2_x.n[y_dim]; + d3_coord[x_dim] += p1 * d->process_topology_2_x.n[x_dim]; + break; case 1: - d3_coord[y_dim] += p0 * d->process_topology_2_y.n[y_dim]; - d3_coord[x_dim] += p1 * d->process_topology_2_y.n[x_dim]; - break; + d3_coord[y_dim] += p0 * d->process_topology_2_y.n[y_dim]; + d3_coord[x_dim] += p1 * d->process_topology_2_y.n[x_dim]; + break; case 2: - d3_coord[x_dim] += p0 * d->process_topology_2_z.n[x_dim]; - d3_coord[y_dim] += p1 * d->process_topology_2_z.n[y_dim]; - break; + d3_coord[x_dim] += p0 * d->process_topology_2_z.n[x_dim]; + d3_coord[y_dim] += p1 * d->process_topology_2_z.n[y_dim]; + break; } if (p1 == p1max) { p0++; @@ -1755,98 +1755,98 @@ static void redistribute_2_and_3(const complex_t *a, p1++; } // create a dataype for my pth subrarray in the 3d distribution - - + + //d3_array_start holds the starting index of the chunk in the cubes local coordinates(note the cubes local coord system is actually the same as the grids global coord system, by set up) - - d3_array_start[x_dim] = d3_coord[x_dim] % cube_sizes[x_dim]; - d3_array_start[y_dim] = d3_coord[y_dim] % cube_sizes[y_dim]; - d3_array_start[z_dim] = d3_coord[z_dim] % cube_sizes[z_dim]; - + + d3_array_start[x_dim] = d3_coord[x_dim] % cube_sizes[x_dim]; + d3_array_start[y_dim] = d3_coord[y_dim] % cube_sizes[y_dim]; + d3_array_start[z_dim] = d3_coord[z_dim] % cube_sizes[z_dim]; + //make starting point so that it coincides with the starting point of the pencil from the pencils coordinate system. (for z_pencils nothing needs to be changed, since it already //has the coordinate system of the grid, however, the x and y pencils have different starting points of the subchunk in their coord systems.) if(z_dim==0 || z_dim ==1){ d3_array_start[2]=d3_array_start[2]+subsizes[2]-1; } if(print_me && (self==me))fprintf(stderr,"D3_array_start is (%d,%d,%d) and subsizes is (%d,%d,%d) \n",d3_array_start[0],d3_array_start[1],d3_array_start[2],subsizes[0],subsizes[1],subsizes[2]); - - - //If sending cube chunks to pencils, need to fill those chunks with data here. The chunks are filled in the order - //such that when the pencil recieves the chunk, in its local array indexing, it assumes that the array is already + + + //If sending cube chunks to pencils, need to fill those chunks with data here. The chunks are filled in the order + //such that when the pencil recieves the chunk, in its local array indexing, it assumes that the array is already //filled such that it is contiguous. Therefore, complicated for-loops below fill the array in the cubes local indexing to match what the pencil will - //expect. + //expect. if(direction == REDISTRIBUTE_3_TO_2){ int64_t ch_indx=0; int dims_size=cube_sizes[0]*cube_sizes[1]*cube_sizes[2]; if((self == me) && print_me)fprintf(stderr, "%d, %d, MAKE 3D Chunk...\n", self,d3_peer); switch(z_dim){ - case 0: - for(int i2=d3_array_start[y_dim];i2>d3_array_start[y_dim]-subsizes[y_dim];i2--){//perhaps y_dim - for(int i1=d3_array_start[x_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - d->d3_chunk[ch_indx]=a[local_indx]; - ch_indx++; - } - } - } - break; - case 1: - for(int i0=d3_array_start[y_dim];i0d3_array_start[x_dim]-subsizes[x_dim];i2--){ - for(int i1=d3_array_start[z_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - d->d3_chunk[ch_indx]=a[local_indx]; - ch_indx++; - } - } - } - - break; - case 2: - for(int i0=d3_array_start[x_dim];i0process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - d->d3_chunk[ch_indx]=a[local_indx]; - ch_indx++; - } - } - } - - break; + case 0: + for(int i2=d3_array_start[y_dim];i2>d3_array_start[y_dim]-subsizes[y_dim];i2--){//perhaps y_dim + for(int i1=d3_array_start[x_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + d->d3_chunk[ch_indx]=a[local_indx]; + ch_indx++; + } + } + } + break; + case 1: + for(int i0=d3_array_start[y_dim];i0d3_array_start[x_dim]-subsizes[x_dim];i2--){ + for(int i1=d3_array_start[z_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + d->d3_chunk[ch_indx]=a[local_indx]; + ch_indx++; + } + } + } + + break; + case 2: + for(int i0=d3_array_start[x_dim];i0process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + d->d3_chunk[ch_indx]=a[local_indx]; + ch_indx++; + } + } + } + + break; } } - + if (DEBUG_CONDITION || ((self == me) && print_me)) { fprintf(stderr, - "%d: pencil_sizes=(%d,%d,%d), cube_sizes=(%d,%d,%d), subsizes=(%d,%d,%d), d3_coord=(%d,%d,%d), d3_array_start=(%d,%d,%d) \n", - self, - pencil_sizes[0], pencil_sizes[1], pencil_sizes[2], - cube_sizes[0], cube_sizes[1], cube_sizes[2], - subsizes[0], subsizes[1], subsizes[2], - d3_coord[0], d3_coord[1], d3_coord[2], - d3_array_start[0],d3_array_start[1],d3_array_start[2]); + "%d: pencil_sizes=(%d,%d,%d), cube_sizes=(%d,%d,%d), subsizes=(%d,%d,%d), d3_coord=(%d,%d,%d), d3_array_start=(%d,%d,%d) \n", + self, + pencil_sizes[0], pencil_sizes[1], pencil_sizes[2], + cube_sizes[0], cube_sizes[1], cube_sizes[2], + subsizes[0], subsizes[1], subsizes[2], + d3_coord[0], d3_coord[1], d3_coord[2], + d3_array_start[0],d3_array_start[1],d3_array_start[2]); } - + // what peer in the 2d distribution owns this subarray? for (int i = 0; i < 3; ++i) { switch(z_dim){ - case 0: - d2_peer_coord[i] = d3_coord[i] / d->process_topology_2_x.n[i]; - break; - case 1: - d2_peer_coord[i] = d3_coord[i] / d->process_topology_2_y.n[i]; - break; - case 2: - d2_peer_coord[i] = d3_coord[i] / d->process_topology_2_z.n[i]; - break; + case 0: + d2_peer_coord[i] = d3_coord[i] / d->process_topology_2_x.n[i]; + break; + case 1: + d2_peer_coord[i] = d3_coord[i] / d->process_topology_2_y.n[i]; + break; + case 2: + d2_peer_coord[i] = d3_coord[i] / d->process_topology_2_z.n[i]; + break; } } d2_peer_coord[z_dim] = 0;//since these are pencils, there is no two pencils in this direction. @@ -1855,16 +1855,16 @@ static void redistribute_2_and_3(const complex_t *a, //find its rank case 0: Rank_x_pencils(&d2_peer,d2_peer_coord,d); - break; + break; case 1: Rank_y_pencils(&d2_peer,d2_peer_coord,d); - break; + break; case 2: - Rank_z_pencils(&d2_peer,d2_peer_coord,d); - break; + Rank_z_pencils(&d2_peer,d2_peer_coord,d); + break; } if((self == me) && print_me)fprintf(stderr, "%d, %d, %d Made it before comm!...\n", self,p, npeers); - + // record the communication to be done in a schedule. Make sure to map each grid to the correct rank if (direction == REDISTRIBUTE_3_TO_2) { recv_peer = d->rankmap[d3_peer]; @@ -1878,24 +1878,24 @@ static void redistribute_2_and_3(const complex_t *a, //comunication of the chunks: //if print_mess boolean is set to true, then the code runs without sending any messages, and is used to test which messages would be sent in the entire run. //(designed to debug comm hangups, if they occur). - + if(direction == REDISTRIBUTE_3_TO_2){ - + if((self == me) && print_mess)fprintf(stderr, " I am %d, making request to recieve from %d...\n", self,recv_peer); if(!print_mess)MPI_Irecv((void *) d->d2_chunk, chunk_size, MPI_DOUBLE_COMPLEX, recv_peer, 0, d->process_topology_1.cart, &req1); - + if((self == me) && print_mess)fprintf(stderr, " I am %d, making request to send to %d...\n", self,send_peer); if(!print_mess)MPI_Isend((void *) d->d3_chunk, chunk_size, MPI_DOUBLE_COMPLEX, send_peer, 0, d->process_topology_1.cart, &req2); - + if((self == me) && print_mess)fprintf(stderr, " I am %d, waiting to recieve from %d...\n", self,recv_peer); //fprintf(stderr, " I am %d, waiting to recieve from %d...\n", self,recv_peer); if(!print_mess)MPI_Wait(&req1,MPI_STATUS_IGNORE); - + //if((self == me || self == 1 || self == 2 || self == 3) && print_me)fprintf(stderr, " I am %d, waiting to send to %d...\n", self,send_peer); //fprintf(stderr, " I am %d, waiting to send to %d...\n", self,send_peer); if(self==me && print_mess)fprintf(stderr, " I am %d, waiting to send to %d...\n", self,send_peer); if(!print_mess)MPI_Wait(&req2,MPI_STATUS_IGNORE); - + //fill the local array with the received chunk. int64_t ch_indx=0; int dims_size=pencil_dims[0]*pencil_dims[1]*pencil_dims[2]; @@ -1903,108 +1903,108 @@ static void redistribute_2_and_3(const complex_t *a, if(self==me && print_me)fprintf(stderr,"PENCIL DIMENSION VS. local sizes (%d,%d,%d) vs (%d,%d,%d)\n",pencil_dims[0],pencil_dims[1],pencil_dims[2],local_sizes[0],local_sizes[1],local_sizes[2]); if(self==me && print_me)fprintf(stderr,"DIM_2_ARRAY_START (%d,%d,%d) \n",d2_array_start[0],d2_array_start[1],d2_array_start[2]); for(int i0=d2_array_start[0];i0= dims_size)fprintf(stderr,"WOW, in third for, dims is (%d), we are %d and my rank is %d",dims_size,local_indx,self); - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - b[local_indx]=d->d2_chunk[ch_indx]; - //if((p==0 || p==1 || p==2 || p==3 || p==4 || p==5) && self==me)fprintf(stderr,"(%f,%f) ",real(d->d2_chunk[ch_indx]),imag(d->d2_chunk[ch_indx])); - ch_indx++; - } - } + for(int i1=d2_array_start[1];i1= dims_size)fprintf(stderr,"WOW, in third for, dims is (%d), we are %d and my rank is %d",dims_size,local_indx,self); + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + b[local_indx]=d->d2_chunk[ch_indx]; + //if((p==0 || p==1 || p==2 || p==3 || p==4 || p==5) && self==me)fprintf(stderr,"(%f,%f) ",real(d->d2_chunk[ch_indx]),imag(d->d2_chunk[ch_indx])); + ch_indx++; + } + } } // if((p==0 ||p==1 || p==2 || p==3 || p==4 || p==5) && self==me)fprintf(stderr,"P is %d \n",p); - - } + + } else if (direction == REDISTRIBUTE_2_TO_3) { - + if((self == me) && print_mess)fprintf(stderr, " I am %d, making request to recieve from %d...\n", self,recv_peer); if(!print_mess)MPI_Irecv((void *) d->d3_chunk, chunk_size, MPI_DOUBLE_COMPLEX, recv_peer, 0, d->process_topology_1.cart, &req1); - + if((self == me) && print_mess)fprintf(stderr, " I am %d, making request to send to %d...\n", self,send_peer); if(!print_mess)MPI_Isend((void *) d->d2_chunk, chunk_size, MPI_DOUBLE_COMPLEX, send_peer, 0, d->process_topology_1.cart, &req2); - + if((self == me) && print_mess)fprintf(stderr, " I am %d, waiting to recieve from %d...\n", self,recv_peer); if(!print_mess)MPI_Wait(&req1,MPI_STATUS_IGNORE); - + if((self == me) && print_mess)fprintf(stderr, " I am %d, waiting to send to %d...\n", self,send_peer); if(!print_mess)MPI_Wait(&req2,MPI_STATUS_IGNORE); int64_t ch_indx=0; int dims_size=(d->process_topology_3.n[2])*(d->process_topology_3.n[1])*(d->process_topology_3.n[0]); if(z_dim==0){ - //fill the local array with the received chunk. - - for(int i2=d3_array_start[y_dim];i2>d3_array_start[y_dim]-subsizes[y_dim];i2--){ - for(int i1=d3_array_start[x_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; - //if(local_indx >= dims_size)fprintf(stderr,"WOW, in fourth for, dims is (%d), we are %d and my rank is %d",dims_size,local_indx,self); - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - b[local_indx]=d->d3_chunk[ch_indx]; - // if(p==3 && self==me)fprintf(stderr,"(%f,%f) ",real(d->d3_chunk[ch_indx]),imag(d->d3_chunk[ch_indx])); - ch_indx++; - } - } - } + //fill the local array with the received chunk. + + for(int i2=d3_array_start[y_dim];i2>d3_array_start[y_dim]-subsizes[y_dim];i2--){ + for(int i1=d3_array_start[x_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; + //if(local_indx >= dims_size)fprintf(stderr,"WOW, in fourth for, dims is (%d), we are %d and my rank is %d",dims_size,local_indx,self); + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + b[local_indx]=d->d3_chunk[ch_indx]; + // if(p==3 && self==me)fprintf(stderr,"(%f,%f) ",real(d->d3_chunk[ch_indx]),imag(d->d3_chunk[ch_indx])); + ch_indx++; + } + } + } } else if(z_dim==1){ - for(int i0=d3_array_start[y_dim];i0d3_array_start[x_dim]-subsizes[x_dim];i2--){ - for(int i1=d3_array_start[z_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; - //if(local_indx >= dims_size)fprintf(stderr,"WOW, in fourth for, dims is (%d), we are %d and my rank is %d",dims_size,local_indx,self); - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - b[local_indx]=d->d3_chunk[ch_indx]; - // if(p==0 && self==me)fprintf(stderr,"(%f,%f) ",real(d->d3_chunk[ch_indx]),imag(d->d3_chunk[ch_indx])); - ch_indx++; - } - } - } - + for(int i0=d3_array_start[y_dim];i0d3_array_start[x_dim]-subsizes[x_dim];i2--){ + for(int i1=d3_array_start[z_dim];i1process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; + //if(local_indx >= dims_size)fprintf(stderr,"WOW, in fourth for, dims is (%d), we are %d and my rank is %d",dims_size,local_indx,self); + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + b[local_indx]=d->d3_chunk[ch_indx]; + // if(p==0 && self==me)fprintf(stderr,"(%f,%f) ",real(d->d3_chunk[ch_indx]),imag(d->d3_chunk[ch_indx])); + ch_indx++; + } + } + } + } else if(z_dim==2){ - for(int i0=d3_array_start[x_dim];i0process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; - assert(local_indx < dims_size); - assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); - b[local_indx]=d->d3_chunk[ch_indx]; - // if(p==1 && self==me)fprintf(stderr,"(%f,%f) ",real(d->d3_chunk[ch_indx]),imag(d->d3_chunk[ch_indx])); - ch_indx++; - } - } - } - + for(int i0=d3_array_start[x_dim];i0process_topology_3.n[2]*(d->process_topology_3.n[1]*i0+i1) + i2; + assert(local_indx < dims_size); + assert(ch_indx = 0 && local_indx>=0 && local_indx < dims_size); + b[local_indx]=d->d3_chunk[ch_indx]; + // if(p==1 && self==me)fprintf(stderr,"(%f,%f) ",real(d->d3_chunk[ch_indx]),imag(d->d3_chunk[ch_indx])); + ch_indx++; + } + } + } + } else{ - abort(); + abort(); } } - + if (DEBUG_CONDITION) { fprintf(stderr, - "%d: npeers,p,p0,p1,p1max=(%d,%d,%d,%d,%d), " - "d3_coord=(%d,%d,%d), d2_peer_coord=(%d,%d,%d), " - "d2_coord=(%d,%d,%d), d3_peer_coord=(%d,%d,%d), " - "recv_peer=%d, send_peer=%d\n", - self, - npeers, p, p0, p1, p1max, - d3_coord[0], d3_coord[1], d3_coord[2], - d2_peer_coord[0], d2_peer_coord[1], d2_peer_coord[2], - d2_coord[0], d2_coord[1], d2_coord[2], - d3_peer_coord[0], d3_peer_coord[1], d3_peer_coord[2], - recv_peer, send_peer); + "%d: npeers,p,p0,p1,p1max=(%d,%d,%d,%d,%d), " + "d3_coord=(%d,%d,%d), d2_peer_coord=(%d,%d,%d), " + "d2_coord=(%d,%d,%d), d3_peer_coord=(%d,%d,%d), " + "recv_peer=%d, send_peer=%d\n", + self, + npeers, p, p0, p1, p1max, + d3_coord[0], d3_coord[1], d3_coord[2], + d2_peer_coord[0], d2_peer_coord[1], d2_peer_coord[2], + d2_coord[0], d2_coord[1], d2_coord[2], + d3_peer_coord[0], d3_peer_coord[1], d3_peer_coord[2], + recv_peer, send_peer); } - + if((self == me) && print_me)fprintf(stderr, "%d, %d, %d Made it end-for!...\n", self,p, npeers); } - + //if((self == me) && print_me)fprintf(outfile, " Made it all the way! for z_dim =(%d) and num_proc = (%d)...\n", z_dim, d->process_topology_1.nproc[0]); if((self == me) && print_result){ FILE * outfile; diff --git a/Src/Extern/SWFFT/distribution_c.h b/Src/Extern/SWFFT/distribution_c.h index b6cce694abd..4f915738290 100644 --- a/Src/Extern/SWFFT/distribution_c.h +++ b/Src/Extern/SWFFT/distribution_c.h @@ -42,8 +42,8 @@ * ***************************************************************************** * DISCLAIMER * THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND. NEITHER THE - * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR - * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, + * UNITED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT OF ENERGY, NOR + * UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES, MAKES ANY WARRANTY, * EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY OR RESPONSIBILITY FOR THE * ACCURARY, COMPLETENESS, OR USEFULNESS OF ANY INFORMATION, DATA, APPARATUS, * PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS THAT ITS USE WOULD NOT INFRINGE @@ -115,12 +115,12 @@ typedef struct { // rmap pointer to grid->rank map // debug debugging output /// -void distribution_init(MPI_Comm comm, - const int n[], +void distribution_init(MPI_Comm comm, + const int n[], const int Ndims[], - distribution_t *d, + distribution_t *d, const int* rmap, - bool debug); + bool debug); /// @@ -192,8 +192,8 @@ void distribution_3_to_1(const complex_t *a, /// void distribution_2_to_3(const complex_t *a, complex_t *b, - distribution_t *d, - int dim_z); + distribution_t *d, + int dim_z); /// // redistribute a 3-d to a 2-d data distribution @@ -203,8 +203,8 @@ void distribution_2_to_3(const complex_t *a, /// void distribution_3_to_2(const complex_t *a, complex_t *b, - distribution_t *d, - int dim_z); + distribution_t *d, + int dim_z); /// diff --git a/Src/Extern/amrdata/AMReX_AmrData.H b/Src/Extern/amrdata/AMReX_AmrData.H index 6a6d27259a5..eda36f6288b 100644 --- a/Src/Extern/amrdata/AMReX_AmrData.H +++ b/Src/Extern/amrdata/AMReX_AmrData.H @@ -24,15 +24,15 @@ namespace amrex { class Interpolater; class AmrData { - - protected: + + protected: // List of grids at each level, level 0 being coarsest. Vector > dataGrids; // [level][component] vector > > dataGridsDefined; // [level][component][index] Vector > visMF; // [level][whichMultiFab] Vector compIndexToVisMFMap; // [nComp] Vector compIndexToVisMFComponentMap; // [nComp] - + int finestLevel; // most refined level available Vector refRatio; // refinement ratio between level L and L+1 Vector< Vector > dxLevel; // [level][dim]; // grid spacing @@ -50,23 +50,23 @@ class AmrData { Vector< Vector< FArrayBox *> > regions; // [lev][nReg] ptrs to bndry data BoxArray fabBoxArray; // used only for fileType == Amrvis::FAB string plotFileVersion; - + Amrvis::FileType fileType; bool bCartGrid, bShowBody; int vCartGrid; // ---- the CartGrid version bool bTerrain; Vector levelSteps; - + public: AmrData(); ~AmrData(); - + bool ReadData(const string &filename, Amrvis::FileType filetype); bool ReadNonPlotfileData(const string &filename, Amrvis::FileType filetype); - + const Vector &PlotVarNames() const { return plotVars; } - - int FinestLevel() const { return finestLevel; } + + int FinestLevel() const { return finestLevel; } const Vector &RefRatio() const { return refRatio; } const BoxArray &boxArray(int level) const { if(fileType == Amrvis::FAB || (fileType == Amrvis::MULTIFAB && level == 0)) { @@ -77,12 +77,12 @@ class AmrData { return visMF[level][0]->boxArray(); } } - + const DistributionMapping& DistributionMap (int level) const { if(fileType == Amrvis::FAB || (fileType == Amrvis::MULTIFAB && level == 0)) { - return dataGrids[0][0]->DistributionMap(); + return dataGrids[0][0]->DistributionMap(); } else { - return dataGrids[level][0]->DistributionMap(); + return dataGrids[level][0]->DistributionMap(); } } @@ -92,7 +92,7 @@ class AmrData { const Vector &ProbSize() const { return probSize; } const Vector &ProbLo() const { return probLo; } const Vector &ProbHi() const { return probHi; } - + // return physical location of cell at index ix, level lev // cellLoc = location of cell center // loNodeLoc = location of node (vertex) on lower left hand corner @@ -109,31 +109,31 @@ class AmrData { // and the intvect at the given finest level void IntVectFromLocation(const int finestFillLevel, const Vector &location, IntVect &ivLoc, int &ivLevel, IntVect &ivFinestFillLev); - + const Vector< Vector< Vector > > &GridLocLo() const { return gridLocLo; } const Vector< Vector< Vector > > &GridLocHi() const { return gridLocHi; } const Vector< Vector > &DxLevel() const { return dxLevel; } const Vector &LevelSteps() const { return levelSteps; } - - int NComp() const { return nComp; } - int BoundaryWidth() const { return boundaryWidth; } - int NGrow() const { return nGrow; } - int CoordSys() const { return coordSys; } - Real Time() const { return time; } + + int NComp() const { return nComp; } + int BoundaryWidth() const { return boundaryWidth; } + int NGrow() const { return nGrow; } + int CoordSys() const { return coordSys; } + Real Time() const { return time; } const string &PlotFileVersion() const { return plotFileVersion; } - + // fill a databox using conservative interpolation void FillVar(FArrayBox *destFab, const Box &destBox, int finestFillLevel, const string &varName, int procWithFabs); void FillVar(Vector &destFabs, const Vector &destBoxes, int finestFillLevel, const string &varName, int procWithFabs); void FillVar(MultiFab &destMultiFab, int finestFillLevel, - const Vector &varNames, const Vector &destFillComps); + const Vector &varNames, const Vector &destFillComps); void FillVar(MultiFab &destMultiFab, int finestFillLevel, - const string &varname, int destcomp = 0); - + const string &varname, int destcomp = 0); + const string &GetFileName() const { return fileName; } - + void SetFileType(Amrvis::FileType type); Amrvis::FileType GetFileType() const { return fileType; } bool CartGrid() const { return bCartGrid; } @@ -145,21 +145,21 @@ class AmrData { bool Terrain() const { return bTerrain; } void SetBoundaryWidth(int width) { boundaryWidth = width; } - + bool CanDerive(const string &name) const; bool CanDerive(const Vector &names) const; int NumDeriveFunc() const; void ListDeriveFunc(std::ostream &os) const; int StateNumber(const string &statename) const; - + // return the finest level <= startLevel that fully contains box b // b is defined on startLevel int FinestContainingLevel(const Box &b, int startLevel) const; - + // return the finest level <= startLevel that intersects box b // b is defined on startLevel int FinestIntersectingLevel(const Box &b, int startLevel) const; - + // number of grids at level which intersect b int NIntersectingGrids(int level, const Box &b) const; MultiFab &GetGrids(int level, int componentIndex); @@ -167,24 +167,24 @@ class AmrData { void FlushGrids(); // Clear all internal field data void FlushGrids(int componentIndex); // Clear all internal field data associated with this component - + // calculate the min and max values of derived on onBox at level // return false if onBox did not intersect any grids bool MinMax(const Box &onBox, const string &derived, int level, Real &dataMin, Real &dataMax); - + static void SetVerbose(bool tf) { verbose = tf; } static bool Verbose() { return verbose; } static void SetSkipPltLines(int spl) { skipPltLines = spl; } static void SetStaticBoundaryWidth(int bw) { sBoundaryWidth = bw; } - + private: string fileName; static Amrvis::FileType defaultFileType; static bool verbose; static int skipPltLines; static int sBoundaryWidth; - + // fill on interior by piecewise constant interpolation void FillInterior(FArrayBox &dest, int level, const Box &subbox); void Interp(FArrayBox &fine, FArrayBox &crse, diff --git a/Src/Extern/amrdata/AMReX_AmrData.cpp b/Src/Extern/amrdata/AMReX_AmrData.cpp index ee86b11d8b0..ee02ad6f9a9 100644 --- a/Src/Extern/amrdata/AMReX_AmrData.cpp +++ b/Src/Extern/amrdata/AMReX_AmrData.cpp @@ -79,25 +79,25 @@ extern "C" { void FORT_CINTERP(amrex::Real *fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), const int *fblo, const int *fbhi, const int &nvar, const int &lratio, - const amrex::Real *crse, const int &clo, const int &chi, - const int *cslo, const int *cshi, - const int *fslo, const int *fshi, - amrex::Real *cslope, const int &c_len, - amrex::Real *fslope, amrex::Real *fdat, const int &f_len, - amrex::Real *foff); + const amrex::Real *crse, const int &clo, const int &chi, + const int *cslo, const int *cshi, + const int *fslo, const int *fshi, + amrex::Real *cslope, const int &c_len, + amrex::Real *fslope, amrex::Real *fdat, const int &f_len, + amrex::Real *foff); #endif void FORT_PCINTERP(amrex::Real *fine, AMREX_ARLIM_P(flo), AMREX_ARLIM_P(fhi), const int *fblo, const int *fbhi, - const int &lrat, const int &nvar, - const amrex::Real *crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), - const int *cblo, const int *cbhi, - amrex::Real *temp, const int &tlo, const int &thi); + const int &lrat, const int &nvar, + const amrex::Real *crse, AMREX_ARLIM_P(clo), AMREX_ARLIM_P(chi), + const int *cblo, const int *cbhi, + amrex::Real *temp, const int &tlo, const int &thi); #if (BL_SPACEDIM != 1) void FORT_CARTGRIDMINMAX (amrex::Real *data, AMREX_ARLIM_P(dlo), AMREX_ARLIM_P(dhi), - const amrex::Real *vfrac, const amrex::Real &vfeps, - amrex::Real &dmin, amrex::Real &dmax); + const amrex::Real *vfrac, const amrex::Real &vfeps, + amrex::Real &dmin, amrex::Real &dmax); #endif } @@ -236,12 +236,12 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { if(verbose) { if(ParallelDescriptor::IOProcessor()) { cout << "Plot file version: " << plotFileVersion << endl; - if(bCartGrid) { - cout << ":::: Found a CartGrid file: version " << vCartGrid << endl; - } - if(bTerrain) { - cout << ":::: Found a Terrain file type." << endl; - } + if(bCartGrid) { + cout << ":::: Found a CartGrid file: version " << vCartGrid << endl; + } + if(bTerrain) { + cout << ":::: Found a Terrain file type." << endl; + } } } @@ -263,31 +263,31 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { isPltIn.getline(plotVarName, Amrvis::LINELENGTH); // eat white space left by op<< for(i = 0; i < nComp; ++i) { isPltIn.getline(plotVarName, Amrvis::LINELENGTH); - mytrim(plotVarName); - if(bCartGrid) { // check various permutations of vfrac name - if(strcmp(plotVarName, "vol_frac") == 0) { - cout << "+++++++++++++ found bad vfrac name: " << plotVarName << endl; - strcpy(plotVarName, "vfrac"); - cout << "+++++++++++++ now: " << plotVarName << endl; - } - if(strcmp(plotVarName, "volfrac") == 0) { - cout << "+++++++++++++ found bad vfrac name: " << plotVarName << endl; - strcpy(plotVarName, "vfrac"); - cout << "+++++++++++++ now: " << plotVarName << endl; - } - if(strcmp(plotVarName, "vfrac") == 0) { + mytrim(plotVarName); + if(bCartGrid) { // check various permutations of vfrac name + if(strcmp(plotVarName, "vol_frac") == 0) { + cout << "+++++++++++++ found bad vfrac name: " << plotVarName << endl; + strcpy(plotVarName, "vfrac"); + cout << "+++++++++++++ now: " << plotVarName << endl; + } + if(strcmp(plotVarName, "volfrac") == 0) { + cout << "+++++++++++++ found bad vfrac name: " << plotVarName << endl; + strcpy(plotVarName, "vfrac"); + cout << "+++++++++++++ now: " << plotVarName << endl; + } + if(strcmp(plotVarName, "vfrac") == 0) { bVFracFound = true; - } - } + } + } plotVars[i] = plotVarName; if(ParallelDescriptor::IOProcessor()) { VSHOWVAL(verbose, plotVarName); - } + } } if(bCartGrid && bVFracFound == false) { - cerr << endl << " ~~~~ Error: no vfrac found for a " << plotFileVersion - << " file." << endl << endl; - return false; + cerr << endl << " ~~~~ Error: no vfrac found for a " << plotFileVersion + << " file." << endl << endl; + return false; } int spacedim; @@ -299,40 +299,40 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { } if(spacedim != BL_SPACEDIM) { if(ParallelDescriptor::IOProcessor()) { - cerr << endl << " ~~~~ Error: You are using " << BL_SPACEDIM - << "D amrvis " - << "to look at a " << spacedim << "D file." << endl << endl; - } - return false; + cerr << endl << " ~~~~ Error: You are using " << BL_SPACEDIM + << "D amrvis " + << "to look at a " << spacedim << "D file." << endl << endl; + } + return false; } if(finestLevel < 0) { if(ParallelDescriptor::IOProcessor()) { cerr << "Error in AmrData: bad finestLevel = " << finestLevel << endl; - } + } return false; } for(i = 0; i < BL_SPACEDIM; ++i) { isPltIn >> probLo[i]; if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "probLo[" << i << "] = " << probLo[i] << endl; - } - } + cout << "probLo[" << i << "] = " << probLo[i] << endl; + } + } } for(i = 0; i < BL_SPACEDIM; ++i) { isPltIn >> probHi[i]; if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "probHi[" << i << "] = " << probHi[i] << endl; - } - } + cout << "probHi[" << i << "] = " << probHi[i] << endl; + } + } } if(verbose) { if(ParallelDescriptor::IOProcessor()) { - if(finestLevel > 0) { - cout << "Resizing refRatio to size = " << finestLevel << endl; - } - } + if(finestLevel > 0) { + cout << "Resizing refRatio to size = " << finestLevel << endl; + } + } } if(finestLevel == 0) { refRatio.resize(1, 1); @@ -347,31 +347,31 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { bIVRefRatio = true; } for(i = 0; i < finestLevel; ++i) { - // try to guess if refRatio is an IntVect - if(bIVRefRatio) { // it is an IntVect - IntVect ivRefRatio; - isPltIn >> ivRefRatio; + // try to guess if refRatio is an IntVect + if(bIVRefRatio) { // it is an IntVect + IntVect ivRefRatio; + isPltIn >> ivRefRatio; if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "IntVect refRatio[" << i << "] = " << ivRefRatio << endl; - } - } - refRatio[i] = ivRefRatio[0]; // non-uniform ref ratios not supported - } else { + cout << "IntVect refRatio[" << i << "] = " << ivRefRatio << endl; + } + } + refRatio[i] = ivRefRatio[0]; // non-uniform ref ratios not supported + } else { isPltIn >> refRatio[i]; - } + } if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "refRatio[" << i << "] = " << refRatio[i] << endl; - } - } + cout << "refRatio[" << i << "] = " << refRatio[i] << endl; + } + } } for(i = 0; i < finestLevel; ++i ) { if(refRatio[i] < 2 || refRatio[i] > 32 ) { if(ParallelDescriptor::IOProcessor()) { cerr << "Error in AmrData: bad refRatio at level " << i << " = " - << refRatio[i] << endl; - } + << refRatio[i] << endl; + } return false; } } @@ -382,16 +382,16 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { maxDomain.resize(finestLevel + 1); for(i = 0; i <= finestLevel; ++i) { isPltIn >> probDomain[i]; - if(verbose) { + if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "probDomain[" << i << "] = " << probDomain[i] << endl; - } - } + cout << "probDomain[" << i << "] = " << probDomain[i] << endl; + } + } if( ! probDomain[i].ok()) { if(ParallelDescriptor::IOProcessor()) { cerr << "Error in AmrData: bad probDomain[" << i << "] = " - << probDomain[i] << endl; - } + << probDomain[i] << endl; + } return false; } } @@ -399,17 +399,17 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { while(isPltIn.get() != '\n') { ; // do nothing } -#if 0 +#if 0 char lstepbuff[128]; isPltIn.getline(lstepbuff, 128); // ignore levelsteps--some files have - // finestlevel of these, others have - // finestlevel + 1 + // finestlevel of these, others have + // finestlevel + 1 + - if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "Ignored levelSteps = " << lstepbuff << endl; - } + cout << "Ignored levelSteps = " << lstepbuff << endl; + } } #else // Get level steps - why do some have fl+1 ? (note above...just throw away last one here) @@ -426,22 +426,22 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { for(i = 0; i <= finestLevel; ++i) { cout << "LevelSteps[" << i << "] = " << levelSteps[i] << endl; } - } + } } #endif - + dxLevel.resize(finestLevel + 1); for(i = 0; i <= finestLevel; ++i) { dxLevel[i].resize(BL_SPACEDIM); for(k = 0; k < BL_SPACEDIM; ++k) { - isPltIn >> dxLevel[i][k]; - if(verbose) { + isPltIn >> dxLevel[i][k]; + if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "dxLevel[" << i << "][" << k << "] = " - << dxLevel[i][k] << endl; - } - } - } + cout << "dxLevel[" << i << "][" << k << "] = " + << dxLevel[i][k] << endl; + } + } + } } vfEps.resize(finestLevel + 1); // must resize these even if not cartGrid @@ -460,10 +460,10 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { if(probSize[i] <= 0.0 ) { if(ParallelDescriptor::IOProcessor()) { cerr << "Error in AmrData: bad probSize[" << i << "] = " - << probSize[i] << endl; - } + << probSize[i] << endl; + } return false; - } + } } isPltIn >> coordSys; @@ -510,15 +510,15 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { regions[lev].resize(nRegions); i = 0; while(bli != bndry_boxes.end()) { - regions[lev][i] = new FArrayBox(*bli, nComp); - if(verbose) { + regions[lev][i] = new FArrayBox(*bli, nComp); + if(verbose) { if(ParallelDescriptor::IOProcessor()) { - cout << "BNDRY REGION " << i << " : " << *bli << endl; - cout << " numPts = " << bli->numPts() << endl; - } - } - ++i; - ++bli; + cout << "BNDRY REGION " << i << " : " << *bli << endl; + cout << " numPts = " << bli->numPts() << endl; + } + } + ++i; + ++bli; } } @@ -526,7 +526,7 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { if(width > 0) { if(ParallelDescriptor::IOProcessor()) { cerr << "Error in AmrData: Boundary width > 0 not supported: width = " - << width << endl; + << width << endl; } return false; } // end if(width...) @@ -552,16 +552,16 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { } if(i != lev) { if(ParallelDescriptor::IOProcessor()) { - cerr << "Level misrestart:mismatch on restart" << endl; + cerr << "Level misrestart:mismatch on restart" << endl; cerr << "Error in AmrData: Level mismatch: read level " << lev - << " while expecting level " << i << endl; - } + << " while expecting level " << i << endl; + } return false; } if(nGrids < 1) { if(ParallelDescriptor::IOProcessor()) { cerr << "Error in AmrData: bad nGrids = " << nGrids << endl; - } + } return false; } @@ -570,13 +570,13 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { for(int iloc = 0; iloc < nGrids; ++iloc) { gridLocLo[i][iloc].resize(BL_SPACEDIM); gridLocHi[i][iloc].resize(BL_SPACEDIM); - for(int iDim = 0; iDim < BL_SPACEDIM; ++iDim) { - isPltIn >> gridLocLo[i][iloc][iDim] >> gridLocHi[i][iloc][iDim]; + for(int iDim = 0; iDim < BL_SPACEDIM; ++iDim) { + isPltIn >> gridLocLo[i][iloc][iDim] >> gridLocHi[i][iloc][iDim]; if(ParallelDescriptor::IOProcessor()) { VSHOWVAL(verbose, gridLocLo[i][iloc][iDim]); VSHOWVAL(verbose, gridLocHi[i][iloc][iDim]); - } - } + } + } } // here we account for multiple multifabs in a plot file @@ -603,21 +603,21 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { visMF[i].resize(currentVisMF + 1); // this preserves previous ones visMF[i][currentVisMF] = new VisMF(mfName); - const BoxArray& ba = visMF[i][currentVisMF]->boxArray(); - if (!dmap) dmap.reset(new DistributionMapping(ba)); - int iComp(currentIndexComp); + const BoxArray& ba = visMF[i][currentVisMF]->boxArray(); + if (!dmap) dmap.reset(new DistributionMapping(ba)); + int iComp(currentIndexComp); nGrow = visMF[i][currentVisMF]->nGrow(); currentIndexComp += visMF[i][currentVisMF]->nComp(); - int currentVisMFComponent(0); + int currentVisMFComponent(0); for( ; iComp < currentIndexComp; ++iComp) { // make single component multifabs // defer reading the MultiFab data - dataGrids[i][iComp] = new MultiFab(ba, *dmap, - 1, - visMF[i][currentVisMF]->nGrow(), - Fab_noallocate); + dataGrids[i][iComp] = new MultiFab(ba, *dmap, + 1, + visMF[i][currentVisMF]->nGrow(), + Fab_noallocate); dataGridsDefined[i][iComp].resize(visMF[i][currentVisMF]->size(), - false); + false); compIndexToVisMFMap[iComp] = currentVisMF; compIndexToVisMFComponentMap[iComp] = currentVisMFComponent; ++currentVisMFComponent; @@ -639,133 +639,133 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { BoxList outside = amrex::boxDiff(reg1, reg2); if(outside.size() > 0) { // parts of the bndry have not been filled from the input - // data, must extending from interior regions + // data, must extending from interior regions for(int idir(0); idir < BL_SPACEDIM; ++idir) { Box bx(amrex::adjCellLo(inbox,idir,boundaryWidth)); - Box br(bx); - for(k = 0; k < BL_SPACEDIM; ++k) { - if(k != idir) { - br.grow(k,boundaryWidth); - } - } - - br.shift(idir,1); - FArrayBox tmpreg(br,nComp); - Box reg_bx = tmpreg.box(); - reg_bx &= inbox; - FillInterior(tmpreg,lev,reg_bx); - br.shift(idir,-1); - FArrayBox tmpreg_lo(br,nComp); - tmpreg_lo.copy(tmpreg,tmpreg.box(),0,tmpreg_lo.box(),0,nComp); + Box br(bx); + for(k = 0; k < BL_SPACEDIM; ++k) { + if(k != idir) { + br.grow(k,boundaryWidth); + } + } + + br.shift(idir,1); + FArrayBox tmpreg(br,nComp); + Box reg_bx = tmpreg.box(); + reg_bx &= inbox; + FillInterior(tmpreg,lev,reg_bx); + br.shift(idir,-1); + FArrayBox tmpreg_lo(br,nComp); + tmpreg_lo.copy(tmpreg,tmpreg.box(),0,tmpreg_lo.box(),0,nComp); // now fill out tmp region along idir direction - Box b_lo(amrex::adjCellLo(inbox,idir,1)); - for(k = 1; k < boundaryWidth; ++k) { - Box btmp(b_lo); - btmp.shift(idir, -k); - tmpreg_lo.copy(tmpreg_lo,b_lo,0,btmp,0,nComp); - } - - // now fill out temp bndry region - Box b_src, b_dest; - int n; - for(k = 1; k < BL_SPACEDIM; ++k) { - int kdir = (idir + k) % BL_SPACEDIM; - b_dest = amrex::adjCellLo(bx, kdir, 1); - b_src = b_dest; - b_src = b_src.shift(kdir, 1); + Box b_lo(amrex::adjCellLo(inbox,idir,1)); + for(k = 1; k < boundaryWidth; ++k) { + Box btmp(b_lo); + btmp.shift(idir, -k); + tmpreg_lo.copy(tmpreg_lo,b_lo,0,btmp,0,nComp); + } + + // now fill out temp bndry region + Box b_src, b_dest; + int n; + for(k = 1; k < BL_SPACEDIM; ++k) { + int kdir = (idir + k) % BL_SPACEDIM; + b_dest = amrex::adjCellLo(bx, kdir, 1); + b_src = b_dest; + b_src = b_src.shift(kdir, 1); for(n = 1; n <= boundaryWidth; ++n) { - tmpreg_lo.copy(tmpreg_lo, b_src, 0, b_dest, 0, nComp); - b_dest.shift(kdir, -1); - } + tmpreg_lo.copy(tmpreg_lo, b_src, 0, b_dest, 0, nComp); + b_dest.shift(kdir, -1); + } - b_dest = amrex::adjCellHi(bx,kdir,1); - b_src = b_dest; - b_src.shift(kdir,-1); + b_dest = amrex::adjCellHi(bx,kdir,1); + b_src = b_dest; + b_src.shift(kdir,-1); for(n = 1; n <= boundaryWidth; ++n) { - tmpreg_lo.copy(tmpreg_lo,b_src,0,b_dest,0,nComp); - b_dest.shift(kdir,1); - } - bx.grow(kdir,boundaryWidth); - } - - // now copy into real bndry regions - for(j = 0; j < nRegions; ++j) { - FArrayBox *p = regions[lev][j]; - Box p_box = p->box(); - BoxList::iterator bli = outside.begin(); - while(bli != outside.end()) { + tmpreg_lo.copy(tmpreg_lo,b_src,0,b_dest,0,nComp); + b_dest.shift(kdir,1); + } + bx.grow(kdir,boundaryWidth); + } + + // now copy into real bndry regions + for(j = 0; j < nRegions; ++j) { + FArrayBox *p = regions[lev][j]; + Box p_box = p->box(); + BoxList::iterator bli = outside.begin(); + while(bli != outside.end()) { Box ovlp(p_box); - ovlp &= *bli; - ovlp &= br; - if(ovlp.ok()) { - p->copy(tmpreg_lo, ovlp); - } - ++bli; + ovlp &= *bli; + ovlp &= br; + if(ovlp.ok()) { + p->copy(tmpreg_lo, ovlp); + } + ++bli; } - } // end for j + } // end for j // now work on the high side of the bndry region bx = amrex::adjCellHi(inbox,idir,boundaryWidth); - br = bx; - for(k = 0; k < BL_SPACEDIM; ++k) { - if(k != idir) br.grow(k, boundaryWidth); - } - - br.shift(idir,-1); - FArrayBox tmpreg2(br,nComp); - reg_bx = tmpreg2.box(); - reg_bx &= inbox; - FillInterior(tmpreg2,lev,reg_bx); - br.shift(idir,1); - FArrayBox tmpreg_hi(br,nComp); - tmpreg_hi.copy(tmpreg2,tmpreg2.box(),0,tmpreg_hi.box(),0,nComp); + br = bx; + for(k = 0; k < BL_SPACEDIM; ++k) { + if(k != idir) br.grow(k, boundaryWidth); + } + + br.shift(idir,-1); + FArrayBox tmpreg2(br,nComp); + reg_bx = tmpreg2.box(); + reg_bx &= inbox; + FillInterior(tmpreg2,lev,reg_bx); + br.shift(idir,1); + FArrayBox tmpreg_hi(br,nComp); + tmpreg_hi.copy(tmpreg2,tmpreg2.box(),0,tmpreg_hi.box(),0,nComp); // now fill out tmp region along idir direction - Box b_hi(amrex::adjCellHi(inbox,idir,1)); - for(k = 1; k < boundaryWidth; ++k) { - Box btmp(b_hi); - btmp.shift(idir,k); - tmpreg_hi.copy(tmpreg_hi,b_hi,0,btmp,0,nComp); - } - - // now fill out temp bndry region - for(k = 1; k < BL_SPACEDIM; ++k) { - int kdir = (idir + k) % BL_SPACEDIM; - b_dest = amrex::adjCellLo(bx, kdir, 1); - b_src = b_dest; - b_src.shift(kdir, 1); + Box b_hi(amrex::adjCellHi(inbox,idir,1)); + for(k = 1; k < boundaryWidth; ++k) { + Box btmp(b_hi); + btmp.shift(idir,k); + tmpreg_hi.copy(tmpreg_hi,b_hi,0,btmp,0,nComp); + } + + // now fill out temp bndry region + for(k = 1; k < BL_SPACEDIM; ++k) { + int kdir = (idir + k) % BL_SPACEDIM; + b_dest = amrex::adjCellLo(bx, kdir, 1); + b_src = b_dest; + b_src.shift(kdir, 1); for(n = 1; n <= boundaryWidth; ++n) { - tmpreg_hi.copy(tmpreg_hi, b_src, 0, b_dest, 0, nComp); - b_dest.shift(kdir,-1); - } + tmpreg_hi.copy(tmpreg_hi, b_src, 0, b_dest, 0, nComp); + b_dest.shift(kdir,-1); + } - b_dest = amrex::adjCellHi(bx, kdir, 1); - b_src = b_dest; - b_src.shift(kdir, -1); + b_dest = amrex::adjCellHi(bx, kdir, 1); + b_src = b_dest; + b_src.shift(kdir, -1); for(n = 1; n <= boundaryWidth; ++n) { - tmpreg_hi.copy(tmpreg_hi, b_src, 0, b_dest, 0, nComp); - b_dest.shift(kdir, 1); - } - bx.grow(kdir, boundaryWidth); - } - - // now copy into real bndry regions - for(j = 0; j < nRegions; ++j) { - FArrayBox *p = regions[lev][j]; - Box p_box = p->box(); - BoxList::iterator bli = outside.begin(); - while(bli != outside.end()) { + tmpreg_hi.copy(tmpreg_hi, b_src, 0, b_dest, 0, nComp); + b_dest.shift(kdir, 1); + } + bx.grow(kdir, boundaryWidth); + } + + // now copy into real bndry regions + for(j = 0; j < nRegions; ++j) { + FArrayBox *p = regions[lev][j]; + Box p_box = p->box(); + BoxList::iterator bli = outside.begin(); + while(bli != outside.end()) { Box ovlp(p_box); - ovlp &= *bli; - ovlp &= br; - if(ovlp.ok()) { - p->copy(tmpreg_hi, ovlp); - } - ++bli; + ovlp &= *bli; + ovlp &= br; + if(ovlp.ok()) { + p->copy(tmpreg_hi, ovlp); + } + ++bli; } - } // end for j + } // end for j } // end for(idir...) } // end if(outside.size())... @@ -780,8 +780,8 @@ bool AmrData::ReadData(const string &filename, Amrvis::FileType filetype) { HiNodeLoc(0,maxDomain[0].bigEnd(),p_hi); for(i = 0; i < BL_SPACEDIM; ++i) { probLo[i] = p_lo[i]; - probHi[i] = p_hi[i]; - probSize[i] = p_hi[i] - p_lo[i]; + probHi[i] = p_hi[i]; + probSize[i] = p_hi[i] - p_lo[i]; } for(lev = 0; lev <= finestLevel; ++lev) { probDomain[lev] = maxDomain[lev]; @@ -961,8 +961,8 @@ bool AmrData::ReadNonPlotfileData(const string &filename, Amrvis::FileType /*fil zvMin = tempVisMF.min(0, iComp); // init with first value zvMax = tempVisMF.max(0, iComp); // init with first value for(int ic(0); ic < tempVisMF.size(); ++ic) { - zvMin = std::min(zvMin, tempVisMF.min(ic, iComp)); - zvMax = std::max(zvMax, tempVisMF.max(ic, iComp)); + zvMin = std::min(zvMin, tempVisMF.min(ic, iComp)); + zvMax = std::max(zvMax, tempVisMF.max(ic, iComp)); } levelZeroValue = zvMin; newfab->setVal(levelZeroValue); @@ -972,11 +972,11 @@ bool AmrData::ReadNonPlotfileData(const string &filename, Amrvis::FileType /*fil int idx; for(int icr(0); icr < newfab->box().length(1); ++icr) { for(int icc(0); icc < newfab->box().length(0); ++icc) { - idx = icc + (icr * newfab->box().length(0)); - BL_ASSERT(idx < newfab->box().numPts()); - if((icc + icr) % 5 == 0) { + idx = icc + (icr * newfab->box().length(0)); + BL_ASSERT(idx < newfab->box().numPts()); + if((icc + icr) % 5 == 0) { dptr[idx] = zvMax; - } + } } } #endif @@ -1002,16 +1002,16 @@ bool AmrData::ReadNonPlotfileData(const string &filename, Amrvis::FileType /*fil while(currentIndexComp < nComp) { visMF[LevelOne].resize(currentVisMF + 1); // this preserves previous ones visMF[LevelOne][currentVisMF] = new VisMF(filename); - const BoxArray& ba = visMF[LevelOne][currentVisMF]->boxArray(); - if (!dmap) dmap.reset(new DistributionMapping(ba)); + const BoxArray& ba = visMF[LevelOne][currentVisMF]->boxArray(); + if (!dmap) dmap.reset(new DistributionMapping(ba)); int iComp(currentIndexComp); mfNGrow = visMF[LevelOne][currentVisMF]->nGrow(); currentIndexComp += visMF[LevelOne][currentVisMF]->nComp(); for(int currentVisMFComponent(0); iComp < currentIndexComp; ++iComp) { // make single component multifabs for level one dataGrids[1][iComp] = - new MultiFab(ba, *dmap, - 1, visMF[LevelOne][currentVisMF]->nGrow(), + new MultiFab(ba, *dmap, + 1, visMF[LevelOne][currentVisMF]->nGrow(), Fab_noallocate); dataGridsDefined[LevelOne][iComp].resize(visMF[LevelOne][currentVisMF]->size(), false); compIndexToVisMFMap[iComp] = currentVisMF; @@ -1059,7 +1059,7 @@ void AmrData::HiNodeLoc(int lev, IntVect ix, Vector &pos) const { void AmrData::IntVectFromLocation(const int finestFillLevel, const Vector &location, IntVect &ivLoc, int &ivLevel, - IntVect &ivFinestFillLev) + IntVect &ivFinestFillLev) { BL_ASSERT(location.size() == BL_SPACEDIM); BL_ASSERT(finestFillLevel <= finestLevel); @@ -1069,7 +1069,7 @@ void AmrData::IntVectFromLocation(const int finestFillLevel, for(int i(0); i < BL_SPACEDIM; ++i) { ival = probDomain[finestFillLevel].smallEnd()[i] + (static_cast ( (location[i] - probLo[i]) / - dxLevel[finestFillLevel][i] ) ); + dxLevel[finestFillLevel][i] ) ); ivFinestFillLev.setVal(i, ival); } Box fflBox(ivFinestFillLev, ivFinestFillLev); @@ -1077,7 +1077,7 @@ void AmrData::IntVectFromLocation(const int finestFillLevel, for(int i(0); i < BL_SPACEDIM; ++i) { ival = probDomain[ivLevel].smallEnd()[i] + (static_cast ( (location[i] - probLo[i]) / - dxLevel[ivLevel][i] ) ); + dxLevel[ivLevel][i] ) ); ivLoc.setVal(i, ival); } } @@ -1085,7 +1085,7 @@ void AmrData::IntVectFromLocation(const int finestFillLevel, // --------------------------------------------------------------- void AmrData::FillVar(FArrayBox *destFab, const Box &destBox, - int finestFillLevel, const string &varname, int procWithFabs) + int finestFillLevel, const string &varname, int procWithFabs) { Vector destFabs(1); Vector destBoxes(1); @@ -1098,7 +1098,7 @@ void AmrData::FillVar(FArrayBox *destFab, const Box &destBox, // --------------------------------------------------------------- void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, - const string &varname, int destcomp) + const string &varname, int destcomp) { int numFillComps(1); Vector varNames(numFillComps); @@ -1111,8 +1111,8 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, // --------------------------------------------------------------- void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, - const Vector &varNames, - const Vector &destFillComps) + const Vector &varNames, + const Vector &destFillComps) { // This function fills the destMultiFab which is defined on // the finestFillLevel. @@ -1152,7 +1152,7 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, // ensure the required grids are in memory for(currentLevel = 0; currentLevel <= finestFillLevel; ++currentLevel) { for(int iBox = 0; iBox < destBoxes.size(); ++iBox) { - Box tempCoarseBox(destBoxes[iBox]); + Box tempCoarseBox(destBoxes[iBox]); if(currentLevel != finestFillLevel) { tempCoarseBox.coarsen(cumulativeRefRatios[currentLevel]); } @@ -1168,11 +1168,11 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, } BoxArray localMFBoxes(destBoxes.size()); // These are the ones - // we want to fillpatch. + // we want to fillpatch. Vector< Vector< Vector< Vector > > > fillBoxId; Vector< Vector< Vector< Vector > > > fillBoxIdBAs; - // [grid][level][fillablesubbox][oldnew] - // oldnew not used here + // [grid][level][fillablesubbox][oldnew] + // oldnew not used here Vector< Vector< Vector > > savedFineBox; // [grid][level][fillablesubbox] fillBoxId.resize(destBoxes.size()); @@ -1180,7 +1180,7 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, savedFineBox.resize(destBoxes.size()); for(int iBox(0); iBox < destBoxes.size(); ++iBox) { if(destMultiFab.DistributionMap()[iBox] == myProc) { - localMFBoxes.set(iBox, destBoxes[iBox]); + localMFBoxes.set(iBox, destBoxes[iBox]); fillBoxId[iBox].resize(finestFillLevel + 1); fillBoxIdBAs[iBox].resize(finestFillLevel + 1); savedFineBox[iBox].resize(finestFillLevel + 1); @@ -1193,7 +1193,7 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, // Do this for all local fab boxes. for(int ibox(0); ibox < localMFBoxes.size(); ++ibox) { if(destMultiFab.DistributionMap()[ibox] != myProc) { - continue; + continue; } unfilledBoxesOnThisLevel.clear(); BL_ASSERT(unfilledBoxesOnThisLevel.ixType() == boxType); @@ -1208,15 +1208,15 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, unfillableBoxesOnThisLevel.clear(); const Box ¤tPDomain = probDomain[currentLevel]; - int ufbLength(unfilledBoxesOnThisLevel.size()); + int ufbLength(unfilledBoxesOnThisLevel.size()); fillBoxId[ibox][currentLevel].resize(ufbLength); fillBoxIdBAs[ibox][currentLevel].resize(ufbLength); savedFineBox[ibox][currentLevel].resize(ufbLength); int currentBLI(0); for(BoxList::iterator bli = unfilledBoxesOnThisLevel.begin(); - bli != unfilledBoxesOnThisLevel.end(); ++bli) - { + bli != unfilledBoxesOnThisLevel.end(); ++bli) + { BL_ASSERT(bli->ok()); Box coarseDestBox(*bli); Box fineTruncDestBox(coarseDestBox & currentPDomain); @@ -1227,7 +1227,7 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, tempCoarseBox = fineTruncDestBox; } else { tempCoarseBox = fineTruncDestBox; - // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv tempCoarseBox.coarsen(cumulativeRefRatios[currentLevel]); } @@ -1238,14 +1238,14 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, fillBoxId[ibox][currentLevel][currentBLI].resize(1); fillBoxIdBAs[ibox][currentLevel][currentBLI].resize(1); - fillBoxId[ibox][currentLevel][currentBLI][0] = - multiFabCopyDesc.AddBox(stateDataMFId[currentLevel], - tempCoarseBox, &tempUnfillableBoxes, - srcComp, 0, 1); + fillBoxId[ibox][currentLevel][currentBLI][0] = + multiFabCopyDesc.AddBox(stateDataMFId[currentLevel], + tempCoarseBox, &tempUnfillableBoxes, + srcComp, 0, 1); fillBoxIdBAs[ibox][currentLevel][currentBLI][0] = BoxArray(amrex::complementIn(tempCoarseBox, - tempUnfillableBoxes)); + tempUnfillableBoxes)); unfillableBoxesOnThisLevel.catenate(tempUnfillableBoxes); ++currentBLI; @@ -1262,10 +1262,10 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, Box coarseLocalMFBox(localMFBoxes[ibox]); coarseLocalMFBox.coarsen(cumulativeRefRatios[currentLevel]); unfilledBoxesOnThisLevel.intersect(coarseLocalMFBox); - // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - if(currentLevel != 0) { + // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + if(currentLevel != 0) { unfilledBoxesOnThisLevel.coarsen(refRatio[currentLevel - 1]); - } + } if(currentLevel == 0) { BoxList unfilledInside = @@ -1284,7 +1284,7 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, for(int currentIndex = 0; currentIndex < destBoxes.size(); ++currentIndex) { if(destMultiFab.DistributionMap()[currentIndex] != myProc) { - continue; + continue; } for(int currLevel(0); currLevel <= finestFillLevel; ++currLevel) { for(int currentBox(0); @@ -1292,12 +1292,12 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, ++currentBox) { Box tempCoarseBox( - fillBoxId[currentIndex][currLevel][currentBox][0].box()); + fillBoxId[currentIndex][currLevel][currentBox][0].box()); FArrayBox tempCoarseDestFab(tempCoarseBox, 1); tempCoarseDestFab.setVal(1.e30); multiFabCopyDesc.FillFab(stateDataMFId[currLevel], - fillBoxId[currentIndex][currLevel][currentBox][0], - tempCoarseDestFab); + fillBoxId[currentIndex][currLevel][currentBox][0], + tempCoarseDestFab); Box intersectDestBox(savedFineBox[currentIndex][currLevel][currentBox]); intersectDestBox &= destMultiFab[currentIndex].box(); @@ -1315,11 +1315,11 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, // Interpolate up to fine patch. tempCurrentFillPatchedFab.resize(intersectDestBox, nFillComps); tempCurrentFillPatchedFab.setVal(1.e30); - BL_ASSERT(intersectDestBox.ok()); - BL_ASSERT(tempCoarseDestFab.box().ok()); - PcInterp(tempCurrentFillPatchedFab, - tempCoarseDestFab, intersectDestBox, - cumulativeRefRatios[currLevel]); + BL_ASSERT(intersectDestBox.ok()); + BL_ASSERT(tempCoarseDestFab.box().ok()); + PcInterp(tempCurrentFillPatchedFab, + tempCoarseDestFab, intersectDestBox, + cumulativeRefRatios[currLevel]); copyFromThisFab = &tempCurrentFillPatchedFab; copyFromTheseBoxes = &fboxes; } else { @@ -1349,7 +1349,7 @@ void AmrData::FillVar(MultiFab &destMultiFab, int finestFillLevel, // --------------------------------------------------------------- void AmrData::FillVar(Vector &destFabs, const Vector &destBoxes, - int finestFillLevel, const string &varname, int procWithFabs) + int finestFillLevel, const string &varname, int procWithFabs) { // @@ -1382,7 +1382,7 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe // ensure the required grids are in memory for(currentLevel = 0; currentLevel <= finestFillLevel; ++currentLevel) { for(int iBox = 0; iBox < destBoxes.size(); ++iBox) { - Box tempCoarseBox(destBoxes[iBox]); + Box tempCoarseBox(destBoxes[iBox]); if(currentLevel != finestFillLevel) { tempCoarseBox.coarsen(cumulativeRefRatios[currentLevel]); } @@ -1400,8 +1400,8 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe Vector localMFBoxes; // These are the ones we want to fillpatch. Vector< Vector< Vector< Vector > > > fillBoxId; Vector< Vector< Vector< Vector > > > fillBoxIdBAs; - // [grid][level][fillablesubbox][oldnew] - // oldnew not used here + // [grid][level][fillablesubbox][oldnew] + // oldnew not used here Vector< Vector< Vector > > savedFineBox; // [grid][level][fillablesubbox] if(myproc == procWithFabs) { localMFBoxes = destBoxes; @@ -1433,15 +1433,15 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe unfillableBoxesOnThisLevel.clear(); const Box ¤tPDomain = probDomain[currentLevel]; - int ufbLength = unfilledBoxesOnThisLevel.size(); + int ufbLength = unfilledBoxesOnThisLevel.size(); fillBoxId[ibox][currentLevel].resize(ufbLength); fillBoxIdBAs[ibox][currentLevel].resize(ufbLength); savedFineBox[ibox][currentLevel].resize(ufbLength); int currentBLI(0); for(BoxList::iterator bli = unfilledBoxesOnThisLevel.begin(); - bli != unfilledBoxesOnThisLevel.end(); ++bli) - { + bli != unfilledBoxesOnThisLevel.end(); ++bli) + { BL_ASSERT(bli->ok()); Box coarseDestBox(*bli); Box fineTruncDestBox(coarseDestBox & currentPDomain); @@ -1452,7 +1452,7 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe tempCoarseBox = fineTruncDestBox; } else { tempCoarseBox = fineTruncDestBox; - // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv tempCoarseBox.coarsen(cumulativeRefRatios[currentLevel]); } @@ -1463,14 +1463,14 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe fillBoxId[ibox][currentLevel][currentBLI].resize(1); fillBoxIdBAs[ibox][currentLevel][currentBLI].resize(1); - fillBoxId[ibox][currentLevel][currentBLI][0] = - multiFabCopyDesc.AddBox(stateDataMFId[currentLevel], - tempCoarseBox, &tempUnfillableBoxes, - srcComp, destComp, numFillComps); + fillBoxId[ibox][currentLevel][currentBLI][0] = + multiFabCopyDesc.AddBox(stateDataMFId[currentLevel], + tempCoarseBox, &tempUnfillableBoxes, + srcComp, destComp, numFillComps); fillBoxIdBAs[ibox][currentLevel][currentBLI][0] = BoxArray(amrex::complementIn(tempCoarseBox, - tempUnfillableBoxes)); + tempUnfillableBoxes)); unfillableBoxesOnThisLevel.catenate(tempUnfillableBoxes); ++currentBLI; @@ -1487,10 +1487,10 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe Box coarseLocalMFBox(localMFBoxes[ibox]); coarseLocalMFBox.coarsen(cumulativeRefRatios[currentLevel]); unfilledBoxesOnThisLevel.intersect(coarseLocalMFBox); - // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv - if(currentLevel != 0) { + // check this vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv + if(currentLevel != 0) { unfilledBoxesOnThisLevel.coarsen(refRatio[currentLevel - 1]); - } + } if(currentLevel == 0) { BoxList unfilledInside = @@ -1509,20 +1509,20 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe for(int currentIndex = 0; currentIndex < destBoxes.size(); ++currentIndex) { for(int currLevel = 0; currLevel <= finestFillLevel; ++currLevel) { - if(myproc != procWithFabs) { - break; - } + if(myproc != procWithFabs) { + break; + } for(int currentBox(0); currentBox < fillBoxId[currentIndex][currLevel].size(); ++currentBox) { Box tempCoarseBox( - fillBoxId[currentIndex][currLevel][currentBox][0].box()); + fillBoxId[currentIndex][currLevel][currentBox][0].box()); FArrayBox tempCoarseDestFab(tempCoarseBox, numFillComps); tempCoarseDestFab.setVal(1.e30); multiFabCopyDesc.FillFab(stateDataMFId[currLevel], - fillBoxId[currentIndex][currLevel][currentBox][0], - tempCoarseDestFab); + fillBoxId[currentIndex][currLevel][currentBox][0], + tempCoarseDestFab); Box intersectDestBox(savedFineBox[currentIndex][currLevel][currentBox]); intersectDestBox &= destFabs[currentIndex]->box(); @@ -1540,12 +1540,12 @@ void AmrData::FillVar(Vector &destFabs, const Vector &destBoxe // Interpolate up to fine patch. tempCurrentFillPatchedFab.resize(intersectDestBox, numFillComps); tempCurrentFillPatchedFab.setVal(1.e30); - BL_ASSERT(intersectDestBox.ok()); - BL_ASSERT( tempCoarseDestFab.box().ok()); - PcInterp(tempCurrentFillPatchedFab, - tempCoarseDestFab, - intersectDestBox, - cumulativeRefRatios[currLevel]); + BL_ASSERT(intersectDestBox.ok()); + BL_ASSERT( tempCoarseDestFab.box().ok()); + PcInterp(tempCurrentFillPatchedFab, + tempCoarseDestFab, + intersectDestBox, + cumulativeRefRatios[currLevel]); copyFromThisFab = &tempCurrentFillPatchedFab; copyFromTheseBoxes = &fboxes; } else { @@ -1678,7 +1678,7 @@ int AmrData::FinestIntersectingLevel(const Box &b, int startLevel) const { for(int iBox(0); iBox < visMFBA.size(); ++iBox) { if(visMFBA[iBox].intersects(levelBox)) { return level; - } + } } levelBox.coarsen(refRatio[level - 1]); @@ -1764,7 +1764,7 @@ void AmrData::FlushGrids(int componentIndex) { // --------------------------------------------------------------- bool AmrData::MinMax(const Box &onBox, const string &derived, int level, - Real &dataMin, Real &dataMax) + Real &dataMin, Real &dataMax) { BL_ASSERT(level >= 0 && level <= finestLevel); BL_ASSERT(onBox.ok()); @@ -1821,12 +1821,12 @@ bool AmrData::MinMax(const Box &onBox, const string &derived, int level, Real vfVisMFMin(visMF[level][vfWhichVisMF]->min(gdx, vfWhichVisMFComponent)); Real vfVisMFMax(visMF[level][vfWhichVisMF]->max(gdx, vfWhichVisMFComponent)); if(vfVisMFMin > (1.0 - vfEps[level])) { // no cg body in this grid - ++cCountAllFluid; + ++cCountAllFluid; dataMin = std::min(dataMin, visMFMin); dataMax = std::max(dataMax, visMFMax); valid = true; - } else if(vfVisMFMax >= vfEps[level] ) { - ++cCountMixed; + } else if(vfVisMFMax >= vfEps[level] ) { + ++cCountMixed; if(visMFMin < dataMin || visMFMax > dataMax) { // do it the hard way DefineFab(level, compIndex, gdx); DefineFab(level, vfIndex, gdx); @@ -1839,7 +1839,7 @@ bool AmrData::MinMax(const Box &onBox, const string &derived, int level, overlap &= gpli.validbox(); Real vfMaxVal = (*dataGrids[level][vfIndex])[gpli].max(overlap, 0); if(vfMaxVal >= vfEps[level]) { - ++cCountMixedFort; + ++cCountMixedFort; valid = true; FORT_CARTGRIDMINMAX(ddat, AMREX_ARLIM(dlo), AMREX_ARLIM(dhi), vdat, vfEps[level], @@ -1847,14 +1847,14 @@ bool AmrData::MinMax(const Box &onBox, const string &derived, int level, dataMin = std::min(dataMin, minVal); dataMax = std::max(dataMax, maxVal); } - } else { - ++cCountMixedSkipped; - } - } else { // all body - ++cCountAllBody; - } + } else { + ++cCountMixedSkipped; + } + } else { // all body + ++cCountAllBody; + } } else if(onBox.intersects(visMF[level][whichVisMF]->boxArray()[gdx])) { - ++iCount; + ++iCount; if(visMFMin < dataMin || visMFMax > dataMax) { // do it the hard way DefineFab(level, compIndex, gdx); DefineFab(level, vfIndex, gdx); @@ -1867,7 +1867,7 @@ bool AmrData::MinMax(const Box &onBox, const string &derived, int level, overlap &= gpli.validbox(); Real vfMaxVal = (*dataGrids[level][vfIndex])[gpli].max(overlap, 0); if(vfMaxVal >= vfEps[level]) { - ++iCountMixedFort; + ++iCountMixedFort; valid = true; FORT_CARTGRIDMINMAX(ddat, AMREX_ARLIM(dlo), AMREX_ARLIM(dhi), vdat, vfEps[level], @@ -1875,11 +1875,11 @@ bool AmrData::MinMax(const Box &onBox, const string &derived, int level, dataMin = std::min(dataMin, minVal); dataMax = std::max(dataMax, maxVal); } else { - ++iCountAllBody; - } - } else { - ++iCountMixedSkipped; - } + ++iCountAllBody; + } + } else { + ++iCountMixedSkipped; + } } else { ++outsideCount; } @@ -1913,18 +1913,18 @@ bool AmrData::MinMax(const Box &onBox, const string &derived, int level, int whichVisMF(compIndexToVisMFMap[compIndex]); int whichVisMFComponent(compIndexToVisMFComponentMap[compIndex]); Real visMFMin(visMF[level][whichVisMF]->min(gpli.index(), - whichVisMFComponent)); + whichVisMFComponent)); Real visMFMax(visMF[level][whichVisMF]->max(gpli.index(), - whichVisMFComponent)); + whichVisMFComponent)); if(onBox.contains(gpli.validbox())) { - dataMin = std::min(dataMin, visMFMin); - dataMax = std::max(dataMax, visMFMax); - valid = true; + dataMin = std::min(dataMin, visMFMin); + dataMax = std::max(dataMax, visMFMax); + valid = true; } else if(onBox.intersects(visMF[level][whichVisMF]-> - boxArray()[gpli.index()])) + boxArray()[gpli.index()])) { if(visMFMin < dataMin || visMFMax > dataMax) { // do it the hard way - DefineFab(level, compIndex, gpli.index()); + DefineFab(level, compIndex, gpli.index()); valid = true; overlap = onBox; overlap &= gpli.validbox(); diff --git a/Src/Extern/amrdata/AMReX_DataServices.H b/Src/Extern/amrdata/AMReX_DataServices.H index 180f6b0eb0e..bb2ff6673b1 100644 --- a/Src/Extern/amrdata/AMReX_DataServices.H +++ b/Src/Extern/amrdata/AMReX_DataServices.H @@ -118,10 +118,10 @@ class DataServices { //! PointValueRequest void PointValue(int pointBoxArraySize, Box *pointBoxArray, - const string ¤tDerived, - int coarsestLevelToSearch, int finestLevelToSearch, - int &intersectedLevel, Box &intersectedGrid, - Real &dataPointValue, bool &bPointIsValid); + const string ¤tDerived, + int coarsestLevelToSearch, int finestLevelToSearch, + int &intersectedLevel, Box &intersectedGrid, + Real &dataPointValue, bool &bPointIsValid); //! LineValuesRequest void LineValues(int lineBoxArraySize, Box *lineBoxArray, int whichDir, diff --git a/Src/Extern/amrdata/AMReX_DataServices.cpp b/Src/Extern/amrdata/AMReX_DataServices.cpp index a4d583f9d63..84f9d8ce557 100644 --- a/Src/Extern/amrdata/AMReX_DataServices.cpp +++ b/Src/Extern/amrdata/AMReX_DataServices.cpp @@ -255,8 +255,8 @@ void DataServices::Init(const string &filename, const Amrvis::FileType &filetype if(regOutputStats_H.TraceDataValid()) { // if(bIOP) { -// cout << "Calling InitRegionTimeRanges." << endl; -// } +// cout << "Calling InitRegionTimeRanges." << endl; +// } RegionsProfStats::OpenAllStreams(fileName); Box myBox(procBoxArray[myProc]); // bRegionDataAvailable = regOutputStats_H.InitRegionTimeRanges(myBox); @@ -264,8 +264,8 @@ void DataServices::Init(const string &filename, const Amrvis::FileType &filetype RegionsProfStats::CloseAllStreams(); regOutputStats_H.SetFNames(blProfStats_H.BLPFNames()); // if(bIOP) { -// cout << "Finished InitRegionTimeRanges." << endl; -// } +// cout << "Finished InitRegionTimeRanges." << endl; +// } } else { bTraceDataAvailable = false; } @@ -276,7 +276,7 @@ void DataServices::Init(const string &filename, const Amrvis::FileType &filetype if(bParseFilterFile) { if(bIOP) cout << "Parsing filter file." << endl; ParseFilterFile(); - } + } if( ! regOutputStats_H.TimeRangeInitialized()) { regOutputStats_H.InitFilterTimeRanges(); @@ -334,15 +334,15 @@ void DataServices::InitRegionTimeRanges() { if(bTraceDataAvailable) { if(regOutputStats_H.TraceDataValid()) { if(bIOP) { - cout << "Calling InitRegionTimeRanges." << endl; - } + cout << "Calling InitRegionTimeRanges." << endl; + } RegionsProfStats::OpenAllStreams(fileName); Box myBox(procBoxArray[myProc]); bRegionDataAvailable = regOutputStats_H.InitRegionTimeRanges(myBox); RegionsProfStats::CloseAllStreams(); if(bIOP) { - cout << "Finished InitRegionTimeRanges." << endl; - } + cout << "Finished InitRegionTimeRanges." << endl; + } } else { bTraceDataAvailable = false; } @@ -392,12 +392,12 @@ void DataServices::SetBatchMode() { // --------------------------------------------------------------- void DataServices::SetFabOutSize(int iSize) { if (profiler) // Unused with profiling data - { + { if(iSize == 1 || iSize == 8 || iSize == 32) { dsFabOutSize = iSize; } else { cerr << "Warning: DataServices::SetFabOutSize: size must be 1, 8 or 32 only." - << " Defaulting to native." << endl; + << " Defaulting to native." << endl; dsFabOutSize = 0; } } @@ -525,7 +525,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { bool bDeleteDS(false); BL_ASSERT(DataServices::dsArray[whichDSIndex]->numberOfUsers >= 0); if(ParallelDescriptor::IOProcessor()) { - bDeleteDS = (DataServices::dsArray[whichDSIndex]->numberOfUsers == 0); + bDeleteDS = (DataServices::dsArray[whichDSIndex]->numberOfUsers == 0); } { @@ -550,13 +550,13 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { int derivedLength, derivedLengthPadded; if(ParallelDescriptor::IOProcessor()) { - destFab = (FArrayBox *) va_arg(ap, void *); + destFab = (FArrayBox *) va_arg(ap, void *); const Box *boxRef = (const Box *) va_arg(ap, void *); - destBox = *boxRef; + destBox = *boxRef; fineFillLevel = va_arg(ap, int); const string *derivedRef = (const string *) va_arg(ap, void *); derivedTemp = *derivedRef; - derivedLength = derivedTemp.length(); + derivedLength = derivedTemp.length(); } ParallelDescriptor::Bcast(&destBox, 1, 0); @@ -604,13 +604,13 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { if(ParallelDescriptor::IOProcessor()) { const string *fabFileNameRef = (const string *) va_arg(ap, void *); - fabFileName = *fabFileNameRef; + fabFileName = *fabFileNameRef; const Box *boxRef = (const Box *) va_arg(ap, void *); - destBox = *boxRef; + destBox = *boxRef; fineFillLevel = va_arg(ap, int); const string *derivedRef = (const string *) va_arg(ap, void *); derivedTemp = *derivedRef; - derivedLength = derivedTemp.length(); + derivedLength = derivedTemp.length(); } ParallelDescriptor::Bcast(&destBox, 1, 0); @@ -643,9 +643,9 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { if(ParallelDescriptor::IOProcessor()) { const string *fabFileNameRef = (const string *) va_arg(ap, void *); - fabFileName = *fabFileNameRef; + fabFileName = *fabFileNameRef; const Box *boxRef = (const Box *) va_arg(ap, void *); - destBox = *boxRef; + destBox = *boxRef; fineFillLevel = va_arg(ap, int); } @@ -668,7 +668,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { slicenum = va_arg(ap, int); const string *derivedRef = (const string *) va_arg(ap, void *); derivedTemp = *derivedRef; - derivedLength = derivedTemp.length(); + derivedLength = derivedTemp.length(); } ParallelDescriptor::Bcast(&slicedir, 1, 0); @@ -715,10 +715,10 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { int derivedLength, derivedLengthPadded; if(ParallelDescriptor::IOProcessor()) { const Box *boxRef = (const Box *) va_arg(ap, void *); - box = *boxRef; + box = *boxRef; const string *derivedRef = (const string *) va_arg(ap, void *); - derivedTemp = *derivedRef; - derivedLength = derivedTemp.length(); + derivedTemp = *derivedRef; + derivedLength = derivedTemp.length(); } ParallelDescriptor::Bcast(&box, 1, 0); @@ -744,7 +744,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { Box box; if(ParallelDescriptor::IOProcessor()) { const Box *boxRef = (const Box *) va_arg(ap, void *); - box = *boxRef; + box = *boxRef; } ParallelDescriptor::Bcast(&box, 1, 0); @@ -767,9 +767,9 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { const Box *boxRef = (const Box *) va_arg(ap, void *); const string *derivedRef = (const string *) va_arg(ap, void *); level = va_arg(ap, int); - box = *boxRef; - derivedTemp = *derivedRef; - derivedLength = derivedTemp.length(); + box = *boxRef; + derivedTemp = *derivedRef; + derivedLength = derivedTemp.length(); } ParallelDescriptor::Bcast(&box, 1, 0); @@ -794,9 +794,9 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { Real *dataMinRef = va_arg(ap, Real *); Real *dataMaxRef = va_arg(ap, Real *); bool *minMaxValidRef = va_arg(ap, bool *); - *dataMinRef = dataMin; - *dataMaxRef = dataMax; - *minMaxValidRef = minMaxValid; + *dataMinRef = dataMin; + *dataMaxRef = dataMax; + *minMaxValidRef = minMaxValid; } } break; @@ -828,7 +828,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { pointBoxArrayTempPtr = (Box *) va_arg(ap, void *); const string *derivedRef = (const string *) va_arg(ap, void *); derivedTemp = *derivedRef; - derivedLength = derivedTemp.length(); + derivedLength = derivedTemp.length(); coarsestLevelToSearch = va_arg(ap, int); finestLevelToSearch = va_arg(ap, int); } @@ -845,9 +845,9 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { derivedCharPtr = new char[derivedLengthPadded]; if(ParallelDescriptor::IOProcessor()) { strcpy(derivedCharPtr, derivedTemp.c_str()); - for(int iBox = 0; iBox < pointBoxArraySize; ++iBox) { - pointBoxArrayPtr[iBox] = pointBoxArrayTempPtr[iBox]; - } + for(int iBox = 0; iBox < pointBoxArraySize; ++iBox) { + pointBoxArrayPtr[iBox] = pointBoxArrayTempPtr[iBox]; + } } ParallelDescriptor::Bcast(derivedCharPtr, derivedLengthPadded, 0); ParallelDescriptor::Bcast(pointBoxArrayPtr, pointBoxArraySize, 0); @@ -862,13 +862,13 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { bool bPointIsValid; ds->PointValue(pointBoxArraySize, pointBoxArrayPtr, - derived, - coarsestLevelToSearch, - finestLevelToSearch, - intersectedLevel, - intersectedBox, - dataPointValue, - bPointIsValid); + derived, + coarsestLevelToSearch, + finestLevelToSearch, + intersectedLevel, + intersectedBox, + dataPointValue, + bPointIsValid); // set the return values if(ParallelDescriptor::IOProcessor()) { @@ -876,10 +876,10 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { Box *intersectedBoxRef = (Box *) va_arg(ap, void *); Real *dataPointValueRef = va_arg(ap, Real *); bool *bPointIsValidRef = va_arg(ap, bool *); - *intersectedLevelRef = intersectedLevel; - *intersectedBoxRef = intersectedBox; - *dataPointValueRef = dataPointValue; - *bPointIsValidRef = bPointIsValid; + *intersectedLevelRef = intersectedLevel; + *intersectedBoxRef = intersectedBox; + *dataPointValueRef = dataPointValue; + *bPointIsValidRef = bPointIsValid; } // dont need to broadcast the return values--only the IOProcessor uses them @@ -962,7 +962,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { delete [] lineBoxArrayPtr; - } + } break; case InvalidRequestType: @@ -1043,7 +1043,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { int maxSmallImageLength, refRatioAll; bool *proxMapPtr; - if (ParallelDescriptor::IOProcessor()) + if (ParallelDescriptor::IOProcessor()) { plotfileNamePtr = (std::string *) va_arg(ap, void *); maxSmallImageLength = va_arg(ap, int); @@ -1055,11 +1055,11 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { amrex::BroadcastBool(*proxMapPtr, ParallelDescriptor::MyProc(), ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); // --- Broadcast Ints - ParallelDescriptor::Bcast(&maxSmallImageLength, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); - ParallelDescriptor::Bcast(&refRatioAll, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); - + ParallelDescriptor::Bcast(&maxSmallImageLength, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + ParallelDescriptor::Bcast(&refRatioAll, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + // --- Broadcast String - amrex::BroadcastString(*plotfileNamePtr, ParallelDescriptor::MyProc(), ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + amrex::BroadcastString(*plotfileNamePtr, ParallelDescriptor::MyProc(), ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); ds->RunSendsPF(*plotfileNamePtr, maxSmallImageLength, *proxMapPtr, refRatioAll); } @@ -1086,7 +1086,7 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { // Get passed data mpiFuncNames = *((std::map *) va_arg(ap, void *)); plotfileName = *((std::string *) va_arg(ap, void *)); - subTimeRange = *((BLProfStats::TimeRange *) va_arg(ap, void *)); + subTimeRange = *((BLProfStats::TimeRange *) va_arg(ap, void *)); maxSmallImageLength = va_arg(ap, int); refRatioAll = va_arg(ap, int); nTimeSlots = va_arg(ap, int); @@ -1110,13 +1110,13 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { amrex::BroadcastBool(statsCollected, ParallelDescriptor::MyProc(), ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); // --- Broadcast Ints - ParallelDescriptor::Bcast(&maxSmallImageLength, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); - ParallelDescriptor::Bcast(&refRatioAll, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); - ParallelDescriptor::Bcast(&nTimeSlots, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + ParallelDescriptor::Bcast(&maxSmallImageLength, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + ParallelDescriptor::Bcast(&refRatioAll, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + ParallelDescriptor::Bcast(&nTimeSlots, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); // --- Broadcast Reals - ParallelDescriptor::Bcast(&start, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); - ParallelDescriptor::Bcast(&stop, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + ParallelDescriptor::Bcast(&start, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); + ParallelDescriptor::Bcast(&stop, 1, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); // --- Broadcast Map as 2 Arrays amrex::BroadcastArray(mapFirst, ParallelDescriptor::MyProc(), ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::Communicator()); @@ -1130,9 +1130,9 @@ void DataServices::Dispatch(DSRequestType requestType, DataServices *ds, ...) { plotfileName = fileNameAndmapSecond.front(); for(int i(0); i(mapFirst[i+1], fileNameAndmapSecond[i])); + mpiFuncNames.insert(std::pair(mapFirst[i+1], fileNameAndmapSecond[i])); } - } + } ds->RunTimelinePF(mpiFuncNames, plotfileName, subTimeRange, maxSmallImageLength, refRatioAll, nTimeSlots, statsCollected); @@ -1229,9 +1229,9 @@ bool DataServices::DumpSlice(int slicedir, int slicenum, cout << endl; if( ! amrData.ProbDomain()[iWTL].contains(sliceBox)) { cerr << "Error: sliceBox = " << sliceBox << " slicedir " << slicenum - << " on Level " << iWTL - << " not in probDomain: " << amrData.ProbDomain()[iWTL] - << endl; + << " on Level " << iWTL + << " not in probDomain: " << amrData.ProbDomain()[iWTL] + << endl; return false; } bool bWF = WriteFab(sliceFile, sliceBox, iWTL, varname); @@ -1287,9 +1287,9 @@ bool DataServices::DumpSlice(int slicedir, int slicenum) { // dump all vars cout << endl; if( ! amrData.ProbDomain()[iWTL].contains(sliceBox)) { cerr << "Error: sliceBox = " << sliceBox << " slicedir " << slicenum - << " on Level " << iWTL - << " not in probDomain: " << amrData.ProbDomain()[iWTL] - << endl; + << " on Level " << iWTL + << " not in probDomain: " << amrData.ProbDomain()[iWTL] + << endl; return false; } bool bWF = WriteFab(sliceFile, sliceBox, iWTL); @@ -1330,7 +1330,7 @@ bool DataServices::DumpSlice(const Box &b, const string &varname) { b.bigEnd(Amrvis::XDIR), b.bigEnd(Amrvis::YDIR), b.bigEnd(Amrvis::ZDIR), iWTL); #endif if (count >= N) { - amrex::Abort("DataServices::DumpSlice(3): slicechar buffer too small"); + amrex::Abort("DataServices::DumpSlice(3): slicechar buffer too small"); } sliceFile += slicechar; sliceFile += ".fab"; @@ -1340,9 +1340,9 @@ bool DataServices::DumpSlice(const Box &b, const string &varname) { if( ! amrData.ProbDomain()[iWTL].contains(b)) { cerr << "Slice box not in probDomain: " - << amrData.ProbDomain()[iWTL] - << " on Level " << iWTL - << endl; + << amrData.ProbDomain()[iWTL] + << " on Level " << iWTL + << endl; return false; } bool bWF = WriteFab(sliceFile, b, iWTL, varname); @@ -1381,7 +1381,7 @@ bool DataServices::DumpSlice(const Box &b) { // dump all vars b.bigEnd(Amrvis::XDIR), b.bigEnd(Amrvis::YDIR), b.bigEnd(Amrvis::ZDIR), iWTL); #endif if (count >= N) - amrex::Abort("DataServices::DumpSlice(4): slicechar buffer too small"); + amrex::Abort("DataServices::DumpSlice(4): slicechar buffer too small"); sliceFile += slicechar; sliceFile += ".fab"; cout << "sliceFile = " << sliceFile << endl; @@ -1390,9 +1390,9 @@ bool DataServices::DumpSlice(const Box &b) { // dump all vars if( ! amrData.ProbDomain()[iWTL].contains(b)) { cerr << "Slice box not in probDomain: " - << amrData.ProbDomain()[iWTL] - << " on Level " << iWTL - << endl; + << amrData.ProbDomain()[iWTL] + << " on Level " << iWTL + << endl; return false; } bool bWF = WriteFab(sliceFile, b, iWTL); @@ -1403,8 +1403,8 @@ bool DataServices::DumpSlice(const Box &b) { // dump all vars // --------------------------------------------------------------- bool DataServices::FillVar(FArrayBox *destFab, const Box &destBox, - int finestFillLevel, const string &varname, - int procWithFab) + int finestFillLevel, const string &varname, + int procWithFab) { if( ! bAmrDataOk) { return false; @@ -1417,7 +1417,7 @@ bool DataServices::FillVar(FArrayBox *destFab, const Box &destBox, // --------------------------------------------------------------- bool DataServices::FillVar(MultiFab &destMultiFab, int finestFillLevel, - const string &varname) + const string &varname) { if( ! bAmrDataOk) { return false; @@ -1437,7 +1437,7 @@ bool DataServices::FillVar(MultiFab &destMultiFab, int finestFillLevel, // // bool DataServices::WriteFab(const string &fname, const Box ®ion, int lev, - const string &varname) + const string &varname) { if( ! bAmrDataOk) { return false; @@ -1452,7 +1452,7 @@ bool DataServices::WriteFab(const string &fname, const Box ®ion, int lev, destFabs[0] = &data; destBoxes[0] = region; amrData.FillVar(destFabs, destBoxes, lev, varname, - ParallelDescriptor::IOProcessorNumber()); + ParallelDescriptor::IOProcessorNumber()); bool bWF(true); if(ParallelDescriptor::IOProcessor()) { @@ -1511,7 +1511,7 @@ bool DataServices::WriteFab(const string &fname, const Box ®ion, int lev) { destFabs[0] = &tempdata; destBoxes[0] = region; amrData.FillVar(destFabs, destBoxes, lev, amrData.PlotVarNames()[ivar], - ParallelDescriptor::IOProcessorNumber()); + ParallelDescriptor::IOProcessorNumber()); int srccomp(0); int destcomp(ivar); int ncomp(1); @@ -1587,13 +1587,13 @@ int DataServices::NumDeriveFunc() const { // --------------------------------------------------------------- void DataServices::PointValue(int /*pointBoxArraySize*/, Box *pointBoxArray, - const string ¤tDerived, - int coarsestLevelToSearch, - int finestLevelToSearch, - int &intersectedLevel, - Box &intersectedGrid, - Real &dataPointValue, - bool &bPointIsValid) + const string ¤tDerived, + int coarsestLevelToSearch, + int finestLevelToSearch, + int &intersectedLevel, + Box &intersectedGrid, + Real &dataPointValue, + bool &bPointIsValid) { bPointIsValid = false; if( ! bAmrDataOk) { @@ -1601,8 +1601,8 @@ void DataServices::PointValue(int /*pointBoxArraySize*/, Box *pointBoxArray, } intersectedLevel = - amrData.FinestContainingLevel(pointBoxArray[finestLevelToSearch], - finestLevelToSearch); + amrData.FinestContainingLevel(pointBoxArray[finestLevelToSearch], + finestLevelToSearch); if(intersectedLevel < coarsestLevelToSearch) { return; @@ -1627,7 +1627,7 @@ void DataServices::PointValue(int /*pointBoxArraySize*/, Box *pointBoxArray, destFab = new FArrayBox(destBox, 1); } amrData.FillVar(destFab, destBox, intersectedLevel, currentDerived, - ParallelDescriptor::IOProcessorNumber()); + ParallelDescriptor::IOProcessorNumber()); if(ParallelDescriptor::IOProcessor()) { dataPointValue = (destFab->dataPtr())[0]; @@ -1674,7 +1674,7 @@ void DataServices::LineValues(int /*lineBoxArraySize*/, Box *lineBoxArray, int w // --------------------------------------------------------------- bool DataServices::MinMax(const Box &onBox, const string &derived, int level, - Real &dataMin, Real &dataMax, bool &minMaxValid) + Real &dataMin, Real &dataMax, bool &minMaxValid) { minMaxValid = amrData.MinMax(onBox, derived, level, dataMin, dataMax); return minMaxValid; @@ -1709,7 +1709,7 @@ void DataServices::ParseFilterFile() // ---------------------------------------------------------------------- void DataServices::WriteSummary(std::ostream &os, bool bWriteAverage, int whichProc, bool bUseTrace, - bool graphTopPct) + bool graphTopPct) { bool bIOP(ParallelDescriptor::IOProcessor()); if(bUseTrace) { @@ -1779,7 +1779,7 @@ void DataServices::CheckProfData() if((yyin = fopen(regFileName_H.c_str(), "r"))) { yyparse(®ionsOutputStats); fclose(yyin); - } + } else { cerr << "DataServices::CheckProfData: Cannot open file " << regPrefix_H << endl; } @@ -1803,7 +1803,7 @@ void DataServices::CheckProfData() cout << "Number of regions = " << regionsOutputStats.RegionNumbers().size() << endl; regionsOutputStats.CheckRegionsData(); } - + ParallelDescriptor::Barrier(); if(bIOP) { cout << "---------------- finished checking profiling data." << endl << endl; } if(bIOP) { cout << "Check Time = " << ParallelDescriptor::second() - dstart << " s." << endl; } @@ -2334,7 +2334,7 @@ BLProfStats::TimeRange DataServices::FindCalcTimeRange() bool bIOP(ParallelDescriptor::IOProcessor()); int myProc(ParallelDescriptor::MyProc()); int nProcs(ParallelDescriptor::NProcs()); - + const Vector &commHeaderFileNames = CommProfStats::GetHeaderFileNames(); BLProfStats::TimeRange calcTimeRange(std::numeric_limits::max(), -std::numeric_limits::max()); @@ -2373,7 +2373,7 @@ void DataServices::RunTimelinePF(std::map &mpiFuncNames, BLProfStats::TimeRange &subTimeRange, int maxSmallImageLength, int refRatioAll, int nTimeSlots, - bool &statsCollected) + bool &statsCollected) { BL_PROFILE("DataServices::RunTimelinePF()"); @@ -2481,7 +2481,7 @@ void DataServices::RunTimelinePF(std::map &mpiFuncNames, yyparse(&commOutputStats); fclose(yyin); - + if(hfnI_I == 0) { // this assumes all headers have the same nametag and barrier names // ---- this part encodes the name tag name into the NameTag cfType value nameTagNames = commOutputStats.NameTagNames(); @@ -2722,7 +2722,7 @@ void DataServices::MakeRegionPlt(std::string &plotfileName) // ---------------------------------------------------------------------- void DataServices::RunACTPF(std::string &plotfileName, int maxSmallImageLength, int refRatioAll, - const Vector &actFNames) + const Vector &actFNames) { #if (BL_SPACEDIM != 2) cout << "**** Error: DataServices::RunACTPF is only supported for 2D" << endl; @@ -2735,7 +2735,7 @@ void DataServices::RunACTPF(std::string &plotfileName, if(BL_SPACEDIM != 2) { if(bIOP) { cout << "DataServices::RunACTPF: bRunACTPF only supported for 2D." - << endl; + << endl; } return; } @@ -2824,7 +2824,7 @@ void DataServices::RunACTPF(std::string &plotfileName, if( ! bSameNCalls || ncMin != ncMax) { if(bIOP) { cout << "**** bSameNCalls == false for: " << whichFuncName - << " ::: unsupported." << endl; + << " ::: unsupported." << endl; SHOWVAL(ncMin); SHOWVAL(ncMax); SHOWVAL(whichFuncNCalls); diff --git a/Src/Extern/amrdata/AMReX_FABUTIL_1D.F b/Src/Extern/amrdata/AMReX_FABUTIL_1D.F index 97c7bea91b4..5692a76831b 100644 --- a/Src/Extern/amrdata/AMReX_FABUTIL_1D.F +++ b/Src/Extern/amrdata/AMReX_FABUTIL_1D.F @@ -1,4 +1,4 @@ -c ::: SCCS stuff "@(#)FABUTIL_2D.F 3.1\t6/25/93" +c ::: SCCS stuff "@(#)FABUTIL_2D.F 3.1\t6/25/93" #define FORT_CINTERP cinterp1d #define FORT_PCINTERP pcinterp1d @@ -26,28 +26,28 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, REAL_T fdat(flen) REAL_T voff(flen) -c ::: NOTE: data must be sent in so that +c ::: NOTE: data must be sent in so that c ::: cslope(1,*) and crse(1,*) are associated with c ::: the same cell c ::: :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: c ::: cinterp: conservative interpolation from coarse grid to c ::: subregion of fine grid defined by (fblo,fbhi) -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fine grid array c ::: flo,fhi => (const) index limits of fine grid c ::: fblo,fbhi => (const) subregion of fine grid to get values c ::: nvar => (const) number of variables in state vector c ::: lratio => (const) refinement ratio between levels -c ::: +c ::: c ::: crse => (const) coarse grid data widended by 1 zone c ::: and unrolled c ::: clo,chi => (const) one dimensional limits of crse grid c ::: cslo,cshi => (const) coarse grid index limits where c ::: slopes are to be defined. This is c ::: the projection of (fblo,fbhi) down -c ::: to the coarse level +c ::: to the coarse level c ::: fslo,fshi => (const) fine grid index limits where c ::: slopes are needed. This is the c ::: refinement of (cslo,cshi) and @@ -91,22 +91,22 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, leny = fbhi(2)-fblo(2)+1 maxlen = max(lenx,leny) if (maxlen .eq. lenx) then - do 100 i = fblo(1), fbhi(1) + do 100 i = fblo(1), fbhi(1) fn = i-fslo(1)+1 ioff = mod(fn-1,lratio) voff(fn) = float(ioff)-hafrat 100 continue else - do 110 j = fblo(2), fbhi(2) + do 110 j = fblo(2), fbhi(2) fn = j-fslo(2)+1 joff = mod(fn-1,lratio) voff(fn) = float(joff)-hafrat 110 continue end if - do 120 n = 1, nvar + do 120 n = 1, nvar c ::: ::::: compute slopes in x direction - do 130 i = 1, clen + do 130 i = 1, clen cen = half*(crse(i+ist,n)-crse(i-ist,n)) forw = crse(i+ist,n)-crse(i,n) back = crse(i,n)-crse(i-ist,n) @@ -114,7 +114,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, cslope(i,1)=merge(slp,zero,forw*back>=0.0d0) 130 continue c ::: ::::: compute slopes in y direction - do 140 i = 1, clen + do 140 i = 1, clen cen = half*(crse(i+jst,n)-crse(i-jst,n)) forw = crse(i+jst,n)-crse(i,n) back = crse(i,n)-crse(i-jst,n) @@ -122,14 +122,14 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, cslope(i,2)=merge(slp,zero,forw*back>=0.0d0) 140 continue if (maxlen .eq. lenx) then - do 150 jc = cblo(2), cbhi(2) + do 150 jc = cblo(2), cbhi(2) c ::: ..,.......::::: strip out a fine grid slope vector - do 160 ioff = 1, lratio + do 160 ioff = 1, lratio icc = clo + ist + jst*(jc-jslo) istart = ioff iend = ioff + (ncbx-1)*lratio - do 170 fn = istart, iend, lratio + do 170 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fdat(fn) = crse(icc,n) @@ -137,7 +137,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, 170 continue 160 continue - do 180 joff = 0, lratio-1 + do 180 joff = 0, lratio-1 j = lratio*jc + joff if (j .lt. fblo(2)) then goto 180 @@ -149,7 +149,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, end if yoff = float(joff)-hafrat - do 190 i = fblo(1), fbhi(1) + do 190 i = fblo(1), fbhi(1) fn = i-fslo(1)+1 fine(i,j,n) = fdat(fn) + volratio* (voff(fn)* $ fslope(fn,1)+yoff*fslope(fn,2)) @@ -158,14 +158,14 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, 181 continue 150 continue else - do 200 ic = cblo(1), cbhi(1) + do 200 ic = cblo(1), cbhi(1) c ::: ..,.......::::: strip out a fine grid slope vector - do 210 joff = 1, lratio + do 210 joff = 1, lratio icc = clo + ist*(ic-islo) + jst istart = joff iend = joff + (ncby-1)*lratio - do 220 fn = istart, iend, lratio + do 220 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fdat(fn) = crse(icc,n) @@ -173,7 +173,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, 220 continue 210 continue - do 230 ioff = 0, lratio-1 + do 230 ioff = 0, lratio-1 i = lratio*ic + ioff if (i .lt. fblo(1)) then goto 230 @@ -185,7 +185,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, end if xoff = float(ioff)-hafrat - do 240 j = fblo(2), fbhi(2) + do 240 j = fblo(2), fbhi(2) fn = j-fslo(2)+1 fine(i,j,n) = fdat(fn) + volratio* (xoff* $ fslope(fn,1)+voff(fn)*fslope(fn,2)) @@ -219,7 +219,7 @@ subroutine FORT_PCINTERP (fine,floi1,fhii1,fblo, fbhi,lrat, c ::: pcinterp: use piecewise constant interpolation to define c ::: values on the subregion of the fine FAB defined c ::: by (fblo,fbhi). -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fab to get interpolated values c ::: flo,fhi => (const) index limits of fine @@ -235,14 +235,14 @@ subroutine FORT_PCINTERP (fine,floi1,fhii1,fblo, fbhi,lrat, integer ixproj,ii,ll ixproj(ii,ll) = (ii + ll*iabs(ii))/ll - iabs(ii) - do 130 n = 1, nvar - do 140 ioff = 0, lrat-1 - do 150 ic = cblo(1),cbhi(1) + do 130 n = 1, nvar + do 140 ioff = 0, lrat-1 + do 150 ic = cblo(1),cbhi(1) i = lrat*ic + ioff temp(i) = crse(ic,n) 150 continue 140 continue - do 160 i = fblo(1), fbhi(1) + do 160 i = fblo(1), fbhi(1) fine(i,n) = temp(i) 160 continue 130 continue diff --git a/Src/Extern/amrdata/AMReX_FABUTIL_2D.F b/Src/Extern/amrdata/AMReX_FABUTIL_2D.F index 446ad02947c..31391d8733d 100644 --- a/Src/Extern/amrdata/AMReX_FABUTIL_2D.F +++ b/Src/Extern/amrdata/AMReX_FABUTIL_2D.F @@ -1,4 +1,4 @@ -c ::: SCCS stuff "@(#)FABUTIL_2D.F 3.1\t6/25/93" +c ::: SCCS stuff "@(#)FABUTIL_2D.F 3.1\t6/25/93" #define FORT_CINTERP cinterp2d #define FORT_PCINTERP pcinterp2d @@ -26,28 +26,28 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, REAL_T fdat(flen) REAL_T voff(flen) -c ::: NOTE: data must be sent in so that +c ::: NOTE: data must be sent in so that c ::: cslope(1,*) and crse(1,*) are associated with c ::: the same cell c ::: :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: c ::: cinterp: conservative interpolation from coarse grid to c ::: subregion of fine grid defined by (fblo,fbhi) -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fine grid array c ::: flo,fhi => (const) index limits of fine grid c ::: fblo,fbhi => (const) subregion of fine grid to get values c ::: nvar => (const) number of variables in state vector c ::: lratio => (const) refinement ratio between levels -c ::: +c ::: c ::: crse => (const) coarse grid data widended by 1 zone c ::: and unrolled c ::: clo,chi => (const) one dimensional limits of crse grid c ::: cslo,cshi => (const) coarse grid index limits where c ::: slopes are to be defined. This is c ::: the projection of (fblo,fbhi) down -c ::: to the coarse level +c ::: to the coarse level c ::: fslo,fshi => (const) fine grid index limits where c ::: slopes are needed. This is the c ::: refinement of (cslo,cshi) and @@ -89,22 +89,22 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, leny = fbhi(2)-fblo(2)+1 maxlen = max(lenx,leny) if (maxlen .eq. lenx) then - do 100 i = fblo(1), fbhi(1) + do 100 i = fblo(1), fbhi(1) fn = i-fslo(1)+1 ioff = mod(fn-1,lratio) voff(fn) = float(ioff)-hafrat 100 continue else - do 110 j = fblo(2), fbhi(2) + do 110 j = fblo(2), fbhi(2) fn = j-fslo(2)+1 joff = mod(fn-1,lratio) voff(fn) = float(joff)-hafrat 110 continue end if - do 120 n = 1, nvar + do 120 n = 1, nvar c ::: ::::: compute slopes in x direction - do 130 i = 1, clen + do 130 i = 1, clen cen = half*(crse(i+ist,n)-crse(i-ist,n)) forw = crse(i+ist,n)-crse(i,n) back = crse(i,n)-crse(i-ist,n) @@ -112,7 +112,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, cslope(i,1)=merge(slp,zero,forw*back>=0.0d0) 130 continue c ::: ::::: compute slopes in y direction - do 140 i = 1, clen + do 140 i = 1, clen cen = half*(crse(i+jst,n)-crse(i-jst,n)) forw = crse(i+jst,n)-crse(i,n) back = crse(i,n)-crse(i-jst,n) @@ -120,14 +120,14 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, cslope(i,2)=merge(slp,zero,forw*back>=0.0d0) 140 continue if (maxlen .eq. lenx) then - do 150 jc = cblo(2), cbhi(2) + do 150 jc = cblo(2), cbhi(2) c ::: ..,.......::::: strip out a fine grid slope vector - do 160 ioff = 1, lratio + do 160 ioff = 1, lratio icc = clo + ist + jst*(jc-jslo) istart = ioff iend = ioff + (ncbx-1)*lratio - do 170 fn = istart, iend, lratio + do 170 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fdat(fn) = crse(icc,n) @@ -135,7 +135,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, 170 continue 160 continue - do 180 joff = 0, lratio-1 + do 180 joff = 0, lratio-1 j = lratio*jc + joff if (j .lt. fblo(2)) then goto 180 @@ -147,7 +147,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, end if yoff = float(joff)-hafrat - do 190 i = fblo(1), fbhi(1) + do 190 i = fblo(1), fbhi(1) fn = i-fslo(1)+1 fine(i,j,n) = fdat(fn) + volratio* (voff(fn)* $ fslope(fn,1)+yoff*fslope(fn,2)) @@ -156,14 +156,14 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, 181 continue 150 continue else - do 200 ic = cblo(1), cbhi(1) + do 200 ic = cblo(1), cbhi(1) c ::: ..,.......::::: strip out a fine grid slope vector - do 210 joff = 1, lratio + do 210 joff = 1, lratio icc = clo + ist*(ic-islo) + jst istart = joff iend = joff + (ncby-1)*lratio - do 220 fn = istart, iend, lratio + do 220 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fdat(fn) = crse(icc,n) @@ -171,7 +171,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, 220 continue 210 continue - do 230 ioff = 0, lratio-1 + do 230 ioff = 0, lratio-1 i = lratio*ic + ioff if (i .lt. fblo(1)) then goto 230 @@ -183,7 +183,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,fhii1,fhii2, fblo,fbhi, end if xoff = float(ioff)-hafrat - do 240 j = fblo(2), fbhi(2) + do 240 j = fblo(2), fbhi(2) fn = j-fslo(2)+1 fine(i,j,n) = fdat(fn) + volratio* (xoff* $ fslope(fn,1)+voff(fn)*fslope(fn,2)) @@ -218,7 +218,7 @@ subroutine FORT_PCINTERP (fine,floi1,floi2,fhii1,fhii2,fblo, c ::: pcinterp: use piecewise constant interpolation to define c ::: values on the subregion of the fine FAB defined c ::: by (fblo,fbhi). -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fab to get interpolated values c ::: flo,fhi => (const) index limits of fine @@ -234,16 +234,16 @@ subroutine FORT_PCINTERP (fine,floi1,floi2,fhii1,fhii2,fblo, integer ixproj,ii,ll ixproj(ii,ll) = (ii + ll*iabs(ii))/ll - iabs(ii) - do 120 j = fblo(2), fbhi(2) + do 120 j = fblo(2), fbhi(2) jc = ixproj(j,lrat) - do 130 n = 1, nvar - do 140 ioff = 0, lrat-1 - do 150 ic = cblo(1),cbhi(1) + do 130 n = 1, nvar + do 140 ioff = 0, lrat-1 + do 150 ic = cblo(1),cbhi(1) i = lrat*ic + ioff temp(i) = crse(ic,jc,n) 150 continue 140 continue - do 160 i = fblo(1), fbhi(1) + do 160 i = fblo(1), fbhi(1) fine(i,j,n) = temp(i) 160 continue 130 continue diff --git a/Src/Extern/amrdata/AMReX_FABUTIL_3D.F b/Src/Extern/amrdata/AMReX_FABUTIL_3D.F index d8e4ba66f82..cdc8cce9ff6 100644 --- a/Src/Extern/amrdata/AMReX_FABUTIL_3D.F +++ b/Src/Extern/amrdata/AMReX_FABUTIL_3D.F @@ -1,4 +1,4 @@ -c ::: SCCS stuff "@(#)FABUTIL_3D.F 3.1\t6/25/93" +c ::: SCCS stuff "@(#)FABUTIL_3D.F 3.1\t6/25/93" #define FORT_CINTERP cinterp3d #define FORT_PCINTERP pcinterp3d @@ -60,28 +60,28 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, REAL_T fdat(flen) REAL_T voff(flen) -c ::: NOTE: data must be sent in so that +c ::: NOTE: data must be sent in so that c ::: cslope(1,*) and crse(1,*) are associated with c ::: the same cell c ::: :::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: c ::: cinterp: conservative interpolation from coarse grid to c ::: subregion of fine grid defined by (fblo,fbhi) -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fine grid array c ::: flo,fhi => (const) index limits of fine grid c ::: fblo,fbhi => (const) subregion of fine grid to get values c ::: nvar => (const) number of variables in state vector c ::: lratio => (const) refinement ratio between levels -c ::: +c ::: c ::: crse => (const) coarse grid data widended by 1 zone c ::: and unrolled c ::: clo,chi => (const) one dimensional limits of crse grid c ::: cslo,cshi => (const) coarse grid index limits where c ::: slopes are to be defined. This is c ::: the projection of (fblo,fbhi) down -c ::: to the coarse level +c ::: to the coarse level c ::: fslo,fshi => (const) fine grid index limits where c ::: slopes are needed. This is the c ::: refinement of (cslo,cshi) and @@ -129,28 +129,28 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, lenz = fbhi(3)-fblo(3)+1 maxlen = max(lenx,leny,lenz) if (maxlen .eq. lenx) then - do 100 i = fblo(1), fbhi(1) + do 100 i = fblo(1), fbhi(1) fn = i-fslo(1)+1 ioff = mod(fn-1,lratio) voff(fn) = float(ioff)-hafrat 100 continue else if (maxlen .eq. leny) then - do 110 j = fblo(2), fbhi(2) + do 110 j = fblo(2), fbhi(2) fn = j-fslo(2)+1 joff = mod(fn-1,lratio) voff(fn) = float(joff)-hafrat 110 continue else - do 120 k = fblo(3), fbhi(3) + do 120 k = fblo(3), fbhi(3) fn = k-fslo(3)+1 koff = mod(fn-1,lratio) voff(fn) = float(koff)-hafrat 120 continue end if - do 130 n = 1, nvar + do 130 n = 1, nvar c ::: ::::: compute slopes in x direction - do 140 i = 1, clen + do 140 i = 1, clen cen = half*(crse(i+ist,n)-crse(i-ist,n)) forw = crse(i+ist,n)-crse(i,n) back = crse(i,n)-crse(i-ist,n) @@ -158,7 +158,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, cslope(i,1)=merge(slp,zero,forw*back>=0.0d0) 140 continue c ::: ::::: compute slopes in y direction - do 150 i = 1, clen + do 150 i = 1, clen cen = half*(crse(i+jst,n)-crse(i-jst,n)) forw = crse(i+jst,n)-crse(i,n) back = crse(i,n)-crse(i-jst,n) @@ -166,7 +166,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, cslope(i,2)=merge(slp,zero,forw*back>=0.0d0) 150 continue c ::: ::::: compute slopes in z direction - do 160 i = 1, clen + do 160 i = 1, clen cen = half*(crse(i+kst,n)-crse(i-kst,n)) forw = crse(i+kst,n)-crse(i,n) back = crse(i,n)-crse(i-kst,n) @@ -175,16 +175,16 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, 160 continue if (maxlen .eq. lenx) then - do 170 kc = cblo(3), cbhi(3) - do 180 jc = cblo(2), cbhi(2) + do 170 kc = cblo(3), cbhi(3) + do 180 jc = cblo(2), cbhi(2) c ::: ..,.......::::: strip out a fine grid slope vector - do 190 ioff = 1, lratio + do 190 ioff = 1, lratio icc = clo + ist + jst*(jc-jslo) + kst*(kc- $ kslo) istart = ioff iend = ioff + (ncbx-1)*lratio - do 200 fn = istart, iend, lratio + do 200 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -193,7 +193,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, 200 continue 190 continue - do 210 koff = 0, lratio-1 + do 210 koff = 0, lratio-1 k = lratio*kc + koff if (k .lt. fblo(3)) then goto 210 @@ -204,7 +204,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, c --- break --- end if zoff = float(koff)-hafrat - do 220 joff = 0, lratio-1 + do 220 joff = 0, lratio-1 j = lratio*jc + joff if (j .lt. fblo(2)) then goto 220 @@ -216,9 +216,9 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, end if yoff = float(joff)-hafrat - do 230 i = fblo(1), fbhi(1) + do 230 i = fblo(1), fbhi(1) fn = i-fslo(1)+1 - fine(i,j,k,n) = fdat(fn) + volratio* + fine(i,j,k,n) = fdat(fn) + volratio* $ (voff(fn)*fslope(fn,1)+yoff*fslope(fn, $ 2)+ zoff*fslope(fn,3)) 230 continue @@ -229,16 +229,16 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, 180 continue 170 continue else if (maxlen .eq. leny) then - do 240 kc = cblo(3), cbhi(3) - do 250 ic = cblo(1), cbhi(1) + do 240 kc = cblo(3), cbhi(3) + do 250 ic = cblo(1), cbhi(1) c ::: ..,.......::::: strip out a fine grid slope vector - do 260 joff = 1, lratio + do 260 joff = 1, lratio icc = clo + ist*(ic-islo) + jst + kst*(kc- $ kslo) istart = joff iend = joff + (ncby-1)*lratio - do 270 fn = istart, iend, lratio + do 270 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -247,7 +247,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, 270 continue 260 continue - do 280 koff = 0, lratio-1 + do 280 koff = 0, lratio-1 k = lratio*kc + koff if (k .lt. fblo(3)) then goto 280 @@ -258,7 +258,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, c --- break --- end if zoff = float(koff)-hafrat - do 290 ioff = 0, lratio-1 + do 290 ioff = 0, lratio-1 i = lratio*ic + ioff if (i .lt. fblo(1)) then goto 290 @@ -270,9 +270,9 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, end if xoff = float(ioff)-hafrat - do 300 j = fblo(2), fbhi(2) + do 300 j = fblo(2), fbhi(2) fn = j-fslo(2)+1 - fine(i,j,k,n) = fdat(fn) + volratio* + fine(i,j,k,n) = fdat(fn) + volratio* $ (xoff*fslope(fn,1)+voff(fn)*fslope(fn, $ 2)+ zoff*fslope(fn,3)) 300 continue @@ -283,16 +283,16 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, 250 continue 240 continue else - do 310 jc = cblo(2), cbhi(2) - do 320 ic = cblo(1), cbhi(1) + do 310 jc = cblo(2), cbhi(2) + do 320 ic = cblo(1), cbhi(1) c ::: ..,.......::::: strip out a fine grid slope vector - do 330 koff = 1, lratio - icc = clo + ist*(ic-islo) + jst*(jc-jslo) + + do 330 koff = 1, lratio + icc = clo + ist*(ic-islo) + jst*(jc-jslo) + $ kst istart = koff iend = koff + (ncbz-1)*lratio - do 340 fn = istart, iend, lratio + do 340 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -301,7 +301,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, 340 continue 330 continue - do 350 joff = 0, lratio-1 + do 350 joff = 0, lratio-1 j = lratio*jc + joff if (j .lt. fblo(2)) then goto 350 @@ -312,7 +312,7 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, c --- break --- end if yoff = float(joff)-hafrat - do 360 ioff = 0, lratio-1 + do 360 ioff = 0, lratio-1 i = lratio*ic + ioff if (i .lt. fblo(1)) then goto 360 @@ -324,10 +324,10 @@ subroutine FORT_CINTERP (fine,floi1,floi2,floi3,fhii1,fhii2,fhii3, end if xoff = float(ioff)-hafrat - do 370 k = fblo(3), fbhi(3) + do 370 k = fblo(3), fbhi(3) fn = k-fslo(3)+1 - fine(i,j,k,n) = fdat(fn) + volratio* - $ (xoff*fslope(fn,1)+yoff*fslope(fn,2)+ + fine(i,j,k,n) = fdat(fn) + volratio* + $ (xoff*fslope(fn,1)+yoff*fslope(fn,2)+ $ voff(fn)*fslope(fn,3)) 370 continue 360 continue @@ -363,7 +363,7 @@ subroutine FORT_PCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, c ::: pcinterp: use piecewise constant interpolation to define c ::: values on the subregion of the fine FAB defined c ::: by (fblo,fbhi). -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fab to get interpolated values c ::: flo,fhi => (const) index limits of fine @@ -379,18 +379,18 @@ subroutine FORT_PCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, integer ixproj,ii,ll ixproj(ii,ll) = (ii + ll*iabs(ii))/ll - iabs(ii) - do 130 k = fblo(3), fbhi(3) + do 130 k = fblo(3), fbhi(3) kc = ixproj(k,lrat) - do 140 j = fblo(2), fbhi(2) + do 140 j = fblo(2), fbhi(2) jc = ixproj(j,lrat) - do 150 n = 1, nvar - do 160 ioff = 0, lrat-1 - do 170 ic = cblo(1),cbhi(1) + do 150 n = 1, nvar + do 160 ioff = 0, lrat-1 + do 170 ic = cblo(1),cbhi(1) i = lrat*ic + ioff temp(i) = crse(ic,jc,kc,n) 170 continue 160 continue - do 180 i = fblo(1), fbhi(1) + do 180 i = fblo(1), fbhi(1) fine(i,j,k,n) = temp(i) 180 continue 150 continue @@ -403,7 +403,7 @@ subroutine FORT_PCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, c ::: -------------------------------------------------------------- subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, - $ fhii3,fbloi1,fbloi2,fbloi3, fbhii1,fbhii2,fbhii3,nvar,lratio, + $ fhii3,fbloi1,fbloi2,fbloi3, fbhii1,fbhii2,fbhii3,nvar,lratio, $ crse,clo,chi,cblo,cbhi, fslo,fshi,cvloi1,cvloi2,cvloi3,cvhii1, $ cvhii2,cvhii3, cslope,clen,fslope,fdat,flen,voff, cvc1,fvc1,cvc2, $ fvc2,cvc3,fvc3) @@ -432,7 +432,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, REAL_T fvc2(fbloi2 :fbhii2+1) REAL_T fvc3(fbloi3 :fbhii3+1) -c ::: NOTE: data must be sent in so that +c ::: NOTE: data must be sent in so that c ::: cslope(1,*) and crse(1,*) are associated with c ::: the same cell @@ -440,21 +440,21 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, c ::: vcinterp: conservative interpolation in volume coordinates c ::: from coarse grid to c ::: subregion of fine grid defined by (fblo,fbhi) -c ::: +c ::: c ::: Inputs/Outputs c ::: fine <=> (modify) fine grid array c ::: flo,fhi => (const) index limits of fine grid c ::: fblo,fbhi => (const) subregion of fine grid to get values c ::: nvar => (const) number of variables in state vector c ::: lratio => (const) refinement ratio between levels -c ::: +c ::: c ::: crse => (const) coarse grid data widended by 1 zone c ::: and unrolled c ::: clo,chi => (const) one dimensional limits of crse grid c ::: cslo,cshi => (const) coarse grid index limits where c ::: slopes are to be defined. This is c ::: the projection of (fblo,fbhi) down -c ::: to the coarse level +c ::: to the coarse level c ::: fslo,fshi => (const) fine grid index limits where c ::: slopes are needed. This is the c ::: refinement of (cslo,cshi) and @@ -507,7 +507,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, lenz = fbhii3-fbloi3+1 maxlen = max(lenx,leny,lenz) if (maxlen .eq. lenx) then - do 380 i = fbloi1, fbhii1 + do 380 i = fbloi1, fbhii1 fn = i-fslo(1)+1 ic = IX_PROJ(i,lratio) fcen = half*(fvc1(i)+fvc1(i+1)) @@ -515,7 +515,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, voff(fn) = (fcen-ccen)/(cvc1(ic+1)-cvc1(ic)) 380 continue else if (maxlen .eq. leny) then - do 390 j = fbloi2, fbhii2 + do 390 j = fbloi2, fbhii2 fn = j-fslo(2)+1 jc = IX_PROJ(j,lratio) fcen = half*(fvc2(j)+fvc2(j+1)) @@ -523,7 +523,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, voff(fn) = (fcen-ccen)/(cvc2(jc+1)-cvc2(jc)) 390 continue else - do 400 k = fbloi3, fbhii3 + do 400 k = fbloi3, fbhii3 fn = k-fslo(3)+1 kc = IX_PROJ(k,lratio) fcen = half*(fvc3(k)+fvc3(k+1)) @@ -531,10 +531,10 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, voff(fn) = (fcen-ccen)/(cvc3(kc+1)-cvc3(kc)) 400 continue end if - do 410 n = 1, nvar + do 410 n = 1, nvar c ::: ::::: compute slopes in x direction - do 420 i = 1, clen + do 420 i = 1, clen cen = half*(crse(i+ist,n)-crse(i-ist,n)) forw = crse(i+ist,n)-crse(i,n) back = crse(i,n)-crse(i-ist,n) @@ -542,7 +542,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, cslope(i,1)=merge(slp,zero,forw*back>=0.0d0) 420 continue c ::: ::::: compute slopes in y direction - do 430 i = 1, clen + do 430 i = 1, clen cen = half*(crse(i+jst,n)-crse(i-jst,n)) forw = crse(i+jst,n)-crse(i,n) back = crse(i,n)-crse(i-jst,n) @@ -550,7 +550,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, cslope(i,2)=merge(slp,zero,forw*back>=0.0d0) 430 continue c ::: ::::: compute slopes in z direction - do 440 i = 1, clen + do 440 i = 1, clen cen = half*(crse(i+kst,n)-crse(i-kst,n)) forw = crse(i+kst,n)-crse(i,n) back = crse(i,n)-crse(i-kst,n) @@ -559,16 +559,16 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, 440 continue if (maxlen .eq. lenx) then - do 450 kc = cblo(3), cbhi(3) - do 460 jc = cblo(2), cbhi(2) + do 450 kc = cblo(3), cbhi(3) + do 460 jc = cblo(2), cbhi(2) c ::: ..,.......::::: strip out a fine grid slope vector - do 470 ioff = 1, lratio + do 470 ioff = 1, lratio icc = clo + ist + jst*(jc-jslo) + kst*(kc- $ kslo) istart = ioff iend = ioff + (ncbx-1)*lratio - do 480 fn = istart, iend, lratio + do 480 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -577,7 +577,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, 480 continue 470 continue - do 490 koff = 0, lratio-1 + do 490 koff = 0, lratio-1 k = lratio*kc + koff if (k .lt. fbloi3) then goto 490 @@ -590,7 +590,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, fcen = half*(fvc3(k)+fvc3(k+1)) ccen = half*(cvc3(kc)+cvc3(kc+1)) zoff = (fcen-ccen)/(cvc3(kc+1)-cvc3(kc)) - do 500 joff = 0, lratio-1 + do 500 joff = 0, lratio-1 j = lratio*jc + joff if (j .lt. fbloi2) then goto 500 @@ -604,7 +604,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, ccen = half*(cvc2(jc)+cvc2(jc+1)) yoff = (fcen-ccen)/(cvc2(jc+1)-cvc2(jc)) - do 510 i = fbloi1, fbhii1 + do 510 i = fbloi1, fbhii1 fn = i-fslo(1)+1 fine(i,j,k,n) = fdat(fn) + voff(fn)* $ fslope(fn,1)+yoff*fslope(fn,2)+ zoff* @@ -617,16 +617,16 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, 460 continue 450 continue else if (maxlen .eq. leny) then - do 520 kc = cblo(3), cbhi(3) - do 530 ic = cblo(1), cbhi(1) + do 520 kc = cblo(3), cbhi(3) + do 530 ic = cblo(1), cbhi(1) c ::: ..,.......::::: strip out a fine grid slope vector - do 540 joff = 1, lratio + do 540 joff = 1, lratio icc = clo + ist*(ic-islo) + jst + kst*(kc- $ kslo) istart = joff iend = joff + (ncby-1)*lratio - do 550 fn = istart, iend, lratio + do 550 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -635,7 +635,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, 550 continue 540 continue - do 560 koff = 0, lratio-1 + do 560 koff = 0, lratio-1 k = lratio*kc + koff if (k .lt. fbloi3) then goto 560 @@ -648,7 +648,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, fcen = half*(fvc3(k)+fvc3(k+1)) ccen = half*(cvc3(kc)+cvc3(kc+1)) zoff = (fcen-ccen)/(cvc3(kc+1)-cvc3(kc)) - do 570 ioff = 0, lratio-1 + do 570 ioff = 0, lratio-1 i = lratio*ic + ioff if (i .lt. fbloi1) then goto 570 @@ -662,10 +662,10 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, ccen = half*(cvc1(ic)+cvc1(ic+1)) xoff = (fcen-ccen)/(cvc1(ic+1)-cvc1(ic)) - do 580 j = fbloi2, fbhii2 + do 580 j = fbloi2, fbhii2 fn = j-fslo(2)+1 fine(i,j,k,n) = fdat(fn) + xoff* - $ fslope(fn,1)+voff(fn)*fslope(fn,2)+ + $ fslope(fn,1)+voff(fn)*fslope(fn,2)+ $ zoff*fslope(fn,3) 580 continue 570 continue @@ -675,16 +675,16 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, 530 continue 520 continue else - do 590 jc = cblo(2), cbhi(2) - do 600 ic = cblo(1), cbhi(1) + do 590 jc = cblo(2), cbhi(2) + do 600 ic = cblo(1), cbhi(1) c ::: ..,.......::::: strip out a fine grid slope vector - do 610 koff = 1, lratio - icc = clo + ist*(ic-islo) + jst*(jc-jslo) + + do 610 koff = 1, lratio + icc = clo + ist*(ic-islo) + jst*(jc-jslo) + $ kst istart = koff iend = koff + (ncbz-1)*lratio - do 620 fn = istart, iend, lratio + do 620 fn = istart, iend, lratio fslope(fn,1) = cslope(icc,1) fslope(fn,2) = cslope(icc,2) fslope(fn,3) = cslope(icc,3) @@ -693,7 +693,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, 620 continue 610 continue - do 630 joff = 0, lratio-1 + do 630 joff = 0, lratio-1 j = lratio*jc + joff if (j .lt. fbloi2) then goto 630 @@ -706,7 +706,7 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, fcen = half*(fvc2(j)+fvc2(j+1)) ccen = half*(cvc2(jc)+cvc2(jc+1)) yoff = (fcen-ccen)/(cvc2(jc+1)-cvc2(jc)) - do 640 ioff = 0, lratio-1 + do 640 ioff = 0, lratio-1 i = lratio*ic + ioff if (i .lt. fbloi1) then goto 640 @@ -720,10 +720,10 @@ subroutine FORT_VCINTERP (fine,floi1,floi2,floi3,fhii1,fhii2, ccen = half*(cvc1(ic)+cvc1(ic+1)) xoff = (fcen-ccen)/(cvc1(ic+1)-cvc1(ic)) - do 650 k = fbloi3, fbhii3 + do 650 k = fbloi3, fbhii3 fn = k-fslo(3)+1 fine(i,j,k,n) = fdat(fn) + xoff* - $ fslope(fn,1)+yoff*fslope(fn,2)+ + $ fslope(fn,1)+yoff*fslope(fn,2)+ $ voff(fn)*fslope(fn,3) 650 continue 640 continue diff --git a/Src/Extern/amrdata/AMReX_XYPlotDataList.H b/Src/Extern/amrdata/AMReX_XYPlotDataList.H index 30de44c4601..385b3f7e6a3 100644 --- a/Src/Extern/amrdata/AMReX_XYPlotDataList.H +++ b/Src/Extern/amrdata/AMReX_XYPlotDataList.H @@ -44,7 +44,7 @@ class XYPlotDataListLink { public: XYPlotDataListLink(const Real *d, int sX, int len) : xypdllLength(len), startXi(sX) - { + { BL_ASSERT(len != 0); endXi = sX + len; xypdllData = new Real[len]; @@ -74,11 +74,11 @@ class XYPlotDataListLink { class XYPlotDataList { public: XYPlotDataList(const string &derived, int minlevel, int maxlevel, - int gridlinein, - const amrex::Vector &ratiolist, - const amrex::Vector &dx, - const amrex::Vector &intersectpoint, - Real startx = 0.0); + int gridlinein, + const amrex::Vector &ratiolist, + const amrex::Vector &dx, + const amrex::Vector &intersectpoint, + Real startx = 0.0); XYPlotDataList(XYPlotDataList *src); ~XYPlotDataList(); @@ -104,7 +104,7 @@ class XYPlotDataList { amrex::Vector &YVal(int level) { return xypdlYVal[level]; } protected: - amrex::Vector > dataSets; + amrex::Vector > dataSets; amrex::Vector xypdlRatios; // Used for insertion of data sets beginning // in the middle of a single cell [level] amrex::Vector dX; // dX in the proper direction [level] diff --git a/Src/Extern/amrdata/AMReX_XYPlotDataList.cpp b/Src/Extern/amrdata/AMReX_XYPlotDataList.cpp index 7949f4b009c..5aa4f8803ef 100644 --- a/Src/Extern/amrdata/AMReX_XYPlotDataList.cpp +++ b/Src/Extern/amrdata/AMReX_XYPlotDataList.cpp @@ -1,5 +1,5 @@ // ------------------------------------------------------------------- -// XYPlotDataList.cpp +// XYPlotDataList.cpp // ------------------------------------------------------------------- #include #include @@ -12,10 +12,10 @@ using namespace amrex; // ------------------------------------------------------------------- XYPlotDataList::XYPlotDataList(const string &derived, int minlevel, int maxlevel, int gridlinein, - const Vector &ratiolist, - const Vector &dx, - const Vector &intersectpoint, - Real offsetx) + const Vector &ratiolist, + const Vector &dx, + const Vector &intersectpoint, + Real offsetx) : dataSets(maxlevel + 1), xypdlRatios(ratiolist), dX(dx), @@ -148,7 +148,7 @@ for(int iCurLevel(maxLevel); iCurLevel >= minLevel; --iCurLevel) { if(fabBoxLists[ilev].isEmpty()) { // set correct index type fabBoxLists[ilev].convert(probDomain.ixType()); } - fillBoxLists[ilev].clear(); + fillBoxLists[ilev].clear(); fillBoxLists[ilev].convert(unfilledBoxLists[ilev].ixType()); fillBoxLists[ilev].join(unfilledBoxLists[ilev]); fillBoxLists[ilev].intersect(fabBoxLists[ilev]); @@ -175,7 +175,7 @@ for(int iCurLevel(maxLevel); iCurLevel >= minLevel; --iCurLevel) { { (*bli).setSmall(isd, (*(fabBoxLists[ilev - 1].begin())).smallEnd(isd)); (*bli).setBig(isd, (*(fabBoxLists[ilev - 1].begin())).bigEnd(isd)); - } + } } } } @@ -218,10 +218,10 @@ for(int iCurLevel(maxLevel); iCurLevel >= minLevel; --iCurLevel) { li != (dataSets[obLev]).end(); ++li) { XYPlotDataListLink *xypd = *li; - if(i >= xypd->StartXi() && i < xypd->EndXi()) { - Real yval(xypd->XYPDLLData()[i - xypd->StartXi()]); + if(i >= xypd->StartXi() && i < xypd->EndXi()) { + Real yval(xypd->XYPDLLData()[i - xypd->StartXi()]); xypdlYVal[iCurLevel][xIndex] = yval; - } + } } ++xIndex; } diff --git a/Src/Extern/amrdata/CMakeLists.txt b/Src/Extern/amrdata/CMakeLists.txt index 137b7d0090f..a2c281d227b 100644 --- a/Src/Extern/amrdata/CMakeLists.txt +++ b/Src/Extern/amrdata/CMakeLists.txt @@ -7,7 +7,7 @@ target_sources(amrex AMReX_XYPlotDataList.H AMReX_XYPlotDataList.cpp AMReX_DataServices.H - AMReX_DataServices.cpp + AMReX_DataServices.cpp AMReX_AmrvisConstants.H AMReX_FABUTIL_${AMReX_SPACEDIM}D.F ) diff --git a/Src/Extern/hpgmg/BL_HPGMG.H b/Src/Extern/hpgmg/BL_HPGMG.H index e69132ed2c0..992ba557433 100644 --- a/Src/Extern/hpgmg/BL_HPGMG.H +++ b/Src/Extern/hpgmg/BL_HPGMG.H @@ -31,8 +31,8 @@ #endif typedef struct { - int subtype; // e.g. used to calculate normal to domain for BC's - struct {int i, j, k;}dim; // dimensions of the block to copy + int subtype; // e.g. used to calculate normal to domain for BC's + struct {int i, j, k;}dim; // dimensions of the block to copy struct {int box, i, j, k, jStride, kStride;double * __restrict__ ptr;}read,write; // coordinates in the read grid to extract data, // coordinates in the write grid to insert data @@ -42,17 +42,17 @@ typedef struct { typedef struct { - int num_recvs; // number of neighbors by type - int num_sends; // number of neighbors by type - int * __restrict__ recv_ranks; // MPI rank of each neighbor... recv_ranks[neighbor] - int * __restrict__ send_ranks; // MPI rank of each neighbor... send_ranks[neighbor] - int * __restrict__ recv_sizes; // size of each MPI recv buffer... recv_sizes[neighbor] - int * __restrict__ send_sizes; // size of each MPI send buffer... send_sizes[neighbor] - double ** __restrict__ recv_buffers; // MPI recv buffer for each neighbor... recv_buffers[neighbor][ recv_sizes[neighbor] ] - double ** __restrict__ send_buffers; // MPI send buffer for each neighbor... send_buffers[neighbor][ send_sizes[neighbor] ] - int allocated_blocks[3]; // number of blocks allocated (not necessarily used) each list... - int num_blocks[3]; // number of blocks in each list... num_blocks[pack,local,unpack] - blockCopy_type * blocks[3]; // list of block copies... blocks[pack,local,unpack] + int num_recvs; // number of neighbors by type + int num_sends; // number of neighbors by type + int * __restrict__ recv_ranks; // MPI rank of each neighbor... recv_ranks[neighbor] + int * __restrict__ send_ranks; // MPI rank of each neighbor... send_ranks[neighbor] + int * __restrict__ recv_sizes; // size of each MPI recv buffer... recv_sizes[neighbor] + int * __restrict__ send_sizes; // size of each MPI send buffer... send_sizes[neighbor] + double ** __restrict__ recv_buffers; // MPI recv buffer for each neighbor... recv_buffers[neighbor][ recv_sizes[neighbor] ] + double ** __restrict__ send_buffers; // MPI send buffer for each neighbor... send_buffers[neighbor][ send_sizes[neighbor] ] + int allocated_blocks[3]; // number of blocks allocated (not necessarily used) each list... + int num_blocks[3]; // number of blocks in each list... num_blocks[pack,local,unpack] + blockCopy_type * blocks[3]; // list of block copies... blocks[pack,local,unpack] #ifdef BL_USE_MPI MPI_Request * __restrict__ requests; MPI_Status * __restrict__ status; @@ -60,60 +60,60 @@ typedef struct { } communicator_type; typedef struct { - int global_box_id; // used to inded into level->rank_of_box - struct {int i, j, k;}low; // global coordinates of the first (non-ghost) element of subdomain - int dim; // dimension of this box's core (owned) - int ghosts; // ghost zone depth - int jStride,kStride,volume; // useful for offsets - int numVectors; // - double ** __restrict__ vectors; // vectors[c] = pointer to 3D array for vector c for one box + int global_box_id; // used to inded into level->rank_of_box + struct {int i, j, k;}low; // global coordinates of the first (non-ghost) element of subdomain + int dim; // dimension of this box's core (owned) + int ghosts; // ghost zone depth + int jStride,kStride,volume; // useful for offsets + int numVectors; // + double ** __restrict__ vectors; // vectors[c] = pointer to 3D array for vector c for one box } box_type; typedef struct { - double h; // grid spacing at this level - int active; // I am an active process (I have work to do on this or subsequent levels) - int num_ranks; // total number of MPI ranks - int my_rank; // my MPI rank - int box_dim; // dimension of each cubical box (not counting ghost zones) - int box_ghosts; // ghost zone depth for each box - int box_jStride,box_kStride,box_volume; // useful for offsets - int numVectors; // number of vectors stored in each box - int tag; // tag each level uniquely... FIX... replace with sub commuicator - struct {int i, j, k;}boxes_in; // total number of boxes in i,j,k across this level - struct {int i, j, k;}dim; // global dimensions at this level (NOTE: dim.i == boxes_in.i * box_dim) - - int * rank_of_box; // 3D array containing rank of each box. i-major ordering - int num_my_boxes; // number of boxes owned by this rank - box_type * my_boxes; // pointer to array of boxes owned by this rank + double h; // grid spacing at this level + int active; // I am an active process (I have work to do on this or subsequent levels) + int num_ranks; // total number of MPI ranks + int my_rank; // my MPI rank + int box_dim; // dimension of each cubical box (not counting ghost zones) + int box_ghosts; // ghost zone depth for each box + int box_jStride,box_kStride,box_volume; // useful for offsets + int numVectors; // number of vectors stored in each box + int tag; // tag each level uniquely... FIX... replace with sub commuicator + struct {int i, j, k;}boxes_in; // total number of boxes in i,j,k across this level + struct {int i, j, k;}dim; // global dimensions at this level (NOTE: dim.i == boxes_in.i * box_dim) + + int * rank_of_box; // 3D array containing rank of each box. i-major ordering + int num_my_boxes; // number of boxes owned by this rank + box_type * my_boxes; // pointer to array of boxes owned by this rank // create flattened FP data... useful for CUDA/OpenMP4/OpenACC when you want to copy an entire vector to/from an accelerator - double ** __restrict__ vectors; // vectors[v][box][k][j][i] = pointer to 5D array for vector v encompasing all boxes on this process... + double ** __restrict__ vectors; // vectors[v][box][k][j][i] = pointer to 5D array for vector v encompasing all boxes on this process... double * __restrict__ vectors_base; // pointer used for malloc/free. vectors[v] are shifted from this for alignment - int allocated_blocks; // number of blocks allocated by this rank (note, this represents a flattening of the box/cell hierarchy to facilitate threading) - int num_my_blocks; // number of blocks owned by this rank (note, this represents a flattening of the box/cell hierarchy to facilitate threading) - blockCopy_type * my_blocks; // pointer to array of blocks owned by this rank (note, this represents a flattening of the box/cell hierarchy to facilitate threading) + int allocated_blocks; // number of blocks allocated by this rank (note, this represents a flattening of the box/cell hierarchy to facilitate threading) + int num_my_blocks; // number of blocks owned by this rank (note, this represents a flattening of the box/cell hierarchy to facilitate threading) + blockCopy_type * my_blocks; // pointer to array of blocks owned by this rank (note, this represents a flattening of the box/cell hierarchy to facilitate threading) struct { - int type; // BC_PERIODIC or BC_DIRICHLET + int type; // BC_PERIODIC or BC_DIRICHLET int allocated_blocks[STENCIL_MAX_SHAPES];// number of blocks allocated (not necessarily used) for boundary conditions on this level for [shape] int num_blocks[STENCIL_MAX_SHAPES];// number of blocks used for boundary conditions on this level for [shape] blockCopy_type * blocks[STENCIL_MAX_SHAPES];// pointer to array of blocks used for boundary conditions on this level for [shape] - } boundary_condition; // boundary conditions on this level + } boundary_condition; // boundary conditions on this level communicator_type exchange_ghosts[STENCIL_MAX_SHAPES];// mini program that performs a neighbor ghost zone exchange for [shape] - communicator_type restriction[4]; // mini program that performs restriction and agglomeration for [0=cell centered, 1=i-face, 2=j-face, 3-k-face] - communicator_type interpolation; // mini program that performs interpolation and dissemination... + communicator_type restriction[4]; // mini program that performs restriction and agglomeration for [0=cell centered, 1=i-face, 2=j-face, 3-k-face] + communicator_type interpolation; // mini program that performs interpolation and dissemination... #ifdef BL_USE_MPI - MPI_Comm MPI_COMM_ALLREDUCE; // MPI sub communicator for just the ranks that have boxes on this level or any subsequent level... + MPI_Comm MPI_COMM_ALLREDUCE; // MPI sub communicator for just the ranks that have boxes on this level or any subsequent level... #endif - double dominant_eigenvalue_of_DinvA; // estimate on the dominate eigenvalue of D^{-1}A - int must_subtract_mean; // e.g. Poisson with Periodic BC's + double dominant_eigenvalue_of_DinvA; // estimate on the dominate eigenvalue of D^{-1}A + int must_subtract_mean; // e.g. Poisson with Periodic BC's double * __restrict__ RedBlack_base; // allocated pointer... will be aligned for the first non ghost zone element - double * __restrict__ RedBlack_FP; // Red/Black Mask (i.e. 0.0 or 1.0) for even/odd planes (2*kStride). + double * __restrict__ RedBlack_FP; // Red/Black Mask (i.e. 0.0 or 1.0) for even/odd planes (2*kStride). int num_threads; - double * __restrict__ fluxes; // temporary array used to hold the flux values used by FV operators + double * __restrict__ fluxes; // temporary array used to hold the flux values used by FV operators // statistics information... struct { @@ -157,10 +157,10 @@ typedef struct { } level_type; typedef struct { - int num_ranks; // total number of MPI ranks for MPI_COMM_WORLD - int my_rank; // my MPI rank for MPI_COMM_WORLD - int num_levels; // depth of the v-cycle - level_type ** levels; // array of pointers to levels + int num_ranks; // total number of MPI ranks for MPI_COMM_WORLD + int my_rank; // my MPI rank for MPI_COMM_WORLD + int num_levels; // depth of the v-cycle + level_type ** levels; // array of pointers to levels struct { uint64_t MGBuild; // total time spent building the coefficients... diff --git a/Src/Extern/hpgmg/BL_HPGMG.cpp b/Src/Extern/hpgmg/BL_HPGMG.cpp index f0dca2a234c..5fa763673ae 100644 --- a/Src/Extern/hpgmg/BL_HPGMG.cpp +++ b/Src/Extern/hpgmg/BL_HPGMG.cpp @@ -454,8 +454,8 @@ void ConvertToHPGMGLevel (const MultiFab& mf, for(j=0;jmaxLevel(); + return amrcore->maxLevel(); } void amrex_fi_get_ref_ratio (int* ref_ratio, const FAmrCore* amrcore) { - int n = amrcore->maxLevel(); - for (int i = 0; i < n; ++i) { - ref_ratio[i] = amrcore->MaxRefRatio(i); - } + int n = amrcore->maxLevel(); + for (int i = 0; i < n; ++i) { + ref_ratio[i] = amrcore->MaxRefRatio(i); + } } int amrex_fi_get_finest_level (const FAmrCore* amrcore) { - return amrcore->finestLevel(); + return amrcore->finestLevel(); } void amrex_fi_get_boxarray (const BoxArray*& ba, int lev, const FAmrCore* amrcore) { - const BoxArray& ba_ = amrcore->boxArray(lev); - ba = &ba_; + const BoxArray& ba_ = amrcore->boxArray(lev); + ba = &ba_; } void amrex_fi_get_distromap (const DistributionMapping*& dm, int lev, const FAmrCore* amrcore) { - const DistributionMapping& dm_ = amrcore->DistributionMap(lev); - dm = &dm_; + const DistributionMapping& dm_ = amrcore->DistributionMap(lev); + dm = &dm_; } void amrex_fi_get_geometry (const Geometry*& geom, int lev, const FAmrCore* amrcore) { - const Geometry& geom_ = amrcore->Geom(lev); - geom = &geom_; + const Geometry& geom_ = amrcore->Geom(lev); + geom = &geom_; } void amrex_fi_make_base_grids (BoxArray*& ba, const FAmrCore* amrcore) { - ba = new BoxArray(amrcore->MakeBaseGrids()); + ba = new BoxArray(amrcore->MakeBaseGrids()); } void amrex_fi_set_finest_level (int new_finest_level, FAmrCore* amrcore) { - amrcore->SetFinestLevel(new_finest_level); + amrcore->SetFinestLevel(new_finest_level); } void amrex_fi_set_boxarray (int lev, const BoxArray* ba, FAmrCore* amrcore) { - if (ba) { - amrcore->SetBoxArray(lev, *ba); - } else { - amrcore->ClearBoxArray(lev); - } + if (ba) { + amrcore->SetBoxArray(lev, *ba); + } else { + amrcore->ClearBoxArray(lev); + } } void amrex_fi_set_distromap (int lev, const DistributionMapping* dm, FAmrCore* amrcore) { - if (dm) { - amrcore->SetDistributionMap(lev, *dm); - } else { - amrcore->ClearDistributionMap(lev); - } + if (dm) { + amrcore->SetDistributionMap(lev, *dm); + } else { + amrcore->ClearDistributionMap(lev); + } } void amrex_fi_set_geometry (int lev, const Geometry* gm, FAmrCore* amrcore) @@ -87,38 +87,38 @@ extern "C" { } void amrex_fi_make_new_grids (int baselev, Real time, int* new_finest, const BoxArray** ba, - FAmrCore* amrcore) - { - Vector new_grids(baselev+1); - new_grids[baselev] = *ba[baselev]; - amrcore->MakeNewGrids(baselev, time, *new_finest, new_grids); - for (int lev = baselev+1; lev <= *new_finest; ++lev) { - delete ba[lev]; - ba[lev] = new BoxArray(new_grids[lev]); - } + FAmrCore* amrcore) + { + Vector new_grids(baselev+1); + new_grids[baselev] = *ba[baselev]; + amrcore->MakeNewGrids(baselev, time, *new_finest, new_grids); + for (int lev = baselev+1; lev <= *new_finest; ++lev) { + delete ba[lev]; + ba[lev] = new BoxArray(new_grids[lev]); + } } void amrex_fi_init_from_scratch (Real t, FAmrCore* amrcore) { - amrcore->InitFromScratch(t); + amrcore->InitFromScratch(t); } void amrex_fi_init_virtual_functions (FAmrCore::make_level_funptr_t mk_lev_scrtch, FAmrCore::make_level_funptr_t mk_lev_crse, FAmrCore::make_level_funptr_t mk_lev_re, - FAmrCore::clear_level_funptr_t clr_lev, - FAmrCore::error_est_funptr_t err_est, - FAmrCore* amrcore) + FAmrCore::clear_level_funptr_t clr_lev, + FAmrCore::error_est_funptr_t err_est, + FAmrCore* amrcore) { - amrcore->make_new_level_from_scratch = mk_lev_scrtch; + amrcore->make_new_level_from_scratch = mk_lev_scrtch; amrcore->make_new_level_from_coarse = mk_lev_crse; amrcore->remake_level = mk_lev_re; - amrcore->clear_level = clr_lev; - amrcore->error_est = err_est; + amrcore->clear_level = clr_lev; + amrcore->error_est = err_est; } void amrex_fi_regrid (int baselev, Real t, FAmrCore* amrcore) { - amrcore->regrid(baselev, t); + amrcore->regrid(baselev, t); } } diff --git a/Src/F_Interfaces/AmrCore/AMReX_amrcore_mod.F90 b/Src/F_Interfaces/AmrCore/AMReX_amrcore_mod.F90 index 679a2463842..d73f3d36e5a 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_amrcore_mod.F90 +++ b/Src/F_Interfaces/AmrCore/AMReX_amrcore_mod.F90 @@ -2,7 +2,7 @@ module amrex_amrcore_module use iso_c_binding use amrex_base_module - + implicit none private @@ -138,7 +138,7 @@ subroutine amrex_fi_set_distromap (lev, dm, amrcore) bind(c) type(c_ptr), value :: dm type(c_ptr), value :: amrcore end subroutine amrex_fi_set_distromap - + subroutine amrex_fi_set_geometry (lev, gm, amrcore) bind(c) import implicit none diff --git a/Src/F_Interfaces/AmrCore/AMReX_fillpatch_fi.cpp b/Src/F_Interfaces/AmrCore/AMReX_fillpatch_fi.cpp index fdb62891ad2..c088e6d63f0 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_fillpatch_fi.cpp +++ b/Src/F_Interfaces/AmrCore/AMReX_fillpatch_fi.cpp @@ -7,20 +7,20 @@ namespace { // THIS MUST BE CONSISTENT WITH amrex_interpolater_module in AMReX_interpolater_mod.F90!!! static Vector interp = { - &amrex::pc_interp, // 0 + &amrex::pc_interp, // 0 &amrex::node_bilinear_interp, // 1 &amrex::cell_bilinear_interp, // 2 - &amrex::quadratic_interp, // 3 - &amrex::lincc_interp, // 4 - &amrex::cell_cons_interp, // 5 - &amrex::protected_interp, // 6 + &amrex::quadratic_interp, // 3 + &amrex::lincc_interp, // 4 + &amrex::cell_cons_interp, // 5 + &amrex::protected_interp, // 6 &amrex::quartic_interp, // 7 &amrex::face_divfree_interp // 8 }; } namespace { - + typedef void (*INTERP_HOOK) (const int* lo, const int*hi, Real* d, const int* dlo, const int* dhi, const int nd, const int icomp, const int ncomp); @@ -83,41 +83,41 @@ namespace { extern "C" { void amrex_fi_fillpatch_single (MultiFab* mf, Real time, MultiFab* smf[], Real stime[], int ns, - int scomp, int dcomp, int ncomp, const Geometry* geom, + int scomp, int dcomp, int ncomp, const Geometry* geom, FPhysBC::fill_physbc_funptr_t fill) { FPhysBC pbc(fill, geom); - amrex::FillPatchSingleLevel(*mf, time, Vector{smf, smf+ns}, - Vector{stime, stime+ns}, - scomp, dcomp, ncomp, *geom, pbc, 0); + amrex::FillPatchSingleLevel(*mf, time, Vector{smf, smf+ns}, + Vector{stime, stime+ns}, + scomp, dcomp, ncomp, *geom, pbc, 0); } void amrex_fi_fillpatch_two (MultiFab* mf, Real time, - MultiFab* cmf[], Real ct[], int nc, - MultiFab* fmf[], Real ft[], int nf, - int scomp, int dcomp, int ncomp, - const Geometry* cgeom, const Geometry* fgeom, - FPhysBC::fill_physbc_funptr_t cfill, + MultiFab* cmf[], Real ct[], int nc, + MultiFab* fmf[], Real ft[], int nf, + int scomp, int dcomp, int ncomp, + const Geometry* cgeom, const Geometry* fgeom, + FPhysBC::fill_physbc_funptr_t cfill, FPhysBC::fill_physbc_funptr_t ffill, - int rr, int interp_id, - int* lo_bc[], int* hi_bc[], + int rr, int interp_id, + int* lo_bc[], int* hi_bc[], INTERP_HOOK pre_interp, INTERP_HOOK post_interp) { - Vector bcs(ncomp); - for (int i = 0; i < ncomp; ++i) { - bcs.emplace_back(lo_bc[i+scomp], hi_bc[i+scomp]); - } + Vector bcs(ncomp); + for (int i = 0; i < ncomp; ++i) { + bcs.emplace_back(lo_bc[i+scomp], hi_bc[i+scomp]); + } FPhysBC cbc(cfill, cgeom); FPhysBC fbc(ffill, fgeom); - amrex::FillPatchTwoLevels(*mf, time, - Vector{cmf, cmf+nc}, Vector{ct, ct+nc}, - Vector{fmf, fmf+nf}, Vector{ft, ft+nf}, - scomp, dcomp, ncomp, - *cgeom, *fgeom, - cbc, 0, fbc, 0, - IntVect{AMREX_D_DECL(rr,rr,rr)}, - interp[interp_id], bcs, 0, + amrex::FillPatchTwoLevels(*mf, time, + Vector{cmf, cmf+nc}, Vector{ct, ct+nc}, + Vector{fmf, fmf+nf}, Vector{ft, ft+nf}, + scomp, dcomp, ncomp, + *cgeom, *fgeom, + cbc, 0, fbc, 0, + IntVect{AMREX_D_DECL(rr,rr,rr)}, + interp[interp_id], bcs, 0, FIInterpHook(pre_interp), FIInterpHook(post_interp)); } @@ -138,7 +138,7 @@ extern "C" { bcs[d].resize(ncomp); for (int i = 0; i < ncomp; ++i) - { bcs[d].emplace_back(lo_bc[d*(scomp+ncomp)+i+scomp], + { bcs[d].emplace_back(lo_bc[d*(scomp+ncomp)+i+scomp], hi_bc[d*(scomp+ncomp)+i+scomp]); } } @@ -160,15 +160,15 @@ extern "C" FPhysBC(ffill[1], fgeom), FPhysBC(ffill[2], fgeom)) }; - amrex::FillPatchTwoLevels(Array{AMREX_D_DECL(mf[0], mf[1], mf[2])}, + amrex::FillPatchTwoLevels(Array{AMREX_D_DECL(mf[0], mf[1], mf[2])}, time, - va_cmf, Vector{ct, ct+nc}, - va_fmf, Vector{ft, ft+nf}, - scomp, dcomp, ncomp, - *cgeom, *fgeom, - cbc, 0, fbc, 0, - IntVect{AMREX_D_DECL(rr,rr,rr)}, - interp[interp_id], bcs, 0, + va_cmf, Vector{ct, ct+nc}, + va_fmf, Vector{ft, ft+nf}, + scomp, dcomp, ncomp, + *cgeom, *fgeom, + cbc, 0, fbc, 0, + IntVect{AMREX_D_DECL(rr,rr,rr)}, + interp[interp_id], bcs, 0, FIArrInterpHook(pre_interp), FIArrInterpHook(post_interp)); } @@ -182,14 +182,14 @@ extern "C" int* lo_bc[], int* hi_bc[], INTERP_HOOK pre_interp, INTERP_HOOK post_interp) { - Vector bcs(ncomp); - for (int i = 0; i < ncomp; ++i) { - bcs.emplace_back(lo_bc[i+scomp], hi_bc[i+scomp]); - } + Vector bcs(ncomp); + for (int i = 0; i < ncomp; ++i) { + bcs.emplace_back(lo_bc[i+scomp], hi_bc[i+scomp]); + } FPhysBC cbc(cfill, cgeom); FPhysBC fbc(ffill, fgeom); - amrex::InterpFromCoarseLevel(*mf, time, *cmf, + amrex::InterpFromCoarseLevel(*mf, time, *cmf, scomp, dcomp, ncomp, *cgeom, *fgeom, cbc, 0, fbc, 0, diff --git a/Src/F_Interfaces/AmrCore/AMReX_fillpatch_mod.F90 b/Src/F_Interfaces/AmrCore/AMReX_fillpatch_mod.F90 index 6d758f1c068..294f86e4fa6 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_fillpatch_mod.F90 +++ b/Src/F_Interfaces/AmrCore/AMReX_fillpatch_mod.F90 @@ -116,7 +116,7 @@ subroutine amrex_fillpatch_single (mf, told, mfold, tnew, mfnew, geom, fill_phys ! scomp-1 and dcomp-1 because of Fortran index starts with 1 call amrex_fi_fillpatch_single(mf%p, time, smf, stime, ns, scomp-1, dcomp-1, ncomp, geom%p, & & c_funloc(fill_physbc)) - + end subroutine amrex_fillpatch_single subroutine amrex_fillpatch_two (mf, told_c, mfold_c, tnew_c, mfnew_c, geom_c, fill_physbc_c, & @@ -139,7 +139,7 @@ subroutine amrex_fillpatch_two (mf, told_c, mfold_c, tnew_c, mfnew_c, geom_c, fi type(c_ptr) :: lo_bc_ptr(scomp+ncomp-1), hi_bc_ptr(scomp+ncomp-1) type(c_funptr) :: pre_interp_ptr, post_interp_ptr integer :: ncrse, nfine, i - + ! coarse level teps = 1.e-4_amrex_real * (tnew_c - told_c) if (abs(time-tnew_c) .lt. teps) then @@ -350,7 +350,7 @@ subroutine amrex_fillcoarsepatch (mf, told_c, mfold_c, tnew_c, mfnew_c, & type(c_ptr) :: lo_bc_ptr(scomp+ncomp-1), hi_bc_ptr(scomp+ncomp-1) type(c_funptr) :: pre_interp_ptr, post_interp_ptr integer :: i - + ! coarse level teps = 1.e-4_amrex_real * (tnew_c - told_c) if (abs(time-tnew_c) .lt. teps) then diff --git a/Src/F_Interfaces/AmrCore/AMReX_fluxregister_fi.cpp b/Src/F_Interfaces/AmrCore/AMReX_fluxregister_fi.cpp index ee98cbb7ee0..edbc3970bad 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_fluxregister_fi.cpp +++ b/Src/F_Interfaces/AmrCore/AMReX_fluxregister_fi.cpp @@ -5,7 +5,7 @@ using namespace amrex; extern "C" { - void amrex_fi_new_fluxregister (FluxRegister*& flux_reg, const BoxArray* ba, + void amrex_fi_new_fluxregister (FluxRegister*& flux_reg, const BoxArray* ba, const DistributionMapping* dm, int rr, int flev, int ncomp) { flux_reg = new FluxRegister(*ba, *dm, IntVect(AMREX_D_DECL(rr,rr,rr)), flev, ncomp); @@ -27,14 +27,14 @@ extern "C" void amrex_fi_fluxregister_fineadd_1fab_1dir (FluxRegister* flux_reg, const Real* fabdata, const int* flo, const int* fhi, int dir, int boxno, int zeroFirst, int nfluxes, Real scale) { Box bx; - bx = Box(IntVect(flo), IntVect(fhi)); - bx.shiftHalf(dir,-1); + bx = Box(IntVect(flo), IntVect(fhi)); + bx.shiftHalf(dir,-1); - BL_ASSERT(flux_reg->nComp() == nfluxes); - if (zeroFirst) - flux_reg->FineSetVal(dir, boxno, 0, flux_reg->nComp(), 0.0, RunOn::Cpu); + BL_ASSERT(flux_reg->nComp() == nfluxes); + if (zeroFirst) + flux_reg->FineSetVal(dir, boxno, 0, flux_reg->nComp(), 0.0, RunOn::Cpu); const FArrayBox fab(bx, nfluxes, const_cast(fabdata)); - flux_reg->FineAdd(fab, dir, boxno, 0, 0, flux_reg->nComp(), scale, RunOn::Cpu); + flux_reg->FineAdd(fab, dir, boxno, 0, 0, flux_reg->nComp(), scale, RunOn::Cpu); } void amrex_fi_fluxregister_crseinit (FluxRegister* flux_reg, MultiFab* flxs[], Real scale) @@ -59,7 +59,7 @@ extern "C" flux_reg->setVal(val); } - void amrex_fi_fluxregister_reflux (FluxRegister* flux_reg, MultiFab* mf, Real scale, + void amrex_fi_fluxregister_reflux (FluxRegister* flux_reg, MultiFab* mf, Real scale, const Geometry* geom) { MultiFab vol; diff --git a/Src/F_Interfaces/AmrCore/AMReX_fluxregister_mod.F90 b/Src/F_Interfaces/AmrCore/AMReX_fluxregister_mod.F90 index a54b18a68fd..3e40c9cd517 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_fluxregister_mod.F90 +++ b/Src/F_Interfaces/AmrCore/AMReX_fluxregister_mod.F90 @@ -3,7 +3,7 @@ module amrex_fluxregister_module use iso_c_binding use amrex_base_module - + implicit none private @@ -81,7 +81,7 @@ subroutine amrex_fi_fluxregister_crseadd (fr, flxs, scale, geom) bind(c) end subroutine amrex_fi_fluxregister_crseadd subroutine amrex_fi_fluxregister_setval (fr, val) bind(c) - import + import implicit none type(c_ptr), value :: fr real(amrex_real), value :: val @@ -114,7 +114,7 @@ subroutine amrex_fluxregister_build (fr, ba, dm, ref_ratio, fine_lev, ncomp) fr%flev = fine_lev call amrex_fi_new_fluxregister(fr%p, ba%p, dm%p, ref_ratio, fine_lev, ncomp) end subroutine amrex_fluxregister_build - + impure elemental subroutine amrex_fluxregister_destroy (this) type(amrex_fluxregister), intent(inout) :: this if (this%owner) then @@ -125,7 +125,7 @@ impure elemental subroutine amrex_fluxregister_destroy (this) this%owner = .false. this%p = c_null_ptr end subroutine amrex_fluxregister_destroy - + subroutine amrex_fluxregister_assign (dst, src) class(amrex_fluxregister), intent(inout) :: dst type (amrex_fluxregister), intent(in ) :: src @@ -159,7 +159,7 @@ subroutine amrex_fluxregister_fineadd_1fab (this, fluxfabs, gridIdx, scale, zero do dir = 1, amrex_spacedim associate(fab => fluxfabs(dir) ) flo(1:amrex_spacedim) = fab%bx%lo(1:amrex_spacedim) - fhi(1:amrex_spacedim) = fab%bx%hi(1:amrex_spacedim) + fhi(1:amrex_spacedim) = fab%bx%hi(1:amrex_spacedim) fp => fab%dataPtr() cp = c_loc(fp(flo(1),flo(2),flo(3),1)) nc = fab%nc diff --git a/Src/F_Interfaces/AmrCore/AMReX_tagbox_fi.cpp b/Src/F_Interfaces/AmrCore/AMReX_tagbox_fi.cpp index 8e801303819..e6917a42409 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_tagbox_fi.cpp +++ b/Src/F_Interfaces/AmrCore/AMReX_tagbox_fi.cpp @@ -7,14 +7,14 @@ extern "C" { void amrex_fi_tagboxarray_dataptr (TagBoxArray* tag, MFIter* mfi, char*& dp, int lo[3], int hi[3]) { - TagBox& fab = (*tag)[*mfi]; - dp = fab.dataPtr(); - const Box& bx = fab.box(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - } + TagBox& fab = (*tag)[*mfi]; + dp = fab.dataPtr(); + const Box& bx = fab.box(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + } } } diff --git a/Src/F_Interfaces/AmrCore/AMReX_tagbox_mod.F90 b/Src/F_Interfaces/AmrCore/AMReX_tagbox_mod.F90 index 490d9595fe7..1ff0cb60b84 100644 --- a/Src/F_Interfaces/AmrCore/AMReX_tagbox_mod.F90 +++ b/Src/F_Interfaces/AmrCore/AMReX_tagbox_mod.F90 @@ -55,5 +55,5 @@ function amrex_tagboxarray_dataPtr (this, mfi) result(dp) call c_f_pointer(cp, fp, shape=n) dp(bx%lo(1):,bx%lo(2):,bx%lo(3):,1:) => fp end function amrex_tagboxarray_dataPtr - + end module amrex_tagbox_module diff --git a/Src/F_Interfaces/Base/AMReX_FPhysBC.cpp b/Src/F_Interfaces/Base/AMReX_FPhysBC.cpp index 950970d4bad..062737f9008 100644 --- a/Src/F_Interfaces/Base/AMReX_FPhysBC.cpp +++ b/Src/F_Interfaces/Base/AMReX_FPhysBC.cpp @@ -7,8 +7,8 @@ amrex::FPhysBC::operator() (MultiFab& mf, int scomp, int ncomp, IntVect const& / Real time, int /*bccomp*/) { if (fill_physbc != nullptr) { - fill_physbc(&mf, scomp+1, ncomp, time, geom); + fill_physbc(&mf, scomp+1, ncomp, time, geom); } else { - amrex::Abort("FPhysBC::fill_physbc is null"); + amrex::Abort("FPhysBC::fill_physbc is null"); } } diff --git a/Src/F_Interfaces/Base/AMReX_box_fi.cpp b/Src/F_Interfaces/Base/AMReX_box_fi.cpp index 61a69f508f9..700e76c59e9 100644 --- a/Src/F_Interfaces/Base/AMReX_box_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_box_fi.cpp @@ -8,7 +8,7 @@ extern "C" { void amrex_fi_print_box (const int* lo, const int* hi, const int* nodal) { - Box box {IntVect(lo), IntVect(hi), IntVect(nodal)}; - AllPrint() << box << "\n"; + Box box {IntVect(lo), IntVect(hi), IntVect(nodal)}; + AllPrint() << box << "\n"; } } diff --git a/Src/F_Interfaces/Base/AMReX_box_mod.F90 b/Src/F_Interfaces/Base/AMReX_box_mod.F90 index fe3ab2497c8..1bb5d88f503 100644 --- a/Src/F_Interfaces/Base/AMReX_box_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_box_mod.F90 @@ -180,7 +180,7 @@ function amrex_box_intersects_fp (this, p) result(r) phi = ubound(p) seclo = max(this%lo, plo(1:3)) sechi = min(this%hi, phi(1:3)) - r = all(seclo(1:ndims) .le. sechi(1:ndims)) + r = all(seclo(1:ndims) .le. sechi(1:ndims)) end function amrex_box_intersects_fp function amrex_box_contains_box (this, bx) result(r) diff --git a/Src/F_Interfaces/Base/AMReX_boxarray_fi.cpp b/Src/F_Interfaces/Base/AMReX_boxarray_fi.cpp index 2895a9667c9..248ae076b6f 100644 --- a/Src/F_Interfaces/Base/AMReX_boxarray_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_boxarray_fi.cpp @@ -9,11 +9,11 @@ extern "C" { void amrex_fi_new_boxarray (BoxArray*& ba, int lo[3], int hi[3]) { - IntVect small(lo), big(hi); - ba = new BoxArray(Box(small,big)); + IntVect small(lo), big(hi); + ba = new BoxArray(Box(small,big)); } - void amrex_fi_new_boxarray_from_bxfarr (BoxArray*& ba, const int* bxs, + void amrex_fi_new_boxarray_from_bxfarr (BoxArray*& ba, const int* bxs, const int nsides, const int ndims, const int nbxs) { amrex::ignore_unused(nsides); @@ -27,18 +27,18 @@ extern "C" { bl.push_back(Box{lo,hi}); bxs += 2*ndims; } - ba = new BoxArray(bl); + ba = new BoxArray(bl); } void amrex_fi_delete_boxarray (BoxArray* ba) { - delete ba; + delete ba; } void amrex_fi_clone_boxarray (BoxArray*& bao, const BoxArray* bai) { - delete bao; - bao = new BoxArray(*bai); + delete bao; + bao = new BoxArray(*bai); } void amrex_fi_boxarray_maxsize (BoxArray* ba, int sz[]) @@ -51,16 +51,16 @@ extern "C" { { const Box& bx = (*ba)[i]; const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - for (int idim = 0; idim < BL_SPACEDIM; ++idim) { - lo[idim] = lov[idim]; - hi[idim] = hiv[idim]; - } + const int* hiv = bx.hiVect(); + for (int idim = 0; idim < BL_SPACEDIM; ++idim) { + lo[idim] = lov[idim]; + hi[idim] = hiv[idim]; + } } void amrex_fi_print_boxarray (const BoxArray* ba) { - AllPrint() << *ba; + AllPrint() << *ba; } void amrex_fi_boxarray_nodal_type (const BoxArray* ba, int inodal[3]) diff --git a/Src/F_Interfaces/Base/AMReX_distromap_fi.cpp b/Src/F_Interfaces/Base/AMReX_distromap_fi.cpp index 2d10dcb5c36..66d744cde5c 100644 --- a/Src/F_Interfaces/Base/AMReX_distromap_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_distromap_fi.cpp @@ -8,7 +8,7 @@ extern "C" { void amrex_fi_new_distromap (DistributionMapping*& dm, const BoxArray* ba) { - dm = new DistributionMapping(*ba); + dm = new DistributionMapping(*ba); } void amrex_fi_new_distromap_from_pmap (DistributionMapping*& dm, const int* pmap, const int plen) @@ -19,25 +19,25 @@ extern "C" { void amrex_fi_delete_distromap (DistributionMapping* dm) { - delete dm; + delete dm; } void amrex_fi_clone_distromap (DistributionMapping*& dmo, const DistributionMapping* dmi) { - delete dmo; - dmo = new DistributionMapping(*dmi); + delete dmo; + dmo = new DistributionMapping(*dmi); } void amrex_fi_distromap_get_pmap (const DistributionMapping* dm, int* pmap, const int plen) { - Long dmsize = dm->size(); - AMREX_ASSERT(plen >= dmsize); - for (int i = 0; i < dmsize && i < plen; ++i) + Long dmsize = dm->size(); + AMREX_ASSERT(plen >= dmsize); + for (int i = 0; i < dmsize && i < plen; ++i) pmap[i] = (*dm)[i]; } void amrex_fi_print_distromap (const DistributionMapping* dm) { - AllPrint() << *dm; + AllPrint() << *dm; } } diff --git a/Src/F_Interfaces/Base/AMReX_distromap_mod.F90 b/Src/F_Interfaces/Base/AMReX_distromap_mod.F90 index 259bb3fc086..26316ffab21 100644 --- a/Src/F_Interfaces/Base/AMReX_distromap_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_distromap_mod.F90 @@ -19,7 +19,7 @@ module amrex_distromap_module procedure :: move => amrex_distromap_move ! transfer ownership procedure :: get_pmap => amrex_distromap_get_pmap ! fill caller-owned array of PEs procedure, private :: amrex_distromap_assign - procedure, private :: amrex_distromap_install + procedure, private :: amrex_distromap_install #if !defined(__GFORTRAN__) || (__GNUC__ > 4) final :: amrex_distromap_destroy #endif diff --git a/Src/F_Interfaces/Base/AMReX_geometry_fi.cpp b/Src/F_Interfaces/Base/AMReX_geometry_fi.cpp index 112fb1c6c85..d50f22c2220 100644 --- a/Src/F_Interfaces/Base/AMReX_geometry_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_geometry_fi.cpp @@ -6,38 +6,38 @@ extern "C" { void amrex_fi_new_geometry (Geometry*& geom, int lo[3], int hi[3]) { - const Box domain{IntVect{AMREX_D_DECL(lo[0],lo[1],lo[2])}, + const Box domain{IntVect{AMREX_D_DECL(lo[0],lo[1],lo[2])}, IntVect{AMREX_D_DECL(hi[0],hi[1],hi[2])}}; - geom = new Geometry(domain); + geom = new Geometry(domain); } void amrex_fi_delete_geometry (Geometry* geom) { - delete geom; + delete geom; } void amrex_fi_geometry_get_pmask (int is_per[3]) { Geometry* gg = AMReX::top()->getDefaultGeometry(); - for (int i = 0; i < BL_SPACEDIM; ++i) - is_per[i] = gg->isPeriodic(i); + for (int i = 0; i < BL_SPACEDIM; ++i) + is_per[i] = gg->isPeriodic(i); } void amrex_fi_geometry_get_probdomain (Real problo[3], Real probhi[3]) { Geometry* gg = AMReX::top()->getDefaultGeometry(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - problo[i] = gg->ProbLo(i); - probhi[i] = gg->ProbHi(i); - } + for (int i = 0; i < BL_SPACEDIM; ++i) { + problo[i] = gg->ProbLo(i); + probhi[i] = gg->ProbHi(i); + } } void amrex_fi_geometry_get_intdomain (const Geometry* geom, int lo[3], int hi[3]) { - const Box& bx = geom->Domain(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = bx.smallEnd(i); - hi[i] = bx.bigEnd(i); - } + const Box& bx = geom->Domain(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = bx.smallEnd(i); + hi[i] = bx.bigEnd(i); + } } } diff --git a/Src/F_Interfaces/Base/AMReX_geometry_mod.F90 b/Src/F_Interfaces/Base/AMReX_geometry_mod.F90 index 1ae3d0b074a..d65a8f6d980 100644 --- a/Src/F_Interfaces/Base/AMReX_geometry_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_geometry_mod.F90 @@ -16,7 +16,7 @@ module amrex_geometry_module public :: amrex_is_periodic, amrex_is_any_periodic, amrex_is_all_periodic public :: amrex_geometry_finalize - logical, save :: amrex_pmask(3) = .false. + logical, save :: amrex_pmask(3) = .false. real(amrex_real), save :: amrex_problo(3) = 0.0_amrex_real real(amrex_real), save :: amrex_probhi(3) = 1.0_amrex_real @@ -198,5 +198,5 @@ function amrex_geometry_get_ploc (this, iloc) result (ploc) ploc(i) = amrex_problo(i) + (iloc(i) - this%domain%lo(i)) * this%dx(i) end do end function amrex_geometry_get_ploc - + end module amrex_geometry_module diff --git a/Src/F_Interfaces/Base/AMReX_init_mod.F90 b/Src/F_Interfaces/Base/AMReX_init_mod.F90 index ccd66c68769..2aca3c1157e 100644 --- a/Src/F_Interfaces/Base/AMReX_init_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_init_mod.F90 @@ -60,7 +60,7 @@ end subroutine amrex_fi_init call amrex_fi_init(cmd_string%data, amrex_parallel_communicator(), l_arg_parmparse, cfp) initialized = .true. end subroutine amrex_init - + subroutine amrex_finalize () use amrex_geometry_module, only: amrex_geometry_finalize interface diff --git a/Src/F_Interfaces/Base/AMReX_multifab_fi.cpp b/Src/F_Interfaces/Base/AMReX_multifab_fi.cpp index 3bb4c3832a2..09382041345 100644 --- a/Src/F_Interfaces/Base/AMReX_multifab_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_multifab_fi.cpp @@ -7,13 +7,13 @@ using namespace amrex; extern "C" { - void amrex_fi_new_multifab (MultiFab*& mf, const BoxArray*& ba, - const DistributionMapping*& dm, - int nc, const int* ng, const int* nodal) + void amrex_fi_new_multifab (MultiFab*& mf, const BoxArray*& ba, + const DistributionMapping*& dm, + int nc, const int* ng, const int* nodal) { - mf = new MultiFab(amrex::convert(*ba, IntVect(nodal)), *dm, nc, IntVect(ng)); - ba = &(mf->boxArray()); - dm = &(mf->DistributionMap()); + mf = new MultiFab(amrex::convert(*ba, IntVect(nodal)), *dm, nc, IntVect(ng)); + ba = &(mf->boxArray()); + dm = &(mf->DistributionMap()); } void amrex_fi_new_multifab_alias (MultiFab*& mf, const MultiFab* srcmf, int comp, int ncomp) @@ -23,12 +23,12 @@ extern "C" { void amrex_fi_delete_multifab (MultiFab* mf) { - delete mf; + delete mf; } int amrex_fi_multifab_ncomp (const MultiFab* mf) { - return mf->nComp(); + return mf->nComp(); } void amrex_fi_multifab_ngrow (const MultiFab* mf, int* ngv) @@ -39,68 +39,68 @@ extern "C" { const BoxArray* amrex_fi_multifab_boxarray (const MultiFab* mf) { - return &(mf->boxArray()); + return &(mf->boxArray()); } const DistributionMapping* amrex_fi_multifab_distromap (const MultiFab* mf) { - return &(mf->DistributionMap()); + return &(mf->DistributionMap()); } - + void amrex_fi_multifab_dataptr_iter (MultiFab* mf, MFIter* mfi, Real*& dp, int lo[3], int hi[3]) { - FArrayBox& fab = (*mf)[*mfi]; - dp = fab.dataPtr(); - const Box& bx = fab.box(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - } + FArrayBox& fab = (*mf)[*mfi]; + dp = fab.dataPtr(); + const Box& bx = fab.box(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + } } void amrex_fi_multifab_dataptr_int (MultiFab* mf, int igrd, Real*& dp, int lo[3], int hi[3]) { - FArrayBox& fab = (*mf)[igrd]; - dp = fab.dataPtr(); - const Box& bx = fab.box(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - } + FArrayBox& fab = (*mf)[igrd]; + dp = fab.dataPtr(); + const Box& bx = fab.box(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + } } Real amrex_fi_multifab_min (const MultiFab* mf, int comp, int nghost) { - return mf->min(comp,nghost); + return mf->min(comp,nghost); } Real amrex_fi_multifab_max (const MultiFab* mf, int comp, int nghost) { - return mf->max(comp,nghost); + return mf->max(comp,nghost); } Real amrex_fi_multifab_sum (const MultiFab* mf, int comp) { - return mf->sum(comp); + return mf->sum(comp); } Real amrex_fi_multifab_norm0 (const MultiFab* mf, int comp) { - return mf->norm0(comp); + return mf->norm0(comp); } Real amrex_fi_multifab_norm1 (const MultiFab* mf, int comp) { - return mf->norm1(comp); + return mf->norm1(comp); } Real amrex_fi_multifab_norm2 (const MultiFab* mf, int comp) { - return mf->norm2(comp); + return mf->norm2(comp); } void amrex_fi_multifab_setval (MultiFab* mf, Real val, int ic, int nc, const int* ng) @@ -178,10 +178,10 @@ extern "C" { dstmf->ParallelCopy(*srcmf,srccomp,dstcomp,nc,sg,dg,geom->periodicity()); } - void amrex_fi_multifab_fill_boundary (MultiFab* mf, const Geometry* geom, - int c, int nc, int cross) + void amrex_fi_multifab_fill_boundary (MultiFab* mf, const Geometry* geom, + int c, int nc, int cross) { - mf->FillBoundary(c, nc, geom->periodicity(), cross); + mf->FillBoundary(c, nc, geom->periodicity(), cross); } void amrex_fi_build_owner_imultifab (iMultiFab*& msk, const BoxArray*& ba, @@ -216,13 +216,13 @@ extern "C" { // iMultiFab - void amrex_fi_new_imultifab (iMultiFab*& imf, const BoxArray*& ba, - const DistributionMapping*& dm, - int nc, const int* ng, const int* nodal) + void amrex_fi_new_imultifab (iMultiFab*& imf, const BoxArray*& ba, + const DistributionMapping*& dm, + int nc, const int* ng, const int* nodal) { - imf = new iMultiFab(amrex::convert(*ba, IntVect(nodal)), *dm, nc, IntVect(ng)); - ba = &(imf->boxArray()); - dm = &(imf->DistributionMap()); + imf = new iMultiFab(amrex::convert(*ba, IntVect(nodal)), *dm, nc, IntVect(ng)); + ba = &(imf->boxArray()); + dm = &(imf->DistributionMap()); } void amrex_fi_new_imultifab_alias (iMultiFab*& mf, const iMultiFab* srcmf, int comp, int ncomp) @@ -232,20 +232,20 @@ extern "C" { void amrex_fi_delete_imultifab (iMultiFab* imf) { - delete imf; + delete imf; } void amrex_fi_imultifab_dataptr (iMultiFab* imf, MFIter* mfi, int*& dp, int lo[3], int hi[3]) { - IArrayBox& fab = (*imf)[*mfi]; - dp = fab.dataPtr(); - const Box& bx = fab.box(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - } + IArrayBox& fab = (*imf)[*mfi]; + dp = fab.dataPtr(); + const Box& bx = fab.box(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + } } void amrex_fi_imultifab_setval (iMultiFab* imf, int val, int ic, int nc, const int* ng) @@ -304,117 +304,117 @@ extern "C" { void amrex_fi_delete_mfiter (MFIter* mfi) { - delete mfi; + delete mfi; } void amrex_fi_increment_mfiter (MFIter* mfi, int* isvalid) { - ++(*mfi); - *isvalid = mfi->isValid(); + ++(*mfi); + *isvalid = mfi->isValid(); } void amrex_fi_mfiter_is_valid (MFIter* mfi, int* isvalid) { - *isvalid = mfi->isValid(); + *isvalid = mfi->isValid(); } int amrex_fi_mfiter_grid_index (MFIter* mfi) { - return mfi->index(); + return mfi->index(); } int amrex_fi_mfiter_local_tile_index (MFIter* mfi) { - return mfi->LocalTileIndex(); + return mfi->LocalTileIndex(); } void amrex_fi_mfiter_tilebox (MFIter* mfi, int lo[3], int hi[3], int nodal[3]) { - const Box& bx = mfi->tilebox(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - const IntVect& t = bx.type(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - nodal[i] = t[i]; - } + const Box& bx = mfi->tilebox(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + const IntVect& t = bx.type(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + nodal[i] = t[i]; + } } void amrex_fi_mfiter_tilebox_iv (MFIter* mfi, int lo[3], int hi[3], const int nodal[3]) { const Box& bx = mfi->tilebox(IntVect(nodal)); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - } + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + } } void amrex_fi_mfiter_nodaltilebox (MFIter* mfi, int dir, int lo[3], int hi[3], int nodal[3]) { - const Box& bx = mfi->nodaltilebox(dir); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - const IntVect& t = bx.type(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - nodal[i] = t[i]; - } + const Box& bx = mfi->nodaltilebox(dir); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + const IntVect& t = bx.type(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + nodal[i] = t[i]; + } } void amrex_fi_mfiter_growntilebox (MFIter* mfi, int lo[3], int hi[3], int ng, int nodal[3]) { - const Box& bx = mfi->growntilebox(ng); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - const IntVect& t = bx.type(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - nodal[i] = t[i]; - } + const Box& bx = mfi->growntilebox(ng); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + const IntVect& t = bx.type(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + nodal[i] = t[i]; + } } void amrex_fi_mfiter_grownnodaltilebox (MFIter* mfi, int lo[3], int hi[3], int dir, int ng, int nodal[3]) { - const Box& bx = mfi->grownnodaltilebox(dir, ng); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - const IntVect& t = bx.type(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - nodal[i] = t[i]; - } + const Box& bx = mfi->grownnodaltilebox(dir, ng); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + const IntVect& t = bx.type(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + nodal[i] = t[i]; + } } void amrex_fi_mfiter_validbox (MFIter* mfi, int lo[3], int hi[3], int nodal[3]) { - const Box& bx = mfi->validbox(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - const IntVect& t = bx.type(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - nodal[i] = t[i]; - } + const Box& bx = mfi->validbox(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + const IntVect& t = bx.type(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + nodal[i] = t[i]; + } } void amrex_fi_mfiter_fabbox (MFIter* mfi, int lo[3], int hi[3], int nodal[3]) { - const Box& bx = mfi->fabbox(); - const int* lov = bx.loVect(); - const int* hiv = bx.hiVect(); - const IntVect& t = bx.type(); - for (int i = 0; i < BL_SPACEDIM; ++i) { - lo[i] = lov[i]; - hi[i] = hiv[i]; - nodal[i] = t[i]; - } + const Box& bx = mfi->fabbox(); + const int* lov = bx.loVect(); + const int* hiv = bx.hiVect(); + const IntVect& t = bx.type(); + for (int i = 0; i < BL_SPACEDIM; ++i) { + lo[i] = lov[i]; + hi[i] = hiv[i]; + nodal[i] = t[i]; + } } } diff --git a/Src/F_Interfaces/Base/AMReX_multifab_mod.F90 b/Src/F_Interfaces/Base/AMReX_multifab_mod.F90 index c7d82825788..81ce03b9f3d 100644 --- a/Src/F_Interfaces/Base/AMReX_multifab_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_multifab_mod.F90 @@ -131,7 +131,7 @@ module amrex_multifab_module type, public :: amrex_mfiter type(c_ptr) :: p = c_null_ptr - integer ,private :: counter = -1 + integer ,private :: counter = -1 contains generic :: assignment(=) => amrex_mfiter_assign ! will abort if called procedure :: clear => amrex_mfiter_clear @@ -161,7 +161,7 @@ module amrex_multifab_module ! interfaces to c++ functions - interface + interface subroutine amrex_fi_new_multifab (mf,ba,dm,nc,ng,nodal) bind(c) import implicit none @@ -169,7 +169,7 @@ subroutine amrex_fi_new_multifab (mf,ba,dm,nc,ng,nodal) bind(c) integer(c_int), value :: nc integer(c_int), intent(in) :: ng(3), nodal(3) end subroutine amrex_fi_new_multifab - + subroutine amrex_fi_new_multifab_alias (mf, srcmf, comp, ncomp) bind(c) import implicit none @@ -627,7 +627,7 @@ subroutine amrex_multifab_build_a (mf, ba, dm, nc, ng, nodal) mf%owner = .true. mf%nc = nc mf%ng(1:ndims) = ng(1:ndims) - inodal = 0 + inodal = 0 if (present(nodal)) then do dir = 1, ndims if (nodal(dir)) inodal(dir) = 1 @@ -1032,7 +1032,7 @@ subroutine amrex_multifab_fill_boundary_c (this, geom, c, nc, cross) integer, intent(in) :: c, nc logical, intent(in), optional :: cross integer :: lcross - lcross = 0 + lcross = 0 if (present(cross)) then if (cross) then lcross = 1 @@ -1420,7 +1420,7 @@ logical function amrex_mfiter_next (this) this%counter = this%counter + 1 if (this%counter == 1) then call amrex_fi_mfiter_is_valid(this%p, isvalid) - else + else call amrex_fi_increment_mfiter(this%p, isvalid) end if if (isvalid .eq. 1) then diff --git a/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp b/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp index 09dd3f47d2a..93bf211dc33 100644 --- a/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_multifabutil_fi.cpp @@ -5,10 +5,10 @@ using namespace amrex; extern "C" { void amrex_fi_average_down (const MultiFab* S_fine, MultiFab* S_crse, - const Geometry* fgeom, const Geometry* cgeom, - int scomp, int ncomp, int rr) + const Geometry* fgeom, const Geometry* cgeom, + int scomp, int ncomp, int rr) { - amrex::average_down(*S_fine, *S_crse, *fgeom, *cgeom, scomp, ncomp, rr); + amrex::average_down(*S_fine, *S_crse, *fgeom, *cgeom, scomp, ncomp, rr); } void amrex_fi_average_cellcenter_to_face (MultiFab* fc[], const MultiFab* cc, const Geometry* geom) diff --git a/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 b/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 index 1305369a0c1..4bf1114dc5d 100644 --- a/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_parallel_mod.F90 @@ -23,7 +23,7 @@ module amrex_parallel_module public :: amrex_parallel_init public :: amrex_parallel_finalize public :: amrex_parallel_communicator - public :: amrex_parallel_myproc + public :: amrex_parallel_myproc public :: amrex_parallel_ioprocessor public :: amrex_parallel_nprocs public :: amrex_parallel_reduce_sum @@ -101,7 +101,7 @@ subroutine amrex_parallel_init (comm) else call amrex_abort("amrex_parallel_init: size of amrex_real is unknown") end if -#endif +#endif end subroutine amrex_parallel_init subroutine amrex_parallel_finalize () diff --git a/Src/F_Interfaces/Base/AMReX_physbc_fi.cpp b/Src/F_Interfaces/Base/AMReX_physbc_fi.cpp index 712f0421cad..6f054a11d5b 100644 --- a/Src/F_Interfaces/Base/AMReX_physbc_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_physbc_fi.cpp @@ -6,11 +6,11 @@ extern "C" { void amrex_fi_new_physbc (FPhysBC*& pbc, FPhysBC::fill_physbc_funptr_t fill, const Geometry* geom) { - pbc = new FPhysBC(fill, geom); + pbc = new FPhysBC(fill, geom); } void amrex_fi_delete_physbc (FPhysBC* pbc) { - delete pbc; + delete pbc; } } diff --git a/Src/F_Interfaces/Base/AMReX_physbc_mod.F90 b/Src/F_Interfaces/Base/AMReX_physbc_mod.F90 index d8692ed88ac..6831fe81514 100644 --- a/Src/F_Interfaces/Base/AMReX_physbc_mod.F90 +++ b/Src/F_Interfaces/Base/AMReX_physbc_mod.F90 @@ -67,7 +67,7 @@ impure elemental subroutine amrex_physbc_destroy (this) this%owner = .false. this%p = c_null_ptr end subroutine amrex_physbc_destroy - + subroutine amrex_physbc_assign (dst, src) class(amrex_physbc), intent(inout) :: dst type (amrex_physbc), intent(in ) :: src diff --git a/Src/F_Interfaces/Base/AMReX_plotfile_fi.cpp b/Src/F_Interfaces/Base/AMReX_plotfile_fi.cpp index 13b4e70f9bf..c569eef6077 100644 --- a/Src/F_Interfaces/Base/AMReX_plotfile_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_plotfile_fi.cpp @@ -6,21 +6,21 @@ using namespace amrex; extern "C" { void amrex_fi_write_plotfile (const char* name, int nlevs, const MultiFab* mf[], - const char* varname[], const Geometry* geom[], - Real time, const int level_steps[], const int ref_ratio[]) + const char* varname[], const Geometry* geom[], + Real time, const int level_steps[], const int ref_ratio[]) { - Vector mfarr {mf, mf+nlevs}; - Vector varnamearr {varname, varname+mf[0]->nComp()}; - Vector geomarr; - for (int lev = 0; lev < nlevs; ++lev) { - geomarr.emplace_back(*geom[lev]); - } - Vector lsarr {level_steps, level_steps+nlevs}; - Vector rrarr; - for (int lev = 0; lev < nlevs-1; ++lev) { - rrarr.emplace_back(AMREX_D_DECL(ref_ratio[lev],ref_ratio[lev],ref_ratio[lev])); - } - amrex::WriteMultiLevelPlotfile(name, nlevs, mfarr, varnamearr, geomarr, - time, lsarr, rrarr); + Vector mfarr {mf, mf+nlevs}; + Vector varnamearr {varname, varname+mf[0]->nComp()}; + Vector geomarr; + for (int lev = 0; lev < nlevs; ++lev) { + geomarr.emplace_back(*geom[lev]); + } + Vector lsarr {level_steps, level_steps+nlevs}; + Vector rrarr; + for (int lev = 0; lev < nlevs-1; ++lev) { + rrarr.emplace_back(AMREX_D_DECL(ref_ratio[lev],ref_ratio[lev],ref_ratio[lev])); + } + amrex::WriteMultiLevelPlotfile(name, nlevs, mfarr, varnamearr, geomarr, + time, lsarr, rrarr); } } diff --git a/Src/F_Interfaces/Base/AMReX_vismf_fi.cpp b/Src/F_Interfaces/Base/AMReX_vismf_fi.cpp index d2a957af817..f5e6ef05ba4 100644 --- a/Src/F_Interfaces/Base/AMReX_vismf_fi.cpp +++ b/Src/F_Interfaces/Base/AMReX_vismf_fi.cpp @@ -8,12 +8,12 @@ extern "C" { void amrex_fi_write_multifab (const MultiFab* mf, const char* name) { - VisMF::Write(*mf, std::string(name)); + VisMF::Write(*mf, std::string(name)); } void amrex_fi_read_multifab (MultiFab* mf, const char* name) { - BL_ASSERT(mf != nullptr); - VisMF::Read(*mf, std::string(name)); + BL_ASSERT(mf != nullptr); + VisMF::Read(*mf, std::string(name)); } } diff --git a/Src/F_Interfaces/LinearSolvers/AMReX_linop_mod.F90 b/Src/F_Interfaces/LinearSolvers/AMReX_linop_mod.F90 index c0ff184ca1f..00e31a60742 100644 --- a/Src/F_Interfaces/LinearSolvers/AMReX_linop_mod.F90 +++ b/Src/F_Interfaces/LinearSolvers/AMReX_linop_mod.F90 @@ -81,5 +81,5 @@ subroutine amrex_linop_set_level_bc (this, lev, bcdata) type(amrex_multifab), intent(in) :: bcdata call amrex_fi_linop_set_level_bc(this%p, lev, bcdata%p) end subroutine amrex_linop_set_level_bc - + end module amrex_linop_module diff --git a/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_fi.cpp b/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_fi.cpp index bff278111a2..57186660a61 100644 --- a/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_fi.cpp +++ b/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_fi.cpp @@ -100,12 +100,12 @@ { mlmg->setBottomVerbose(n); } - + void amrex_fi_multigrid_set_always_use_bnorm (MLMG* mlmg, int f) { mlmg->setAlwaysUseBNorm(f); } - + void amrex_fi_multigrid_set_final_fill_bc (MLMG* mlmg, int f) { mlmg->setFinalFillBC(f); diff --git a/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_mod.F90 b/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_mod.F90 index 65d426ae474..bc7ae734d01 100644 --- a/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_mod.F90 +++ b/Src/F_Interfaces/LinearSolvers/AMReX_multigrid_mod.F90 @@ -60,7 +60,7 @@ subroutine amrex_fi_new_multigrid (mg, linop) bind(c) type(c_ptr), intent(inout) :: mg type(c_ptr), value :: linop end subroutine amrex_fi_new_multigrid - + subroutine amrex_fi_delete_multigrid (mg) bind(c) import implicit none @@ -247,7 +247,7 @@ subroutine amrex_multigrid_comp_residual (mg, res, sol, rhs) call amrex_fi_multigrid_comp_residual(mg%p, pres, psol, prhs) end subroutine amrex_multigrid_comp_residual - + subroutine amrex_multigrid_set_verbose (mg, v) class(amrex_multigrid), intent(inout) :: mg integer, intent(in) :: v diff --git a/Src/F_Interfaces/LinearSolvers/AMReX_poisson_mod.F90 b/Src/F_Interfaces/LinearSolvers/AMReX_poisson_mod.F90 index 76a94046219..3690419f22d 100644 --- a/Src/F_Interfaces/LinearSolvers/AMReX_poisson_mod.F90 +++ b/Src/F_Interfaces/LinearSolvers/AMReX_poisson_mod.F90 @@ -18,7 +18,7 @@ module amrex_poisson_module end type amrex_poisson ! interfaces to C++ functions - + interface subroutine amrex_fi_new_poisson (poisson,n,gm,ba,dm,mt,agg,con,mcl) bind(c) import diff --git a/Src/F_Interfaces/Octree/AMReX_octree_fi.cpp b/Src/F_Interfaces/Octree/AMReX_octree_fi.cpp index 7ec953c85bb..7bc6d248090 100644 --- a/Src/F_Interfaces/Octree/AMReX_octree_fi.cpp +++ b/Src/F_Interfaces/Octree/AMReX_octree_fi.cpp @@ -173,7 +173,7 @@ extern "C" { for (int i = 0; i < n; ++i) { a_copy[i] = (*leaves)[i]; } - + delete leaves; } diff --git a/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp b/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp index 60f77c5a174..3e80eeb86ff 100644 --- a/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp +++ b/Src/F_Interfaces/Particle/AMReX_particlecontainer_fi.cpp @@ -18,12 +18,12 @@ extern "C" { void amrex_fi_new_particlecontainer (FParticleContainer*& particlecontainer, AmrCore* amrcore) { - particlecontainer = new FParticleContainer(amrcore); + particlecontainer = new FParticleContainer(amrcore); } void amrex_fi_delete_particlecontainer (FParticleContainer* particlecontainer) { - delete particlecontainer; + delete particlecontainer; } void amrex_fi_get_next_particle_id (Long& id) @@ -65,7 +65,7 @@ extern "C" { void amrex_fi_particle_redistribute (FParticleContainer* particlecontainer, int lev_min, int lev_max, int ng) { - particlecontainer->Redistribute(lev_min, lev_max, ng); + particlecontainer->Redistribute(lev_min, lev_max, ng); } void amrex_fi_get_particles_mfi(FParticleContainer* particlecontainer, diff --git a/Src/LinearSolvers/MLMG/AMReX_MLABecLap_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLABecLap_2D_K.H index 7f1306672f0..6b63e84949c 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLABecLap_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLABecLap_2D_K.H @@ -377,7 +377,7 @@ void abec_gsrb_with_line_solve ( phi(i,j,0,n) = u_ls(j-lo.y); } } - } + } } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLABecLap_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLABecLap_3D_K.H index 4e6ba8c418a..841fd79a044 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLABecLap_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLABecLap_3D_K.H @@ -418,21 +418,21 @@ void abec_gsrb_with_line_solve ( const auto vhi = amrex::ubound(vbox); // amrex::Print() << "GSRB LS " << box << " " << dhx << " " << dhy << " " << dhz << std::endl; - + // idir is the direction in which we will do the tridiagonal solve -- // it should be the direction in which the mesh spacing is much larger // than in the other directions int idir = 2; int ilen = hi.z - lo.z + 1; - + if ( (dhx <= dhy) && (dhz <= dhy) ) { - idir = 1; - ilen = hi.y - lo.y + 1; - } + idir = 1; + ilen = hi.y - lo.y + 1; + } if ( (dhy <= dhx) && (dhz <= dhx) ) { - idir = 0; - ilen = hi.x - lo.x + 1; - } + idir = 0; + ilen = hi.x - lo.x + 1; + } // This assertion should be moved outside the kernel for performance! if (ilen > 32) amrex::Abort("abec_gsrb_with_line_solve is hard-wired to be no longer than 32"); @@ -444,8 +444,8 @@ void abec_gsrb_with_line_solve ( Array1D u_ls; Array1D gam; - if (idir == 2) { - for (int n = 0; n < nc; ++n) { + if (idir == 2) { + for (int n = 0; n < nc; ++n) { for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { @@ -510,7 +510,7 @@ void abec_gsrb_with_line_solve ( } } - tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); + tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); for (int k = lo.z; k <= hi.z; ++k) { @@ -520,7 +520,7 @@ void abec_gsrb_with_line_solve ( } } } - } else if (idir == 1) { + } else if (idir == 1) { for (int n = 0; n < nc; ++n) { for (int i = lo.x; i <= hi.x; ++i) { AMREX_PRAGMA_SIMD @@ -585,7 +585,7 @@ void abec_gsrb_with_line_solve ( } } - tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); + tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); for (int j = lo.y; j <= hi.y; ++j) { @@ -594,7 +594,7 @@ void abec_gsrb_with_line_solve ( } } } - } + } } else if (idir == 0) { for (int n = 0; n < nc; ++n) { for (int j = lo.y; j <= hi.y; ++j) { @@ -660,8 +660,8 @@ void abec_gsrb_with_line_solve ( } } - tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); - + tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); + for (int i = lo.x; i <= hi.x; ++i) { phi(i,j,k,n) = u_ls(i-lo.x); @@ -669,8 +669,8 @@ void abec_gsrb_with_line_solve ( } } } - } - } + } + } } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE diff --git a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.cpp index 77e013fcaed..065e88da266 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLABecLaplacian.cpp @@ -11,7 +11,7 @@ MLABecLaplacian::MLABecLaplacian (const Vector& a_geom, const Vector& a_dmap, const LPInfo& a_info, const Vector const*>& a_factory, - const int a_ncomp) + const int a_ncomp) : m_ncomp(a_ncomp) { define(a_geom, a_grids, a_dmap, a_info, a_factory); @@ -123,7 +123,7 @@ MLABecLaplacian::setBCoeffs (int amrlev, MultiFab::Copy(m_b_coeffs[amrlev][0][idim], *beta[idim], icomp, icomp, 1, 0); } } - else + else for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { for (int icomp = 0; icomp < ncomp; ++icomp) { MultiFab::Copy(m_b_coeffs[amrlev][0][idim], *beta[idim], 0, icomp, 1, 0); @@ -187,7 +187,7 @@ MLABecLaplacian::averageDownCoeffsSameAmrLevel (int amrlev, Vector& a, { amrex::average_down(a[mglev-1], a[mglev], 0, 1, ratio); } - + Vector fine {AMREX_D_DECL(&(b[mglev-1][0]), &(b[mglev-1][1]), &(b[mglev-1][2]))}; @@ -291,7 +291,7 @@ MLABecLaplacian::prepareForSolve () for (int alev = 0; alev < m_num_amr_levels; ++alev) { // For now this assumes that overset regions are treated as Dirichlet bc's - if (m_domain_covered[alev] && !m_overset_mask[alev][0]) + if (m_domain_covered[alev] && !m_overset_mask[alev][0]) { if (m_a_scalar == 0.0) { @@ -449,7 +449,7 @@ MLABecLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& #endif for (MFIter mfi(sol,mfi_info); mfi.isValid(); ++mfi) { - const auto& m0 = mm0.array(mfi); + const auto& m0 = mm0.array(mfi); const auto& m1 = mm1.array(mfi); #if (AMREX_SPACEDIM > 1) const auto& m2 = mm2.array(mfi); @@ -460,7 +460,7 @@ MLABecLaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& #endif #endif - const Box& tbx = mfi.tilebox(); + const Box& tbx = mfi.tilebox(); const Box& vbx = mfi.validbox(); const auto& solnfab = sol.array(mfi); const auto& rhsfab = rhs.array(mfi); @@ -631,7 +631,7 @@ MLABecLaplacian::update () for (int alev = 0; alev < m_num_amr_levels; ++alev) { // For now this assumes that overset regions are treated as Dirichlet bc's - if (m_domain_covered[alev] && !m_overset_mask[alev][0]) + if (m_domain_covered[alev] && !m_overset_mask[alev][0]) { if (m_a_scalar == 0.0) { diff --git a/Src/LinearSolvers/MLMG/AMReX_MLALap_1D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLALap_1D_K.H index 6432ad82872..7dffd98ec49 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLALap_1D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLALap_1D_K.H @@ -159,11 +159,11 @@ void mlalap_gsrb (Box const& box, Array4 const& phi, ? f0(vlo.x,0,0) : Real(0.0); Real cf1 = (i == vhi.x && m1(vhi.x+1,0,0) > 0) ? f1(vhi.x,0,0) : Real(0.0); - + Real delta = dhx*(cf0+cf1); - + Real gamma = alpha*a(i,0,0) + dhx*Real(2.0); - + Real rho = dhx*(phi(i-1,0,0) + phi(i+1,0,0)); phi(i,0,0) = (rhs(i,0,0) + rho - phi(i,0,0)*delta) @@ -197,11 +197,11 @@ void mlalap_gsrb_m (Box const& box, Array4 const& phi, Real rel = (probxlo + i *dx) * (probxlo + i *dx); Real rer = (probxlo +(i+1)*dx) * (probxlo +(i+1)*dx); Real rc = (probxlo +(i+Real(.5))*dx) * (probxlo +(i+Real(.5))*dx); - + Real delta = dhx*(rel*cf0 + rer*cf1); - + Real gamma = alpha*a(i,0,0)*rc + dhx*(rel+rer); - + Real rho = dhx*(rel*phi(i-1,0,0) + rer*phi(i+1,0,0)); phi(i,0,0) = (rhs(i,0,0) + rho - phi(i,0,0)*delta) diff --git a/Src/LinearSolvers/MLMG/AMReX_MLALaplacian.cpp b/Src/LinearSolvers/MLMG/AMReX_MLALaplacian.cpp index 860caea3578..89bd1457e14 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLALaplacian.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLALaplacian.cpp @@ -104,7 +104,7 @@ MLALaplacian::averageDownCoeffsToCoarseAmrLevel (int flev) if (m_a_scalar != 0.0) { amrex::average_down(fine_a_coeffs, crse_a_coeffs, 0, 1, mg_coarsen_ratio); - } + } } void @@ -285,7 +285,7 @@ MLALaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs #endif for (MFIter mfi(sol,mfi_info); mfi.isValid(); ++mfi) { - const auto& m0 = mm0.array(mfi); + const auto& m0 = mm0.array(mfi); const auto& m1 = mm1.array(mfi); #if (AMREX_SPACEDIM > 1) const auto& m2 = mm2.array(mfi); @@ -296,7 +296,7 @@ MLALaplacian::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs #endif #endif - const Box& tbx = mfi.tilebox(); + const Box& tbx = mfi.tilebox(); const Box& vbx = mfi.validbox(); const auto& solnfab = sol.array(mfi); const auto& rhsfab = rhs.array(mfi); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H index 2040988cad8..45464bbeb9c 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.H @@ -36,9 +36,9 @@ public: * 2 means iterations exceeded */ int solve (MultiFab& solnL, - const MultiFab& rhsL, - Real eps_rel, - Real eps_abs); + const MultiFab& rhsL, + Real eps_rel, + Real eps_abs); void setVerbose (int _verbose) { verbose = _verbose; } int getVerbose () const { return verbose; } @@ -48,7 +48,7 @@ public: void setNGhost(int _nghost) {nghost = _nghost;} int getNGhost() {return nghost;} - + Real dotxy (const MultiFab& r, const MultiFab& z, bool local = false); Real norm_inf (const MultiFab& res, bool local = false); int solve_bicgstab (MultiFab& solnL, diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp index f9e51066990..c2c89378142 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLCGSolver.cpp @@ -109,10 +109,10 @@ MLCGSolver::solve_bicgstab (MultiFab& sol, // If singular remove mean from residual // if (Lp.isBottomSingular()) mlmg->makeSolvable(amrlev, mglev, r); - + // Then normalize Lp.normalize(amrlev, mglev, r); - + MultiFab::Copy(sorig,sol,0,0,ncomp,nghost); MultiFab::Copy(rh, r, 0,0,ncomp,nghost); @@ -132,21 +132,21 @@ MLCGSolver::solve_bicgstab (MultiFab& sol, if ( rnorm0 == 0 || rnorm0 < eps_abs ) { if ( verbose > 0 ) - { + { amrex::Print() << "MLCGSolver_BiCGStab: niter = 0," - << ", rnorm = " << rnorm + << ", rnorm = " << rnorm << ", eps_abs = " << eps_abs << std::endl; - } + } return ret; } for (; iter <= maxiter; ++iter) { const Real rho = dotxy(rh,r); - if ( rho == 0 ) - { + if ( rho == 0 ) + { ret = 1; break; - } + } if ( iter == 1 ) { MultiFab::Copy(p,r,0,0,ncomp,nghost); @@ -163,19 +163,19 @@ MLCGSolver::solve_bicgstab (MultiFab& sol, Real rhTv = dotxy(rh,v); if ( rhTv != Real(0.0) ) - { + { alpha = rho/rhTv; - } + } else - { + { ret = 2; break; - } + } sxay(sol, sol, alpha, ph, nghost); sxay(s, r, -alpha, v, nghost); - //Subtract mean from s + //Subtract mean from s // if (Lp.isBottomSingular()) mlmg->makeSolvable(amrlev, mglev, s); - + rnorm = norm_inf(s); if ( verbose > 2 && ParallelDescriptor::IOProcessor() ) @@ -203,13 +203,13 @@ MLCGSolver::solve_bicgstab (MultiFab& sol, BL_PROFILE_VAR_STOP(blp_par); if ( tvals[0] != Real(0.0) ) - { + { omega = tvals[1]/tvals[0]; - } + } else - { + { ret = 3; break; - } + } sxay(sol, sol, omega, sh, nghost); sxay(r, s, -omega, t, nghost); @@ -228,9 +228,9 @@ MLCGSolver::solve_bicgstab (MultiFab& sol, if ( rnorm < eps_rel*rnorm0 || rnorm < eps_abs ) break; if ( omega == 0 ) - { + { ret = 4; break; - } + } rho_1 = rho; } @@ -252,8 +252,8 @@ MLCGSolver::solve_bicgstab (MultiFab& sol, if ( ( ret == 0 || ret == 8 ) && (rnorm < rnorm0) ) { sol.plus(sorig, 0, ncomp, nghost); - } - else + } + else { sol.setVal(0); sol.plus(sorig, 0, ncomp, nghost); @@ -306,9 +306,9 @@ MLCGSolver::solve_cg (MultiFab& sol, { if ( verbose > 0 ) { amrex::Print() << "MLCGSolver_CG: niter = 0," - << ", rnorm = " << rnorm + << ", rnorm = " << rnorm << ", eps_abs = " << eps_abs << std::endl; - } + } return ret; } @@ -336,14 +336,14 @@ MLCGSolver::solve_cg (MultiFab& sol, Real alpha; Real pw = dotxy(p,q); if ( pw != Real(0.0)) - { + { alpha = rho/pw; - } + } else - { + { ret = 1; break; - } - + } + if ( verbose > 2 ) { amrex::Print() << "MLCGSolver_cg:" @@ -367,7 +367,7 @@ MLCGSolver::solve_cg (MultiFab& sol, rho_1 = rho; } - + if ( verbose > 0 ) { amrex::Print() << "MLCGSolver_cg: Final Iteration" @@ -386,8 +386,8 @@ MLCGSolver::solve_cg (MultiFab& sol, if ( ( ret == 0 || ret == 8 ) && (rnorm < rnorm0) ) { sol.plus(sorig, 0, ncomp, nghost); - } - else + } + else { sol.setVal(0); sol.plus(sorig, 0, ncomp, nghost); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellABecLap.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCellABecLap.cpp index 1664b459ec4..77a076070ef 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCellABecLap.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLCellABecLap.cpp @@ -423,7 +423,7 @@ MLCellABecLap::makePETSc () const const Geometry& geom = m_geom[0].back(); const auto& factory = *(m_factory[0].back()); MPI_Comm comm = BottomCommunicator(); - + auto petsc_solver = makePetsc(ba, dm, geom, comm); petsc_solver->setScalars(getAScalar(), getBScalar()); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp index 12a639f7271..0a8cf33f947 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLCellLinOp.cpp @@ -49,7 +49,7 @@ MLCellLinOp::defineAuxData () 1, 0, 0, ncomp); } } - + for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) { m_maskvals[amrlev].resize(m_num_mg_levels[amrlev]); @@ -153,7 +153,7 @@ MLCellLinOp::defineBC () m_bcondloc[amrlev][mglev].reset(new BndryCondLoc(m_grids[amrlev][mglev], m_dmap[amrlev][mglev], ncomp)); - } + } } } @@ -247,7 +247,7 @@ MLCellLinOp::makeNGrids (int grid_size) const b.refine(grid_size); IntVect sz = b.size(); const IntVect nblks {AMREX_D_DECL(sz[0]/grid_size, sz[1]/grid_size, sz[2]/grid_size)}; - + IntVect big = b.smallEnd() + grid_size - 1; b.setBig(big); @@ -321,7 +321,7 @@ MLCellLinOp::interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiF int kc = amrex::coarsen(k,ratio3.z); ffab(i,j,k,n) += cfab(ic,jc,kc,n); }); - } + } } void @@ -410,7 +410,7 @@ MLCellLinOp::fillSolutionBC (int amrlev, MultiFab& sol, const MultiFab* crse_bcd } const int mglev = 0; applyBC(amrlev, mglev, sol, BCMode::Inhomogeneous, StateMode::Solution, - m_bndry_sol[amrlev].get()); + m_bndry_sol[amrlev].get()); } void @@ -1199,7 +1199,7 @@ MLCellLinOp::applyMetricTerm (int amrlev, int mglev, MultiFab& rhs) const { amrex::ignore_unused(amrlev,mglev,rhs); #if (AMREX_SPACEDIM != 3) - + if (!m_has_metric_term) return; const int ncomp = rhs.nComp(); @@ -1255,7 +1255,7 @@ MLCellLinOp::unapplyMetricTerm (int amrlev, int mglev, MultiFab& rhs) const { amrex::ignore_unused(amrlev,mglev,rhs); #if (AMREX_SPACEDIM != 3) - + if (!m_has_metric_term) return; const int ncomp = rhs.nComp(); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H index 6dcd517b4ff..e8aaa87e6cb 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.H @@ -47,7 +47,7 @@ public: Location a_beta_loc); void setBCoeffs (int amrlev, const Array& beta) {setBCoeffs (amrlev, beta, Location::FaceCenter);} - + void setBCoeffs (int amrlev, Real beta); void setBCoeffs (int amrlev, Vector const& beta); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp index be1c91364ed..01304a3e799 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap.cpp @@ -91,7 +91,7 @@ MLEBABecLap::define (const Vector& a_geom, 1, 0, 0, 1); } } - + // Default to cell center; can be re-set to cell centroid via setPhiOnCentroid call m_phi_loc = Location::CellCenter; } @@ -419,7 +419,7 @@ MLEBABecLap::setEBHomogDirichlet (int amrlev, const MultiFab& beta) { if (flag(i,j,k).isSingleValued()) { betaout(i,j,k,n) = betain(i,j,k,n); - } else { + } else { betaout(i,j,k,n) = 0.0; } }); @@ -558,7 +558,7 @@ MLEBABecLap::averageDownCoeffs () { auto& fine_a_coeffs = m_a_coeffs[amrlev]; auto& fine_b_coeffs = m_b_coeffs[amrlev]; - + averageDownCoeffsSameAmrLevel(amrlev, fine_a_coeffs, fine_b_coeffs, amrex::GetVecOfPtrs(m_eb_b_coeffs[0])); averageDownCoeffsToCoarseAmrLevel(amrlev); @@ -636,7 +636,7 @@ MLEBABecLap::prepareForSolve () BL_PROFILE("MLABecLaplacian::prepareForSolve()"); MLCellABecLap::prepareForSolve(); - + averageDownCoeffs(); if (m_eb_phi[0]) { @@ -683,7 +683,7 @@ MLEBABecLap::Fapply (int amrlev, int mglev, MultiFab& out, const MultiFab& in) c const MultiFab& bycoef = m_b_coeffs[amrlev][mglev][1];, const MultiFab& bzcoef = m_b_coeffs[amrlev][mglev][2];); const iMultiFab& ccmask = m_cc_mask[amrlev][mglev]; - + const auto dxinvarr = m_geom[amrlev][mglev].InvCellSizeArray(); auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); @@ -871,7 +871,7 @@ MLEBABecLap::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs, #endif for (MFIter mfi(sol, mfi_info); mfi.isValid(); ++mfi) { - const auto& m0 = mm0.array(mfi); + const auto& m0 = mm0.array(mfi); const auto& m1 = mm1.array(mfi); #if (AMREX_SPACEDIM > 1) const auto& m2 = mm2.array(mfi); @@ -972,12 +972,12 @@ MLEBABecLap::FFlux (int amrlev, const MFIter& mfi, const Array(m_factory[amrlev][mglev].get()); - const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; + auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); + const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; const Real* dxinv = m_geom[amrlev][mglev].InvCellSize(); AMREX_D_TERM(const auto& bx = m_b_coeffs[amrlev][mglev][0][mfi];, @@ -992,7 +992,7 @@ MLEBABecLap::FFlux (int amrlev, const MFIter& mfi, const Array const& fy = flux[1]->array();, Array4 const& fz = flux[2]->array();); - const auto fabtyp = (flags) ? (*flags)[mfi].getType(box) : FabType::regular; + const auto fabtyp = (flags) ? (*flags)[mfi].getType(box) : FabType::regular; if (fabtyp == FabType::covered) { AMREX_HOST_DEVICE_PARALLEL_FOR_4D(xbx, ncomp, i, j, k, n, { @@ -1099,12 +1099,12 @@ MLEBABecLap::compGrad (int amrlev, const Array& grad, const Real dzi = m_geom[amrlev][mglev].InvCellSize(2);); const iMultiFab& ccmask = m_cc_mask[amrlev][mglev]; - auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); - const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; - auto area = (factory) ? factory->getAreaFrac() : - Array{AMREX_D_DECL(nullptr, nullptr, nullptr)}; + auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); + const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; + auto area = (factory) ? factory->getAreaFrac() : + Array{AMREX_D_DECL(nullptr, nullptr, nullptr)}; auto fcent = (factory) ? factory->getFaceCent(): - Array{AMREX_D_DECL(nullptr, nullptr, nullptr)}; + Array{AMREX_D_DECL(nullptr, nullptr, nullptr)}; MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.EnableTiling().SetDynamic(true); @@ -1113,7 +1113,7 @@ MLEBABecLap::compGrad (int amrlev, const Array& grad, #endif for (MFIter mfi(sol, mfi_info); mfi.isValid(); ++mfi) { - const Box& box = mfi.tilebox(); + const Box& box = mfi.tilebox(); auto fabtyp = (flags) ? (*flags)[mfi].getType(box) :FabType::regular; AMREX_D_TERM(Box const& fbx = mfi.nodaltilebox(0);, Box const& fby = mfi.nodaltilebox(1);, @@ -1185,7 +1185,7 @@ MLEBABecLap::compGrad (int amrlev, const Array& grad, Array4 const& ay = area[1]->const_array(mfi);, Array4 const& az = area[2]->const_array(mfi);); - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_phi_loc == Location::CellCenter, + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_phi_loc == Location::CellCenter, "If computing the gradient at face centers we assume phi at cell centers"); AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM ( @@ -1253,7 +1253,7 @@ MLEBABecLap::normalize (int amrlev, int mglev, MultiFab& mf) const Array4 const& bzfab = bzcoef.const_array(mfi);); auto fabtyp = (flags) ? (*flags)[mfi].getType(bx) : FabType::regular; - + if (fabtyp == FabType::regular) { AMREX_LAUNCH_HOST_DEVICE_LAMBDA (bx, tbx, @@ -1288,7 +1288,7 @@ MLEBABecLap::normalize (int amrlev, int mglev, MultiFab& mf) const ccmfab, flagfab, vfracfab, AMREX_D_DECL(apxfab,apyfab,apzfab), AMREX_D_DECL(fcxfab,fcyfab,fczfab), - bafab, bcfab, bebfab, is_eb_dirichlet, + bafab, bcfab, bebfab, is_eb_dirichlet, beta_on_centroid, ncomp); }); } @@ -1390,12 +1390,12 @@ MLEBABecLap::applyBC (int amrlev, int mglev, MultiFab& in, BCMode bc_mode, State const auto& bcondloc = *m_bcondloc[amrlev][mglev]; const auto& ccmask = m_cc_mask[amrlev][mglev]; - + auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; auto area = (factory) ? factory->getAreaFrac() : Array{AMREX_D_DECL(nullptr,nullptr,nullptr)}; - + FArrayBox foofab(Box::TheUnitBox(),ncomp); const auto& foo = foofab.array(); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_2D_K.H index 97377e7907e..509028c4655 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_2D_K.H @@ -33,7 +33,7 @@ void mlebabeclap_adotx_centroid (Box const& box, Array4 const& y, { y(i,j,k,n) = 0.0; } - else if (flag(i,j,k).isRegular() && + else if (flag(i,j,k).isRegular() && ((flag(i-1,j ,k).isRegular() && flag(i+1,j ,k).isRegular() && flag(i ,j-1,k).isRegular() && flag(i ,j+1,k).isRegular()) )) { @@ -170,7 +170,7 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, Array4 const& bc, Array4 const& beb, bool is_dirichlet, Array4 const& phieb, bool is_inhomog, GpuArray const& dxinv, - Real alpha, Real beta, int ncomp, + Real alpha, Real beta, int ncomp, bool beta_on_centroid, bool phi_on_centroid) noexcept { Real dhx = beta*dxinv[0]*dxinv[0]; @@ -205,9 +205,9 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, if (apxm != 0.0 && apxm != 1.0) { int jj = j + static_cast(amrex::Math::copysign(1.0,fcx(i,j,k))); Real fracy = (ccm(i-1,jj,k) || ccm(i,jj,k)) ? amrex::Math::abs(fcx(i,j,k)) : 0.0; - if (beta_on_center && phi_on_center) + if (beta_on_center && phi_on_center) fxm = (1.0-fracy)*fxm + fracy*bX(i,jj,k,n)*(x(i,jj,k,n)-x(i-1,jj,k,n)); - else if (beta_on_centroid && phi_on_center) + else if (beta_on_centroid && phi_on_center) fxm = bX(i,j,k,n) * ( (1.0-fracy)*(x(i, j,k,n)-x(i-1, j,k,n)) + fracy *(x(i,jj,k,n)-x(i-1,jj,k,n)) ); } @@ -216,9 +216,9 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, if (apxp != 0.0 && apxp != 1.0) { int jj = j + static_cast(amrex::Math::copysign(1.0,fcx(i+1,j,k))); Real fracy = (ccm(i,jj,k) || ccm(i+1,jj,k)) ? amrex::Math::abs(fcx(i+1,j,k)) : 0.0; - if (beta_on_center && phi_on_center) + if (beta_on_center && phi_on_center) fxp = (1.0-fracy)*fxp + fracy*bX(i+1,jj,k,n)*(x(i+1,jj,k,n)-x(i,jj,k,n)); - else if (beta_on_centroid && phi_on_center) + else if (beta_on_centroid && phi_on_center) fxp = bX(i+1,j,k,n) * ( (1.0-fracy)*(x(i+1, j,k,n)-x(i, j,k,n)) + fracy *(x(i+1,jj,k,n)-x(i,jj,k,n)) ); } @@ -227,9 +227,9 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, if (apym != 0.0 && apym != 1.0) { int ii = i + static_cast(amrex::Math::copysign(1.0,fcy(i,j,k))); Real fracx = (ccm(ii,j-1,k) || ccm(ii,j,k)) ? amrex::Math::abs(fcy(i,j,k)) : 0.0; - if (beta_on_center && phi_on_center) + if (beta_on_center && phi_on_center) fym = (1.0-fracx)*fym + fracx*bY(ii,j,k,n)*(x(ii,j,k,n)-x(ii,j-1,k,n)); - else if (beta_on_centroid && phi_on_center) + else if (beta_on_centroid && phi_on_center) fym = bY(i,j,k,n) * ( (1.0-fracx)*(x( i,j,k,n)-x( i,j-1,k,n)) + fracx *(x(ii,j,k,n)-x(ii,j-1,k,n)) ); } @@ -238,9 +238,9 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, if (apyp != 0.0 && apyp != 1.0) { int ii = i + static_cast(amrex::Math::copysign(1.0,fcy(i,j+1,k))); Real fracx = (ccm(ii,j,k) || ccm(ii,j+1,k)) ? amrex::Math::abs(fcy(i,j+1,k)) : 0.0; - if (beta_on_center && phi_on_center) + if (beta_on_center && phi_on_center) fyp = (1.0-fracx)*fyp + fracx*bY(ii,j+1,k,n)*(x(ii,j+1,k,n)-x(ii,j,k,n)); - else if (beta_on_centroid && phi_on_center) + else if (beta_on_centroid && phi_on_center) fyp = bY(i,j+1,k,n) * ( (1.0-fracx)*(x( i,j+1,k,n)-x( i,j,k,n)) + fracx *(x(ii,j+1,k,n)-x(ii,j,k,n)) ); } @@ -373,7 +373,7 @@ void mlebabeclap_gsrb (Box const& box, Array4 const& m0, Array4 const& m2, Array4 const& m1, Array4 const& m3, Array4 const& f0, Array4 const& f2, - Array4 const& f1, Array4 const& f3, + Array4 const& f1, Array4 const& f3, Array4 const& ccm, Array4 const& flag, Array4 const& vfrc, Array4 const& apx, Array4 const& apy, @@ -439,9 +439,9 @@ void mlebabeclap_gsrb (Box const& box, ? amrex::Math::abs(fcx(i,j,k)) : 0.0; if (!beta_on_centroid && !phi_on_centroid) { - fxm = (1.0-fracy)*fxm + + fxm = (1.0-fracy)*fxm + fracy *bX(i,jj,k,n)*(phi(i,jj,k,n)-phi(i-1,jj,k,n)); - } + } else if (beta_on_centroid && !phi_on_centroid) { fxm = (1.0-fracy)*( -phi(i-1,j,k,n)) + @@ -461,9 +461,9 @@ void mlebabeclap_gsrb (Box const& box, ? amrex::Math::abs(fcx(i+1,j,k)) : 0.0; if (!beta_on_centroid && !phi_on_centroid) { - fxp = (1.0-fracy)*fxp + + fxp = (1.0-fracy)*fxp + fracy *bX(i+1,jj,k,n)*(phi(i+1,jj,k,n)-phi(i,jj,k,n)); - } + } else if (beta_on_centroid && !phi_on_centroid) { fxp = (1.0-fracy)*(phi(i+1,j,k,n) ) + @@ -483,9 +483,9 @@ void mlebabeclap_gsrb (Box const& box, ? amrex::Math::abs(fcy(i,j,k)) : 0.0; if (!beta_on_centroid && !phi_on_centroid) { - fym = (1.0-fracx)*fym + + fym = (1.0-fracx)*fym + fracx *bY(ii,j,k,n)*(phi(ii,j,k,n)-phi(ii,j-1,k,n)); - } + } else if (beta_on_centroid && !phi_on_centroid) { fym = (1.0-fracx)*( -phi( i,j-1,k,n)) + @@ -506,9 +506,9 @@ void mlebabeclap_gsrb (Box const& box, ? amrex::Math::abs(fcy(i,j+1,k)) : 0.0; if (!beta_on_centroid && !phi_on_centroid) { - fyp = (1.0-fracx)*fyp + + fyp = (1.0-fracx)*fyp + fracx*bY(ii,j+1,k,n)*(phi(ii,j+1,k,n)-phi(ii,j,k,n)); - } + } else if (beta_on_centroid && !phi_on_centroid) { fyp = (1.0-fracx)*(phi(i,j+1,k,n) )+ @@ -560,7 +560,7 @@ void mlebabeclap_gsrb (Box const& box, int ii = i - static_cast(sx); int jj = j - static_cast(sy); - + Real phig_gamma = (1.0 + gx*sx + gy*sy + gx*gy*sx*sy); Real phig = ( - gx*sx - gx*gy*sx*sy) * phi(ii,j ,k,n) + ( - gy*sy - gx*gy*sx*sy) * phi(i ,jj,k,n) @@ -689,7 +689,7 @@ void mlebabeclap_flux_y_0 (Box const& box, Array4 const& fy, Array4 const& gx, Array4 const& sol, Array4 const& apx, Array4 const& fcx, - Array4 const& ccm, + Array4 const& ccm, Real dxi, int ncomp, bool phi_on_centroid) noexcept { amrex::LoopConcurrent(box, ncomp, [=] (int i, int j, int k, int n) noexcept @@ -702,7 +702,7 @@ void mlebabeclap_grad_x (Box const& box, Array4 const& gx, Array4(amrex::Math::copysign(1.0,fcx(i,j,k))); Real fracy = (ccm(i-1,jj,k) || ccm(i,jj,k)) ? amrex::Math::abs(fcx(i,j,k)) : 0.0; - if (!phi_on_centroid) + if (!phi_on_centroid) gxm = (1.0-fracy)*gxm + fracy*(sol(i,jj,k,n)-sol(i-1,jj,k,n)); gx(i,j,k,n) = gxm*dxi; } @@ -725,7 +725,7 @@ void mlebabeclap_grad_y (Box const& box, Array4 const& gy, Array4(amrex::Math::copysign(1.0,fcy(i,j,k))); Real fracx = (ccm(ii,j-1,k) || ccm(ii,j,k)) ? amrex::Math::abs(fcy(i,j,k)) : 0.0; - if (!phi_on_centroid) + if (!phi_on_centroid) gym = (1.0-fracx)*gym + fracx*(sol(ii,j,k,n)-sol(ii,j-1,k,n)); gy(i,j,k,n) = gym*dyi; } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_3D_K.H index 7fc3ab8388b..fa76e481a5a 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_3D_K.H @@ -230,7 +230,7 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, Real dhx = beta*dxinv[0]*dxinv[0]; Real dhy = beta*dxinv[1]*dxinv[1]; Real dhz = beta*dxinv[2]*dxinv[2]; - + bool beta_on_center = !(beta_on_centroid); bool phi_on_center = !( phi_on_centroid); @@ -280,7 +280,7 @@ void mlebabeclap_adotx (Box const& box, Array4 const& y, fracz *(1.0-fracy)*(x(i, j,kk,n)-x(i-1, j,kk,n)) + fracy * fracz *(x(i,jj,kk,n)-x(i-1,jj,kk,n)); fxm *= bX(i,j,k,n); - } + } } Real fxp = bX(i+1,j,k,n)*(x(i+1,j,k,n) - x(i,j,k,n)); @@ -591,7 +591,7 @@ void mlebabeclap_gsrb (Box const& box, { Real cf0 = (i == vlo.x && m0(vlo.x-1,j,k) > 0) ? f0(vlo.x,j,k,n) : 0.0; - Real cf1 = (j == vlo.y && m1(i,vlo.y-1,k) > 0) + Real cf1 = (j == vlo.y && m1(i,vlo.y-1,k) > 0) ? f1(i,vlo.y,k,n) : 0.0; Real cf2 = (k == vlo.z && m2(i,j,vlo.z-1) > 0) ? f2(i,j,vlo.z,n) : 0.0; @@ -617,7 +617,7 @@ void mlebabeclap_gsrb (Box const& box, bZ(i ,j ,k ,n)*phi(i ,j ,k-1,n)); Real delta = dhx*(bX(i,j,k,n)*cf0 + bX(i+1,j,k,n)*cf3) - + dhy*(bY(i,j,k,n)*cf1 + bY(i,j+1,k,n)*cf4) + + dhy*(bY(i,j,k,n)*cf1 + bY(i,j+1,k,n)*cf4) + dhz*(bZ(i,j,k,n)*cf2 + bZ(i,j,k+1,n)*cf5); Real res = rhs(i,j,k,n) - (gamma*phi(i,j,k,n) - rho); @@ -652,7 +652,7 @@ void mlebabeclap_gsrb (Box const& box, } else if (beta_on_centroid && !phi_on_centroid) { - fxm = (1.0-fracy)*(1.0-fracz)*( -phi(i-1, j, k,n)) + fxm = (1.0-fracy)*(1.0-fracz)*( -phi(i-1, j, k,n)) + fracy *(1.0-fracz)*(phi(i,jj,k ,n)-phi(i-1,jj, k,n)) +(1.0-fracy)* fracz *(phi(i,j ,kk,n)-phi(i-1, j,kk,n)) + fracy * fracz *(phi(i,jj,kk,n)-phi(i-1,jj,kk,n)); @@ -736,14 +736,14 @@ void mlebabeclap_gsrb (Box const& box, ? amrex::Math::abs(fcy(i,j+1,k,1)) : 0.0; if (!beta_on_centroid && !phi_on_centroid) { - fyp = (1.0-fracx)*(1.0-fracz)*fyp + fyp = (1.0-fracx)*(1.0-fracz)*fyp + fracx *(1.0-fracz)*bY(ii,j+1,k ,n)*(phi(ii,j+1,k ,n)-phi(ii,j,k ,n)) + (1.0-fracx)* fracz *bY(i ,j+1,kk,n)*(phi(i ,j+1,kk,n)-phi(i ,j,kk,n)) + fracx * fracz *bY(ii,j+1,kk,n)*(phi(ii,j+1,kk,n)-phi(ii,j,kk,n)); } else if (beta_on_centroid && !phi_on_centroid) { - fyp = (1.0-fracx)*(1.0-fracz)*(phi( i,j+1, k,n) ) + fyp = (1.0-fracx)*(1.0-fracz)*(phi( i,j+1, k,n) ) + fracx *(1.0-fracz)*(phi(ii,j+1, k,n)-phi(ii,j, k,n)) + (1.0-fracx)* fracz *(phi( i,j+1,kk,n)-phi( i,j,kk,n)) + fracx * fracz *(phi(ii,j+1,kk,n)-phi(ii,j,kk,n)); @@ -796,7 +796,7 @@ void mlebabeclap_gsrb (Box const& box, ? amrex::Math::abs(fcz(i,j,k+1,1)) : 0.0; if (!beta_on_centroid && !phi_on_centroid) { - fzp = (1.0-fracx)*(1.0-fracy)*fzp + fzp = (1.0-fracx)*(1.0-fracy)*fzp + fracx *(1.0-fracy)*bZ(ii,j ,k+1,n)*(phi(ii,j ,k+1,n)-phi(ii,j ,k,n)) + (1.0-fracx)* fracy *bZ(i ,jj,k+1,n)*(phi(i ,jj,k+1,n)-phi(i ,jj,k,n)) + fracx * fracy *bZ(ii,jj,k+1,n)*(phi(ii,jj,k+1,n)-phi(ii,jj,k,n)); @@ -820,14 +820,14 @@ void mlebabeclap_gsrb (Box const& box, dhy*(apym*sym-apyp*syp) + dhz*(apzm*szm-apzp*szp)); - Real rho = -vfrcinv * - (dhx*(apxm*fxm-apxp*fxp) + - dhy*(apym*fym-apyp*fyp) + + Real rho = -vfrcinv * + (dhx*(apxm*fxm-apxp*fxp) + + dhy*(apym*fym-apyp*fyp) + dhz*(apzm*fzm-apzp*fzp)); - Real delta = -vfrcinv * - (dhx*(apxm*oxm-apxp*oxp) + - dhy*(apym*oym-apyp*oyp) + + Real delta = -vfrcinv * + (dhx*(apxm*oxm-apxp*oxp) + + dhy*(apym*oym-apyp*oyp) + dhz*(apzm*ozm-apzp*ozp)); if (is_dirichlet) { @@ -917,7 +917,7 @@ void mlebabeclap_flux_x (Box const& box, Array4 const& fx, Array4 const& fy, Array4 const& fz, Array4 const& fz, Array4 const& gx, Array4 const& sol, Array4 const& apx, Array4 const& fcx, - Array4 const& ccm, + Array4 const& ccm, Real dxi, int ncomp, bool phi_on_centroid) noexcept { amrex::LoopConcurrent(box, ncomp, [=] (int i, int j, int k, int n) noexcept diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp index ba08cc2c122..b5a0de61811 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensorOp.cpp @@ -292,7 +292,7 @@ MLEBTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, auto factory = dynamic_cast(m_factory[amrlev][mglev].get()); const FabArray* flags = (factory) ? &(factory->getMultiEBCellFlagFab()) : nullptr; - + MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.SetDynamic(true); #ifdef AMREX_USE_OMP @@ -392,7 +392,7 @@ MLEBTensorOp::compCrossTerms(int amrlev, int mglev, MultiFab const& mf) const Array const& etamf = m_b_coeffs[amrlev][mglev]; Array const& kapmf = m_kappa[amrlev][mglev]; Array& fluxmf = m_tauflux[amrlev][mglev]; - + MFItInfo mfi_info; if (Gpu::notInLaunchRegion()) mfi_info.EnableTiling().SetDynamic(true); #ifdef AMREX_USE_OMP @@ -401,99 +401,99 @@ MLEBTensorOp::compCrossTerms(int amrlev, int mglev, MultiFab const& mf) const for (MFIter mfi(mf, mfi_info); mfi.isValid(); ++mfi) { const Box& bx = mfi.tilebox(); - AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, - Box const ybx = mfi.nodaltilebox(1);, - Box const zbx = mfi.nodaltilebox(2);); - - // grow by 1 because of corners - auto fabtyp = (flags) ? (*flags)[mfi].getType(amrex::grow(bx,1)) : FabType::regular; - - if (fabtyp == FabType::covered) { - AMREX_D_TERM(Array4 const& fxfab = fluxmf[0].array(mfi);, - Array4 const& fyfab = fluxmf[1].array(mfi);, - Array4 const& fzfab = fluxmf[2].array(mfi);); - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { + AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, + Box const ybx = mfi.nodaltilebox(1);, + Box const zbx = mfi.nodaltilebox(2);); + + // grow by 1 because of corners + auto fabtyp = (flags) ? (*flags)[mfi].getType(amrex::grow(bx,1)) : FabType::regular; + + if (fabtyp == FabType::covered) { + AMREX_D_TERM(Array4 const& fxfab = fluxmf[0].array(mfi);, + Array4 const& fyfab = fluxmf[1].array(mfi);, + Array4 const& fzfab = fluxmf[2].array(mfi);); + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { AMREX_LOOP_4D(txbx, AMREX_SPACEDIM, i, j, k, n, { fxfab(i,j,k,n) = 0.0; }); - } - , ybx, tybx, - { + } + , ybx, tybx, + { AMREX_LOOP_4D(tybx, AMREX_SPACEDIM, i, j, k, n, { fyfab(i,j,k,n) = 0.0; }); - } - , zbx, tzbx, - { + } + , zbx, tzbx, + { AMREX_LOOP_4D(tzbx, AMREX_SPACEDIM, i, j, k, n, { fzfab(i,j,k,n) = 0.0; }); - } - ); - } else { - AMREX_D_TERM(Array4 const fxfab = fluxmf[0].array(mfi);, - Array4 const fyfab = fluxmf[1].array(mfi);, - Array4 const fzfab = fluxmf[2].array(mfi);); - Array4 const vfab = mf.const_array(mfi); - AMREX_D_TERM(Array4 const etaxfab = etamf[0].const_array(mfi);, - Array4 const etayfab = etamf[1].const_array(mfi);, - Array4 const etazfab = etamf[2].const_array(mfi);); - AMREX_D_TERM(Array4 const kapxfab = kapmf[0].const_array(mfi);, - Array4 const kapyfab = kapmf[1].const_array(mfi);, - Array4 const kapzfab = kapmf[2].const_array(mfi);); - - if (fabtyp == FabType::regular) - { - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); - } - , ybx, tybx, - { - mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); - } - , zbx, tzbx, - { - mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); - } - ); - } - else - { - AMREX_D_TERM(Array4 const& apx = area[0]->const_array(mfi);, - Array4 const& apy = area[1]->const_array(mfi);, - Array4 const& apz = area[2]->const_array(mfi);); - Array4 const& flag = flags->const_array(mfi); - - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM - ( xbx, txbx, - { - mlebtensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,apx,flag,dxinv); - } - , ybx, tybx, - { - mlebtensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,apy,flag,dxinv); - } - , zbx, tzbx, - { - mlebtensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,apz,flag,dxinv); - } - ); - } - } + } + ); + } else { + AMREX_D_TERM(Array4 const fxfab = fluxmf[0].array(mfi);, + Array4 const fyfab = fluxmf[1].array(mfi);, + Array4 const fzfab = fluxmf[2].array(mfi);); + Array4 const vfab = mf.const_array(mfi); + AMREX_D_TERM(Array4 const etaxfab = etamf[0].const_array(mfi);, + Array4 const etayfab = etamf[1].const_array(mfi);, + Array4 const etazfab = etamf[2].const_array(mfi);); + AMREX_D_TERM(Array4 const kapxfab = kapmf[0].const_array(mfi);, + Array4 const kapyfab = kapmf[1].const_array(mfi);, + Array4 const kapzfab = kapmf[2].const_array(mfi);); + + if (fabtyp == FabType::regular) + { + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mltensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,dxinv); + } + , ybx, tybx, + { + mltensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,dxinv); + } + , zbx, tzbx, + { + mltensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,dxinv); + } + ); + } + else + { + AMREX_D_TERM(Array4 const& apx = area[0]->const_array(mfi);, + Array4 const& apy = area[1]->const_array(mfi);, + Array4 const& apz = area[2]->const_array(mfi);); + Array4 const& flag = flags->const_array(mfi); + + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM + ( xbx, txbx, + { + mlebtensor_cross_terms_fx(txbx,fxfab,vfab,etaxfab,kapxfab,apx,flag,dxinv); + } + , ybx, tybx, + { + mlebtensor_cross_terms_fy(tybx,fyfab,vfab,etayfab,kapyfab,apy,flag,dxinv); + } + , zbx, tzbx, + { + mlebtensor_cross_terms_fz(tzbx,fzfab,vfab,etazfab,kapzfab,apz,flag,dxinv); + } + ); + } + } } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { fluxmf[idim].FillBoundary(0, AMREX_SPACEDIM, geom.periodicity()); } } - + void MLEBTensorOp::compFlux (int amrlev, const Array& fluxes, MultiFab& sol, Location loc) const @@ -535,71 +535,71 @@ MLEBTensorOp::compFlux (int amrlev, const Array& fluxe if (fabtyp == FabType::regular) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& nbx = mfi.nodaltilebox(idim); - Array4 dst = fluxes[idim]->array(mfi); - Array4 src = fluxmf[idim].array(mfi); - AMREX_HOST_DEVICE_FOR_4D (nbx, ncomp, i, j, k, n, - { - dst(i,j,k,n) += bscalar*src(i,j,k,n); - }); - } + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + const Box& nbx = mfi.nodaltilebox(idim); + Array4 dst = fluxes[idim]->array(mfi); + Array4 src = fluxmf[idim].array(mfi); + AMREX_HOST_DEVICE_FOR_4D (nbx, ncomp, i, j, k, n, + { + dst(i,j,k,n) += bscalar*src(i,j,k,n); + }); + } } else if ( loc==Location::FaceCenter ) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& nbx = mfi.nodaltilebox(idim); - Array4 dst = fluxes[idim]->array(mfi); - Array4 src = fluxmf[idim].array(mfi); - Array4 const& ap = area[idim]->array(mfi); - - AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( nbx, tbx, - { - mlebtensor_flux_0(tbx, dst, src, ap, bscalar); - }); - } + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + const Box& nbx = mfi.nodaltilebox(idim); + Array4 dst = fluxes[idim]->array(mfi); + Array4 src = fluxmf[idim].array(mfi); + Array4 const& ap = area[idim]->array(mfi); + + AMREX_LAUNCH_HOST_DEVICE_LAMBDA ( nbx, tbx, + { + mlebtensor_flux_0(tbx, dst, src, ap, bscalar); + }); + } } - else // loc==Location::FaceCentroid - { - const iMultiFab& ccmask = m_cc_mask[amrlev][mglev]; + else // loc==Location::FaceCentroid + { + const iMultiFab& ccmask = m_cc_mask[amrlev][mglev]; - AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, + AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, Box const ybx = mfi.nodaltilebox(1);, Box const zbx = mfi.nodaltilebox(2);); - AMREX_D_TERM(Array4 fx = fluxmf[0].const_array(mfi);, - Array4 fy = fluxmf[1].const_array(mfi);, - Array4 fz = fluxmf[2].const_array(mfi);); - AMREX_D_TERM(Array4 Ax = fluxes[0]->array(mfi);, - Array4 Ay = fluxes[1]->array(mfi);, - Array4 Az = fluxes[2]->array(mfi);); - - const auto& fcent = factory->getFaceCent(); - AMREX_D_TERM(Array4 const& apx = area[0]->const_array(mfi);, - Array4 const& apy = area[1]->const_array(mfi);, - Array4 const& apz = area[2]->const_array(mfi);); - AMREX_D_TERM(Array4 const& fcx = fcent[0]->const_array(mfi);, - Array4 const& fcy = fcent[1]->const_array(mfi);, - Array4 const& fcz = fcent[2]->const_array(mfi);); - Array4 const& msk = ccmask.const_array(mfi); - - int face_only = 0; - - AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM ( + AMREX_D_TERM(Array4 fx = fluxmf[0].const_array(mfi);, + Array4 fy = fluxmf[1].const_array(mfi);, + Array4 fz = fluxmf[2].const_array(mfi);); + AMREX_D_TERM(Array4 Ax = fluxes[0]->array(mfi);, + Array4 Ay = fluxes[1]->array(mfi);, + Array4 Az = fluxes[2]->array(mfi);); + + const auto& fcent = factory->getFaceCent(); + AMREX_D_TERM(Array4 const& apx = area[0]->const_array(mfi);, + Array4 const& apy = area[1]->const_array(mfi);, + Array4 const& apz = area[2]->const_array(mfi);); + AMREX_D_TERM(Array4 const& fcx = fcent[0]->const_array(mfi);, + Array4 const& fcy = fcent[1]->const_array(mfi);, + Array4 const& fcz = fcent[2]->const_array(mfi);); + Array4 const& msk = ccmask.const_array(mfi); + + int face_only = 0; + + AMREX_LAUNCH_HOST_DEVICE_LAMBDA_DIM ( xbx, txbx, - { - mlebtensor_flux_x(txbx, Ax, fx, apx, fcx, bscalar, msk, face_only, xbx); - } - , ybx, tybx, - { - mlebtensor_flux_y(tybx, Ay, fy, apy, fcy, bscalar, msk, face_only, ybx); - } - , zbx, tzbx, - { - mlebtensor_flux_z(tzbx, Az, fz, apz, fcz, bscalar, msk, face_only, zbx); - } - ); - - } + { + mlebtensor_flux_x(txbx, Ax, fx, apx, fcx, bscalar, msk, face_only, xbx); + } + , ybx, tybx, + { + mlebtensor_flux_y(tybx, Ay, fy, apy, fcy, bscalar, msk, face_only, ybx); + } + , zbx, tzbx, + { + mlebtensor_flux_z(tzbx, Az, fz, apz, fcz, bscalar, msk, face_only, zbx); + } + ); + + } } } @@ -641,7 +641,7 @@ MLEBTensorOp::compVelGrad (int amrlev, const Array& fl if (fabtyp == FabType::regular) { - + Array4 const vfab = sol.const_array(mfi); AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, Box const ybx = mfi.nodaltilebox(1);, @@ -671,9 +671,9 @@ MLEBTensorOp::compVelGrad (int amrlev, const Array& fl ); // The derivatives are put in the array with the following order: -// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 -// in 2D: dU/dx, dV/dx, dU/dy, dV/dy -// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz +// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 +// in 2D: dU/dx, dV/dx, dU/dy, dV/dy +// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -696,7 +696,7 @@ MLEBTensorOp::compVelGrad (int amrlev, const Array& fl } else // loc==Location::FaceCentroid { - + amrex::Abort("compVelGrad not yet implemented for cut-cells "); } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H index 16faf7cd7c5..512c58a7ef1 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_2D_K.H @@ -47,7 +47,7 @@ void mlebtensor_cross_terms_fx (Box const& box, Array4 const& fx, +(vel(i-1,jlop,0,0)-vel(i-1,jlom,0,0))*wlo); Real dvdy = (0.5*dyi) * ((vel(i ,jhip,0,1)-vel(i ,jhim,0,1))*whi +(vel(i-1,jlop,0,1)-vel(i-1,jlom,0,1))*wlo); - Real divu = dvdy; + Real divu = dvdy; Real xif = kapx(i,j,0); Real mun = 0.75*(etax(i,j,0,0)-xif); // restore the original eta Real mut = etax(i,j,0,1); @@ -93,7 +93,7 @@ void mlebtensor_cross_terms_fy (Box const& box, Array4 const& fy, Real dvdx = (0.5*dxi) * ((vel(ihip,j ,0,1)-vel(ihim,j ,0,1))*whi +(vel(ilop,j-1,0,1)-vel(ilom,j-1,0,1))*wlo); - Real divu = dudx; + Real divu = dudx; Real xif = kapy(i,j,0); Real mun = 0.75*(etay(i,j,0,1)-xif); // restore the original eta Real mut = etay(i,j,0,0); @@ -257,10 +257,10 @@ void mlebtensor_cross_terms (Box const& box, Array4 const& Ax, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlebtensor_flux_0 (Box const& box, - Array4 const& Ax, - Array4 const& fx, - Array4 const& ap, - Real bscalar) noexcept + Array4 const& Ax, + Array4 const& fx, + Array4 const& ap, + Real bscalar) noexcept { const auto lo = amrex::lbound(box); const auto hi = amrex::ubound(box); @@ -269,78 +269,78 @@ void mlebtensor_flux_0 (Box const& box, for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - if (ap(i,j,k) != 0.0) { - for (int n=0; n const& Ax, - Array4 const& fx, Array4 const& apx, - Array4 const& fcx, - Real const bscalar, Array4 const& ccm, - int face_only, Box const& xbox) noexcept + Array4 const& fx, Array4 const& apx, + Array4 const& fcx, + Real const bscalar, Array4 const& ccm, + int face_only, Box const& xbox) noexcept { int lof = xbox.smallEnd(0); int hif = xbox.bigEnd(0); amrex::LoopConcurrent(box, [=] (int i, int j, int k) noexcept { if (!face_only || lof == i || hif == i) { - if (apx(i,j,k) == 1.0) { - for (int n=0; n= 0.0 ? 1 : -1); - Real fracy = (ccm(i-1,jj,0) || ccm(i,jj,0)) ? amrex::Math::abs(fcx(i,j,0,0)) : 0.0; - fxm_0 = (1.0-fracy)*fxm_0 + fracy*fx(i,jj,0,0); - fxm_1 = (1.0-fracy)*fxm_1 + fracy*fx(i,jj,0,1); + int jj = j + (fcx(i,j,0,0) >= 0.0 ? 1 : -1); + Real fracy = (ccm(i-1,jj,0) || ccm(i,jj,0)) ? amrex::Math::abs(fcx(i,j,0,0)) : 0.0; + fxm_0 = (1.0-fracy)*fxm_0 + fracy*fx(i,jj,0,0); + fxm_1 = (1.0-fracy)*fxm_1 + fracy*fx(i,jj,0,1); Ax(i,j,k,0) += bscalar*fxm_0; - Ax(i,j,k,1) += bscalar*fxm_1; - } - //else MLEBABec::compFlux already set Ax = 0 + Ax(i,j,k,1) += bscalar*fxm_1; + } + //else MLEBABec::compFlux already set Ax = 0 } }); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlebtensor_flux_y (Box const& box, Array4 const& Ay, - Array4 const& fy, Array4 const& apy, - Array4 const& fcy, - Real const bscalar, Array4 const& ccm, - int face_only, Box const& ybox) noexcept + Array4 const& fy, Array4 const& apy, + Array4 const& fcy, + Real const bscalar, Array4 const& ccm, + int face_only, Box const& ybox) noexcept { int lof = ybox.smallEnd(1); int hif = ybox.bigEnd(1); amrex::LoopConcurrent(box, [=] (int i, int j, int k) noexcept { if (!face_only || lof == j || hif == j) { - if (apy(i,j,k) == 1.0) { - for (int n=0; n= 0.0 ? 1 : -1); - Real fracx = (ccm(ii,j-1,0) || ccm(ii,j,0)) ? amrex::Math::abs(fcy(i,j,0,0)) : 0.0; - fym_0 = (1.0-fracx)*fym_0 + fracx*fy(ii,j,0,0); - fym_1 = (1.0-fracx)*fym_1 + fracx*fy(ii,j,0,1); + int ii = i + (fcy(i,j,0,0) >= 0.0 ? 1 : -1); + Real fracx = (ccm(ii,j-1,0) || ccm(ii,j,0)) ? amrex::Math::abs(fcy(i,j,0,0)) : 0.0; + fym_0 = (1.0-fracx)*fym_0 + fracx*fy(ii,j,0,0); + fym_1 = (1.0-fracx)*fym_1 + fracx*fy(ii,j,0,1); Ay(i,j,k,0) += bscalar*fym_0; - Ay(i,j,k,1) += bscalar*fym_1; - } - //else MLEBABec::compFlux already set Ay = 0 + Ay(i,j,k,1) += bscalar*fym_1; + } + //else MLEBABec::compFlux already set Ay = 0 } }); } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H index 38eef862a44..6598af3f2e1 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLEBTensor_3D_K.H @@ -508,10 +508,10 @@ void mlebtensor_cross_terms (Box const& box, Array4 const& Ax, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlebtensor_flux_0 (Box const& box, - Array4 const& Ax, - Array4 const& fx, - Array4 const& ap, - Real bscalar) noexcept + Array4 const& Ax, + Array4 const& fx, + Array4 const& ap, + Real bscalar) noexcept { const auto lo = amrex::lbound(box); const auto hi = amrex::ubound(box); @@ -520,153 +520,153 @@ void mlebtensor_flux_0 (Box const& box, for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - if (ap(i,j,k) != 0.0) { - for (int n=0; n const& Ax, - Array4 const& fx, Array4 const& apx, - Array4 const& fcx, - Real const bscalar, Array4 const& ccm, - int face_only, Box const& xbox) noexcept + Array4 const& fx, Array4 const& apx, + Array4 const& fcx, + Real const bscalar, Array4 const& ccm, + int face_only, Box const& xbox) noexcept { int lof = xbox.smallEnd(0); int hif = xbox.bigEnd(0); amrex::LoopConcurrent(box, [=] (int i, int j, int k) noexcept { if (!face_only || lof == i || hif == i) { - if (apx(i,j,k) == 1.0) { - for (int n=0; n= 0.0 ? 1 : -1); - int kk = k + (fcx(i,j,k,1) >= 0.0 ? 1 : -1); - Real fracy = (ccm(i-1,jj,k) || ccm(i,jj,k)) ? amrex::Math::abs(fcx(i,j,k,0)) : 0.0; - Real fracz = (ccm(i-1,j,kk) || ccm(i,j,kk)) ? amrex::Math::abs(fcx(i,j,k,1)) : 0.0; - fxm_0 = (1.0-fracy)*(1.0-fracz) *fxm_0 - + fracy*(1.0-fracz) * fx(i,jj,k ,0) - + fracz*(1.0-fracy) * fx(i,j ,kk,0) - + fracy* fracz * fx(i,jj,kk,0); - fxm_1 = (1.0-fracy)*(1.0-fracz) *fxm_1 + if (apx(i,j,k) == 1.0) { + for (int n=0; n= 0.0 ? 1 : -1); + int kk = k + (fcx(i,j,k,1) >= 0.0 ? 1 : -1); + Real fracy = (ccm(i-1,jj,k) || ccm(i,jj,k)) ? amrex::Math::abs(fcx(i,j,k,0)) : 0.0; + Real fracz = (ccm(i-1,j,kk) || ccm(i,j,kk)) ? amrex::Math::abs(fcx(i,j,k,1)) : 0.0; + fxm_0 = (1.0-fracy)*(1.0-fracz) *fxm_0 + + fracy*(1.0-fracz) * fx(i,jj,k ,0) + + fracz*(1.0-fracy) * fx(i,j ,kk,0) + + fracy* fracz * fx(i,jj,kk,0); + fxm_1 = (1.0-fracy)*(1.0-fracz) *fxm_1 + fracy*(1.0-fracz) * fx(i,jj,k ,1) + fracz*(1.0-fracy) * fx(i,j ,kk,1) + fracy* fracz * fx(i,jj,kk,1); - fxm_2 = (1.0-fracy)*(1.0-fracz) *fxm_2 + fxm_2 = (1.0-fracy)*(1.0-fracz) *fxm_2 + fracy*(1.0-fracz) * fx(i,jj,k ,2) + fracz*(1.0-fracy) * fx(i,j ,kk,2) + fracy* fracz * fx(i,jj,kk,2); - Ax(i,j,k,0) += bscalar*fxm_0; - Ax(i,j,k,1) += bscalar*fxm_1; - Ax(i,j,k,2) += bscalar*fxm_2; - } - } + Ax(i,j,k,0) += bscalar*fxm_0; + Ax(i,j,k,1) += bscalar*fxm_1; + Ax(i,j,k,2) += bscalar*fxm_2; + } + } }); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlebtensor_flux_y (Box const& box, Array4 const& Ay, - Array4 const& fy, Array4 const& apy, - Array4 const& fcy, - Real const bscalar, Array4 const& ccm, - int face_only, Box const& ybox) noexcept + Array4 const& fy, Array4 const& apy, + Array4 const& fcy, + Real const bscalar, Array4 const& ccm, + int face_only, Box const& ybox) noexcept { int lof = ybox.smallEnd(1); int hif = ybox.bigEnd(1); amrex::LoopConcurrent(box, [=] (int i, int j, int k) noexcept { if (!face_only || lof == j || hif == j) { - if (apy(i,j,k) == 1.0) { - for (int n=0; n= 0.0 ? 1 : -1); - int kk = k + (fcy(i,j,k,1) >= 0.0 ? 1 : -1); - Real fracx = (ccm(ii,j-1,k) || ccm(ii,j,k)) ? amrex::Math::abs(fcy(i,j,k,0)) : 0.0; - Real fracz = (ccm(i,j-1,kk) || ccm(i,j,kk)) ? amrex::Math::abs(fcy(i,j,k,1)) : 0.0; - fym_0 = (1.0-fracx)*(1.0-fracz) *fym_0 + Real fym_1 = fy(i,j,k,1); + Real fym_2 = fy(i,j,k,2); + + int ii = i + (fcy(i,j,k,0) >= 0.0 ? 1 : -1); + int kk = k + (fcy(i,j,k,1) >= 0.0 ? 1 : -1); + Real fracx = (ccm(ii,j-1,k) || ccm(ii,j,k)) ? amrex::Math::abs(fcy(i,j,k,0)) : 0.0; + Real fracz = (ccm(i,j-1,kk) || ccm(i,j,kk)) ? amrex::Math::abs(fcy(i,j,k,1)) : 0.0; + fym_0 = (1.0-fracx)*(1.0-fracz) *fym_0 + fracx*(1.0-fracz) * fy(ii,j,k ,0) + fracz*(1.0-fracx) * fy(i ,j,kk,0) + fracx* fracz * fy(ii,j,kk,0); - fym_1 = (1.0-fracx)*(1.0-fracz) *fym_1 + fym_1 = (1.0-fracx)*(1.0-fracz) *fym_1 + fracx*(1.0-fracz) * fy(ii,j,k ,1) + fracz*(1.0-fracx) * fy(i ,j,kk,1) + fracx* fracz * fy(ii,j,kk,1); - fym_2 = (1.0-fracx)*(1.0-fracz) *fym_2 + fym_2 = (1.0-fracx)*(1.0-fracz) *fym_2 + fracx*(1.0-fracz) * fy(ii,j,k ,2) + fracz*(1.0-fracx) * fy(i ,j,kk,2) + fracx* fracz * fy(ii,j,kk,2); - Ay(i,j,k,0) += bscalar*fym_0; - Ay(i,j,k,1) += bscalar*fym_1; - Ay(i,j,k,2) += bscalar*fym_2; - } - //else MLEBABec::compFlux already set Ay = 0 + Ay(i,j,k,0) += bscalar*fym_0; + Ay(i,j,k,1) += bscalar*fym_1; + Ay(i,j,k,2) += bscalar*fym_2; + } + //else MLEBABec::compFlux already set Ay = 0 } }); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlebtensor_flux_z (Box const& box, Array4 const& Az, - Array4 const& fz, Array4 const& apz, - Array4 const& fcz, - Real const bscalar, Array4 const& ccm, - int face_only, Box const& zbox) noexcept + Array4 const& fz, Array4 const& apz, + Array4 const& fcz, + Real const bscalar, Array4 const& ccm, + int face_only, Box const& zbox) noexcept { int lof = zbox.smallEnd(2); int hif = zbox.bigEnd(2); amrex::LoopConcurrent(box, [=] (int i, int j, int k) noexcept { if (!face_only || lof == k || hif == k) { - if (apz(i,j,k) == 1.0) { - for (int n=0; n= 0.0 ? 1 : -1); - int jj = j + (fcz(i,j,k,1) >= 0.0 ? 1 : -1); - Real fracx = (ccm(ii,j,k-1) || ccm(ii,j,k)) ? amrex::Math::abs(fcz(i,j,k,0)) : 0.0; - Real fracy = (ccm(i,jj,k-1) || ccm(i,jj,k)) ? amrex::Math::abs(fcz(i,j,k,1)) : 0.0; - fzm_0 = (1.0-fracx)*(1.0-fracy) *fzm_0 + Real fzm_1 = fz(i,j,k,1); + Real fzm_2 = fz(i,j,k,2); + + int ii = i + (fcz(i,j,k,0) >= 0.0 ? 1 : -1); + int jj = j + (fcz(i,j,k,1) >= 0.0 ? 1 : -1); + Real fracx = (ccm(ii,j,k-1) || ccm(ii,j,k)) ? amrex::Math::abs(fcz(i,j,k,0)) : 0.0; + Real fracy = (ccm(i,jj,k-1) || ccm(i,jj,k)) ? amrex::Math::abs(fcz(i,j,k,1)) : 0.0; + fzm_0 = (1.0-fracx)*(1.0-fracy) *fzm_0 + fracx*(1.0-fracy) * fz(ii,j ,k,0) + fracy*(1.0-fracx) * fz(i ,jj,k,0) + fracx* fracy * fz(ii,jj,k,0); - fzm_1 = (1.0-fracx)*(1.0-fracy) *fzm_1 + fzm_1 = (1.0-fracx)*(1.0-fracy) *fzm_1 + fracx*(1.0-fracy) * fz(ii,j ,k,1) + fracy*(1.0-fracx) * fz(i ,jj,k,1) + fracx* fracy * fz(ii,jj,k,1); - fzm_2 = (1.0-fracx)*(1.0-fracy) *fzm_2 + fzm_2 = (1.0-fracx)*(1.0-fracy) *fzm_2 + fracx*(1.0-fracy) * fz(ii,j ,k,2) + fracy*(1.0-fracx) * fz(i ,jj,k,2) + fracx* fracy * fz(ii,jj,k,2); - Az(i,j,k,0) += bscalar*fzm_0; - Az(i,j,k,1) += bscalar*fzm_1; - Az(i,j,k,2) += bscalar*fzm_2; - } - //else MLEBABec::compFlux already set Az = 0 - } + Az(i,j,k,0) += bscalar*fzm_0; + Az(i,j,k,1) += bscalar*fzm_1; + Az(i,j,k,2) += bscalar*fzm_2; + } + //else MLEBABec::compFlux already set Az = 0 + } }); } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H index 06b3b6289da..9c32d3b7902 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.H @@ -137,15 +137,15 @@ public: /** - * \brief For cell-centered solves only: if we want to do a linear solve - * where the boundary conditions on the coarsest AMR level of the solve - * come from a coarser level (e.g. the base AMR level of the solve is > 0 - * and does not cover the entire domain), we must explicitly provide the coarser - * data. Boundary conditions from a coarser level are always Dirichlet. + * \brief For cell-centered solves only: if we want to do a linear solve + * where the boundary conditions on the coarsest AMR level of the solve + * come from a coarser level (e.g. the base AMR level of the solve is > 0 + * and does not cover the entire domain), we must explicitly provide the coarser + * data. Boundary conditions from a coarser level are always Dirichlet. * The MultiFab crse does not need to have ghost cells and is at a coarser * resolution than the coarsest AMR level of the solve; it is used to supply - * (interpolated) boundary conditions for the solve. - * NOTE: If this is called, it must be called before `setLevelBC`. + * (interpolated) boundary conditions for the solve. + * NOTE: If this is called, it must be called before `setLevelBC`. * If crse is nullptr, then the bc values are assumed to be zero. * * \param crse @@ -154,7 +154,7 @@ public: void setCoarseFineBC (const MultiFab* crse, int crse_ratio) noexcept; /** - * \brief For cell-centered solves only: this must be called for each level. + * \brief For cell-centered solves only: this must be called for each level. * Argument `levelbcdata` is * used to supply Dirichlet or Neumann bc at the * physical domain; if those data are homogeneous we can pass nullptr instead * of levelbcdata. Regardless, this function must be called. diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp index ee1bdea4658..792e2ad5eeb 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp.cpp @@ -286,12 +286,12 @@ MLLinOp::defineGrids (const Vector& a_geom, Vector agg_flag; domainboxes.push_back(dbx); boundboxes.push_back(bbx); - agg_flag.push_back(false); + agg_flag.push_back(false); #if (AMREX_SPACEDIM > 1) - if (info.do_semicoarsening) - { - int num_semicoarsening_level = 0; + if (info.do_semicoarsening) + { + int num_semicoarsening_level = 0; IntVect rr_0(AMREX_D_DECL(mg_coarsen_ratio,1,1)); bool is_coarsenable_x = ( dbx.coarsenable(rr_0, mg_domain_min_width) && bbx.coarsenable(rr_0, mg_box_min_width)); @@ -303,14 +303,14 @@ MLLinOp::defineGrids (const Vector& a_geom, bool is_coarsenable_z = ( dbx.coarsenable(rr_2, mg_domain_min_width) && bbx.coarsenable(rr_2, mg_box_min_width)); #endif - IntVect rr_vec(mg_coarsen_ratio); + IntVect rr_vec(mg_coarsen_ratio); #if (AMREX_SPACEDIM == 2) while ( is_coarsenable_x || is_coarsenable_y ) #endif #if (AMREX_SPACEDIM == 3) while ( is_coarsenable_x || is_coarsenable_y || is_coarsenable_z ) #endif - { + { #if (AMREX_SPACEDIM >= 2) int r0 = (is_coarsenable_x) ? mg_coarsen_ratio : 1; int r1 = (is_coarsenable_y) ? mg_coarsen_ratio : 1; @@ -339,21 +339,21 @@ MLLinOp::defineGrids (const Vector& a_geom, #endif #endif #if (AMREX_SPACEDIM == 2) - if (!(is_coarsenable_x && is_coarsenable_y)) + if (!(is_coarsenable_x && is_coarsenable_y)) #endif #if (AMREX_SPACEDIM == 3) if (!(is_coarsenable_x && is_coarsenable_y && is_coarsenable_z)) #endif - { - num_semicoarsening_level++; - if (num_semicoarsening_level > info.max_semicoarsening_level) break; - } - } + { + num_semicoarsening_level++; + if (num_semicoarsening_level > info.max_semicoarsening_level) break; + } + } } - else + else #endif - { + { while ( dbx.coarsenable(mg_coarsen_ratio,mg_domain_min_width) && bbx.coarsenable(mg_coarsen_ratio,mg_box_min_width)) { @@ -364,7 +364,7 @@ MLLinOp::defineGrids (const Vector& a_geom, bool to_agg = (bbx.d_numPts() / nbxs) < 0.999*threshold_npts; agg_flag.push_back(to_agg); } - } + } int first_agglev = std::distance(agg_flag.begin(), std::find(agg_flag.begin(),agg_flag.end(),1)); @@ -401,19 +401,19 @@ MLLinOp::defineGrids (const Vector& a_geom, for (int lev = 1; lev < last_coarsenableto_lev; ++lev) { m_geom[0].emplace_back(amrex::coarsen(a_geom[0].Domain(),rr),rb,coord,is_per); - + m_grids[0].push_back(a_grids[0]); m_grids[0].back().coarsen(rr); - + m_dmap[0].push_back(a_dmap[0]); - + rr *= mg_coarsen_ratio; } for (int lev = last_coarsenableto_lev; lev < nmaxlev; ++lev) { m_geom[0].emplace_back(domainboxes[lev],rb,coord,is_per); - + m_grids[0].emplace_back(boundboxes[lev]); m_grids[0].back().maxSize(info.agg_grid_size); @@ -465,7 +465,7 @@ MLLinOp::defineGrids (const Vector& a_geom, { m_dmap[0].push_back(a_dmap[0]); } - + ++(m_num_mg_levels[0]); rr *= mg_coarsen_ratio; } @@ -498,12 +498,12 @@ MLLinOp::defineGrids (const Vector& a_geom, IntVect rr_vec(rr/mg_coarsen_ratio); #if (AMREX_SPACEDIM == 2) while ( (num_semicoarsening_level < info.max_semicoarsening_level + 1) && - (m_num_mg_levels[0] < info.max_coarsening_level + 1) && + (m_num_mg_levels[0] < info.max_coarsening_level + 1) && (is_coarsenable_x || is_coarsenable_y ) ) #endif #if (AMREX_SPACEDIM == 3) while ( (num_semicoarsening_level < info.max_semicoarsening_level + 1) && - (m_num_mg_levels[0] < info.max_coarsening_level + 1) && + (m_num_mg_levels[0] < info.max_coarsening_level + 1) && (is_coarsenable_x || is_coarsenable_y || is_coarsenable_z) ) #endif { @@ -757,7 +757,7 @@ MLLinOp::makeSubCommunicator (const DistributionMapping& dm) std::sort(newgrp_ranks.begin(), newgrp_ranks.end()); auto last = std::unique(newgrp_ranks.begin(), newgrp_ranks.end()); newgrp_ranks.erase(last, newgrp_ranks.end()); - + if (flag_verbose_linop) { Print() << "MLLinOp::makeSubCommunicator(): called for " << newgrp_ranks.size() << " ranks" << std::endl; } @@ -824,10 +824,10 @@ MLLinOp::makeAgglomeratedDMap (const Vector& ba, Vector >& sfc = DistributionMapping::makeSFC(ba[i]); - + const int nprocs = ParallelContext::NProcsSub(); AMREX_ASSERT(static_cast(sfc.size()) == nprocs); - + Vector pmap(ba[i].size()); for (int iproc = 0; iproc < nprocs; ++iproc) { int grank = ParallelContext::local_to_global_rank(iproc); @@ -858,7 +858,7 @@ MLLinOp::makeConsolidatedDMap (const Vector& ba, Vector pmap(pmap_fine.size()); - ParallelContext::global_to_local_rank(pmap.data(), pmap_fine.data(), pmap.size()); + ParallelContext::global_to_local_rank(pmap.data(), pmap_fine.data(), pmap.size()); if (strategy == 1) { for (auto& x: pmap) { x /= ratio; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLLinOp_nd.F90 b/Src/LinearSolvers/MLMG/AMReX_MLLinOp_nd.F90 index 816e8b0e7fd..8f01872a8a1 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLLinOp_nd.F90 +++ b/Src/LinearSolvers/MLMG/AMReX_MLLinOp_nd.F90 @@ -23,7 +23,7 @@ module amrex_mllinop_nd_module integer, parameter :: yhi_dir = 4 integer, parameter :: zhi_dir = 5 #endif - + private public :: amrex_mllinop_apply_bc @@ -78,7 +78,7 @@ subroutine amrex_mllinop_apply_bc (lo, hi, phi, hlo, hhi, mask, mlo, mhi, & end do end do #if (AMREX_SPACEDIM >= 2) - case (ylo_dir) + case (ylo_dir) do k = lo(3), hi(3) do i = lo(1), hi(1) if (mask(i,lo(2)-1,k) .gt. 0) then @@ -215,28 +215,28 @@ subroutine amrex_mllinop_apply_bc (lo, hi, phi, hlo, hhi, mask, mlo, mhi, & ! Corners in XY plane if (cdir==xlo_dir .or. cdir==ylo_dir) then do k = lo(3), hi(3) - phi(lo(1)-1,lo(2)-1,k,n) = & + phi(lo(1)-1,lo(2)-1,k,n) = & 0.5d0*(2.d0*phi(lo(1),lo(2)-1,k,n) - phi(lo(1)+1,lo(2)-1,k,n)) + & 0.5d0*(2.d0*phi(lo(1)-1,lo(2),k,n) - phi(lo(1)-1,lo(2)+1,k,n)) end do end if if (cdir==xhi_dir .or. cdir==ylo_dir) then do k = lo(3), hi(3) - phi(hi(1)+1,lo(2)-1,k,n) = & + phi(hi(1)+1,lo(2)-1,k,n) = & 0.5d0*(2.d0*phi(hi(1),lo(2)-1,k,n) - phi(hi(1)-1,lo(2)-1,k,n)) + & 0.5d0*(2.d0*phi(hi(1)+1,lo(2),k,n) - phi(hi(1)+1,lo(2)+1,k,n)) end do end if if (cdir==xlo_dir .or. cdir==yhi_dir) then do k = lo(3), hi(3) - phi(lo(1)-1,hi(2)+1,k,n) = & + phi(lo(1)-1,hi(2)+1,k,n) = & 0.5d0*(2.d0*phi(lo(1),hi(2)+1,k,n) - phi(lo(1)+1,hi(2)+1,k,n)) + & 0.5d0*(2.d0*phi(lo(1)-1,hi(2),k,n) - phi(lo(1)-1,hi(2)-1,k,n)) end do end if if (cdir==xhi_dir .or. cdir==yhi_dir) then do k = lo(3), hi(3) - phi(hi(1)+1,hi(2)+1,k,n) = & + phi(hi(1)+1,hi(2)+1,k,n) = & 0.5d0*(2.d0*phi(hi(1),hi(2)+1,k,n) - phi(hi(1)-1,hi(2)+1,k,n)) + & 0.5d0*(2.d0*phi(hi(1)+1,hi(2),k,n) - phi(hi(1)+1,hi(2)-1,k,n)) end do @@ -245,28 +245,28 @@ subroutine amrex_mllinop_apply_bc (lo, hi, phi, hlo, hhi, mask, mlo, mhi, & ! Corners in YZ plane if (cdir==zlo_dir .or. cdir==ylo_dir) then do i = lo(1), hi(1) - phi(i,lo(2)-1,lo(3)-1,n) = & + phi(i,lo(2)-1,lo(3)-1,n) = & 0.5d0*(2.d0*phi(i,lo(2)-1,lo(3),n) - phi(i,lo(2)-1,lo(3)+1,n)) + & 0.5d0*(2.d0*phi(i,lo(2),lo(3)-1,n) - phi(i,lo(2)+1,lo(3)-1,n)) end do end if if (cdir==zhi_dir .or. cdir==ylo_dir) then do i = lo(1), hi(1) - phi(i,lo(2)-1,hi(3)+1,n) = & + phi(i,lo(2)-1,hi(3)+1,n) = & 0.5d0*(2.d0*phi(i,lo(2)-1,hi(3),n) - phi(i,lo(2)-1,hi(3)-1,n)) + & 0.5d0*(2.d0*phi(i,lo(2),hi(3)+1,n) - phi(i,lo(2)+1,hi(3)+1,n)) end do end if if (cdir==zlo_dir .or. cdir==yhi_dir) then do i = lo(1), hi(1) - phi(i,hi(2)+1,lo(3)-1,n) = & + phi(i,hi(2)+1,lo(3)-1,n) = & 0.5d0*(2.d0*phi(i,hi(2)+1,lo(3),n) - phi(i,hi(2)+1,lo(3)+1,n)) + & 0.5d0*(2.d0*phi(i,hi(2),lo(3)-1,n) - phi(i,hi(2)-1,lo(3)-1,n)) end do end if if (cdir==zhi_dir .or. cdir==yhi_dir) then do i = lo(1), hi(1) - phi(i,hi(2)+1,hi(3)+1,n) = & + phi(i,hi(2)+1,hi(3)+1,n) = & 0.5d0*(2.d0*phi(i,hi(2)+1,hi(3),n) - phi(i,hi(2)+1,hi(3)-1,n)) + & 0.5d0*(2.d0*phi(i,hi(2),hi(3)+1,n) - phi(i,hi(2)-1,hi(3)+1,n)) end do @@ -274,28 +274,28 @@ subroutine amrex_mllinop_apply_bc (lo, hi, phi, hlo, hhi, mask, mlo, mhi, & ! Corners in XZ plane if (cdir==xlo_dir .or. cdir==zlo_dir) then do j = lo(2), hi(2) - phi(lo(1)-1,j,lo(3)-1,n) = & + phi(lo(1)-1,j,lo(3)-1,n) = & 0.5d0*(2.d0*phi(lo(1), j,lo(3)-1,n) - phi(lo(1)+1,j,lo(3)-1,n)) + & 0.5d0*(2.d0*phi(lo(1)-1,j,lo(3),n) - phi(lo(1)-1,j,lo(3)+1,n)) end do end if if (cdir==xhi_dir .or. cdir==zlo_dir) then do j = lo(2), hi(2) - phi(hi(1)+1,j,lo(3)-1,n) = & + phi(hi(1)+1,j,lo(3)-1,n) = & 0.5d0*(2.d0*phi(hi(1),j,lo(3)-1,n) - phi(hi(1)-1,j,lo(3)-1,n)) + & 0.5d0*(2.d0*phi(hi(1)+1,j,lo(3),n) - phi(hi(1)+1,j,lo(3)+1,n)) end do end if if (cdir==xlo_dir .or. cdir==zhi_dir) then do j = lo(2), hi(2) - phi(lo(1)-1,j,hi(3)+1,n) = & + phi(lo(1)-1,j,hi(3)+1,n) = & 0.5d0*(2.d0*phi(lo(1),j,hi(3)+1,n) - phi(lo(1)+1,j,hi(3)+1,n)) + & 0.5d0*(2.d0*phi(lo(1)-1,j,hi(3),n) - phi(lo(1)-1,j,hi(3)-1,n)) end do end if if (cdir==xhi_dir .or. cdir==zhi_dir) then do j = lo(2), hi(2) - phi(hi(1)+1,j,hi(3)+1,n) = & + phi(hi(1)+1,j,hi(3)+1,n) = & 0.5d0*(2.d0*phi(hi(1),j,hi(3)+1,n) - phi(hi(1)-1,j,hi(3)+1,n)) + & 0.5d0*(2.d0*phi(hi(1)+1,j,hi(3),n) - phi(hi(1)+1,j,hi(3)-1,n)) end do @@ -306,5 +306,5 @@ subroutine amrex_mllinop_apply_bc (lo, hi, phi, hlo, hhi, mask, mlo, mhi, & end do end subroutine amrex_mllinop_apply_bc - + end module amrex_mllinop_nd_module diff --git a/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp b/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp index 6b419f75468..1b98ff67762 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLMG.cpp @@ -177,7 +177,7 @@ MLMG::solve (const Vector& a_sol, const Vector& a_rh << composite_norminf << ", " << composite_norminf/max_norm << "\n"; } - amrex::Abort("MLMG failing so lets stop here"); + amrex::Abort("MLMG failing so lets stop here"); } } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLMGBndry.cpp b/Src/LinearSolvers/MLMG/AMReX_MLMGBndry.cpp index 91851135234..84fa3b739c1 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLMGBndry.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLMGBndry.cpp @@ -58,7 +58,7 @@ MLMGBndry::setBoxBC (RealTuple& bloc, BCTuple& bctag, const Box& bx, const Box& { const Orientation face = fi(); const int dir = face.coordDir(); - + if (domain[face] == bx[face] && !is_periodic[dir]) { // All physical bc values are located on face. diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H index 2d7b7a07ba9..1fb5d9f17a8 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_2D_K.H @@ -163,7 +163,7 @@ void mlndlap_avgdown_coeff_y (int i, int j, int k, Array4 const& crse, AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_semi_avgdown_coeff (int i, int j, int k, Array4 const& crse, Array4 const& fine, int idir) noexcept -{ +{ if (idir == 1) { Real a = fine(2*i ,j,k); Real b = fine(2*i+1,j,k); @@ -695,8 +695,8 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& Real fmx2y = Real(2.0)*facy - facx; if (is_rz) { - amrex::Abort("mlndlap_gauss_seidel_with_line_solve_aa is not implemented in r-z 2D "); - } + amrex::Abort("mlndlap_gauss_seidel_with_line_solve_aa is not implemented in r-z 2D "); + } const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); @@ -742,7 +742,7 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& a_ls(j-lo.y) = fmx2y*(sig(i-1,j-1,k)+sig(i,j-1,k)); b_ls(j-lo.y) = s0; - c_ls(j-lo.y) = fmx2y*(sig(i-1,j ,k)+sig(i,j ,k)); + c_ls(j-lo.y) = fmx2y*(sig(i-1,j ,k)+sig(i,j ,k)); u_ls(j-lo.y) = 0.; r_ls(j-lo.y) = rhs(i,j,k) - Ax; } @@ -995,11 +995,11 @@ void mlndlap_semi_interpadd_aa (int i, int j, int, Array4 const& fine, fine(i,j,0) += crse(ic,jc,0); } } - } else if (idir == 0 ) { - if (!msk(i,j,0)) { + } else if (idir == 0 ) { + if (!msk(i,j,0)) { int ic = i; int jc = amrex::coarsen(j,2); - bool j_is_odd = (ic*2 != i); + bool j_is_odd = (ic*2 != i); if (j_is_odd) { // Node on Y line fine(i,j,0) += aa_interp_line_y(crse,sig,i,j,ic,jc); @@ -1008,7 +1008,7 @@ void mlndlap_semi_interpadd_aa (int i, int j, int, Array4 const& fine, fine(i,j,0) += crse(ic,jc,0); } } - } + } } namespace { @@ -2084,8 +2084,8 @@ void mlndlap_mknewu_eb (int i, int j, int, Array4 const& u, Array4 const& u, Array4 const& p, - Real sig, Array4 const& vfrac, - Array4 const& intg, GpuArray const& dxinv) noexcept + Real sig, Array4 const& vfrac, + Array4 const& intg, GpuArray const& dxinv) noexcept { Real facx = Real(0.5)*dxinv[0]; Real facy = Real(0.5)*dxinv[1]; diff --git a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_3D_K.H index 1b1b29e0fea..0365bb5adbc 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLNodeLap_3D_K.H @@ -236,7 +236,7 @@ void mlndlap_semi_avgdown_coeff (int i, int j, int k, Array4 const& crse, if (idir == 2) { Real cl = fine(2*i ,2*j,k) + fine(2*i ,2*j+1,k); Real cr = fine(2*i+1,2*j,k) + fine(2*i+1,2*j+1,k); - crse(i,j,k) = cl*cr/(cl+cr); + crse(i,j,k) = cl*cr/(cl+cr); } else if (idir == 1) { Real cl = fine(2*i ,j,2*k) + fine(2*i ,j,2*k+1); Real cr = fine(2*i+1,j,2*k) + fine(2*i+1,j,2*k+1); @@ -1372,7 +1372,7 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& Real f4xm2ym2z = Real(4.0)*facx - Real(2.0)*facy - Real(2.0)*facz; Real fm2x4ym2z = -Real(2.0)*facx + Real(4.0)*facy - Real(2.0)*facz; Real fm2xm2y4z = -Real(2.0)*facx - Real(2.0)*facy + Real(4.0)*facz; - + const auto lo = amrex::lbound(bx); const auto hi = amrex::ubound(bx); @@ -1397,23 +1397,23 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& Array1D a_ls,b_ls,c_ls,u_ls,r_ls,gam; - if ( idir == 2 ) + if ( idir == 2 ) { - for (int j = lo.y; j <= hi.y; ++j) + for (int j = lo.y; j <= hi.y; ++j) { for (int i = lo.x; i <= hi.x; ++i) { for (int k = lo.z; k <= hi.z; ++k) { - if (msk(i,j,k)) + if (msk(i,j,k)) { a_ls(k-lo.z) = Real(0.); b_ls(k-lo.z) = Real(1.); c_ls(k-lo.z) = Real(0.); u_ls(k-lo.z) = Real(0.); r_ls(k-lo.z) = Real(0.); - } - else + } + else { Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1) + sig(i-1,j-1,k )+sig(i,j-1,k )+sig(i-1,j,k )+sig(i,j,k )); @@ -1453,7 +1453,7 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); for (int k = lo.z; k <= hi.z; ++k) - { + { sol(i,j,k) = u_ls(k-lo.z); } } @@ -1473,7 +1473,7 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& c_ls(j-lo.y) = Real(0.); u_ls(j-lo.y) = Real(0.); r_ls(j-lo.y) = Real(0.); - } + } else { Real s0 = Real(-4.0)*fxyz*(sig(i-1,j-1,k-1)+sig(i,j-1,k-1)+sig(i-1,j,k-1)+sig(i,j,k-1) @@ -1520,8 +1520,8 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& } } } - } - else if (idir == 0) + } + else if (idir == 0) { for (int j = lo.y; j <= hi.y; ++j) { @@ -1529,7 +1529,7 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& { for (int i = lo.x; i <= hi.x; ++i) { - if (msk(i,j,k)) + if (msk(i,j,k)) { a_ls(i-lo.x) = Real(0.); b_ls(i-lo.x) = Real(1.); @@ -1572,7 +1572,7 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& c_ls(i-lo.x) = f4xm2ym2z*(sig(i ,j-1,k-1)+sig(i ,j,k-1)+sig(i ,j-1,k)+sig(i ,j,k)); u_ls(i-lo.x) = Real(0.); r_ls(i-lo.x) = rhs(i,j,k) - Ax; - } + } } tridiagonal_solve(a_ls, b_ls, c_ls, r_ls, u_ls, gam, ilen); @@ -1582,11 +1582,11 @@ void mlndlap_gauss_seidel_with_line_solve_aa (Box const& bx, Array4 const& } } } - } + } else { - amrex::Abort("mlndlap_gauss_seidel_with_line_solve_aa is wrong direction."); - } + amrex::Abort("mlndlap_gauss_seidel_with_line_solve_aa is wrong direction."); + } } // @@ -1676,27 +1676,27 @@ AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mlndlap_semi_restriction (int i, int j, int k, Array4 const& crse, Array4 const& fine, Array4 const& msk, int idir) noexcept { - if (idir == 2) + if (idir == 2) { int ii = i*2; int jj = j*2; int kk = k; if (msk(ii,jj,kk)) { crse(i,j,k) = Real(0.0); - } else { // use 2-D formula + } else { // use 2-D formula crse(i,j,k) = Real(1./16.)*(fine(ii-1,jj-1,kk) + Real(2.)*fine(ii ,jj-1,kk) + fine(ii+1,jj-1,kk) + Real(2.)*fine(ii-1,jj ,kk) + Real(4.)*fine(ii ,jj ,kk) + Real(2.)*fine(ii+1,jj ,kk) + fine(ii-1,jj+1,kk) + Real(2.)*fine(ii ,jj+1,kk) + fine(ii+1,jj+1,kk)); } - } - else if (idir == 1) + } + else if (idir == 1) { int ii = i*2; int jj = j; int kk = k*2; if (msk(ii,jj,kk)) { crse(i,j,k) = Real(0.0); - } else { // use 2-D formula + } else { // use 2-D formula crse(i,j,k) = Real(1./16.)*(fine(ii-1,jj,kk-1) + Real(2.)*fine(ii ,jj,kk-1) + fine(ii+1,jj,kk-1) + Real(2.)*fine(ii-1,jj ,kk) + Real(4.)*fine(ii ,jj,kk ) + Real(2.)*fine(ii+1,jj,kk ) + fine(ii-1,jj,kk+1) + Real(2.)*fine(ii ,jj,kk+1) + fine(ii+1,jj,kk+1)); @@ -1709,13 +1709,13 @@ void mlndlap_semi_restriction (int i, int j, int k, Array4 const& crse, int kk = k*2; if (msk(ii,jj,kk)) { crse(i,j,k) = Real(0.0); - } else { // use 2-D formula + } else { // use 2-D formula crse(i,j,k) = Real(1./16.)*(fine(ii,jj-1,kk-1) + Real(2.)*fine(ii ,jj,kk-1) + fine(ii,jj+1,kk-1) + Real(2.)*fine(ii,jj-1 ,kk) + Real(4.)*fine(ii ,jj,kk ) + Real(2.)*fine(ii,jj+1,kk ) + fine(ii,jj-1,kk+1) + Real(2.)*fine(ii ,jj,kk+1) + fine(ii,jj+1,kk+1)); } } - else + else { amrex::Abort("mlndlap_semi_restriction semi direction wrong semi-direction. "); } @@ -1904,9 +1904,9 @@ void mlndlap_semi_interpadd_aa (int i, int j, int k, Array4 const& fine, Array4 const& crse, Array4 const& sig, Array4 const& msk, int idir) noexcept { - if (idir == 2 ) + if (idir == 2 ) { - if (!msk(i,j,k)) { + if (!msk(i,j,k)) { int ic = amrex::coarsen(i,2); int jc = amrex::coarsen(j,2); int kc = k; @@ -1923,10 +1923,10 @@ void mlndlap_semi_interpadd_aa (int i, int j, int k, Array4 const& fine, // Node on Y line fine(i,j,k) += aa_interp_line_y(crse,sig,i,j,k,ic,jc,kc); } else { - // Node coincident with coarse node + // Node coincident with coarse node fine(i,j,k) += crse(ic,jc,kc); } - } + } } else if (idir ==1 ){ if (!msk(i,j,k)) { int ic = amrex::coarsen(i,2); @@ -1937,7 +1937,7 @@ void mlndlap_semi_interpadd_aa (int i, int j, int k, Array4 const& fine, if (i_is_odd && k_is_odd) { // Node on a X-Z face - fine(i,j,k) += aa_interp_face_xz(crse,sig,i,j,k,ic,jc,kc); + fine(i,j,k) += aa_interp_face_xz(crse,sig,i,j,k,ic,jc,kc); } else if (i_is_odd) { // Node on X line fine(i,j,k) += aa_interp_line_x(crse,sig,i,j,k,ic,jc,kc); @@ -1971,7 +1971,7 @@ void mlndlap_semi_interpadd_aa (int i, int j, int k, Array4 const& fine, fine(i,j,k) += crse(ic,jc,kc); } } - } else { + } else { amrex::Abort("mlndlap_semi_interpolation semi direction wrong semi-direction. "); } } @@ -6881,7 +6881,7 @@ void mlndlap_mknewu_eb_c (int i, int j, int k, Array4 const& u, Array4 const& garr = grad.array(mfi); - Array4 const& solarr = sol.const_array(mfi); + Array4 const& garr = grad.array(mfi); + Array4 const& solarr = sol.const_array(mfi); - AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, AMREX_SPACEDIM, i, j, k, n, + AMREX_HOST_DEVICE_PARALLEL_FOR_4D ( bx, AMREX_SPACEDIM, i, j, k, n, { garr(i,j,k,n) = 0.0; }); #ifdef AMREX_USE_EB - bool regular = !factory; - if (factory) - { - auto type = (*flags)[mfi].getType(bx); - Array4 const& vfracarr = vfrac->const_array(mfi); - Array4 const& intgarr = intg->const_array(mfi); - if (type == FabType::covered) - { } - else if (type == FabType::singlevalued) - { - AMREX_HOST_DEVICE_FOR_3D(bx, i, j, k, + bool regular = !factory; + if (factory) + { + auto type = (*flags)[mfi].getType(bx); + Array4 const& vfracarr = vfrac->const_array(mfi); + Array4 const& intgarr = intg->const_array(mfi); + if (type == FabType::covered) + { } + else if (type == FabType::singlevalued) + { + AMREX_HOST_DEVICE_FOR_3D(bx, i, j, k, { - mlndlap_mknewu_eb_c(i,j,k, garr, solarr, sigma, vfracarr, intgarr, dxinv); + mlndlap_mknewu_eb_c(i,j,k, garr, solarr, sigma, vfracarr, intgarr, dxinv); }); - } - else - { - regular = true; - } - } - if (regular) + } + else + { + regular = true; + } + } + if (regular) #endif - { + { #if (AMREX_SPACEDIM == 2) - AMREX_HOST_DEVICE_PARALLEL_FOR_3D (bx, i, j, k, - { - mlndlap_mknewu_c(i,j,k,garr,solarr,sigma,dxinv,is_rz); - }); + AMREX_HOST_DEVICE_PARALLEL_FOR_3D (bx, i, j, k, + { + mlndlap_mknewu_c(i,j,k,garr,solarr,sigma,dxinv,is_rz); + }); #else - AMREX_HOST_DEVICE_PARALLEL_FOR_3D (bx, i, j, k, + AMREX_HOST_DEVICE_PARALLEL_FOR_3D (bx, i, j, k, { - mlndlap_mknewu_c(i,j,k,garr,solarr,sigma,dxinv); - }); + mlndlap_mknewu_c(i,j,k,garr,solarr,sigma,dxinv); + }); #endif - } + } } } @@ -1391,7 +1391,7 @@ MLNodeLaplacian::interpolation (int amrlev, int fmglev, MultiFab& fine, const Mu }); } else - { + { Array4 const& sfab = sigma[0]->const_array(mfi); if (regular_coarsening) { diff --git a/Src/LinearSolvers/MLMG/AMReX_MLPoisson.cpp b/Src/LinearSolvers/MLMG/AMReX_MLPoisson.cpp index 1af1d36c704..9f202b61a39 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLPoisson.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLPoisson.cpp @@ -69,7 +69,7 @@ MLPoisson::prepareForSolve () if (m_domain_covered[alev] && !m_overset_mask[alev][0]) { m_is_singular[alev] = true; - } + } } } } @@ -231,7 +231,7 @@ MLPoisson::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs, i #endif for (MFIter mfi(sol,mfi_info); mfi.isValid(); ++mfi) { - const auto& m0 = mm0.array(mfi); + const auto& m0 = mm0.array(mfi); const auto& m1 = mm1.array(mfi); #if (AMREX_SPACEDIM > 1) const auto& m2 = mm2.array(mfi); @@ -242,7 +242,7 @@ MLPoisson::Fsmooth (int amrlev, int mglev, MultiFab& sol, const MultiFab& rhs, i #endif #endif - const Box& tbx = mfi.tilebox(); + const Box& tbx = mfi.tilebox(); const Box& vbx = mfi.validbox(); const auto& solnfab = sol.array(mfi); const auto& rhsfab = rhs.array(mfi); @@ -580,7 +580,7 @@ MLPoisson::makeNLinOp (int grid_size) const nop->setACoeffs(0, alpha); - return r; + return r; } } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLPoisson_1D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLPoisson_1D_K.H index 691c24dda07..a10d9dbef54 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLPoisson_1D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLPoisson_1D_K.H @@ -83,7 +83,7 @@ void mlpoisson_flux_xface_m (Box const& box, Array4 const& fx, const auto lo = amrex::lbound(box); int i = lo.x; - Real re = (probxlo + i*dx) * (probxlo + i*dx); + Real re = (probxlo + i*dx) * (probxlo + i*dx); fx(i,0,0) = dxinv*re*(sol(i,0,0)-sol(i-1,0,0)); i += xlen; re = (probxlo + i*dx) * (probxlo + i*dx); @@ -187,7 +187,7 @@ void mlpoisson_gsrb_m (Box const& box, Array4 const& phi, Array4& fluxes, MultiFab& sol, Location loc) const override; - + void compVelGrad (int amrlev, const Array& fluxes, MultiFab& sol, Location loc) const; - + protected: bool m_needs_update = true; @@ -102,7 +102,7 @@ public: // for cuda void applyBCTensor (int amrlev, int mglev, MultiFab& vel, BCMode bc_mode, StateMode s_mode, const MLMGBndry* bndry - ) const; + ) const; }; } diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp index ddb7f60c0bb..57853594648 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensorOp.cpp @@ -332,13 +332,13 @@ MLTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, const auto& myhi = maskvals[Orientation(1,Orientation::high)].array(mfi); const auto& bvxlo = (bndry != nullptr) ? - (*bndry)[Orientation(0,Orientation::low )].array(mfi) : foo; + (*bndry)[Orientation(0,Orientation::low )].array(mfi) : foo; const auto& bvylo = (bndry != nullptr) ? - (*bndry)[Orientation(1,Orientation::low )].array(mfi) : foo; + (*bndry)[Orientation(1,Orientation::low )].array(mfi) : foo; const auto& bvxhi = (bndry != nullptr) ? - (*bndry)[Orientation(0,Orientation::high)].array(mfi) : foo; + (*bndry)[Orientation(0,Orientation::high)].array(mfi) : foo; const auto& bvyhi = (bndry != nullptr) ? - (*bndry)[Orientation(1,Orientation::high)].array(mfi) : foo; + (*bndry)[Orientation(1,Orientation::high)].array(mfi) : foo; #if (AMREX_SPACEDIM == 2) @@ -355,11 +355,11 @@ MLTensorOp::applyBCTensor (int amrlev, int mglev, MultiFab& vel, const auto& mzhi = maskvals[Orientation(2,Orientation::high)].array(mfi); const auto& bvzlo = (bndry != nullptr) ? - (*bndry)[Orientation(2,Orientation::low )].array(mfi) : foo; + (*bndry)[Orientation(2,Orientation::low )].array(mfi) : foo; const auto& bvzhi = (bndry != nullptr) ? - (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; + (*bndry)[Orientation(2,Orientation::high)].array(mfi) : foo; - // only edge vals used in 3D stencil + // only edge vals used in 3D stencil AMREX_HOST_DEVICE_FOR_1D ( 12, iedge, { mltensor_fill_edges(iedge, vbx, velfab, @@ -414,7 +414,7 @@ MLTensorOp::compFlux (int amrlev, const Array& fluxes, AMREX_D_TERM(Box const xbx = mfi.nodaltilebox(0);, Box const ybx = mfi.nodaltilebox(1);, Box const zbx = mfi.nodaltilebox(2);); - AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM);, + AMREX_D_TERM(fluxfab_tmp[0].resize(xbx,AMREX_SPACEDIM);, fluxfab_tmp[1].resize(ybx,AMREX_SPACEDIM);, fluxfab_tmp[2].resize(zbx,AMREX_SPACEDIM);); AMREX_D_TERM(Elixir fxeli = fluxfab_tmp[0].elixir();, @@ -439,11 +439,11 @@ MLTensorOp::compFlux (int amrlev, const Array& fluxes, } ); - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const Box& nbx = mfi.nodaltilebox(idim); + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + const Box& nbx = mfi.nodaltilebox(idim); Array4 dst = fluxes[idim]->array(mfi); - Array4 src = fluxfab_tmp[idim].const_array(); - AMREX_HOST_DEVICE_PARALLEL_FOR_4D (nbx, ncomp, i, j, k, n, + Array4 src = fluxfab_tmp[idim].const_array(); + AMREX_HOST_DEVICE_PARALLEL_FOR_4D (nbx, ncomp, i, j, k, n, { dst(i,j,k,n) += bscalar*src(i,j,k,n); }); @@ -510,11 +510,11 @@ MLTensorOp::compVelGrad (int amrlev, const Array& flux ); // The derivatives are put in the array with the following order: -// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 -// in 2D: dU/dx, dV/dx, dU/dy, dV/dy -// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz - - +// component: 0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 +// in 2D: dU/dx, dV/dx, dU/dy, dV/dy +// in 3D: dU/dx, dV/dx, dW/dx, dU/dy, dV/dy, dW/dy, dU/dz, dV/dz, dW/dz + + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const Box& nbx = mfi.nodaltilebox(idim); Array4 dst = fluxes[idim]->array(mfi); diff --git a/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H b/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H index 12e5f381570..a4a4c7df9ef 100644 --- a/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H +++ b/Src/LinearSolvers/MLMG/AMReX_MLTensor_3D_K.H @@ -518,7 +518,7 @@ void mltensor_fill_corners (int icorner, Box const& vbox, // vbox: the valid box } } } - + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE void mltensor_fill_edges (int iedge, Box const& vbox, // vbox: the valid box Array4 const& vel, @@ -536,7 +536,7 @@ void mltensor_fill_edges (int iedge, Box const& vbox, // vbox: the valid box Array4 const& bcvalzhi, GpuArray const& bct, GpuArray const& bcl, - int inhomog, int maxorder, + int inhomog, int maxorder, GpuArray const& dxinv, Box const& domain) noexcept { constexpr int oxlo = 0; @@ -1148,7 +1148,7 @@ void mltensor_cross_terms_fz (Box const& box, Array4 const& fz, fz(i,j,k,2) = -mun*(-twoThirds*divu) - xif*divu; } } - } + } } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -1237,19 +1237,19 @@ void mltensor_vel_grads_fx (Box const& box, Array4 const& fx, for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - + Real dudx = (vel(i,j,k,0) - vel(i-1,j,k,0))*dxi; Real dvdx = (vel(i,j,k,1) - vel(i-1,j,k,1))*dxi; Real dwdx = (vel(i,j,k,2) - vel(i-1,j,k,2))*dxi; - + Real dudy = (vel(i,j+1,k,0)+vel(i-1,j+1,k,0)-vel(i,j-1,k,0)-vel(i-1,j-1,k,0))*(Real(0.25)*dyi); Real dvdy = (vel(i,j+1,k,1)+vel(i-1,j+1,k,1)-vel(i,j-1,k,1)-vel(i-1,j-1,k,1))*(Real(0.25)*dyi); Real dwdy = (vel(i,j+1,k,2)+vel(i-1,j+1,k,2)-vel(i,j-1,k,2)-vel(i-1,j-1,k,2))*(Real(0.25)*dyi); - + Real dudz = (vel(i,j,k+1,0)+vel(i-1,j,k+1,0)-vel(i,j,k-1,0)-vel(i-1,j,k-1,0))*(Real(0.25)*dzi); Real dvdz = (vel(i,j,k+1,1)+vel(i-1,j,k+1,1)-vel(i,j,k-1,1)-vel(i-1,j,k-1,1))*(Real(0.25)*dzi); Real dwdz = (vel(i,j,k+1,2)+vel(i-1,j,k+1,2)-vel(i,j,k-1,2)-vel(i-1,j,k-1,2))*(Real(0.25)*dzi); - + fx(i,j,k,0) = dudx; fx(i,j,k,1) = dvdx; fx(i,j,k,2) = dwdx; @@ -1259,7 +1259,7 @@ void mltensor_vel_grads_fx (Box const& box, Array4 const& fx, fx(i,j,k,6) = dudz; fx(i,j,k,7) = dvdz; fx(i,j,k,8) = dwdz; - + } } } @@ -1280,19 +1280,19 @@ void mltensor_vel_grads_fy (Box const& box, Array4 const& fy, for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - + Real dudx = (vel(i+1,j,k,0)+vel(i+1,j-1,k,0)-vel(i-1,j,k,0)-vel(i-1,j-1,k,0))*(Real(0.25)*dxi); Real dvdx = (vel(i+1,j,k,1)+vel(i+1,j-1,k,1)-vel(i-1,j,k,1)-vel(i-1,j-1,k,1))*(Real(0.25)*dxi); Real dwdx = (vel(i+1,j,k,2)+vel(i+1,j-1,k,2)-vel(i-1,j,k,2)-vel(i-1,j-1,k,2))*(Real(0.25)*dxi); - + Real dudy = (vel(i,j,k,0) - vel(i,j-1,k,0))*dyi; Real dvdy = (vel(i,j,k,1) - vel(i,j-1,k,1))*dyi; Real dwdy = (vel(i,j,k,2) - vel(i,j-1,k,2))*dyi; - + Real dudz = (vel(i,j,k+1,0)+vel(i,j-1,k+1,0)-vel(i,j,k-1,0)-vel(i,j-1,k-1,0))*(Real(0.25)*dzi); Real dvdz = (vel(i,j,k+1,1)+vel(i,j-1,k+1,1)-vel(i,j,k-1,1)-vel(i,j-1,k-1,1))*(Real(0.25)*dzi); Real dwdz = (vel(i,j,k+1,2)+vel(i,j-1,k+1,2)-vel(i,j,k-1,2)-vel(i,j-1,k-1,2))*(Real(0.25)*dzi); - + fy(i,j,k,0) = dudx; fy(i,j,k,1) = dvdx; fy(i,j,k,2) = dwdx; @@ -1323,19 +1323,19 @@ void mltensor_vel_grads_fz (Box const& box, Array4 const& fz, for (int j = lo.y; j <= hi.y; ++j) { AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { - + Real dudx = (vel(i+1,j,k,0)+vel(i+1,j,k-1,0)-vel(i-1,j,k,0)-vel(i-1,j,k-1,0))*(Real(0.25)*dxi); Real dvdx = (vel(i+1,j,k,1)+vel(i+1,j,k-1,1)-vel(i-1,j,k,1)-vel(i-1,j,k-1,1))*(Real(0.25)*dxi); Real dwdx = (vel(i+1,j,k,2)+vel(i+1,j,k-1,2)-vel(i-1,j,k,2)-vel(i-1,j,k-1,2))*(Real(0.25)*dxi); - + Real dudy = (vel(i,j+1,k,0)+vel(i,j+1,k-1,0)-vel(i,j-1,k,0)-vel(i,j-1,k-1,0))*(Real(0.25)*dyi); Real dvdy = (vel(i,j+1,k,1)+vel(i,j+1,k-1,1)-vel(i,j-1,k,1)-vel(i,j-1,k-1,1))*(Real(0.25)*dyi); Real dwdy = (vel(i,j+1,k,2)+vel(i,j+1,k-1,2)-vel(i,j-1,k,2)-vel(i,j-1,k-1,2))*(Real(0.25)*dyi); - + Real dudz = (vel(i,j,k,0) - vel(i,j,k-1,0))*dzi; Real dvdz = (vel(i,j,k,1) - vel(i,j,k-1,1))*dzi; Real dwdz = (vel(i,j,k,2) - vel(i,j,k-1,2))*dzi; - + fz(i,j,k,0) = dudx; fz(i,j,k,1) = dvdx; fz(i,j,k,2) = dwdx; @@ -1348,7 +1348,7 @@ void mltensor_vel_grads_fz (Box const& box, Array4 const& fz, } } - } + } } } diff --git a/Src/LinearSolvers/Projections/AMReX_MacProjector.H b/Src/LinearSolvers/Projections/AMReX_MacProjector.H index b3e54c9aa12..98c084ff6b9 100644 --- a/Src/LinearSolvers/Projections/AMReX_MacProjector.H +++ b/Src/LinearSolvers/Projections/AMReX_MacProjector.H @@ -136,7 +136,7 @@ public: // // Setters and getters // - void setVerbose (int v) noexcept + void setVerbose (int v) noexcept { m_verbose = v; m_mlmg->setVerbose(m_verbose); } diff --git a/Src/LinearSolvers/Projections/AMReX_NodalProjector.cpp b/Src/LinearSolvers/Projections/AMReX_NodalProjector.cpp index a71b52e63a7..f6e8c83be05 100644 --- a/Src/LinearSolvers/Projections/AMReX_NodalProjector.cpp +++ b/Src/LinearSolvers/Projections/AMReX_NodalProjector.cpp @@ -293,7 +293,7 @@ NodalProjector::project ( Real a_rtol, Real a_atol ) MultiFab::Add( *m_vel[lev], m_fluxes[lev], 0, 0, AMREX_SPACEDIM, 0); // set m_fluxes = grad(phi) - m_linop->compGrad(lev,m_fluxes[lev],m_phi[lev]); + m_linop->compGrad(lev,m_fluxes[lev],m_phi[lev]); } // diff --git a/Src/Particle/AMReX_ArrayOfStructs.H b/Src/Particle/AMReX_ArrayOfStructs.H index 80beee39c2c..50ce5883716 100644 --- a/Src/Particle/AMReX_ArrayOfStructs.H +++ b/Src/Particle/AMReX_ArrayOfStructs.H @@ -76,7 +76,7 @@ public: RealType* dataPtr () { return data(); } std::pair dataShape () const { - return std::make_pair(SizeInReal, static_cast(m_data.size())); + return std::make_pair(SizeInReal, static_cast(m_data.size())); } void push_back (const ParticleType& p) { return m_data.push_back(p); } diff --git a/Src/Particle/AMReX_BinIterator.H b/Src/Particle/AMReX_BinIterator.H index ae3fc89c11c..05bf2eb5064 100644 --- a/Src/Particle/AMReX_BinIterator.H +++ b/Src/Particle/AMReX_BinIterator.H @@ -13,33 +13,33 @@ template struct BinIterator { using index_type = unsigned int; - + struct iterator { AMREX_GPU_HOST_DEVICE - iterator (index_type start, index_type stop, const index_type* a_perm, const T* a_items) + iterator (index_type start, index_type stop, const index_type* a_perm, const T* a_items) : m_items(a_items), m_perm(a_perm), m_index(start), m_stop(stop) {} - + AMREX_GPU_HOST_DEVICE void operator++ () { ++m_index;; } - + AMREX_GPU_HOST_DEVICE bool operator!= (iterator const& /*rhs*/) const { return m_index < m_stop; } - + AMREX_GPU_HOST_DEVICE std::pair operator* () const { return std::make_pair(m_perm[m_index], m_items[m_perm[m_index]]); } - + private: const T* m_items; const index_type* m_perm; index_type m_index; index_type m_stop; }; - + AMREX_GPU_HOST_DEVICE iterator begin () const { @@ -57,18 +57,18 @@ struct BinIterator return iterator(m_offsets_ptr[m_i+1], m_offsets_ptr[m_i+1], m_permutation_ptr, m_items); } - AMREX_GPU_HOST_DEVICE + AMREX_GPU_HOST_DEVICE BinIterator (index_type i, const index_type *offsets_ptr, const index_type *permutation_ptr, const T* items) : m_i(i), m_offsets_ptr(offsets_ptr), m_permutation_ptr(permutation_ptr), m_items(items) {} - + private: const index_type m_i; const index_type * m_offsets_ptr; const index_type * m_permutation_ptr; const T * m_items; - static constexpr index_type m_not_found = std::numeric_limits::max(); + static constexpr index_type m_not_found = std::numeric_limits::max(); }; } diff --git a/Src/Particle/AMReX_NeighborParticlesGPUImpl.H b/Src/Particle/AMReX_NeighborParticlesGPUImpl.H index b7efbfcf046..58a57b3f89e 100644 --- a/Src/Particle/AMReX_NeighborParticlesGPUImpl.H +++ b/Src/Particle/AMReX_NeighborParticlesGPUImpl.H @@ -73,7 +73,7 @@ buildNeighborMask () for (MFIter mfi(ba, dmap); mfi.isValid(); ++mfi) { int grid = mfi.index(); - int num_codes = 0; + int num_codes = 0; std::set neighbor_grids; for (auto pit=pshifts.cbegin(); pit!=pshifts.cend(); ++pit) @@ -107,7 +107,7 @@ buildNeighborMask () amrex::grow(ba[mfi], -m_num_neighbor_cells), m_num_neighbor_cells); m_grid_map[grid].resize(bl.size()); - m_code_offsets[grid].push_back(0); + m_code_offsets[grid].push_back(0); for (int i = 0; i < static_cast(bl.size()); ++i) { const Box& box = bl[i]; @@ -124,14 +124,14 @@ buildNeighborMask () code.grid_id = isec_grids[isec.first]; code.periodic_shift = isec_pshifts[isec.first]; m_grid_map[grid][i].push_back(code); - m_code_array[grid].push_back(code); + m_code_array[grid].push_back(code); const int global_rank = dmap[code.grid_id]; neighbor_procs.push_back(ParallelContext::global_to_local_rank(global_rank)); - ++num_codes; + ++num_codes; } - m_code_offsets[grid].push_back(m_code_array[grid].size()); + m_code_offsets[grid].push_back(m_code_array[grid].size()); } - } + } RemoveDuplicates(neighbor_procs); } } @@ -168,58 +168,58 @@ buildNeighborCopyOp () Array4 const& mask_arr = m_neighbor_mask_ptr->array(mfi); - Gpu::DeviceVector counts(np + 1, 0); - Gpu::DeviceVector offsets(np + 1); - auto p_counts = counts.dataPtr(); - auto p_offsets = offsets.dataPtr(); + Gpu::DeviceVector counts(np + 1, 0); + Gpu::DeviceVector offsets(np + 1); + auto p_counts = counts.dataPtr(); + auto p_offsets = offsets.dataPtr(); ParticleType* p_ptr = &(aos[0]); - auto p_code_array = m_code_array[gid].dataPtr(); - auto p_code_offsets = m_code_offsets[gid].dataPtr(); - AMREX_FOR_1D ( np, i, + auto p_code_array = m_code_array[gid].dataPtr(); + auto p_code_offsets = m_code_offsets[gid].dataPtr(); + AMREX_FOR_1D ( np, i, { IntVect iv = getParticleCell(p_ptr[i], plo, dxi, domain); - int code = mask_arr(iv); - if (code >= 0) + int code = mask_arr(iv); + if (code >= 0) { int start = p_code_offsets[code]; int stop = p_code_offsets[code+1]; p_counts[i] += stop - start; - } - }); + } + }); amrex::Gpu::exclusive_scan(counts.begin(), counts.end(), offsets.begin()); - int num_copies; + int num_copies; Gpu::dtoh_memcpy(&num_copies, offsets.data()+np, sizeof(int)); - neighbor_copy_op.resize(gid, lev, num_copies); + neighbor_copy_op.resize(gid, lev, num_copies); - auto p_boxes = neighbor_copy_op.m_boxes[lev][gid].dataPtr(); + auto p_boxes = neighbor_copy_op.m_boxes[lev][gid].dataPtr(); auto p_levs = neighbor_copy_op.m_levels[lev][gid].dataPtr(); - auto p_src_indices = neighbor_copy_op.m_src_indices[lev][gid].dataPtr(); - auto p_periodic_shift = neighbor_copy_op.m_periodic_shift[lev][gid].dataPtr(); + auto p_src_indices = neighbor_copy_op.m_src_indices[lev][gid].dataPtr(); + auto p_periodic_shift = neighbor_copy_op.m_periodic_shift[lev][gid].dataPtr(); - AMREX_FOR_1D ( np, i, + AMREX_FOR_1D ( np, i, { IntVect iv = getParticleCell(p_ptr[i], plo, dxi, domain); - int code = mask_arr(iv); - if (code >= 0) - { - int k = 0; - int start = p_code_offsets[code]; - int stop = p_code_offsets[code+1]; - for (int j = start; j < stop; ++j) - { + int code = mask_arr(iv); + if (code >= 0) + { + int k = 0; + int start = p_code_offsets[code]; + int stop = p_code_offsets[code+1]; + for (int j = start; j < stop; ++j) + { int index = p_offsets[i]+k; p_boxes[index] = p_code_array[j].grid_id; p_levs[index] = 0; p_periodic_shift[index] = p_code_array[j].periodic_shift; p_src_indices[index] = i; ++k; - } - } - }); + } + } + }); Gpu::streamSynchronize(); } diff --git a/Src/Particle/AMReX_ParGDB.H b/Src/Particle/AMReX_ParGDB.H index a519c117da1..8637bc29657 100644 --- a/Src/Particle/AMReX_ParGDB.H +++ b/Src/Particle/AMReX_ParGDB.H @@ -64,18 +64,18 @@ public: ParGDB () { ; } ParGDB (const Geometry & geom, - const DistributionMapping & dmap, - const BoxArray & ba); + const DistributionMapping & dmap, + const BoxArray & ba); ParGDB (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr); + const Vector & dmap, + const Vector & ba, + const Vector & rr); ParGDB (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr); + const Vector & dmap, + const Vector & ba, + const Vector & rr); virtual ~ParGDB () {;} @@ -100,7 +100,7 @@ public: virtual const Vector& boxArray () const override; virtual void SetParticleBoxArray (int level, const BoxArray& new_ba) override; - virtual void SetParticleDistributionMap (int level, const DistributionMapping& new_dm) override; + virtual void SetParticleDistributionMap (int level, const DistributionMapping& new_dm) override; virtual void SetParticleGeometry (int level, const Geometry& new_geom) override; virtual void ClearParticleBoxArray (int level) override; @@ -130,13 +130,13 @@ bool ParGDBBase::OnSameGrids (int level, const MultiFab& mf) const { return (mf.DistributionMap() == ParticleDistributionMap(level) && - mf.boxArray().CellEqual(ParticleBoxArray(level))); + mf.boxArray().CellEqual(ParticleBoxArray(level))); } inline ParGDB::ParGDB (const Geometry & geom, - const DistributionMapping & dmap, - const BoxArray & ba) + const DistributionMapping & dmap, + const BoxArray & ba) : m_geom(1,geom), m_dmap(1,dmap), @@ -146,9 +146,9 @@ ParGDB::ParGDB (const Geometry & geom, inline ParGDB::ParGDB (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr) + const Vector & dmap, + const Vector & ba, + const Vector & rr) : m_geom(geom), m_dmap(dmap), @@ -159,9 +159,9 @@ ParGDB::ParGDB (const Vector & geom, inline ParGDB::ParGDB (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr) + const Vector & dmap, + const Vector & ba, + const Vector & rr) : m_geom(geom), m_dmap(dmap), diff --git a/Src/Particle/AMReX_Particle.H b/Src/Particle/AMReX_Particle.H index 4e4d3463a72..6bdea4bb46f 100644 --- a/Src/Particle/AMReX_Particle.H +++ b/Src/Particle/AMReX_Particle.H @@ -356,7 +356,7 @@ Particle::NextID () next = the_next_id++; if (next > LastParticleID) - amrex::Abort("Particle::NextID() -- too many particles"); + amrex::Abort("Particle::NextID() -- too many particles"); return next; } @@ -367,7 +367,7 @@ Particle::UnprotectedNextID () { Long next = the_next_id++; if (next > LastParticleID) - amrex::Abort("Particle::NextID() -- too many particles"); + amrex::Abort("Particle::NextID() -- too many particles"); return next; } diff --git a/Src/Particle/AMReX_ParticleCommunication.cpp b/Src/Particle/AMReX_ParticleCommunication.cpp index 2a336d2ca24..d9d91de00eb 100644 --- a/Src/Particle/AMReX_ParticleCommunication.cpp +++ b/Src/Particle/AMReX_ParticleCommunication.cpp @@ -74,7 +74,7 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) auto box_buffer_indices = map.allBucketsOnProc(i); Long nbytes = 0; for (auto bucket : box_buffer_indices) - { + { int dst = map.bucketToGrid(bucket); int lev = map.bucketToLevel(bucket); AMREX_ASSERT(m_box_counts_h[bucket] <= std::numeric_limits::max()); @@ -87,9 +87,9 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) snd_data[i].push_back(lev); snd_data[i].push_back(MyProc); nbytes += 4*sizeof(int); - } - m_Snds[i] = nbytes; - m_NumSnds += nbytes; + } + m_Snds[i] = nbytes; + m_NumSnds += nbytes; } doHandShake(m_Snds, m_Rcvs); @@ -121,7 +121,7 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) } m_RcvProc.resize(0); - m_rOffset.resize(0); + m_rOffset.resize(0); std::size_t TotRcvBytes = 0; for (auto i : m_neighbor_procs) { @@ -132,7 +132,7 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) TotRcvBytes += m_Rcvs[i]; } } - + m_nrcvs = m_RcvProc.size(); m_build_stats.resize(0); @@ -142,20 +142,20 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) m_build_rreqs.resize(m_nrcvs); m_rcv_data.resize(TotRcvBytes/sizeof(int)); - + for (int i = 0; i < m_nrcvs; ++i) { const auto Who = m_RcvProc[i]; const auto offset = m_rOffset[i]; const auto Cnt = m_Rcvs[Who]; - + AMREX_ASSERT(Cnt > 0); AMREX_ASSERT(Cnt < std::numeric_limits::max()); AMREX_ASSERT(Who >= 0 && Who < NProcs); - + m_build_rreqs[i] = ParallelDescriptor::Arecv((char*) (m_rcv_data.dataPtr() + offset), Cnt, Who, SeqNum, ParallelContext::CommunicatorSub()).req(); } - + for (auto i : m_neighbor_procs) { if (i == MyProc) continue; @@ -174,7 +174,7 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) m_snd_counts.resize(0); m_snd_offsets.resize(0); m_snd_pad_correction_h.resize(0); - + m_snd_offsets.push_back(0); m_snd_pad_correction_h.push_back(0); for (int i = 0; i < NProcs; ++i) @@ -192,7 +192,7 @@ void ParticleCopyPlan::buildMPIStart (const ParticleBufferMap& map, Long psize) { m_snd_pad_correction_h[i] = m_snd_offsets[i] - m_snd_pad_correction_h[i]; } - + m_snd_pad_correction_d.resize(m_snd_pad_correction_h.size()); Gpu::copy(Gpu::hostToDevice, m_snd_pad_correction_h.begin(), m_snd_pad_correction_h.end(), m_snd_pad_correction_d.begin()); diff --git a/Src/Particle/AMReX_ParticleContainerBase.H b/Src/Particle/AMReX_ParticleContainerBase.H index 893357d0168..3d802af7dbe 100644 --- a/Src/Particle/AMReX_ParticleContainerBase.H +++ b/Src/Particle/AMReX_ParticleContainerBase.H @@ -31,7 +31,7 @@ public: ParticleContainerBase (ParGDBBase* gdb) : m_verbose(0), - m_gdb(gdb) + m_gdb(gdb) {} ParticleContainerBase (const Geometry & geom, @@ -39,7 +39,7 @@ public: const BoxArray & ba) : m_verbose(0), - m_gdb_object(geom,dmap,ba) + m_gdb_object(geom,dmap,ba) { m_gdb = &m_gdb_object; } @@ -48,9 +48,9 @@ public: const Vector & dmap, const Vector & ba, const Vector & rr) - : + : m_verbose(0), - m_gdb_object(geom,dmap,ba,rr) + m_gdb_object(geom,dmap,ba,rr) { m_gdb = &m_gdb_object; } @@ -59,9 +59,9 @@ public: const Vector & dmap, const Vector & ba, const Vector & rr) - : + : m_verbose(0), - m_gdb_object(geom,dmap,ba, [&]() -> Vector { + m_gdb_object(geom,dmap,ba, [&]() -> Vector { Vector ref_ratio; for (int i = 0; i < static_cast(rr.size()); ++i) { @@ -81,13 +81,13 @@ public: void Define (ParGDBBase* gdb) { m_gdb = gdb;} void Define (const Geometry & geom, - const DistributionMapping & dmap, - const BoxArray & ba); + const DistributionMapping & dmap, + const BoxArray & ba); void Define (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr); + const Vector & dmap, + const Vector & ba, + const Vector & rr); void reserveData (); void resizeData (); @@ -140,14 +140,14 @@ public: //! \param lev The level. //! const BoxArray& ParticleBoxArray (int lev) const - { return m_gdb->ParticleBoxArray(lev); } + { return m_gdb->ParticleBoxArray(lev); } //! \brief Get the DistributionMapping for a given level //! //! \param lev The level. //! const DistributionMapping& ParticleDistributionMap (int lev) const - { return m_gdb->ParticleDistributionMap(lev); } + { return m_gdb->ParticleDistributionMap(lev); } //! \brief Get the Geometry for a given level //! diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 9af7156947a..1f314ce494d 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -97,11 +97,11 @@ template bool ParticleContainer ::Where (const P& p, - ParticleLocData& pld, - int lev_min, - int lev_max, - int nGrow, - int local_grid) const + ParticleLocData& pld, + int lev_min, + int lev_max, + int nGrow, + int local_grid) const { AMREX_ASSERT(m_gdb != 0); @@ -143,13 +143,13 @@ ParticleContainer if (grid >= 0) { const Box& bx = ba.getCellCenteredBox(grid); - pld.m_lev = lev; - pld.m_grid = grid; - pld.m_tile = getTileIndex(iv, bx, do_tiling, tile_size, pld.m_tilebox); - pld.m_cell = iv; - pld.m_gridbox = bx; + pld.m_lev = lev; + pld.m_grid = grid; + pld.m_tile = getTileIndex(iv, bx, do_tiling, tile_size, pld.m_tilebox); + pld.m_cell = iv; + pld.m_gridbox = bx; pld.m_grown_gridbox = amrex::grow(bx, nGrow); - return true; + return true; } } @@ -161,10 +161,10 @@ template ::EnforcePeriodicWhere (ParticleType& p, - ParticleLocData& pld, - int lev_min, - int lev_max, - int local_grid) const + ParticleLocData& pld, + int lev_min, + int lev_max, + int local_grid) const { AMREX_ASSERT(m_gdb != 0); @@ -182,31 +182,31 @@ ParticleContainer std::vector< std::pair > isects; for (int lev = lev_max; lev >= lev_min; lev--) { - int grid; + int grid; IntVect iv; const BoxArray& ba = ParticleBoxArray(lev); AMREX_ASSERT(ba.ixType().cellCentered()); - if (local_grid < 0) { + if (local_grid < 0) { iv = Index(p_prime, lev); ba.intersections(Box(iv, iv), isects, true, 0); grid = isects.empty() ? -1 : isects[0].first; - } else { + } else { iv = Index(p_prime, lev); if (ba[local_grid].contains(iv)) { grid = local_grid; } - else - { - ba.intersections(Box(iv, iv), isects, true, 0); - grid = isects.empty() ? -1 : isects[0].first; - if(grid == -1) - { - grid = (*redistribute_mask_ptr)[local_grid](Index(p, lev), 0); - } - } - } + else + { + ba.intersections(Box(iv, iv), isects, true, 0); + grid = isects.empty() ? -1 : isects[0].first; + if(grid == -1) + { + grid = (*redistribute_mask_ptr)[local_grid](Index(p, lev), 0); + } + } + } if (grid >= 0) { AMREX_D_TERM(p.pos(0) = p_prime.pos(0);, @@ -217,7 +217,7 @@ ParticleContainer pld.m_lev = lev; pld.m_grid = grid; - pld.m_tile = getTileIndex(iv, bx, do_tiling, tile_size, pld.m_tilebox); + pld.m_tile = getTileIndex(iv, bx, do_tiling, tile_size, pld.m_tilebox); pld.m_cell = iv; pld.m_gridbox = bx; pld.m_grown_gridbox = bx; @@ -267,13 +267,13 @@ Reset (ParticleType& p, if (!ok) { // invalidate the particle. - if (verbose) { + if (verbose) { amrex::AllPrint()<< "Invalidating out-of-domain particle: " << p << '\n'; - } + } - AMREX_ASSERT(p.id() > 0); + AMREX_ASSERT(p.id() > 0); - p.id() = -p.id(); + p.id() = -p.id(); } return pld; @@ -606,8 +606,8 @@ ParticleContainer::In // If mf is not defined on the particle_box_array, then we need // to make a temporary mf_pointer here and copy it into mf at the end. mf_pointer = new MultiFab(ParticleBoxArray(lev), - ParticleDistributionMap(lev), - mf.nComp(),mf.nGrow()); + ParticleDistributionMap(lev), + mf.nComp(),mf.nGrow()); } ParticleLocData pld; @@ -616,7 +616,7 @@ ParticleContainer::In const auto& pbox = kv.second.GetArrayOfStructs(); FArrayBox& fab = (*mf_pointer)[gid]; for (int k = 0; k < pbox.numParticles(); ++ k) { - const ParticleType& p = pbox[k]; + const ParticleType& p = pbox[k]; if (p.id() > 0) { Where(p, pld); AMREX_ASSERT(pld.m_grid == gid); @@ -1093,7 +1093,7 @@ ParticleContainer auto p_periodic_shift = op.m_periodic_shift[lev][gid].dataPtr(); auto p_ptr = &(aos[0]); - AMREX_FOR_1D ( num_move, i, + AMREX_FOR_1D ( num_move, i, { const auto& p = p_ptr[i + num_stay]; if (p.id() < 0) @@ -1248,7 +1248,7 @@ ParticleContainer // we resize these buffers outside the parallel region for (int lev = lev_min; lev <= lev_max; lev++) { for (MFIter mfi(*m_dummy_mf[lev], this->do_tiling ? this->tile_size : IntVect::TheZeroVector()); - mfi.isValid(); ++mfi) { + mfi.isValid(); ++mfi) { auto index = std::make_pair(mfi.index(), mfi.LocalTileIndex()); tmp_local[lev][index].resize(num_threads); soa_local[lev][index].resize(num_threads); @@ -1301,7 +1301,7 @@ ParticleContainer ParticleType& p = aos[pindex]; if (p.id() < 0) - { + { aos[pindex] = aos[last]; for (int comp = 0; comp < NumRealComps(); comp++) soa.GetRealData(comp)[pindex] = soa.GetRealData(comp)[last]; @@ -1362,7 +1362,7 @@ ParticleContainer } for (int comp = 0; comp < NumIntComps(); comp++) { if (h_communicate_int_comp[comp]) { - std::memcpy(dst, &soa.GetIntData(comp)[pindex], sizeof(int)); + std::memcpy(dst, &soa.GetIntData(comp)[pindex], sizeof(int)); dst += sizeof(int); } } @@ -1371,7 +1371,7 @@ ParticleContainer } if (p.id() < 0) - { + { aos[pindex] = aos[last]; for (int comp = 0; comp < NumRealComps(); comp++) soa.GetRealData(comp)[pindex] = soa.GetRealData(comp)[last]; @@ -1643,7 +1643,7 @@ RedistributeMPI (std::map >& not_ours, if (nrcvs > 0) { ParallelDescriptor::Waitall(rreqs, stats); - BL_PROFILE_VAR_START(blp_locate); + BL_PROFILE_VAR_START(blp_locate); int npart = TotRcvBytes / superparticle_size; @@ -1671,7 +1671,7 @@ RedistributeMPI (std::map >& not_ours, } } - BL_PROFILE_VAR_STOP(blp_locate); + BL_PROFILE_VAR_STOP(blp_locate); BL_PROFILE_VAR_START(blp_copy); @@ -1717,19 +1717,19 @@ RedistributeMPI (std::map >& not_ours, } } #else - Vector, Gpu::HostVector > > host_particles; - host_particles.reserve(15); - host_particles.resize(finestLevel()+1); + Vector, Gpu::HostVector > > host_particles; + host_particles.reserve(15); + host_particles.resize(finestLevel()+1); - Vector, - std::vector > > > host_real_attribs; - host_real_attribs.reserve(15); - host_real_attribs.resize(finestLevel()+1); + Vector, + std::vector > > > host_real_attribs; + host_real_attribs.reserve(15); + host_real_attribs.resize(finestLevel()+1); - Vector, - std::vector > > > host_int_attribs; - host_int_attribs.reserve(15); - host_int_attribs.resize(finestLevel()+1); + Vector, + std::vector > > > host_int_attribs; + host_int_attribs.reserve(15); + host_int_attribs.resize(finestLevel()+1); ipart = 0; for (int i = 0; i < nrcvs; ++i) @@ -1781,42 +1781,42 @@ RedistributeMPI (std::map >& not_ours, } } - for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) - { - for (auto& kv : host_particles[host_lev]) { - auto grid = kv.first.first; - auto tile = kv.first.second; - const auto& src_tile = kv.second; + for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) + { + for (auto& kv : host_particles[host_lev]) { + auto grid = kv.first.first; + auto tile = kv.first.second; + const auto& src_tile = kv.second; - auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; - auto old_size = dst_tile.GetArrayOfStructs().size(); - auto new_size = old_size + src_tile.size(); - dst_tile.resize(new_size); + auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; + auto old_size = dst_tile.GetArrayOfStructs().size(); + auto new_size = old_size + src_tile.size(); + dst_tile.resize(new_size); - Gpu::copy(Gpu::hostToDevice, + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); - for (int i = 0; i < NumRealComps(); ++i) { + for (int i = 0; i < NumRealComps(); ++i) { Gpu::copy(Gpu::hostToDevice, host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } + } - for (int i = 0; i < NumIntComps(); ++i) { + for (int i = 0; i < NumIntComps(); ++i) { Gpu::copy(Gpu::hostToDevice, host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size); - } - } - } + } + } + } - Gpu::Device::streamSynchronize(); + Gpu::Device::streamSynchronize(); #endif - BL_PROFILE_VAR_STOP(blp_copy); + BL_PROFILE_VAR_STOP(blp_copy); } #else amrex::ignore_unused(not_ours,lev_min,lev_max,nGrow,local); @@ -1917,8 +1917,8 @@ AssignCellDensitySingleLevel (int rho_index, // If mf_to_be_filled is not defined on the particle_box_array, then we need // to make a temporary here and copy into mf_to_be_filled at the end. mf_pointer = new MultiFab(ParticleBoxArray(lev), - ParticleDistributionMap(lev), - ncomp, mf_to_be_filled.nGrow()); + ParticleDistributionMap(lev), + ncomp, mf_to_be_filled.nGrow()); } // We must have ghost cells for each FAB so that a particle in one grid can spread diff --git a/Src/Particle/AMReX_ParticleHDF5.H b/Src/Particle/AMReX_ParticleHDF5.H index 291a57dac03..a5da39933c7 100644 --- a/Src/Particle/AMReX_ParticleHDF5.H +++ b/Src/Particle/AMReX_ParticleHDF5.H @@ -34,7 +34,7 @@ ParticleContainer tmp_real_comp_names.push_back(real_comp_names[i]); } } - + Vector write_int_comp; Vector tmp_int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) @@ -54,7 +54,7 @@ ParticleContainer WriteHDF5ParticleData(dir, name, write_real_comp, write_int_comp, tmp_real_comp_names, tmp_int_comp_names); - + } template ss << "real_comp" << i; real_comp_names.push_back(ss.str()); } - + Vector write_int_comp; Vector int_comp_names; for (int i = 0; i < NStructInt + NumIntComps(); ++i ) @@ -82,7 +82,7 @@ ParticleContainer ss << "int_comp" << i; int_comp_names.push_back(ss.str()); } - + WriteHDF5ParticleData(dir, name, write_real_comp, write_int_comp, real_comp_names, int_comp_names); } @@ -362,7 +362,7 @@ ParticleContainer Long nparticles = 0; Long maxnextid; - + if(usePrePost) { nparticles = nparticlesPrePost; @@ -371,14 +371,14 @@ ParticleContainer { nparticles = 0; maxnextid = ParticleType::NextID(); - + for (int lev = 0; lev < m_particles.size(); lev++) { const auto& pmap = m_particles[lev]; for (const auto& kv : pmap) { const auto& aos = kv.second.GetArrayOfStructs(); - for (int k = 0; k < aos.numParticles(); ++k) + for (int k = 0; k < aos.numParticles(); ++k) { // Only count (and checkpoint) valid particles. const ParticleType& p = aos[k]; @@ -386,7 +386,7 @@ ParticleContainer } } } - + ParallelDescriptor::ReduceLongSum(nparticles, IOProcNumber); ParticleType::NextID(maxnextid); ParallelDescriptor::ReduceLongMax(maxnextid, IOProcNumber); @@ -419,14 +419,14 @@ ParticleContainer std::string HDF5FileName = pdir; if ( ! HDF5FileName.empty() && HDF5FileName[HDF5FileName.size()-1] != '/') HDF5FileName += '/'; - + HDF5FileName += name; - if (std::string::npos == HDF5FileName.find("h5") && std::string::npos == HDF5FileName.find("hdf5")) + if (std::string::npos == HDF5FileName.find("h5") && std::string::npos == HDF5FileName.find("hdf5")) HDF5FileName += ".h5"; HdrFileNamePrePost = HDF5FileName; - + if (ParallelDescriptor::IOProcessor()) { - + // Have only one rank to create and write metadata (header) fapl = H5Pcreate (H5P_FILE_ACCESS); @@ -438,7 +438,7 @@ ParticleContainer // Create the HDF5 file fid = H5Fcreate(HDF5FileName.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - if (fid < 0) + if (fid < 0) amrex::FileOpenFailed(HDF5FileName.c_str()); // @@ -458,22 +458,22 @@ ParticleContainer int num_output_real = 0; for (int i = 0; i < NumRealComps() + NStructReal; ++i) if (write_real_comp[i]) ++num_output_real; - + int num_output_int = 0; for (int i = 0; i < NumIntComps() + NStructInt; ++i) if (write_int_comp[i]) ++num_output_int; - + // AMREX_SPACEDIM and N for sanity checking. int ndim = AMREX_SPACEDIM; grp = H5Gcreate(fid, "Chombo_global", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CreateWriteHDF5AttrInt(grp, "SpaceDim", 1, &ndim); H5Gclose(grp); - - // The number of extra real parameters + + // The number of extra real parameters int ncomp = num_output_real + num_output_int; CreateWriteHDF5AttrInt(fid, "num_component", 1, &ncomp); CreateWriteHDF5AttrInt(fid, "num_component_real", 1, &num_output_real); - + char comp_name[128]; // Real component names for (int i = 0; i < NStructReal + NumRealComps(); ++i ) { @@ -483,10 +483,10 @@ ParticleContainer CreateWriteHDF5AttrString(fid, comp_name, real_comp_names[i].c_str()); } } - - // The number of extra int parameters + + // The number of extra int parameters CreateWriteHDF5AttrInt(fid, "num_component_int", 1, &num_output_int); - + // int component names for (int i = 0; i < NStructInt + NumIntComps(); ++i ) { if (write_int_comp[i]) { @@ -528,7 +528,7 @@ ParticleContainer int mfs_size = 2 * AMREX_SPACEDIM; hsize_t mfs_dim = (hsize_t)ngrids; - + hid_t mfs_dset_space = H5Screate_simple(1, &mfs_dim, NULL); hid_t mfs_dset= H5Dcreate(grp, "boxes", comp_dtype, mfs_dset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); @@ -573,7 +573,7 @@ ParticleContainer // All process open the file fid = H5Fopen(HDF5FileName.c_str(), H5F_ACC_RDWR, fapl); - if (fid < 0) + if (fid < 0) FileOpenFailed(HDF5FileName.c_str()); char level_name[32]; @@ -583,38 +583,38 @@ ParticleContainer grp = H5Gopen(fid, level_name, H5P_DEFAULT); bool gotsome; - if(usePrePost) + if(usePrePost) gotsome = (nParticlesAtLevelPrePost[lev] > 0); else gotsome = (NumberOfParticlesAtLevel(lev) > 0); - MFInfo info; - info.SetAlloc(false); - MultiFab state(ParticleBoxArray(lev), ParticleDistributionMap(lev), 1, 0, info); + MFInfo info; + info.SetAlloc(false); + MultiFab state(ParticleBoxArray(lev), ParticleDistributionMap(lev), 1, 0, info); // We eventually want to write out the file name and the offset // into that file into which each grid of particles is written. Vector which(state.size(),0); Vector count(state.size(),0); Vector where(state.size(),0); - - bool groupSets(false), setBuf(true); - + + bool groupSets(false), setBuf(true); + if (gotsome) - { + { WriteParticlesHDF5(grp, lev, count, where); - - if(usePrePost) { + + if(usePrePost) { whichPrePost[lev] = which; countPrePost[lev] = count; wherePrePost[lev] = where; - } else { + } else { ParallelDescriptor::ReduceIntSum (which.dataPtr(), which.size(), IOProcNumber); ParallelDescriptor::ReduceIntSum (count.dataPtr(), count.size(), IOProcNumber); ParallelDescriptor::ReduceLongSum(where.dataPtr(), where.size(), IOProcNumber); - } + } } H5Gclose(grp); @@ -623,13 +623,13 @@ ParticleContainer H5Tclose(comp_dtype); H5Pclose(fapl); H5Fclose(fid); - + if (m_verbose > 1) { auto stoptime = amrex::second() - strttime; - + ParallelDescriptor::ReduceRealMax(stoptime, IOProcNumber); - + amrex::Print() << "ParticleContainer::CheckpointHDF5() time: " << stoptime << '\n'; } } @@ -661,18 +661,18 @@ ParticleContainer tile_map[grid].push_back(tile); // Only write out valid particles. - int cnt = 0; - for (int k = 0; k < kv.second.GetArrayOfStructs().numParticles(); ++k) - { - const ParticleType& p = kv.second.GetArrayOfStructs()[k]; - if (p.id() > 0) { + int cnt = 0; + for (int k = 0; k < kv.second.GetArrayOfStructs().numParticles(); ++k) + { + const ParticleType& p = kv.second.GetArrayOfStructs()[k]; + if (p.id() > 0) { cnt++; - } - } + } + } count[grid] += cnt; } - + MFInfo info; info.SetAlloc(false); MultiFab state(ParticleBoxArray(lev), ParticleDistributionMap(lev), 1, 0, info); @@ -693,7 +693,7 @@ ParticleContainer // Get the size for each mf so we know the amount of data from each rank for (MFIter mfi(state); mfi.isValid(); ++mfi) { const int grid = mfi.index(); - if (count[grid] == 0) + if (count[grid] == 0) continue; int_size = count[grid] * (2 + NStructInt + NArrayInt); @@ -710,63 +710,63 @@ ParticleContainer #ifdef BL_USE_MPI // Collect the number of mf and total size of mf from each rank MPI_Allgather(&my_mfi_cnt, 1, ParallelDescriptor::Mpi_typemap::type(), &(all_mfi_cnt[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); - for (int i = 0; i < ParallelDescriptor::NProcs(); i++) + for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_mfi += all_mfi_cnt[i]; - // Create the int data - MPI_Allgather(&my_mfi_int_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), + // Create the int data + MPI_Allgather(&my_mfi_int_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), &(all_mfi_int_total_size[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); #else all_mfi_cnt[0] = &my_mfi_cnt; all_mfi_int_total_size[0] = my_mfi_int_total_size; #endif - - for (int i = 0; i < ParallelDescriptor::NProcs(); i++) - total_int_size += all_mfi_int_total_size[i]; - int_dset_space = H5Screate_simple(1, &total_int_size, NULL); + for (int i = 0; i < ParallelDescriptor::NProcs(); i++) + total_int_size += all_mfi_int_total_size[i]; + + int_dset_space = H5Screate_simple(1, &total_int_size, NULL); int_dset_id = H5Dcreate(grp, "data:datatype=0", H5T_NATIVE_INT, int_dset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Sclose(int_dset_space); int_file_offset = 0; - for (int i = 0; i < ParallelDescriptor::MyProc(); i++) + for (int i = 0; i < ParallelDescriptor::MyProc(); i++) int_file_offset += all_mfi_int_total_size[i]; my_int_offset = int_file_offset; my_int_count = 0; - - // Create the real data + + // Create the real data #ifdef BL_USE_MPI - MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), + MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap::type(), &(all_mfi_real_total_size[0]), 1, ParallelDescriptor::Mpi_typemap::type(), ParallelDescriptor::Communicator()); #else all_mfi_real_total_size[0] = my_mfi_real_total_size; #endif - for (int i = 0; i < ParallelDescriptor::NProcs(); i++) + for (int i = 0; i < ParallelDescriptor::NProcs(); i++) total_real_size += all_mfi_real_total_size[i]; - real_dset_space = H5Screate_simple(1, &total_real_size, NULL); - if (sizeof(typename ParticleType::RealType) == 4) - real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_FLOAT, real_dset_space, + real_dset_space = H5Screate_simple(1, &total_real_size, NULL); + if (sizeof(typename ParticleType::RealType) == 4) + real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_FLOAT, real_dset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - else - real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_DOUBLE, real_dset_space, + else + real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_DOUBLE, real_dset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); H5Sclose(real_dset_space); real_file_offset = 0; - for (int i = 0; i < ParallelDescriptor::MyProc(); i++) + for (int i = 0; i < ParallelDescriptor::MyProc(); i++) real_file_offset += all_mfi_real_total_size[i]; my_real_offset = real_file_offset; my_real_count = 0; for (MFIter mfi(state); mfi.isValid(); ++mfi) { const int grid = mfi.index(); - - if (count[grid] == 0) + + if (count[grid] == 0) continue; - + // First write out the integer data in binary. const int iChunkSize = 2 + NStructInt + NArrayInt; Vector istuff(count[grid]*iChunkSize); @@ -795,10 +795,10 @@ ParticleContainer /* writeIntData(istuff.dataPtr(), istuff.size(), ofs); */ my_int_count = istuff.size(); - int_mem_space = H5Screate_simple(1, &my_int_count, NULL); - /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */ + int_mem_space = H5Screate_simple(1, &my_int_count, NULL); + /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */ /* my_int_offset << ", my_int_count = " << my_int_count << ", total_int_size = " << total_int_size << '\n'; */ - int_dset_space = H5Screate_simple(1, &total_int_size, NULL); + int_dset_space = H5Screate_simple(1, &total_int_size, NULL); H5Sselect_hyperslab (int_dset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL); H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl, istuff.dataPtr()); H5Sclose(int_mem_space); @@ -806,12 +806,12 @@ ParticleContainer my_int_offset += my_int_count; - + // Write the Real data in binary. const int rChunkSize = AMREX_SPACEDIM + NStructReal + NArrayReal; Vector rstuff(count[grid]*rChunkSize); typename ParticleType::RealType* rptr = rstuff.dataPtr(); - + for (unsigned i = 0; i < tile_map[grid].size(); i++) { const auto& pbox = m_particles[lev].at(std::make_pair(grid, tile_map[grid][i])); for (int pindex = 0; pindex < pbox.GetArrayOfStructs().numParticles(); ++pindex) { @@ -835,10 +835,10 @@ ParticleContainer } my_real_count = rstuff.size(); - real_mem_space = H5Screate_simple(1, &my_real_count, NULL); - /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_real_offset = " << */ + real_mem_space = H5Screate_simple(1, &my_real_count, NULL); + /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_real_offset = " << */ /* my_real_offset << ", my_real_count = " << my_real_count << ", total_real_size = " << total_real_size << '\n'; */ - real_dset_space = H5Screate_simple(1, &total_real_size, NULL); + real_dset_space = H5Screate_simple(1, &total_real_size, NULL); H5Sselect_hyperslab (real_dset_space, H5S_SELECT_SET, &my_real_offset, NULL, &my_real_count, NULL); if (sizeof(typename ParticleType::RealType) == 4) H5Dwrite(real_dset_id, H5T_NATIVE_FLOAT, real_mem_space, real_dset_space, dxpl, rstuff.dataPtr()); @@ -856,15 +856,15 @@ ParticleContainer H5Dclose(int_dset_id); // Create and write the size dataset - offset_space = H5Screate_simple(1, &total_mfi, NULL); + offset_space = H5Screate_simple(1, &total_mfi, NULL); offset_id = H5Dcreate(grp, "nparticles_grid", H5T_NATIVE_INT, offset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); my_int_offset = 0; - for (int i = 0; i < ParallelDescriptor::MyProc(); i++) + for (int i = 0; i < ParallelDescriptor::MyProc(); i++) my_int_offset += all_mfi_cnt[i]; my_int_count = my_mfi_cnt; - int_mem_space = H5Screate_simple(1, &my_int_count, NULL); - /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */ + int_mem_space = H5Screate_simple(1, &my_int_count, NULL); + /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */ /* my_int_offset << ", my_int_count = " << my_int_count << ", total_mfi = " << total_mfi << '\n'; */ H5Sselect_hyperslab (offset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL); @@ -904,9 +904,9 @@ ParticleContainer BL_PROFILE("ParticleContainer::RestartHDF5()"); AMREX_ASSERT(!dir.empty()); AMREX_ASSERT(!file.empty()); - + const auto strttime = amrex::second(); - + /* int DATA_Digits_Read(5); */ /* ParmParse pp("particles"); */ /* pp.query("datadigits_read",DATA_Digits_Read); */ @@ -916,7 +916,7 @@ ParticleContainer fullname += '/'; fullname += file; - if (std::string::npos == fullname.find("h5") && std::string::npos == fullname.find("hdf5")) + if (std::string::npos == fullname.find("h5") && std::string::npos == fullname.find("hdf5")) fullname += ".h5"; hid_t fid, dset, grp, fapl, attr, atype, dspace, int_dset, real_dset; @@ -1013,7 +1013,7 @@ ParticleContainer } if (nr != NStructReal + NumRealComps()) amrex::Abort("ParticleContainer::Restart(): nr != NStructReal + NumRealComps()"); - + aname = "num_component_int"; int ni; ret = ReadHDF5AttrInt(fid, aname.c_str(), &ni); @@ -1024,7 +1024,7 @@ ParticleContainer } if (ni != NStructInt + NumIntComps()) amrex::Abort("ParticleContainer::Restart(): ni != NStructInt"); - + aname = "nparticles"; Long nparticles; ret = ReadHDF5AttrLong(fid, aname.c_str(), &nparticles); @@ -1034,7 +1034,7 @@ ParticleContainer amrex::Abort(msg.c_str()); } AMREX_ASSERT(nparticles >= 0); - + aname = "maxnextid"; Long maxnextid; ret = ReadHDF5AttrLong(fid, aname.c_str(), &maxnextid); @@ -1045,7 +1045,7 @@ ParticleContainer } AMREX_ASSERT(maxnextid > 0); ParticleType::NextID(maxnextid); - + aname = "finest_level"; int finest_level_in_file; ret = ReadHDF5AttrInt(fid, aname.c_str(), &finest_level_in_file); @@ -1055,7 +1055,7 @@ ParticleContainer amrex::Abort(msg.c_str()); } AMREX_ASSERT(finest_level_in_file >= 0); - + // Determine whether this is a dual-grid restart or not. Vector particle_box_arrays(finest_level_in_file + 1); bool dual_grid = false; @@ -1087,7 +1087,7 @@ ParticleContainer dual_grid = true; break; } - + gname = "level_" + std::to_string(lev); grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT); if (grp < 0) { @@ -1095,7 +1095,7 @@ ParticleContainer msg += gname; amrex::Abort(msg.c_str()); } - + dset = H5Dopen(grp, "boxes", H5P_DEFAULT); dspace = H5Dget_space(dset); hsize_t ngrid; @@ -1110,7 +1110,7 @@ ParticleContainer H5Sclose(dspace); H5Dclose(dset); H5Gclose(grp); - + /* particle_box_arrays[lev].readFrom(phdr_file); */ particle_box_arrays[lev] = ParticleBoxArray(lev); for (int i = 0; i < ngrid; i++) { @@ -1118,7 +1118,7 @@ ParticleContainer particle_box_arrays[lev][i] = tmp; } - if (! particle_box_arrays[lev].CellEqual(ParticleBoxArray(lev))) + if (! particle_box_arrays[lev].CellEqual(ParticleBoxArray(lev))) dual_grid = true; } if (dual_grid) { @@ -1128,7 +1128,7 @@ ParticleContainer SetParticleDistributionMap(lev, pdm); } } - + Vector ngrids(finest_level_in_file+1); for (int lev = 0; lev <= finest_level_in_file; lev++) { gname = "level_" + std::to_string(lev); @@ -1154,13 +1154,13 @@ ParticleContainer H5Gclose(grp); } - + resizeData(); - + if (finest_level_in_file > finestLevel()) { m_particles.resize(finest_level_in_file+1); } - + for (int lev = 0; lev <= finest_level_in_file; lev++) { gname = "level_" + std::to_string(lev); @@ -1203,35 +1203,35 @@ ParticleContainer grids_to_read.push_back(mfi.index()); } } else { - + // we lost a level on restart. we still need to read in particles // on finer levels, and put them in the right place via Redistribute() - + const int rank = ParallelDescriptor::MyProc(); const int NReaders = MaxReaders(); if (rank >= NReaders) return; - + const int Navg = ngrids[lev] / NReaders; const int Nleft = ngrids[lev] - Navg * NReaders; - + int lo, hi; if (rank < Nleft) { lo = rank*(Navg + 1); hi = lo + Navg + 1; - } + } else { lo = rank * Navg + Nleft; hi = lo + Navg; } - + for (int i = lo; i < hi; ++i) { grids_to_read.push_back(i); } } - + for(int igrid = 0; igrid < static_cast(grids_to_read.size()); ++igrid) { const int grid = grids_to_read[igrid]; - + if (how == "single") { ReadParticlesHDF5(offset[grid], count[grid], grid, lev, int_dset, real_dset, finest_level_in_file); } @@ -1249,13 +1249,13 @@ ParticleContainer H5Dclose(real_dset); H5Gclose(grp); } // end for level - + H5Fclose(fid); - + Redistribute(); - + AMREX_ASSERT(OK()); - + if (m_verbose > 1) { auto stoptime = amrex::second() - strttime; ParallelDescriptor::ReduceRealMax(stoptime, ParallelDescriptor::IOProcessorNumber()); @@ -1284,20 +1284,20 @@ ParticleContainer int_fspace = H5Dget_space(int_dset); hsize_t int_cnt = cnt*iChunkSize; hsize_t int_offset = offset*iChunkSize; - int_dspace = H5Screate_simple(1, &int_cnt, NULL); + int_dspace = H5Screate_simple(1, &int_cnt, NULL); H5Sselect_hyperslab (int_fspace, H5S_SELECT_SET, &int_offset, NULL, &int_cnt, NULL); H5Dread(int_dset, H5T_NATIVE_INT, int_dspace, int_fspace, H5P_DEFAULT, istuff.dataPtr()); H5Sclose(int_fspace); H5Sclose(int_dspace); - + // Then the real data in binary. const int rChunkSize = AMREX_SPACEDIM + NStructReal + NumRealComps(); Vector rstuff(cnt*rChunkSize); real_fspace = H5Dget_space(real_dset); hsize_t real_cnt = cnt*rChunkSize; hsize_t real_offset = offset*rChunkSize; - real_dspace = H5Screate_simple(1, &real_cnt, NULL); + real_dspace = H5Screate_simple(1, &real_cnt, NULL); H5Sselect_hyperslab (real_fspace, H5S_SELECT_SET, &real_offset, NULL, &real_cnt, NULL); if (sizeof(RTYPE) == 4) H5Dread(real_dset, H5T_NATIVE_FLOAT, real_dspace, real_fspace, H5P_DEFAULT, rstuff.dataPtr()); @@ -1306,11 +1306,11 @@ ParticleContainer H5Sclose(real_fspace); H5Sclose(real_dspace); - + // Now reassemble the particles. int* iptr = istuff.dataPtr(); RTYPE* rptr = rstuff.dataPtr(); - + ParticleType p; ParticleLocData pld; @@ -1355,59 +1355,59 @@ ParticleContainer } locateParticle(p, pld, 0, finestLevel(), 0); - - std::pair ind(grd, pld.m_tile); + + std::pair ind(grd, pld.m_tile); host_real_attribs[lev][ind].resize(NumRealComps()); host_int_attribs[lev][ind].resize(NumIntComps()); - - // add the struct - host_particles[lev][ind].push_back(p); - // add the real... - for (int icomp = 0; icomp < NumRealComps(); icomp++) { + // add the struct + host_particles[lev][ind].push_back(p); + + // add the real... + for (int icomp = 0; icomp < NumRealComps(); icomp++) { host_real_attribs[lev][ind][icomp].push_back(*rptr); ++rptr; - } - - // ... and int array data - for (int icomp = 0; icomp < NumIntComps(); icomp++) { + } + + // ... and int array data + for (int icomp = 0; icomp < NumIntComps(); icomp++) { host_int_attribs[lev][ind][icomp].push_back(*iptr); ++iptr; - } + } } for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) { - for (auto& kv : host_particles[host_lev]) { - auto grid = kv.first.first; - auto tile = kv.first.second; - const auto& src_tile = kv.second; - - auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile); - auto old_size = dst_tile.GetArrayOfStructs().size(); - auto new_size = old_size + src_tile.size(); - dst_tile.resize(new_size); - - Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), + for (auto& kv : host_particles[host_lev]) { + auto grid = kv.first.first; + auto tile = kv.first.second; + const auto& src_tile = kv.second; + + auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile); + auto old_size = dst_tile.GetArrayOfStructs().size(); + auto new_size = old_size + src_tile.size(); + dst_tile.resize(new_size); + + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); - - for (int i = 0; i < NumRealComps(); ++i) { + + for (int i = 0; i < NumRealComps(); ++i) { Gpu::copy(Gpu::hostToDevice, host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } - - for (int i = 0; i < NumIntComps(); ++i) { + } + + for (int i = 0; i < NumIntComps(); ++i) { Gpu::copy(Gpu::hostToDevice, host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size); - } - } + } + } } - + Gpu::streamSynchronize(); } diff --git a/Src/Particle/AMReX_ParticleIO.H b/Src/Particle/AMReX_ParticleIO.H index dc71f8b227a..63085a20c00 100644 --- a/Src/Particle/AMReX_ParticleIO.H +++ b/Src/Particle/AMReX_ParticleIO.H @@ -667,9 +667,9 @@ ParticleContainer typename ParticleType::RealType* rptr = rstuff.dataPtr(); for (unsigned i = 0; i < tile_map[grid].size(); i++) { - auto ptile_index = std::make_pair(grid, tile_map[grid][i]); + auto ptile_index = std::make_pair(grid, tile_map[grid][i]); const auto& pbox = m_particles[lev].at(ptile_index); - const auto& pflags = particle_io_flags[lev].at(ptile_index); + const auto& pflags = particle_io_flags[lev].at(ptile_index); for (int pindex = 0; pindex < pbox.GetArrayOfStructs().numParticles(); ++pindex) { const auto& aos = pbox.GetArrayOfStructs(); const auto& p = aos[pindex]; @@ -1057,56 +1057,56 @@ ParticleContainer locateParticle(p, pld, 0, finestLevel(), 0); - std::pair ind(grd, pld.m_tile); + std::pair ind(grd, pld.m_tile); host_real_attribs[lev][ind].resize(NumRealComps()); host_int_attribs[lev][ind].resize(NumIntComps()); - // add the struct - host_particles[lev][ind].push_back(p); + // add the struct + host_particles[lev][ind].push_back(p); - // add the real... - for (int icomp = 0; icomp < NumRealComps(); icomp++) { + // add the real... + for (int icomp = 0; icomp < NumRealComps(); icomp++) { host_real_attribs[lev][ind][icomp].push_back(ParticleReal(*rptr)); ++rptr; - } + } - // ... and int array data - for (int icomp = 0; icomp < NumIntComps(); icomp++) { + // ... and int array data + for (int icomp = 0; icomp < NumIntComps(); icomp++) { host_int_attribs[lev][ind][icomp].push_back(*iptr); ++iptr; - } + } } for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) { - for (auto& kv : host_particles[host_lev]) { - auto grid = kv.first.first; - auto tile = kv.first.second; - const auto& src_tile = kv.second; + for (auto& kv : host_particles[host_lev]) { + auto grid = kv.first.first; + auto tile = kv.first.second; + const auto& src_tile = kv.second; - auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile); - auto old_size = dst_tile.GetArrayOfStructs().size(); - auto new_size = old_size + src_tile.size(); - dst_tile.resize(new_size); + auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile); + auto old_size = dst_tile.GetArrayOfStructs().size(); + auto new_size = old_size + src_tile.size(); + dst_tile.resize(new_size); - Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); - for (int i = 0; i < NumRealComps(); ++i) { + for (int i = 0; i < NumRealComps(); ++i) { Gpu::copy(Gpu::hostToDevice, host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } + } - for (int i = 0; i < NumIntComps(); ++i) { + for (int i = 0; i < NumIntComps(); ++i) { Gpu::copy(Gpu::hostToDevice, host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size); - } - } + } + } } Gpu::streamSynchronize(); @@ -1131,11 +1131,11 @@ ParticleContainer auto& pmap = m_particles[lev]; for (const auto& kv : pmap) { const auto& aos = kv.second.GetArrayOfStructs(); - auto np = aos.numParticles(); - Gpu::HostVector host_aos(np); - Gpu::copy(Gpu::deviceToHost, aos.begin(), aos.begin() + np, host_aos.begin()); - for (int k = 0; k < np; ++k) { - const ParticleType& p = host_aos[k]; + auto np = aos.numParticles(); + Gpu::HostVector host_aos(np); + Gpu::copy(Gpu::deviceToHost, aos.begin(), aos.begin() + np, host_aos.begin()); + for (int k = 0; k < np; ++k) { + const ParticleType& p = host_aos[k]; if (p.id() > 0) // // Only count (and checkpoint) valid particles. @@ -1200,19 +1200,19 @@ ParticleContainer if (!File.good()) amrex::FileOpenFailed(filename); - for (int lev = 0; lev < m_particles.size(); lev++) { - auto& pmap = m_particles[lev]; - for (const auto& kv : pmap) { + for (int lev = 0; lev < m_particles.size(); lev++) { + auto& pmap = m_particles[lev]; + for (const auto& kv : pmap) { const auto& aos = kv.second.GetArrayOfStructs(); const auto& soa = kv.second.GetStructOfArrays(); - auto np = aos.numParticles(); - Gpu::HostVector host_aos(np); - Gpu::copy(Gpu::deviceToHost, aos.begin(), aos.end(), host_aos.begin()); + auto np = aos.numParticles(); + Gpu::HostVector host_aos(np); + Gpu::copy(Gpu::deviceToHost, aos.begin(), aos.end(), host_aos.begin()); - for (int index = 0; index < np; ++index) { - const ParticleType* it = &host_aos[index]; - if (it->id() > 0) { + for (int index = 0; index < np; ++index) { + const ParticleType* it = &host_aos[index]; + if (it->id() > 0) { // write out the particle struct first... AMREX_D_TERM(File << it->pos(0) << ' ', diff --git a/Src/Particle/AMReX_ParticleInit.H b/Src/Particle/AMReX_ParticleInit.H index 6018dd0e43f..b7aef420999 100644 --- a/Src/Particle/AMReX_ParticleInit.H +++ b/Src/Particle/AMReX_ParticleInit.H @@ -74,7 +74,7 @@ ParticleContainer resizeData(); IntVect lNrep(AMREX_D_DECL(1,1,1)); - + if (Nrep != nullptr) lNrep = *Nrep; @@ -83,8 +83,8 @@ ParticleContainer Gpu::HostVector nparticles; Vector > nreals; - if (extradata > NStructReal) nreals.resize(extradata - NStructReal); - + if (extradata > NStructReal) nreals.resize(extradata - NStructReal); + if (MyProc < NReaders) { VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); @@ -104,12 +104,12 @@ ParticleContainer ifs >> cnt >> std::ws; - ParticleLocData pld; + ParticleLocData pld; ParticleType p, p_rep; Vector r; if (extradata > NStructReal) r.resize(extradata - NStructReal); - + const int Chunk = cnt / NReaders; for (int i = 0; i < MyProc*Chunk; i++) @@ -268,13 +268,13 @@ ParticleContainer host_particles.reserve(15); host_particles.resize(finestLevel()+1); - Vector, std::array, NArrayReal > > > host_real_attribs; + Vector, std::array, NArrayReal > > > host_real_attribs; host_real_attribs.reserve(15); host_real_attribs.resize(finestLevel()+1); - + if (m_verbose > 0) { amrex::Print() << "Redistributing from processor " - << nr*NRedist_chunk << " to " + << nr*NRedist_chunk << " to " << (nr+1)*NRedist_chunk-1 << '\n'; } for (int which = nr*NRedist_chunk; which < (nr+1)*NRedist_chunk; which++) @@ -284,18 +284,18 @@ ParticleContainer while (!nparticles.empty()) { ParticleType& p = nparticles.back(); - Where(p, pld); - + Where(p, pld); + host_particles[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)].push_back(p); - + for (int n = NStructReal; n < extradata; n++) { Real rdata = nreals[n-NStructReal].back(); host_real_attribs[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)][n-NStructReal].push_back(rdata); } - + nparticles.pop_back(); - + for (int n = NStructReal; n < extradata; n++) { nreals[n-NStructReal].pop_back(); @@ -303,7 +303,7 @@ ParticleContainer } } } - + for (int lev = 0; lev < static_cast(host_particles.size()); ++lev) { for (auto& kv : host_particles[lev]) @@ -311,12 +311,12 @@ ParticleContainer auto grid = kv.first.first; auto tile = kv.first.second; const auto& src_tile = kv.second; - + auto& dst_tile = GetParticles(lev)[std::make_pair(grid,tile)]; auto old_size = dst_tile.GetArrayOfStructs().size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); @@ -325,18 +325,18 @@ ParticleContainer host_real_attribs[lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } + } } } - - Redistribute(); + + Redistribute(); } if (m_verbose > 0) { if (NRedist*NRedist_chunk < NReaders) { amrex::Print() << "Redistributing from processor " - << NRedist*NRedist_chunk << " to " + << NRedist*NRedist_chunk << " to " << NReaders << '\n'; } } @@ -346,29 +346,29 @@ ParticleContainer host_particles.reserve(15); host_particles.resize(finestLevel()+1); - Vector, std::array, NArrayReal > > > host_real_attribs; + Vector, std::array, NArrayReal > > > host_real_attribs; host_real_attribs.reserve(15); host_real_attribs.resize(finestLevel()+1); - + if (which == MyProc) { - while (!nparticles.empty()) + while (!nparticles.empty()) { ParticleType& p = nparticles.back(); - Where(p, pld); + Where(p, pld); - host_particles[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)].push_back(p); + host_particles[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)].push_back(p); for (int n = NStructReal; n < extradata; n++) { Real rdata = nreals[n-NStructReal].back(); host_real_attribs[pld.m_lev][std::make_pair(pld.m_grid, pld.m_tile)][n-NStructReal].push_back(rdata); } - - nparticles.pop_back(); + + nparticles.pop_back(); for (int n = NStructReal; n < extradata; n++) { nreals[n-NStructReal].pop_back(); - } + } } } @@ -379,12 +379,12 @@ ParticleContainer auto grid = kv.first.first; auto tile = kv.first.second; const auto& src_tile = kv.second; - + auto& dst_tile = GetParticles(lev)[std::make_pair(grid,tile)]; auto old_size = dst_tile.GetArrayOfStructs().size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); @@ -393,10 +393,10 @@ ParticleContainer host_real_attribs[lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } + } } } - + Redistribute(); } @@ -404,7 +404,7 @@ ParticleContainer { const int IOProcNumber = ParallelDescriptor::IOProcessorNumber(); - Long num_particles = how_many; + Long num_particles = how_many; ParallelDescriptor::ReduceLongSum(num_particles, IOProcNumber); @@ -414,7 +414,7 @@ ParticleContainer } else { - Long num_particles_read = how_many_read; + Long num_particles_read = how_many_read; ParallelDescriptor::ReduceLongSum(num_particles_read, IOProcNumber); @@ -432,9 +432,9 @@ ParticleContainer ByteSpread(); auto runtime = amrex::second() - strttime; - + ParallelDescriptor::ReduceRealMax(runtime, ParallelDescriptor::IOProcessorNumber()); - + amrex::Print() << "InitFromAsciiFile() time: " << runtime << '\n'; } } @@ -491,7 +491,7 @@ InitFromBinaryFile (const std::string& file, tmp_particles.reserve(15); // So we don't ever have to do any copying on a resize. tmp_particles.resize(finestLevel()+1); - + resizeData(); // @@ -501,7 +501,7 @@ InitFromBinaryFile (const std::string& file, Long NP = 0; VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); - + std::ifstream ifs; // @@ -699,7 +699,7 @@ InitFromBinaryFile (const std::string& file, Vector, Gpu::HostVector > > host_particles; host_particles.reserve(15); host_particles.resize(finestLevel()+1); - + if (readers.find(MyProc) != readers.end()) { ParticleType p; @@ -812,17 +812,17 @@ InitFromBinaryFile (const std::string& file, auto grid = kv.first.first; auto tile = kv.first.second; const auto& src_tile = kv.second; - + auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; auto old_size = dst_tile.GetArrayOfStructs().size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); } } - + Redistribute(); // @@ -855,7 +855,7 @@ InitFromBinaryFile (const std::string& file, // if (m_verbose > 0) { - Long num_particles_read = how_many_read; + Long num_particles_read = how_many_read; ParallelDescriptor::ReduceLongSum(num_particles_read, ParallelDescriptor::IOProcessorNumber()); @@ -948,10 +948,10 @@ InitRandom (Long icount, geom.ProbLength(1), geom.ProbLength(2)) }; - // We will enforce that the particles are within the containing_bx. + // We will enforce that the particles are within the containing_bx. // If containing_bx is not passed in, it defaults to the full domain. if (!containing_bx.ok()) containing_bx = geom.ProbDomain(); - + // containing_bx is assumed to lie within the domain. if (!geom.ProbDomain().contains(containing_bx)) { @@ -1003,11 +1003,11 @@ InitRandom (Long icount, host_particles.reserve(15); host_particles.resize(finestLevel()+1); - Vector, std::array, NArrayReal > > > host_real_attribs; + Vector, std::array, NArrayReal > > > host_real_attribs; host_real_attribs.reserve(15); host_real_attribs.resize(finestLevel()+1); - Vector, std::array, NArrayInt > > > host_int_attribs; + Vector, std::array, NArrayInt > > > host_int_attribs; host_int_attribs.reserve(15); host_int_attribs.resize(finestLevel()+1); @@ -1048,7 +1048,7 @@ InitRandom (Long icount, // add the real... for (int i = 0; i < NArrayReal; i++) { host_real_attribs[pld.m_lev][ind][i].push_back(pdata.real_array_data[i]); - } + } // ... and int array data for (int i = 0; i < NArrayInt; i++) { @@ -1056,37 +1056,37 @@ InitRandom (Long icount, } cnt++; - } + } } - for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) + for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) { for (auto& kv : host_particles[host_lev]) { auto grid = kv.first.first; auto tile = kv.first.second; const auto& src_tile = kv.second; - + auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; auto old_size = dst_tile.GetArrayOfStructs().size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); - for (int i = 0; i < NArrayReal; ++i) { + for (int i = 0; i < NArrayReal; ++i) { Gpu::copy(Gpu::hostToDevice, host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } + } - for (int i = 0; i < NArrayInt; ++i) { + for (int i = 0; i < NArrayInt; ++i) { Gpu::copy(Gpu::hostToDevice, host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size); - } + } } } @@ -1100,18 +1100,18 @@ InitRandom (Long icount, if (MyProc == 0) { M += (icount % NProcs); } - + ParticleLocData pld; Vector, Gpu::HostVector > > host_particles; host_particles.reserve(15); host_particles.resize(finestLevel()+1); - Vector, std::array, NArrayReal > > > host_real_attribs; + Vector, std::array, NArrayReal > > > host_real_attribs; host_real_attribs.reserve(15); host_real_attribs.resize(finestLevel()+1); - Vector, std::array, NArrayInt > > > host_int_attribs; + Vector, std::array, NArrayInt > > > host_int_attribs; host_int_attribs.reserve(15); host_int_attribs.resize(finestLevel()+1); @@ -1149,18 +1149,18 @@ InitRandom (Long icount, AMREX_ASSERT(pld.m_lev >= 0 && pld.m_lev <= finestLevel()); std::pair ind(pld.m_grid, pld.m_tile); - // add the struct - host_particles[pld.m_lev][ind].push_back(p); + // add the struct + host_particles[pld.m_lev][ind].push_back(p); - // add the real... - for (int i = 0; i < NArrayReal; i++) { - host_real_attribs[pld.m_lev][ind][i].push_back(pdata.real_array_data[i]); - } + // add the real... + for (int i = 0; i < NArrayReal; i++) { + host_real_attribs[pld.m_lev][ind][i].push_back(pdata.real_array_data[i]); + } - // ... and int array data - for (int i = 0; i < NArrayInt; i++) { - host_int_attribs[pld.m_lev][ind][i].push_back(pdata.int_array_data[i]); - } + // ... and int array data + for (int i = 0; i < NArrayInt; i++) { + host_int_attribs[pld.m_lev][ind][i].push_back(pdata.int_array_data[i]); + } } for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) @@ -1169,28 +1169,28 @@ InitRandom (Long icount, auto grid = kv.first.first; auto tile = kv.first.second; const auto& src_tile = kv.second; - + auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; auto old_size = dst_tile.GetArrayOfStructs().size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - + Gpu::copy(Gpu::hostToDevice, src_tile.begin(), src_tile.end(), dst_tile.GetArrayOfStructs().begin() + old_size); - for (int i = 0; i < NArrayReal; ++i) { - Gpu::copy(Gpu::hostToDevice, + for (int i = 0; i < NArrayReal; ++i) { + Gpu::copy(Gpu::hostToDevice, host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } + } - for (int i = 0; i < NArrayInt; ++i) { + for (int i = 0; i < NArrayInt; ++i) { Gpu::copy(Gpu::hostToDevice, host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size); - } + } } } // Let Redistribute() sort out where the particles beLong. @@ -1257,7 +1257,7 @@ ParticleContainer p.pos(0) = static_cast(grid_box.lo(0) + (dist(mt) + icnt) / icount_per_box); p.pos(1) = static_cast(grid_box.lo(1) + (dist(mt) + jcnt) / icount_per_box); p.pos(2) = static_cast(grid_box.lo(2) + (dist(mt) + kcnt) / icount_per_box); - + for (int i = 0; i < AMREX_SPACEDIM; i++) AMREX_ASSERT(p.pos(i) < grid_box.hi(i)); @@ -1279,20 +1279,20 @@ ParticleContainer amrex::Abort("ParticleContainer::InitRandomPerBox(): invalid particle"); } AMREX_ASSERT(pld.m_lev >= 0 && pld.m_lev <= finestLevel()); - std::pair ind(pld.m_grid, pld.m_tile); + std::pair ind(pld.m_grid, pld.m_tile); // add the struct m_particles[pld.m_lev][ind].push_back(p); - + // add the real... for (int i = 0; i < NArrayReal; i++) { m_particles[pld.m_lev][ind].push_back_real(i, static_cast(pdata.real_array_data[i])); } - + // ... and int array data for (int i = 0; i < NArrayInt; i++) { m_particles[pld.m_lev][ind].push_back_int(i, pdata.int_array_data[i]); - } + } } } } } @@ -1321,8 +1321,8 @@ InitOnePerCell (Real x_off, Real y_off, Real z_off, const ParticleInitData& pdat // Note that x_off, y_off and z_off are the offsets from the lower left corner // in each cell as measured in units of dx, so they must be in [0,1] - AMREX_ASSERT(x_off >= 0. && y_off >= 0. && z_off >= 0.); - AMREX_ASSERT(x_off <= 1. && y_off <= 1. && z_off <= 1.); + AMREX_ASSERT(x_off >= 0. && y_off >= 0. && z_off >= 0.); + AMREX_ASSERT(x_off <= 1. && y_off <= 1. && z_off <= 1.); const int IOProc = ParallelDescriptor::IOProcessorNumber(); const auto strttime = amrex::second(); @@ -1368,11 +1368,11 @@ InitOnePerCell (Real x_off, Real y_off, Real z_off, const ParticleInitData& pdat { amrex::Abort("ParticleContainer::InitOnePerCell(): invalid particle"); } - AMREX_ASSERT(pld.m_lev >= 0 && pld.m_lev <= finestLevel()); - std::pair ind(pld.m_grid, pld.m_tile); + AMREX_ASSERT(pld.m_lev >= 0 && pld.m_lev <= finestLevel()); + std::pair ind(pld.m_grid, pld.m_tile); // add the struct - m_particles[pld.m_lev][ind].push_back(p); + m_particles[pld.m_lev][ind].push_back(p); // add the real... for (int i = 0; i < NArrayReal; i++) { @@ -1382,13 +1382,13 @@ InitOnePerCell (Real x_off, Real y_off, Real z_off, const ParticleInitData& pdat // ... and int array data for (int i = 0; i < NArrayInt; i++) { m_particles[pld.m_lev][ind].push_back_int(i, pdata.int_array_data[i]); - } + } } } - + if (m_verbose > 1) { auto stoptime = amrex::second() - strttime; - + ParallelDescriptor::ReduceRealMax(stoptime,IOProc); amrex::Print() << "ParticleContainer::InitOnePerCell() time: " << stoptime << '\n'; @@ -1431,11 +1431,11 @@ InitNRandomPerCell (int n_per_cell, const ParticleInitData& pdata) host_particles.reserve(15); host_particles.resize(finestLevel()+1); - Vector, std::array, NArrayReal > > > host_real_attribs; + Vector, std::array, NArrayReal > > > host_real_attribs; host_real_attribs.reserve(15); host_real_attribs.resize(finestLevel()+1); - Vector, std::array, NArrayInt > > > host_int_attribs; + Vector, std::array, NArrayInt > > > host_int_attribs; host_int_attribs.reserve(15); host_int_attribs.resize(finestLevel()+1); @@ -1474,7 +1474,7 @@ InitNRandomPerCell (int n_per_cell, const ParticleInitData& pdata) amrex::Abort("ParticleContainer::InitNRandomPerCell(): invalid particle"); } AMREX_ASSERT(pld.m_lev >= 0 && pld.m_lev <= finestLevel()); - std::pair ind(pld.m_grid, pld.m_tile); + std::pair ind(pld.m_grid, pld.m_tile); // add the struct host_particles[pld.m_lev][ind].push_back(p); @@ -1482,7 +1482,7 @@ InitNRandomPerCell (int n_per_cell, const ParticleInitData& pdata) // add the real... for (int i = 0; i < NArrayReal; i++) { host_real_attribs[pld.m_lev][ind][i].push_back(pdata.real_array_data[i]); - } + } // ... and int array data for (int i = 0; i < NArrayInt; i++) { @@ -1491,36 +1491,36 @@ InitNRandomPerCell (int n_per_cell, const ParticleInitData& pdata) } } - for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) + for (int host_lev = 0; host_lev < static_cast(host_particles.size()); ++host_lev) { for (auto& kv : host_particles[host_lev]) { auto gid = kv.first.first; auto tid = kv.first.second; const auto& src_tid = kv.second; - + auto& dst_tile = GetParticles(host_lev)[std::make_pair(gid,tid)]; auto old_size = dst_tile.GetArrayOfStructs().size(); auto new_size = old_size + src_tid.size(); dst_tile.resize(new_size); - + Gpu::copy(Gpu::hostToDevice, src_tid.begin(), src_tid.end(), dst_tile.GetArrayOfStructs().begin() + old_size); - - for (int i = 0; i < NArrayReal; ++i) + + for (int i = 0; i < NArrayReal; ++i) { Gpu::copy(Gpu::hostToDevice, host_real_attribs[host_lev][std::make_pair(gid,tid)][i].begin(), host_real_attribs[host_lev][std::make_pair(gid,tid)][i].end(), dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size); - } - - for (int i = 0; i < NArrayInt; ++i) + } + + for (int i = 0; i < NArrayInt; ++i) { Gpu::copy(Gpu::hostToDevice, host_int_attribs[host_lev][std::make_pair(gid,tid)][i].begin(), host_int_attribs[host_lev][std::make_pair(gid,tid)][i].end(), dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size); - } + } } } @@ -1529,9 +1529,9 @@ InitNRandomPerCell (int n_per_cell, const ParticleInitData& pdata) if (m_verbose > 1) { auto stoptime = amrex::second() - strttime; - + ParallelDescriptor::ReduceRealMax(stoptime,IOProc); - + amrex::Print() << "ParticleContainer::InitNRandomPerCell() time: " << stoptime << '\n'; } diff --git a/Src/Particle/AMReX_ParticleMPIUtil.H b/Src/Particle/AMReX_ParticleMPIUtil.H index 09dda701947..51d43455345 100644 --- a/Src/Particle/AMReX_ParticleMPIUtil.H +++ b/Src/Particle/AMReX_ParticleMPIUtil.H @@ -8,7 +8,7 @@ namespace amrex { -#ifdef AMREX_USE_MPI +#ifdef AMREX_USE_MPI Long CountSnds(const std::map >& not_ours, Vector& Snds); diff --git a/Src/Particle/AMReX_ParticleMesh.H b/Src/Particle/AMReX_ParticleMesh.H index fded434db54..145a32ae6a8 100644 --- a/Src/Particle/AMReX_ParticleMesh.H +++ b/Src/Particle/AMReX_ParticleMesh.H @@ -13,9 +13,9 @@ void ParticleToMesh (PC const& pc, MF& mf, int lev, F&& f) { BL_PROFILE("amrex::ParticleToMesh"); - + MultiFab* mf_pointer = pc.OnSameGrids(lev, mf) ? - &mf : new MultiFab(pc.ParticleBoxArray(lev), + &mf : new MultiFab(pc.ParticleBoxArray(lev), pc.ParticleDistributionMap(lev), mf.nComp(), mf.nGrowVect()); mf_pointer->setVal(0.); @@ -30,11 +30,11 @@ ParticleToMesh (PC const& pc, MF& mf, int lev, F&& f) const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); const auto& aos = tile.GetArrayOfStructs(); - const auto pstruct = aos().dataPtr(); + const auto pstruct = aos().dataPtr(); FArrayBox& fab = (*mf_pointer)[pti]; auto fabarr = fab.array(); - + AMREX_FOR_1D( np, i, { f(pstruct[i], fabarr); @@ -54,7 +54,7 @@ ParticleToMesh (PC const& pc, MF& mf, int lev, F&& f) const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); const auto& aos = tile.GetArrayOfStructs(); - const auto pstruct = aos().dataPtr(); + const auto pstruct = aos().dataPtr(); FArrayBox& fab = (*mf_pointer)[pti]; @@ -63,12 +63,12 @@ ParticleToMesh (PC const& pc, MF& mf, int lev, F&& f) local_fab.resize(tile_box,mf_pointer->nComp()); local_fab.setVal(0.0); auto fabarr = local_fab.array(); - + AMREX_FOR_1D( np, i, { f(pstruct[i], fabarr); }); - + fab.atomicAdd(local_fab, tile_box, tile_box, 0, 0, mf_pointer->nComp()); } } @@ -90,12 +90,12 @@ MeshToParticle (PC& pc, MF const& mf, int lev, F&& f) BL_PROFILE("amrex::MeshToParticle"); MultiFab* mf_pointer = pc.OnSameGrids(lev, mf) ? - const_cast(&mf) : new MultiFab(pc.ParticleBoxArray(lev), + const_cast(&mf) : new MultiFab(pc.ParticleBoxArray(lev), pc.ParticleDistributionMap(lev), mf.nComp(), mf.nGrowVect()); if (mf_pointer != &mf) mf_pointer->copy(mf,0,0,mf.nComp(),mf.nGrowVect(),mf.nGrowVect()); - + auto& plevel = pc.GetParticles(lev); using ParIter = typename PC::ParIterType; #ifdef AMREX_USE_OMP @@ -106,10 +106,10 @@ MeshToParticle (PC& pc, MF const& mf, int lev, F&& f) auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); auto& aos = tile.GetArrayOfStructs(); - auto pstruct = aos().dataPtr(); + auto pstruct = aos().dataPtr(); const FArrayBox& fab = (*mf_pointer)[pti]; - auto fabarr = fab.array(); + auto fabarr = fab.array(); AMREX_FOR_1D( np, i, { diff --git a/Src/Particle/AMReX_ParticleReduce.H b/Src/Particle/AMReX_ParticleReduce.H index 28b31a8f67e..2a885037188 100644 --- a/Src/Particle/AMReX_ParticleReduce.H +++ b/Src/Particle/AMReX_ParticleReduce.H @@ -42,7 +42,7 @@ ReduceSum (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates only on the specified level. - * + * * This version uses "Sum" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -51,13 +51,13 @@ ReduceSum (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev the level to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> auto ReduceSum (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticleType())) @@ -68,7 +68,7 @@ ReduceSum (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticl /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates from the specified lev_min to lev_max. - * + * * This version uses "Sum" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -77,9 +77,9 @@ ReduceSum (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticl * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev_min the minimum level to include * \param lev_max the maximum level to include * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. @@ -127,7 +127,7 @@ ReduceSum (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); for (int i = 0; i < np; ++i) sm += f(ptd.getSuperParticle(i)); } @@ -140,7 +140,7 @@ ReduceSum (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates over all particles on all levels. - * + * * This version uses "Max" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -149,12 +149,12 @@ ReduceSum (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> auto ReduceMax (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) @@ -165,7 +165,7 @@ ReduceMax (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates only on the specified level. - * + * * This version uses "Mas" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -174,9 +174,9 @@ ReduceMax (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev the level to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * @@ -191,7 +191,7 @@ ReduceMax (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticl /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates from the specified lev_min to lev_max. - * + * * This version uses "Max" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -200,14 +200,14 @@ ReduceMax (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticl * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev_min the minimum level to include * \param lev_max the maximum level to include * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> auto ReduceMax (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename PC::SuperParticleType())) @@ -230,7 +230,7 @@ ReduceMax (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); reduce_op.eval(np, reduce_data, [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple {return {f(ptd.getSuperParticle(i))};}); } @@ -251,7 +251,7 @@ ReduceMax (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); for (int i = 0; i < np; ++i) r = std::max(r, f(ptd.getSuperParticle(i))); } @@ -264,7 +264,7 @@ ReduceMax (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates over all particles on all levels. - * + * * This version uses "Min" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -273,12 +273,12 @@ ReduceMax (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> auto ReduceMin (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) @@ -289,7 +289,7 @@ ReduceMin (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates only on the specified level. - * + * * This version uses "Min" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -298,9 +298,9 @@ ReduceMin (PC const& pc, F&& f) -> decltype(f(typename PC::SuperParticleType())) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev the level to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * @@ -315,7 +315,7 @@ ReduceMin (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticl /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates from the specified lev_min to lev_max. - * + * * This version uses "Min" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -324,9 +324,9 @@ ReduceMin (PC const& pc, int lev, F&& f) -> decltype(f(typename PC::SuperParticl * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev_min the minimum level to include * \param lev_max the maximum level to include * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. @@ -354,7 +354,7 @@ ReduceMin (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); reduce_op.eval(np, reduce_data, [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple {return {f(ptd.getSuperParticle(i))};}); } @@ -375,7 +375,7 @@ ReduceMin (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); for (int i = 0; i < np; ++i) r = std::min(r, f(ptd.getSuperParticle(i))); } @@ -388,7 +388,7 @@ ReduceMin (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates over all particles on all levels. - * + * * This version uses "LogicalAnd" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -397,12 +397,12 @@ ReduceMin (PC const& pc, int lev_min, int lev_max, F&& f) -> decltype(f(typename * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> bool ReduceLogicalAnd (PC const& pc, F&& f) @@ -413,7 +413,7 @@ ReduceLogicalAnd (PC const& pc, F&& f) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates only on the specified level. - * + * * This version uses "LogicalAnd" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -422,13 +422,13 @@ ReduceLogicalAnd (PC const& pc, F&& f) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev the level to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> bool ReduceLogicalAnd (PC const& pc, int lev, F&& f) @@ -439,7 +439,7 @@ ReduceLogicalAnd (PC const& pc, int lev, F&& f) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates from the specified lev_min to lev_max. - * + * * This version uses "LogicalAnd" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -448,14 +448,14 @@ ReduceLogicalAnd (PC const& pc, int lev, F&& f) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev_min the minimum level to include * \param lev_max the maximum level to include * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> bool ReduceLogicalAnd (PC const& pc, int lev_min, int lev_max, F&& f) @@ -476,7 +476,7 @@ ReduceLogicalAnd (PC const& pc, int lev_min, int lev_max, F&& f) { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); reduce_op.eval(np, reduce_data, [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple {return {f(ptd.getSuperParticle(i))};}); } @@ -497,7 +497,7 @@ ReduceLogicalAnd (PC const& pc, int lev_min, int lev_max, F&& f) { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); for (int i = 0; i < np; ++i) r = r && f(ptd.getSuperParticle(i)); } @@ -510,7 +510,7 @@ ReduceLogicalAnd (PC const& pc, int lev_min, int lev_max, F&& f) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates over all particles on all levels. - * + * * This version uses "LogicalOr" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -519,12 +519,12 @@ ReduceLogicalAnd (PC const& pc, int lev_min, int lev_max, F&& f) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> bool ReduceLogicalOr (PC const& pc, F&& f) @@ -535,7 +535,7 @@ ReduceLogicalOr (PC const& pc, F&& f) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates only on the specified level. - * + * * This version uses "LogicalOr" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -544,9 +544,9 @@ ReduceLogicalOr (PC const& pc, F&& f) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev the level to operate on * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * @@ -561,7 +561,7 @@ ReduceLogicalOr (PC const& pc, int lev, F&& f) /** * \brief A general reduction method for the particles in a ParticleContainer that can run on either CPUs or GPUs. * This version operates from the specified lev_min to lev_max. - * + * * This version uses "LogicalOr" as the reduction operation. The quantity reduced over is an arbitrary function * of a "superparticle", which contains all the data in the particle type, whether it is stored in AoS or * SoA form. @@ -570,14 +570,14 @@ ReduceLogicalOr (PC const& pc, int lev, F&& f) * call the MPI reduction operations described in ParallelDescriptor if they want that behavior. * * \tparam PC the ParticleContainer type - * \tparam F a function object + * \tparam F a function object * - * \param pc the ParticleContainer to operate on + * \param pc the ParticleContainer to operate on * \param lev_min the minimum level to include * \param lev_max the maximum level to include * \param f a function that takes a "superparticle" and returns the value to be reduced over all particles. * - */ + */ template ::value, int> foo = 0> bool ReduceLogicalOr (PC const& pc, int lev_min, int lev_max, F&& f) @@ -598,9 +598,9 @@ ReduceLogicalOr (PC const& pc, int lev_min, int lev_max, F&& f) { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); reduce_op.eval(np, reduce_data, - [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple {return {f(ptd.getSuperParticle(i))};}); + [=] AMREX_GPU_DEVICE (const int i) -> ReduceTuple {return {f(ptd.getSuperParticle(i))};}); } } @@ -619,7 +619,7 @@ ReduceLogicalOr (PC const& pc, int lev_min, int lev_max, F&& f) { const auto& tile = pti.GetParticleTile(); const auto np = tile.numParticles(); - const auto ptd = tile.getConstParticleTileData(); + const auto ptd = tile.getConstParticleTileData(); for (int i = 0; i < np; ++i) r = r || f(ptd.getSuperParticle(i)); @@ -628,6 +628,6 @@ ReduceLogicalOr (PC const& pc, int lev_min, int lev_max, F&& f) } return r; -} +} } #endif diff --git a/Src/Particle/AMReX_ParticleTransformation.H b/Src/Particle/AMReX_ParticleTransformation.H index d86cb6572c2..c05d9836cf8 100644 --- a/Src/Particle/AMReX_ParticleTransformation.H +++ b/Src/Particle/AMReX_ParticleTransformation.H @@ -96,8 +96,8 @@ void copyParticle (const ParticleTileData& dst, */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void swapParticle (const ParticleTileData& dst, - const ParticleTileData& src, +void swapParticle (const ParticleTileData& dst, + const ParticleTileData& src, int src_i, int dst_i) noexcept { AMREX_ASSERT(dst.m_num_runtime_real == src.m_num_runtime_real); @@ -282,7 +282,7 @@ void transformParticles (DstTile1& dst1, DstTile2& dst2, const SrcTile& src, /** * \brief Conditionally copy particles from src to dst based on the value of mask. - * + * * \tparam DstTile the dst particle tile type * \tparam SrcTile the src particle tile type * \tparam Index the index type, e.g. unsigned int @@ -298,12 +298,12 @@ Index filterParticles (DstTile& dst, const SrcTile& src, const Index* mask) noex { return filterParticles(dst, src, mask, 0, 0, src.numParticles()); } - + /** * \brief Conditionally copy particles from src to dst based on the value of mask. - * This version conditionally copies n particles starting at index src_start, writing + * This version conditionally copies n particles starting at index src_start, writing * the result starting at dst_start. - * + * * \tparam DstTile the dst particle tile type * \tparam SrcTile the src particle tile type * \tparam Index the index type, e.g. unsigned int @@ -327,16 +327,16 @@ Index filterParticles (DstTile& dst, const SrcTile& src, const Index* mask, Index last_mask=0, last_offset=0; Gpu::copyAsync(Gpu::deviceToHost, mask+np-1, mask + np, &last_mask); - Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); - + Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); + auto p_offsets = offsets.dataPtr(); - + const auto src_data = src.getConstParticleTileData(); auto dst_data = dst.getParticleTileData(); AMREX_HOST_DEVICE_FOR_1D( n, i, { - if (mask[src_start+i]) copyParticle(dst_data, src_data, src_start+i, + if (mask[src_start+i]) copyParticle(dst_data, src_data, src_start+i, dst_start+p_offsets[src_start+i]); }); @@ -346,7 +346,7 @@ Index filterParticles (DstTile& dst, const SrcTile& src, const Index* mask, /** * \brief Conditionally copy particles from src to dst based on a predicate. - * + * * \tparam DstTile the dst particle tile type * \tparam SrcTile the src particle tile type * \tparam Pred a function object @@ -355,7 +355,7 @@ Index filterParticles (DstTile& dst, const SrcTile& src, const Index* mask, * \param src the source tile * \param p predicate function - particles will be copied if p returns true * - */ + */ template auto filterParticles (DstTile& dst, const SrcTile& src, Pred&& p) noexcept -> decltype(p(typename SrcTile::ConstParticleTileDataType(), 0)) @@ -365,9 +365,9 @@ auto filterParticles (DstTile& dst, const SrcTile& src, Pred&& p) noexcept /** * \brief Conditionally copy particles from src to dst based on a predicate. - * This version conditionally copies n particles starting at index src_start, writing + * This version conditionally copies n particles starting at index src_start, writing * the result starting at dst_start. - * + * * \tparam DstTile the dst particle tile type * \tparam SrcTile the src particle tile type * \tparam Pred a function object @@ -425,10 +425,10 @@ Index filterAndTransformParticles (DstTile& dst, const SrcTile& src, Index* mask Index last_mask, last_offset; Gpu::copyAsync(Gpu::deviceToHost, mask+np-1, mask + np, &last_mask); - Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); + Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); auto p_offsets = offsets.dataPtr(); - + const auto src_data = src.getConstParticleTileData(); auto dst_data = dst.getParticleTileData(); @@ -444,7 +444,7 @@ Index filterAndTransformParticles (DstTile& dst, const SrcTile& src, Index* mask /** * \brief Conditionally copy particles from src to dst based on a predicate. * A transformation will also be applied to the particles on copy. - * + * * \tparam DstTile the dst particle tile type * \tparam SrcTile the src particle tile type * \tparam Pred a function object @@ -455,7 +455,7 @@ Index filterAndTransformParticles (DstTile& dst, const SrcTile& src, Index* mask * \param p predicate function - particles will be copied if p returns true * \param f defines the transformation that will be applied to the particles on copy * - */ + */ template auto filterAndTransformParticles (DstTile& dst, const SrcTile& src, Pred&& p, F&& f) noexcept -> decltype(p(typename SrcTile::ConstParticleTileDataType(), 0)) @@ -479,7 +479,7 @@ auto filterAndTransformParticles (DstTile& dst, const SrcTile& src, Pred&& p, F& /** * \brief Conditionally copy particles from src to dst1 and dst2 based on the value of mask. * A transformation will also be applied to the particles on copy. - * + * * \tparam DstTile1 the dst1 particle tile type * \tparam DstTile2 the dst2 particle tile type * \tparam SrcTile the src particle tile type @@ -504,10 +504,10 @@ Index filterAndTransformParticles (DstTile1& dst1, DstTile2& dst2, Index last_mask, last_offset; Gpu::copyAsync(Gpu::deviceToHost, mask+np-1, mask + np, &last_mask); - Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); + Gpu::copyAsync(Gpu::deviceToHost, offsets.data()+np-1, offsets.data()+np, &last_offset); auto p_offsets = offsets.dataPtr(); - + const auto src_data = src.getConstParticleTileData(); auto dst_data1 = dst1.getParticleTileData(); auto dst_data2 = dst2.getParticleTileData(); @@ -524,7 +524,7 @@ Index filterAndTransformParticles (DstTile1& dst1, DstTile2& dst2, /** * \brief Conditionally copy particles from src to dst1 and dst2 based on a predicate. * A transformation will also be applied to the particles on copy. - * + * * \tparam DstTile1 the dst1 particle tile type * \tparam DstTile2 the dst2 particle tile type * \tparam SrcTile the src particle tile type @@ -538,7 +538,7 @@ Index filterAndTransformParticles (DstTile1& dst1, DstTile2& dst2, * \param p predicate function - particles will be copied if p returns true * \param f defines the transformation that will be applied to the particles on copy * - */ + */ template auto filterAndTransformParticles (DstTile1& dst1, DstTile2& dst2, const SrcTile& src, Pred&& p, F&& f) noexcept @@ -559,12 +559,12 @@ auto filterAndTransformParticles (DstTile1& dst1, DstTile2& dst2, const SrcTile& return filterAndTransformParticles(dst1, dst2, src, mask.dataPtr(), std::forward(f)); } - + /** - * \brief Gather particles copies particles into contiguous order from an + * \brief Gather particles copies particles into contiguous order from an * arbitrary order. Specifically, the particle at the index inds[i] in src * will be copied to the index i in dst. - * + * * \tparam PTile the particle tile type * \tparam N the size type, e.g. Long * \tparam Index the index type, e.g. unsigned int @@ -581,7 +581,7 @@ void gatherParticles (PTile& dst, const PTile& src, N np, const Index* inds) { const auto src_data = src.getConstParticleTileData(); auto dst_data = dst.getParticleTileData(); - + AMREX_HOST_DEVICE_FOR_1D( np, i, { copyParticle(dst_data, src_data, inds[i], i); @@ -594,7 +594,7 @@ void gatherParticles (PTile& dst, const PTile& src, N np, const Index* inds) * \brief Scatter particles copies particles from contiguous order into an * arbitrary order. Specifically, the particle at the index i in src * will be copied to the index inds[i] in dst. - * + * * \tparam PTile the particle tile type * \tparam N the size type, e.g. Long * \tparam Index the index type, e.g. unsigned int @@ -611,7 +611,7 @@ void scatterParticles (PTile& dst, const PTile& src, N np, const Index* inds) { const auto src_data = src.getConstParticleTileData(); auto dst_data = dst.getParticleTileData(); - + AMREX_HOST_DEVICE_FOR_1D( np, i, { copyParticle(dst_data, src_data, i, inds[i]); diff --git a/Src/Particle/AMReX_ParticleUtil.H b/Src/Particle/AMReX_ParticleUtil.H index 1fb2537ee2c..0470aa62185 100644 --- a/Src/Particle/AMReX_ParticleUtil.H +++ b/Src/Particle/AMReX_ParticleUtil.H @@ -223,7 +223,7 @@ numParticlesOutOfRange (PC const& pc, int lev_min, int lev_max, IntVect nGrow) AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE int getTileIndex (const IntVect& iv, const Box& box, const bool a_do_tiling, - const IntVect& a_tile_size, Box& tbx) + const IntVect& a_tile_size, Box& tbx) { if (a_do_tiling == false) { tbx = box; @@ -239,7 +239,7 @@ int getTileIndex (const IntVect& iv, const Box& box, const bool a_do_tiling, int ts_right = ncells/ntile; int ts_left = ts_right+1; int nleft = ncells - ntile*ts_right; - int ii = i - lo; + int ii = i - lo; int nbndry = nleft*ts_left; if (ii < nbndry) { tileidx = ii / ts_left; // tiles on the left of nbndry have size of ts_left @@ -256,12 +256,12 @@ int getTileIndex (const IntVect& iv, const Box& box, const bool a_do_tiling, IntVect ntiles, ivIndex, tilelo, tilehi; AMREX_D_TERM(int iv0 = amrex::min(amrex::max(iv[0], small[0]), big[0]);, - int iv1 = amrex::min(amrex::max(iv[1], small[1]), big[1]);, - int iv2 = amrex::min(amrex::max(iv[2], small[2]), big[2]);); + int iv1 = amrex::min(amrex::max(iv[1], small[1]), big[1]);, + int iv2 = amrex::min(amrex::max(iv[2], small[2]), big[2]);); AMREX_D_TERM(tiling_1d(iv0, small[0], big[0], a_tile_size[0], ntiles[0], ivIndex[0], tilelo[0], tilehi[0]);, - tiling_1d(iv1, small[1], big[1], a_tile_size[1], ntiles[1], ivIndex[1], tilelo[1], tilehi[1]);, - tiling_1d(iv2, small[2], big[2], a_tile_size[2], ntiles[2], ivIndex[2], tilelo[2], tilehi[2]);); + tiling_1d(iv1, small[1], big[1], a_tile_size[1], ntiles[1], ivIndex[1], tilelo[1], tilehi[1]);, + tiling_1d(iv2, small[2], big[2], a_tile_size[2], ntiles[2], ivIndex[2], tilelo[2], tilehi[2]);); tbx = Box(tilelo, tilehi); @@ -288,8 +288,8 @@ int numTilesInBox (const Box& box, const bool a_do_tiling, const IntVect& a_tile IntVect ntiles; AMREX_D_TERM(tiling_1d(small[0], big[0], a_tile_size[0], ntiles[0]);, - tiling_1d(small[1], big[1], a_tile_size[1], ntiles[1]);, - tiling_1d(small[2], big[2], a_tile_size[2], ntiles[2]);); + tiling_1d(small[1], big[1], a_tile_size[1], ntiles[1]);, + tiling_1d(small[2], big[2], a_tile_size[2], ntiles[2]);); return AMREX_D_TERM(ntiles[0], *=ntiles[1], *=ntiles[2]); } @@ -409,23 +409,23 @@ partitionParticlesByDest (PTile& ptile, const PLocator& ploc, const ParticleBuff } else { - auto p_prime = p; + auto p_prime = p; enforcePeriodic(p_prime, plo, phi, is_per); auto tup = ploc(p_prime, lev_min, lev_max, nGrow); assigned_grid = amrex::get<0>(tup); assigned_lev = amrex::get<1>(tup); - if (assigned_grid >= 0) - { - AMREX_D_TERM(p.pos(0) = p_prime.pos(0);, - p.pos(1) = p_prime.pos(1);, - p.pos(2) = p_prime.pos(2);); - } - else if (lev_min > 0) - { - auto tup = ploc(p, lev_min, lev_max, nGrow); - assigned_grid = amrex::get<0>(tup); - assigned_lev = amrex::get<1>(tup); - } + if (assigned_grid >= 0) + { + AMREX_D_TERM(p.pos(0) = p_prime.pos(0);, + p.pos(1) = p_prime.pos(1);, + p.pos(2) = p_prime.pos(2);); + } + else if (lev_min > 0) + { + auto tup = ploc(p, lev_min, lev_max, nGrow); + assigned_grid = amrex::get<0>(tup); + assigned_lev = amrex::get<1>(tup); + } } return ((assigned_grid == gid) && (assigned_lev == lev) && (getPID(lev, gid) == pid)); diff --git a/Src/Particle/AMReX_Particle_mod_K.H b/Src/Particle/AMReX_Particle_mod_K.H index b3b156af458..fce858890a3 100644 --- a/Src/Particle/AMReX_Particle_mod_K.H +++ b/Src/Particle/AMReX_Particle_mod_K.H @@ -17,20 +17,20 @@ void amrex_deposit_cic (P const& p, int nc, amrex::Array4 const& rh { #if (AMREX_SPACEDIM == 1) amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; - + int i = static_cast(amrex::Math::floor(lx)); - + amrex::Real xint = lx - i; - + amrex::Real sx[2] = {Real(1.0) - xint, xint}; - - for (int ii = 0; ii <= 1; ++ii) { + + for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, 0, 0, 0), static_cast(sx[ii]*p.rdata(0))); } - + for (int comp=1; comp < nc; ++comp) { - for (int ii = 0; ii <= 1; ++ii) { - amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, 0, 0, comp), + for (int ii = 0; ii <= 1; ++ii) { + amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, 0, 0, comp), static_cast(sx[ii]*p.rdata(0)*p.rdata(comp))); } } @@ -47,16 +47,16 @@ void amrex_deposit_cic (P const& p, int nc, amrex::Array4 const& rh amrex::Real sx[2] = {Real(1.0) - xint, xint}; amrex::Real sy[2] = {Real(1.0) - yint, yint}; - for (int jj = 0; jj <= 1; ++jj) { - for (int ii = 0; ii <= 1; ++ii) { + for (int jj = 0; jj <= 1; ++jj) { + for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, 0, 0), static_cast(sx[ii]*sy[jj]*p.rdata(0))); } } for (int comp=1; comp < nc; ++comp) { - for (int jj = 0; jj <= 1; ++jj) { - for (int ii = 0; ii <= 1; ++ii) { + for (int jj = 0; jj <= 1; ++jj) { + for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, 0, comp), static_cast(sx[ii]*sy[jj]*p.rdata(0)*p.rdata(comp))); } @@ -79,9 +79,9 @@ void amrex_deposit_cic (P const& p, int nc, amrex::Array4 const& rh amrex::Real sx[] = {Real(1.0) - xint, xint}; amrex::Real sy[] = {Real(1.0) - yint, yint}; amrex::Real sz[] = {Real(1.0) - zint, zint}; - - for (int kk = 0; kk <= 1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) { + + for (int kk = 0; kk <= 1; ++kk) { + for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, 0), static_cast(sx[ii]*sy[jj]*sz[kk]*p.rdata(0))); @@ -90,8 +90,8 @@ void amrex_deposit_cic (P const& p, int nc, amrex::Array4 const& rh } for (int comp=1; comp < nc; ++comp) { - for (int kk = 0; kk <= 1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) { + for (int kk = 0; kk <= 1; ++kk) { + for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, comp), static_cast(sx[ii]*sy[jj]*sz[kk]*p.rdata(0)*p.rdata(comp))); @@ -121,11 +121,11 @@ void amrex_deposit_particle_dx_cic (P const& p, int nc, amrex::Array4(amrex::Math::floor(lx)); int hi_x = static_cast(amrex::Math::floor(hx)); - + for (int i = lo_x; i <= hi_x; ++i) { if (i < rho.begin.x || i >= rho.end.x) continue; amrex::Real wx = amrex::min(hx - i, amrex::Real(1.0)) - amrex::max(lx - i, amrex::Real(0.0)); - amrex::Real weight = wx*factor; + amrex::Real weight = wx*factor; amrex::Gpu::Atomic::AddNoRet(&rho(i, 0, 0, 0), static_cast(weight*p.rdata(0))); } @@ -133,7 +133,7 @@ void amrex_deposit_particle_dx_cic (P const& p, int nc, amrex::Array4= rho.end.x) continue; - amrex::Real wx = amrex::min(hx - i, amrex::Real(1.0)) - amrex::max(lx - i, amrex::Real(0.0)); + amrex::Real wx = amrex::min(hx - i, amrex::Real(1.0)) - amrex::max(lx - i, amrex::Real(0.0)); amrex::Real weight = wx*factor; amrex::Gpu::Atomic::AddNoRet(&rho(i, 0, 0, comp), static_cast(weight*p.rdata(0)*p.rdata(comp))); } @@ -180,23 +180,23 @@ void amrex_deposit_particle_dx_cic (P const& p, int nc, amrex::Array4(amrex::Math::floor(lx)); int lo_y = static_cast(amrex::Math::floor(ly)); int lo_z = static_cast(amrex::Math::floor(lz)); - + int hi_x = static_cast(amrex::Math::floor(hx)); int hi_y = static_cast(amrex::Math::floor(hy)); int hi_z = static_cast(amrex::Math::floor(hz)); - + for (int k = lo_z; k <= hi_z; ++k) { if (k < rho.begin.z || k >= rho.end.z) continue; amrex::Real wz = amrex::min(hz - k, amrex::Real(1.0)) - amrex::max(lz - k, amrex::Real(0.0)); @@ -241,15 +241,15 @@ void amrex_interpolate_cic (P const& p, int nc, amrex::Array4 { #if (AMREX_SPACEDIM == 1) amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; - + int i = static_cast(amrex::Math::floor(lx)); - + amrex::Real xint = lx - i; - + amrex::Real sx[] = {Real(1.0)-xint, xint}; - for (int comp=0; comp < nc; ++comp) { - for (int ii = 0; ii <= 1; ++ii) { + for (int comp=0; comp < nc; ++comp) { + for (int ii = 0; ii <= 1; ++ii) { amrex::Real acceleration = sx[ii]*acc(i+ii-1,0,0,comp); amrex::ignore_unused(acceleration); } @@ -257,19 +257,19 @@ void amrex_interpolate_cic (P const& p, int nc, amrex::Array4 #elif (AMREX_SPACEDIM == 2) amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; - + int i = static_cast(amrex::Math::floor(lx)); int j = static_cast(amrex::Math::floor(ly)); - + amrex::Real xint = lx - i; amrex::Real yint = ly - j; - + amrex::Real sx[] = {Real(1.0)-xint, xint}; amrex::Real sy[] = {Real(1.0)-yint, yint}; - for (int comp=0; comp < nc; ++comp) { - for (int jj = 0; jj <= 1; ++jj) { - for (int ii = 0; ii <= 1; ++ii) { + for (int comp=0; comp < nc; ++comp) { + for (int jj = 0; jj <= 1; ++jj) { + for (int ii = 0; ii <= 1; ++ii) { amrex::Real acceleration = sx[ii]*sy[jj]*acc(i+ii-1,j+jj-1,0,comp); amrex::ignore_unused(acceleration); } @@ -279,22 +279,22 @@ void amrex_interpolate_cic (P const& p, int nc, amrex::Array4 amrex::Real lx = (p.pos(0) - plo[0]) * dxi[0] + 0.5; amrex::Real ly = (p.pos(1) - plo[1]) * dxi[1] + 0.5; amrex::Real lz = (p.pos(2) - plo[2]) * dxi[2] + 0.5; - + int i = static_cast(amrex::Math::floor(lx)); int j = static_cast(amrex::Math::floor(ly)); int k = static_cast(amrex::Math::floor(lz)); - + amrex::Real xint = lx - i; amrex::Real yint = ly - j; amrex::Real zint = lz - k; - + amrex::Real sx[] = {Real(1.0)-xint, xint}; amrex::Real sy[] = {Real(1.0)-yint, yint}; amrex::Real sz[] = {Real(1.0)-zint, zint}; - for (int comp=0; comp < nc; ++comp) { - for (int kk = 0; kk <= 1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) { + for (int comp=0; comp < nc; ++comp) { + for (int kk = 0; kk <= 1; ++kk) { + for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) { amrex::Real acceleration = sx[ii]*sy[jj]*sz[kk]*acc(i+ii-1,j+jj-1,k+kk-1,comp); amrex::ignore_unused(acceleration); diff --git a/Src/Particle/AMReX_Particles.H b/Src/Particle/AMReX_Particles.H index 21ede04978f..54b6d4cd189 100644 --- a/Src/Particle/AMReX_Particles.H +++ b/Src/Particle/AMReX_Particles.H @@ -222,9 +222,9 @@ public: //! \param A BoxArray, which gives the set of grid boxes //! ParticleContainer (const Geometry & geom, - const DistributionMapping & dmap, - const BoxArray & ba) - : + const DistributionMapping & dmap, + const BoxArray & ba) + : ParticleContainerBase(geom, dmap, ba), h_communicate_real_comp(NArrayReal, true), h_communicate_int_comp(NArrayInt, true), @@ -247,10 +247,10 @@ public: //! refinement ratio between levels n and n+1 //! ParticleContainer (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr) - : + const Vector & dmap, + const Vector & ba, + const Vector & rr) + : ParticleContainerBase(geom, dmap, ba, rr), h_communicate_real_comp(NArrayReal, true), h_communicate_int_comp(NArrayInt, true), @@ -272,10 +272,10 @@ public: //! refinement ratio between levels n and n+1 //! ParticleContainer (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr) - : + const Vector & dmap, + const Vector & ba, + const Vector & rr) + : ParticleContainerBase(geom, dmap, ba, rr), h_communicate_real_comp(NArrayReal, true), h_communicate_int_comp(NArrayInt, true), @@ -319,8 +319,8 @@ public: //! \param A BoxArray, which gives the set of grid boxes //! void Define (const Geometry & geom, - const DistributionMapping & dmap, - const BoxArray & ba) + const DistributionMapping & dmap, + const BoxArray & ba) { this->ParticleContainerBase::Define(geom, dmap, ba); reserveData(); @@ -337,13 +337,13 @@ public: //! refinement ratio between levels n and n+1 //! void Define (const Vector & geom, - const Vector & dmap, - const Vector & ba, - const Vector & rr) + const Vector & dmap, + const Vector & ba, + const Vector & rr) { this->ParticleContainerBase::Define(geom, dmap, ba, rr); reserveData(); - resizeData(); + resizeData(); } //! \brief The total number of tiles on this rank on this level @@ -684,15 +684,15 @@ public: /** * \brief Writes particle data to disk in the AMReX native format. * - * \tparam F function type - * + * \tparam F function type + * * \param dir The base directory into which to write (i.e. "plt00000") * \param name The name of the sub-directory for this particle type (i.e. "Tracer") * \param write_real_comp for each real component, whether or not we include that component in the file * \param write_int_comp for each integer component, whether or not we include that component in the file * \param real_comp_names for each real component, a name to label the data with * \param int_comp_names for each integer component, a name to label the data with - * \param f callable that returns whether a given particle should be written or not + * \param f callable that returns whether a given particle should be written or not */ template void WriteBinaryParticleData (const std::string& dir, @@ -701,7 +701,7 @@ public: const Vector& write_int_comp, const Vector& real_comp_names, const Vector& int_comp_names, - F&& f) const; + F&& f) const; void CheckpointPre (); @@ -820,7 +820,7 @@ public: * \param file The name of the sub-directory for this particle type (i.e. "Tracer") * \param real_comp_names for each real component, a name to label the data with * \param int_comp_names for each integer component, a name to label the data with - * \param f callable that returns whether or not to write each particle + * \param f callable that returns whether or not to write each particle */ template void WritePlotFile (const std::string& dir, const std::string& name, @@ -915,10 +915,10 @@ public: * the user should pass in vectors containing names of all the components, whether * they are written or not. * - * This version also lets you pass in a functor to toggle whether each particle gets output. - * - * \tparam F function type - * + * This version also lets you pass in a functor to toggle whether each particle gets output. + * + * \tparam F function type + * * \param dir The base directory into which to write (i.e. "plt00000") * \param file The name of the sub-directory for this particle type (i.e. "Tracer") * \param write_real_comp for each real component, whether to include that comp in the file @@ -1111,7 +1111,7 @@ protected: */ template bool Where (const P& prt, ParticleLocData& pld, - int lev_min = 0, int lev_max = -1, int nGrow=0, int local_grid=-1) const; + int lev_min = 0, int lev_max = -1, int nGrow=0, int local_grid=-1) const; /** @@ -1158,7 +1158,7 @@ private: int /*grid*/, const ParticleType& /*p*/) {} void RedistributeMPI (std::map >& not_ours, - int lev_min = 0, int lev_max = 0, int nGrow = 0, int local=0); + int lev_min = 0, int lev_max = 0, int nGrow = 0, int local=0); void locateParticle (ParticleType& p, ParticleLocData& pld, int lev_min, int lev_max, int nGrow, int local_grid=-1) const; diff --git a/Src/Particle/AMReX_SparseBins.H b/Src/Particle/AMReX_SparseBins.H index c0857005c28..93635784822 100644 --- a/Src/Particle/AMReX_SparseBins.H +++ b/Src/Particle/AMReX_SparseBins.H @@ -9,15 +9,15 @@ namespace amrex { - + template struct SparseBinIteratorFactory { using index_type = unsigned int; - + SparseBinIteratorFactory (const Gpu::DeviceVector& bins, - const Gpu::DeviceVector& offsets, + const Gpu::DeviceVector& offsets, const Gpu::DeviceVector& permutation, const T* items) : m_bins_ptr(bins.dataPtr()), @@ -37,17 +37,17 @@ struct SparseBinIteratorFactory while (lo < hi) { if (m_bins_ptr[lo] == bin_number) return lo; if (m_bins_ptr[hi] == bin_number) return hi; - + index_type mid = (lo + hi) / 2; index_type mid_value = m_bins_ptr[mid]; if (mid_value == bin_number) return mid; - - mid_value < bin_number ? lo = mid+1 : hi = mid; + + mid_value < bin_number ? lo = mid+1 : hi = mid; } - + return m_not_found; - } - + } + AMREX_GPU_HOST_DEVICE BinIterator getBinIterator (const index_type bin_number) const noexcept { @@ -60,16 +60,16 @@ struct SparseBinIteratorFactory const index_type* m_permutation_ptr; const T* m_items; index_type m_num_bins; - - static constexpr index_type m_not_found = std::numeric_limits::max(); + + static constexpr index_type m_not_found = std::numeric_limits::max(); }; - + /** * \brief A container for storing items in a set of bins using "sparse" storage. - * + * * The underlying data structure consists of three arrays. The first is a sorted - * array of bin numbers with a size equal to the number of non-zero bins. The - * second is an array of size nitems defining a permutation of the items in the container + * array of bin numbers with a size equal to the number of non-zero bins. The + * second is an array of size nitems defining a permutation of the items in the container * that puts them in bin-sorted order. Finally, there is an offsets array that stores, * for each non-zero bin, the offsets into the permutation array where each bin starts. * @@ -79,7 +79,7 @@ struct SparseBinIteratorFactory * * \tparam The type of items we hold * - */ + */ template class SparseBins { @@ -98,15 +98,15 @@ public: * \param nitems the number of items to put in the bins * \param v pointer to the start of the items * \param bx the Box that defines the space over which the bins will be defined - * \param f a function object that maps items to bins - */ + * \param f a function object that maps items to bins + */ template void build (N nitems, T const* v, const Box& bx, F&& f) { BL_PROFILE("SparseBins::build"); m_items = v; - + Gpu::HostVector host_cells(nitems); std::map bins_map; const auto lo = lbound(bx); @@ -129,7 +129,7 @@ public: std::iota(host_perm.begin(), host_perm.end(), 0); std::sort(host_perm.begin(), host_perm.end(), [&](int i, int j) {return host_cells[i] < host_cells[j];}); - + Gpu::HostVector host_bins; Gpu::HostVector host_offsets = {0}; for (const auto& kv : bins_map) @@ -137,10 +137,10 @@ public: host_bins.push_back(kv.first); host_offsets.push_back(host_offsets.back() + kv.second); } - + m_bins.resize(host_bins.size()); Gpu::copy(Gpu::hostToDevice, host_bins.begin(), host_bins.end(), m_bins.begin()); - + m_offsets.resize(host_offsets.size()); Gpu::copy(Gpu::hostToDevice, host_offsets.begin(), host_offsets.end(), m_offsets.begin()); @@ -154,7 +154,7 @@ public: //! \brief the number of bins in the container Long numBins () const noexcept { return m_offsets.size()-1; } - //! \brief returns the pointer to the permutation array + //! \brief returns the pointer to the permutation array index_type* permutationPtr () noexcept { return m_perm.dataPtr(); } //! \brief returns the pointer to the offsets array @@ -162,8 +162,8 @@ public: //! \brief returns the pointer to the array of non-zero bins index_type* getNonZeroBinsPtr() noexcept { return m_bins.dataPtr(); } - - //! \brief returns const pointer to the permutation array + + //! \brief returns const pointer to the permutation array const index_type* permutationPtr () const noexcept { return m_perm.dataPtr(); } //! \brief returns const pointer to the offsets array @@ -177,11 +177,11 @@ public: { return SparseBinIteratorFactory(m_bins, m_offsets, m_perm, m_items); } - + private: const T* m_items; - + Gpu::DeviceVector m_bins; Gpu::DeviceVector m_offsets; Gpu::DeviceVector m_perm; diff --git a/Src/Particle/AMReX_TracerParticle_mod_K.H b/Src/Particle/AMReX_TracerParticle_mod_K.H index 3c38caa1bb8..99d1e590c38 100644 --- a/Src/Particle/AMReX_TracerParticle_mod_K.H +++ b/Src/Particle/AMReX_TracerParticle_mod_K.H @@ -17,10 +17,10 @@ namespace amrex{ template AMREX_GPU_HOST_DEVICE AMREX_INLINE void cic_interpolate (const P& p, - amrex::GpuArray const& plo, - amrex::GpuArray const& dxi, - const amrex::Array4 & uccarr, - amrex::ParticleReal * val, int M) + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Array4 & uccarr, + amrex::ParticleReal * val, int M) { AMREX_ASSERT(val != nullptr); @@ -38,7 +38,7 @@ void cic_interpolate (const P& p, { val[d] = ParticleReal(0.0); for (int ii = 0; ii<=1; ++ii) - { + { val[d] += static_cast(sx[ii]*uccarr(i+ii,0,0,d)); } } @@ -62,9 +62,9 @@ void cic_interpolate (const P& p, { val[d] = ParticleReal(0.0); for (int jj = 0; jj <= 1; ++jj) - { + { for (int ii = 0; ii <= 1; ++ii) - { + { val[d] += static_cast(sx[ii]*sy[jj]*uccarr(i+ii,j+jj,0,d)); } } @@ -92,7 +92,7 @@ void cic_interpolate (const P& p, { val[d] = ParticleReal(0.0); for (int kk = 0; kk<=1; ++kk) - { + { for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) @@ -108,10 +108,10 @@ void cic_interpolate (const P& p, template AMREX_GPU_HOST_DEVICE AMREX_INLINE void cic_interpolate (const P& p, - amrex::GpuArray const& plo, - amrex::GpuArray const& dxi, - const amrex::Array4 & uccarr, - amrex::ParticleReal * val) + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + const amrex::Array4 & uccarr, + amrex::ParticleReal * val) { cic_interpolate(p, plo, dxi, uccarr, val, AMREX_SPACEDIM); } @@ -119,10 +119,10 @@ void cic_interpolate (const P& p, template AMREX_GPU_HOST_DEVICE AMREX_INLINE void mac_interpolate (const P& p, - amrex::GpuArray const& plo, - amrex::GpuArray const& dxi, - amrex::GpuArray,AMREX_SPACEDIM> const& p_uccarr, - amrex::ParticleReal * val) + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi, + amrex::GpuArray,AMREX_SPACEDIM> const& p_uccarr, + amrex::ParticleReal * val) { #if (AMREX_SPACEDIM == 1) @@ -193,11 +193,11 @@ void mac_interpolate (const P& p, val[d] = ParticleReal(0.0); for (int kk = 0; kk <=1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) + for (int jj = 0; jj <= 1; ++jj) { - for (int ii = 0; ii <= 1; ++ii) + for (int ii = 0; ii <= 1; ++ii) { - val[d] += static_cast((p_uccarr[d])(i+ii, j+jj, k+kk ,0)*sx[ii]*sy[jj]*sz[kk]); + val[d] += static_cast((p_uccarr[d])(i+ii, j+jj, k+kk ,0)*sx[ii]*sy[jj]*sz[kk]); } } } diff --git a/Src/Particle/AMReX_TracerParticles.H b/Src/Particle/AMReX_TracerParticles.H index f1f46387836..8217450f83c 100644 --- a/Src/Particle/AMReX_TracerParticles.H +++ b/Src/Particle/AMReX_TracerParticles.H @@ -12,14 +12,14 @@ class TracerParticleContainer public: TracerParticleContainer (ParGDBBase* gdb) - : ParticleContainer(gdb) - {} + : ParticleContainer(gdb) + {} - TracerParticleContainer (const Geometry & geom, - const DistributionMapping & dmap, - const BoxArray & ba) - : ParticleContainer(geom,dmap,ba) - {} + TracerParticleContainer (const Geometry & geom, + const DistributionMapping & dmap, + const BoxArray & ba) + : ParticleContainer(geom,dmap,ba) + {} ~TracerParticleContainer () {} @@ -28,7 +28,7 @@ public: void AdvectWithUcc (const MultiFab& ucc, int level, Real dt); void Timestamp (const std::string& file, const MultiFab& mf, int lev, Real time, - const std::vector& idx); + const std::vector& idx); }; using TracerParIter = ParIter; diff --git a/Src/Particle/AMReX_TracerParticles.cpp b/Src/Particle/AMReX_TracerParticles.cpp index dfad7dd5845..0b4ecebb418 100644 --- a/Src/Particle/AMReX_TracerParticles.cpp +++ b/Src/Particle/AMReX_TracerParticles.cpp @@ -33,23 +33,23 @@ TracerParticleContainer::AdvectWithUmac (MultiFab* umac, int lev, Real dt) if (OnSameGrids(lev, umac[0])) { for (int i = 0; i < AMREX_SPACEDIM; i++) { - umac_pointer[i] = &umac[i]; - } + umac_pointer[i] = &umac[i]; + } } else { for (int i = 0; i < AMREX_SPACEDIM; i++) { - int ng = umac[i].nGrow(); - raii_umac[i].reset(new MultiFab(amrex::convert(m_gdb->ParticleBoxArray(lev), + int ng = umac[i].nGrow(); + raii_umac[i].reset(new MultiFab(amrex::convert(m_gdb->ParticleBoxArray(lev), IntVect::TheDimensionVector(i)), - m_gdb->ParticleDistributionMap(lev), - umac[i].nComp(), ng)); + m_gdb->ParticleDistributionMap(lev), + umac[i].nComp(), ng)); - umac_pointer[i] = raii_umac[i].get(); - umac_pointer[i]->copy(umac[i],0,0,umac[i].nComp(),ng,ng); + umac_pointer[i] = raii_umac[i].get(); + umac_pointer[i]->copy(umac[i],0,0,umac[i].nComp(),ng,ng); } } @@ -107,14 +107,14 @@ TracerParticleContainer::AdvectWithUmac (MultiFab* umac, int lev, Real dt) auto stoptime = amrex::second() - strttime; #ifdef AMREX_LAZY - Lazy::QueueReduction( [=] () mutable { + Lazy::QueueReduction( [=] () mutable { #endif ParallelReduce::Max(stoptime, ParallelContext::IOProcessorNumberSub(), ParallelContext::CommunicatorSub()); amrex::Print() << "TracerParticleContainer::AdvectWithUmac() time: " << stoptime << '\n'; #ifdef AMREX_LAZY - }); + }); #endif } } @@ -187,7 +187,7 @@ TracerParticleContainer::AdvectWithUcc (const MultiFab& Ucc, int lev, Real dt) auto stoptime = amrex::second() - strttime; #ifdef AMREX_LAZY - Lazy::QueueReduction( [=] () mutable { + Lazy::QueueReduction( [=] () mutable { #endif ParallelReduce::Max(stoptime, ParallelContext::IOProcessorNumberSub(), ParallelContext::CommunicatorSub()); @@ -201,10 +201,10 @@ TracerParticleContainer::AdvectWithUcc (const MultiFab& Ucc, int lev, Real dt) void TracerParticleContainer::Timestamp (const std::string& basename, - const MultiFab& mf, - int lev, - Real time, - const std::vector& indices) + const MultiFab& mf, + int lev, + Real time, + const std::vector& indices) { BL_PROFILE("TracerParticleContainer::Timestamp()"); // @@ -240,28 +240,28 @@ TracerParticleContainer::Timestamp (const std::string& basename, for (int iSet = 0; iSet < nSets; ++iSet) { if (mySet == iSet) - { + { // // Do we have any particles at this level that need writing? // bool gotwork = false; const auto& pmap = GetParticles(lev); - for (auto& kv : pmap) { + for (auto& kv : pmap) { const auto& pbox = kv.second.GetArrayOfStructs(); - for (int k = 0; k < pbox.numParticles(); ++k) - { - const ParticleType& p = pbox[k]; - if (p.id() > 0) { - gotwork = true; - break; - } - } - if (gotwork) break; - } + for (int k = 0; k < pbox.numParticles(); ++k) + { + const ParticleType& p = pbox[k]; + if (p.id() > 0) { + gotwork = true; + break; + } + } + if (gotwork) break; + } if (gotwork) - { + { std::string FileName = amrex::Concatenate(basename + '_', MyProc % nOutFiles, 2); VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); @@ -286,48 +286,48 @@ TracerParticleContainer::Timestamp (const std::string& basename, std::vector vals(M); - for (auto& kv : pmap) { - int grid = kv.first.first; - const auto& pbox = kv.second.GetArrayOfStructs(); - const Box& bx = ba[grid]; - const FArrayBox& fab = mf[grid]; + for (auto& kv : pmap) { + int grid = kv.first.first; + const auto& pbox = kv.second.GetArrayOfStructs(); + const Box& bx = ba[grid]; + const FArrayBox& fab = mf[grid]; const auto uccarr = fab.array(); - for (int k = 0; k < pbox.numParticles(); ++k) - { - const ParticleType& p = pbox[k]; + for (int k = 0; k < pbox.numParticles(); ++k) + { + const ParticleType& p = pbox[k]; - if (p.id() <= 0) continue; + if (p.id() <= 0) continue; - const IntVect& iv = Index(p,lev); + const IntVect& iv = Index(p,lev); - if (!bx.contains(iv) && !ba.contains(iv)) continue; + if (!bx.contains(iv) && !ba.contains(iv)) continue; - TimeStampFile << p.id() << ' ' << p.cpu() << ' '; + TimeStampFile << p.id() << ' ' << p.cpu() << ' '; - AMREX_D_TERM(TimeStampFile << p.pos(0) << ' ';, + AMREX_D_TERM(TimeStampFile << p.pos(0) << ' ';, TimeStampFile << p.pos(1) << ' ';, TimeStampFile << p.pos(2) << ' ';); - TimeStampFile << time; - // - // AdvectWithUmac stores the velocity in rdata ... - // - AMREX_D_TERM(TimeStampFile << ' ' << p.rdata(0);, + TimeStampFile << time; + // + // AdvectWithUmac stores the velocity in rdata ... + // + AMREX_D_TERM(TimeStampFile << ' ' << p.rdata(0);, TimeStampFile << ' ' << p.rdata(1);, TimeStampFile << ' ' << p.rdata(2);); - if (M > 0) + if (M > 0) { cic_interpolate(p, plo, dxi, uccarr, &vals[0], M); - for (int i = 0; i < M; i++) + for (int i = 0; i < M; i++) { - TimeStampFile << ' ' << vals[i]; + TimeStampFile << ' ' << vals[i]; } } - TimeStampFile << '\n'; + TimeStampFile << '\n'; } } diff --git a/Src/SDC/AMReX_SDCquadrature.F90 b/Src/SDC/AMReX_SDCquadrature.F90 index cdec1b5412f..571d736b3ed 100644 --- a/Src/SDC/AMReX_SDCquadrature.F90 +++ b/Src/SDC/AMReX_SDCquadrature.F90 @@ -4,7 +4,7 @@ module SDCquadrature_mod use iso_c_binding implicit none - integer, parameter :: qp = c_long_double + integer, parameter :: qp = c_long_double integer, parameter :: dp = c_double real(qp), parameter :: eps = 1.0e-23_qp @@ -144,7 +144,7 @@ subroutine SDC_quadrature(qtype_in,nnodes, nnodes0, nodes, nflags, qmats) bind(C end do end do - + end subroutine SDC_quadrature @@ -273,7 +273,7 @@ subroutine sdc_qnodes(qnodes, flags, qtype, nnodes) bind(c) end subroutine sdc_qnodes - !> Subroutine to compute the quadrature matrices + !> Subroutine to compute the quadrature matrices subroutine sdc_qmats(qmat, smat, dst, src, flags, ndst, nsrc) bind(c) integer(c_int), intent(in), value :: ndst !< Number of destination points integer(c_int), intent(in), value :: nsrc !< Number of source points @@ -281,7 +281,7 @@ subroutine sdc_qmats(qmat, smat, dst, src, flags, ndst, nsrc) bind(c) real(c_long_double), intent(in) :: src(nsrc) !< Source points real(amrex_real), intent(out) :: qmat(ndst-1, nsrc) !< O to dst quadrature weights real(amrex_real), intent(out) :: smat(ndst-1, nsrc) !< dst(m) to dst(m+1) quadrature weights - integer(c_int), intent(in) :: flags(nsrc) + integer(c_int), intent(in) :: flags(nsrc) integer :: i, j, m real(qp) :: q, s, den, p(0:nsrc) @@ -329,7 +329,7 @@ end subroutine sdc_qmats !! !! p = [ a_0, a_1, ..., a_n ]. !! - + !> Function to evaluate real polynomial real(qp) function poly_eval(p, n, x) result(v) bind(c) integer, intent(in), value :: n diff --git a/Src/SDC/AMReX_SDCstruct.H b/Src/SDC/AMReX_SDCstruct.H index a3778e8740b..d66f6b82e03 100644 --- a/Src/SDC/AMReX_SDCstruct.H +++ b/Src/SDC/AMReX_SDCstruct.H @@ -21,7 +21,7 @@ extern "C" { * \param qmats */ void SDC_quadrature(int* qtype_in, int* nnodes, int* nnodes0, - amrex_real* nodes, int* nflags,amrex_real* qmats); + amrex_real* nodes, int* nflags,amrex_real* qmats); } class SDCstruct { diff --git a/Src/SDC/AMReX_SDCstruct.cpp b/Src/SDC/AMReX_SDCstruct.cpp index cb99fceb34c..3c64757c7bb 100644 --- a/Src/SDC/AMReX_SDCstruct.cpp +++ b/Src/SDC/AMReX_SDCstruct.cpp @@ -3,37 +3,37 @@ SDCstruct::SDCstruct(int Nnodes_in,int Npieces_in, MultiFab& sol_in) { - + Nnodes=Nnodes_in; - Npieces=Npieces_in; - + Npieces=Npieces_in; + qnodes= new Real[Nnodes]; - Qall= new Real[4*(Nnodes-1)*Nnodes]; + Qall= new Real[4*(Nnodes-1)*Nnodes]; Nflags= new int[Nnodes]; Qgauss.resize(Nnodes-1, Vector(Nnodes)); Qexp.resize(Nnodes-1, Vector(Nnodes)); Qimp.resize(Nnodes-1, Vector(Nnodes)); - QLU.resize(Nnodes-1, Vector(Nnodes)); + QLU.resize(Nnodes-1, Vector(Nnodes)); // Make the quadrature tables - SDC_quadrature(&qtype, &Nnodes, &Nnodes,qnodes,Nflags, &Qall[0]); + SDC_quadrature(&qtype, &Nnodes, &Nnodes,qnodes,Nflags, &Qall[0]); // Load the quadrature nodes into their spots for ( int j = 0; j < Nnodes-1; ++j) for ( int k = 0; k < Nnodes; ++k) { - Qgauss[j][k]= Qall[0*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; - Qexp[j][k]= Qall[1*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; - Qimp[j][k]= Qall[2*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; - QLU[j][k]= Qall[3*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; + Qgauss[j][k]= Qall[0*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; + Qexp[j][k]= Qall[1*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; + Qimp[j][k]= Qall[2*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; + QLU[j][k]= Qall[3*(Nnodes-1)*(Nnodes) +j*(Nnodes) + k ]; } // Resize the storage sol.resize(Nnodes); res.resize(Nnodes); f.resize(Npieces); - if (Npieces == 3) + if (Npieces == 3) Ithree.resize(Nnodes); // Assign geomety and multifab info @@ -42,19 +42,19 @@ SDCstruct::SDCstruct(int Nnodes_in,int Npieces_in, MultiFab& sol_in) const int Nghost=sol_in.nGrow(); const int Ncomp=sol_in.nComp(); - for (auto& v : f) v.resize(Nnodes); + for (auto& v : f) v.resize(Nnodes); for (int sdc_m = 0; sdc_m < Nnodes; sdc_m++) { sol[sdc_m].define(ba, dm, Ncomp, Nghost); res[sdc_m].define(ba, dm, Ncomp, Nghost); for (int i = 0; i < Npieces; i++) - { - f[i][sdc_m].define(ba, dm, Ncomp, Nghost); - } + { + f[i][sdc_m].define(ba, dm, Ncomp, Nghost); + } if (Npieces == 3) - Ithree[sdc_m].define(ba, dm, Ncomp, Nghost); + Ithree[sdc_m].define(ba, dm, Ncomp, Nghost); } - + } void SDCstruct::SDC_rhs_integrals(Real dt) @@ -66,30 +66,30 @@ void SDCstruct::SDC_rhs_integrals(Real dt) for (int sdc_m = 0; sdc_m < Nnodes-1; sdc_m++) { for ( MFIter mfi(res[sdc_m]); mfi.isValid(); ++mfi ) - { - res[sdc_m][mfi].setVal(0.0); - if (Npieces == 3) - Ithree[sdc_m][mfi].setVal(0.0); - for (int sdc_n = 0; sdc_n < Nnodes; sdc_n++) - { - qij = dt*(Qgauss[sdc_m][sdc_n]-Qexp[sdc_m][sdc_n]); - res[sdc_m][mfi].saxpy(qij,f[0][sdc_n][mfi]); // Explicit part - - qij = dt*(Qgauss[sdc_m][sdc_n]-Qimp[sdc_m][sdc_n]); - res[sdc_m][mfi].saxpy(qij,f[1][sdc_n][mfi]); // Implicit part - } - if (Npieces == 3) - { - for (int sdc_n = 0; sdc_n < Nnodes; sdc_n++) - { // // MISDC pieces - qij = dt*(Qgauss[sdc_m][sdc_n]); // leave off -dt*Qtil and add it later - res[sdc_m][mfi].saxpy(qij,f[2][sdc_n][mfi]); - // Compute seperate integral for f_3 piece - qij = -dt*(Qimp[sdc_m][sdc_n]); - Ithree[sdc_m][mfi].saxpy(qij,f[2][sdc_n][mfi]); - } - } - } + { + res[sdc_m][mfi].setVal(0.0); + if (Npieces == 3) + Ithree[sdc_m][mfi].setVal(0.0); + for (int sdc_n = 0; sdc_n < Nnodes; sdc_n++) + { + qij = dt*(Qgauss[sdc_m][sdc_n]-Qexp[sdc_m][sdc_n]); + res[sdc_m][mfi].saxpy(qij,f[0][sdc_n][mfi]); // Explicit part + + qij = dt*(Qgauss[sdc_m][sdc_n]-Qimp[sdc_m][sdc_n]); + res[sdc_m][mfi].saxpy(qij,f[1][sdc_n][mfi]); // Implicit part + } + if (Npieces == 3) + { + for (int sdc_n = 0; sdc_n < Nnodes; sdc_n++) + { // // MISDC pieces + qij = dt*(Qgauss[sdc_m][sdc_n]); // leave off -dt*Qtil and add it later + res[sdc_m][mfi].saxpy(qij,f[2][sdc_n][mfi]); + // Compute seperate integral for f_3 piece + qij = -dt*(Qimp[sdc_m][sdc_n]); + Ithree[sdc_m][mfi].saxpy(qij,f[2][sdc_n][mfi]); + } + } + } } } @@ -97,38 +97,38 @@ void SDCstruct::SDC_rhs_k_plus_one(MultiFab& sol_new, Real dt,int sdc_m) { // Compute the rhs terms for the implicit solve Real qij; - + // Copy first the initial value MultiFab::Copy(sol_new,sol[0], 0, 0, 1, 0); for ( MFIter mfi(sol_new); mfi.isValid(); ++mfi ) { sol_new[mfi].saxpy(1.0,res[sdc_m][mfi]); for (int sdc_n = 0; sdc_n < sdc_m+1; sdc_n++) - { - qij = dt*Qexp[sdc_m][sdc_n]; - sol_new[mfi].saxpy(qij,f[0][sdc_n][mfi]); - } + { + qij = dt*Qexp[sdc_m][sdc_n]; + sol_new[mfi].saxpy(qij,f[0][sdc_n][mfi]); + } for (int sdc_n = 0; sdc_n < sdc_m+1; sdc_n++) - { - qij = dt*Qimp[sdc_m][sdc_n]; - sol_new[mfi].saxpy(qij,f[1][sdc_n][mfi]); - } + { + qij = dt*Qimp[sdc_m][sdc_n]; + sol_new[mfi].saxpy(qij,f[1][sdc_n][mfi]); + } } - + } void SDCstruct::SDC_rhs_misdc(MultiFab& sol_new, Real dt,int sdc_m) { // Add the terms to the rhs before the second implicit solve Real qij; - + for ( MFIter mfi(sol_new); mfi.isValid(); ++mfi ) { sol_new[mfi].saxpy(1.0,Ithree[sdc_m][mfi]); for (int sdc_n = 0; sdc_n < sdc_m+1; sdc_n++) - { - qij = dt*Qimp[sdc_m][sdc_n]; - sol_new[mfi].saxpy(qij,f[2][sdc_n][mfi]); - } + { + qij = dt*Qimp[sdc_m][sdc_n]; + sol_new[mfi].saxpy(qij,f[2][sdc_n][mfi]); + } } - + } diff --git a/Tests/AsyncOut/multifab/main.cpp b/Tests/AsyncOut/multifab/main.cpp index b7f724e0ac2..7c72af11706 100644 --- a/Tests/AsyncOut/multifab/main.cpp +++ b/Tests/AsyncOut/multifab/main.cpp @@ -42,7 +42,7 @@ void main_main () pp.query("nwork", nwork); pp.query("nwrites", nwrites); - // inputs hierarchy: + // inputs hierarchy: // n_cell > n_boxes_per_rank > n_cell_3d if (n_cell != 0) @@ -66,7 +66,7 @@ void main_main () Gpu::ManagedVector< Array4 > arrs(nwrites); for (int m = 0; m < nwrites; ++m) { - mfs[m].define(ba, dm, 1, 0); + mfs[m].define(ba, dm, 1, 0); } for (MFIter mfi(mfs[0]); mfi.isValid(); ++mfi) { @@ -98,7 +98,7 @@ void main_main () } amrex::Print() << "I/O printing randomly filled multifab with: " - << "\n dimensions = " << ba.minimalBox() + << "\n dimensions = " << ba.minimalBox() << "\n max_grid_size = " << max_grid_size << "\n boxes = " << ba.size() << "\n and nwork = " << nwork << std::endl; @@ -174,7 +174,7 @@ void main_main () // *************************************************************** - amrex::Print() << " AsyncOut " << std::endl; + amrex::Print() << " AsyncOut " << std::endl; { BL_PROFILE_REGION("vismf-async-overlap"); for (int m = 0; m < nwrites; ++m) { diff --git a/Tests/DivFreePatch/main.cpp b/Tests/DivFreePatch/main.cpp index febf2e8da0a..e2e904d6c65 100644 --- a/Tests/DivFreePatch/main.cpp +++ b/Tests/DivFreePatch/main.cpp @@ -110,7 +110,7 @@ Real MFdiff(const MultiFab& lhs, const MultiFab& rhs, max_diff = (max_diff > max_i) ? max_diff : max_i; } - return max_diff; + return max_diff; } // ================================================ @@ -130,7 +130,7 @@ void main_main () int n_cell = 0; int f_offset = 4; - int nghost_c = 1; + int nghost_c = 1; int nghost_f = 2; amrex::Vector c_lo(AMREX_SPACEDIM, 0); @@ -175,8 +175,8 @@ void main_main () // For outputing ghost cells for debugging. Array f_mf_faces_wg; - MultiFab div_fine_wg; - + MultiFab div_fine_wg; + AMREX_D_TERM( IntVect x_face{AMREX_D_DECL(1,0,0)};, IntVect y_face{AMREX_D_DECL(0,1,0)};, IntVect z_face{AMREX_D_DECL(0,0,1)}; ); @@ -375,7 +375,7 @@ void main_main () // Check for errors for (int i=0; i max_i) ? max_diff : max_i; } amrex::Print() << " Fine values maximum change: " << max_diff << std::endl; - // Check fine divergence = coarse divergence in ghost cells. + // Check fine divergence = coarse divergence in ghost cells. calcDiv(f_mf_faces, div_fine, f_geom.CellSizeArray()); amrex::VisMF::Write(div_fine, std::string("pltfiles/fineFP")); @@ -508,14 +508,14 @@ void main_main () // Check for errors for (int i=0; inorm1(); + if (ParallelDescriptor::IOProcessor()) { + std::cout << points[lev] << " points on level " << lev << std::endl; + } } int nrounds = 1000; { - ParmParse pp; - pp.query("nrounds", nrounds); + ParmParse pp; + pp.query("nrounds", nrounds); } Real err = 0.0; @@ -136,30 +136,30 @@ main (int argc, char* argv[]) auto wt0 = ParallelDescriptor::second(); for (int iround = 0; iround < nrounds; ++iround) { - for (int c=0; c<2; ++c) { - for (int lev = 0; lev < nlevels; ++lev) { - mfs[lev]->FillBoundary_nowait(); - mfs[lev]->FillBoundary_finish(); - } - for (int lev = nlevels-1; lev >= 0; --lev) { - mfs[lev]->FillBoundary_nowait(); - mfs[lev]->FillBoundary_finish(); - } - } - Real e = double(iround+ParallelDescriptor::MyProc()); - ParallelDescriptor::ReduceRealMax(e); - err += e; + for (int c=0; c<2; ++c) { + for (int lev = 0; lev < nlevels; ++lev) { + mfs[lev]->FillBoundary_nowait(); + mfs[lev]->FillBoundary_finish(); + } + for (int lev = nlevels-1; lev >= 0; --lev) { + mfs[lev]->FillBoundary_nowait(); + mfs[lev]->FillBoundary_finish(); + } + } + Real e = double(iround+ParallelDescriptor::MyProc()); + ParallelDescriptor::ReduceRealMax(e); + err += e; } - + ParallelDescriptor::Barrier(); auto wt1 = ParallelDescriptor::second(); if (ParallelDescriptor::IOProcessor()) { std::cout << "Using MPI" << std::endl; - std::cout << "----------------------------------------------" << std::endl; - std::cout << "Fill Boundary Time: " << wt1-wt0 << std::endl; - std::cout << "----------------------------------------------" << std::endl; - std::cout << "ignore this line " << err << std::endl; + std::cout << "----------------------------------------------" << std::endl; + std::cout << "Fill Boundary Time: " << wt1-wt0 << std::endl; + std::cout << "----------------------------------------------" << std::endl; + std::cout << "ignore this line " << err << std::endl; } // diff --git a/Tests/GPU/AnyOf/main.cpp b/Tests/GPU/AnyOf/main.cpp index 59201e1f125..2fb923b06c2 100644 --- a/Tests/GPU/AnyOf/main.cpp +++ b/Tests/GPU/AnyOf/main.cpp @@ -68,7 +68,7 @@ void main_main () return ( item > 2.0 ) ; }); amrex::Print() << "Vector, (0, 1): " - << anyof_M << ", " << anyof_N << std::endl; + << anyof_M << ", " << anyof_N << std::endl; } // Redo, confirming works for a single item. @@ -90,7 +90,7 @@ void main_main () }); amrex::Print() << "Vector, both true: " - << anyof_M << ", " << anyof_N << std::endl; + << anyof_M << ", " << anyof_N << std::endl; } amrex::Print() << "\nTesting Box Version.\n One box = 0,1,1,1, all others 0,1,0,1\n"; @@ -114,11 +114,11 @@ void main_main () return ( int(arr(i,j,k) > 2.0) ) ; }); - // Redo, confirming works for a single value. + // Redo, confirming works for a single value. if (bx.contains(IntVect{ncell/3,ncell/2,ncell-1})) { arr(ncell/3,ncell/2,ncell-1) = 1.0; - } + } bool anyof_C = Reduce::AnyOf(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept @@ -133,8 +133,8 @@ void main_main () }); amrex::Print() << "Box #" << mfi.LocalIndex() << " = " - << anyof_A << ", " << anyof_B << ", " - << anyof_C << ", " << anyof_D << std::endl; + << anyof_A << ", " << anyof_B << ", " + << anyof_C << ", " << anyof_D << std::endl; } } diff --git a/Tests/GPU/Fuse/main.cpp b/Tests/GPU/Fuse/main.cpp index fc90f1dc0f3..cd64e322d45 100644 --- a/Tests/GPU/Fuse/main.cpp +++ b/Tests/GPU/Fuse/main.cpp @@ -86,7 +86,7 @@ int main(int argc, char* argv[]) fused_test(mfa,mfb,mfc); } r = r && verify(mfc); - + { BL_PROFILE("fused-test2"); Real t = amrex::second(); diff --git a/Tests/GPU/Vector/main.cpp b/Tests/GPU/Vector/main.cpp index 46c2494dae8..dbd4fd82f9a 100644 --- a/Tests/GPU/Vector/main.cpp +++ b/Tests/GPU/Vector/main.cpp @@ -11,30 +11,30 @@ void test_container() Container v = {1, 2, 4}; v.insert(v.begin(), 0); v.insert(v.begin() + 3, 3); - v.insert(v.end(), 5); + v.insert(v.end(), 5); v.insert(v.begin(), {-2, -1}); - + Container v2; v2.push_back(-5); v2.push_back(-4); - v2.push_back(-3); + v2.push_back(-3); v.insert(v.begin(), v2.begin(), v2.end()); - + v.insert(v.begin(),1,-6); v.insert(v.end(), 2, 6); v.erase(v.end()-1, v.end()); - + v.pop_back(); v.erase(v.begin(), v.begin()+1); - + Container v3; v3.assign(v.begin(), v.end()); - + // v3 should now contain all integers, -5 to 5, inclusive for (int i=-5, index=0; i <= 5; ++i, ++index) { AMREX_ALWAYS_ASSERT(v3[index] == i); - } + } } int main (int argc, char* argv[]) diff --git a/Tests/HDF5Benchmark/main.cpp b/Tests/HDF5Benchmark/main.cpp index 69cd3759322..e0263d3d939 100644 --- a/Tests/HDF5Benchmark/main.cpp +++ b/Tests/HDF5Benchmark/main.cpp @@ -73,7 +73,7 @@ void test () Vector geom(nlevs); geom[0].define(domains[0], &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(domains[lev], &real_box, CoordSys::cartesian, is_per); + geom[lev].define(domains[lev], &real_box, CoordSys::cartesian, is_per); } Vector dmap(nlevs); @@ -278,7 +278,7 @@ void set_grids_nested (Vector& domains, } for (int lev = 1; lev < nlevs; lev++) { - domains[lev] = amrex::refine(domains[lev-1], ref_ratio[lev-1]); + domains[lev] = amrex::refine(domains[lev-1], ref_ratio[lev-1]); } } diff --git a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H index 721da35d7f7..baa438cd1fa 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H +++ b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.H @@ -33,7 +33,7 @@ private: int ref_ratio = 2; int n_cell = 128; int max_grid_size = 64; - + bool composite_solve = true; int prob_type = 1; // 1. Poisson, 2. ABecLaplacian diff --git a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp index 27c0c144506..bd015398d1d 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp +++ b/Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp @@ -84,9 +84,9 @@ MyTest::solvePoisson () for (int ilev = 0; ilev < nlevels; ++ilev) { MLPoisson mlpoisson({geom[ilev]}, {grids[ilev]}, {dmap[ilev]}, info); - + mlpoisson.setMaxOrder(linop_maxorder); - + // This is a 3d problem with Dirichlet BC mlpoisson.setDomainBC({AMREX_D_DECL(LinOpBCType::Dirichlet, LinOpBCType::Dirichlet, @@ -94,7 +94,7 @@ MyTest::solvePoisson () {AMREX_D_DECL(LinOpBCType::Dirichlet, LinOpBCType::Dirichlet, LinOpBCType::Dirichlet)}); - + if (ilev > 0) { mlpoisson.setCoarseFineBC(&solution[ilev-1], ref_ratio); } @@ -117,8 +117,8 @@ MyTest::solvePoisson () mlmg.setBottomSolver(MLMG::BottomSolver::petsc); } #endif - - mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); + + mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); } } } @@ -164,7 +164,7 @@ MyTest::solveABecLaplacian () for (int ilev = 0; ilev < nlevels; ++ilev) { mlabec.setACoeffs(ilev, acoef[ilev]); - + Array face_bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -201,9 +201,9 @@ MyTest::solveABecLaplacian () for (int ilev = 0; ilev < nlevels; ++ilev) { MLABecLaplacian mlabec({geom[ilev]}, {grids[ilev]}, {dmap[ilev]}, info); - + mlabec.setMaxOrder(linop_maxorder); - + // This is a 3d problem with homogeneous Neumann BC mlabec.setDomainBC({AMREX_D_DECL(LinOpBCType::Neumann, LinOpBCType::Neumann, @@ -211,7 +211,7 @@ MyTest::solveABecLaplacian () {AMREX_D_DECL(LinOpBCType::Neumann, LinOpBCType::Neumann, LinOpBCType::Neumann)}); - + if (ilev > 0) { mlabec.setCoarseFineBC(&solution[ilev-1], ref_ratio); } @@ -222,7 +222,7 @@ MyTest::solveABecLaplacian () mlabec.setScalars(ascalar, bscalar); mlabec.setACoeffs(0, acoef[ilev]); - + Array face_bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -251,7 +251,7 @@ MyTest::solveABecLaplacian () } #endif - mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); + mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); } } @@ -306,7 +306,7 @@ MyTest::solveABecLaplacianInhomNeumann () for (int ilev = 0; ilev < nlevels; ++ilev) { mlabec.setACoeffs(ilev, acoef[ilev]); - + Array face_bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -343,9 +343,9 @@ MyTest::solveABecLaplacianInhomNeumann () for (int ilev = 0; ilev < nlevels; ++ilev) { MLABecLaplacian mlabec({geom[ilev]}, {grids[ilev]}, {dmap[ilev]}, info); - + mlabec.setMaxOrder(linop_maxorder); - + // This is a 3d problem with inhomogeneous Neumann BC mlabec.setDomainBC({AMREX_D_DECL(LinOpBCType::inhomogNeumann, LinOpBCType::inhomogNeumann, @@ -353,7 +353,7 @@ MyTest::solveABecLaplacianInhomNeumann () {AMREX_D_DECL(LinOpBCType::inhomogNeumann, LinOpBCType::inhomogNeumann, LinOpBCType::inhomogNeumann)}); - + if (ilev > 0) { mlabec.setCoarseFineBC(&solution[ilev-1], ref_ratio); } @@ -364,7 +364,7 @@ MyTest::solveABecLaplacianInhomNeumann () mlabec.setScalars(ascalar, bscalar); mlabec.setACoeffs(0, acoef[ilev]); - + Array face_bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -393,7 +393,7 @@ MyTest::solveABecLaplacianInhomNeumann () } #endif - mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); + mlmg.solve({&solution[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); } } @@ -462,7 +462,7 @@ MyTest::initData () solution.resize(nlevels); rhs.resize(nlevels); exact_solution.resize(nlevels); - + if (prob_type == 2 || prob_type == 3) { acoef.resize(nlevels); bcoef.resize(nlevels); @@ -485,7 +485,7 @@ MyTest::initData () grids[ilev].define(domain); grids[ilev].maxSize(max_grid_size); domain.grow(-n_cell/4); // fine level cover the middle of the coarse domain - domain.refine(ref_ratio); + domain.refine(ref_ratio); } for (int ilev = 0; ilev < nlevels; ++ilev) diff --git a/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp b/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp index e33c2dbac4e..a7b197adbed 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp +++ b/Tests/LinearSolvers/ABecLaplacian_C/initProb.cpp @@ -129,7 +129,7 @@ MyTest::initProbABecLaplacianInhomNeumann () actual_init_dphi_dx_hi(i,j,k,solnfab,prob_lo,dx); }); } - + if (bx.smallEnd(1) == domain.smallEnd(1)) { Box const& bylo = amrex::adjCellLo(bx, 1); amrex::ParallelFor(bylo, diff --git a/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H b/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H index b0cf8adbed2..b7743273b2d 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H +++ b/Tests/LinearSolvers/ABecLaplacian_C/initProb_K.H @@ -19,7 +19,7 @@ void actual_init_poisson (int i, int j, int k, exact(i,j,k) = (std::sin(tpi*x) * std::sin(tpi*y) * std::sin(tpi*z)) + .25 * (std::sin(fpi*x) * std::sin(fpi*y) * std::sin(fpi*z)); - + rhs(i,j,k) = -fac * (std::sin(tpi*x) * std::sin(tpi*y) * std::sin(tpi*z)) -fac * (std::sin(fpi*x) * std::sin(fpi*y) * std::sin(fpi*z)); } @@ -79,7 +79,7 @@ void actual_init_abeclap (int i, int j, int k, amrex::Real tmp = std::cosh(theta*(r-0.25)); amrex::Real dbdrfac = (sigma-1.)/2./(tmp*tmp) * theta/r; dbdrfac *= b; - + alpha(i,j,k) = 1.; exact(i,j,k) = std::cos(tpi*x) * std::cos(tpi*y) * std::cos(tpi*z) @@ -129,7 +129,7 @@ void actual_init_abeclap_in (int i, int j, int k, amrex::Real tmp = std::cosh(theta*(r-0.25)); amrex::Real dbdrfac = (sigma-1.)/2./(tmp*tmp) * theta/r; dbdrfac *= b; - + alpha(i,j,k) = 1.; exact(i,j,k) = std::sin(tpi*x) * std::cos(tpi*y) * std::cos(tpi*z) @@ -248,5 +248,5 @@ void actual_init_dphi_dz_hi (int i, int j, int k, amrex::Array4 con fz(i,j,k) = std::sin(tpi*x) * std::cos(tpi*y) * (-tpi) * std::sin(tpi*z) + .25 * std::cos(fpi*x) * std::sin(fpi*y) * (-fpi) * std::sin(fpi*z); } - + #endif diff --git a/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-16.sh b/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-16.sh index 5d7bb9201b4..ab3f820b28e 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-16.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-16.sh @@ -5,7 +5,7 @@ #BSUB -J amrex #BSUB -o amrexo.%J #BSUB -e amrexe.%J -module load gcc +module load gcc module load cuda module list set -x diff --git a/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-32.sh b/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-32.sh index 26b7de2f825..0022e1b5a02 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-32.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-32.sh @@ -5,7 +5,7 @@ #BSUB -J amrex #BSUB -o amrexo.%J #BSUB -e amrexe.%J -module load gcc +module load gcc module load cuda module list set -x diff --git a/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-8.sh b/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-8.sh index 54eb21e34c9..4c79fc41235 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-8.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/scalingtest/run-8.sh @@ -5,7 +5,7 @@ #BSUB -J amrex #BSUB -o amrexo.%J #BSUB -e amrexe.%J -module load gcc +module load gcc module load cuda module list set -x diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-1.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-1.sh index b0b6ded650f..3d32c9e6235 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-1.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-1.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 1 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 1-MPI.out @@ -15,4 +15,4 @@ NCELLS=256 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 64 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 64 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-16.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-16.sh index 4974551d142..89865a4bd02 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-16.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-16.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 16 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 16-MPI.out @@ -15,4 +15,4 @@ NCELLS=1024 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 1024 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 1024 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-2.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-2.sh index ad1a4b26d14..ac71c77679b 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-2.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-2.sh @@ -1,7 +1,7 @@ #!/bin/bash -#SBATCH -N 2 +#SBATCH -N 2 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 2-MPI.out @@ -15,4 +15,4 @@ NCELLS=512 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 128 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 128 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-32.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-32.sh index b2951306c18..4c5edfae284 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-32.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-32.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 32 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 32-MPI.out @@ -15,4 +15,4 @@ NCELLS=1024 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 2048 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 2048 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-4.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-4.sh index 3dd0d046973..4f0693a922a 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-4.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-4.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 4 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 4-MPI.out @@ -15,4 +15,4 @@ NCELLS=512 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 256 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 256 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-64.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-64.sh index 52672389c27..eede591b515 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-64.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-64.sh @@ -1,7 +1,7 @@ #!/bin/bash -#SBATCH -N 64 +#SBATCH -N 64 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 64-MPI.out @@ -15,4 +15,4 @@ NCELLS=1024 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 4096 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 4096 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-8.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-8.sh index d88dbf33eb6..66aea91d9bf 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-8.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-mpi-8.sh @@ -1,7 +1,7 @@ #!/bin/bash -#SBATCH -N 8 +#SBATCH -N 8 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 8-MPI.out @@ -15,4 +15,4 @@ NCELLS=512 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 512 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} +srun -n 512 -c 4 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-1.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-1.sh index addf072b12b..31c40068a3f 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-1.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-1.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 1 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 1-OMP.out @@ -15,4 +15,4 @@ NCELLS=256 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 8 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 8 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-16.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-16.sh index 7a5ae76722f..5bcf7daf896 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-16.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-16.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 16 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 16-OMP.out @@ -15,4 +15,4 @@ NCELLS=1024 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 128 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 128 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-2.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-2.sh index 139af6d41ef..516a7422332 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-2.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-2.sh @@ -1,7 +1,7 @@ #!/bin/bash -#SBATCH -N 2 +#SBATCH -N 2 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 2-OMP.out @@ -15,4 +15,4 @@ NCELLS=512 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 16 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 16 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-32.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-32.sh index bfcc1443694..0fe5b8c93d9 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-32.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-32.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 32 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 32-OMP.out @@ -15,4 +15,4 @@ NCELLS=1024 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 256 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 256 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-4.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-4.sh index 20c062abbf1..18673675536 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-4.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-4.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH -N 4 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 4-OMP.out @@ -15,4 +15,4 @@ NCELLS=512 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 32 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 32 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-64.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-64.sh index 2496dc09ddd..90a3bf88a95 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-64.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-64.sh @@ -1,7 +1,7 @@ #!/bin/bash -#SBATCH -N 64 +#SBATCH -N 64 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 64-OMP.out @@ -15,4 +15,4 @@ NCELLS=1024 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 512 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 512 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-8.sh b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-8.sh index f2ea3dc1e24..2586d557572 100644 --- a/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-8.sh +++ b/Tests/LinearSolvers/ABecLaplacian_C/threadmultiple_test/knl-omp-8.sh @@ -1,7 +1,7 @@ #!/bin/bash -#SBATCH -N 8 +#SBATCH -N 8 #SBATCH -C knl -#SBATCH -q debug +#SBATCH -q debug #SBATCH -t 00:10:00 #SBATCH -o 8-OMP.out @@ -15,4 +15,4 @@ NCELLS=512 export MPICH_MAX_THREAD_SAFETY=multiple #run the application: -srun -n 64 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} +srun -n 64 -c 32 --cpu_bind=cores main3d.gnu.mic-knl.TPROF.MPI.OMP.ex inputs.test n_cell=${NCELLS} diff --git a/Tests/LinearSolvers/ABecLaplacian_F/init_prob.F90 b/Tests/LinearSolvers/ABecLaplacian_F/init_prob.F90 index 841585c5c23..7c0d670373e 100644 --- a/Tests/LinearSolvers/ABecLaplacian_F/init_prob.F90 +++ b/Tests/LinearSolvers/ABecLaplacian_F/init_prob.F90 @@ -25,7 +25,7 @@ subroutine init_prob_poisson (geom, solution, rhs, exact_solution) do ilev = 0, size(rhs)-1 !$omp parallel private(rlo,rhi,elo,ehi,bx,mfi,prhs,pexact) call amrex_mfiter_build(mfi, rhs(ilev), tiling=.true.) - + do while (mfi%next()) bx = mfi%tilebox() prhs => rhs(ilev) % dataptr(mfi) @@ -69,7 +69,7 @@ subroutine init_prob_abeclaplacian (geom, solution, rhs, exact_solution, acoef, do ilev = 0, size(rhs)-1 !$omp parallel private(rlo,rhi,elo,ehi,alo,ahi,blo,bhi,bx,gbx,mfi,prhs,pexact,pa,pb) call amrex_mfiter_build(mfi, rhs(ilev), tiling=.true.) - + do while (mfi%next()) bx = mfi%tilebox() gbx = mfi%growntilebox(1) @@ -118,10 +118,10 @@ subroutine actual_init_poisson (lo, hi, rhs, rlo, rhi, exact, elo, ehi, prob_lo, y = prob_lo(2) + dx(2) * (dble(j)+0.5d0) do i = lo(1), hi(1) x = prob_lo(1) + dx(1) * (dble(i)+0.5d0) - + exact(i,j,k) = 1.d0 * (sin(tpi*x) * sin(tpi*y) * sin(tpi*z)) & & + .25d0 * (sin(fpi*x) * sin(fpi*y) * sin(fpi*z)) - + rhs(i,j,k) = -fac * (sin(tpi*x) * sin(tpi*y) * sin(tpi*z)) & & -fac * (sin(fpi*x) * sin(fpi*y) * sin(fpi*z)) end do @@ -157,33 +157,33 @@ subroutine actual_init_abeclapcian (lo, hi, glo, ghi, rhs, rlo, rhi, exact, elo, zc = (prob_hi(3) + prob_lo(3))/2.d0 theta = 0.5d0*log(3.d0) / (w + 1.d-50) - + do k = glo(3), ghi(3) z = prob_lo(3) + dx(3) * (dble(k)+0.5d0) do j = glo(2), ghi(2) y = prob_lo(2) + dx(2) * (dble(j)+0.5d0) do i = glo(1), ghi(1) x = prob_lo(1) + dx(1) * (dble(i)+0.5d0) - + r = sqrt((x-xc)**2 + (y-yc)**2 + (z-zc)**2) - + beta(i,j,k) = (sigma-1.d0)/2.d0*tanh(theta*(r-0.25d0)) + (sigma+1.d0)/2.d0 end do end do end do - + do k = lo(3), hi(3) z = prob_lo(3) + dx(3) * (dble(k)+0.5d0) do j = lo(2), hi(2) y = prob_lo(2) + dx(2) * (dble(j)+0.5d0) do i = lo(1), hi(1) x = prob_lo(1) + dx(1) * (dble(i)+0.5d0) - + r = sqrt((x-xc)**2 + (y-yc)**2 + (z-zc)**2) - + dbdrfac = (sigma-1.d0)/2.d0/(cosh(theta*(r-0.25d0)))**2 * theta/r dbdrfac = dbdrfac * b - + alpha(i,j,k) = 1.d0 exact(i,j,k) = 1.d0 * cos(tpi*x) * cos(tpi*y) * cos(tpi*z) & diff --git a/Tests/LinearSolvers/ABecLaplacian_F/mytest.F90 b/Tests/LinearSolvers/ABecLaplacian_F/mytest.F90 index 9c52632f1b5..be3613d66cd 100644 --- a/Tests/LinearSolvers/ABecLaplacian_F/mytest.F90 +++ b/Tests/LinearSolvers/ABecLaplacian_F/mytest.F90 @@ -40,7 +40,7 @@ module mytest_module type(amrex_multifab), allocatable, save :: acoef(:) type(amrex_multifab), allocatable, save :: bcoef(:) real(amrex_real), save :: ascalar, bscalar - + private public :: init, finalize, solve, write_plotfile @@ -187,7 +187,7 @@ subroutine solve_poisson () call amrex_poisson_build(poisson, geom, ba, dm, & metric_term=.false., agglomeration=agglomeration, consolidation=consolidation, & max_coarsening_level=max_coarsening_level) - + call poisson % set_maxorder(linop_maxorder) ! This is a 3d problem with Dirichlet BC @@ -217,7 +217,7 @@ subroutine solve_poisson () call amrex_poisson_build(poisson, [geom(ilev)], [ba(ilev)], [dm(ilev)], & metric_term=.false., agglomeration=agglomeration, consolidation=consolidation, & max_coarsening_level=max_coarsening_level) - + call poisson % set_maxorder(linop_maxorder) ! The order of the following set bc calls matters. @@ -225,7 +225,7 @@ subroutine solve_poisson () ! This is a 3d problem with Dirichlet BC call poisson % set_domain_bc([amrex_lo_dirichlet, amrex_lo_dirichlet, amrex_lo_dirichlet], & & [amrex_lo_dirichlet, amrex_lo_dirichlet, amrex_lo_dirichlet]) - + if (ilev > 0) then ! use coarse level data to set up bc at corase/fine boundary call poisson % set_coarse_fine_bc(solution(ilev-1), ref_ratio) @@ -315,7 +315,7 @@ subroutine solve_abeclaplacian () else do ilev = 0, max_level - + call amrex_abeclaplacian_build(abeclap, [geom(ilev)], [ba(ilev)], [dm(ilev)], & metric_term=.false., agglomeration=agglomeration, consolidation=consolidation, & max_coarsening_level=max_coarsening_level) diff --git a/Tests/LinearSolvers/CellEB/MyTest.H b/Tests/LinearSolvers/CellEB/MyTest.H index 99363140d6c..4af902ddec3 100644 --- a/Tests/LinearSolvers/CellEB/MyTest.H +++ b/Tests/LinearSolvers/CellEB/MyTest.H @@ -27,7 +27,7 @@ private: int max_grid_size = 64; int is_periodic = 0; int eb_is_dirichlet = 0; - + std::string plot_file_name{"plot"}; amrex::Vector scalars; diff --git a/Tests/LinearSolvers/CellEB/MyTest.cpp b/Tests/LinearSolvers/CellEB/MyTest.cpp index 0d93b77a197..3909592aa47 100644 --- a/Tests/LinearSolvers/CellEB/MyTest.cpp +++ b/Tests/LinearSolvers/CellEB/MyTest.cpp @@ -79,7 +79,7 @@ MyTest::solve () mlmg.setVerbose(verbose); mlmg.setBottomVerbose(bottom_verbose); if (use_hypre) mlmg.setBottomSolver(MLMG::BottomSolver::hypre); - if (use_petsc) mlmg.setBottomSolver(MLMG::BottomSolver::petsc); + if (use_petsc) mlmg.setBottomSolver(MLMG::BottomSolver::petsc); const Real tol_rel = reltol; const Real tol_abs = 0.0; mlmg.solve(amrex::GetVecOfPtrs(phi), amrex::GetVecOfConstPtrs(rhs), tol_rel, tol_abs); @@ -93,14 +93,14 @@ MyTest::writePlotfile () const MultiFab& vfrc = factory[ilev]->getVolFrac(); plotmf[ilev].define(grids[ilev],dmap[ilev],2,0); MultiFab::Copy(plotmf[ilev], phi[ilev], 0, 0, 1, 0); - MultiFab::Copy(plotmf[ilev], vfrc, 0, 1, 1, 0); + MultiFab::Copy(plotmf[ilev], vfrc, 0, 1, 1, 0); } WriteMultiLevelPlotfile(plot_file_name, max_level+1, amrex::GetVecOfConstPtrs(plotmf), {"phi","vfrac"}, geom, 0.0, Vector(max_level+1,0), Vector(max_level,IntVect{2})); - + } void @@ -138,7 +138,7 @@ MyTest::readParameters () pp.query("use_hypre", use_hypre); #endif #ifdef AMREX_USE_PETSC - pp.query("use_petsc",use_petsc); + pp.query("use_petsc",use_petsc); #endif } @@ -166,7 +166,7 @@ MyTest::initGrids () grids[ilev].define(domain); grids[ilev].maxSize(max_grid_size); domain.grow(-n_cell/4); // fine level cover the middle of the coarse domain - domain.refine(ref_ratio); + domain.refine(ref_ratio); } } diff --git a/Tests/LinearSolvers/CellEB/initEB.cpp b/Tests/LinearSolvers/CellEB/initEB.cpp index bd3677e57d7..af062dcc945 100644 --- a/Tests/LinearSolvers/CellEB/initEB.cpp +++ b/Tests/LinearSolvers/CellEB/initEB.cpp @@ -32,7 +32,7 @@ MyTest::initializeEB () EB2::translate(box, {AMREX_D_DECL(-0.5,-0.5,-0.5)}), std::atan(1.0)*0.3, 2), {AMREX_D_DECL(0.5,0.5,0.5)})); - EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); + EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); } else if (geom_type == "two_spheres") { diff --git a/Tests/LinearSolvers/CellEB2/MyTest.H b/Tests/LinearSolvers/CellEB2/MyTest.H index a00ae992d09..c91c3d5cc11 100644 --- a/Tests/LinearSolvers/CellEB2/MyTest.H +++ b/Tests/LinearSolvers/CellEB2/MyTest.H @@ -33,7 +33,7 @@ private: int prob_type = 2; std::string plot_file_name{"plot"}; - + amrex::Vector scalars; // For MLMG solver diff --git a/Tests/LinearSolvers/CellEB2/MyTest.cpp b/Tests/LinearSolvers/CellEB2/MyTest.cpp index f27464387c8..a9c7799e242 100644 --- a/Tests/LinearSolvers/CellEB2/MyTest.cpp +++ b/Tests/LinearSolvers/CellEB2/MyTest.cpp @@ -69,20 +69,20 @@ MyTest::solve () MLEBABecLap mleb (geom, grids, dmap, info, amrex::GetVecOfConstPtrs(factory)); mleb.setMaxOrder(linop_maxorder); - + mleb.setDomainBC(mlmg_lobc, mlmg_hibc); for (int ilev = 0; ilev <= max_level; ++ilev) { mleb.setLevelBC(ilev, &phi[ilev]); } - + mleb.setScalars(scalars[0], scalars[1]); - + for (int ilev = 0; ilev <= max_level; ++ilev) { mleb.setACoeffs(ilev, acoef[ilev]); mleb.setBCoeffs(ilev, amrex::GetArrOfConstPtrs(bcoef[ilev])); } - + if (true) { // In this test we assume EB is Dirichlet. for (int ilev = 0; ilev <= max_level; ++ilev) { mleb.setEBDirichlet(ilev, phieb[ilev], bcoef_eb[ilev]); @@ -101,7 +101,7 @@ MyTest::solve () } else if (use_petsc) { mlmg.setBottomSolver(MLMG::BottomSolver::petsc); } - + const Real tol_rel = reltol; const Real tol_abs = 0.0; mlmg.solve(amrex::GetVecOfPtrs(phi), amrex::GetVecOfConstPtrs(rhs), tol_rel, tol_abs); @@ -143,7 +143,7 @@ MyTest::solve () } else if (use_petsc) { mlmg.setBottomSolver(MLMG::BottomSolver::petsc); } - + const Real tol_rel = reltol; const Real tol_abs = 0.0; mlmg.solve({&phi[ilev]}, {&rhs[ilev]}, tol_rel, tol_abs); @@ -165,7 +165,7 @@ MyTest::solve () Real norm1 = mf.norm1()*AMREX_D_TERM((1.0/n_cell), *(1.0/n_cell), *(1.0/n_cell)); amrex::Print() << "Level " << ilev << ": weighted max and 1 norms " << norminf << ", " << norm1 << std::endl; } - } + } } void @@ -214,7 +214,7 @@ MyTest::writePlotfile () geom, 0.0, Vector(max_level+1,0), Vector(max_level,IntVect{2})); } - + } void diff --git a/Tests/LinearSolvers/CellEB2/initEB.cpp b/Tests/LinearSolvers/CellEB2/initEB.cpp index e06da3d3608..2944db55553 100644 --- a/Tests/LinearSolvers/CellEB2/initEB.cpp +++ b/Tests/LinearSolvers/CellEB2/initEB.cpp @@ -28,7 +28,7 @@ MyTest::initializeEB () EB2::translate(box, {AMREX_D_DECL(-0.5,-0.5,-0.5)}), std::atan(1.0)*0.3, 2), {AMREX_D_DECL(0.5,0.5,0.5)})); - EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); + EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); } else if (geom_type == "flower") { @@ -48,7 +48,7 @@ MyTest::initializeEB () EB2::SphereIF sphere(0.5, {0.5,0.5,0.5}, false); EB2::BoxIF cube({0.1,0.1,0.1}, {0.9,0.9,0.9}, false); auto cubesphere = EB2::makeIntersection(sphere, cube); - + EB2::CylinderIF cylinder_x(0.25, 0, {0.5,0.5,0.5}, false); EB2::CylinderIF cylinder_y(0.25, 1, {0.5,0.5,0.5}, false); EB2::CylinderIF cylinder_z(0.25, 2, {0.5,0.5,0.5}, false); diff --git a/Tests/LinearSolvers/EBConvergenceTest/BC_2D.F90 b/Tests/LinearSolvers/EBConvergenceTest/BC_2D.F90 index 45d14e414ed..602594105c9 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/BC_2D.F90 +++ b/Tests/LinearSolvers/EBConvergenceTest/BC_2D.F90 @@ -10,7 +10,7 @@ subroutine apply_bc(lo, hi, phi, phlo, phhi, & integer, dimension(2), intent(in) :: lo, hi, phlo, phhi real(amrex_real), dimension(2), intent(in) :: dx, problo, probhi real(amrex_real), intent(inout) :: phi(phlo(1):phhi(1), phlo(2):phhi(2)) - + real(amrex_real) :: xl, xh, x, yl, yh, y, denom integer :: i, j @@ -25,7 +25,7 @@ subroutine apply_bc(lo, hi, phi, phlo, phhi, & denom = sqrt((xh-0.5d0)*(xh-0.5d0) + (y-0.5d0)*(y-0.5d0)) phi(hi(1),j) = (xh-0.5d0)/denom end do - + do i = lo(1)+1, hi(1)-1 x = xl + (dble(i) + 0.5d0)*dx(1) denom = sqrt((x-0.5d0)*(x-0.5d0) + (yl-0.5d0)*(yl-0.5d0)) diff --git a/Tests/LinearSolvers/EBConvergenceTest/BC_3D.F90 b/Tests/LinearSolvers/EBConvergenceTest/BC_3D.F90 index 6bb185432c7..acee765145c 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/BC_3D.F90 +++ b/Tests/LinearSolvers/EBConvergenceTest/BC_3D.F90 @@ -12,16 +12,16 @@ subroutine apply_bc(lo, hi, phi, phlo, phhi, & integer, dimension(3), intent(in) :: lo, hi, phlo, phhi real(amrex_real), dimension(3), intent(in) :: dx, problo, probhi real(amrex_real), intent(inout) :: phi(phlo(1):phhi(1), phlo(2):phhi(2), phlo(3):phhi(3)) - + real(amrex_real) :: xl, xh, x, yl, yh, y, zl, zh, z, denom integer :: i, j, k - xl = problo(1) - 0.5d0 + xl = problo(1) - 0.5d0 xh = probhi(1) - 0.5d0 yl = problo(2) - 0.5d0 yh = probhi(2) - 0.5d0 zl = problo(3) - 0.5d0 - zh = probhi(3) - 0.5d0 + zh = probhi(3) - 0.5d0 do k = lo(3)+1, hi(3)-1 z = zl + (dble(k) + 0.5d0)*dx(3) @@ -37,24 +37,24 @@ subroutine apply_bc(lo, hi, phi, phlo, phhi, & phi(i,lo(2),k) = f3(x,yl,z) phi(i,hi(2),k) = f3(x,yh,z) end do - !x-y edges + !x-y edges phi(lo(1),lo(2),k) = f3(xl,yl,z) phi(hi(1),hi(2),k) = f3(xh,yh,z) phi(lo(1),hi(2),k) = f3(xl,yh,z) phi(hi(1),lo(2),k) = f3(xh,yl,z) enddo - - !z-face + + !z-face do j = lo(2)+1, hi(2)-1 y = yl + (dble(j) + 0.5d0)*dx(2) do i = lo(1)+1,hi(2)-1 - x = xl +(dble(i) +0.5d0)*dx(1) + x = xl +(dble(i) +0.5d0)*dx(1) phi(i,j,lo(3)) = f3(x,y,zl) phi(i,j,hi(3)) = f3(x,y,zh) enddo !xz edge phi(lo(1),j,lo(3)) = f3(xl,y,zl) - phi(lo(1),j,hi(3)) = f3(xl,y,zh) + phi(lo(1),j,hi(3)) = f3(xl,y,zh) phi(hi(1),j,hi(3)) = f3(xh,y,zh) phi(hi(1),j,lo(3)) = f3(xh,y,zl) enddo @@ -63,28 +63,28 @@ subroutine apply_bc(lo, hi, phi, phlo, phhi, & do i = lo(1)+1,hi(1)-1 x = xl +(dble(i) +0.5d0)*dx(1) phi(i,lo(2),lo(3)) = f3(x,yl,zl) - phi(i,lo(2),hi(3)) = f3(x,yl,zh) + phi(i,lo(2),hi(3)) = f3(x,yl,zh) phi(i,hi(2),hi(3)) = f3(x,yh,zh) phi(i,hi(2),lo(3)) = f3(x,yh,zl) enddo - !Corners! + !Corners! phi(lo(1),lo(2),lo(3)) = f3(xl,yl,zl) - phi(lo(1),lo(2),hi(3)) = f3(xl,yl,zh) + phi(lo(1),lo(2),hi(3)) = f3(xl,yl,zh) phi(lo(1),hi(2),hi(3)) = f3(xl,yh,zh) phi(lo(1),hi(2),lo(3)) = f3(xl,yh,zl) phi(hi(1),lo(2),lo(3)) = f3(xh,yl,zl) - phi(hi(1),lo(2),hi(3)) = f3(xh,yl,zh) + phi(hi(1),lo(2),hi(3)) = f3(xh,yl,zh) phi(hi(1),hi(2),hi(3)) = f3(xh,yh,zh) phi(hi(1),hi(2),lo(3)) = f3(xh,yh,zl) end subroutine apply_bc - + function f3(x,y,z) real(amrex_real), intent(in) :: x, y, z real(amrex_real) :: f3 - + f3 = (x+z)/sqrt(x*x + y*y + z*z) - end function f3 + end function f3 end module bc diff --git a/Tests/LinearSolvers/EBConvergenceTest/Convergence_Data_Gen.sh b/Tests/LinearSolvers/EBConvergenceTest/Convergence_Data_Gen.sh index 0eced3fadb1..547c1340867 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/Convergence_Data_Gen.sh +++ b/Tests/LinearSolvers/EBConvergenceTest/Convergence_Data_Gen.sh @@ -3,18 +3,18 @@ # Center for Computational Science and Engineering # Lawrence Berkeley National Laboratory -# Runs the EB Elliptic Test Solver for n_cell = 32 to 1024 for 2D -# and n_cell = 16 to 256 for 3D. -# Then converts the multifabs into a .mat file +# Runs the EB Elliptic Test Solver for n_cell = 32 to 1024 for 2D +# and n_cell = 16 to 256 for 3D. +# Then converts the multifabs into a .mat file # After all processing it moves the mat files into the Results folder. -# Within the Results folder there is an octave file for plotting the results +# Within the Results folder there is an octave file for plotting the results -# Make sure you build the Multifab to matlab tools and +# Make sure you build the Multifab to matlab tools and # export PATH="$PATH:/../../../Tools/Postprocessing/C_Src" if [ $1 -eq 2 ]; then -echo "2D Test Commencing!" +echo "2D Test Commencing!" ./main2d.gnu.TEST.MPI.ex inputs n_cell=32 max_grid_size=32 mv phi-0_H phi-0_32_H MultiFabToMatLab2d.gnu.MPI.ex infile=phi-0_32 @@ -38,8 +38,8 @@ MultiFabToMatLab2d.gnu.MPI.ex infile=phi-0_512 ./main2d.gnu.TEST.MPI.ex inputs n_cell=1024 max_grid_size=1024 mv phi-0_H phi-0_1024_H MultiFabToMatLab2d.gnu.MPI.ex infile=phi-0_1024 -elif [ $1 -eq 3 ]; -then +elif [ $1 -eq 3 ]; +then echo "Commencing 3D Test!" ./main3d.gnu.TEST.MPI.ex inputs n_cell=16 max_grid_size=16 mv phi-0_H phi-0_16_H @@ -60,6 +60,6 @@ MultiFabToMatLab3d.gnu.MPI.ex infile=phi-0_128 ./main3d.gnu.TEST.MPI.ex inputs n_cell=256 max_grid_size=256 mv phi-0_H phi-0_256_H MultiFabToMatLab3d.gnu.MPI.ex infile=phi-0_256 -fi +fi mv *.mat Results/ rm phi-0* vfrc-0* diff --git a/Tests/LinearSolvers/EBConvergenceTest/MyTest.H b/Tests/LinearSolvers/EBConvergenceTest/MyTest.H index b86e7be17ad..856eeaf2a9a 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/MyTest.H +++ b/Tests/LinearSolvers/EBConvergenceTest/MyTest.H @@ -24,7 +24,7 @@ private: int ref_ratio = 2; int n_cell = 128; int max_grid_size = 64; - + // For MLMG solver int verbose = 2; int bottom_verbose = 2; diff --git a/Tests/LinearSolvers/EBConvergenceTest/MyTest.cpp b/Tests/LinearSolvers/EBConvergenceTest/MyTest.cpp index 310ed396714..e420f6a9572 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/MyTest.cpp +++ b/Tests/LinearSolvers/EBConvergenceTest/MyTest.cpp @@ -1,6 +1,6 @@ #include "MyTest.H" #include "MyTest_F.H" - + #include #include #include @@ -120,7 +120,7 @@ MyTest::initGrids () grids[ilev].define(domain); grids[ilev].maxSize(max_grid_size); domain.grow(-n_cell/4); // fine level cover the middle of the coarse domain - domain.refine(ref_ratio); + domain.refine(ref_ratio); } } @@ -163,65 +163,65 @@ MyTest::initData () #if (AMREX_SPACEDIM == 2) for (MFIter mfi(rhs[ilev]); mfi.isValid(); ++mfi) { - FArrayBox& rhsfab = rhs[ilev][mfi]; - FArrayBox& afab = acoef[ilev][mfi]; - FArrayBox& bfabx = bcoef[ilev][0][mfi]; + FArrayBox& rhsfab = rhs[ilev][mfi]; + FArrayBox& afab = acoef[ilev][mfi]; + FArrayBox& bfabx = bcoef[ilev][0][mfi]; FArrayBox& bfaby = bcoef[ilev][1][mfi]; const Box& bx = rhsfab.box(); - build_a_2d(bx.loVect(), bx.hiVect(), - geom[ilev].ProbLo(), - geom[ilev].ProbHi(), - BL_TO_FORTRAN_ANYD(afab), - dx); - - build_b_2d(bx.loVect(), bx.hiVect(), - geom[ilev].ProbLo(), - geom[ilev].ProbHi(), + build_a_2d(bx.loVect(), bx.hiVect(), + geom[ilev].ProbLo(), + geom[ilev].ProbHi(), + BL_TO_FORTRAN_ANYD(afab), + dx); + + build_b_2d(bx.loVect(), bx.hiVect(), + geom[ilev].ProbLo(), + geom[ilev].ProbHi(), BL_TO_FORTRAN_ANYD( bfabx), BL_TO_FORTRAN_ANYD( bfaby), dx); // */ - - build_rhs_2d(bx.loVect(), bx.hiVect(), - BL_TO_FORTRAN_ANYD(rhsfab), - BL_TO_FORTRAN_ANYD( afab), + + build_rhs_2d(bx.loVect(), bx.hiVect(), + BL_TO_FORTRAN_ANYD(rhsfab), + BL_TO_FORTRAN_ANYD( afab), BL_TO_FORTRAN_ANYD( bfabx), BL_TO_FORTRAN_ANYD( bfaby), - dx, geom[ilev].ProbLo(), - geom[ilev].ProbHi()); + dx, geom[ilev].ProbLo(), + geom[ilev].ProbHi()); } #elif (AMREX_SPACEDIM ==3) for (MFIter mfi(rhs[ilev]); mfi.isValid(); ++mfi) { - FArrayBox& rhsfab = rhs[ilev][mfi]; - FArrayBox& afab = acoef[ilev][mfi]; - FArrayBox& bfabx = bcoef[ilev][0][mfi]; + FArrayBox& rhsfab = rhs[ilev][mfi]; + FArrayBox& afab = acoef[ilev][mfi]; + FArrayBox& bfabx = bcoef[ilev][0][mfi]; FArrayBox& bfaby = bcoef[ilev][1][mfi]; FArrayBox& bfabz = bcoef[ilev][2][mfi]; const Box& bx = rhsfab.box(); - build_a_3d(bx.loVect(), bx.hiVect(), - geom[ilev].ProbLo(), - geom[ilev].ProbHi(), - BL_TO_FORTRAN_ANYD(afab), - dx); - - build_b_3d(bx.loVect(), bx.hiVect(), - geom[ilev].ProbLo(), - geom[ilev].ProbHi(), + build_a_3d(bx.loVect(), bx.hiVect(), + geom[ilev].ProbLo(), + geom[ilev].ProbHi(), + BL_TO_FORTRAN_ANYD(afab), + dx); + + build_b_3d(bx.loVect(), bx.hiVect(), + geom[ilev].ProbLo(), + geom[ilev].ProbHi(), BL_TO_FORTRAN_ANYD( bfabx), BL_TO_FORTRAN_ANYD( bfaby), BL_TO_FORTRAN_ANYD( bfabz), dx); // */ - - build_rhs_3d(bx.loVect(), bx.hiVect(), - BL_TO_FORTRAN_ANYD(rhsfab), - BL_TO_FORTRAN_ANYD( afab), + + build_rhs_3d(bx.loVect(), bx.hiVect(), + BL_TO_FORTRAN_ANYD(rhsfab), + BL_TO_FORTRAN_ANYD( afab), BL_TO_FORTRAN_ANYD( bfabx), BL_TO_FORTRAN_ANYD( bfaby), - BL_TO_FORTRAN_ANYD( bfabz), - dx, geom[ilev].ProbLo(), - geom[ilev].ProbHi()); + BL_TO_FORTRAN_ANYD( bfabz), + dx, geom[ilev].ProbLo(), + geom[ilev].ProbHi()); } #endif @@ -230,10 +230,10 @@ MyTest::initData () { FArrayBox& fab = phi[ilev][mfi]; const Box& bx = fab.box(); - apply_bc(bx.loVect(), bx.hiVect(), + apply_bc(bx.loVect(), bx.hiVect(), BL_TO_FORTRAN_ANYD(fab), - dx, geom[ilev].ProbLo(), - geom[ilev].ProbHi());// */ + dx, geom[ilev].ProbLo(), + geom[ilev].ProbHi());// */ } phi[ilev].setVal(0.0, 0, 1, 0); diff --git a/Tests/LinearSolvers/EBConvergenceTest/MyTest_F.H b/Tests/LinearSolvers/EBConvergenceTest/MyTest_F.H index 11f3c7a75df..ff47d315bae 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/MyTest_F.H +++ b/Tests/LinearSolvers/EBConvergenceTest/MyTest_F.H @@ -5,60 +5,60 @@ #ifdef __cplusplus extern "C" { -#endif +#endif #if AMREX_SPACEDIM == 2 - void build_rhs_2d(const int* lo, const int* hi, - amrex_real* rhs, const int* rlo, const int* rhi, - const amrex_real* a, const int* alo, const int* ahi, - const amrex_real* bx, const int* bxlo, const int* bxhi, - const amrex_real* by, const int* bylo, const int* byhi, - const amrex_real* dx, const amrex_real* prob_lo, + void build_rhs_2d(const int* lo, const int* hi, + amrex_real* rhs, const int* rlo, const int* rhi, + const amrex_real* a, const int* alo, const int* ahi, + const amrex_real* bx, const int* bxlo, const int* bxhi, + const amrex_real* by, const int* bylo, const int* byhi, + const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi); void build_a_2d(const int* lo, const int* hi, - const amrex_real* prob_lo, + const amrex_real* prob_lo, const amrex_real* prob_hi, - amrex_real* a, const int* alo, const int* ahi, + amrex_real* a, const int* alo, const int* ahi, const amrex_real* dx); void build_b_2d(const int* lo, const int* hi, - const amrex_real* prob_lo, + const amrex_real* prob_lo, const amrex_real* prob_hi, - amrex_real* bx, const int* bxlo, const int* bxhi, - amrex_real* by, const int* bylo, const int* byhi, + amrex_real* bx, const int* bxlo, const int* bxhi, + amrex_real* by, const int* bylo, const int* byhi, const amrex_real* dx); #elif AMREX_SPACEDIM ==3 - void build_rhs_3d(const int* lo, const int* hi, - amrex_real* rhs, const int* rlo, const int* rhi, - const amrex_real* a, const int* alo, const int* ahi, - const amrex_real* bx, const int* bxlo, const int* bxhi, + void build_rhs_3d(const int* lo, const int* hi, + amrex_real* rhs, const int* rlo, const int* rhi, + const amrex_real* a, const int* alo, const int* ahi, + const amrex_real* bx, const int* bxlo, const int* bxhi, const amrex_real* by, const int* bylo, const int* byhi, - const amrex_real* bz, const int* bzlo, const int* bzhi, - const amrex_real* dx, const amrex_real* prob_lo, + const amrex_real* bz, const int* bzlo, const int* bzhi, + const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi); void build_a_3d(const int* lo, const int* hi, - const amrex_real* prob_lo, + const amrex_real* prob_lo, const amrex_real* prob_hi, - amrex_real* a, const int* alo, const int* ahi, + amrex_real* a, const int* alo, const int* ahi, const amrex_real* dx); void build_b_3d(const int* lo, const int* hi, - const amrex_real* prob_lo, + const amrex_real* prob_lo, const amrex_real* prob_hi, - amrex_real* bx, const int* bxlo, const int* bxhi, - amrex_real* by, const int* bylo, const int* byhi, - amrex_real* bz, const int* bzlo, const int* bzhi, + amrex_real* bx, const int* bxlo, const int* bxhi, + amrex_real* by, const int* bylo, const int* byhi, + amrex_real* bz, const int* bzlo, const int* bzhi, const amrex_real* dx); #endif - void apply_bc(const int* lo, const int* hi, - amrex_real* phi, const int* phlo, const int* phhi, - const amrex_real* dx, const amrex_real* prob_lo, - const amrex_real* prob_hi); + void apply_bc(const int* lo, const int* hi, + amrex_real* phi, const int* phlo, const int* phhi, + const amrex_real* dx, const amrex_real* prob_lo, + const amrex_real* prob_hi); #ifdef __cplusplus } #endif -#endif +#endif diff --git a/Tests/LinearSolvers/EBConvergenceTest/RHS.F90 b/Tests/LinearSolvers/EBConvergenceTest/RHS.F90 index 65d81edaea3..318c2b93c8c 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/RHS.F90 +++ b/Tests/LinearSolvers/EBConvergenceTest/RHS.F90 @@ -1,9 +1,9 @@ module rhs use amrex_fort_module, only : amrex_real - + private :: RHS3, dbdx, dbdy, dbdz public :: build_rhs_2D, build_rhs_3D - interface dbdx + interface dbdx module procedure dbdx2 module procedure dbdx3 end interface dbdx @@ -11,15 +11,15 @@ module rhs module procedure dbdy2 module procedure dbdy3 end interface dbdy - interface dbdz + interface dbdz module procedure dbdz3 end interface dbdz -contains +contains subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & a , alo, ahi, & bx , bxlo, bxhi, & - by , bylo, byhi, & + by , bylo, byhi, & dx, problo, probhi) & bind(c, name='build_rhs_2d') implicit none @@ -29,16 +29,16 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & real(amrex_real), intent(in ) :: a ( alo(1): ahi(1), alo(2): ahi(2)) real(amrex_real), intent(in ) :: bx (bxlo(1):bxhi(1), bxlo(2):bxhi(2)) real(amrex_real), intent(in ) :: by (bylo(1):byhi(1), bylo(2):byhi(2)) - + real(amrex_real) :: dxb, dyb, denom, x, y, dyinv, dxinv, term1, term2, b, xl, xh, yl, yh - integer :: i, j - + integer :: i, j + dxinv = 1.d0/dx(1) dyinv = 1.d0/dx(2) yl = problo(2) xl = problo(1) yh = probhi(2) - xh = probhi(1) + xh = probhi(1) do j = lo(2)+1, hi(2)-1 y = yl + (dble(j) + 0.5d0)*dx(2) -0.5d0 do i = lo(1)+1, hi(1)-1 @@ -46,10 +46,10 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & dyb = (by(i,j+1) - by(i,j))*dyinv b = 0.25d0*(bx(i+1,j) + bx(i,j) + by(i,j+1) + by(i,j)) dxb = (bx(i+1,j) - bx(i,j))*dxinv - x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5d0; + x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5d0; ! dxb = dbdx(x,y) ! dyb = dbdy(x,y) - denom = sqrt(x*x + y*y) + denom = sqrt(x*x + y*y) term1 = a(i,j)*x/denom term2 = (y*(y*dxb -x*dyb) - x*b)/(denom**3) rhs(i,j) = term1 - term2 @@ -58,16 +58,16 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & b = 0.25d0*(bx(lo(1)+1,j) + bx(lo(1),j) + by(lo(1),j+1) + by(lo(1),j)) dyb = (by(lo(1), j+1) - by(lo(1),j))*(dyinv) x = xl - 0.5d0 - dxb = (bx(lo(1)+1,j)-bx(lo(1),j))*(dxinv) + dxb = (bx(lo(1)+1,j)-bx(lo(1),j))*(dxinv) ! dxb = dbdx(x,y) ! dyb = dbdy(x,y) denom = sqrt(x*x + y*y) term1 = a(lo(1),j)*x/denom term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) - rhs(lo(1),j) = term1 - term2 + rhs(lo(1),j) = term1 - term2 dyb = (by(hi(1), j+1) - by(hi(1), j))*dyinv - x = xh - 0.5d0 + x = xh - 0.5d0 b = 0.25d0*(bx(hi(1)+1,j) + bx(hi(1),j) + by(hi(1),j+1) + by(hi(1),j)) dxb = (bx(hi(1)+1, j) - bx(hi(1),j))*(dxinv*0.5) ! dxb = dbdx(x,y) @@ -75,14 +75,14 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & denom = sqrt((x)**2 + y*y) term1 = a(hi(1),j)*x/denom term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) - rhs(hi(1),j) = term1 - term2 - enddo + rhs(hi(1),j) = term1 - term2 + enddo !Boundary case ylo, yhi, x inside do i = lo(1)+1, hi(1)-1 - x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5d0; - - y = yl - 0.5d0 + x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5d0; + + y = yl - 0.5d0 b = 0.25d0*(bx(i+1,lo(2)) + bx(i,lo(2)) + by(i,lo(2)+1) + by(i,lo(2))) dyb = (by(i, lo(2)+1) - by(i,lo(2)))*dyinv dxb = (bx(i+1, lo(2)) - bx(i,lo(2)))*dxinv @@ -91,7 +91,7 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & denom = sqrt(x**2 + y**2) term1 = a(i,lo(2))*x/denom term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) - rhs(i,lo(2)) = term1 - term2 + rhs(i,lo(2)) = term1 - term2 y = yh -0.5d0 b = 0.25d0*(bx(i+1,hi(2)) + bx(i,hi(2)) + by(i,hi(2)+1) + by(i,hi(2))) @@ -102,21 +102,21 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & denom = sqrt(x**2 + y**2) term1 = a(i,hi(2))*x/denom term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) - rhs(i,hi(2)) = term1 - term2 - enddo + rhs(i,hi(2)) = term1 - term2 + enddo !Corner cases - !xlo ylo + !xlo ylo b = 0.25d0*(bx(lo(1)+1,lo(2)) + bx(lo(1),lo(2)) + by(lo(1),lo(2)+1) + by(lo(1),lo(2))) x = xl - 0.5d0 - y = yl - 0.5d0 + y = yl - 0.5d0 dxb = (bx(lo(1)+1, lo(2)) - bx(lo(1), lo(2)))*dxinv dyb = (by(lo(1), lo(2)+1) - by(lo(1), lo(2)))*dyinv denom = sqrt(x*x + y*y) term1 = a(lo(1),lo(2))*x/denom term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) - rhs(lo(1), lo(2)) = term1 - term2 - + rhs(lo(1), lo(2)) = term1 - term2 + !xlo yhi b = 0.25d0*(bx(lo(1)+1,hi(2)) + bx(lo(1),hi(2)) + by(lo(1),hi(2)+1) + by(lo(1),hi(2))) y = yh -0.5d0 @@ -127,7 +127,7 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) rhs(lo(1), hi(2)) = term1 - term2 - !xhi yhi + !xhi yhi b = 0.25d0*(bx(hi(1)+1,hi(2)) + bx(hi(1),hi(2)) + by(hi(1),hi(2)+1) + by(hi(1),hi(2))) x = xh -0.5d0 dxb = (bx(hi(1)+1, hi(2)) - bx(hi(1), hi(2)))*dxinv @@ -137,7 +137,7 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) rhs(hi(1), hi(2)) = term1 - term2 - !xhi ylo + !xhi ylo b = 0.25d0*(bx(hi(1)+1,lo(2)) + bx(hi(1),lo(2)) + by(hi(1),lo(2)+1) + by(hi(1),lo(2))) y = yl -0.5d0 dxb = (bx(hi(1)+1, lo(2)) - bx(hi(1), lo(2)))*dxinv @@ -146,7 +146,7 @@ subroutine build_rhs_2D(lo, hi, rhs, rlo, rhi, & term1 = a(hi(1), lo(2))*x/denom term2 = (y*(y*dxb - x*dyb) - x*b)/(denom**3) rhs(hi(1), lo(2)) = term1 - term2 - + end subroutine build_rhs_2D subroutine build_rhs_3D(lo, hi, rhs, rlo, rhi, & @@ -157,18 +157,18 @@ subroutine build_rhs_3D(lo, hi, rhs, rlo, rhi, & dx, problo, probhi) & bind(c, name='build_rhs_3d') implicit none - integer, dimension(3), intent(in) :: lo, hi, rlo, rhi, alo, ahi, bxlo, bxhi, bylo, byhi, bzlo, bzhi + integer, dimension(3), intent(in) :: lo, hi, rlo, rhi, alo, ahi, bxlo, bxhi, bylo, byhi, bzlo, bzhi real(amrex_real), dimension(3), intent(in) :: dx, problo, probhi real(amrex_real), intent(inout) :: rhs( rlo(1): rhi(1), rlo(2): rhi(2), rlo(3): rhi(3)) real(amrex_real), intent(in ) :: a ( alo(1): ahi(1), alo(2): ahi(2), alo(3): ahi(3)) real(amrex_real), intent(in ) :: bx (bxlo(1):bxhi(1),bxlo(2):bxhi(2),bxlo(3):bxhi(3)) real(amrex_real), intent(in ) :: by (bylo(1):byhi(1),bylo(2):byhi(2),bylo(3):byhi(3)) real(amrex_real), intent(in ) :: bz (bzlo(1):bzhi(1),bzlo(2):bzhi(2),bzlo(3):bzhi(3)) - + real(amrex_real) :: dxb, dyb, dzb, denom, x, y, z, dxinv, dyinv, dzinv real(amrex_real) :: term1, term2, b, xl, xh, yl, yh, zl, zh integer :: i, j, k - + dxinv = 1.d0/dx(1) dyinv = 1.d0/dx(2) dzinv = 1.d0/dx(3) @@ -177,97 +177,97 @@ subroutine build_rhs_3D(lo, hi, rhs, rlo, rhi, & xl = problo(1) zh = probhi(3) yh = probhi(2) - xh = probhi(1) - + xh = probhi(1) + do k = lo(3)+1, hi(3)-1 z = zl + (dble(k) + 0.5d0)*dx(3) - 0.5 ! Shifted to center of sphere <0.5 0.5 0.5> do j = lo(2)+1, hi(2)-1 y = yl + (dble(j) + 0.5d0)*dx(2) - 0.5 do i = lo(1)+1, hi(1)-1 !Interior b = 1.d0/6.d0*(bx(i+1,j,k) + bx(i,j,k) + by(i,j+1,k) + by(i,j,k) + bz(i,j,k+1) + bz(i,j,k)) !Average to CC - x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5 + x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5 dzb = (bz(i,j,k+1) - bz(i,j,k))*dzinv dyb = (by(i,j+1,k) - by(i,j,k))*dyinv dxb = (bx(i+1,j,k) - bx(i,j,k))*dxinv rhs(i,j,k) = RHS3(a(i,j,k), b, x, y, z, dxb, dyb, dzb) enddo - !Xlo faces + !Xlo faces x = xl - 0.5d0 b = bx(lo(1), j, k) - dxb = (bx(lo(1)+1,j,k) - b)*dxinv + dxb = (bx(lo(1)+1,j,k) - b)*dxinv dyb = (by(lo(1),j+1,k) - by(lo(1),j,k))*dyinv - dzb = (bz(lo(1),j,k+1) - bz(lo(1),j,k))*dzinv + dzb = (bz(lo(1),j,k+1) - bz(lo(1),j,k))*dzinv rhs(lo(1), j, k) = RHS3(a(lo(1),j,k), b, x, y, z, dxb, dyb, dzb) - !Xhi faces + !Xhi faces x = xh - 0.5d0 b = (bx(hi(1),j,k) + bx(hi(1)+1,j,k))*0.5d0 - dxb = (bx(hi(1)+1,j,k) - b)*dxinv + dxb = (bx(hi(1)+1,j,k) - b)*dxinv dyb = (by(hi(1),j+1,k) - by(hi(1),j,k))*dyinv - dzb = (bz(hi(1),j,k+1) - bz(hi(1),j,k))*dzinv + dzb = (bz(hi(1),j,k+1) - bz(hi(1),j,k))*dzinv rhs(hi(1), j, k) = RHS3(a(hi(1),j,k), b, x, y, z, dxb, dyb, dzb) enddo do i = lo(1)+1, hi(1)-1 x = xl + (dble(i) + 0.5d0)*dx(1) - 0.5d0 !Ylo Faces - y = yl - 0.5d0 + y = yl - 0.5d0 b = by(i,lo(2),k) dxb = (bx(i+1,lo(2),k) - bx(i,lo(2),k))*dxinv dyb = (by(i,lo(2)+1,k) - by(i,lo(2),k))*dyinv dzb = (bz(i,lo(2),k+1) - bz(i,lo(2),k))*dzinv - rhs(i,lo(2),k) = RHS3(a(i,lo(2),k), b, x, y, z, dxb, dyb, dzb) + rhs(i,lo(2),k) = RHS3(a(i,lo(2),k), b, x, y, z, dxb, dyb, dzb) !Yhi Faces - y = yh - 0.5d0 + y = yh - 0.5d0 b = (by(i,hi(2),k) + by(i,hi(2)+1,k))*0.5d0 dxb = (bx(i+1,hi(2),k) - bx(i,hi(2),k))*dxinv dyb = (by(i,hi(2)+1,k) - by(i,hi(2),k))*dyinv dzb = (bz(i,hi(2),k+1) - bz(i,hi(2),k))*dzinv - rhs(i,hi(2),k) = RHS3(a(i,hi(2),k), b, x, y, z, dxb, dyb, dzb) + rhs(i,hi(2),k) = RHS3(a(i,hi(2),k), b, x, y, z, dxb, dyb, dzb) enddo !xlo ylo edge y = yl - 0.5d0 x = xl - 0.5d0 b = 0.5d0*(bx(lo(1), lo(2), k) + by(lo(1),lo(2),k)) - dxb = (bx(lo(1)+1,lo(2),k) - bx(lo(1),lo(2),k))*dxinv + dxb = (bx(lo(1)+1,lo(2),k) - bx(lo(1),lo(2),k))*dxinv dyb = (by(lo(1),lo(2)+1,k) - by(lo(1),lo(2),k))*dyinv - dzb = (bz(lo(1),lo(2),k+1) - bz(lo(1),lo(2),k))*dzinv + dzb = (bz(lo(1),lo(2),k+1) - bz(lo(1),lo(2),k))*dzinv rhs(lo(1), lo(2), k) = RHS3(a(lo(1),lo(2),k), b, x, y, z, dxb, dyb, dzb) !xhi ylo edge - x = xh - 0.5d0 + x = xh - 0.5d0 b = 0.5d0*(bx(hi(1), lo(2), k) + by(hi(1),lo(2),k)) - dxb = (bx(hi(1)+1,lo(2),k) - bx(hi(1),lo(2),k))*dxinv + dxb = (bx(hi(1)+1,lo(2),k) - bx(hi(1),lo(2),k))*dxinv dyb = (by(hi(1),lo(2)+1,k) - by(hi(1),lo(2),k))*dyinv - dzb = (bz(hi(1),lo(2),k+1) - bz(hi(1),lo(2),k))*dzinv + dzb = (bz(hi(1),lo(2),k+1) - bz(hi(1),lo(2),k))*dzinv rhs(hi(1), lo(2), k) = RHS3(a(hi(1),lo(2),k), b, x, y, z, dxb, dyb, dzb) - - !xhi yhi edge + + !xhi yhi edge y = yh - 0.5d0 b = 0.5d0*(bx(hi(1), hi(2), k) + by(hi(1),hi(2),k)) - dxb = (bx(hi(1)+1,hi(2),k) - bx(hi(1),hi(2),k))*dxinv + dxb = (bx(hi(1)+1,hi(2),k) - bx(hi(1),hi(2),k))*dxinv dyb = (by(hi(1),hi(2)+1,k) - by(hi(1),hi(2),k))*dyinv - dzb = (bz(hi(1),hi(2),k+1) - bz(hi(1),hi(2),k))*dzinv + dzb = (bz(hi(1),hi(2),k+1) - bz(hi(1),hi(2),k))*dzinv rhs(hi(1), hi(2), k) = RHS3(a(hi(1),hi(2),k), b, x, y, z, dxb, dyb, dzb) - !xlo yhi edge + !xlo yhi edge x = xl -0.5d0 b = 0.5d0*(bx(lo(1), hi(2), k) + by(lo(1),hi(2),k)) - dxb = (bx(lo(1)+1,hi(2),k) - bx(lo(1),hi(2),k))*dxinv + dxb = (bx(lo(1)+1,hi(2),k) - bx(lo(1),hi(2),k))*dxinv dyb = (by(lo(1),hi(2)+1,k) - by(lo(1),hi(2),k))*dyinv - dzb = (bz(lo(1),hi(2),k+1) - bz(lo(1),hi(2),k))*dzinv + dzb = (bz(lo(1),hi(2),k+1) - bz(lo(1),hi(2),k))*dzinv rhs(lo(1), hi(2), k) = RHS3(a(lo(1),hi(2),k), b, x, y, z, dxb, dyb, dzb) - enddo + enddo + - do j = lo(2)+1,hi(2)-1 y = yl + (dble(j) + 0.5d0)*dx(2) - 0.5d0 do i = lo(1)+1,hi(1)-1 x = xl + (dble(i) + 0.5d0)*dx(1) -0.5d0 - !zlo face + !zlo face z = zl - 0.5d0 b = bz(i,j,lo(3)) dxb = (bx(i+1,j,lo(3))-bx(i,j,lo(3)))*dxinv @@ -275,9 +275,9 @@ subroutine build_rhs_3D(lo, hi, rhs, rlo, rhi, & dzb = (bz(i,j,lo(3)+1)-b)*dzinv rhs(i,j,lo(3)) = RHS3(a(i,j,lo(3)),b,x,y,z,dxb, dyb, dzb) - !zhi face + !zhi face z = zh - 0.5d0 - b = (bz(i,j,hi(3)) + bz(i,j,hi(3)+1))*0.5d0 + b = (bz(i,j,hi(3)) + bz(i,j,hi(3)+1))*0.5d0 dxb = (bx(i+1,j,hi(3))-bx(i,j,hi(3)))*dxinv dyb = (by(i,j+1,hi(3))-by(i,j,hi(3)))*dyinv dzb = (bz(i,j,hi(3)+1)-b)*dzinv @@ -287,171 +287,171 @@ subroutine build_rhs_3D(lo, hi, rhs, rlo, rhi, & z = zl - 0.5d0 x = xl - 0.5d0 b = 0.5d0*(bx(lo(1), j, lo(3)) + bz(lo(1),j,lo(3))) - dxb = (bx(lo(1)+1,j,lo(3)) - bx(lo(1),j,lo(3)))*dxinv + dxb = (bx(lo(1)+1,j,lo(3)) - bx(lo(1),j,lo(3)))*dxinv dyb = (by(lo(1),j+1,lo(3)) - by(lo(1),j,lo(3)))*dyinv - dzb = (bz(lo(1),j,lo(3)+1) - bz(lo(1),j,lo(3)))*dzinv + dzb = (bz(lo(1),j,lo(3)+1) - bz(lo(1),j,lo(3)))*dzinv rhs(lo(1), j, lo(3)) = RHS3(a(lo(1),j,lo(3)), b, x, y, z, dxb, dyb, dzb) !zlo xhi edge - x = xh - 0.5d0 + x = xh - 0.5d0 b = 0.5d0*(bx(hi(1), j, lo(3)) + bz(hi(1),j,lo(3))) - dxb = (bx(hi(1)+1,j,lo(3)) - bx(hi(1),j,lo(3)))*dxinv + dxb = (bx(hi(1)+1,j,lo(3)) - bx(hi(1),j,lo(3)))*dxinv dyb = (by(hi(1),j+1,lo(3)) - by(hi(1),j,lo(3)))*dyinv - dzb = (bz(hi(1),j,lo(3)+1) - bz(hi(1),j,lo(3)))*dzinv + dzb = (bz(hi(1),j,lo(3)+1) - bz(hi(1),j,lo(3)))*dzinv rhs(hi(1), j, lo(3)) = RHS3(a(hi(1),j,lo(3)), b, x, y, z, dxb, dyb, dzb) - - !zhi xhi edge + + !zhi xhi edge z = zh - 0.5d0 b = 0.5d0*(bx(hi(1), j, hi(3)) + bz(hi(1),j,hi(3))) - dxb = (bx(hi(1)+1,j,hi(3)) - bx(hi(1),j,hi(3)))*dxinv + dxb = (bx(hi(1)+1,j,hi(3)) - bx(hi(1),j,hi(3)))*dxinv dyb = (by(hi(1),j+1,hi(3)) - by(hi(1),j,hi(3)))*dyinv - dzb = (bz(hi(1),j,hi(3)+1) - bz(hi(1),j,hi(3)))*dzinv + dzb = (bz(hi(1),j,hi(3)+1) - bz(hi(1),j,hi(3)))*dzinv rhs(hi(1), j, hi(3)) = RHS3(a(hi(1),j,hi(3)), b, x, y, z, dxb, dyb, dzb) - !zhi xlo edge + !zhi xlo edge x = xl -0.5d0 b = 0.5d0*(bx(lo(1), j, hi(3)) + bz(lo(1),j,hi(3))) - dxb = (bx(lo(1)+1,j,hi(3)) - bx(lo(1),j,hi(3)))*dxinv + dxb = (bx(lo(1)+1,j,hi(3)) - bx(lo(1),j,hi(3)))*dxinv dyb = (by(lo(1),j+1,hi(3)) - by(lo(1),j,hi(3)))*dyinv - dzb = (bz(lo(1),j,hi(3)+1) - bz(lo(1),j,hi(3)))*dzinv + dzb = (bz(lo(1),j,hi(3)+1) - bz(lo(1),j,hi(3)))*dzinv rhs(lo(1), j, hi(3)) = RHS3(a(lo(1),j,hi(3)), b, x, y, z, dxb, dyb, dzb) - enddo - - do i = lo(1)+1,hi(1)-1 + enddo + + do i = lo(1)+1,hi(1)-1 !zlo ylo edge z = zl - 0.5d0 y = yl - 0.5d0 b = 0.5d0*(by(i, lo(2), lo(3)) + bz(i,lo(2),lo(3))) - dxb = (bx(i+1,lo(2),lo(3)) - bx(i,lo(2),lo(3)))*dxinv + dxb = (bx(i+1,lo(2),lo(3)) - bx(i,lo(2),lo(3)))*dxinv dyb = (by(i,lo(2)+1,lo(3)) - by(i,lo(2),lo(3)))*dyinv - dzb = (bz(i,lo(2),lo(3)+1) - bz(i,lo(2),lo(3)))*dzinv + dzb = (bz(i,lo(2),lo(3)+1) - bz(i,lo(2),lo(3)))*dzinv rhs(i, lo(2), lo(3)) = RHS3(a(i,lo(2),lo(3)), b, x, y, z, dxb, dyb, dzb) !zlo yhi edge - y = yh - 0.5d0 + y = yh - 0.5d0 b = 0.5d0*(by(i, hi(2), lo(3)) + bz(i,hi(2),lo(3))) - dxb = (bx(i+1,hi(2),lo(3)) - bx(i,hi(2),lo(3)))*dxinv + dxb = (bx(i+1,hi(2),lo(3)) - bx(i,hi(2),lo(3)))*dxinv dyb = (by(i,hi(2)+1,lo(3)) - by(i,hi(2),lo(3)))*dyinv - dzb = (bz(i,hi(2),lo(3)+1) - bz(i,hi(2),lo(3)))*dzinv + dzb = (bz(i,hi(2),lo(3)+1) - bz(i,hi(2),lo(3)))*dzinv rhs(i, hi(2), lo(3)) = RHS3(a(i,hi(2),lo(3)), b, x, y, z, dxb, dyb, dzb) - - !zhi yhi edge + + !zhi yhi edge z = zh - 0.5d0 b = 0.5d0*(by(i, hi(2), hi(3)) + bz(i,hi(2),hi(3))) - dxb = (bx(i+1,hi(2),hi(3)) - bx(i,hi(2),hi(3)))*dxinv + dxb = (bx(i+1,hi(2),hi(3)) - bx(i,hi(2),hi(3)))*dxinv dyb = (by(i,hi(2)+1,hi(3)) - by(i,hi(2),hi(3)))*dyinv - dzb = (bz(i,hi(2),hi(3)+1) - bz(i,hi(2),hi(3)))*dzinv + dzb = (bz(i,hi(2),hi(3)+1) - bz(i,hi(2),hi(3)))*dzinv rhs(i, hi(2), hi(3)) = RHS3(a(i,hi(2),hi(3)), b, x, y, z, dxb, dyb, dzb) - !zhi ylo edge + !zhi ylo edge y = yl -0.5d0 b = 0.5d0*(by(i, lo(2), hi(3)) + bz(i,lo(2),hi(3))) - dxb = (bx(i+1,lo(2),hi(3)) - bx(i,lo(2),hi(3)))*dxinv + dxb = (bx(i+1,lo(2),hi(3)) - bx(i,lo(2),hi(3)))*dxinv dyb = (by(i,lo(2)+1,hi(3)) - by(i,lo(2),hi(3)))*dyinv - dzb = (bz(i,lo(2),hi(3)+1) - bz(i,lo(2),hi(3)))*dzinv + dzb = (bz(i,lo(2),hi(3)+1) - bz(i,lo(2),hi(3)))*dzinv rhs(i, lo(2), hi(3)) = RHS3(a(i,lo(2),hi(3)), b, x, y, z, dxb, dyb, dzb) - enddo + enddo - !Corners - !xlo ylo zlo + !Corners + !xlo ylo zlo z = zl - 0.5d0 y = yl - 0.5d0 - x = xl - 0.5d0 + x = xl - 0.5d0 b = 1.d0/3.d0*(bx(lo(1),lo(2),lo(3)) + by(lo(1), lo(2), lo(3)) + bz(lo(1),lo(2),lo(3))) - dxb = (bx(lo(1)+1,lo(2),lo(3)) - bx(lo(1),lo(2),lo(3)))*dxinv + dxb = (bx(lo(1)+1,lo(2),lo(3)) - bx(lo(1),lo(2),lo(3)))*dxinv dyb = (by(lo(1),lo(2)+1,lo(3)) - by(lo(1),lo(2),lo(3)))*dyinv - dzb = (bz(lo(1),lo(2),lo(3)+1) - bz(lo(1),lo(2),lo(3)))*dzinv + dzb = (bz(lo(1),lo(2),lo(3)+1) - bz(lo(1),lo(2),lo(3)))*dzinv rhs(lo(1), lo(2), lo(3)) = RHS3(a(lo(1),lo(2),lo(3)), b, x, y, z, dxb, dyb, dzb) - !xlo ylo zhi + !xlo ylo zhi z = zh - 0.5d0 b = 1.d0/3.d0*(bx(lo(1),lo(2),hi(3)) + by(lo(1), lo(2), hi(3)) + bz(lo(1),lo(2),hi(3))) - dxb = (bx(lo(1)+1,lo(2),hi(3)) - bx(lo(1),lo(2),hi(3)))*dxinv + dxb = (bx(lo(1)+1,lo(2),hi(3)) - bx(lo(1),lo(2),hi(3)))*dxinv dyb = (by(lo(1),lo(2)+1,hi(3)) - by(lo(1),lo(2),hi(3)))*dyinv - dzb = (bz(lo(1),lo(2),hi(3)+1) - bz(lo(1),lo(2),hi(3)))*dzinv + dzb = (bz(lo(1),lo(2),hi(3)+1) - bz(lo(1),lo(2),hi(3)))*dzinv rhs(lo(1), lo(2), hi(3)) = RHS3(a(lo(1),lo(2),hi(3)), b, x, y, z, dxb, dyb, dzb) - !xlo yhi zlo + !xlo yhi zlo z = zl - 0.5d0 y = yh - 0.5d0 b = 1.d0/3.d0*(bx(lo(1),hi(2),lo(3)) + by(lo(1), hi(2), lo(3)) + bz(lo(1),hi(2),lo(3))) - dxb = (bx(lo(1)+1,hi(2),lo(3)) - bx(lo(1),hi(2),lo(3)))*dxinv + dxb = (bx(lo(1)+1,hi(2),lo(3)) - bx(lo(1),hi(2),lo(3)))*dxinv dyb = (by(lo(1),hi(2)+1,lo(3)) - by(lo(1),hi(2),lo(3)))*dyinv - dzb = (bz(lo(1),hi(2),lo(3)+1) - bz(lo(1),hi(2),lo(3)))*dzinv + dzb = (bz(lo(1),hi(2),lo(3)+1) - bz(lo(1),hi(2),lo(3)))*dzinv rhs(lo(1), hi(2), lo(3)) = RHS3(a(lo(1),hi(2),lo(3)), b, x, y, z, dxb, dyb, dzb) - !xlo yhi zhi + !xlo yhi zhi z = zh - 0.5d0 b = 1.d0/3.d0*(bx(lo(1),hi(2),hi(3)) + by(lo(1), hi(2), hi(3)) + bz(lo(1),hi(2),hi(3))) - dxb = (bx(lo(1)+1,hi(2),hi(3)) - bx(lo(1),hi(2),hi(3)))*dxinv + dxb = (bx(lo(1)+1,hi(2),hi(3)) - bx(lo(1),hi(2),hi(3)))*dxinv dyb = (by(lo(1),hi(2)+1,hi(3)) - by(lo(1),hi(2),hi(3)))*dyinv - dzb = (bz(lo(1),hi(2),hi(3)+1) - bz(lo(1),hi(2),hi(3)))*dzinv + dzb = (bz(lo(1),hi(2),hi(3)+1) - bz(lo(1),hi(2),hi(3)))*dzinv rhs(lo(1), hi(2), hi(3)) = RHS3(a(lo(1),hi(2),hi(3)), b, x, y, z, dxb, dyb, dzb) - !xhi ylo zlo + !xhi ylo zlo z = zl - 0.5d0 y = yl - 0.5d0 - x = xh - 0.5d0 + x = xh - 0.5d0 b = 1.d0/3.d0*(bx(hi(1),lo(2),lo(3)) + by(hi(1), lo(2), lo(3)) + bz(hi(1),lo(2),lo(3))) - dxb = (bx(hi(1)+1,lo(2),lo(3)) - bx(hi(1),lo(2),lo(3)))*dxinv + dxb = (bx(hi(1)+1,lo(2),lo(3)) - bx(hi(1),lo(2),lo(3)))*dxinv dyb = (by(hi(1),lo(2)+1,lo(3)) - by(hi(1),lo(2),lo(3)))*dyinv - dzb = (bz(hi(1),lo(2),lo(3)+1) - bz(hi(1),lo(2),lo(3)))*dzinv + dzb = (bz(hi(1),lo(2),lo(3)+1) - bz(hi(1),lo(2),lo(3)))*dzinv rhs(hi(1), lo(2), lo(3)) = RHS3(a(hi(1),lo(2),lo(3)), b, x, y, z, dxb, dyb, dzb) - !xhi ylo zhi + !xhi ylo zhi z = zh - 0.5d0 b = 1.d0/3.d0*(bx(hi(1),lo(2),hi(3)) + by(hi(1), lo(2), hi(3)) + bz(hi(1),lo(2),hi(3))) - dxb = (bx(hi(1)+1,lo(2),hi(3)) - bx(hi(1),lo(2),hi(3)))*dxinv + dxb = (bx(hi(1)+1,lo(2),hi(3)) - bx(hi(1),lo(2),hi(3)))*dxinv dyb = (by(hi(1),lo(2)+1,hi(3)) - by(hi(1),lo(2),hi(3)))*dyinv - dzb = (bz(hi(1),lo(2),hi(3)+1) - bz(hi(1),lo(2),hi(3)))*dzinv + dzb = (bz(hi(1),lo(2),hi(3)+1) - bz(hi(1),lo(2),hi(3)))*dzinv rhs(hi(1), lo(2), hi(3)) = RHS3(a(hi(1),lo(2),hi(3)), b, x, y, z, dxb, dyb, dzb) - !xhi yhi zlo + !xhi yhi zlo z = zl - 0.5d0 y = yh - 0.5d0 b = 1.d0/3.d0*(bx(hi(1),hi(2),lo(3)) + by(hi(1), hi(2), lo(3)) + bz(hi(1),hi(2),lo(3))) - dxb = (bx(hi(1)+1,hi(2),lo(3)) - bx(hi(1),hi(2),lo(3)))*dxinv + dxb = (bx(hi(1)+1,hi(2),lo(3)) - bx(hi(1),hi(2),lo(3)))*dxinv dyb = (by(hi(1),hi(2)+1,lo(3)) - by(hi(1),hi(2),lo(3)))*dyinv - dzb = (bz(hi(1),hi(2),lo(3)+1) - bz(hi(1),hi(2),lo(3)))*dzinv + dzb = (bz(hi(1),hi(2),lo(3)+1) - bz(hi(1),hi(2),lo(3)))*dzinv rhs(hi(1), hi(2), lo(3)) = RHS3(a(hi(1),hi(2),lo(3)), b, x, y, z, dxb, dyb, dzb) - !xhi yhi zhi + !xhi yhi zhi z = zh - 0.5d0 b = 1.d0/3.d0*(bx(hi(1),hi(2),hi(3)) + by(hi(1), hi(2), hi(3)) + bz(hi(1),hi(2),hi(3))) - dxb = (bx(hi(1)+1,hi(2),hi(3)) - bx(hi(1),hi(2),hi(3)))*dxinv + dxb = (bx(hi(1)+1,hi(2),hi(3)) - bx(hi(1),hi(2),hi(3)))*dxinv dyb = (by(hi(1),hi(2)+1,hi(3)) - by(hi(1),hi(2),hi(3)))*dyinv - dzb = (bz(hi(1),hi(2),hi(3)+1) - bz(hi(1),hi(2),hi(3)))*dzinv + dzb = (bz(hi(1),hi(2),hi(3)+1) - bz(hi(1),hi(2),hi(3)))*dzinv rhs(hi(1), hi(2), hi(3)) = RHS3(a(hi(1),hi(2),hi(3)), b, x, y, z, dxb, dyb, dzb) end subroutine build_rhs_3D function RHS3(a, b, x, y, z, dxb, dyb, dzb) - real(amrex_real), intent(in) :: a, b, x, y, z, dxb, dyb, dzb + real(amrex_real), intent(in) :: a, b, x, y, z, dxb, dyb, dzb real(amrex_real) :: RHS3 - real(amrex_real) :: denom, term1, term2 + real(amrex_real) :: denom, term1, term2 denom = sqrt(x*x + y*y + z*z) - term1 = a*(x+z)/denom - term2 = (x*x - x*z + y*y)*dzb + (y*y - x*z + z*z)*dxb - 2.d0*(x+z)*b -(x*y + y*z)*dyb + term1 = a*(x+z)/denom + term2 = (x*x - x*z + y*y)*dzb + (y*y - x*z + z*z)*dxb - 2.d0*(x+z)*b -(x*y + y*z)*dyb term2 = term2/(denom**3) - RHS3 = term1 - term2 - end function + RHS3 = term1 - term2 + end function function dbdx2(x,y) - real(amrex_real), intent(in) :: x, y + real(amrex_real), intent(in) :: x, y real(amrex_real) :: dbdx2 - dbdx2 = cos(x)*cos(y) + dbdx2 = cos(x)*cos(y) end function dbdx2 function dbdy2(x,y) real(amrex_real), intent(in) :: x, y real(amrex_real) :: dbdy2 - dbdy2 = -sin(x)*sin(y) + dbdy2 = -sin(x)*sin(y) end function dbdy2 function dbdx3(x,y,z) real(amrex_real), intent(in) :: x,y,z real(amrex_real) :: dbdx3 dbdx3 = cos(x)*sin(y)*cos(z) - end function dbdx3 + end function dbdx3 function dbdy3(x,y,z) real(amrex_real), intent(in) :: x,y,z diff --git a/Tests/LinearSolvers/EBConvergenceTest/acoef.F90 b/Tests/LinearSolvers/EBConvergenceTest/acoef.F90 index 94a0b799272..7aaa32e9558 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/acoef.F90 +++ b/Tests/LinearSolvers/EBConvergenceTest/acoef.F90 @@ -1,32 +1,32 @@ module acoef use amrex_fort_module, only : amrex_real -private :: f +private :: f public :: build_a_2d, build_a_3d - interface f + interface f module procedure f2 module procedure f3 - end interface + end interface contains subroutine build_a_2d(lo, hi, problo, probhi, & - a, alo, ahi, & + a, alo, ahi, & dx) bind(c, name='build_a_2d') implicit none integer, dimension(2), intent(in ) :: lo, hi, alo, ahi - real(amrex_real), dimension(2), intent(in ) :: problo, probhi, dx + real(amrex_real), dimension(2), intent(in ) :: problo, probhi, dx real(amrex_real), intent(inout) :: a(alo(1):ahi(1),alo(2):ahi(2)) integer :: i, j - real(amrex_real) :: x, y, xl, yl, xh, yh + real(amrex_real) :: x, y, xl, yl, xh, yh xl = problo(1) yl = problo(2) xh = probhi(1) yh = probhi(2) - + do j = lo(2)+1, hi(2)-1 y = yl + (dble(j) + 0.5d0)*dx(2) do i = lo(1)+1, hi(1)-1 - x = xl + (dble(i) + 0.5d0)*dx(1) + x = xl + (dble(i) + 0.5d0)*dx(1) a(i,j) = f(x,y) enddo a(lo(1),j) = f(xl,y) @@ -45,21 +45,21 @@ subroutine build_a_2d(lo, hi, problo, probhi, & end subroutine build_a_2d subroutine build_a_3d(lo, hi, problo, probhi, & - a, alo, ahi, & + a, alo, ahi, & dx) bind(c, name='build_a_3d') - implicit none + implicit none integer, dimension(3), intent(in) :: lo, hi, alo, ahi - real(amrex_real), dimension(3), intent(in) :: problo, probhi, dx + real(amrex_real), dimension(3), intent(in) :: problo, probhi, dx real(amrex_real), intent(inout) :: a(alo(1):ahi(1),alo(2):ahi(2),alo(3):ahi(3)) - integer :: i, j, k + integer :: i, j, k real(amrex_real) :: x, xl, xh, y, yl, yh, z, zl, zh xl = problo(1) yl = problo(2) zl = problo(3) xh = probhi(1) yh = probhi(2) - zh = probhi(3) + zh = probhi(3) do k = lo(3)+1, hi(3)-1 z = zl + (dble(k) + 0.5d0)*dx(3) do j = lo(2)+1, hi(2)-1 @@ -88,8 +88,8 @@ subroutine build_a_3d(lo, hi, problo, probhi, & a(i,j,lo(3)) = f(x,y,zl) a(i,j,hi(3)) = f(x,y,zh) enddo - a(lo(1),j,lo(3)) = f(xl,y,zl) - a(hi(1),j,lo(3)) = f(xh,y,zl) + a(lo(1),j,lo(3)) = f(xl,y,zl) + a(hi(1),j,lo(3)) = f(xh,y,zl) a(lo(1),j,hi(3)) = f(xl,y,zh) a(hi(1),j,hi(3)) = f(xh,y,zh) enddo @@ -105,11 +105,11 @@ subroutine build_a_3d(lo, hi, problo, probhi, & end subroutine build_a_3d function f2(x,y) - real(amrex_real), intent(in) :: x, y - real(amrex_real) :: f2 - f2 = cos(x)*cos(y) - end function f2 - + real(amrex_real), intent(in) :: x, y + real(amrex_real) :: f2 + f2 = cos(x)*cos(y) + end function f2 + function f3(x,y,z) real(amrex_real), intent(in) :: x, y, z real(amrex_real) :: f3 diff --git a/Tests/LinearSolvers/EBConvergenceTest/bcoef.F90 b/Tests/LinearSolvers/EBConvergenceTest/bcoef.F90 index 5e20c83decd..d977f9d57bc 100644 --- a/Tests/LinearSolvers/EBConvergenceTest/bcoef.F90 +++ b/Tests/LinearSolvers/EBConvergenceTest/bcoef.F90 @@ -1,25 +1,25 @@ module bcoef use amrex_fort_module, only : amrex_real -private :: f +private :: f public :: build_b_2d, build_b_3d - interface f + interface f module procedure f2 module procedure f3 - end interface + end interface contains subroutine build_b_2d(lo, hi, problo, probhi, & - bx, bxlo, bxhi, & - by, bylo, byhi, & + bx, bxlo, bxhi, & + by, bylo, byhi, & dx) bind(c, name='build_b_2d') implicit none - integer, dimension(2), intent(in ) :: lo, hi, bxlo, bxhi, bylo, byhi - real(amrex_real), dimension(2), intent(in ) :: problo, probhi, dx + integer, dimension(2), intent(in ) :: lo, hi, bxlo, bxhi, bylo, byhi + real(amrex_real), dimension(2), intent(in ) :: problo, probhi, dx real(amrex_real), intent(inout) :: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2)) real(amrex_real), intent(inout) :: by(bylo(1):byhi(1),bylo(2):byhi(2)) integer :: i, j - real(amrex_real) :: xc, yc, xf, yf, xl, yl, xh, yh + real(amrex_real) :: xc, yc, xf, yf, xl, yl, xh, yh xl = problo(1) yl = problo(2) xh = probhi(1) @@ -32,10 +32,10 @@ subroutine build_b_2d(lo, hi, problo, probhi, & !interior xf = xl + dble(i)*dx(1) xc = xl + (dble(i) + 0.5d0)*dx(1) - bx(i,j) = f(xf, yc) + bx(i,j) = f(xf, yc) by(i,j) = f(xc, yf) - enddo - !xfaces + enddo + !xfaces xf = xl + dble(hi(1)+1)*dx(1) bx(lo(1),j) = f(xl,yc) by(lo(1),j) = f(xl,yf) @@ -78,37 +78,37 @@ subroutine build_b_2d(lo, hi, problo, probhi, & end subroutine build_b_2d subroutine build_b_3d(lo, hi, problo, probhi, & - bx, bxlo, bxhi, & - by, bylo, byhi, & + bx, bxlo, bxhi, & + by, bylo, byhi, & bz, bzlo, bzhi, & dx) bind(c, name='build_b_3d') - implicit none - integer, dimension(3), intent(in) :: lo, hi, bxlo, bxhi, bylo, byhi, bzlo, bzhi - real(amrex_real), dimension(3), intent(in) :: problo, probhi, dx + implicit none + integer, dimension(3), intent(in) :: lo, hi, bxlo, bxhi, bylo, byhi, bzlo, bzhi + real(amrex_real), dimension(3), intent(in) :: problo, probhi, dx real(amrex_real), intent(inout) :: bx(bxlo(1):bxhi(1),bxlo(2):bxhi(2),bxlo(3):bxhi(3)) real(amrex_real), intent(inout) :: by(bylo(1):byhi(1),bylo(2):byhi(2),bylo(3):byhi(3)) real(amrex_real), intent(inout) :: bz(bzlo(1):bzhi(1),bzlo(2):bzhi(2),bzlo(3):bzhi(3)) - integer :: i, j, k + integer :: i, j, k real(amrex_real) :: xc, xf, xl, xh, yc, yf, yl, yh, zc, zf, zl, zh xl = problo(1) yl = problo(2) zl = problo(3) xh = probhi(1) yh = probhi(2) - zh = probhi(3) + zh = probhi(3) do k = lo(3)+1,hi(3)-1 zf = zl + dble(k)*dx(3) - zc = zl + (dble(k) + 0.5d0)*dx(3) + zc = zl + (dble(k) + 0.5d0)*dx(3) do j = lo(2)+1,hi(2)-1 yf = yl + dble(j)*dx(2) yc = yl + (dble(j) + 0.5d0)*dx(2) do i = lo(1)+1,hi(1)-1 ! interior - xf = xl + dble(i)*dx(1) + xf = xl + dble(i)*dx(1) xc = xl + (dble(i) + 0.5d0)*dx(1) bx(i,j,k) = f(xf,yc,zc) - by(i,j,k) = f(xc,yf,zc) + by(i,j,k) = f(xc,yf,zc) bz(i,j,k) = f(xc,yc,zf) enddo xf = xl + dble(hi(1))*dx(1) @@ -134,10 +134,10 @@ subroutine build_b_3d(lo, hi, problo, probhi, & bx(i,hi(2),k) = f(xf,yh,zc) by(i,hi(2),k) = f(xc,yf,zc) bz(i,hi(2),k) = f(xc,yh,zf) - by(i,hi(2)+1,k) = f(xc,yh,zc) + by(i,hi(2)+1,k) = f(xc,yh,zc) enddo -! xy edges +! xy edges bx(lo(1),lo(2),k) = f(xl,yl,zc) by(lo(1),lo(2),k) = f(xl,yl,zc) @@ -166,7 +166,7 @@ subroutine build_b_3d(lo, hi, problo, probhi, & yf = yl + dble(j)*dx(2) do i = lo(1)+1,hi(1)-1 ! z face - xc = xl + (dble(i) + 0.5d0)*dx(1) + xc = xl + (dble(i) + 0.5d0)*dx(1) xf = xl + dble(i)*dx(1) bx(i,j,lo(3)) = f(xf,yc,zl) by(i,j,lo(3)) = f(xc,yf,zl) @@ -177,7 +177,7 @@ subroutine build_b_3d(lo, hi, problo, probhi, & bz(i,j,hi(3)) = f(xc,yc,zf) bz(i,j,hi(3)+1) = f(xc,yc,zh) enddo -! xz edges +! xz edges xf = xl + (dble(hi(1)))*dx(1) bx(lo(1),j,lo(3)) = f(xl,yc,zl) by(lo(1),j,lo(3)) = f(xl,yf,zl) @@ -202,7 +202,7 @@ subroutine build_b_3d(lo, hi, problo, probhi, & ! yz edges do i = lo(1)+1,hi(1)-1 - xc = xl +(dble(i) + 0.5)*dx(1) + xc = xl +(dble(i) + 0.5)*dx(1) xf = xl +dble(i)*dx(1) bx(i,lo(2),lo(3)) = f(xf,yl,zl) by(i,lo(2),lo(3)) = f(xc,yl,zl) @@ -249,7 +249,7 @@ subroutine build_b_3d(lo, hi, problo, probhi, & by(lo(1),hi(2)+1,hi(3)) = f(xl,yh,zh) bz(lo(1),hi(2),hi(3)) = f(xl,yh,zf) bz(lo(1),hi(2),hi(3)+1) = f(xl,yh,zh) - + bx(hi(1),lo(2),lo(3)) = f(xf,yl,zl) bx(hi(1)+1,lo(2),lo(3)) = f(xh,yl,zl) by(hi(1),lo(2),lo(3)) = f(xh,yl,zl) @@ -277,11 +277,11 @@ subroutine build_b_3d(lo, hi, problo, probhi, & end subroutine build_b_3d function f2(x,y) - real(amrex_real), intent(in) :: x, y - real(amrex_real) :: f2 - f2 = sin(x)*cos(y) - end function f2 - + real(amrex_real), intent(in) :: x, y + real(amrex_real) :: f2 + f2 = sin(x)*cos(y) + end function f2 + function f3(x,y,z) real(amrex_real), intent(in) :: x, y, z real(amrex_real) :: f3 diff --git a/Tests/LinearSolvers/EBTensor/MyTest.cpp b/Tests/LinearSolvers/EBTensor/MyTest.cpp index 92a2e459a95..b20afdf4dab 100644 --- a/Tests/LinearSolvers/EBTensor/MyTest.cpp +++ b/Tests/LinearSolvers/EBTensor/MyTest.cpp @@ -140,7 +140,7 @@ void MyTest::readParameters () { ParmParse pp; - + pp.query("n_cell", n_cell); pp.query("max_grid_size", max_grid_size); diff --git a/Tests/LinearSolvers/EBTensor/MyTestPlotfile.cpp b/Tests/LinearSolvers/EBTensor/MyTestPlotfile.cpp index 6709ff55f64..c51cc95ea4c 100644 --- a/Tests/LinearSolvers/EBTensor/MyTestPlotfile.cpp +++ b/Tests/LinearSolvers/EBTensor/MyTestPlotfile.cpp @@ -17,7 +17,7 @@ MyTest::writePlotfile () const const int ncomp = AMREX_SPACEDIM*4 + 2; Vector varname = #if (AMREX_SPACEDIM == 2) - {"u", "v", "uexact", "vexact", "xerror", "yerror", + {"u", "v", "uexact", "vexact", "xerror", "yerror", "xrhs", "yrhs", "eta", "vfrc"}; #else {"u", "v", "w", "uexact", "vexact", "wexact", "xerror", "yerror", "zerror", diff --git a/Tests/LinearSolvers/EBTensor/MyTest_2D_K.H b/Tests/LinearSolvers/EBTensor/MyTest_2D_K.H index c309eaa6a42..b53b5907472 100644 --- a/Tests/LinearSolvers/EBTensor/MyTest_2D_K.H +++ b/Tests/LinearSolvers/EBTensor/MyTest_2D_K.H @@ -4,7 +4,7 @@ #include inline void init (amrex::Real x, amrex::Real y, amrex::Real R2, - amrex::Real& u, amrex::Real& v, + amrex::Real& u, amrex::Real& v, amrex::Real& urhs, amrex::Real& vrhs, amrex::Real& eta) { diff --git a/Tests/LinearSolvers/EBflux_grad/MyTest.H b/Tests/LinearSolvers/EBflux_grad/MyTest.H index a0ba0e07da0..17ccdec1604 100644 --- a/Tests/LinearSolvers/EBflux_grad/MyTest.H +++ b/Tests/LinearSolvers/EBflux_grad/MyTest.H @@ -24,7 +24,7 @@ private: int ref_ratio = 2; int n_cell = 128; int max_grid_size = 64; - + // For MLMG solver int verbose = 2; int bottom_verbose = 2; @@ -43,8 +43,8 @@ private: amrex::Vector > factory; amrex::Vector phi; - amrex::Vector> flux; - amrex::Vector> grad; + amrex::Vector> flux; + amrex::Vector> grad; amrex::Vector rhs; amrex::Vector acoef; amrex::Vector > bcoef; diff --git a/Tests/LinearSolvers/EBflux_grad/MyTest.cpp b/Tests/LinearSolvers/EBflux_grad/MyTest.cpp index 9056348c10d..e10fddb6c9a 100644 --- a/Tests/LinearSolvers/EBflux_grad/MyTest.cpp +++ b/Tests/LinearSolvers/EBflux_grad/MyTest.cpp @@ -71,15 +71,15 @@ MyTest::solve () const Real tol_rel = reltol; const Real tol_abs = 0.0; mlmg.solve(amrex::GetVecOfPtrs(phi), amrex::GetVecOfConstPtrs(rhs), tol_rel, tol_abs); - mlmg.getFluxes(amrex::GetVecOfArrOfPtrs(flux)); + mlmg.getFluxes(amrex::GetVecOfArrOfPtrs(flux)); mlmg.getGradSolution(amrex::GetVecOfArrOfPtrs(grad)); for (int ilev = 0; ilev <= max_level; ++ilev) { amrex::VisMF::Write(phi[0], "phi-"+std::to_string(ilev)); } for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - amrex::VisMF::Write(flux[0][idim], "flux-"+std::to_string(idim)); - amrex::VisMF::Write(grad[0][idim], "grad-"+std::to_string(idim)); + amrex::VisMF::Write(flux[0][idim], "flux-"+std::to_string(idim)); + amrex::VisMF::Write(grad[0][idim], "grad-"+std::to_string(idim)); } } @@ -130,7 +130,7 @@ MyTest::initGrids () grids[ilev].define(domain); grids[ilev].maxSize(max_grid_size); domain.grow(-n_cell/4); // fine level cover the middle of the coarse domain - domain.refine(ref_ratio); + domain.refine(ref_ratio); } } @@ -144,8 +144,8 @@ MyTest::initData () rhs.resize(nlevels); acoef.resize(nlevels); bcoef.resize(nlevels); - flux.resize(1); - grad.resize(1); + flux.resize(1); + grad.resize(1); for (int ilev = 0; ilev < nlevels; ++ilev) { dmap[ilev].define(grids[ilev]); diff --git a/Tests/LinearSolvers/EBflux_grad/initEB.cpp b/Tests/LinearSolvers/EBflux_grad/initEB.cpp index 8f5da6d50ad..1e8832c2e2e 100644 --- a/Tests/LinearSolvers/EBflux_grad/initEB.cpp +++ b/Tests/LinearSolvers/EBflux_grad/initEB.cpp @@ -43,7 +43,7 @@ MyTest::initializeEB () EB2::translate(box, {AMREX_D_DECL(-0.5,-0.5,-0.5)}), std::atan(1.0)*0.3, 2), {AMREX_D_DECL(0.5,0.5,0.5)})); - EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); + EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); } else if (geom_type == "two_spheres") { diff --git a/Tests/LinearSolvers/LeastSquares/MyTest.cpp b/Tests/LinearSolvers/LeastSquares/MyTest.cpp index 3fef89b9889..3f61b3bf1e0 100644 --- a/Tests/LinearSolvers/LeastSquares/MyTest.cpp +++ b/Tests/LinearSolvers/LeastSquares/MyTest.cpp @@ -134,7 +134,7 @@ MyTest::compute_gradient () grad_x_arr(i,j,k,n) = (apx(i,j,k) == 0.0) ? 0.0 : grad_x_of_phi_on_centroids(i, j, k, n, phi_arr, phi_eb_arr, - flag, ccent, bcent, + flag, ccent, bcent, yloc_on_xface, is_eb_dirichlet, is_eb_inhomog); @@ -170,7 +170,7 @@ MyTest::compute_gradient () grad_x_arr(i,j,k,n) = (apx(i,j,k) == 0.0) ? 0.0 : grad_x_of_phi_on_centroids_extdir(i, j, k, n, phi_arr, phi_eb_arr, flag, ccent, bcent, vfrac, - yloc_on_xface, zloc_on_xface, + yloc_on_xface, zloc_on_xface, is_eb_dirichlet, is_eb_inhomog, on_x_face, domlo_x, domhi_x, on_y_face, domlo_y, domhi_y, diff --git a/Tests/LinearSolvers/LeastSquares/README.md b/Tests/LinearSolvers/LeastSquares/README.md index a867970fef2..44128aa6d8e 100644 --- a/Tests/LinearSolvers/LeastSquares/README.md +++ b/Tests/LinearSolvers/LeastSquares/README.md @@ -4,7 +4,7 @@ This directory contains tests for: 1. Slope computation on face centroids using the Least Square method, 2. Laplacian operator computation when applying the explicit stencil, -Given a function defined on cell centroids. +Given a function defined on cell centroids. # Building & Running 1. Build using `make` diff --git a/Tests/LinearSolvers/LeastSquares/initData.cpp b/Tests/LinearSolvers/LeastSquares/initData.cpp index 4ac7d6ebc21..77de5479061 100644 --- a/Tests/LinearSolvers/LeastSquares/initData.cpp +++ b/Tests/LinearSolvers/LeastSquares/initData.cpp @@ -93,26 +93,26 @@ MyTest::initData () bcoef[ilev][idim].setVal(1.0); } - if(use_poiseuille) { - initializePoiseuilleData(ilev); - } - else { - // Test a custom polynomial function - for (MFIter mfi(phi[ilev]); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.fabbox(); - Array4 const& fab = phi[ilev].array(mfi); - const auto dx = geom[ilev].CellSizeArray(); + if(use_poiseuille) { + initializePoiseuilleData(ilev); + } + else { + // Test a custom polynomial function + for (MFIter mfi(phi[ilev]); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.fabbox(); + Array4 const& fab = phi[ilev].array(mfi); + const auto dx = geom[ilev].CellSizeArray(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rx = (i+0.5)*dx[0]; - Real ry = (j+0.5)*dx[1]; - // fab(i,j,k) = std::sin(rx*2.*pi + 43.5)*std::sin(ry*2.*pi + 89.); - fab(i,j,k) = rx*(1.-rx)*ry*(1.-ry); - }); - } - } + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + Real rx = (i+0.5)*dx[0]; + Real ry = (j+0.5)*dx[1]; + // fab(i,j,k) = std::sin(rx*2.*pi + 43.5)*std::sin(ry*2.*pi + 89.); + fab(i,j,k) = rx*(1.-rx)*ry*(1.-ry); + }); + } + } } } diff --git a/Tests/LinearSolvers/LeastSquares/initEB.cpp b/Tests/LinearSolvers/LeastSquares/initEB.cpp index 54ad8ed6711..d4fb19ac0e9 100644 --- a/Tests/LinearSolvers/LeastSquares/initEB.cpp +++ b/Tests/LinearSolvers/LeastSquares/initEB.cpp @@ -32,7 +32,7 @@ MyTest::initializeEB () EB2::translate(box, {AMREX_D_DECL(-0.5,-0.5,-0.5)}), std::atan(1.0)*0.3, 2), {AMREX_D_DECL(0.5,0.5,0.5)})); - EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); + EB2::Build(gshop, geom.back(), max_level, max_level+max_coarsening_level); } else if (geom_type == "two_spheres") { @@ -64,17 +64,17 @@ MyTest::initializeEB () Vector pt_on_top_wall(3); Real height; Real rotation = 0.0; - + pp.getarr("channel_pt_on_top_wall", pt_on_top_wall, 0, 3); pp.get("channel_rotation", rotation); pp.get("channel_height", height); rotation = (rotation/180.) * M_PI; - EB2::PlaneIF left({AMREX_D_DECL(pt_on_top_wall[0],pt_on_top_wall[1],0.0)}, - {AMREX_D_DECL(-std::sin(rotation),std::cos(rotation),0.0)}, + EB2::PlaneIF left({AMREX_D_DECL(pt_on_top_wall[0],pt_on_top_wall[1],0.0)}, + {AMREX_D_DECL(-std::sin(rotation),std::cos(rotation),0.0)}, fluid_inside); - EB2::PlaneIF right({AMREX_D_DECL(pt_on_top_wall[0], pt_on_top_wall[1] - (height/std::cos(rotation)),0.0)}, - {AMREX_D_DECL(std::sin(rotation),-std::cos(rotation),0.0)}, + EB2::PlaneIF right({AMREX_D_DECL(pt_on_top_wall[0], pt_on_top_wall[1] - (height/std::cos(rotation)),0.0)}, + {AMREX_D_DECL(std::sin(rotation),-std::cos(rotation),0.0)}, fluid_inside); auto channel = EB2::makeUnion(left, right); auto gshop = EB2::makeShop(channel); @@ -87,7 +87,7 @@ MyTest::initializeEB () Vector pt_on_top_wall(3); Real height; Vector rotation(2); - + pp.getarr("channel_pt_on_top_wall", pt_on_top_wall, 0, 3); pp.getarr("channel_rotation", rotation, 0, 2); pp.get("channel_height", height); @@ -96,21 +96,21 @@ MyTest::initializeEB () Real gamma = (rotation[1]/180.) * M_PI; Vector norm(3); - Real norm_mag = std::sqrt(std::sin(alpha)*std::sin(alpha) + - std::cos(alpha)*std::cos(alpha)*std::cos(gamma)*std::cos(gamma) + + Real norm_mag = std::sqrt(std::sin(alpha)*std::sin(alpha) + + std::cos(alpha)*std::cos(alpha)*std::cos(gamma)*std::cos(gamma) + std::sin(gamma)*std::sin(gamma)); norm[0] = -std::sin(gamma) / norm_mag; norm[1] = std::cos(alpha)*std::cos(gamma) / norm_mag; norm[2] = -std::sin(alpha) / norm_mag; - EB2::PlaneIF left({AMREX_D_DECL(pt_on_top_wall[0],pt_on_top_wall[1],pt_on_top_wall[2])}, - {AMREX_D_DECL(norm[0],norm[1],norm[2])}, - fluid_inside); + EB2::PlaneIF left({AMREX_D_DECL(pt_on_top_wall[0],pt_on_top_wall[1],pt_on_top_wall[2])}, + {AMREX_D_DECL(norm[0],norm[1],norm[2])}, + fluid_inside); EB2::PlaneIF right({AMREX_D_DECL(pt_on_top_wall[0] - height*norm[0], - pt_on_top_wall[1] - height*norm[1], - pt_on_top_wall[2] - height*norm[2])}, - {AMREX_D_DECL(-norm[0],-norm[1],-norm[2])}, + pt_on_top_wall[1] - height*norm[1], + pt_on_top_wall[2] - height*norm[2])}, + {AMREX_D_DECL(-norm[0],-norm[1],-norm[2])}, fluid_inside); auto channel = EB2::makeUnion(left, right); diff --git a/Tests/LinearSolvers/LeastSquares/initPoiseuilleData.cpp b/Tests/LinearSolvers/LeastSquares/initPoiseuilleData.cpp index 345c53f5bb5..f517c09eeec 100644 --- a/Tests/LinearSolvers/LeastSquares/initPoiseuilleData.cpp +++ b/Tests/LinearSolvers/LeastSquares/initPoiseuilleData.cpp @@ -2,8 +2,8 @@ void MyTest::initializePoiseuilleData(int ilev) { #if (AMREX_SPACEDIM == 2) - initializePoiseuilleDataFor2D(ilev); + initializePoiseuilleDataFor2D(ilev); #else - initializePoiseuilleDataFor3D(ilev); + initializePoiseuilleDataFor3D(ilev); #endif } diff --git a/Tests/LinearSolvers/MAC_Projection_EB/main.cpp b/Tests/LinearSolvers/MAC_Projection_EB/main.cpp index 2b66caa245e..1211e620d45 100644 --- a/Tests/LinearSolvers/MAC_Projection_EB/main.cpp +++ b/Tests/LinearSolvers/MAC_Projection_EB/main.cpp @@ -160,18 +160,18 @@ int main (int argc, char* argv[]) // such as BaseFab, FArrayBox, FabArray, and MultiFab EBFArrayBoxFactory factory(eb_level, geom, grids, dmap, ng_ebs, ebs); - // allocate face-centered velocities and face-centered beta coefficient + // allocate face-centered velocities and face-centered beta coefficient for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { vel[idim].define (amrex::convert(grids,IntVect::TheDimensionVector(idim)), dmap, 1, 1, - MFInfo(), factory); + MFInfo(), factory); beta[idim].define(amrex::convert(grids,IntVect::TheDimensionVector(idim)), dmap, 1, 0, - MFInfo(), factory); + MFInfo(), factory); beta[idim].setVal(1.0); // set beta to 1 } - // If we want to supply a non-zero S we must allocate and fill it outside the solver + // If we want to supply a non-zero S we must allocate and fill it outside the solver // MultiFab S(grids, dmap, 1, 0, MFInfo(), factory); - // Set S here ... + // Set S here ... // store plotfile variables; velocity and processor id plotfile_mf.define(grids, dmap, AMREX_SPACEDIM+1, 0, MFInfo(), factory); @@ -195,34 +195,34 @@ int main (int argc, char* argv[]) {geom}, // the geometry object lp_info); // structure for passing info to the operator - // Here we specifiy the desired divergence S - // MacProjector macproj({amrex::GetArrOfPtrs(vel)}, // face-based velocity - // {amrex::GetArrOfConstPtrs(beta)}, // beta - // {geom}, // the geometry object - // lp_info, // structure for passing info to the operator - // {&S}); // defines the specified RHS divergence + // Here we specifiy the desired divergence S + // MacProjector macproj({amrex::GetArrOfPtrs(vel)}, // face-based velocity + // {amrex::GetArrOfConstPtrs(beta)}, // beta + // {geom}, // the geometry object + // lp_info, // structure for passing info to the operator + // {&S}); // defines the specified RHS divergence // Set bottom-solver to use hypre instead of native BiCGStab if (use_hypre) macproj.getMLMG().setBottomSolver(MLMG::BottomSolver::hypre); - // Hard-wire the boundary conditions to be Neumann on the low x-face, Dirichlet - // on the high x-face, and periodic in the other two directions - // (the first argument is for the low end, the second is for the high end) + // Hard-wire the boundary conditions to be Neumann on the low x-face, Dirichlet + // on the high x-face, and periodic in the other two directions + // (the first argument is for the low end, the second is for the high end) macproj.setDomainBC({AMREX_D_DECL(LinOpBCType::Neumann, LinOpBCType::Periodic, LinOpBCType::Periodic)}, - {AMREX_D_DECL(LinOpBCType::Dirichlet, - LinOpBCType::Periodic, - LinOpBCType::Periodic)}); + {AMREX_D_DECL(LinOpBCType::Dirichlet, + LinOpBCType::Periodic, + LinOpBCType::Periodic)}); macproj.setVerbose(mg_verbose); macproj.getMLMG().setBottomVerbose(bottom_verbose); - // Define the relative tolerance + // Define the relative tolerance Real reltol = 1.e-8; - // Define the absolute tolerance; note that this argument is optional + // Define the absolute tolerance; note that this argument is optional Real abstol = 1.e-15; amrex::Print() << " \n********************************************************************" << std::endl; @@ -232,13 +232,13 @@ int main (int argc, char* argv[]) amrex::Print() << " The maximum grid size is " << max_grid_size << std::endl; amrex::Print() << "******************************************************************** \n" << std::endl; - // Solve for phi and subtract from the velocity to make it divergence-free - // Note that the normal velocities are at face centers (not centroids) + // Solve for phi and subtract from the velocity to make it divergence-free + // Note that the normal velocities are at face centers (not centroids) macproj.project(reltol,abstol); - // If we want to use phi elsewhere, we can pass in an array in which to return the solution - // MultiFab phi_inout(grids, dmap, 1, 1, MFInfo(), factory); - // macproj.project_center_vels({&phi_inout},reltol,abstol,MLMG::Location::FaceCenter); + // If we want to use phi elsewhere, we can pass in an array in which to return the solution + // MultiFab phi_inout(grids, dmap, 1, 1, MFInfo(), factory); + // macproj.project_center_vels({&phi_inout},reltol,abstol,MLMG::Location::FaceCenter); amrex::Print() << " \n********************************************************************" << std::endl; amrex::Print() << " Done!" << std::endl; diff --git a/Tests/LinearSolvers/MLMG/fort_3d.F90 b/Tests/LinearSolvers/MLMG/fort_3d.F90 index ef236b49c18..717e60fa53b 100644 --- a/Tests/LinearSolvers/MLMG/fort_3d.F90 +++ b/Tests/LinearSolvers/MLMG/fort_3d.F90 @@ -27,33 +27,33 @@ subroutine fort_set_coef (lo, hi, exact, elo, ehi, alpha, alo, ahi, beta, blo, b zc = (prob_hi(3) + prob_lo(3))/2.d0 theta = 0.5d0*log(3.d0) / (w + 1.d-50) - + do k = lo(3)-1, hi(3)+1 z = prob_lo(3) + dx(3) * (dble(k)+0.5d0) do j = lo(2)-1, hi(2)+1 y = prob_lo(2) + dx(2) * (dble(j)+0.5d0) do i = lo(1)-1, hi(1)+1 x = prob_lo(1) + dx(1) * (dble(i)+0.5d0) - + r = sqrt((x-xc)**2 + (y-yc)**2 + (z-zc)**2) - + beta(i,j,k) = (sigma-1.d0)/2.d0*tanh(theta*(r-0.25d0)) + (sigma+1.d0)/2.d0 end do end do end do - + do k = lo(3), hi(3) z = prob_lo(3) + dx(3) * (dble(k)+0.5d0) do j = lo(2), hi(2) y = prob_lo(2) + dx(2) * (dble(j)+0.5d0) do i = lo(1), hi(1) x = prob_lo(1) + dx(1) * (dble(i)+0.5d0) - + r = sqrt((x-xc)**2 + (y-yc)**2 + (z-zc)**2) - + dbdrfac = (sigma-1.d0)/2.d0/(cosh(theta*(r-0.25d0)))**2 * theta/r dbdrfac = dbdrfac * b - + alpha(i,j,k) = 1.d0 if (bct .eq. 'p' .or. bct .eq. 'n') then diff --git a/Tests/LinearSolvers/MLMG/init_prob.cpp b/Tests/LinearSolvers/MLMG/init_prob.cpp index a4d1238763e..36058c31655 100644 --- a/Tests/LinearSolvers/MLMG/init_prob.cpp +++ b/Tests/LinearSolvers/MLMG/init_prob.cpp @@ -26,7 +26,7 @@ void init_prob_parms () pp.query("b" , b); pp.query("sigma", sigma); pp.query("w" , w); - + std::string bc_type_s; pp.query("bc_type", bc_type_s); if (bc_type_s == "Dirichlet") { diff --git a/Tests/LinearSolvers/MLMG/main.cpp b/Tests/LinearSolvers/MLMG/main.cpp index e0a5e6f97de..699d1bfa0df 100644 --- a/Tests/LinearSolvers/MLMG/main.cpp +++ b/Tests/LinearSolvers/MLMG/main.cpp @@ -42,7 +42,7 @@ namespace { int main (int argc, char* argv[]) { amrex::Initialize(argc, argv); - + { BL_PROFILE("main()"); @@ -51,13 +51,13 @@ int main (int argc, char* argv[]) Vector geom; Vector grids; build_geometry_and_grids(geom, grids); - + Vector soln(nlevels); Vector exact(nlevels); Vector alpha(nlevels); Vector beta(nlevels); Vector rhs(nlevels); - + for (int ilev = 0; ilev < nlevels; ++ilev) { DistributionMapping dm{grids[ilev]}; @@ -73,7 +73,7 @@ int main (int argc, char* argv[]) for (auto& mf : soln) { mf.setVal(0.0); // initial guess } - + solve_with_mlmg (geom, ref_ratio, soln, alpha, beta, rhs, exact); #ifdef AMREX_SOFT_PERF_COUNTERS @@ -143,16 +143,16 @@ void build_geometry_and_grids (Vector& geom, Vector& grids) nlevels = max_level + 1; geom.resize(nlevels); grids.resize(nlevels); - + IntVect dom0_lo {IntVect::TheZeroVector()}; IntVect dom0_hi {AMREX_D_DECL(n_cell-1, n_cell-1, n_cell-1)}; - + Box dom0 {dom0_lo, dom0_hi}; BoxArray ba0{dom0}; - + grids[0] = ba0; grids[0].maxSize(max_grid_size); - + for (int ilev=1, n=grids.size(); ilev < n; ++ilev) { ba0.grow(-n_cell/4); @@ -161,11 +161,11 @@ void build_geometry_and_grids (Vector& geom, Vector& grids) grids[ilev].maxSize(max_grid_size); } } - + std::array prob_lo{AMREX_D_DECL(0.,0.,0.)}; std::array prob_hi{AMREX_D_DECL(1.,1.,1.)}; RealBox real_box{prob_lo, prob_hi}; - + const int coord = 0; // Cartesian coordinates std::array is_periodic{AMREX_D_DECL(0,0,0)}; if (prob::bc_type == MLLinOp::BCType::Periodic) @@ -176,7 +176,7 @@ void build_geometry_and_grids (Vector& geom, Vector& grids) IntVect dom0_lo {IntVect::TheZeroVector()}; IntVect dom0_hi {AMREX_D_DECL(n_cell-1, n_cell-1, n_cell-1)}; Box dom0 {dom0_lo, dom0_hi}; - + geom[0].define(dom0, &real_box, coord, is_periodic.data()); for (int ilev=1, n=grids.size(); ilev < n; ++ilev) { @@ -201,7 +201,7 @@ readBoxList (const std::string& file, Box& domain) amrex::Error(msg.c_str()); } boxspec >> domain; - + int numbox = 0; boxspec >> numbox; @@ -210,7 +210,7 @@ readBoxList (const std::string& file, Box& domain) Box tmpbox; boxspec >> tmpbox; if( !domain.contains(tmpbox) ) - { + { std::cerr << "readBoxList: bogus box " << tmpbox << '\n'; exit(1); } diff --git a/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.H b/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.H index ab19a8a1daf..b1b6e914d86 100644 --- a/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.H +++ b/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.H @@ -9,12 +9,12 @@ // solids.uccs.edu // // Date: September 3, 2019 -// +// // Description: This file defines the multi-component linear operator // // See also: ./MCNodalLinOp.cpp (for implementation of class) // ./main.cpp (for execution of the tutorial) -// +// #ifndef MCNodalLinOp_H #define MCNodalLinOp_H @@ -29,84 +29,84 @@ class MCNodalLinOp : public MLNodeLinOp { public : - MCNodalLinOp () {} - virtual ~MCNodalLinOp (); - void define (const Vector& a_geom, const Vector& a_grids, - const Vector& a_dmap, - const LPInfo& a_info = LPInfo(), - const Vector const*>& a_factory = {}); + MCNodalLinOp () {} + virtual ~MCNodalLinOp (); + void define (const Vector& a_geom, const Vector& a_grids, + const Vector& a_dmap, + const LPInfo& a_info = LPInfo(), + const Vector const*>& a_factory = {}); - // Set the number of components that the operator will work with. - // Default = 1 - void setNComp(int a_ncomp) {ncomp = a_ncomp;} - // Set the coefficients for the operator. The input (a_coeff) must have - // ncomp x ncomp ( x ncomp) components. It is the matrix of coefficients - // used in the definition of the operator. - void setCoeff(const Vector a_coeff) {coeff = a_coeff;} + // Set the number of components that the operator will work with. + // Default = 1 + void setNComp(int a_ncomp) {ncomp = a_ncomp;} + // Set the coefficients for the operator. The input (a_coeff) must have + // ncomp x ncomp ( x ncomp) components. It is the matrix of coefficients + // used in the definition of the operator. + void setCoeff(const Vector a_coeff) {coeff = a_coeff;} protected: - // Apply the operator, which is defined to be (in index notation) - // D(phi)_i = coeff_{ij} * Laplacian(phi_j) - // where coeff_{ij} is stored as a vector, passed in using - // the setCoeff function - virtual void Fapply (int amrlev, int mglev,MultiFab& out,const MultiFab& in) const override ; - // Compute the diagonal of the operator - void Diag (int amrlev, int mglev, MultiFab& diag); + // Apply the operator, which is defined to be (in index notation) + // D(phi)_i = coeff_{ij} * Laplacian(phi_j) + // where coeff_{ij} is stored as a vector, passed in using + // the setCoeff function + virtual void Fapply (int amrlev, int mglev,MultiFab& out,const MultiFab& in) const override ; + // Compute the diagonal of the operator + void Diag (int amrlev, int mglev, MultiFab& diag); protected: - void Diagonal (bool recompute=false); - // Run a Gauss-Siedel operation on the solution. - // (Note: this implementation uses just Diag and Fapply; no operator-specific code needed) - virtual void Fsmooth (int amrlev, int mglev, MultiFab& x,const MultiFab& b) const override; - // Divide out by the diagonal. - // (Note: this implementation uses just Diag, no operator-specific code needed) - virtual void normalize (int amrlev, int mglev, MultiFab& mf) const override; - // Resolve the residual at the coarse/fine boundary. - // (Note: unlike other nodal solvers, this is NOT operator-specific. Instead is just performs a - // restriction from the fine level to the coarse. MUST be used with CFStrategy::ghostnodes) - virtual void reflux (int crse_amrlev, MultiFab& res, const MultiFab& crse_sol, const MultiFab& crse_rhs, - MultiFab& fine_res, MultiFab& fine_sol, const MultiFab& fine_rhs) const override; + void Diagonal (bool recompute=false); + // Run a Gauss-Siedel operation on the solution. + // (Note: this implementation uses just Diag and Fapply; no operator-specific code needed) + virtual void Fsmooth (int amrlev, int mglev, MultiFab& x,const MultiFab& b) const override; + // Divide out by the diagonal. + // (Note: this implementation uses just Diag, no operator-specific code needed) + virtual void normalize (int amrlev, int mglev, MultiFab& mf) const override; + // Resolve the residual at the coarse/fine boundary. + // (Note: unlike other nodal solvers, this is NOT operator-specific. Instead is just performs a + // restriction from the fine level to the coarse. MUST be used with CFStrategy::ghostnodes) + virtual void reflux (int crse_amrlev, MultiFab& res, const MultiFab& crse_sol, const MultiFab& crse_rhs, + MultiFab& fine_res, MultiFab& fine_sol, const MultiFab& fine_rhs) const override; protected: - // Usual MG restriction - virtual void restriction (int amrlev, int cmglev, MultiFab& crse, MultiFab& fine) const final; - // Usual MG interpolation - virtual void interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const override final; - // Usual averageDownSolutionRHS - virtual void averageDownSolutionRHS (int camrlev, MultiFab& crse_sol, MultiFab& crse_rhs, const MultiFab& fine_sol, const MultiFab& fine_rhs) final; - // Usual prepareForSolve - virtual void prepareForSolve () final; - // isSingular always returns 0 (singular operators not supported) - virtual bool isSingular (int amrlev) const final {return (amrlev == 0) ? m_is_bottom_singular : false; } - // isBottomSingular always returns 0 (singular operators not supported) - virtual bool isBottomSingular () const final { return m_is_bottom_singular; } - // Usual applyBC - virtual void applyBC (int amrlev, int mglev, MultiFab& phi, BCMode bc_mode, amrex::MLLinOp::StateMode /**/, bool skip_fillboundary=false) const final; - // Usual fixUpResidualMask - virtual void fixUpResidualMask (int amrlev, iMultiFab& resmsk) final; + // Usual MG restriction + virtual void restriction (int amrlev, int cmglev, MultiFab& crse, MultiFab& fine) const final; + // Usual MG interpolation + virtual void interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const override final; + // Usual averageDownSolutionRHS + virtual void averageDownSolutionRHS (int camrlev, MultiFab& crse_sol, MultiFab& crse_rhs, const MultiFab& fine_sol, const MultiFab& fine_rhs) final; + // Usual prepareForSolve + virtual void prepareForSolve () final; + // isSingular always returns 0 (singular operators not supported) + virtual bool isSingular (int amrlev) const final {return (amrlev == 0) ? m_is_bottom_singular : false; } + // isBottomSingular always returns 0 (singular operators not supported) + virtual bool isBottomSingular () const final { return m_is_bottom_singular; } + // Usual applyBC + virtual void applyBC (int amrlev, int mglev, MultiFab& phi, BCMode bc_mode, amrex::MLLinOp::StateMode /**/, bool skip_fillboundary=false) const final; + // Usual fixUpResidualMask + virtual void fixUpResidualMask (int amrlev, iMultiFab& resmsk) final; public: - // Return the number of ghost cells - // Note: MUST return 2! - virtual int getNGrow() const override final {return 2;} - // Return the number of components - virtual int getNComp() const override final {return ncomp;} - // Overrides solutionResidual to add call to realFillBoundary - virtual void solutionResidual (int amrlev, MultiFab& resid, MultiFab& x, const MultiFab& b, - const MultiFab* crse_bcdata=nullptr) override final; - // Overrides correctionResidual to add call to realFillBoundary - virtual void correctionResidual (int amrlev, int mglev, MultiFab& resid, MultiFab& x, const MultiFab& b, - BCMode bc_mode, const MultiFab* crse_bcdata=nullptr) override final; + // Return the number of ghost cells + // Note: MUST return 2! + virtual int getNGrow() const override final {return 2;} + // Return the number of components + virtual int getNComp() const override final {return ncomp;} + // Overrides solutionResidual to add call to realFillBoundary + virtual void solutionResidual (int amrlev, MultiFab& resid, MultiFab& x, const MultiFab& b, + const MultiFab* crse_bcdata=nullptr) override final; + // Overrides correctionResidual to add call to realFillBoundary + virtual void correctionResidual (int amrlev, int mglev, MultiFab& resid, MultiFab& x, const MultiFab& b, + BCMode bc_mode, const MultiFab* crse_bcdata=nullptr) override final; private: - // Usual buildMasks - void buildMasks (); + // Usual buildMasks + void buildMasks (); + + bool m_is_bottom_singular = false; + bool m_masks_built = false; + + int ncomp = 1; + Vector coeff = {1.0}; - bool m_is_bottom_singular = false; - bool m_masks_built = false; - - int ncomp = 1; - Vector coeff = {1.0}; - protected: - bool m_diagonal_computed = false; - amrex::Vector > > m_diag; + bool m_diagonal_computed = false; + amrex::Vector > > m_diag; }; #endif diff --git a/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.cpp b/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.cpp index 1e274058ac4..ac6d58880bf 100644 --- a/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.cpp +++ b/Tests/LinearSolvers/MultiComponent/MCNodalLinOp.cpp @@ -9,348 +9,348 @@ // solids.uccs.edu // // Date: September 3, 2019 -// +// // Description: This file implements the multi-component linear operator // // See also: ./MCNodalLinOp.H (for definitions and documentation of each function) // ./main.cpp (for execution of the tutorial) -// +// #include #include "MCNodalLinOp.H" using namespace amrex; -void MCNodalLinOp::Fapply (int amrlev, int mglev, MultiFab& a_out,const MultiFab& a_in) const +void MCNodalLinOp::Fapply (int amrlev, int mglev, MultiFab& a_out,const MultiFab& a_in) const { - BL_PROFILE("MCNodalLinOp::Fapply()"); - - a_out.setVal(0.0); - amrex::Box domain(m_geom[amrlev][mglev].Domain()); - domain.convert(amrex::IntVect::TheNodeVector()); - domain.grow(-1); // Shrink domain so we don't operate on any boundaries - const Real* DX = m_geom[amrlev][mglev].CellSize(); - - amrex::AsyncArray aa(coeff.data(), coeff.size()); - amrex::Real const* dcoeff = aa.data(); - const int N = getNComp(); - - for (MFIter mfi(a_out, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Box bx = mfi.tilebox(); - bx.grow(1); // Expand to cover first layer of ghost nodes - bx = bx & domain; // Take intersection of box and the problem domain - - amrex::Array4 const& in = a_in.array(mfi); - amrex::Array4 const& out = a_out.array(mfi); - - for (int n = 0; n < getNComp(); n++) - amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { - out(i,j,k,n) = 0.0; - for (int m = 0; m < N; m++) - { - out(i,j,k,n) += dcoeff[ncomp*n + m] * - ( AMREX_D_TERM(- (in(i-1,j,k,m) - 2.0 * in(i,j,k,m) + in(i+1,j,k,m)) / DX[0] / DX[0], - - (in(i,j-1,k,m) - 2.0 * in(i,j,k,m) + in(i,j+1,k,m)) / DX[1] / DX[1], - - (in(i,j,k-1,m) - 2.0 * in(i,j,k,m) + in(i,j,k+1,m)) / DX[2] / DX[2])) ; - } - - }); - } + BL_PROFILE("MCNodalLinOp::Fapply()"); + + a_out.setVal(0.0); + amrex::Box domain(m_geom[amrlev][mglev].Domain()); + domain.convert(amrex::IntVect::TheNodeVector()); + domain.grow(-1); // Shrink domain so we don't operate on any boundaries + const Real* DX = m_geom[amrlev][mglev].CellSize(); + + amrex::AsyncArray aa(coeff.data(), coeff.size()); + amrex::Real const* dcoeff = aa.data(); + const int N = getNComp(); + + for (MFIter mfi(a_out, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Box bx = mfi.tilebox(); + bx.grow(1); // Expand to cover first layer of ghost nodes + bx = bx & domain; // Take intersection of box and the problem domain + + amrex::Array4 const& in = a_in.array(mfi); + amrex::Array4 const& out = a_out.array(mfi); + + for (int n = 0; n < getNComp(); n++) + amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { + out(i,j,k,n) = 0.0; + for (int m = 0; m < N; m++) + { + out(i,j,k,n) += dcoeff[ncomp*n + m] * + ( AMREX_D_TERM(- (in(i-1,j,k,m) - 2.0 * in(i,j,k,m) + in(i+1,j,k,m)) / DX[0] / DX[0], + - (in(i,j-1,k,m) - 2.0 * in(i,j,k,m) + in(i,j+1,k,m)) / DX[1] / DX[1], + - (in(i,j,k-1,m) - 2.0 * in(i,j,k,m) + in(i,j,k+1,m)) / DX[2] / DX[2])) ; + } + + }); + } } -void MCNodalLinOp::Diag (int amrlev, int mglev, MultiFab& a_diag) +void MCNodalLinOp::Diag (int amrlev, int mglev, MultiFab& a_diag) { - a_diag.setVal(1.0); - amrex::Box domain(m_geom[amrlev][mglev].Domain()); - domain.convert(amrex::IntVect::TheNodeVector()); - domain.grow(-1); // Shrink domain so we don't operate on any boundaries - const Real* DX = m_geom[amrlev][mglev].CellSize(); - - amrex::AsyncArray aa(coeff.data(), coeff.size()); - amrex::Real const* dcoeff = aa.data(); - const int N = getNComp(); - - for (MFIter mfi(a_diag, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Box bx = mfi.tilebox(); - bx.grow(1); // Expand to cover first layer of ghost nodes - bx = bx & domain; // Take intersection of box and the problem domain - - amrex::Array4 const& diag = a_diag.array(mfi); - - for (int n = 0; n < getNComp(); n++) - amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { - diag(i,j,k,n) = dcoeff[N*n + n] * - ( AMREX_D_TERM(+ 2.0 / DX[0] / DX[0], - + 2.0 / DX[1] / DX[1], - + 2.0 / DX[2] / DX[2]) ); - }); - } + a_diag.setVal(1.0); + amrex::Box domain(m_geom[amrlev][mglev].Domain()); + domain.convert(amrex::IntVect::TheNodeVector()); + domain.grow(-1); // Shrink domain so we don't operate on any boundaries + const Real* DX = m_geom[amrlev][mglev].CellSize(); + + amrex::AsyncArray aa(coeff.data(), coeff.size()); + amrex::Real const* dcoeff = aa.data(); + const int N = getNComp(); + + for (MFIter mfi(a_diag, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Box bx = mfi.tilebox(); + bx.grow(1); // Expand to cover first layer of ghost nodes + bx = bx & domain; // Take intersection of box and the problem domain + + amrex::Array4 const& diag = a_diag.array(mfi); + + for (int n = 0; n < getNComp(); n++) + amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { + diag(i,j,k,n) = dcoeff[N*n + n] * + ( AMREX_D_TERM(+ 2.0 / DX[0] / DX[0], + + 2.0 / DX[1] / DX[1], + + 2.0 / DX[2] / DX[2]) ); + }); + } } void MCNodalLinOp::Diagonal (bool recompute) { - if ( !recompute && m_diagonal_computed ) return; - m_diagonal_computed = true; - - for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) - { - for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) - { - Diag(amrlev,mglev,*m_diag[amrlev][mglev]); - } - } + if ( !recompute && m_diagonal_computed ) return; + m_diagonal_computed = true; + + for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) + { + for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) + { + Diag(amrlev,mglev,*m_diag[amrlev][mglev]); + } + } } void MCNodalLinOp::Fsmooth (int amrlev, int mglev, amrex::MultiFab& a_x, const amrex::MultiFab& a_b) const { - BL_PROFILE("MCNodalLinOp::Fsmooth()"); - - amrex::Box domain(m_geom[amrlev][mglev].Domain()); - domain.convert(amrex::IntVect::TheNodeVector()); - domain.grow(-1); // Shrink domain so we don't operate on any boundaries - - //int ncomp = getNComp(); - int nghost = getNGrow(); - - Real omega = 2./3.; // Damping factor (very important!) - - MultiFab _Ax(a_x.boxArray(), a_x.DistributionMap(), ncomp, nghost); - MultiFab _Dx(a_x.boxArray(), a_x.DistributionMap(), ncomp, nghost); - MultiFab _Rx(a_x.boxArray(), a_x.DistributionMap(), ncomp, nghost); - - if (!m_diagonal_computed) amrex::Abort("MCNodalLinOp::Diagonal() must be called before using Fsmooth"); - - // This is a JACOBI iteration, not Gauss-Seidel. - // So we need to do twice the number of iterations to get the same behavior as GS. - for (int ctr = 0; ctr < 2; ctr++) - { - Fapply(amrlev,mglev,_Ax,a_x); // find Ax - - amrex::MultiFab::Copy(_Dx,a_x,0,0,ncomp,nghost); // Dx = x - amrex::MultiFab::Multiply(_Dx,*m_diag[amrlev][mglev],0,0,ncomp,nghost); // Dx *= diag (Dx = x*diag) - - amrex::MultiFab::Copy(_Rx,_Ax,0,0,ncomp,nghost); // Rx = Ax - amrex::MultiFab::Subtract(_Rx,_Dx,0,0,ncomp,nghost); // Rx -= Dx (Rx = Ax - Dx) - - - //for (MFIter mfi(a_x, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) - for (MFIter mfi(a_x, false); mfi.isValid(); ++mfi) - { - //Box bx = mfi.tilebox(); - Box bx = mfi.validbox(); - bx.grow(1); // Expand to cover first layer of ghost nodes - bx = bx & domain; // Take intersection of box and the problem domain - - amrex::Array4 const& x = a_x.array(mfi); - - amrex::Array4 const& b = a_b.array(mfi); - amrex::Array4 const& Rx = _Rx.array(mfi); - amrex::Array4 const& diag = (*m_diag[amrlev][mglev]).array(mfi); - - for (int n = 0; n < getNComp(); n++) - amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { - x(i,j,k,n) = (1.-omega)*x(i,j,k,n) + omega*(b(i,j,k,n) - Rx(i,j,k,n))/diag(i,j,k,n); - }); - } - } - amrex::Geometry geom = m_geom[amrlev][mglev]; - a_x.setMultiGhost(true); - a_x.FillBoundary(); - nodalSync(amrlev, mglev, a_x); + BL_PROFILE("MCNodalLinOp::Fsmooth()"); + + amrex::Box domain(m_geom[amrlev][mglev].Domain()); + domain.convert(amrex::IntVect::TheNodeVector()); + domain.grow(-1); // Shrink domain so we don't operate on any boundaries + + //int ncomp = getNComp(); + int nghost = getNGrow(); + + Real omega = 2./3.; // Damping factor (very important!) + + MultiFab _Ax(a_x.boxArray(), a_x.DistributionMap(), ncomp, nghost); + MultiFab _Dx(a_x.boxArray(), a_x.DistributionMap(), ncomp, nghost); + MultiFab _Rx(a_x.boxArray(), a_x.DistributionMap(), ncomp, nghost); + + if (!m_diagonal_computed) amrex::Abort("MCNodalLinOp::Diagonal() must be called before using Fsmooth"); + + // This is a JACOBI iteration, not Gauss-Seidel. + // So we need to do twice the number of iterations to get the same behavior as GS. + for (int ctr = 0; ctr < 2; ctr++) + { + Fapply(amrlev,mglev,_Ax,a_x); // find Ax + + amrex::MultiFab::Copy(_Dx,a_x,0,0,ncomp,nghost); // Dx = x + amrex::MultiFab::Multiply(_Dx,*m_diag[amrlev][mglev],0,0,ncomp,nghost); // Dx *= diag (Dx = x*diag) + + amrex::MultiFab::Copy(_Rx,_Ax,0,0,ncomp,nghost); // Rx = Ax + amrex::MultiFab::Subtract(_Rx,_Dx,0,0,ncomp,nghost); // Rx -= Dx (Rx = Ax - Dx) + + + //for (MFIter mfi(a_x, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(a_x, false); mfi.isValid(); ++mfi) + { + //Box bx = mfi.tilebox(); + Box bx = mfi.validbox(); + bx.grow(1); // Expand to cover first layer of ghost nodes + bx = bx & domain; // Take intersection of box and the problem domain + + amrex::Array4 const& x = a_x.array(mfi); + + amrex::Array4 const& b = a_b.array(mfi); + amrex::Array4 const& Rx = _Rx.array(mfi); + amrex::Array4 const& diag = (*m_diag[amrlev][mglev]).array(mfi); + + for (int n = 0; n < getNComp(); n++) + amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { + x(i,j,k,n) = (1.-omega)*x(i,j,k,n) + omega*(b(i,j,k,n) - Rx(i,j,k,n))/diag(i,j,k,n); + }); + } + } + amrex::Geometry geom = m_geom[amrlev][mglev]; + a_x.setMultiGhost(true); + a_x.FillBoundary(); + nodalSync(amrlev, mglev, a_x); } void MCNodalLinOp::normalize (int amrlev, int mglev, MultiFab& a_x) const { - BL_PROFILE("MCNodalLinOp::normalize()"); - int nghost = 1; - amrex::MultiFab::Divide(a_x,*m_diag[amrlev][mglev],0,0,ncomp,nghost); // Dx *= diag (Dx = x*diag) + BL_PROFILE("MCNodalLinOp::normalize()"); + int nghost = 1; + amrex::MultiFab::Divide(a_x,*m_diag[amrlev][mglev],0,0,ncomp,nghost); // Dx *= diag (Dx = x*diag) } MCNodalLinOp::~MCNodalLinOp () {} void MCNodalLinOp::define (const Vector& a_geom, - const Vector& a_grids, - const Vector& a_dmap, - const LPInfo& a_info, - const Vector const*>& a_factory) + const Vector& a_grids, + const Vector& a_dmap, + const LPInfo& a_info, + const Vector const*>& a_factory) { - BL_PROFILE("MCNodalLinOp::~Operator()"); - - // This makes sure grids are node-centered; - Vector cc_grids = a_grids; - for (auto& ba : cc_grids) { - ba.enclosedCells(); - } - - MLNodeLinOp::define(a_geom, a_grids, a_dmap, a_info, a_factory); - - int nghost = 2; - // Resize the multifab containing the operator diagonal - m_diag.resize(m_num_amr_levels); - for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) - { - m_diag[amrlev].resize(m_num_mg_levels[amrlev]); - - for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) - { - m_diag[amrlev][mglev].reset(new MultiFab(amrex::convert(m_grids[amrlev][mglev], amrex::IntVect::TheNodeVector()), - m_dmap[amrlev][mglev], getNComp(), nghost)); - } - } - - m_lobc.resize(getNComp(),{{AMREX_D_DECL(BCType::Dirichlet,BCType::Dirichlet,BCType::Dirichlet)}}); - m_hibc.resize(getNComp(),{{AMREX_D_DECL(BCType::Dirichlet,BCType::Dirichlet,BCType::Dirichlet)}}); + BL_PROFILE("MCNodalLinOp::~Operator()"); + + // This makes sure grids are node-centered; + Vector cc_grids = a_grids; + for (auto& ba : cc_grids) { + ba.enclosedCells(); + } + + MLNodeLinOp::define(a_geom, a_grids, a_dmap, a_info, a_factory); + + int nghost = 2; + // Resize the multifab containing the operator diagonal + m_diag.resize(m_num_amr_levels); + for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) + { + m_diag[amrlev].resize(m_num_mg_levels[amrlev]); + + for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) + { + m_diag[amrlev][mglev].reset(new MultiFab(amrex::convert(m_grids[amrlev][mglev], amrex::IntVect::TheNodeVector()), + m_dmap[amrlev][mglev], getNComp(), nghost)); + } + } + + m_lobc.resize(getNComp(),{{AMREX_D_DECL(BCType::Dirichlet,BCType::Dirichlet,BCType::Dirichlet)}}); + m_hibc.resize(getNComp(),{{AMREX_D_DECL(BCType::Dirichlet,BCType::Dirichlet,BCType::Dirichlet)}}); } void MCNodalLinOp::buildMasks () { - BL_PROFILE("MCNodalLinOp::buildMasks()"); - if (m_masks_built) return; - - m_masks_built = true; - - m_is_bottom_singular = false; - - { - std::vector< std::pair > isects; - IArrayBox ccfab; - - for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) - { - for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) - { - const Geometry& geom = m_geom[amrlev][mglev]; - const auto& period = geom.periodicity(); - const Box& ccdomain = geom.Domain(); - const std::vector& pshifts = period.shiftIntVect(); - - Box ccdomain_p = ccdomain; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - if (geom.isPeriodic(idim)) { - ccdomain_p.grow(idim, 1); - } - } - - { - auto& dmask = *m_dirichlet_mask[amrlev][mglev]; - const BoxArray& ccba = m_grids[amrlev][mglev]; - - for (MFIter mfi(dmask, MFItInfo().SetDynamic(true)); mfi.isValid(); ++mfi) - { - const Box& ndbx = mfi.validbox(); - const Box& ccbx = amrex::enclosedCells(ndbx); - const Box& ccbxg1 = amrex::grow(ccbx,1); - - ccfab.resize(ccbxg1); - ccfab.setVal(1); - ccfab.setComplement(2,ccdomain_p,0,1); - - for (const auto& iv : pshifts) - { - ccba.intersections(ccbxg1+iv, isects); - for (const auto& is : isects) - { - ccfab.setVal(0, is.second-iv, 0, 1); - } - } - } - } - } - } - } - - for (int amrlev = 0; amrlev < m_num_amr_levels-1; ++amrlev) - { - iMultiFab& cc_mask = *m_cc_fine_mask[amrlev]; - iMultiFab& nd_mask = *m_nd_fine_mask[amrlev]; - LayoutData& has_cf = *m_has_fine_bndry[amrlev]; - const BoxArray& fba = m_grids[amrlev+1][0]; - const BoxArray& cfba = amrex::coarsen(fba, AMRRefRatio(amrlev)); - - const Box& ccdom = m_geom[amrlev][0].Domain(); - - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(AMRRefRatio(amrlev) == 2, "ref_ratio != 0 not supported"); - - cc_mask.setVal(0); // coarse by default - - const std::vector& pshifts = m_geom[amrlev][0].periodicity().shiftIntVect(); - - { - std::vector< std::pair > isects; - - for (MFIter mfi(cc_mask); mfi.isValid(); ++mfi) - { - has_cf[mfi] = 0; - IArrayBox& fab = cc_mask[mfi]; - const Box& bx = fab.box(); - for (const auto& iv : pshifts) - { - cfba.intersections(bx+iv, isects); - for (const auto& is : isects) - { - fab.setVal(1, is.second-iv, 0, 1); - } - if (!isects.empty()) has_cf[mfi] = 1; - } + BL_PROFILE("MCNodalLinOp::buildMasks()"); + if (m_masks_built) return; + + m_masks_built = true; + + m_is_bottom_singular = false; + + { + std::vector< std::pair > isects; + IArrayBox ccfab; + + for (int amrlev = 0; amrlev < m_num_amr_levels; ++amrlev) + { + for (int mglev = 0; mglev < m_num_mg_levels[amrlev]; ++mglev) + { + const Geometry& geom = m_geom[amrlev][mglev]; + const auto& period = geom.periodicity(); + const Box& ccdomain = geom.Domain(); + const std::vector& pshifts = period.shiftIntVect(); + + Box ccdomain_p = ccdomain; + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + if (geom.isPeriodic(idim)) { + ccdomain_p.grow(idim, 1); + } + } + + { + auto& dmask = *m_dirichlet_mask[amrlev][mglev]; + const BoxArray& ccba = m_grids[amrlev][mglev]; + + for (MFIter mfi(dmask, MFItInfo().SetDynamic(true)); mfi.isValid(); ++mfi) + { + const Box& ndbx = mfi.validbox(); + const Box& ccbx = amrex::enclosedCells(ndbx); + const Box& ccbxg1 = amrex::grow(ccbx,1); + + ccfab.resize(ccbxg1); + ccfab.setVal(1); + ccfab.setComplement(2,ccdomain_p,0,1); + + for (const auto& iv : pshifts) + { + ccba.intersections(ccbxg1+iv, isects); + for (const auto& is : isects) + { + ccfab.setVal(0, is.second-iv, 0, 1); + } + } + } + } + } + } + } + + for (int amrlev = 0; amrlev < m_num_amr_levels-1; ++amrlev) + { + iMultiFab& cc_mask = *m_cc_fine_mask[amrlev]; + iMultiFab& nd_mask = *m_nd_fine_mask[amrlev]; + LayoutData& has_cf = *m_has_fine_bndry[amrlev]; + const BoxArray& fba = m_grids[amrlev+1][0]; + const BoxArray& cfba = amrex::coarsen(fba, AMRRefRatio(amrlev)); + + const Box& ccdom = m_geom[amrlev][0].Domain(); + + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(AMRRefRatio(amrlev) == 2, "ref_ratio != 0 not supported"); + + cc_mask.setVal(0); // coarse by default + + const std::vector& pshifts = m_geom[amrlev][0].periodicity().shiftIntVect(); + + { + std::vector< std::pair > isects; + + for (MFIter mfi(cc_mask); mfi.isValid(); ++mfi) + { + has_cf[mfi] = 0; + IArrayBox& fab = cc_mask[mfi]; + const Box& bx = fab.box(); + for (const auto& iv : pshifts) + { + cfba.intersections(bx+iv, isects); + for (const auto& is : isects) + { + fab.setVal(1, is.second-iv, 0, 1); + } + if (!isects.empty()) has_cf[mfi] = 1; + } mlndlap_fillbc_cc(mfi.validbox(), fab.array(), ccdom, LoBC(0), HiBC(0)); - } - } + } + } - for (MFIter mfi(nd_mask,true); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); + for (MFIter mfi(nd_mask,true); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); Array4 const& nmsk = nd_mask.array(mfi); Array4 const& cmsk = cc_mask.const_array(mfi); amrex::ParallelFor(bx, [=] (int i, int j, int k) noexcept { mlndlap_set_nodal_mask(i,j,k,nmsk,cmsk); }); - } - } + } + } - auto& has_cf = *m_has_fine_bndry[m_num_amr_levels-1]; + auto& has_cf = *m_has_fine_bndry[m_num_amr_levels-1]; - for (MFIter mfi(has_cf); mfi.isValid(); ++mfi) - { - has_cf[mfi] = 0; - } + for (MFIter mfi(has_cf); mfi.isValid(); ++mfi) + { + has_cf[mfi] = 0; + } - { - int amrlev = 0; - int mglev = m_num_mg_levels[amrlev]-1; - const iMultiFab& omask = *m_owner_mask[amrlev][mglev]; - m_bottom_dot_mask.define(omask.boxArray(), omask.DistributionMap(), 1, 0); + { + int amrlev = 0; + int mglev = m_num_mg_levels[amrlev]-1; + const iMultiFab& omask = *m_owner_mask[amrlev][mglev]; + m_bottom_dot_mask.define(omask.boxArray(), omask.DistributionMap(), 1, 0); - const Geometry& geom = m_geom[amrlev][mglev]; - Box nddomain = amrex::surroundingNodes(geom.Domain()); + const Geometry& geom = m_geom[amrlev][mglev]; + Box nddomain = amrex::surroundingNodes(geom.Domain()); - for (MFIter mfi(m_bottom_dot_mask,true); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); + for (MFIter mfi(m_bottom_dot_mask,true); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); Array4 const& dfab = m_bottom_dot_mask.array(mfi); Array4 const& sfab = omask.const_array(mfi); mlndlap_set_dot_mask(bx, dfab, sfab, nddomain, LoBC(0), HiBC(0)); - } - } + } + } } void MCNodalLinOp::fixUpResidualMask (int amrlev, iMultiFab& resmsk) { - BL_PROFILE("MCNodalLinOp::fixUpResidualMask()"); + BL_PROFILE("MCNodalLinOp::fixUpResidualMask()"); - if (!m_masks_built) buildMasks(); + if (!m_masks_built) buildMasks(); - const iMultiFab& cfmask = *m_nd_fine_mask[amrlev]; + const iMultiFab& cfmask = *m_nd_fine_mask[amrlev]; #ifdef AMREX_USE_OMP #pragma omp parallel #endif - for (MFIter mfi(resmsk,true); mfi.isValid(); ++mfi) - { + for (MFIter mfi(resmsk,true); mfi.isValid(); ++mfi) + { const Box& bx = mfi.tilebox(); Array4 const& rmsk = resmsk.array(mfi); Array4 const& fmsk = cfmask.const_array(mfi); @@ -358,299 +358,299 @@ void MCNodalLinOp::fixUpResidualMask (int amrlev, iMultiFab& resmsk) { if (fmsk(i,j,k) == crse_fine_node) rmsk(i,j,k) = 1; }); - } + } } void MCNodalLinOp::prepareForSolve () { - BL_PROFILE("MCNodalLinOp::prepareForSolve()"); - MLNodeLinOp::prepareForSolve(); - buildMasks(); - Diagonal(true); + BL_PROFILE("MCNodalLinOp::prepareForSolve()"); + MLNodeLinOp::prepareForSolve(); + buildMasks(); + Diagonal(true); } void MCNodalLinOp::restriction (int amrlev, int cmglev, MultiFab& crse, MultiFab& fine) const { - BL_PROFILE("MCNodalLinOp::restriction()"); + BL_PROFILE("MCNodalLinOp::restriction()"); - applyBC(amrlev, cmglev-1, fine, BCMode::Homogeneous, StateMode::Solution); + applyBC(amrlev, cmglev-1, fine, BCMode::Homogeneous, StateMode::Solution); - amrex::Box cdomain = m_geom[amrlev][cmglev].Domain(); - cdomain.convert(amrex::IntVect::TheNodeVector()); - cdomain.grow(-1); + amrex::Box cdomain = m_geom[amrlev][cmglev].Domain(); + cdomain.convert(amrex::IntVect::TheNodeVector()); + cdomain.grow(-1); - bool need_parallel_copy = !amrex::isMFIterSafe(crse, fine); - MultiFab cfine; - if (need_parallel_copy) { - const BoxArray& ba = amrex::coarsen(fine.boxArray(), 2); - cfine.define(ba, fine.DistributionMap(), fine.nComp(), fine.nGrow()); - } + bool need_parallel_copy = !amrex::isMFIterSafe(crse, fine); + MultiFab cfine; + if (need_parallel_copy) { + const BoxArray& ba = amrex::coarsen(fine.boxArray(), 2); + cfine.define(ba, fine.DistributionMap(), fine.nComp(), fine.nGrow()); + } - MultiFab* pcrse = (need_parallel_copy) ? &cfine : &crse; + MultiFab* pcrse = (need_parallel_copy) ? &cfine : &crse; pcrse->setVal(0.0); - for (MFIter mfi(*pcrse, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Box bx = mfi.tilebox(); - bx = bx & cdomain; - - amrex::Array4 const& fdata = fine.array(mfi); - amrex::Array4 const& cdata = pcrse->array(mfi); - - for (int n = 0; n < crse.nComp(); n++) - { - // I,J,K == coarse coordinates - // i,j,k == fine coordinates - amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int I, int J, int K) { - int i=2*I, j=2*J, k=2*K; -#if AMREX_SPACEDIM == 2 - cdata(I,J,K,n) = - (fdata(i-1,j-1,k,n) + fdata(i-1,j+1,k,n) + fdata(i+1,j-1,k,n) + fdata(i+1,j+1,k,n)) / 16.0 - + - (fdata(i-1,j,k,n) + fdata(i,j-1,k,n) + fdata(i+1,j,k,n) + fdata(i,j+1,k,n)) / 8.0 - + - fdata(i,j,k,n) / 4.0; -#elif AMREX_SPACEDIM == 3 - cdata(I,J,K,n) = - (fdata(i-1,j-1,k-1,n) + fdata(i-1,j-1,k+1,n) + fdata(i-1,j+1,k-1,n) + fdata(i-1,j+1,k+1,n) + - fdata(i+1,j-1,k-1,n) + fdata(i+1,j-1,k+1,n) + fdata(i+1,j+1,k-1,n) + fdata(i+1,j+1,k+1,n)) / 64.0 - + - (fdata(i,j-1,k-1,n) + fdata(i,j-1,k+1,n) + fdata(i,j+1,k-1,n) + fdata(i,j+1,k+1,n) + - fdata(i-1,j,k-1,n) + fdata(i+1,j,k-1,n) + fdata(i-1,j,k+1,n) + fdata(i+1,j,k+1,n) + - fdata(i-1,j-1,k,n) + fdata(i-1,j+1,k,n) + fdata(i+1,j-1,k,n) + fdata(i+1,j+1,k,n)) / 32.0 - + - (fdata(i-1,j,k,n) + fdata(i,j-1,k,n) + fdata(i,j,k-1,n) + - fdata(i+1,j,k,n) + fdata(i,j+1,k,n) + fdata(i,j,k+1,n)) / 16.0 - + - fdata(i,j,k,n) / 8.0; -#endif - }); - } - } - - if (need_parallel_copy) { - crse.ParallelCopy(cfine); - } - - amrex::Geometry geom = m_geom[amrlev][cmglev]; - crse.setMultiGhost(true); - crse.FillBoundary(); - nodalSync(amrlev, cmglev, crse); + for (MFIter mfi(*pcrse, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Box bx = mfi.tilebox(); + bx = bx & cdomain; + + amrex::Array4 const& fdata = fine.array(mfi); + amrex::Array4 const& cdata = pcrse->array(mfi); + + for (int n = 0; n < crse.nComp(); n++) + { + // I,J,K == coarse coordinates + // i,j,k == fine coordinates + amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int I, int J, int K) { + int i=2*I, j=2*J, k=2*K; +#if AMREX_SPACEDIM == 2 + cdata(I,J,K,n) = + (fdata(i-1,j-1,k,n) + fdata(i-1,j+1,k,n) + fdata(i+1,j-1,k,n) + fdata(i+1,j+1,k,n)) / 16.0 + + + (fdata(i-1,j,k,n) + fdata(i,j-1,k,n) + fdata(i+1,j,k,n) + fdata(i,j+1,k,n)) / 8.0 + + + fdata(i,j,k,n) / 4.0; +#elif AMREX_SPACEDIM == 3 + cdata(I,J,K,n) = + (fdata(i-1,j-1,k-1,n) + fdata(i-1,j-1,k+1,n) + fdata(i-1,j+1,k-1,n) + fdata(i-1,j+1,k+1,n) + + fdata(i+1,j-1,k-1,n) + fdata(i+1,j-1,k+1,n) + fdata(i+1,j+1,k-1,n) + fdata(i+1,j+1,k+1,n)) / 64.0 + + + (fdata(i,j-1,k-1,n) + fdata(i,j-1,k+1,n) + fdata(i,j+1,k-1,n) + fdata(i,j+1,k+1,n) + + fdata(i-1,j,k-1,n) + fdata(i+1,j,k-1,n) + fdata(i-1,j,k+1,n) + fdata(i+1,j,k+1,n) + + fdata(i-1,j-1,k,n) + fdata(i-1,j+1,k,n) + fdata(i+1,j-1,k,n) + fdata(i+1,j+1,k,n)) / 32.0 + + + (fdata(i-1,j,k,n) + fdata(i,j-1,k,n) + fdata(i,j,k-1,n) + + fdata(i+1,j,k,n) + fdata(i,j+1,k,n) + fdata(i,j,k+1,n)) / 16.0 + + + fdata(i,j,k,n) / 8.0; +#endif + }); + } + } + + if (need_parallel_copy) { + crse.ParallelCopy(cfine); + } + + amrex::Geometry geom = m_geom[amrlev][cmglev]; + crse.setMultiGhost(true); + crse.FillBoundary(); + nodalSync(amrlev, cmglev, crse); } void MCNodalLinOp::interpolation (int amrlev, int fmglev, MultiFab& fine, const MultiFab& crse) const { - BL_PROFILE("MCNodalLinOp::interpolation()"); - amrex::Box fdomain = m_geom[amrlev][fmglev].Domain(); fdomain.convert(amrex::IntVect::TheNodeVector()); - - bool need_parallel_copy = !amrex::isMFIterSafe(crse, fine); - MultiFab cfine; - const MultiFab* cmf = &crse; - if (need_parallel_copy) { - const BoxArray& ba = amrex::coarsen(fine.boxArray(), 2); - cfine.define(ba, fine.DistributionMap(), crse.nComp(), crse.nGrow()); - cfine.ParallelCopy(crse); - cmf = &cfine; - } - - for (MFIter mfi(fine, false); mfi.isValid(); ++mfi) - { - const Box& fine_bx = mfi.validbox() & fdomain; - const Box& course_bx = amrex::coarsen(fine_bx,2); - const Box& tmpbx = amrex::refine(course_bx,2); - FArrayBox tmpfab; - tmpfab.resize(tmpbx,fine.nComp()); - tmpfab.setVal(0.0); - const amrex::FArrayBox &crsefab = (*cmf)[mfi]; - - amrex::Array4 const& cdata = crsefab.const_array(); - amrex::Array4 const& fdata = tmpfab.array(); - - for (int n = 0; n < crse.nComp(); n++) - { - // I,J,K == coarse coordinates - // i,j,k == fine coordinates - amrex::ParallelFor (fine_bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { - - int I=i/2, J=j/2, K=k/2; - - if (i%2 == 0 && j%2 == 0 && k%2 ==0) // Coincident - fdata(i,j,k,n) = cdata(I,J,K,n); - else if (j%2 == 0 && k%2 == 0) // X Edge - fdata(i,j,k,n) = 0.5 * (cdata(I,J,K,n) + cdata(I+1,J,K,n)); - else if (k%2 == 0 && i%2 == 0) // Y Edge - fdata(i,j,k,n) = 0.5 * (cdata(I,J,K,n) + cdata(I,J+1,K,n)); - else if (i%2 == 0 && j%2 == 0) // Z Edge - fdata(i,j,k,n) = 0.5 * (cdata(I,J,K,n) + cdata(I,J,K+1,n)); - else if (i%2 == 0) // X Face - fdata(i,j,k,n) = 0.25 * (cdata(I,J,K,n) + cdata(I,J+1,K,n) + - cdata(I,J,K+1,n) + cdata(I,J+1,K+1,n)); - else if (j%2 == 0) // Y Face - fdata(i,j,k,n) = 0.25 * (cdata(I,J,K,n) + cdata(I,J,K+1,n) + - cdata(I+1,J,K,n) + cdata(I+1,J,K+1,n)); - else if (k%2 == 0) // Z Face - fdata(i,j,k,n) = 0.25 * (cdata(I,J,K,n) + cdata(I+1,J,K,n) + - cdata(I,J+1,K,n) + cdata(I+1,J+1,K,n)); - else // Center - fdata(i,j,k,n) = 0.125 * (cdata(I,J,K,n) + - cdata(I+1,J,K,n) + cdata(I,J+1,K,n) + cdata(I,J,K+1,n) + - cdata(I,J+1,K+1,n) + cdata(I+1,J,K+1,n) + cdata(I+1,J+1,K,n) + - cdata(I+1,J+1,K+1,n)); - }); - } - fine[mfi].plus(tmpfab,fine_bx,fine_bx,0,0,fine.nComp()); - } - amrex::Geometry geom = m_geom[amrlev][fmglev]; - fine.setMultiGhost(true); - fine.FillBoundary(); - nodalSync(amrlev, fmglev, fine); + BL_PROFILE("MCNodalLinOp::interpolation()"); + amrex::Box fdomain = m_geom[amrlev][fmglev].Domain(); fdomain.convert(amrex::IntVect::TheNodeVector()); + + bool need_parallel_copy = !amrex::isMFIterSafe(crse, fine); + MultiFab cfine; + const MultiFab* cmf = &crse; + if (need_parallel_copy) { + const BoxArray& ba = amrex::coarsen(fine.boxArray(), 2); + cfine.define(ba, fine.DistributionMap(), crse.nComp(), crse.nGrow()); + cfine.ParallelCopy(crse); + cmf = &cfine; + } + + for (MFIter mfi(fine, false); mfi.isValid(); ++mfi) + { + const Box& fine_bx = mfi.validbox() & fdomain; + const Box& course_bx = amrex::coarsen(fine_bx,2); + const Box& tmpbx = amrex::refine(course_bx,2); + FArrayBox tmpfab; + tmpfab.resize(tmpbx,fine.nComp()); + tmpfab.setVal(0.0); + const amrex::FArrayBox &crsefab = (*cmf)[mfi]; + + amrex::Array4 const& cdata = crsefab.const_array(); + amrex::Array4 const& fdata = tmpfab.array(); + + for (int n = 0; n < crse.nComp(); n++) + { + // I,J,K == coarse coordinates + // i,j,k == fine coordinates + amrex::ParallelFor (fine_bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { + + int I=i/2, J=j/2, K=k/2; + + if (i%2 == 0 && j%2 == 0 && k%2 ==0) // Coincident + fdata(i,j,k,n) = cdata(I,J,K,n); + else if (j%2 == 0 && k%2 == 0) // X Edge + fdata(i,j,k,n) = 0.5 * (cdata(I,J,K,n) + cdata(I+1,J,K,n)); + else if (k%2 == 0 && i%2 == 0) // Y Edge + fdata(i,j,k,n) = 0.5 * (cdata(I,J,K,n) + cdata(I,J+1,K,n)); + else if (i%2 == 0 && j%2 == 0) // Z Edge + fdata(i,j,k,n) = 0.5 * (cdata(I,J,K,n) + cdata(I,J,K+1,n)); + else if (i%2 == 0) // X Face + fdata(i,j,k,n) = 0.25 * (cdata(I,J,K,n) + cdata(I,J+1,K,n) + + cdata(I,J,K+1,n) + cdata(I,J+1,K+1,n)); + else if (j%2 == 0) // Y Face + fdata(i,j,k,n) = 0.25 * (cdata(I,J,K,n) + cdata(I,J,K+1,n) + + cdata(I+1,J,K,n) + cdata(I+1,J,K+1,n)); + else if (k%2 == 0) // Z Face + fdata(i,j,k,n) = 0.25 * (cdata(I,J,K,n) + cdata(I+1,J,K,n) + + cdata(I,J+1,K,n) + cdata(I+1,J+1,K,n)); + else // Center + fdata(i,j,k,n) = 0.125 * (cdata(I,J,K,n) + + cdata(I+1,J,K,n) + cdata(I,J+1,K,n) + cdata(I,J,K+1,n) + + cdata(I,J+1,K+1,n) + cdata(I+1,J,K+1,n) + cdata(I+1,J+1,K,n) + + cdata(I+1,J+1,K+1,n)); + }); + } + fine[mfi].plus(tmpfab,fine_bx,fine_bx,0,0,fine.nComp()); + } + amrex::Geometry geom = m_geom[amrlev][fmglev]; + fine.setMultiGhost(true); + fine.FillBoundary(); + nodalSync(amrlev, fmglev, fine); } - + void MCNodalLinOp::averageDownSolutionRHS (int camrlev, MultiFab& crse_sol, MultiFab&, - const MultiFab& fine_sol, const MultiFab&) + const MultiFab& fine_sol, const MultiFab&) { - BL_PROFILE("MCNodalLinOp::averageDownSolutionRHS()"); - const auto& amrrr = AMRRefRatio(camrlev); - amrex::average_down(fine_sol, crse_sol, 0, crse_sol.nComp(), amrrr); - if (isSingular(0)) amrex::Abort("Singular operators not supported!"); + BL_PROFILE("MCNodalLinOp::averageDownSolutionRHS()"); + const auto& amrrr = AMRRefRatio(camrlev); + amrex::average_down(fine_sol, crse_sol, 0, crse_sol.nComp(), amrrr); + if (isSingular(0)) amrex::Abort("Singular operators not supported!"); } void MCNodalLinOp::applyBC (int amrlev, int mglev, MultiFab& phi, BCMode, - amrex::MLLinOp::StateMode , bool skip_fillboundary) const + amrex::MLLinOp::StateMode , bool skip_fillboundary) const { - BL_PROFILE("MCNodalLinOp::applyBC()"); - const Geometry& geom = m_geom[amrlev][mglev]; - if (!skip_fillboundary) {phi.setMultiGhost(true); phi.FillBoundary();} + BL_PROFILE("MCNodalLinOp::applyBC()"); + const Geometry& geom = m_geom[amrlev][mglev]; + if (!skip_fillboundary) {phi.setMultiGhost(true); phi.FillBoundary();} } void MCNodalLinOp::reflux (int crse_amrlev, - MultiFab& res, const MultiFab&, const MultiFab&, - MultiFab& fine_res, MultiFab&, const MultiFab&) const + MultiFab& res, const MultiFab&, const MultiFab&, + MultiFab& fine_res, MultiFab&, const MultiFab&) const { - BL_PROFILE("MCNodalLinOp::reflux()"); - - amrex::Box cdomain(m_geom[crse_amrlev][0].Domain()); - cdomain.convert(amrex::IntVect::TheNodeVector()); - - const Geometry& cgeom = m_geom[crse_amrlev ][0]; - - const BoxArray& fba = fine_res.boxArray(); - const DistributionMapping& fdm = fine_res.DistributionMap(); - - MultiFab fine_res_for_coarse(amrex::coarsen(fba, 2), fdm, ncomp, 2); - fine_res_for_coarse.ParallelCopy(res,0,0,ncomp,0,0,cgeom.periodicity()); - - applyBC(crse_amrlev+1, 0, fine_res, BCMode::Inhomogeneous, StateMode::Solution); - - const int coarse_fine_node = 1; - const int fine_fine_node = 2; - - amrex::iMultiFab nodemask(amrex::coarsen(fba,2), fdm, 1, 2); - nodemask.ParallelCopy(*m_nd_fine_mask[crse_amrlev],0,0,1,0,0,cgeom.periodicity()); - - amrex::iMultiFab cellmask(amrex::convert(amrex::coarsen(fba,2),amrex::IntVect::TheCellVector()), fdm, 1, 2); - cellmask.ParallelCopy(*m_cc_fine_mask[crse_amrlev],0,0,1,1,1,cgeom.periodicity()); - - for (MFIter mfi(fine_res_for_coarse, false); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.validbox(); - - amrex::Array4 const& nmask = nodemask.array(mfi); - amrex::Array4 const& cdata = fine_res_for_coarse.array(mfi); - amrex::Array4 const& fdata = fine_res.array(mfi); - - const Dim3 lo= amrex::lbound(cdomain), hi = amrex::ubound(cdomain); - - for (int n = 0; n < fine_res.nComp(); n++) - { - // I,J,K == coarse coordinates - // i,j,k == fine coordinates - amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int I, int J, int K) { - int i=I*2, j=J*2, k=K*2; - - if (nmask(I,J,K) == fine_fine_node || nmask(I,J,K) == coarse_fine_node) - { - if ((I == lo.x || I == hi.x) && - (J == lo.y || J == hi.y) && - (K == lo.z || K == hi.z)) // Corner - cdata(I,J,K,n) = fdata(i,j,k,n); - else if ((J == lo.y || J == hi.y) && - (K == lo.z || K == hi.z)) // X edge - cdata(I,J,K,n) = 0.25*fdata(i-1,j,k,n) + 0.5*fdata(i,j,k,n) + 0.25*fdata(i+1,j,k,n); - else if ((K == lo.z || K == hi.z) && - (I == lo.x || I == hi.x)) // Y edge - cdata(I,J,K,n) = 0.25*fdata(i,j-1,k,n) + 0.5*fdata(i,j,k,n) + 0.25*fdata(i,j+1,k,n); - else if ((I == lo.x || I == hi.x) && - (J == lo.y || J == hi.y)) // Z edge - cdata(I,J,K,n) = 0.25*fdata(i,j,k-1,n) + 0.5*fdata(i,j,k,n) + 0.25*fdata(i,j,k+1,n); - else if (I == lo.x || I == hi.x) // X face - cdata(I,J,K,n) = - (+ fdata(i,j-1,k-1,n) + 2.0*fdata(i,j,k-1,n) + fdata(i,j+1,k-1,n) - + 2.0*fdata(i,j-1,k ,n) + 4.0*fdata(i,j,k ,n) + 2.0*fdata(i,j+1,k ,n) - + fdata(i,j-1,k+1,n) + 2.0*fdata(i,j,k+1,n) + fdata(i,j+1,k+1,n))/16.0; - else if (J == lo.y || J == hi.y) // Y face - cdata(I,J,K,n) = - (+ fdata(i-1,j,k-1,n) + 2.0*fdata(i-1,j,k,n) + fdata(i-1,j,k+1,n) - + 2.0*fdata(i ,j,k-1,n) + 4.0*fdata(i ,j,k,n) + 2.0*fdata(i ,j,k+1,n) - + fdata(i+1,j,k-1,n) + 2.0*fdata(i+1,j,k,n) + fdata(i+1,j,k+1,n))/16.0; - else if (K == lo.z || K == hi.z) // Z face - cdata(I,J,K,n) = - (+ fdata(i-1,j-1,k,n) + 2.0*fdata(i,j-1,k,n) + fdata(i+1,j-1,k,n) - + 2.0*fdata(i-1,j ,k,n) + 4.0*fdata(i,j ,k,n) + 2.0*fdata(i+1,j ,k,n) - + fdata(i-1,j+1,k,n) + 2.0*fdata(i,j+1,k,n) + fdata(i+1,j+1,k,n))/16.0; - else // Interior - cdata(I,J,K,n) = - (fdata(i-1,j-1,k-1,n) + fdata(i-1,j-1,k+1,n) + fdata(i-1,j+1,k-1,n) + fdata(i-1,j+1,k+1,n) + - fdata(i+1,j-1,k-1,n) + fdata(i+1,j-1,k+1,n) + fdata(i+1,j+1,k-1,n) + fdata(i+1,j+1,k+1,n)) / 64.0 - + - (fdata(i,j-1,k-1,n) + fdata(i,j-1,k+1,n) + fdata(i,j+1,k-1,n) + fdata(i,j+1,k+1,n) + - fdata(i-1,j,k-1,n) + fdata(i+1,j,k-1,n) + fdata(i-1,j,k+1,n) + fdata(i+1,j,k+1,n) + - fdata(i-1,j-1,k,n) + fdata(i-1,j+1,k,n) + fdata(i+1,j-1,k,n) + fdata(i+1,j+1,k,n)) / 32.0 - + - (fdata(i-1,j,k,n) + fdata(i,j-1,k,n) + fdata(i,j,k-1,n) + - fdata(i+1,j,k,n) + fdata(i,j+1,k,n) + fdata(i,j,k+1,n)) / 16.0 - + - fdata(i,j,k,n) / 8.0; - } - - }); - } - } - - // Copy the fine residual restricted onto the coarse grid - // into the final residual. - res.ParallelCopy(fine_res_for_coarse,0,0,ncomp,0,0,cgeom.periodicity()); - - const int mglev = 0; - - // Sync up ghost nodes - amrex::Geometry geom = m_geom[crse_amrlev][mglev]; - res.setMultiGhost(true); - res.FillBoundary(); - nodalSync(crse_amrlev,mglev, res); + BL_PROFILE("MCNodalLinOp::reflux()"); + + amrex::Box cdomain(m_geom[crse_amrlev][0].Domain()); + cdomain.convert(amrex::IntVect::TheNodeVector()); + + const Geometry& cgeom = m_geom[crse_amrlev ][0]; + + const BoxArray& fba = fine_res.boxArray(); + const DistributionMapping& fdm = fine_res.DistributionMap(); + + MultiFab fine_res_for_coarse(amrex::coarsen(fba, 2), fdm, ncomp, 2); + fine_res_for_coarse.ParallelCopy(res,0,0,ncomp,0,0,cgeom.periodicity()); + + applyBC(crse_amrlev+1, 0, fine_res, BCMode::Inhomogeneous, StateMode::Solution); + + const int coarse_fine_node = 1; + const int fine_fine_node = 2; + + amrex::iMultiFab nodemask(amrex::coarsen(fba,2), fdm, 1, 2); + nodemask.ParallelCopy(*m_nd_fine_mask[crse_amrlev],0,0,1,0,0,cgeom.periodicity()); + + amrex::iMultiFab cellmask(amrex::convert(amrex::coarsen(fba,2),amrex::IntVect::TheCellVector()), fdm, 1, 2); + cellmask.ParallelCopy(*m_cc_fine_mask[crse_amrlev],0,0,1,1,1,cgeom.periodicity()); + + for (MFIter mfi(fine_res_for_coarse, false); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.validbox(); + + amrex::Array4 const& nmask = nodemask.array(mfi); + amrex::Array4 const& cdata = fine_res_for_coarse.array(mfi); + amrex::Array4 const& fdata = fine_res.array(mfi); + + const Dim3 lo= amrex::lbound(cdomain), hi = amrex::ubound(cdomain); + + for (int n = 0; n < fine_res.nComp(); n++) + { + // I,J,K == coarse coordinates + // i,j,k == fine coordinates + amrex::ParallelFor (bx,[=] AMREX_GPU_DEVICE(int I, int J, int K) { + int i=I*2, j=J*2, k=K*2; + + if (nmask(I,J,K) == fine_fine_node || nmask(I,J,K) == coarse_fine_node) + { + if ((I == lo.x || I == hi.x) && + (J == lo.y || J == hi.y) && + (K == lo.z || K == hi.z)) // Corner + cdata(I,J,K,n) = fdata(i,j,k,n); + else if ((J == lo.y || J == hi.y) && + (K == lo.z || K == hi.z)) // X edge + cdata(I,J,K,n) = 0.25*fdata(i-1,j,k,n) + 0.5*fdata(i,j,k,n) + 0.25*fdata(i+1,j,k,n); + else if ((K == lo.z || K == hi.z) && + (I == lo.x || I == hi.x)) // Y edge + cdata(I,J,K,n) = 0.25*fdata(i,j-1,k,n) + 0.5*fdata(i,j,k,n) + 0.25*fdata(i,j+1,k,n); + else if ((I == lo.x || I == hi.x) && + (J == lo.y || J == hi.y)) // Z edge + cdata(I,J,K,n) = 0.25*fdata(i,j,k-1,n) + 0.5*fdata(i,j,k,n) + 0.25*fdata(i,j,k+1,n); + else if (I == lo.x || I == hi.x) // X face + cdata(I,J,K,n) = + (+ fdata(i,j-1,k-1,n) + 2.0*fdata(i,j,k-1,n) + fdata(i,j+1,k-1,n) + + 2.0*fdata(i,j-1,k ,n) + 4.0*fdata(i,j,k ,n) + 2.0*fdata(i,j+1,k ,n) + + fdata(i,j-1,k+1,n) + 2.0*fdata(i,j,k+1,n) + fdata(i,j+1,k+1,n))/16.0; + else if (J == lo.y || J == hi.y) // Y face + cdata(I,J,K,n) = + (+ fdata(i-1,j,k-1,n) + 2.0*fdata(i-1,j,k,n) + fdata(i-1,j,k+1,n) + + 2.0*fdata(i ,j,k-1,n) + 4.0*fdata(i ,j,k,n) + 2.0*fdata(i ,j,k+1,n) + + fdata(i+1,j,k-1,n) + 2.0*fdata(i+1,j,k,n) + fdata(i+1,j,k+1,n))/16.0; + else if (K == lo.z || K == hi.z) // Z face + cdata(I,J,K,n) = + (+ fdata(i-1,j-1,k,n) + 2.0*fdata(i,j-1,k,n) + fdata(i+1,j-1,k,n) + + 2.0*fdata(i-1,j ,k,n) + 4.0*fdata(i,j ,k,n) + 2.0*fdata(i+1,j ,k,n) + + fdata(i-1,j+1,k,n) + 2.0*fdata(i,j+1,k,n) + fdata(i+1,j+1,k,n))/16.0; + else // Interior + cdata(I,J,K,n) = + (fdata(i-1,j-1,k-1,n) + fdata(i-1,j-1,k+1,n) + fdata(i-1,j+1,k-1,n) + fdata(i-1,j+1,k+1,n) + + fdata(i+1,j-1,k-1,n) + fdata(i+1,j-1,k+1,n) + fdata(i+1,j+1,k-1,n) + fdata(i+1,j+1,k+1,n)) / 64.0 + + + (fdata(i,j-1,k-1,n) + fdata(i,j-1,k+1,n) + fdata(i,j+1,k-1,n) + fdata(i,j+1,k+1,n) + + fdata(i-1,j,k-1,n) + fdata(i+1,j,k-1,n) + fdata(i-1,j,k+1,n) + fdata(i+1,j,k+1,n) + + fdata(i-1,j-1,k,n) + fdata(i-1,j+1,k,n) + fdata(i+1,j-1,k,n) + fdata(i+1,j+1,k,n)) / 32.0 + + + (fdata(i-1,j,k,n) + fdata(i,j-1,k,n) + fdata(i,j,k-1,n) + + fdata(i+1,j,k,n) + fdata(i,j+1,k,n) + fdata(i,j,k+1,n)) / 16.0 + + + fdata(i,j,k,n) / 8.0; + } + + }); + } + } + + // Copy the fine residual restricted onto the coarse grid + // into the final residual. + res.ParallelCopy(fine_res_for_coarse,0,0,ncomp,0,0,cgeom.periodicity()); + + const int mglev = 0; + + // Sync up ghost nodes + amrex::Geometry geom = m_geom[crse_amrlev][mglev]; + res.setMultiGhost(true); + res.FillBoundary(); + nodalSync(crse_amrlev,mglev, res); } void MCNodalLinOp::solutionResidual (int amrlev, MultiFab& resid, MultiFab& x, const MultiFab& b, - const MultiFab*) + const MultiFab*) { - const int mglev = 0; - apply(amrlev, mglev, resid, x, BCMode::Inhomogeneous, StateMode::Solution); - MultiFab::Xpay(resid, -1.0, b, 0, 0, ncomp, 2); - amrex::Geometry geom = m_geom[amrlev][mglev]; - resid.setMultiGhost(true); - resid.FillBoundary(); + const int mglev = 0; + apply(amrlev, mglev, resid, x, BCMode::Inhomogeneous, StateMode::Solution); + MultiFab::Xpay(resid, -1.0, b, 0, 0, ncomp, 2); + amrex::Geometry geom = m_geom[amrlev][mglev]; + resid.setMultiGhost(true); + resid.FillBoundary(); } void MCNodalLinOp::correctionResidual (int amrlev, int mglev, MultiFab& resid, MultiFab& x, const MultiFab& b, - BCMode, const MultiFab* ) + BCMode, const MultiFab* ) { - resid.setVal(0.0); - apply(amrlev, mglev, resid, x, BCMode::Homogeneous, StateMode::Correction); - MultiFab::Xpay(resid, -1.0, b, 0, 0, ncomp, resid.nGrow()); - amrex::Geometry geom = m_geom[amrlev][mglev]; - resid.setMultiGhost(true); - resid.FillBoundary(); + resid.setVal(0.0); + apply(amrlev, mglev, resid, x, BCMode::Homogeneous, StateMode::Correction); + MultiFab::Xpay(resid, -1.0, b, 0, 0, ncomp, resid.nGrow()); + amrex::Geometry geom = m_geom[amrlev][mglev]; + resid.setMultiGhost(true); + resid.FillBoundary(); } diff --git a/Tests/LinearSolvers/MultiComponent/main.cpp b/Tests/LinearSolvers/MultiComponent/main.cpp index 34b3a29f7f2..1c9854187a5 100644 --- a/Tests/LinearSolvers/MultiComponent/main.cpp +++ b/Tests/LinearSolvers/MultiComponent/main.cpp @@ -9,15 +9,15 @@ // solids.uccs.edu // // Date: September 3, 2019 -// +// // Description: This tutorial demonstrates how to implement a multi-component // nodal linear operator. This tutorial demonstrates the // "CFStrategy::ghostnodes" method for computing the reflux at -// the coarse/fine boundary. +// the coarse/fine boundary. // // See also: ./MCNodalLinOp.H, ./MCNodalLinOp.cpp // for implementation of the MC linear operator. -// +// #include #include @@ -33,200 +33,200 @@ int main (int argc, char* argv[]) { Initialize(argc, argv); { - - // - // Read in mesh structure (# amr levels, # nodes) - // Geometry will be a nnodes x nnodes (x nnodes) grid - // refined in the center up to nlevels. - // - struct { - int nlevels = 3; - int nnodes = 32; - int max_grid_size = 10000000; - } mesh; - { - ParmParse pp("mesh"); - pp.query("nlevels",mesh.nlevels); - pp.query("nnodes",mesh.nnodes); - pp.query("max_grid_size",mesh.max_grid_size); - } - - // - // Read in linear operator parameters: - // ncomp = number of components for a MC solve - // coeff = ncomp x ncomp list of coefficients - // - struct { - int ncomp=1; - Vector coeff = {1.0}; - } op; - { - ParmParse pp("op"); - pp.query("ncomp",op.ncomp); - pp.queryarr("coeff",op.coeff); - } - // - // Read in MLMG solver parameters - // - struct { - int verbose = -1; - int bottom_verbose = -1; - int max_iter = -1; - int fixed_iter = -1; - int max_fmg_iter = -1; - int linop_maxorder = -1; - int agglomeration = -1; - int consolidation = -1; - int max_coarsening_level = -1 ; - } mlmg; - { - ParmParse pp("mlmg"); - pp.query("verbose",mlmg.verbose); - pp.query("bottom_verbose",mlmg.bottom_verbose ); - pp.query("max_iter",mlmg.max_iter); - pp.query("max_fmg_iter",mlmg.max_fmg_iter); - pp.query("agglomeration",mlmg.agglomeration); - pp.query("consolidation",mlmg.consolidation); - pp.query("max_coarsening_level",mlmg.max_coarsening_level); - pp.query("fixed_iter",mlmg.fixed_iter); - } - - - // - // Initialize geometry and grids - // - Vector geom; - Vector cgrids, ngrids; - Vector dmap; - Vector solution, rhs; - geom.resize(mesh.nlevels); - cgrids.resize(mesh.nlevels); - ngrids.resize(mesh.nlevels); - dmap.resize(mesh.nlevels); - solution.resize(mesh.nlevels); - rhs.resize(mesh.nlevels); - RealBox rb({AMREX_D_DECL(-0.5,-0.5,-0.5)}, - {AMREX_D_DECL(0.5,0.5,0.5)}); - Geometry::Setup(&rb, 0); - Box NDomain(IntVect{AMREX_D_DECL(0,0,0)}, - IntVect{AMREX_D_DECL(mesh.nnodes,mesh.nnodes,mesh.nnodes)}, - IntVect::TheNodeVector()); - Box CDomain = convert(NDomain, IntVect::TheCellVector()); - - // - // Refine the grid - // - Box domain = CDomain; - for (int ilev = 0; ilev < mesh.nlevels; ++ilev) - { - geom[ilev].define(domain); - domain.refine(2); - } - Box cdomain = CDomain; - for (int ilev = 0; ilev < mesh.nlevels; ++ilev) - { - cgrids[ilev].define(cdomain); - cgrids[ilev].maxSize(mesh.max_grid_size); // TODO - cdomain.grow(-mesh.nnodes/4); - cdomain.refine(2); - ngrids[ilev] = cgrids[ilev]; - ngrids[ilev].convert(IntVect::TheNodeVector()); - } - - // - // Initialize the solution and rhs fabs. - // Initialize the RHS fab to the function: - // RHS[0] = x1*(1-x1) * x2(1-x2) * x3(1-x3) - // RHS[1] = 0 - // RHS[2] = 0 ... etc - // - int nghost = 2; - for (int ilev = 0; ilev < mesh.nlevels; ++ilev) - { - dmap [ilev].define(cgrids[ilev]); - solution[ilev].define(ngrids[ilev], dmap[ilev], op.ncomp, nghost); - solution[ilev].setVal(0.0); - solution[ilev].setMultiGhost(true); - rhs [ilev].define(ngrids[ilev], dmap[ilev], op.ncomp, nghost); - rhs [ilev].setVal(0.0); - rhs [ilev].setMultiGhost(true); - - Box domain(geom[ilev].Domain()); - const Real AMREX_D_DECL( dx = geom[ilev].CellSize()[0], - dy = geom[ilev].CellSize()[1], - dz = geom[ilev].CellSize()[2]); - const Real AMREX_D_DECL( minx = geom[ilev].ProbLo()[0], - miny = geom[ilev].ProbLo()[1], - minz = geom[ilev].ProbLo()[2]); - domain.convert(IntVect::TheNodeVector()); - domain.grow(-1); // Shrink domain so we don't operate on any boundaries - for (MFIter mfi(solution[ilev], TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Box bx = mfi.tilebox(); - bx.grow(1); // Expand to cover first layer of ghost nodes - bx = bx & domain; // Take intersection of box and the problem domain - - Array4 const& RHS = rhs[ilev].array(mfi); - for (int n = 0; n < op.ncomp; n++) - ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { - - Real AMREX_D_DECL(x1 = i*dx + minx, - x2 = j*dy + miny, - x3 = k*dz + minz); - - if (n==0) RHS(i,j,k,n) = AMREX_D_TERM( (x1-0.5)*(x1+0.5), - * (x2-0.5)*(x2+0.5), - * (x3-0.5)*(x3+0.5)); - else RHS(i,j,k,n) = 0.0; - }); - } - rhs[ilev].FillBoundary(false,true); - } - - // - // Set params to be passed to MLMG solver - // - LPInfo info; - if (mlmg.agglomeration >= 0) info.setAgglomeration(mlmg.agglomeration); - if (mlmg.consolidation >= 0) info.setConsolidation(mlmg.consolidation); - if (mlmg.max_coarsening_level >= 0) info.setMaxCoarseningLevel(mlmg.max_coarsening_level); - - // - // Initialize the MCNodalLinOp linear operator - // (see ./MCNodalLinOp.cpp, ./MCNodalLinOp.H for implementation) - // - MCNodalLinOp linop; - linop.setNComp(op.ncomp); - linop.setCoeff(op.coeff); - linop.define(geom,cgrids,dmap,info); - linop.setDomainBC({AMREX_D_DECL(amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet)}, - {AMREX_D_DECL(amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet)}); - for (int ilev = 0; ilev < mesh.nlevels; ++ilev) linop.setLevelBC(ilev,&solution[ilev]); - - // - // Initialize the MLMG solver - // - MLMG solver(linop); - if (mlmg.verbose >= 0) solver.setVerbose(mlmg.verbose); - if (mlmg.bottom_verbose >= 0) solver.setBottomVerbose(mlmg.bottom_verbose); - if (mlmg.fixed_iter >= 0) solver.setFixedIter(mlmg.fixed_iter); - if (mlmg.max_iter >= 0) solver.setMaxIter(mlmg.max_iter); - if (mlmg.max_fmg_iter >= 0)solver.setMaxFmgIter(mlmg.max_fmg_iter); - // IMPORTANT! Use the "CFStrategy::ghostnodes" strategy to avoid - // having to implement a complicated "reflux" routine! - solver.setCFStrategy(MLMG::CFStrategy::ghostnodes); - - // - // Perform the solve - // - Real tol_rel = 1E-8, tol_abs = 1E-8; - solver.solve(GetVecOfPtrs(solution),GetVecOfConstPtrs(rhs),tol_rel,tol_abs); - - // - // Write the output to ./solution - // - WriteMLMF ("solution",GetVecOfConstPtrs(solution),geom); + // + // Read in mesh structure (# amr levels, # nodes) + // Geometry will be a nnodes x nnodes (x nnodes) grid + // refined in the center up to nlevels. + // + struct { + int nlevels = 3; + int nnodes = 32; + int max_grid_size = 10000000; + } mesh; + { + ParmParse pp("mesh"); + pp.query("nlevels",mesh.nlevels); + pp.query("nnodes",mesh.nnodes); + pp.query("max_grid_size",mesh.max_grid_size); + } + + // + // Read in linear operator parameters: + // ncomp = number of components for a MC solve + // coeff = ncomp x ncomp list of coefficients + // + struct { + int ncomp=1; + Vector coeff = {1.0}; + } op; + { + ParmParse pp("op"); + pp.query("ncomp",op.ncomp); + pp.queryarr("coeff",op.coeff); + } + + // + // Read in MLMG solver parameters + // + struct { + int verbose = -1; + int bottom_verbose = -1; + int max_iter = -1; + int fixed_iter = -1; + int max_fmg_iter = -1; + int linop_maxorder = -1; + int agglomeration = -1; + int consolidation = -1; + int max_coarsening_level = -1 ; + } mlmg; + { + ParmParse pp("mlmg"); + pp.query("verbose",mlmg.verbose); + pp.query("bottom_verbose",mlmg.bottom_verbose ); + pp.query("max_iter",mlmg.max_iter); + pp.query("max_fmg_iter",mlmg.max_fmg_iter); + pp.query("agglomeration",mlmg.agglomeration); + pp.query("consolidation",mlmg.consolidation); + pp.query("max_coarsening_level",mlmg.max_coarsening_level); + pp.query("fixed_iter",mlmg.fixed_iter); + } + + + // + // Initialize geometry and grids + // + Vector geom; + Vector cgrids, ngrids; + Vector dmap; + Vector solution, rhs; + geom.resize(mesh.nlevels); + cgrids.resize(mesh.nlevels); + ngrids.resize(mesh.nlevels); + dmap.resize(mesh.nlevels); + solution.resize(mesh.nlevels); + rhs.resize(mesh.nlevels); + RealBox rb({AMREX_D_DECL(-0.5,-0.5,-0.5)}, + {AMREX_D_DECL(0.5,0.5,0.5)}); + Geometry::Setup(&rb, 0); + Box NDomain(IntVect{AMREX_D_DECL(0,0,0)}, + IntVect{AMREX_D_DECL(mesh.nnodes,mesh.nnodes,mesh.nnodes)}, + IntVect::TheNodeVector()); + Box CDomain = convert(NDomain, IntVect::TheCellVector()); + + // + // Refine the grid + // + Box domain = CDomain; + for (int ilev = 0; ilev < mesh.nlevels; ++ilev) + { + geom[ilev].define(domain); + domain.refine(2); + } + Box cdomain = CDomain; + for (int ilev = 0; ilev < mesh.nlevels; ++ilev) + { + cgrids[ilev].define(cdomain); + cgrids[ilev].maxSize(mesh.max_grid_size); // TODO + cdomain.grow(-mesh.nnodes/4); + cdomain.refine(2); + ngrids[ilev] = cgrids[ilev]; + ngrids[ilev].convert(IntVect::TheNodeVector()); + } + + // + // Initialize the solution and rhs fabs. + // Initialize the RHS fab to the function: + // RHS[0] = x1*(1-x1) * x2(1-x2) * x3(1-x3) + // RHS[1] = 0 + // RHS[2] = 0 ... etc + // + int nghost = 2; + for (int ilev = 0; ilev < mesh.nlevels; ++ilev) + { + dmap [ilev].define(cgrids[ilev]); + solution[ilev].define(ngrids[ilev], dmap[ilev], op.ncomp, nghost); + solution[ilev].setVal(0.0); + solution[ilev].setMultiGhost(true); + rhs [ilev].define(ngrids[ilev], dmap[ilev], op.ncomp, nghost); + rhs [ilev].setVal(0.0); + rhs [ilev].setMultiGhost(true); + + Box domain(geom[ilev].Domain()); + const Real AMREX_D_DECL( dx = geom[ilev].CellSize()[0], + dy = geom[ilev].CellSize()[1], + dz = geom[ilev].CellSize()[2]); + const Real AMREX_D_DECL( minx = geom[ilev].ProbLo()[0], + miny = geom[ilev].ProbLo()[1], + minz = geom[ilev].ProbLo()[2]); + domain.convert(IntVect::TheNodeVector()); + domain.grow(-1); // Shrink domain so we don't operate on any boundaries + for (MFIter mfi(solution[ilev], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Box bx = mfi.tilebox(); + bx.grow(1); // Expand to cover first layer of ghost nodes + bx = bx & domain; // Take intersection of box and the problem domain + + Array4 const& RHS = rhs[ilev].array(mfi); + for (int n = 0; n < op.ncomp; n++) + ParallelFor (bx,[=] AMREX_GPU_DEVICE(int i, int j, int k) { + + Real AMREX_D_DECL(x1 = i*dx + minx, + x2 = j*dy + miny, + x3 = k*dz + minz); + + if (n==0) RHS(i,j,k,n) = AMREX_D_TERM( (x1-0.5)*(x1+0.5), + * (x2-0.5)*(x2+0.5), + * (x3-0.5)*(x3+0.5)); + else RHS(i,j,k,n) = 0.0; + }); + } + rhs[ilev].FillBoundary(false,true); + } + + // + // Set params to be passed to MLMG solver + // + LPInfo info; + if (mlmg.agglomeration >= 0) info.setAgglomeration(mlmg.agglomeration); + if (mlmg.consolidation >= 0) info.setConsolidation(mlmg.consolidation); + if (mlmg.max_coarsening_level >= 0) info.setMaxCoarseningLevel(mlmg.max_coarsening_level); + + // + // Initialize the MCNodalLinOp linear operator + // (see ./MCNodalLinOp.cpp, ./MCNodalLinOp.H for implementation) + // + MCNodalLinOp linop; + linop.setNComp(op.ncomp); + linop.setCoeff(op.coeff); + linop.define(geom,cgrids,dmap,info); + linop.setDomainBC({AMREX_D_DECL(amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet)}, + {AMREX_D_DECL(amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet,amrex::MLLinOp::BCType::Dirichlet)}); + for (int ilev = 0; ilev < mesh.nlevels; ++ilev) linop.setLevelBC(ilev,&solution[ilev]); + + // + // Initialize the MLMG solver + // + MLMG solver(linop); + if (mlmg.verbose >= 0) solver.setVerbose(mlmg.verbose); + if (mlmg.bottom_verbose >= 0) solver.setBottomVerbose(mlmg.bottom_verbose); + if (mlmg.fixed_iter >= 0) solver.setFixedIter(mlmg.fixed_iter); + if (mlmg.max_iter >= 0) solver.setMaxIter(mlmg.max_iter); + if (mlmg.max_fmg_iter >= 0)solver.setMaxFmgIter(mlmg.max_fmg_iter); + // IMPORTANT! Use the "CFStrategy::ghostnodes" strategy to avoid + // having to implement a complicated "reflux" routine! + solver.setCFStrategy(MLMG::CFStrategy::ghostnodes); + + // + // Perform the solve + // + Real tol_rel = 1E-8, tol_abs = 1E-8; + solver.solve(GetVecOfPtrs(solution),GetVecOfConstPtrs(rhs),tol_rel,tol_abs); + + // + // Write the output to ./solution + // + WriteMLMF ("solution",GetVecOfConstPtrs(solution),geom); } Finalize(); diff --git a/Tests/LinearSolvers/Nodal_Projection_EB/main.cpp b/Tests/LinearSolvers/Nodal_Projection_EB/main.cpp index 2acf1f7b66c..b1aa6c001de 100644 --- a/Tests/LinearSolvers/Nodal_Projection_EB/main.cpp +++ b/Tests/LinearSolvers/Nodal_Projection_EB/main.cpp @@ -15,8 +15,8 @@ void write_plotfile(const Geometry& geom, const MultiFab& plotmf) { std::string plotfile_name("plt00000"); - amrex::Print() << "Writing " << plotfile_name << std::endl; - + amrex::Print() << "Writing " << plotfile_name << std::endl; + #if (AMREX_SPACEDIM == 2) EB_WriteSingleLevelPlotfile(plotfile_name, plotmf, { "proc" ,"xvel", "yvel" }, @@ -36,7 +36,7 @@ int main (int argc, char* argv[]) amrex::Initialize(argc, argv); auto strt_time = amrex::second(); - + { int mg_verbose = 0; int bottom_verbose = 0; @@ -57,7 +57,7 @@ int main (int argc, char* argv[]) } #ifndef AMREX_USE_HYPRE - if (use_hypre == 1) + if (use_hypre == 1) amrex::Abort("Cant use hypre if we dont build with USE_HYPRE=TRUE"); #endif @@ -105,7 +105,7 @@ int main (int argc, char* argv[]) int direction = 2; Real height = -1.0; // Putting a negative number for height means it extends beyond the domain - // The "false" below is the boolean that determines if the fluid is inside ("true") or + // The "false" below is the boolean that determines if the fluid is inside ("true") or // outside ("false") the object(s) Array obstacles{ @@ -125,16 +125,16 @@ int main (int argc, char* argv[]) auto all = EB2::makeUnion(group_1,group_2,group_3); auto gshop9 = EB2::makeShop(all); EB2::Build(gshop9, geom, required_coarsening_level, max_coarsening_level); - + const EB2::IndexSpace& eb_is = EB2::IndexSpace::top(); const EB2::Level& eb_level = eb_is.getLevel(geom); - + // options are basic, volume, or full EBSupport ebs = EBSupport::full; - + // number of ghost cells for each of the 3 EBSupport types Vector ng_ebs = {2,2,2}; - + // This object provides access to the EB database in the format of basic AMReX objects // such as BaseFab, FArrayBox, FabArray, and MultiFab EBFArrayBoxFactory factory(eb_level, geom, grids, dmap, ng_ebs, ebs); @@ -149,10 +149,10 @@ int main (int argc, char* argv[]) // and then perform the projection: // // vel = vel - sigma * grad(phi) - // + // // - // Create the cell-centered velocity field we want to project + // Create the cell-centered velocity field we want to project // MultiFab vel(grids, dmap, AMREX_SPACEDIM, 1, MFInfo(), factory); @@ -162,7 +162,7 @@ int main (int argc, char* argv[]) // // Setup linear operator, AKA the nodal Laplacian - // + // LPInfo lp_info; // If we want to use hypre to solve the full problem we do not need to coarsen the GMG stencils @@ -189,115 +189,115 @@ int main (int argc, char* argv[]) matrix.setHarmonicAverage(false); // - // Compute RHS + // Compute RHS // - // NOTE: it's up to the user to compute the RHS. as opposed to the MAC projection + // NOTE: it's up to the user to compute the RHS. as opposed to the MAC projection // // NOTE: do this operation AFTER setting up the linear operator so // that compRHS method can be used - // + // // RHS is nodal const BoxArray & nd_grids = amrex::convert(grids, IntVect::TheNodeVector()); // nodal grids - + // Multifab to host RHS MultiFab rhs(nd_grids, dmap, 1, 1, MFInfo(), factory); - + // Cell-centered contributions to RHS MultiFab S_cc(grids, dmap, 1, 1, MFInfo(), factory); S_cc.setVal(0.0); // Set it to zero for this example - + // Node-centered contributions to RHS MultiFab S_nd(nd_grids, dmap, 1, 1, MFInfo(), factory); - S_nd.setVal(0.0); // Set it to zero for this example - + S_nd.setVal(0.0); // Set it to zero for this example + // Compute RHS -- vel must be cell-centered matrix.compRHS({&rhs}, {&vel}, {&S_nd}, {&S_cc}); - + // // Create the cell-centered sigma field and set it to 1 for this example // MultiFab sigma(grids, dmap, 1, 1, MFInfo(), factory); sigma.setVal(1.0); - - // Set sigma + + // Set sigma matrix.setSigma(0, sigma); - + // // Create node-centered phi // MultiFab phi(nd_grids, dmap, 1, 1, MFInfo(), factory); phi.setVal(0.0); - + // // Setup MLMG solver // MLMG nodal_solver(matrix); - + // We can specify the maximum number of iterations // nodal_solver.setMaxIter(nodal_mg_maxiter); // nodal_solver.setBottomMaxIter(nodal_mg_bottom_maxiter); - + nodal_solver.setVerbose(mg_verbose); nodal_solver.setBottomVerbose(bottom_verbose); - // Set bottom-solver to use hypre instead of native BiCGStab + // Set bottom-solver to use hypre instead of native BiCGStab // ( we could also have set this to cg, bicgcg, cgbicg) - // if (use_hypre_as_full_solver || use_hypre_as_bottom_solver) + // if (use_hypre_as_full_solver || use_hypre_as_bottom_solver) // nodal_solver.setBottomSolver(MLMG::BottomSolver::hypre); - + // Define the relative tolerance Real reltol = 1.e-8; - + // Define the absolute tolerance; note that this argument is optional Real abstol = 1.e-15; - - amrex::Print() << " \n********************************************************************" << std::endl; + + amrex::Print() << " \n********************************************************************" << std::endl; amrex::Print() << " Let's project the initial velocity to find " << std::endl; amrex::Print() << " the flow field around the obstacles ... " << std::endl; amrex::Print() << " The domain has " << n_cell_x << " cells in the x-direction " << std::endl; - amrex::Print() << " The maximum grid size is " << max_grid_size << std::endl; - amrex::Print() << "******************************************************************** \n" << std::endl; - + amrex::Print() << " The maximum grid size is " << max_grid_size << std::endl; + amrex::Print() << "******************************************************************** \n" << std::endl; + // // Solve div( sigma * grad(phi) ) = RHS // nodal_solver.solve( {&phi}, {&rhs}, reltol, abstol); - - amrex::Print() << " \n********************************************************************" << std::endl; + + amrex::Print() << " \n********************************************************************" << std::endl; amrex::Print() << " Done solving the equation " << std::endl; amrex::Print() << " ... now subtracting off sigmna grad phi from vel" << std::endl; - + // // Create cell-centered multifab to hold value of -sigma*grad(phi) at cell-centers - // + // MultiFab fluxes(grids, dmap, AMREX_SPACEDIM, 1, MFInfo(), factory); fluxes.setVal(0.0); - + // Get fluxes from solver nodal_solver.getFluxes( {&fluxes} ); - + + // + // Apply projection explicitly -- vel = vel - sigma * grad(phi) // - // Apply projection explicitly -- vel = vel - sigma * grad(phi) - // MultiFab::Add( vel, fluxes, 0, 0, AMREX_SPACEDIM, 0); - + amrex::Print() << " ... now done with full projection operation" << std::endl; - amrex::Print() << "******************************************************************** \n" << std::endl; - + amrex::Print() << "******************************************************************** \n" << std::endl; + // Store plotfile variables; velocity and processor id MultiFab plotfile_mf(grids, dmap, AMREX_SPACEDIM+1, 0, MFInfo(), factory); - + // copy processor id into plotfile_mf plotfile_mf.setVal(ParallelDescriptor::MyProc(), 0, 1); plotfile_mf.setVal(ParallelDescriptor::MyProc(), 0, 1); - + // copy velocity into plotfile MultiFab::Copy(plotfile_mf, vel, 0, 1, AMREX_SPACEDIM, 0); - write_plotfile(geom, plotfile_mf); + write_plotfile(geom, plotfile_mf); } - + auto stop_time = amrex::second() - strt_time; amrex::Print() << "Total run time " << stop_time << std::endl; diff --git a/Tests/LinearSolvers/NodeEB/MyTest.cpp b/Tests/LinearSolvers/NodeEB/MyTest.cpp index aade032ddef..5ac57db3cb4 100644 --- a/Tests/LinearSolvers/NodeEB/MyTest.cpp +++ b/Tests/LinearSolvers/NodeEB/MyTest.cpp @@ -44,7 +44,7 @@ MyTest::solve () mlmg_hibc[idim] = LinOpBCType::Dirichlet; } } - + LPInfo info; info.setMaxCoarseningLevel(max_coarsening_level); info.setAgglomerationGridSize(agg_grid_size); @@ -118,7 +118,7 @@ MyTest::writePlotfile () void MyTest::readParameters () -{ +{ ParmParse pp; pp.query("max_level", max_level); pp.query("n_cell", n_cell); @@ -188,7 +188,7 @@ MyTest::initGrids () grids[ilev].define(domain); grids[ilev].maxSize(max_grid_size); domain.grow(-n_cell/4); // fine level cover the middle of the coarse domain - domain.refine(ref_ratio); + domain.refine(ref_ratio); } } @@ -241,9 +241,9 @@ MyTest::initData () fab(i,j,k,0) += 2.0*r*ry/r*fac; fab(i,j,k,1) -= 2.0*r*rx/r*fac; }); - } + } #if (AMREX_SPACEDIM > 2) - else if (cylinder_direction == 1) + else if (cylinder_direction == 1) { amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { @@ -254,8 +254,8 @@ MyTest::initData () fab(i,j,k,0) -= 2.0*r*rz/r*fac; fab(i,j,k,2) += 2.0*r*rx/r*fac; }); - } - else if (cylinder_direction == 0) + } + else if (cylinder_direction == 0) { amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { @@ -266,7 +266,7 @@ MyTest::initData () fab(i,j,k,1) += 2.0*r*rz/r*fac; fab(i,j,k,2) -= 2.0*r*ry/r*fac; }); - } + } #endif } } diff --git a/Tests/MultiBlock/Advection/main.cpp b/Tests/MultiBlock/Advection/main.cpp index 13a31428fcf..f30361fe471 100644 --- a/Tests/MultiBlock/Advection/main.cpp +++ b/Tests/MultiBlock/Advection/main.cpp @@ -30,7 +30,7 @@ enum num_components { three_components = 3 }; static constexpr IntVect e_x = IntVect::TheDimensionVector(ix); static constexpr IntVect e_y = IntVect::TheDimensionVector(iy); class AdvectionAmrCore : public AmrCore { - + public: AdvectionAmrCore(Direction vel, Geometry const& level_0_geom, AmrInfo const& amr_info = AmrInfo()) @@ -159,7 +159,7 @@ struct OnesidedMultiBlockBoundaryFn { cached_dest_bd_key = dest->mass.getBDKey(); cached_src_bd_key = src->mass.getBDKey(); } - + return ParallelCopy_nowait(no_local_copy, dest->mass, src->mass, *cmd, packing); } @@ -291,7 +291,7 @@ void MyMain() { amrex::Print() << "Step #" << step << ", Time Point = " << time_point << '\n'; time_point += dt; - step += 1; + step += 1; WritePlotfiles(core_x, core_y, time_point, step); } } diff --git a/Tests/MultiBlock/IndexType/main.cpp b/Tests/MultiBlock/IndexType/main.cpp index 3cfe48a4e41..c014b499ac2 100644 --- a/Tests/MultiBlock/IndexType/main.cpp +++ b/Tests/MultiBlock/IndexType/main.cpp @@ -30,7 +30,7 @@ amrex::iMultiFab InitializeMultiFab(const amrex::Box& domain) const int ny = domain.length(1); for (amrex::MFIter mfi(mf); mfi.isValid(); ++mfi) { auto array = mf.array(mfi); - ParallelFor(mfi.tilebox(), [=](int i, int j, int k) + ParallelFor(mfi.tilebox(), [=](int i, int j, int k) { array(i,j,k) = i + j*nx + k*nx*ny; }); @@ -50,7 +50,7 @@ bool ParallelCopyWithItselfIsCorrect(amrex::iMultiFab& mf, const amrex::Box& dom int fails = 0; for (amrex::MFIter mfi(mf); mfi.isValid(); ++mfi) { const amrex::Box section = dest_box & mfi.tilebox(); - if (section.isEmpty()) continue; + if (section.isEmpty()) continue; auto array = mf.const_array(mfi); amrex::LoopOnCpu(section, [&](int i, int j, int k) { @@ -91,7 +91,7 @@ bool ParallelCopyFaceToFace(amrex::iMultiFab& dest, const amrex::Box& domain_des int ddir = GetFaceDir(domain_dest.ixType()); const amrex::Box src_box = GetFaceBoundary(domain_src, src_side); const amrex::Box dest_box = GetFaceBoundary(domain_dest, dest_side); - + // Default construction is identity amrex::NonLocalBC::MultiBlockIndexMapping dtos{}; // Change permutation to get a correct mapping between index types @@ -110,7 +110,7 @@ bool ParallelCopyFaceToFace(amrex::iMultiFab& dest, const amrex::Box& domain_des const int ny = domain_src.length(1); for (amrex::MFIter mfi(dest); mfi.isValid(); ++mfi) { const amrex::Box section = dest_box & mfi.tilebox(); - if (section.isEmpty()) continue; + if (section.isEmpty()) continue; auto darray = dest.const_array(mfi); auto sarray = src.const_array(mfi); amrex::LoopOnCpu(section, [&](int i, int j, int k) diff --git a/Tests/Particles/AssignDensity/main.cpp b/Tests/Particles/AssignDensity/main.cpp index c95298a2d54..c1260b9a89a 100644 --- a/Tests/Particles/AssignDensity/main.cpp +++ b/Tests/Particles/AssignDensity/main.cpp @@ -35,8 +35,8 @@ void test_assign_density(TestParams& parms) // This sets the boundary conditions to be doubly or triply periodic int is_per[BL_SPACEDIM]; - for (int i = 0; i < BL_SPACEDIM; i++) - is_per[i] = 1; + for (int i = 0; i < BL_SPACEDIM; i++) + is_per[i] = 1; Geometry geom(domain, &real_box, CoordSys::cartesian, is_per); BoxArray ba(domain); @@ -71,14 +71,14 @@ void test_assign_density(TestParams& parms) MyParticleContainer::ParticleInitData pdata = {mass, AMREX_D_DECL(1.0, 2.0, 3.0)}; myPC.InitRandom(num_particles, iseed, pdata, serialize); myPC.AssignCellDensitySingleLevel(0, partMF, 0, 4, 0); - + // myPC.AssignDensitySingleLevel(0, partMF, 0, 4, 0); // myPC.InterpolateSingleLevel(acceleration, 0); MultiFab::Copy(density, partMF, 0, 0, 1, 0); - WriteSingleLevelPlotfile("plt00000", partMF, + WriteSingleLevelPlotfile("plt00000", partMF, {"density", "vx", "vy", "vz"}, geom, 0.0, 0); @@ -88,11 +88,11 @@ void test_assign_density(TestParams& parms) int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + ParmParse pp; - + TestParams parms; - + pp.get("nx", parms.nx); pp.get("ny", parms.ny); pp.get("nz", parms.nz); @@ -100,10 +100,10 @@ int main(int argc, char* argv[]) pp.get("nppc", parms.nppc); if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) amrex::Abort("Must specify at least one particle per cell"); - + parms.verbose = false; pp.query("verbose", parms.verbose); - + if (parms.verbose && ParallelDescriptor::IOProcessor()) { std::cout << std::endl; std::cout << "Number of particles per cell : "; @@ -111,8 +111,8 @@ int main(int argc, char* argv[]) std::cout << "Size of domain : "; std::cout << parms.nx << " " << parms.ny << " " << parms.nz << std::endl; } - + test_assign_density(parms); - + amrex::Finalize(); } diff --git a/Tests/Particles/AssignMultiLevelDensity/main.cpp b/Tests/Particles/AssignMultiLevelDensity/main.cpp index 532501b4c92..a32075761fe 100644 --- a/Tests/Particles/AssignMultiLevelDensity/main.cpp +++ b/Tests/Particles/AssignMultiLevelDensity/main.cpp @@ -20,9 +20,9 @@ struct TestParams { void test_assign_density(TestParams& parms) { - + int nlevs = parms.nlevs; - + RealBox real_box; for (int n = 0; n < BL_SPACEDIM; n++) { real_box.setLo(n, 0.0); @@ -35,9 +35,9 @@ void test_assign_density(TestParams& parms) fine_box.setLo(n,0.25); fine_box.setHi(n,0.75); } - - IntVect domain_lo(AMREX_D_DECL(0 , 0, 0)); - IntVect domain_hi(AMREX_D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); + + IntVect domain_lo(AMREX_D_DECL(0 , 0, 0)); + IntVect domain_hi(AMREX_D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); const Box domain(domain_lo, domain_hi); // Define the refinement ratio @@ -50,27 +50,27 @@ void test_assign_density(TestParams& parms) for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = 1; - // This defines a Geometry object which is useful for writing the plotfiles + // This defines a Geometry object which is useful for writing the plotfiles Vector geom(nlevs); geom[0].define(domain, &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), - &real_box, CoordSys::cartesian, is_per); + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_per); } Vector ba(nlevs); ba[0].define(domain); - + if (nlevs > 1) { int n_fine = parms.nx*rr[0]; - IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4)); + IntVect refined_lo(AMREX_D_DECL(n_fine/4,n_fine/4,n_fine/4)); IntVect refined_hi(AMREX_D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1)); // Build a box for the level 1 domain Box refined_patch(refined_lo, refined_hi); ba[1].define(refined_patch); } - + // break the BoxArrays at both levels into max_grid_size^3 boxes for (int lev = 0; lev < nlevs; lev++) { ba[lev].maxSize(parms.max_grid_size); @@ -129,8 +129,8 @@ void test_assign_density(TestParams& parms) outputMF[lev] = density[lev].get(); outputRR[lev] = IntVect(AMREX_D_DECL(2, 2, 2)); } - - WriteMultiLevelPlotfile("plt00000", output_levs, outputMF, + + WriteMultiLevelPlotfile("plt00000", output_levs, outputMF, varnames, geom, 0.0, level_steps, outputRR); myPC.Checkpoint("plt00000", "particle0", true, particle_varnames); } @@ -138,11 +138,11 @@ void test_assign_density(TestParams& parms) int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + ParmParse pp; - + TestParams parms; - + pp.get("nx", parms.nx); pp.get("ny", parms.ny); pp.get("nz", parms.nz); @@ -151,10 +151,10 @@ int main(int argc, char* argv[]) pp.get("nppc", parms.nppc); if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) amrex::Abort("Must specify at least one particle per cell"); - + parms.verbose = false; pp.query("verbose", parms.verbose); - + if (parms.verbose && ParallelDescriptor::IOProcessor()) { std::cout << std::endl; std::cout << "Number of particles per cell : "; @@ -164,8 +164,8 @@ int main(int argc, char* argv[]) std::cout << parms.nlevs << std::endl; std::cout << parms.nx << " " << parms.ny << " " << parms.nz << std::endl; } - + test_assign_density(parms); - + amrex::Finalize(); } diff --git a/Tests/Particles/AsyncIO/main.cpp b/Tests/Particles/AsyncIO/main.cpp index 24f5ee1d2e6..7d4c47fa273 100644 --- a/Tests/Particles/AsyncIO/main.cpp +++ b/Tests/Particles/AsyncIO/main.cpp @@ -200,8 +200,8 @@ void test_async_io(TestParams& parms) Vector geom(nlevs); geom[0].define(domain, &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), - &real_box, CoordSys::cartesian, is_per); + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_per); } Vector ba(nlevs); diff --git a/Tests/Particles/GhostsAndVirtuals/main.cpp b/Tests/Particles/GhostsAndVirtuals/main.cpp index b1469d42b95..501a2240bfb 100644 --- a/Tests/Particles/GhostsAndVirtuals/main.cpp +++ b/Tests/Particles/GhostsAndVirtuals/main.cpp @@ -53,8 +53,8 @@ void test_ghosts_and_virtuals (TestParams& parms) Vector geom(nlevs); geom[0].define(domain, &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), - &real_box, CoordSys::cartesian, is_per); + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_per); } Vector ba(nlevs); diff --git a/Tests/Particles/InitFromAscii/main.cpp b/Tests/Particles/InitFromAscii/main.cpp index 2ab7c87dccc..fe8adafeb84 100644 --- a/Tests/Particles/InitFromAscii/main.cpp +++ b/Tests/Particles/InitFromAscii/main.cpp @@ -24,17 +24,17 @@ void test_init_ascii (TestParams& parms) real_box.setLo(n, 0.0); real_box.setHi(n, 1.0); } - + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); IntVect domain_hi(AMREX_D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); const Box domain(domain_lo, domain_hi); - + // This sets the boundary conditions to be doubly or triply periodic int is_per[AMREX_SPACEDIM]; - for (int i = 0; i < AMREX_SPACEDIM; i++) + for (int i = 0; i < AMREX_SPACEDIM; i++) is_per[i] = 1; Geometry geom(domain, &real_box, CoordSys::cartesian, is_per); - + BoxArray ba(domain); ba.maxSize(parms.max_grid_size); @@ -67,16 +67,16 @@ void test_init_ascii (TestParams& parms) int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + ParmParse pp; - - TestParams parms; + + TestParams parms; pp.get("nx", parms.nx); pp.get("ny", parms.ny); pp.get("nz", parms.nz); pp.get("max_grid_size", parms.max_grid_size); - + test_init_ascii(parms); - + amrex::Finalize(); } diff --git a/Tests/Particles/Intersection/main.cpp b/Tests/Particles/Intersection/main.cpp index c0dedb85d94..910cb1d9765 100644 --- a/Tests/Particles/Intersection/main.cpp +++ b/Tests/Particles/Intersection/main.cpp @@ -26,15 +26,15 @@ void testIntersection() { TestParams params; get_test_params(params, "intersect"); - + int is_per[BL_SPACEDIM]; for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = params.is_periodic; - + Vector rr(params.nlevs-1); for (int lev = 1; lev < params.nlevs; lev++) rr[lev-1] = IntVect(AMREX_D_DECL(2,2,2)); - + RealBox real_box; for (int n = 0; n < AMREX_SPACEDIM; n++) { @@ -45,14 +45,14 @@ void testIntersection() IntVect domain_lo(AMREX_D_DECL(0 , 0, 0)); IntVect domain_hi(AMREX_D_DECL(params.size[0]-1, params.size[1]-1, params.size[2]-1)); const Box base_domain(domain_lo, domain_hi); - + Vector geom(params.nlevs); geom[0].define(base_domain, &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < params.nlevs; lev++) { geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), &real_box, CoordSys::cartesian, is_per); } - + Vector ba(params.nlevs); IntVect lo = IntVect(AMREX_D_DECL(0, 0, 0)); IntVect size = params.size; @@ -72,21 +72,21 @@ void testIntersection() auto assign_grid = ploc[lev].getGridAssignor(); - for (int i = 0; i < ba[lev].size(); ++i) + for (int i = 0; i < ba[lev].size(); ++i) { const Box& box = ba[lev][i]; Gpu::HostVector host_cells; for (IntVect iv = box.smallEnd(); iv <= box.bigEnd(); box.next(iv)) host_cells.push_back(iv); //host_cells.push_back(box.smallEnd()); - + int num_cells = host_cells.size(); - + Gpu::DeviceVector device_cells(num_cells); Gpu::copy(Gpu::hostToDevice, host_cells.begin(), host_cells.end(), device_cells.begin()); - + Gpu::DeviceVector device_grids(num_cells); - + const auto cells_ptr = device_cells.dataPtr(); const auto grids_ptr = device_grids.dataPtr(); amrex::ParallelFor(num_cells, [=] AMREX_GPU_DEVICE (int j) noexcept @@ -97,15 +97,15 @@ void testIntersection() ReduceOps reduce_op; ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; - + reduce_op.eval(num_cells, reduce_data, [=] AMREX_GPU_DEVICE (int j) -> ReduceTuple { return {grids_ptr[j] != i}; - }); - + }); + ReduceTuple hv = reduce_data.value(); - + int num_wrong = amrex::get<0>(hv); AMREX_ALWAYS_ASSERT(num_wrong == 0); } @@ -115,8 +115,8 @@ void testIntersection() int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + testIntersection(); - + amrex::Finalize(); } diff --git a/Tests/Particles/NeighborParticles/CheckPair.H b/Tests/Particles/NeighborParticles/CheckPair.H index 5edc5cb4b9c..8018e9c49e9 100644 --- a/Tests/Particles/NeighborParticles/CheckPair.H +++ b/Tests/Particles/NeighborParticles/CheckPair.H @@ -11,8 +11,8 @@ struct CheckPair { amrex::Real d0 = (p1.pos(0) - p2.pos(0)); amrex::Real d1 = (p1.pos(1) - p2.pos(1)); - amrex::Real d2 = (p1.pos(2) - p2.pos(2)); - amrex::Real dsquared = d0*d0 + d1*d1 + d2*d2; + amrex::Real d2 = (p1.pos(2) - p2.pos(2)); + amrex::Real dsquared = d0*d0 + d1*d1 + d2*d2; return (dsquared <= 25.0*Params::cutoff*Params::cutoff); } }; diff --git a/Tests/Particles/NeighborParticles/Constants.H b/Tests/Particles/NeighborParticles/Constants.H index 2e26b1939b9..18c94dbe255 100644 --- a/Tests/Particles/NeighborParticles/Constants.H +++ b/Tests/Particles/NeighborParticles/Constants.H @@ -5,7 +5,7 @@ namespace Params { - // This is designed to represent MFiX-like conditions where the grid spacing is + // This is designed to represent MFiX-like conditions where the grid spacing is // roughly 2.5 times the particle diameter. In main.cpp we set grid spacing to 1 // so here we set cutoff to diameter = 1/2.5 --> cutoff = 0.2 static constexpr amrex::Real cutoff = 0.2 ; diff --git a/Tests/Particles/NeighborParticles/MDParticleContainer.cpp b/Tests/Particles/NeighborParticles/MDParticleContainer.cpp index c81f2d667b6..5cb42ea4066 100644 --- a/Tests/Particles/NeighborParticles/MDParticleContainer.cpp +++ b/Tests/Particles/NeighborParticles/MDParticleContainer.cpp @@ -155,11 +155,11 @@ std::pair MDParticleContainer::minAndMaxDistance() auto nbor_data = m_neighbor_list[lev][index].data(); ParticleType* pstruct = aos().dataPtr(); - Gpu::DeviceScalar min_d_gpu(min_d); - Gpu::DeviceScalar max_d_gpu(max_d); + Gpu::DeviceScalar min_d_gpu(min_d); + Gpu::DeviceScalar max_d_gpu(max_d); - Real* pmin_d = min_d_gpu.dataPtr(); - Real* pmax_d = max_d_gpu.dataPtr(); + Real* pmin_d = min_d_gpu.dataPtr(); + Real* pmax_d = max_d_gpu.dataPtr(); AMREX_FOR_1D ( np, i, { @@ -175,15 +175,15 @@ std::pair MDParticleContainer::minAndMaxDistance() r2 = amrex::max(r2, Params::min_r*Params::min_r); Real r = sqrt(r2); - Gpu::Atomic::Min(pmin_d, r); - Gpu::Atomic::Max(pmax_d, r); + Gpu::Atomic::Min(pmin_d, r); + Gpu::Atomic::Max(pmax_d, r); } }); - // Gpu::Device::streamSynchronize(); + // Gpu::Device::streamSynchronize(); - min_d = std::min(min_d, min_d_gpu.dataValue()); - max_d = std::max(max_d, max_d_gpu.dataValue()); + min_d = std::min(min_d, min_d_gpu.dataValue()); + max_d = std::max(max_d, max_d_gpu.dataValue()); } ParallelDescriptor::ReduceRealMin(min_d, ParallelDescriptor::IOProcessorNumber()); ParallelDescriptor::ReduceRealMax(max_d, ParallelDescriptor::IOProcessorNumber()); @@ -359,7 +359,7 @@ void MDParticleContainer::checkNeighborList() auto nbor_data = m_neighbor_list[lev][index].data(); ParticleType* pstruct = aos().dataPtr(); - // ON DEVIDE: + // ON DEVIDE: // AMREX_FOR_1D ( np, i, // ON HOST: // for (int i = 0; i < np; i++) @@ -380,20 +380,20 @@ void MDParticleContainer::checkNeighborList() Real dx = p1.pos(0) - p2.pos(0); Real dy = p1.pos(1) - p2.pos(1); Real dz = p1.pos(2) - p2.pos(2); - + Real r2 = dx*dx + dy*dy + dz*dz; - Real cutoff_sq = 25.0*Params::cutoff*Params::cutoff; + Real cutoff_sq = 25.0*Params::cutoff*Params::cutoff; - if (r2 <= cutoff_sq) - { + if (r2 <= cutoff_sq) + { Gpu::Atomic::AddNoRet(&(p_full_count[i]),1); full_nbors.push_back(p2.id()); - } + } } for (const auto& p2 : nbor_data.getNeighbors(i)) - { + { Gpu::Atomic::AddNoRet(&(p_neighbor_count[i]),1); nbor_nbors.push_back(p2.id()); } @@ -411,10 +411,10 @@ void MDParticleContainer::checkNeighborList() // amrex::PrintToFile("neighbor_test") << " there are " << nbor_nbors.size() << " " << // full_nbors.size() << " list / full neighbors of particle " << i << std::endl; - + // Loop over particles in my neighbor list for (int cnt = 0; cnt < nbor_nbors.size(); cnt++) - { + { // std::cout << " NBORS " << nbor_nbors[cnt] << " " << full_nbors[cnt] << std::endl; if (nbor_nbors[cnt] != full_nbors[cnt]) { diff --git a/Tests/Particles/NeighborParticles/README.md b/Tests/Particles/NeighborParticles/README.md index e4bc48d6a5d..5e8f30fdb80 100644 --- a/Tests/Particles/NeighborParticles/README.md +++ b/Tests/Particles/NeighborParticles/README.md @@ -1,6 +1,6 @@ This unit test -(1) initializes particles, +(1) initializes particles, (2) counts how many particles with which grid id it "owns" (only for grid 0) -- here it only owns the particles it started with, e.g. 512 = 8^3 if num_ppc = 1 @@ -8,16 +8,16 @@ This unit test (3) calls FillNeighbors (4) counts how many particles with which grid id it "owns" (only for grid 0) -- here it should have - its original particles plus particles from every other grid, e.g. 8^3 , 6x8^2 , 12x8 , 8x1 from grids + its original particles plus particles from every other grid, e.g. 8^3 , 6x8^2 , 12x8 , 8x1 from grids 0 to 26 (if num_ppc = 1) (5) calls UpdateNeighbors (6) counts how many particles with which grid id it "owns" (only for grid 0) -- answer should be identical to previous call -(7) calls reset_test_id which sets the "test_id" of every particle (including "ghosts") to the (local)grid id +(7) calls reset_test_id which sets the "test_id" of every particle (including "ghosts") to the (local)grid id -(8) counts how many particles with which grid id it "owns" (only for grid 0) -- should be +(8) counts how many particles with which grid id it "owns" (only for grid 0) -- should be e.g. 1000 = 8^3 + 6x8^2 + 12x8 + 8x1 (if num_ppc = 1) from grid 0 and no others (9) calls UpdateNeighbors diff --git a/Tests/Particles/ParticleIterator/main.cpp b/Tests/Particles/ParticleIterator/main.cpp index 63e21b76c34..215a987301c 100644 --- a/Tests/Particles/ParticleIterator/main.cpp +++ b/Tests/Particles/ParticleIterator/main.cpp @@ -25,15 +25,15 @@ int main(int argc, char* argv[]) real_box.setHi(n,1.0); } - IntVect domain_lo(0 , 0, 0); - IntVect domain_hi(ncell-1, ncell-1, ncell-1); - + IntVect domain_lo(0 , 0, 0); + IntVect domain_hi(ncell-1, ncell-1, ncell-1); + const Box domain(domain_lo, domain_hi); Vector rr(nlevs-1); for (int lev = 1; lev < nlevs; lev++) rr[lev-1] = 2; - + int is_per[BL_SPACEDIM]; for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = 1; @@ -41,7 +41,7 @@ int main(int argc, char* argv[]) geom[0].define(domain, &real_box, coord, is_per); Vector ba(nlevs); - ba[0].define(domain); + ba[0].define(domain); for (int lev = 0; lev < nlevs; lev++) ba[lev].maxSize(max_grid_size); @@ -65,6 +65,6 @@ int main(int argc, char* argv[]) for (ParIter<1+BL_SPACEDIM> mfi(MyPC, 0); mfi.isValid(); ++mfi) { amrex::AllPrintToFile("particle_iterator_out") << mfi.index() << " " << mfi.tileIndex() << "\n"; } - + amrex::Finalize(); } diff --git a/Tests/Particles/ParticleMesh/main.cpp b/Tests/Particles/ParticleMesh/main.cpp index 30f2ee05ade..02a10d22cdd 100644 --- a/Tests/Particles/ParticleMesh/main.cpp +++ b/Tests/Particles/ParticleMesh/main.cpp @@ -33,8 +33,8 @@ void testParticleMesh(TestParams& parms) // This sets the boundary conditions to be doubly or triply periodic int is_per[BL_SPACEDIM]; - for (int i = 0; i < BL_SPACEDIM; i++) - is_per[i] = 1; + for (int i = 0; i < BL_SPACEDIM; i++) + is_per[i] = 1; Geometry geom(domain, &real_box, CoordSys::cartesian, is_per); BoxArray ba(domain); @@ -85,9 +85,9 @@ void testParticleMesh(TestParams& parms) amrex::Real sx[] = {1.-xint, xint}; amrex::Real sy[] = {1.-yint, yint}; amrex::Real sz[] = {1.-zint, zint}; - - for (int kk = 0; kk <= 1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) { + + for (int kk = 0; kk <= 1; ++kk) { + for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, 0), sx[ii]*sy[jj]*sz[kk]*p.rdata(0)); @@ -95,9 +95,9 @@ void testParticleMesh(TestParams& parms) } } - for (int comp=1; comp < nc; ++comp) { - for (int kk = 0; kk <= 1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) { + for (int comp=1; comp < nc; ++comp) { + for (int kk = 0; kk <= 1; ++kk) { + for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) { amrex::Gpu::Atomic::AddNoRet(&rho(i+ii-1, j+jj-1, k+kk-1, comp), sx[ii]*sy[jj]*sz[kk]*p.rdata(0)*p.rdata(comp)); @@ -131,9 +131,9 @@ void testParticleMesh(TestParams& parms) amrex::Real sy[] = {1.-yint, yint}; amrex::Real sz[] = {1.-zint, zint}; - for (int comp=0; comp < nc; ++comp) { - for (int kk = 0; kk <= 1; ++kk) { - for (int jj = 0; jj <= 1; ++jj) { + for (int comp=0; comp < nc; ++comp) { + for (int kk = 0; kk <= 1; ++kk) { + for (int jj = 0; jj <= 1; ++jj) { for (int ii = 0; ii <= 1; ++ii) { p.rdata(4+comp) += sx[ii]*sy[jj]*sz[kk]*acc(i+ii-1,j+jj-1,k+kk-1,comp); } @@ -141,8 +141,8 @@ void testParticleMesh(TestParams& parms) } } }); - - WriteSingleLevelPlotfile("plot", partMF, + + WriteSingleLevelPlotfile("plot", partMF, {"density", "vx", "vy", "vz"}, geom, 0.0, 0); @@ -152,11 +152,11 @@ void testParticleMesh(TestParams& parms) int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + ParmParse pp; - + TestParams parms; - + pp.get("nx", parms.nx); pp.get("ny", parms.ny); pp.get("nz", parms.nz); @@ -164,10 +164,10 @@ int main(int argc, char* argv[]) pp.get("nppc", parms.nppc); if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) amrex::Abort("Must specify at least one particle per cell"); - + parms.verbose = false; pp.query("verbose", parms.verbose); - + if (parms.verbose && ParallelDescriptor::IOProcessor()) { std::cout << std::endl; std::cout << "Number of particles per cell : "; @@ -175,8 +175,8 @@ int main(int argc, char* argv[]) std::cout << "Size of domain : "; std::cout << parms.nx << " " << parms.ny << " " << parms.nz << std::endl; } - + testParticleMesh(parms); - + amrex::Finalize(); } diff --git a/Tests/Particles/ParticleTransformations/main.cpp b/Tests/Particles/ParticleTransformations/main.cpp index 590cee345db..1bdf722291b 100644 --- a/Tests/Particles/ParticleTransformations/main.cpp +++ b/Tests/Particles/ParticleTransformations/main.cpp @@ -15,11 +15,11 @@ void get_position_unit_cell(Real* r, const IntVect& nppc, int i_part) int nx = nppc[0]; int ny = nppc[1]; int nz = nppc[2]; - + int ix_part = i_part/(ny * nz); int iy_part = (i_part % (ny * nz)) % ny; int iz_part = (i_part % (ny * nz)) / ny; - + r[0] = (0.5+ix_part)/nx; r[1] = (0.5+iy_part)/ny; r[2] = (0.5+iz_part)/nz; @@ -33,7 +33,7 @@ class TestParticleContainer using ParticleTileType = ParticleTile; - + TestParticleContainer (const amrex::Geometry & a_geom, const amrex::DistributionMapping & a_dmap, const amrex::BoxArray & a_ba) @@ -43,10 +43,10 @@ class TestParticleContainer void InitParticles (const amrex::IntVect& a_num_particles_per_cell) { BL_PROFILE("InitParticles"); - const int lev = 0; + const int lev = 0; const Real* dx = Geom(lev).CellSize(); const Real* plo = Geom(lev).ProbLo(); - + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], *a_num_particles_per_cell[1], *a_num_particles_per_cell[2]); @@ -63,21 +63,21 @@ class TestParticleContainer for (int i_part=0; i_part AMREX_GPU_HOST_DEVICE int operator() (const SrcData& src, int i) const noexcept @@ -194,7 +194,7 @@ struct KeepOddFilter }; struct KeepEvenFilter -{ +{ template AMREX_GPU_HOST_DEVICE int operator() (const SrcData& src, int i) const noexcept @@ -210,16 +210,16 @@ void transformParticles (PC& pc, F&& f) using ParIter = typename PC::ParConstIterType; using ParticleTileType = typename PC::ParticleTileType; - + for (int lev = 0; lev <= pc.finestLevel(); ++lev) { for(ParIter pti(pc, lev); pti.isValid(); ++pti) { auto& ptile = pc.ParticlesAt(lev, pti); - + ParticleTileType ptile_tmp; ptile_tmp.resize(ptile.size()); - + amrex::transformParticles(ptile_tmp, ptile, std::forward(f)); ptile.swap(ptile_tmp); } @@ -240,10 +240,10 @@ void twoWayTransformParticles (PC& dst1, PC& dst2, const PC& src, F&& f) auto& ptile_src = src.ParticlesAt(lev, pti); auto& ptile_dst1 = dst1.DefineAndReturnParticleTile(lev, pti); auto& ptile_dst2 = dst2.DefineAndReturnParticleTile(lev, pti); - + ptile_dst1.resize(ptile_src.size()); ptile_dst2.resize(ptile_src.size()); - + amrex::transformParticles(ptile_dst1, ptile_dst2, ptile_src, std::forward(f)); } } @@ -258,10 +258,10 @@ void testTwoWayTransform (const PC& pc) pc1.copyParticles(pc); auto np_old = pc1.TotalNumberOfParticles(); - + PC pc2(pc.Geom(0), pc.ParticleDistributionMap(0), pc.ParticleBoxArray(0)); PC pc3(pc.Geom(0), pc.ParticleDistributionMap(0), pc.ParticleBoxArray(0)); - + twoWayTransformParticles(pc2, pc3, pc1, TwoWayTransformer(2, 3)); auto mx1 = amrex::ReduceMax(pc1, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); @@ -273,14 +273,14 @@ void testTwoWayTransform (const PC& pc) auto np_new = pc2.TotalNumberOfParticles(); AMREX_ALWAYS_ASSERT(np_old == np_new); - + auto mx3 = amrex::ReduceMax(pc3, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); AMREX_ALWAYS_ASSERT(mx3 == 3*mx1); np_new = pc3.TotalNumberOfParticles(); - AMREX_ALWAYS_ASSERT(np_old == np_new); + AMREX_ALWAYS_ASSERT(np_old == np_new); } template @@ -290,16 +290,16 @@ void filterParticles (PC& pc, F&& f) using ParIter = typename PC::ParIterType; using ParticleTileType = typename PC::ParticleTileType; - + for (int lev = 0; lev <= pc.finestLevel(); ++lev) { for(ParIter pti(pc, lev); pti.isValid(); ++pti) { auto& ptile = pc.ParticlesAt(lev, pti); - + ParticleTileType ptile_tmp; ptile_tmp.resize(ptile.size()); - + auto num_output = amrex::filterParticles(ptile_tmp, ptile, std::forward(f)); ptile.swap(ptile_tmp); @@ -315,16 +315,16 @@ void filterAndTransformParticles (PC& pc, Pred&& p, F&& f) using ParIter = typename PC::ParIterType; using ParticleTileType = typename PC::ParticleTileType; - + for (int lev = 0; lev <= pc.finestLevel(); ++lev) { for(ParIter pti(pc, lev); pti.isValid(); ++pti) { auto& ptile = pc.ParticlesAt(lev, pti); - + ParticleTileType ptile_tmp; ptile_tmp.resize(ptile.size()); - + auto num_output = amrex::filterAndTransformParticles(ptile_tmp, ptile, std::forward(p), std::forward(f)); ptile.swap(ptile_tmp); ptile.resize(num_output); @@ -337,7 +337,7 @@ void twoWayFilterAndTransformParticles (PC& dst1, PC& dst2, const PC& src, Pred& { BL_PROFILE("twoWayFilterAndTransformParticles"); - using ParIter = typename PC::ParConstIterType; + using ParIter = typename PC::ParConstIterType; for (int lev = 0; lev <= src.finestLevel(); ++lev) { for(ParIter pti(src, lev); pti.isValid(); ++pti) @@ -345,10 +345,10 @@ void twoWayFilterAndTransformParticles (PC& dst1, PC& dst2, const PC& src, Pred& auto& ptile_src = src.ParticlesAt(lev, pti); auto& ptile_dst1 = dst1.DefineAndReturnParticleTile(lev, pti); auto& ptile_dst2 = dst2.DefineAndReturnParticleTile(lev, pti); - + ptile_dst1.resize(ptile_src.size()); ptile_dst2.resize(ptile_src.size()); - + auto num_output = amrex::filterAndTransformParticles(ptile_dst1, ptile_dst2, ptile_src, @@ -369,11 +369,11 @@ void testTransform (const PC& pc) pc2.copyParticles(pc); auto np_old = pc2.TotalNumberOfParticles(); - + auto mx1 = amrex::ReduceMax(pc2, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); transformParticles(pc2, Transformer(2)); - + auto mx2 = amrex::ReduceMax(pc2, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); AMREX_ALWAYS_ASSERT(2*mx1 == mx2); @@ -381,14 +381,14 @@ void testTransform (const PC& pc) auto np_new = pc2.TotalNumberOfParticles(); AMREX_ALWAYS_ASSERT(np_old == np_new); - + pc2.clearParticles(); pc2.copyParticles(pc); transformParticles(pc2, Transformer(3)); - + auto mx3 = amrex::ReduceMax(pc2, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); - AMREX_ALWAYS_ASSERT(mx3 == 3*mx1); + AMREX_ALWAYS_ASSERT(mx3 == 3*mx1); } template @@ -438,13 +438,13 @@ void testFilterAndTransform (const PC& pc) auto mx1 = amrex::ReduceMax(pc2, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); auto np_old = pc2.TotalNumberOfParticles(); - + filterAndTransformParticles(pc2, KeepOddFilter(), Transformer(3)); auto mx2 = amrex::ReduceMax(pc2, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); - + auto np_new = pc2.TotalNumberOfParticles(); - + AMREX_ALWAYS_ASSERT(2*np_new == np_old); AMREX_ALWAYS_ASSERT(mx2 == 3*mx1); } @@ -458,21 +458,21 @@ void testTwoWayFilterAndTransform (const PC& pc) pc1.copyParticles(pc); auto np_old = pc1.TotalNumberOfParticles(); - + PC pc2(pc.Geom(0), pc.ParticleDistributionMap(0), pc.ParticleBoxArray(0)); PC pc3(pc.Geom(0), pc.ParticleDistributionMap(0), pc.ParticleBoxArray(0)); - + twoWayFilterAndTransformParticles(pc2, pc3, pc1, KeepEvenFilter(), TwoWayTransformer(2, 3)); - + auto mx1 = amrex::ReduceMax(pc1, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); auto mx2 = amrex::ReduceMax(pc2, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); auto mx3 = amrex::ReduceMax(pc3, [=] AMREX_GPU_HOST_DEVICE (const PType& p) -> int { return p.idata(NSI+1); }); - + auto np_new2 = pc2.TotalNumberOfParticles(); auto np_new3 = pc2.TotalNumberOfParticles(); - + AMREX_ALWAYS_ASSERT(2*np_new2 == np_old); AMREX_ALWAYS_ASSERT(2*np_new3 == np_old); AMREX_ALWAYS_ASSERT(mx2 == 2*mx1); @@ -528,7 +528,7 @@ void testTransformations () for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = 1; Geometry geom(domain, &real_box, coord, is_per); - + BoxArray ba(domain); ba.maxSize(params.max_grid_size); DistributionMapping dm(ba); @@ -552,6 +552,6 @@ void testTransformations () testTwoWayTransform(pc); testTwoWayFilterAndTransform(pc); - + amrex::Print() << "pass \n"; } diff --git a/Tests/Particles/SparseBins/main.cpp b/Tests/Particles/SparseBins/main.cpp index c224dc41cbe..6e4fe00b08f 100644 --- a/Tests/Particles/SparseBins/main.cpp +++ b/Tests/Particles/SparseBins/main.cpp @@ -26,15 +26,15 @@ void testIntersection() { TestParams params; get_test_params(params, "intersect"); - + int is_per[BL_SPACEDIM]; for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = params.is_periodic; - + Vector rr(params.nlevs-1); for (int lev = 1; lev < params.nlevs; lev++) rr[lev-1] = IntVect(AMREX_D_DECL(2,2,2)); - + RealBox real_box; for (int n = 0; n < AMREX_SPACEDIM; n++) { @@ -45,14 +45,14 @@ void testIntersection() IntVect domain_lo(AMREX_D_DECL(0 , 0, 0)); IntVect domain_hi(AMREX_D_DECL(params.size[0]-1, params.size[1]-1, params.size[2]-1)); const Box base_domain(domain_lo, domain_hi); - + Vector geom(params.nlevs); geom[0].define(base_domain, &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < params.nlevs; lev++) { geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), &real_box, CoordSys::cartesian, is_per); } - + Vector ba(params.nlevs); IntVect lo = IntVect(AMREX_D_DECL(0, 0, 0)); IntVect size = params.size; @@ -66,15 +66,15 @@ void testIntersection() { auto num_boxes = ba[0].size(); - + } } int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + testIntersection(); - + amrex::Finalize(); } diff --git a/Tests/Particles/TypeDescriptor/main.cpp b/Tests/Particles/TypeDescriptor/main.cpp index b00c035132b..6f80feecd31 100644 --- a/Tests/Particles/TypeDescriptor/main.cpp +++ b/Tests/Particles/TypeDescriptor/main.cpp @@ -16,34 +16,34 @@ void testIntIO(const IntDescriptor& id_out) { std::string data_file_name = "int_data.dat"; std::string header_file_name = "int_header_H"; - + amrex::Vector idata_out; for (int i = -99; i <= 100; ++i) { idata_out.push_back(i); } - + std::ofstream ofs; ofs.open(data_file_name.c_str(), std::ios::out|std::ios::binary); writeIntData(idata_out.data(), idata_out.size(), ofs, id_out); ofs.close(); - + ofs.open(header_file_name.c_str(), std::ios::out); ofs << id_out << "\n"; ofs.close(); - + IntDescriptor id_in; std::ifstream ifs; ifs.open(header_file_name.c_str(), std::ios::in); ifs >> id_in; ifs.close(); - + AMREX_ALWAYS_ASSERT(id_out == id_in); - + amrex::Vector idata_in(idata_out.size()); ifs.open(data_file_name.c_str(), std::ios::in|std::ios::binary); readIntData(idata_in.data(), idata_in.size(), ifs, id_in); ifs.close(); - + for (int i = 0; i < static_cast(idata_in.size()); ++i) { AMREX_ALWAYS_ASSERT(idata_in[i] == idata_out[i]); } @@ -53,7 +53,7 @@ void testLongIO(const IntDescriptor& id_out) { std::string data_file_name = "long_data.dat"; std::string header_file_name = "long_header_H"; - + amrex::Vector idata_out; for (int i = -99; i <= 100; ++i) { idata_out.push_back(i); @@ -63,24 +63,24 @@ void testLongIO(const IntDescriptor& id_out) { ofs.open(data_file_name.c_str(), std::ios::out|std::ios::binary); writeLongData(idata_out.data(), idata_out.size(), ofs, id_out); ofs.close(); - + ofs.open(header_file_name.c_str(), std::ios::out); ofs << id_out << "\n"; ofs.close(); - + IntDescriptor id_in; std::ifstream ifs; ifs.open(header_file_name.c_str(), std::ios::in); ifs >> id_in; ifs.close(); - + AMREX_ALWAYS_ASSERT(id_out == id_in); - + amrex::Vector idata_in(idata_out.size()); ifs.open(data_file_name.c_str(), std::ios::in|std::ios::binary); readLongData(idata_in.data(), idata_in.size(), ifs, id_in); ifs.close(); - + for (int i = 0; i < static_cast(idata_in.size()); ++i) { AMREX_ALWAYS_ASSERT(idata_in[i] == idata_out[i]); } @@ -90,7 +90,7 @@ void testRealIO(const RealDescriptor& rd_out) { std::string data_file_name = "real_data.dat"; std::string header_file_name = "real_header_H"; - + amrex::Vector rdata_out; for (int i = -99; i <= 100; ++i) { rdata_out.push_back(amrex::Random()); @@ -100,26 +100,26 @@ void testRealIO(const RealDescriptor& rd_out) { ofs.open(data_file_name.c_str(), std::ios::out|std::ios::binary); writeRealData(rdata_out.data(), rdata_out.size(), ofs, rd_out); ofs.close(); - + ofs.open(header_file_name.c_str(), std::ios::out); ofs << rd_out << "\n"; ofs.close(); - + RealDescriptor rd_in; std::ifstream ifs; ifs.open(header_file_name.c_str(), std::ios::in); ifs >> rd_in; ifs.close(); - + AMREX_ALWAYS_ASSERT(rd_out == rd_in); - + amrex::Vector rdata_in(rdata_out.size()); ifs.open(data_file_name.c_str(), std::ios::in|std::ios::binary); readRealData(rdata_in.data(), rdata_in.size(), ifs, rd_in); ifs.close(); - + for (int i = 0; i < static_cast(rdata_in.size()); ++i) { - if (rd_in == FPC::Native32RealDescriptor() || + if (rd_in == FPC::Native32RealDescriptor() || rd_in == FPC::Ieee32NormalRealDescriptor()) { AMREX_ALWAYS_ASSERT(std::abs(rdata_in[i] - rdata_out[i]) <= 1e-7); } else{ @@ -132,7 +132,7 @@ void testFloatIO(const RealDescriptor& rd_out) { std::string data_file_name = "float_data.dat"; std::string header_file_name = "float_header_H"; - + amrex::Vector rdata_out; for (int i = -99; i <= 100; ++i) { rdata_out.push_back(amrex::Random()); @@ -142,26 +142,26 @@ void testFloatIO(const RealDescriptor& rd_out) { ofs.open(data_file_name.c_str(), std::ios::out|std::ios::binary); writeFloatData(rdata_out.data(), rdata_out.size(), ofs, rd_out); ofs.close(); - + ofs.open(header_file_name.c_str(), std::ios::out); ofs << rd_out << "\n"; ofs.close(); - + RealDescriptor rd_in; std::ifstream ifs; ifs.open(header_file_name.c_str(), std::ios::in); ifs >> rd_in; ifs.close(); - + AMREX_ALWAYS_ASSERT(rd_out == rd_in); - + amrex::Vector rdata_in(rdata_out.size()); ifs.open(data_file_name.c_str(), std::ios::in|std::ios::binary); readFloatData(rdata_in.data(), rdata_in.size(), ifs, rd_in); ifs.close(); - + for (int i = 0; i < static_cast(rdata_in.size()); ++i) { - if (rd_in == FPC::Native32RealDescriptor() || + if (rd_in == FPC::Native32RealDescriptor() || rd_in == FPC::Ieee32NormalRealDescriptor()) { AMREX_ALWAYS_ASSERT(std::abs(rdata_in[i] - rdata_out[i]) <= 1e-7); } else{ @@ -174,7 +174,7 @@ void testDoubleIO(const RealDescriptor& rd_out) { std::string data_file_name = "double_data.dat"; std::string header_file_name = "double_header_H"; - + amrex::Vector rdata_out; for (int i = -99; i <= 100; ++i) { rdata_out.push_back(amrex::Random()); @@ -184,26 +184,26 @@ void testDoubleIO(const RealDescriptor& rd_out) { ofs.open(data_file_name.c_str(), std::ios::out|std::ios::binary); writeDoubleData(rdata_out.data(), rdata_out.size(), ofs, rd_out); ofs.close(); - + ofs.open(header_file_name.c_str(), std::ios::out); ofs << rd_out << "\n"; ofs.close(); - + RealDescriptor rd_in; std::ifstream ifs; ifs.open(header_file_name.c_str(), std::ios::in); ifs >> rd_in; ifs.close(); - + AMREX_ALWAYS_ASSERT(rd_out == rd_in); - + amrex::Vector rdata_in(rdata_out.size()); ifs.open(data_file_name.c_str(), std::ios::in|std::ios::binary); readDoubleData(rdata_in.data(), rdata_in.size(), ifs, rd_in); ifs.close(); - + for (int i = 0; i < static_cast(rdata_in.size()); ++i) { - if (rd_in == FPC::Native32RealDescriptor() || + if (rd_in == FPC::Native32RealDescriptor() || rd_in == FPC::Ieee32NormalRealDescriptor()) { AMREX_ALWAYS_ASSERT(std::abs(rdata_in[i] - rdata_out[i]) <= 1e-7); } else{ @@ -215,27 +215,27 @@ void testDoubleIO(const RealDescriptor& rd_out) { int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + IntDescriptor little16(2, IntDescriptor::ReverseOrder); IntDescriptor little32(4, IntDescriptor::ReverseOrder); IntDescriptor little64(8, IntDescriptor::ReverseOrder); - + IntDescriptor big16(2, IntDescriptor::NormalOrder); IntDescriptor big32(4, IntDescriptor::NormalOrder); IntDescriptor big64(8, IntDescriptor::NormalOrder); - + testIntIO(little16); testIntIO(little32); testIntIO(little64); - + testIntIO(big16); testIntIO(big32); testIntIO(big64); - + testLongIO(little16); testLongIO(little32); testLongIO(little64); - + testLongIO(big16); testLongIO(big32); testLongIO(big64); @@ -256,6 +256,6 @@ int main(int argc, char* argv[]) testDoubleIO(FPC::Ieee64NormalRealDescriptor()); amrex::Print() << "passed!" << std::endl; - + amrex::Finalize(); } diff --git a/Tools/AMRProfParser/TestCodes/AMRProfTest0.cpp b/Tools/AMRProfParser/TestCodes/AMRProfTest0.cpp index 1d4c56ddea5..1a0a704651c 100644 --- a/Tools/AMRProfParser/TestCodes/AMRProfTest0.cpp +++ b/Tools/AMRProfParser/TestCodes/AMRProfTest0.cpp @@ -42,7 +42,7 @@ void InitRegion() { amrex::Print(Print::AllProcs) << myProc << "::InitRegion = " << sleepTime << " s." << endl; - + SleepProcTimes(sleepTime); BL_PROFILE_REGION_STOP("R::InitRegion"); @@ -62,7 +62,7 @@ void ComputeRegion() { amrex::Print() << "Compute Region." << std::endl; SleepProcTimes(sleeptime); - + amrex::Print(Print::AllProcs) << myProc << "::ComputeRegion myProc = " << sleeptime << " s." << endl; @@ -83,7 +83,7 @@ void ConcludeRegion() { amrex::Print(Print::AllProcs) << myProc << "::ConcludeRegion = " << sleepTime << " s." << endl; - + BL_PROFILE_VAR_STOP(concluderegion); BL_PROFILE_REGION_STOP("R::ConcludeRegion"); } diff --git a/Tools/AMRProfParser/TestCodes/ProfWaitTest.cpp b/Tools/AMRProfParser/TestCodes/ProfWaitTest.cpp index 451d24e0613..9dc94131937 100644 --- a/Tools/AMRProfParser/TestCodes/ProfWaitTest.cpp +++ b/Tools/AMRProfParser/TestCodes/ProfWaitTest.cpp @@ -52,7 +52,7 @@ void WaitRegion() { { ParallelDescriptor::Send(&data[j], 1, i, SeqNum); j++; - } + } } for (int i=0; i &lineTokens = amrex::Tokenize(fileLine, delimString); - cout << "lineTokens.size() = " << lineTokens.size() << endl; - for(int i(0); i < lineTokens.size(); ++i) { - cout << i << "::token = " << lineTokens[i] << endl; - } + char delims[1]; + //delims[0] = '\011'; + delims[0] = ' '; + //std::string delimString(delims); + std::string delimString(" "); + const std::vector &lineTokens = amrex::Tokenize(fileLine, delimString); + cout << "lineTokens.size() = " << lineTokens.size() << endl; + for(int i(0); i < lineTokens.size(); ++i) { + cout << i << "::token = " << lineTokens[i] << endl; + } } } } diff --git a/Tools/C_util/AmrDeriveTecplot/AmrDeriveTecplot.cpp b/Tools/C_util/AmrDeriveTecplot/AmrDeriveTecplot.cpp index 8a26e2e1654..28a95ea08b0 100644 --- a/Tools/C_util/AmrDeriveTecplot/AmrDeriveTecplot.cpp +++ b/Tools/C_util/AmrDeriveTecplot/AmrDeriveTecplot.cpp @@ -50,7 +50,7 @@ struct Node std::ostream& operator<< (std::ostream& os, const Node& node) { - os << "Node: IntVect=" << node.iv << ", level=" << node.level << ", grid=" << node.grid << ", type="; + os << "Node: IntVect=" << node.iv << ", level=" << node.level << ", grid=" << node.grid << ", type="; if (node.type==Node::INIT) os << "INIT"; else if (node.type==Node::COVERED) @@ -85,41 +85,41 @@ struct Element }; static -void +void print_usage (int, char* argv[]) { - std::cerr - << endl - << " usage:" << endl - << endl - << " " << argv[0] << " [ParmParse input file] [additional keyword=input]" << endl - << endl - << " The first argument is a file name unless it contains an =." << endl - << " With no arguments prints this help." << endl + std::cerr + << endl + << " usage:" << endl + << endl + << " " << argv[0] << " [ParmParse input file] [additional keyword=input]" << endl + << endl + << " The first argument is a file name unless it contains an =." << endl + << " With no arguments prints this help." << endl << endl - << " keywords:" << endl - << endl - << " box = int list of two subbox coords, LL and UR (default all)" << endl - << " comps = integer comp list (overrides sComp and nComp)" << endl - << " finestLevel = <#> finest level to use (default all)" << endl - << " help = prints this help" << endl - << " infile = (required)" << endl - << " nComp = <#> number of comps (default all)" << endl - << " nGrowPer = <#> number of lev-0 cells by which to" << endl - << " extend periodic boundaries (default 0)" << endl - << " sComp = start comp (default 0)" << endl - << " connect_cc = Generate flattened structure by connecting cells centers," + << " keywords:" << endl + << endl + << " box = int list of two subbox coords, LL and UR (default all)" << endl + << " comps = integer comp list (overrides sComp and nComp)" << endl + << " finestLevel = <#> finest level to use (default all)" << endl + << " help = prints this help" << endl + << " infile = (required)" << endl + << " nComp = <#> number of comps (default all)" << endl + << " nGrowPer = <#> number of lev-0 cells by which to" << endl + << " extend periodic boundaries (default 0)" << endl + << " sComp = start comp (default 0)" << endl + << " connect_cc = Generate flattened structure by connecting cells centers," << " otherwise, generate node at all cell corners and copy cc" << " value out (default 1)" << endl - << endl - << " if nGrowPer > 1 then these additional keywords are needed:" << endl - << endl - << " geometry.coord_sys = <0 Cartesion, 1 rz>" << endl - << " geometry.is_periodic = <0, false or 1, true for each axis>" << endl - << " geometry.prob_lo = " << endl - << " geometry.prob_hi = " << endl - << endl; + << endl + << " if nGrowPer > 1 then these additional keywords are needed:" << endl + << endl + << " geometry.coord_sys = <0 Cartesion, 1 rz>" << endl + << " geometry.is_periodic = <0, false or 1, true for each axis>" << endl + << " geometry.prob_lo = " << endl + << " geometry.prob_hi = " << endl + << endl; amrex::Finalize(); exit(1); } @@ -136,7 +136,7 @@ GetBndryCells (const BoxArray& ba, BoxList gcells, bcells; for (int i = 0; i < ba.size(); ++i) - gcells.join(amrex::boxDiff(amrex::grow(ba[i],ngrow),ba[i])); + gcells.join(amrex::boxDiff(amrex::grow(ba[i],ngrow),ba[i])); // // Now strip out intersections with original BoxArray. // @@ -219,7 +219,7 @@ Collate(Vector& NodeRaw, #if BL_USE_MPI const int IOProc = ParallelDescriptor::IOProcessorNumber(); BL_ASSERT(IOProc==0); - const int nProcs = ParallelDescriptor::NProcs(); + const int nProcs = ParallelDescriptor::NProcs(); Vector nmdataR(nProcs,0); Vector offsetR(nProcs,0); // @@ -295,7 +295,7 @@ Collate(Vector& NodeRaw, for (int i = 1; i < nProcs; i++) { const int nodeOffset = offsetR[i]/nCompPerNode; - for (int j = 0; j < nmdataI[i]; j++) + for (int j = 0; j < nmdataI[i]; j++) EltRaw[offsetI[i]+j] += nodeOffset; } } @@ -335,7 +335,7 @@ main (int argc, outfile_DEF = infile+".dat"; #endif - // + // bool connect_cc = true; pp.query("connect_cc",connect_cc); std::string outfile(outfile_DEF); pp.query("outfile",outfile); @@ -423,7 +423,7 @@ main (int argc, if (nGrowPer>0 && geom[lev].isAnyPeriodic() && gridArray[lev].size()>0) { - //const Box& probDomain = amrData.ProbDomain()[lev]; + //const Box& probDomain = amrData.ProbDomain()[lev]; const BoxArray& ba = amrData.boxArray(lev); BoxList bladd; Vector shifts; @@ -454,7 +454,7 @@ main (int argc, const int nGrow = 1; typedef BaseFab NodeFab; - typedef FabArray MultiNodeFab; + typedef FabArray MultiNodeFab; PArray nodes(Nlev,PArrayManage); std::cerr << "Before nodes allocated" << endl; @@ -474,7 +474,7 @@ main (int argc, for (IntVect iv=box.smallEnd(); iv<=box.bigEnd(); box.next(iv)) ifab(iv,0) = Node(iv,lev,fai.index(),Node::VALID); } - + if (lev != 0) { const int ref = amrData.RefRatio()[lev-1]; @@ -525,7 +525,7 @@ main (int argc, { const BoxArray coarsenedFineBoxes = BoxArray(gridArray[lev+1]).coarsen(amrData.RefRatio()[lev]); - + for (MFIter fai(nodes[lev]); fai.isValid(); ++fai) { NodeFab& ifab = nodes[lev][fai]; @@ -549,7 +549,7 @@ main (int argc, { if (ifab(iv,0).type == Node::VALID) { - if (ifab(iv,0).level != lev) + if (ifab(iv,0).level != lev) std::cout << "bad level: " << ifab(iv,0) << std::endl; nodeMap[ifab(iv,0)] = cnt++; } @@ -565,7 +565,7 @@ main (int argc, { for (MFIter fai(nodes[lev]); fai.isValid(); ++fai) { - NodeFab& ifab = nodes[lev][fai]; + NodeFab& ifab = nodes[lev][fai]; Box box = ifab.box() & subboxArray[lev]; for (int dir=0; dirsecond] = (*it).first; } std::cerr << "Final nodeVect built (" << nodeVect.size() << " nodes)" << endl; - + nodeMap.clear(); elements.clear(); nodes.clear(); @@ -693,11 +693,11 @@ main (int argc, for (MFIter mfi(pData); mfi.isValid(); ++mfi) pData[mfi].copy(pDataNG[mfi]); amrData.FillVar(pData,lev,names[comps[i]],0); - pData.EnforcePeriodicity(geom[lev]); + pData.EnforcePeriodicity(geom[lev]); for (MFIter mfi(pData); mfi.isValid(); ++mfi) pDataNG[mfi].copy(pData[mfi]); fileData[lev].copy(pDataNG,0,i,1); - } + } } if (fileData[lev].max(0) > 1.e29) @@ -752,7 +752,7 @@ main (int argc, if (jGrid<0) { bool found_it = false; - + // Try same grid as last time, otherwise search list if (node.level==levPrev && grids[jGridPrev].contains(iv)) { @@ -769,11 +769,11 @@ main (int argc, } BL_ASSERT(found_it); } - // Remember these for next time - levPrev = node.level; - jGridPrev = jGrid; + // Remember these for next time + levPrev = node.level; + jGridPrev = jGrid; - Vector ivt; + Vector ivt; if (connect_cc) { ivt.resize(1,iv); } else { @@ -788,30 +788,30 @@ main (int argc, #endif } - for (int j=0; j> inames[n]; for (int n = 0; n < n_var; n++) os << inames[n] << '\n'; for (int n = 0; n < nnames.size(); n++) os << nnames[n] << '\n'; - + int sdim; is >> sdim; os << sdim << '\n'; - + Real time; is >> time; os << time << '\n'; - + int oFinestLevel; is >> oFinestLevel; - + BL_ASSERT(oFinestLevel>=finestLevel); os << finestLevel << '\n'; - + Vector probLo(BL_SPACEDIM); for (int i = 0; i < BL_SPACEDIM; i++) is >> probLo[i]; for (int i = 0; i < BL_SPACEDIM; i++) os << probLo[i] << ' '; os << '\n'; - + Vector probHi(BL_SPACEDIM); for (int i = 0; i < BL_SPACEDIM; i++) is >> probHi[i]; for (int i = 0; i < BL_SPACEDIM; i++) os << probHi[i] << ' '; os << '\n'; - + Vector refRatio(oFinestLevel); for (int i = 0; i < oFinestLevel; i++) is >> refRatio[i]; for (int i = 0; i < finestLevel; i++) os << refRatio[i] << ' '; os << '\n'; - + Vector probDomain(oFinestLevel+1); for (int i = 0; i <= oFinestLevel; i++) is >> probDomain[i]; for (int i = 0; i <= finestLevel; i++) os << probDomain[i] << ' '; os << '\n'; - + int tmpI; for (int i = 0; i <= oFinestLevel; i++) is >> tmpI; for (int i = 0; i <= finestLevel; i++) os << 0 << ' '; os << '\n'; - + Real dx[BL_SPACEDIM]; for (int i = 0; i <= oFinestLevel; i++) { @@ -117,11 +117,11 @@ AppendToPlotFile(AmrData& amrData, os << '\n'; } } - + int coordSys; is >> coordSys; os << coordSys << '\n'; - + int bndry; is >> bndry; os << bndry << '\n'; // The bndry data width. @@ -130,7 +130,7 @@ AppendToPlotFile(AmrData& amrData, // Write out level by level. // std::string mfBaseName_Unique = mfBaseName; // Possibly modified if same name already used - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) { // @@ -143,7 +143,7 @@ AppendToPlotFile(AmrData& amrData, std::string PathName(oFile); PathName += '/'; PathName += buf; - + if (ParallelDescriptor::IOProcessor()) { int ilev,ngrd; @@ -154,7 +154,7 @@ AppendToPlotFile(AmrData& amrData, int tmpI; is >> tmpI; os << tmpI << '\n'; - + Real glocl,gloch; for (int i = 0; i < nGrids; ++i) { @@ -177,20 +177,20 @@ AppendToPlotFile(AmrData& amrData, // // Force other processors to wait until directory is "built". // - // std::cout << "Above Barrier" << std::endl; + // std::cout << "Above Barrier" << std::endl; ParallelDescriptor::Barrier(); // std::cout << "Below Barrir" << std::endl; std::string RelativePathNameNEW(buf); RelativePathNameNEW += '/'; RelativePathNameNEW += mfBaseName_Unique; - + if (ParallelDescriptor::IOProcessor()) { - // account for multiple multifabs + // account for multiple multifabs int currentIndexComp(0); while(currentIndexComp < n_var) { - + std::string RelativePathNameOLD; is >> RelativePathNameOLD; @@ -229,11 +229,11 @@ AppendToPlotFile(AmrData& amrData, PathName += '/'; PathName += mfBaseName; // std::cout << "Aboe Vismf" < newComps = concatVectors(divuCompNums, vortCompNums); Vector oldCompNums(nOldComps); - for (int i = 0; i < nOldComps; i++) + for (int i = 0; i < nOldComps; i++) oldCompNums[i] = i; Vector allCompNums(nComps); - for (int i = 0; i < nComps; i++) + for (int i = 0; i < nComps; i++) allCompNums[i] = i; // Make list of output multifabs @@ -264,7 +264,7 @@ main (int argc, char* argv[]) // TODO // this part needs to be checked -- it appears that - // FlushGrids works on all levels at once -- but we are + // FlushGrids works on all levels at once -- but we are // looping over levels for (int i = 0; i < oldCompNums.size(); i++) { diff --git a/Tools/C_util/AugmentPlotfile/AugmentPlotfile_F.H b/Tools/C_util/AugmentPlotfile/AugmentPlotfile_F.H index d8be48b83c4..24901b5a105 100644 --- a/Tools/C_util/AugmentPlotfile/AugmentPlotfile_F.H +++ b/Tools/C_util/AugmentPlotfile/AugmentPlotfile_F.H @@ -1,6 +1,6 @@ #ifndef _AUGPLT_F_H_ -#define _AUGPLT_F_H_ +#define _AUGPLT_F_H_ //#ifdef BL_LANG_FORT //# define FORT_VORT_3D vort_3d @@ -48,7 +48,7 @@ extern "C" { const int* vel, const int* divu, amrex_real* delta ); - + void FORT_COPY_3D( const int* srclo , const int* srchi, const amrex_real* srcd, const int* srcNComps, diff --git a/Tools/C_util/Convergence/Add.cpp b/Tools/C_util/Convergence/Add.cpp index e8158da20a2..c9d9da5107a 100644 --- a/Tools/C_util/Convergence/Add.cpp +++ b/Tools/C_util/Convergence/Add.cpp @@ -72,8 +72,8 @@ main (int argc, FArrayBox dataI, dataE; dataI.readFrom(is); - - dataI.plus(factor); + + dataI.plus(factor); dataI.writeOn(os); diff --git a/Tools/C_util/Convergence/ComparePlotfiles.cpp b/Tools/C_util/Convergence/ComparePlotfiles.cpp index 1ed64d142a3..e37692eeb75 100644 --- a/Tools/C_util/Convergence/ComparePlotfiles.cpp +++ b/Tools/C_util/Convergence/ComparePlotfiles.cpp @@ -20,14 +20,14 @@ PrintUsage (const char* progName) << "L1 = sum(|diff_ijk|)/npts_coarsedomain" << std::endl << "L2 = sqrt[sum(diff_ijk^2)]/sqrt(npts_coarsedomain)" << std::endl << "(only single-level supported)" << std::endl << std::endl; - + Print() << "Usage:" << '\n'; Print() << progName << '\n'; Print() << " infile1 = inputFileName1" << '\n'; Print() << " reffile = refinedPlotFile" << '\n'; Print() << " diffile = differenceFileName" << '\n'; Print() << " (If not specified no file is written)" << '\n' << '\n'; - + Print() << "You can either point to the plotfile base directory itself, e.g." << std::endl << " infile=plt00000" << std::endl << "Or the raw data itself, e.g." << std::endl @@ -37,7 +37,7 @@ PrintUsage (const char* progName) << "The program will first try appending 'Level_0/Cell'" << std::endl << "onto the specified filenames." << std::endl << "If that _H file doesn't exist, it tries using the full specified name" << std::endl << std::endl; - + exit(1); } @@ -84,7 +84,7 @@ main (int argc, // storage for the input coarse and fine MultiFabs MultiFab mf_c, mf_f; - + // read in plotfiles, 'coarse' and 'fine' to MultiFabs // note: fine could be the same resolution as coarse VisMF::Read(mf_c, iFile1); @@ -118,12 +118,12 @@ main (int argc, Print() << "npts in coarse domain = " << bx_c.numPts() << std::endl; Print() << "npts in fine domain = " << bx_f.numPts() << std::endl; long npts_coarsedomain = bx_c.numPts(); - + // assume ref_ratio is the same in each direction int rr = bx_f.length(0)/bx_c.length(0); Print() << "ref_ratio = " << rr << std::endl; - + // check to make sure refinement ratio is an integer for (int i=0; i& fine = mf_f2.array(mfi); @@ -188,7 +188,7 @@ main (int argc, #if (AMREX_SPACEDIM==3) for (int kk=0; kk& norm0, - amrex::Vector& norm1, - amrex::Vector& norm2, - bool verbose); + amrex::Vector& norm0, + amrex::Vector& norm1, + amrex::Vector& norm2, + bool verbose); void ComputeAmrDataInt (amrex::AmrData& amrData, amrex::Vector& norm1, - bool verbose); + bool verbose); #endif diff --git a/Tools/C_util/Convergence/ComputeAmrDataNorms.cpp b/Tools/C_util/Convergence/ComputeAmrDataNorms.cpp index cc5709225a3..9d0bbcd4cb7 100644 --- a/Tools/C_util/Convergence/ComputeAmrDataNorms.cpp +++ b/Tools/C_util/Convergence/ComputeAmrDataNorms.cpp @@ -17,37 +17,37 @@ using namespace amrex; void ComputeAmrDataNorms (AmrData& amrData, - Vector& norm0, - Vector& norm1, - Vector& norm2, - bool verbose) + Vector& norm0, + Vector& norm1, + Vector& norm2, + bool verbose) { std::string oFile, iFileDir, oFileDir; - + if (verbose) { - ParmParse pp; - pp.query("outfile", oFile); - if (oFile.empty()) - amrex::Abort("You must specify `outfile' if run in verbose mode"); + ParmParse pp; + pp.query("outfile", oFile); + if (oFile.empty()) + amrex::Abort("You must specify `outfile' if run in verbose mode"); } - + int finestLevel = amrData.FinestLevel(); int nComp = amrData.NComp(); norm0.clear(); norm0.resize(nComp,0.0); norm1.clear(); norm1.resize(nComp,0.0); norm2.clear(); norm2.resize(nComp,0.0); - + Vector refMult(finestLevel + 1, 1); for (int iLevel=finestLevel-1; iLevel>=0; --iLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; - int vol = 1; - for (int i=0; i error(finestLevel+1); - + for (int iLevel = finestLevel; iLevel>=0; --iLevel) { const BoxArray& ba = amrData.boxArray(iLevel); - DistributionMapping dm {ba}; - - error[iLevel] = new MultiFab(ba, dm, nComp, 0); - for (int iComp=0; iCompcopy(data,0,iComp,1); - } - - // Zero out the error covered by fine grid - long covered_volume = 0; - if (iLevel != finestLevel) - { - int ref_ratio = amrData.RefRatio()[iLevel]; - BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); - for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) - { - for (int iGrid=0; iGridcopy(data,0,iComp,1); + } + + // Zero out the error covered by fine grid + long covered_volume = 0; + if (iLevel != finestLevel) + { + int ref_ratio = amrData.RefRatio()[iLevel]; + BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); + for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) + { + for (int iGrid=0; iGrid 0.0) - { - // Convert volume in numPts to volume in number of fine cells - total_volume += long(level_volume); - - // Get norms at this level - Vector n0(nComp,0.0), n1(nComp,0.0), n2(nComp,0.0); - for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) - { - FArrayBox& fab = (*error[iLevel])[mfi]; - const Box& fabbox = mfi.validbox(); - FArrayBox vwFab(fabbox,nComp); - FArrayBox vwFabSqrd(fabbox,nComp); - - // compute volume-weighted error - vwFab.copy(fab,0,0,nComp); - vwFab.mult(refMult[iLevel]); - - vwFabSqrd.copy(fab,0,0,nComp); - vwFabSqrd.mult(fab,0,0,nComp); - vwFabSqrd.mult(refMult[iLevel]); - - for (int iComp=0; iComp 0.0) + { + // Convert volume in numPts to volume in number of fine cells + total_volume += long(level_volume); + + // Get norms at this level + Vector n0(nComp,0.0), n1(nComp,0.0), n2(nComp,0.0); + for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) + { + FArrayBox& fab = (*error[iLevel])[mfi]; + const Box& fabbox = mfi.validbox(); + FArrayBox vwFab(fabbox,nComp); + FArrayBox vwFabSqrd(fabbox,nComp); + + // compute volume-weighted error + vwFab.copy(fab,0,0,nComp); + vwFab.mult(refMult[iLevel]); + + vwFabSqrd.copy(fab,0,0,nComp); + vwFabSqrd.mult(fab,0,0,nComp); + vwFabSqrd.mult(refMult[iLevel]); + + for (int iComp=0; iComp& norm1, - bool verbose) + Vector& norm1, + bool verbose) { std::string oFile, iFileDir, oFileDir; - + if (verbose) { - ParmParse pp; - pp.query("outfile", oFile); - if (oFile.empty()) - amrex::Abort("You must specify `outfile' if run in verbose mode"); + ParmParse pp; + pp.query("outfile", oFile); + if (oFile.empty()) + amrex::Abort("You must specify `outfile' if run in verbose mode"); } - + int finestLevel = amrData.FinestLevel(); int nComp = amrData.NComp(); norm1.clear(); norm1.resize(nComp,0.0); - + Vector refMult(finestLevel + 1, 1); for (int iLevel=finestLevel-1; iLevel>=0; --iLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; - int vol = 1; - for (int i=0; i error(finestLevel+1); - + for (int iLevel = finestLevel; iLevel>=0; --iLevel) { const BoxArray& ba = amrData.boxArray(iLevel); - DistributionMapping dm {ba}; - - error[iLevel] = new MultiFab(ba, dm, nComp, 0); - for (int iComp=0; iCompcopy(data,0,iComp,1); - } - - // Zero out the error covered by fine grid - long covered_volume = 0; - if (iLevel != finestLevel) - { - int ref_ratio = amrData.RefRatio()[iLevel]; - BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); - for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) - { - for (int iGrid=0; iGridcopy(data,0,iComp,1); + } + + // Zero out the error covered by fine grid + long covered_volume = 0; + if (iLevel != finestLevel) + { + int ref_ratio = amrData.RefRatio()[iLevel]; + BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); + for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) + { + for (int iGrid=0; iGrid 0.0) - { - // Convert volume in numPts to volume in number of fine cells - total_volume += long(level_volume); - - // Get norms at this level - Vector n1(nComp,0.0); - for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) - { - FArrayBox& fab = (*error[iLevel])[mfi]; - const Box& fabbox = mfi.validbox(); - FArrayBox vwFab(fabbox,nComp); - - // compute volume-weighted error - vwFab.copy(fab,0,0,nComp); - for (int iComp=0; iComp 0.0) + { + // Convert volume in numPts to volume in number of fine cells + total_volume += long(level_volume); + + // Get norms at this level + Vector n1(nComp,0.0); + for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) + { + FArrayBox& fab = (*error[iLevel])[mfi]; + const Box& fabbox = mfi.validbox(); + FArrayBox vwFab(fabbox,nComp); + + // compute volume-weighted error + vwFab.copy(fab,0,0,nComp); + for (int iComp=0; iCompgetDomain()); - Real time = 0; int level_step = 0; + Real time = 0; int level_step = 0; WriteSingleLevelPlotfile(filename, *a_data, names, geom, time, level_step); @@ -120,7 +120,7 @@ viewMF(const MultiFab* a_data) } -void +void maxMinMF(const MultiFab* a_data) { amrex::Print() << "max min for multifab " <& derives = amrData1.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; - + // // Compute the error // Vector error(finestLevel+1); - + if (ParallelDescriptor::IOProcessor()) std::cout << "Level L"<< norm << " norm of Error in Each Component" << std::endl << "-----------------------------------------------" << std::endl; @@ -186,8 +186,8 @@ main (int argc, // // Construct MultiFab for errors // - error[iLevel] = new MultiFab(ba2Coarse, dm2Coarse, nComp, 0); - error[iLevel]->setVal(GARBAGE); + error[iLevel] = new MultiFab(ba2Coarse, dm2Coarse, nComp, 0); + error[iLevel]->setVal(GARBAGE); // // For each component, average the fine fields down and calculate @@ -215,16 +215,16 @@ main (int argc, // int index = mfi.index(); - const Box& bx = ba2Coarse[index]; + const Box& bx = ba2Coarse[index]; FArrayBox data2Coarse(bx, 1); int ncCoarse = 1; FORT_CV_AVGDOWN(data2Coarse.dataPtr(), - ARLIM(bx.loVect()), ARLIM(bx.hiVect()), + ARLIM(bx.loVect()), ARLIM(bx.hiVect()), &ncCoarse, data2Fine[mfi].dataPtr(), ARLIM(data2Fine[mfi].loVect()), - ARLIM(data2Fine[mfi].hiVect()), + ARLIM(data2Fine[mfi].hiVect()), bx.loVect(), bx.hiVect(), refine_ratio.getVect()); @@ -266,8 +266,8 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), + int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -286,7 +286,7 @@ main (int argc, } else { - int rc = MPI_Send(norms.dataPtr(), nComp, datatype, + int rc = MPI_Send(norms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -320,9 +320,9 @@ main (int argc, if (!difFile.empty()) WritePlotFile(error, amrData1, difFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); } @@ -330,16 +330,16 @@ main (int argc, bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2) + const AmrData& amrd2) { const Vector& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i& derives = amrData1.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; - + // // Compute the error // Vector error(finestLevel+1); - + if (ParallelDescriptor::IOProcessor()) std::cout << "Level L"<< norm << " norm of Error in Each Component" << std::endl << "-----------------------------------------------" << std::endl; - + int ratio = 1; Vector sum_norms(nComp); for (int iComp = 0; iComp < nComp; iComp++) @@ -215,7 +215,7 @@ main (int argc, baf = BoxArray(amrData1.boxArray(iLevel+1)).coarsen(refine_ratio); } - + // Copy the data at the coarse level one component at a time new_data1.copy(data1,0,0,1); @@ -228,24 +228,24 @@ main (int argc, // Create the Coarsened version of data2 // int index = mfi.index(); - - const Box& bx = ba2Coarse[index]; + + const Box& bx = ba2Coarse[index]; FArrayBox data2Coarse(bx, 1); int ncCoarse = 1; - + FORT_CV_AVGDOWN(data2Coarse.dataPtr(), - ARLIM(bx.loVect()), ARLIM(bx.hiVect()), + ARLIM(bx.loVect()), ARLIM(bx.hiVect()), &ncCoarse, data2Fine[mfi].dataPtr(), ARLIM(data2Fine[mfi].loVect()), - ARLIM(data2Fine[mfi].hiVect()), + ARLIM(data2Fine[mfi].hiVect()), bx.loVect(), bx.hiVect(), refine_ratio.getVect()); - // + // // Calculate the errors on this FAB for this component // @@ -255,11 +255,11 @@ main (int argc, if (iLevel > isects = baf.intersections(bx); - + for (int ii = 0; ii < isects.size(); ii++) (*error[iLevel])[mfi].setVal(0,isects[ii].second,iComp,1); } - + Real grdL2 = (*error[iLevel])[mfi].norm(norm, iComp, 1); if (norm != 0) @@ -290,8 +290,8 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), + int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -310,7 +310,7 @@ main (int argc, } else { - int rc = MPI_Send(norms.dataPtr(), nComp, datatype, + int rc = MPI_Send(norms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -338,7 +338,7 @@ main (int argc, std::cout << norms[iComp] << " "; sum_norms[iComp] = sum_norms[iComp] + norms[iComp]; } - + std::cout << std::endl; } } @@ -352,12 +352,12 @@ main (int argc, std::cout << sum_norms[iComp] << " "; } } - + if (!difFile.empty()) WritePlotFile(error, amrData1, difFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); } @@ -365,16 +365,16 @@ main (int argc, bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2) + const AmrData& amrd2) { const Vector& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; isetVal(GARBAGE); + error[iLevel] = new MultiFab(ba2Coarse, nComp, 0); + error[iLevel]->setVal(GARBAGE); // // For each component, compute errors by striding through fine @@ -217,24 +217,24 @@ main (int argc, FArrayBox data2Coarse(ba2Coarse[index], 1); int ncCoarse = 1; - int i, j, k; - IntVect cidx, fidx; - const int *lo = data2Coarse.loVect(); - const int *hi = data2Coarse.hiVect(); + int i, j, k; + IntVect cidx, fidx; + const int *lo = data2Coarse.loVect(); + const int *hi = data2Coarse.hiVect(); - for (i=lo[0]; i<=hi[0]; i++) { - cidx[0] = i; fidx[0] = i * refine_ratio[0]; + for (i=lo[0]; i<=hi[0]; i++) { + cidx[0] = i; fidx[0] = i * refine_ratio[0]; - for (j=lo[1]; j<=hi[1]; j++) { - cidx[1] = j; fidx[1] = j * refine_ratio[1]; + for (j=lo[1]; j<=hi[1]; j++) { + cidx[1] = j; fidx[1] = j * refine_ratio[1]; - for (k=lo[2]; k<=hi[2]; k++) { - cidx[2] = k; fidx[2] = k * refine_ratio[2]; + for (k=lo[2]; k<=hi[2]; k++) { + cidx[2] = k; fidx[2] = k * refine_ratio[2]; - data2Coarse(cidx) = data2Fine[mfi](fidx); - } - } - } + data2Coarse(cidx) = data2Fine[mfi](fidx); + } + } + } // // Calculate the errors on this FAB for this component @@ -324,7 +324,7 @@ main (int argc, WritePlotFile(error, amrData1, difFile, verbose); for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); } @@ -332,16 +332,16 @@ main (int argc, bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2) + const AmrData& amrd2) { const Vector& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i& derives = amrData1.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; - + // // Compute the error // Vector error(finestLevel+1); - + if (ParallelDescriptor::IOProcessor()) std::cout << "Level L"<< norm << " norm of Error in Each Component" << std::endl << "-----------------------------------------------" << std::endl; @@ -161,11 +161,11 @@ main (int argc, const BoxArray& ba1 = amrData1.boxArray(iLevel); const BoxArray& ba2 = amrData2.boxArray(iLevel); - /* + /* if (ba1.size() != ba2.size()) - std::cout << "Warning: BoxArray lengths are not the same at level " - << iLevel << std::endl; - */ + std::cout << "Warning: BoxArray lengths are not the same at level " + << iLevel << std::endl; + */ // // Construct refinement ratio, build the coarsened boxarray @@ -174,19 +174,19 @@ main (int argc, const Box& domain1 = amrData1.ProbDomain()[iLevel]; const Box& domain2 = amrData2.ProbDomain()[iLevel]; - int nodal_dir = -1; - for (int i=0; isetVal(GARBAGE); + error[iLevel] = new MultiFab(ba2Coarse, dm2Coarse, nComp, 0); + error[iLevel]->setVal(GARBAGE); // // For each component, average the fine fields down and calculate @@ -240,20 +240,20 @@ main (int argc, int ncCoarse = 1; Box box2c = ba2Coarse[mfi]; - IntVect loiv = box2c.smallEnd(); - IntVect hiiv = box2c.bigEnd(); + IntVect loiv = box2c.smallEnd(); + IntVect hiiv = box2c.bigEnd(); FORT_CV_AVGDOWN_STAG(&nodal_dir, - data2Coarse.dataPtr(), + data2Coarse.dataPtr(), ARLIM(data2Coarse.loVect()), - ARLIM(data2Coarse.hiVect()), - &ncCoarse, - data2Fine[mfi].dataPtr(), - ARLIM(data2Fine[mfi].loVect()), - ARLIM(data2Fine[mfi].hiVect()), - loiv.getVect(), - hiiv.getVect(), - refine_ratio.getVect()); + ARLIM(data2Coarse.hiVect()), + &ncCoarse, + data2Fine[mfi].dataPtr(), + ARLIM(data2Fine[mfi].loVect()), + ARLIM(data2Fine[mfi].hiVect()), + loiv.getVect(), + hiiv.getVect(), + refine_ratio.getVect()); // @@ -293,8 +293,8 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), + int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -313,7 +313,7 @@ main (int argc, } else { - int rc = MPI_Send(norms.dataPtr(), nComp, datatype, + int rc = MPI_Send(norms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -347,9 +347,9 @@ main (int argc, if (!difFile.empty()) WritePlotFile(error, amrData1, difFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); } @@ -357,16 +357,16 @@ main (int argc, bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2) + const AmrData& amrd2) { const Vector& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i& derives = amrDataI.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; // // Compute the error // Vector error(finestLevel+1); - + if (ParallelDescriptor::IOProcessor()) std::cout << "L"<< norm << " norm of Error in Each Component" << std::endl << "-----------------------------------------" << std::endl; @@ -143,8 +143,8 @@ main (int argc, } DistributionMapping dmI(baI); - error[iLevel] = new MultiFab(baI, dmI, nComp, 0); - error[iLevel]->setVal(GARBAGE); + error[iLevel] = new MultiFab(baI, dmI, nComp, 0); + error[iLevel]->setVal(GARBAGE); DistributionMapping dmE(baE); MultiFab dataI(baI, dmI, nComp, 0); @@ -165,7 +165,7 @@ main (int argc, // Output Statistics // if (ParallelDescriptor::IOProcessor()) - std::cout << "Level: " << iLevel << std::endl; + std::cout << "Level: " << iLevel << std::endl; Vector norms(nComp,0); @@ -193,8 +193,8 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), + int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -212,7 +212,7 @@ main (int argc, } else { - int rc = MPI_Send(norms.dataPtr(), nComp, datatype, + int rc = MPI_Send(norms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -244,9 +244,9 @@ main (int argc, if (!difFile.empty()) WritePlotFile(error, amrDataI, difFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); diff --git a/Tools/C_util/Convergence/DiffSameGrid2.cpp b/Tools/C_util/Convergence/DiffSameGrid2.cpp index ccbb884d992..03812bab8df 100644 --- a/Tools/C_util/Convergence/DiffSameGrid2.cpp +++ b/Tools/C_util/Convergence/DiffSameGrid2.cpp @@ -29,10 +29,10 @@ void PrintUsage (const char* progName) { std::cout << "\nThis utility performs a diff operation between two" << std::endl - << "plotfiles which have the exact same grids." << std::endl - << "Both absolute and relative differences are reported." << std::endl - << "Note: no attempt has been made to prevent a divide by 0, " << std::endl - << "so some relative errors may not be well defined." << std::endl; + << "plotfiles which have the exact same grids." << std::endl + << "Both absolute and relative differences are reported." << std::endl + << "Note: no attempt has been made to prevent a divide by 0, " << std::endl + << "so some relative errors may not be well defined." << std::endl; std::cout << '\n'; std::cout << "Usage:" << '\n'; std::cout << progName << '\n'; @@ -50,16 +50,16 @@ PrintUsage (const char* progName) static bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2) + const AmrData& amrd2) { const Vector& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i& derives = amrDataI.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; // // Compute the absolute and relative errors @@ -139,25 +139,25 @@ main (int argc, if (ParallelDescriptor::IOProcessor()) std::cout << "L"<< norm << " norm of Absolute and Relative Error in Each Component" << std::endl - << std::setfill('-') << std::setw(80) << "-" << std::setfill(' ') << std::endl; + << std::setfill('-') << std::setw(80) << "-" << std::setfill(' ') << std::endl; for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) { const BoxArray& baI = amrDataI.boxArray(iLevel); const BoxArray& baE = amrDataE.boxArray(iLevel); - DistributionMapping dm {baI}; + DistributionMapping dm {baI}; if (baI != baE) { std::cout << "ERROR: BoxArrays are not the same at level " << iLevel << std::endl; ParallelDescriptor::Abort(); } - aerror[iLevel] = new MultiFab(baI, dm, nComp, 0); - aerror[iLevel]->setVal(GARBAGE); + aerror[iLevel] = new MultiFab(baI, dm, nComp, 0); + aerror[iLevel]->setVal(GARBAGE); - rerror[iLevel] = new MultiFab(baI, dm, nComp, 0); - rerror[iLevel]->setVal(GARBAGE); + rerror[iLevel] = new MultiFab(baI, dm, nComp, 0); + rerror[iLevel]->setVal(GARBAGE); MultiFab dataI(baI, dm, nComp, 0); MultiFab dataE(baE, dm, nComp, 0); @@ -165,30 +165,30 @@ main (int argc, amrDataI.FillVar(dataI, iLevel, derives, destComps); amrDataE.FillVar(dataE, iLevel, derives, destComps); - // this part needs to be checked -- it appears that - // FlushGrids works on all levels at once -- but we are - // looping over levels + // this part needs to be checked -- it appears that + // FlushGrids works on all levels at once -- but we are + // looping over levels for (int i = 0; i < destComps.size(); i++) { amrDataI.FlushGrids(destComps[i]); amrDataE.FlushGrids(destComps[i]); } - // aerror will contain the absolute errors + // aerror will contain the absolute errors (*aerror[iLevel]).copy(dataI); (*aerror[iLevel]).minus(dataE, 0, nComp, 0); - // rerror will contain the relative errors + // rerror will contain the relative errors (*rerror[iLevel]).copy(dataI); (*rerror[iLevel]).minus(dataE, 0, nComp, 0); - (*rerror[iLevel]).divide(dataI, 0, nComp, 0); + (*rerror[iLevel]).divide(dataI, 0, nComp, 0); // // Output Statistics // if (ParallelDescriptor::IOProcessor()) - std::cout << "Level: " << iLevel << std::endl; + std::cout << "Level: " << iLevel << std::endl; Vector anorms(nComp,0); Vector rnorms(nComp,0); @@ -197,23 +197,23 @@ main (int argc, { for (int iComp = 0; iComp < nComp; iComp++) { - - // compute the norm of the absolute and relative errors for the - // current FAB - const Real agrdL2 = (*aerror[iLevel])[mfi].norm(norm, iComp, 1); - const Real rgrdL2 = (*rerror[iLevel])[mfi].norm(norm, iComp, 1); - - // Adding the current norm to that from the previous FABs. - // note: right now, we are storing anorm**norm, so that the - // summation makes sense. - if (norm != 0) { - anorms[iComp] = anorms[iComp] + pow(agrdL2, norm); - rnorms[iComp] = rnorms[iComp] + pow(rgrdL2, norm); - } - else { - anorms[iComp] = std::max(anorms[iComp], agrdL2); - rnorms[iComp] = std::max(rnorms[iComp], rgrdL2); - } + + // compute the norm of the absolute and relative errors for the + // current FAB + const Real agrdL2 = (*aerror[iLevel])[mfi].norm(norm, iComp, 1); + const Real rgrdL2 = (*rerror[iLevel])[mfi].norm(norm, iComp, 1); + + // Adding the current norm to that from the previous FABs. + // note: right now, we are storing anorm**norm, so that the + // summation makes sense. + if (norm != 0) { + anorms[iComp] = anorms[iComp] + pow(agrdL2, norm); + rnorms[iComp] = rnorms[iComp] + pow(rgrdL2, norm); + } + else { + anorms[iComp] = std::max(anorms[iComp], agrdL2); + rnorms[iComp] = std::max(rnorms[iComp], rgrdL2); + } } } @@ -230,19 +230,19 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(atmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, - proc, - ParallelDescriptor::Communicator(), + int rc = MPI_Recv(atmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, + proc, + ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) ParallelDescriptor::Abort(rc); - rc = MPI_Recv(rtmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, - ParallelDescriptor::NProcs() + proc, - ParallelDescriptor::Communicator(), + rc = MPI_Recv(rtmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, + ParallelDescriptor::NProcs() + proc, + ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -251,14 +251,14 @@ main (int argc, for (int iComp = 0; iComp < nComp; iComp++) { - if (norm != 0) { - anorms[iComp] = anorms[iComp] + atmp[iComp]; - rnorms[iComp] = rnorms[iComp] + rtmp[iComp]; - } - else { - anorms[iComp] = std::max(anorms[iComp], atmp[iComp]); - rnorms[iComp] = std::max(rnorms[iComp], rtmp[iComp]); - } + if (norm != 0) { + anorms[iComp] = anorms[iComp] + atmp[iComp]; + rnorms[iComp] = rnorms[iComp] + rtmp[iComp]; + } + else { + anorms[iComp] = std::max(anorms[iComp], atmp[iComp]); + rnorms[iComp] = std::max(rnorms[iComp], rtmp[iComp]); + } } } @@ -266,7 +266,7 @@ main (int argc, } else { - int rc = MPI_Send(anorms.dataPtr(), nComp, datatype, + int rc = MPI_Send(anorms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -275,7 +275,7 @@ main (int argc, ParallelDescriptor::Abort(rc); - rc = MPI_Send(rnorms.dataPtr(), nComp, datatype, + rc = MPI_Send(rnorms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::NProcs() + ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -285,7 +285,7 @@ main (int argc, } #endif - // normalize the norms and print out + // normalize the norms and print out Real vol = 1.0; for (int dir = 0; dir < BL_SPACEDIM; dir++) vol *= amrDataI.DxLevel()[iLevel][dir]; @@ -296,18 +296,18 @@ main (int argc, { if (norm != 0) { - anorms[iComp] = anorms[iComp] * vol; - anorms[iComp] = pow(anorms[iComp], (1.0/norm)); + anorms[iComp] = anorms[iComp] * vol; + anorms[iComp] = pow(anorms[iComp], (1.0/norm)); - rnorms[iComp] = rnorms[iComp] * vol; - rnorms[iComp] = pow(rnorms[iComp], (1.0/norm)); + rnorms[iComp] = rnorms[iComp] * vol; + rnorms[iComp] = pow(rnorms[iComp], (1.0/norm)); } if (anorms[iComp] != 0.0) failed = true; - std::cout << " " << std::setw(32) << derives[iComp] - << ": " << std::setw(20) << anorms[iComp] - << std::setw(20) << rnorms[iComp] << std::endl; + std::cout << " " << std::setw(32) << derives[iComp] + << ": " << std::setw(20) << anorms[iComp] + << std::setw(20) << rnorms[iComp] << std::endl; } std::cout << std::endl; } @@ -320,7 +320,7 @@ main (int argc, // optionally dump out a plotfile containing the absolute errors if (!difFile.empty()) WritePlotFile(aerror, amrDataI, difFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) { delete aerror[iLevel]; delete rerror[iLevel]; diff --git a/Tools/C_util/Convergence/DiffSameGridRefined.cpp b/Tools/C_util/Convergence/DiffSameGridRefined.cpp index 9c74668623a..ce2cf352ce3 100644 --- a/Tools/C_util/Convergence/DiffSameGridRefined.cpp +++ b/Tools/C_util/Convergence/DiffSameGridRefined.cpp @@ -59,7 +59,7 @@ PrintUsage (const char* progName) bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2); + const AmrData& amrd2); IntVect getRefRatio(const Box& crse, @@ -106,7 +106,7 @@ main (int argc, DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); - + DataServices dataServices1(iFile1, fileType); DataServices dataServices2(iFile2, fileType); @@ -115,13 +115,13 @@ main (int argc, // - // Generate AmrData Objects + // Generate AmrData Objects // AmrData& amrData1 = dataServices1.AmrDataRef(); AmrData& amrData2 = dataServices2.AmrDataRef(); // - // Initial Tests + // Initial Tests // if (!amrDatasHaveSameDerives(amrData1,amrData2)) amrex::Abort("ERROR: Plotfiles do not have the same state variables"); @@ -133,15 +133,15 @@ main (int argc, int finestLevel = amrData1.FinestLevel(); const Vector& derives = amrData1.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; - + // // Compute the error // Vector error(finestLevel+1); - + if (ParallelDescriptor::IOProcessor()) std::cout << "Level L"<< norm << " norm of Error in Each Component" << std::endl << "-----------------------------------------------" << std::endl; @@ -156,7 +156,7 @@ main (int argc, if (ba1.size() != ba2.size()) { - std::cout << "ERROR: BoxArray lengths are not the same at level " + std::cout << "ERROR: BoxArray lengths are not the same at level " << iLevel << std::endl; ParallelDescriptor::Abort(); } @@ -165,8 +165,8 @@ main (int argc, // Construct MultiFab for errors // DistributionMapping dm(ba1); - error[iLevel] = new MultiFab(ba1, dm, nComp, 0); - error[iLevel]->setVal(GARBAGE); + error[iLevel] = new MultiFab(ba1, dm, nComp, 0); + error[iLevel]->setVal(GARBAGE); // // Construct refinement ratio, build the coarsened boxarray @@ -209,7 +209,7 @@ main (int argc, // int index = mfi.index(); - const Box& bx = ba2Coarse[index]; + const Box& bx = ba2Coarse[index]; FArrayBox data2Coarse(bx, 1); int ncCoarse = data2Coarse.nComp(); @@ -218,7 +218,7 @@ main (int argc, &ncCoarse, data2Fine[mfi].dataPtr(), ARLIM(data2Fine[mfi].loVect()), - ARLIM(data2Fine[mfi].hiVect()), + ARLIM(data2Fine[mfi].hiVect()), bx.loVect(), bx.hiVect(), refine_ratio.getVect()); @@ -261,8 +261,8 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), + int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -281,7 +281,7 @@ main (int argc, } else { - int rc = MPI_Send(norms.dataPtr(), nComp, datatype, + int rc = MPI_Send(norms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); @@ -315,9 +315,9 @@ main (int argc, if (!difFile.empty()) WritePlotFile(error, amrData1, difFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); } @@ -325,16 +325,16 @@ main (int argc, bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2) + const AmrData& amrd2) { const Vector& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i #define GARBAGE 666.e+40 -using namespace amrex; +using namespace amrex; static void PrintUsage (const char* progName) @@ -53,11 +53,11 @@ finestLevelCoveringDomain(const AmrData& amrData); IntVect getRefRatio(const Box& crse, - const Box& fine); + const Box& fine); bool amrDatasHaveSameDerives(const AmrData& amrd1, - const AmrData& amrd2); + const AmrData& amrd2); int main (int argc, char* argv[]) @@ -98,7 +98,7 @@ main (int argc, DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); - + DataServices dataServicesC(iFile, fileType); DataServices dataServicesF(eFile, fileType); @@ -115,43 +115,43 @@ main (int argc, int exact_level = finestLevelCoveringDomain(amrDataF); if (exact_level < 0) { - std::cout << "Exact data does not contain a level covering the domain" << '\n'; - amrex::Abort(); + std::cout << "Exact data does not contain a level covering the domain" << '\n'; + amrex::Abort(); } if (verbose) - std::cout << "Using level = " << exact_level << " in 'exact' file" << '\n'; - + std::cout << "Using level = " << exact_level << " in 'exact' file" << '\n'; + int finestLevel = amrDataC.FinestLevel(); - + // // Compute the error // Vector error(finestLevel+1); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) { const BoxArray& crseBA = amrDataC.boxArray(iLevel); - int nComp = amrDataC.NComp(); - const Box& domainC = amrDataC.ProbDomain()[iLevel]; - const Box& domainF = amrDataF.ProbDomain()[exact_level]; - IntVect refine_ratio = getRefRatio(domainC, domainF); - if (refine_ratio == IntVect()) - amrex::Error("Cannot find refinement ratio from data to exact"); + int nComp = amrDataC.NComp(); + const Box& domainC = amrDataC.ProbDomain()[iLevel]; + const Box& domainF = amrDataF.ProbDomain()[exact_level]; + IntVect refine_ratio = getRefRatio(domainC, domainF); + if (refine_ratio == IntVect()) + amrex::Error("Cannot find refinement ratio from data to exact"); - if (ParallelDescriptor::IOProcessor()) - std::cout << "Ratio for level " << iLevel << " is " << refine_ratio << std::endl; + if (ParallelDescriptor::IOProcessor()) + std::cout << "Ratio for level " << iLevel << " is " << refine_ratio << std::endl; DistributionMapping dm(crseBA); - error[iLevel] = new MultiFab(crseBA, dm, nComp, 0); - error[iLevel]->setVal(GARBAGE); + error[iLevel] = new MultiFab(crseBA, dm, nComp, 0); + error[iLevel]->setVal(GARBAGE); - for (int iComp=0; iCompcopy(aveExact,0,iComp,nc); - + // Subtract coarse data from coarsened exact data MultiFab& data = amrDataC.GetGrids(iLevel,iComp); BL_ASSERT(data.boxArray() == error[iLevel]->boxArray()); - Real norm_before = (*error[iLevel]).norm2(iComp); + Real norm_before = (*error[iLevel]).norm2(iComp); - if (ParallelDescriptor::IOProcessor()) - { - std::cout << "DOING ICOMP " << iComp << std::endl; - std::cout << "BEFORE: NORM OF ERROR " << norm_before << std::endl; - } - for (MFIter dmfi(data); dmfi.isValid(); ++dmfi) + if (ParallelDescriptor::IOProcessor()) { - (*error[iLevel])[dmfi].minus(data[dmfi],0,iComp,1); - + std::cout << "DOING ICOMP " << iComp << std::endl; + std::cout << "BEFORE: NORM OF ERROR " << norm_before << std::endl; } - Real norm_after = (*error[iLevel]).norm2(iComp); + for (MFIter dmfi(data); dmfi.isValid(); ++dmfi) + { + (*error[iLevel])[dmfi].minus(data[dmfi],0,iComp,1); + + } + Real norm_after = (*error[iLevel]).norm2(iComp); - if (ParallelDescriptor::IOProcessor()) - std::cout << "AFTER: NORM OF ERROR " << norm_after << std::endl; + if (ParallelDescriptor::IOProcessor()) + std::cout << "AFTER: NORM OF ERROR " << norm_after << std::endl; } } WritePlotFile(error, amrDataC, oFile, verbose); - + for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) - delete error[iLevel]; + delete error[iLevel]; amrex::Finalize(); } @@ -213,19 +213,19 @@ finestLevelCoveringDomain(const AmrData& amr_data) for (int iLevel=finest_level; iLevel>=0; --iLevel) { - const BoxArray& ba = amr_data.boxArray(iLevel); - BoxDomain bd; - bd.add(BoxList(ba)); - BoxDomain complement = amrex::complementIn(domain_array[iLevel],bd); - if (complement.isEmpty()) - return iLevel; + const BoxArray& ba = amr_data.boxArray(iLevel); + BoxDomain bd; + bd.add(BoxList(ba)); + BoxDomain complement = amrex::complementIn(domain_array[iLevel],bd); + if (complement.isEmpty()) + return iLevel; } return -1; } IntVect getRefRatio(const Box& crse, - const Box& fine) + const Box& fine) { // Compute refinement ratio between crse and fine boxes, return invalid // IntVect if there is none suitable @@ -234,7 +234,7 @@ getRefRatio(const Box& crse, int Nrr = 0; Nrr = pp.countval("ref_ratio",Nrr); BL_ASSERT(Nrr==0 || Nrr==BL_SPACEDIM || Nrr==1); - if (Nrr>0) + if (Nrr>0) { pp.queryarr("ref_ratio",rr_in,0,Nrr); return IntVect(rr_in); @@ -242,28 +242,28 @@ getRefRatio(const Box& crse, IntVect ref_ratio; for (int i=0; i& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); int length = derives1.size(); if (length != derives2.size()) - return false; + return false; for (int i=0; i& names = amrData.PlotVarNames(); - int maxl = 0; - for (int i=0; i& names = amrData.PlotVarNames(); + int maxl = 0; + for (int i=0; i& names = amrData.PlotVarNames(); - int maxl = 0; - for (int i=0; i& names = amrData.PlotVarNames(); + int maxl = 0; + for (int i=0; i& derives = amrDataI.PlotVarNames(); Vector destComps(nComp); - for (int i = 0; i < nComp; i++) + for (int i = 0; i < nComp; i++) destComps[i] = i; - + // // Compute the error // Vector error(finestLevel+1); - + if (ParallelDescriptor::IOProcessor()) std::cout << "Level L"<< norm << " norm of Error in Each Component" << std::endl << "-----------------------------------------------" << std::endl; @@ -139,7 +139,7 @@ main (int argc, { norms[iComp] = std::max(norms[iComp], grdL2); } - + } } @@ -153,8 +153,8 @@ main (int argc, if (proc != ParallelDescriptor::IOProcessorNumber()) { MPI_Status stat; - int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, - MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), + int rc = MPI_Recv(tmp.dataPtr(), nComp, datatype, + MPI_ANY_SOURCE, proc, ParallelDescriptor::Communicator(), &stat); if (rc != MPI_SUCCESS) @@ -173,7 +173,7 @@ main (int argc, } else { - int rc = MPI_Send(norms.dataPtr(), nComp, datatype, + int rc = MPI_Send(norms.dataPtr(), nComp, datatype, ParallelDescriptor::IOProcessorNumber(), ParallelDescriptor::MyProc(), ParallelDescriptor::Communicator()); diff --git a/Tools/C_util/Convergence/PltFileScalConvRate.cpp b/Tools/C_util/Convergence/PltFileScalConvRate.cpp index 072dee925cb..49c56ce94ad 100644 --- a/Tools/C_util/Convergence/PltFileScalConvRate.cpp +++ b/Tools/C_util/Convergence/PltFileScalConvRate.cpp @@ -76,7 +76,7 @@ main (int argc, DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); - + DataServices dataServicesC(cFile, fileType); DataServices dataServicesF(fFile, fileType); @@ -90,38 +90,38 @@ main (int argc, // Write norms to screen if (ParallelDescriptor::IOProcessor()) { - const Vector& names = amrDataC.PlotVarNames(); - int maxl = 0; - for (int i=0; i& names = amrDataC.PlotVarNames(); + int maxl = 0; + for (int i=0; i& derives1 = amrd1.PlotVarNames(); const Vector& derives2 = amrd2.PlotVarNames(); diff --git a/Tools/C_util/DiffMultiFab/diffmultifab.cpp b/Tools/C_util/DiffMultiFab/diffmultifab.cpp index 2741ee6ae89..8680648b739 100644 --- a/Tools/C_util/DiffMultiFab/diffmultifab.cpp +++ b/Tools/C_util/DiffMultiFab/diffmultifab.cpp @@ -8,14 +8,14 @@ #include #ifdef AMREX_USE_MPI -#define MY_USE_MPI 1 -#else +#define MY_USE_MPI 1 +#else #define MY_USE_MPI 0 -#endif +#endif using namespace amrex; -void +void print_usage (int, char* argv[]) { @@ -26,7 +26,7 @@ print_usage (int, int main(int argc, char* argv[]) { - static_assert(MY_USE_MPI!=1, "cannot work with MPI" ); + static_assert(MY_USE_MPI!=1, "cannot work with MPI" ); amrex::Initialize(argc,argv); { if (argc < 3) @@ -35,10 +35,10 @@ int main(int argc, char* argv[]) std::string name1, name2; int ngrow = -1; { - ParmParse pp; - pp.get("infile1", name1); - pp.get("infile2", name2); - pp.query("ngrow", ngrow); + ParmParse pp; + pp.get("infile1", name1); + pp.get("infile2", name2); + pp.query("ngrow", ngrow); } MultiFab mf1, mf2; @@ -54,9 +54,9 @@ int main(int argc, char* argv[]) Abort(" ngrow bigger than infile1's ngrow! "); if (ngrow > mf2.nGrow()) Abort(" ngrow bigger than infile2's ngrow! "); - + if (mf1.boxArray() != mf2.boxArray()) { - Abort("The two multifabs have different BoxArray"); + Abort("The two multifabs have different BoxArray"); } const int ncomp = mf1.nComp(); @@ -72,7 +72,7 @@ int main(int argc, char* argv[]) mf2_min[icomp] = mf2.min(icomp,ngrow); mf2_max[icomp] = mf2.max(icomp,ngrow); } -*/ +*/ MultiFab mfdiff(mf1.boxArray(), mf1.DistributionMap(), ncomp, ngrow); @@ -80,18 +80,18 @@ int main(int argc, char* argv[]) MultiFab::Subtract(mfdiff, mf2, 0, 0, ncomp, ngrow); for (int icomp = 0; icomp < ncomp; ++icomp) { - Print() << "Component " << icomp << std::endl; - Print() << "diff Min,max: " << mfdiff.min(icomp,ngrow) - << " , " << mfdiff.max(icomp,ngrow) << std::endl; + Print() << "Component " << icomp << std::endl; + Print() << "diff Min,max: " << mfdiff.min(icomp,ngrow) + << " , " << mfdiff.max(icomp,ngrow) << std::endl; } -/* if (ncomp > 1) { - Print() << " for component " << icomp; - } - Print() << " 1st mf min,max: " << mf1_min[icomp] - << ", " << mf1_max[icomp] +/* if (ncomp > 1) { + Print() << " for component " << icomp; + } + Print() << " 1st mf min,max: " << mf1_min[icomp] + << ", " << mf1_max[icomp] << ", 2nd mf min,max: "; - Print() << mf2_min[icomp] - << ", " << mf2_max[icomp] << "\n"; + Print() << mf2_min[icomp] + << ", " << mf2_max[icomp] << "\n"; } */ Print() << "Writing mfdiff" << std::endl; diff --git a/Tools/C_util/Statistics/AVGDOWN_2D.F b/Tools/C_util/Statistics/AVGDOWN_2D.F index 08cdd1f35d4..ed0d11c99ff 100644 --- a/Tools/C_util/Statistics/AVGDOWN_2D.F +++ b/Tools/C_util/Statistics/AVGDOWN_2D.F @@ -17,9 +17,9 @@ subroutine FORT_CV_AVGDOWN ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -69,9 +69,9 @@ subroutine FORT_CV_AVGDOWN ( crse(ic,jc,n) = crse(ic,jc,n) * vol_inv end do end do - + end do - + end subroutine FORT_AVGDOWN ( @@ -83,9 +83,9 @@ subroutine FORT_AVGDOWN ( c ---------------------------------------------------------- c Volume-weight average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c cv = coarse grid volume array c fv = fine grid volume array diff --git a/Tools/C_util/Statistics/AVGDOWN_3D.F b/Tools/C_util/Statistics/AVGDOWN_3D.F index 00ba6f54a11..c60a3812c56 100644 --- a/Tools/C_util/Statistics/AVGDOWN_3D.F +++ b/Tools/C_util/Statistics/AVGDOWN_3D.F @@ -17,9 +17,9 @@ subroutine FORT_CV_AVGDOWN ( c ---------------------------------------------------------- c Average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c lo,hi = index limits of overlap (crse grid) c ratios = IntVect refinement ratio @@ -52,7 +52,7 @@ subroutine FORT_CV_AVGDOWN ( end do end do end do -c +c c sum fine data c do koff = 0, lratz-1 @@ -81,7 +81,7 @@ subroutine FORT_CV_AVGDOWN ( end do end do - + end subroutine FORT_AVGDOWN ( @@ -93,9 +93,9 @@ subroutine FORT_AVGDOWN ( c ---------------------------------------------------------- c Volume-weight average the fine grid data onto the coarse c grid. Overlap is given in coarse grid coordinates. -c +c c crse = coarse grid data -c nvar = number of components in arrays +c nvar = number of components in arrays c fine = fine grid data c cv = coarse grid volume array c fv = fine grid volume array diff --git a/Tools/C_util/Statistics/AVGDOWN_F.H b/Tools/C_util/Statistics/AVGDOWN_F.H index 87e8f6b2916..f9d42b14d96 100644 --- a/Tools/C_util/Statistics/AVGDOWN_F.H +++ b/Tools/C_util/Statistics/AVGDOWN_F.H @@ -16,20 +16,20 @@ #include extern "C" { - + void FORT_CV_AVGDOWN (const amrex_real* crse, ARLIM_P(clo), ARLIM_P(chi), - const int* nvar, - const amrex_real* fine, ARLIM_P(fine_lo), ARLIM_P(fine_hi), - const int* lo, const int* hi, const int* ratio); - + const int* nvar, + const amrex_real* fine, ARLIM_P(fine_lo), ARLIM_P(fine_hi), + const int* lo, const int* hi, const int* ratio); + void FORT_AVGDOWN (const amrex_real* crse, ARLIM_P(clo), ARLIM_P(chi), const int* nvar, const amrex_real* fine, ARLIM_P(fine_lo), ARLIM_P(fine_hi), - const amrex_real* cvol, ARLIM_P(cv_lo), ARLIM_P(cv_hi), + const amrex_real* cvol, ARLIM_P(cv_lo), ARLIM_P(cv_hi), const amrex_real* fvol, ARLIM_P(fv_lo), ARLIM_P(fv_hi), const int* lo, const int* hi, const int* ratio); - + } - + #endif #endif /*_AVGDOWN_F_H_*/ diff --git a/Tools/C_util/Statistics/ComputeAmrDataStat.H b/Tools/C_util/Statistics/ComputeAmrDataStat.H index 5cdb4d6029b..64366b6bb8f 100644 --- a/Tools/C_util/Statistics/ComputeAmrDataStat.H +++ b/Tools/C_util/Statistics/ComputeAmrDataStat.H @@ -2,111 +2,111 @@ void ComputeAmrDataMeanVar (AmrData& amrData, - Vector& mean, - Vector& variance, - int sComp, - int nComp, - bool verbose); + Vector& mean, + Vector& variance, + int sComp, + int nComp, + bool verbose); void ComputeAmrDataMeanVar (AmrData& amrData, - Vector cNames, - Vector bas, - Vector& mean, - Vector& variance); + Vector cNames, + Vector bas, + Vector& mean, + Vector& variance); void ComputeMeanVarMF (MultiFab& mf, - Vector& mean, - Vector& variance); + Vector& mean, + Vector& variance); void ComputeAmrDataList (AmrData& amrData, - Vector mean, - Vector variance, - int sComp, - int nComp); + Vector mean, + Vector variance, + int sComp, + int nComp); void ComputeAmrDataPDF (AmrData& amrData, - Real** icount, - int nBin, - Vector cNames); + Real** icount, + int nBin, + Vector cNames); void ComputeAmrDataPDF (AmrData& amrData, - Real** icount, - int nBin, - Vector cNames, - Vector bas); + Real** icount, + int nBin, + Vector cNames, + Vector bas); void ComputeAmrDataVAR (AmrData& amrData, - int nBin, - Vector cNames, - Vector barr, - std::string oFile); + int nBin, + Vector cNames, + Vector barr, + std::string oFile); void VariogramUniform (AmrData& amrData, - Vector cNames, - Vector barr, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile); + Vector cNames, + Vector barr, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile); void VariogramCross(AmrData& amrData, - Vector cNames, - MultiFab& mf, - Vector barr, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile); + Vector cNames, + MultiFab& mf, + Vector barr, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile); void VariogramUniformMF (const MultiFab& mf, - Vector dx, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile); + Vector dx, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile); void VariogramUniformMFG (const MultiFab& mf, - Vector dx, - IntVect sm, - IntVect bg, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile); + Vector dx, + IntVect sm, + IntVect bg, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile); void TakeDifferenceFine(AmrData& amrDataf, - AmrData& amrDatac, - Vector cNames, - Vector barr, - std::string oFile); + AmrData& amrDatac, + Vector cNames, + Vector barr, + std::string oFile); void TakeDifferenceCrse(AmrData& amrDataf, - AmrData& amrDatac, - Vector cNames, - Vector barr, - std::string oFile); + AmrData& amrDatac, + Vector cNames, + Vector barr, + std::string oFile); void TakeDifferenceSum(AmrData& amrDataf, - AmrData& amrDatac, - Vector cNames, - Vector barr, - std::string oFile); + AmrData& amrDatac, + Vector cNames, + Vector barr, + std::string oFile); void TakeDifferenceMean(AmrData& amrDataf, - Vector cNames, - Vector barr, - Vector rratio, - std::string oFile); + Vector cNames, + Vector barr, + Vector rratio, + std::string oFile); diff --git a/Tools/C_util/Statistics/ComputeAmrDataStat.cpp b/Tools/C_util/Statistics/ComputeAmrDataStat.cpp index 64dab1a71c4..95624a1831f 100644 --- a/Tools/C_util/Statistics/ComputeAmrDataStat.cpp +++ b/Tools/C_util/Statistics/ComputeAmrDataStat.cpp @@ -16,36 +16,36 @@ // Determine the volume-averaged mean for an AMR data. void ComputeAmrDataMeanVar (AmrData& amrData, - Vector& mean, - Vector& variance, - int sComp, - int nComp, - bool verbose) + Vector& mean, + Vector& variance, + int sComp, + int nComp, + bool verbose) { std::string oFile, iFileDir, oFileDir; - + if (verbose) { ParmParse pp; pp.query("outfile", oFile); if (oFile.empty()) - amrex::Abort("You must specify `outfile' if run in verbose mode"); + amrex::Abort("You must specify `outfile' if run in verbose mode"); } - + int finestLevel = amrData.FinestLevel(); mean.clear(); mean.resize(nComp,0.0); variance.clear(); variance.resize(nComp,0.0); - + Vector refMult(finestLevel + 1, 1); for (int iLevel=finestLevel-1; iLevel>=0; --iLevel) { int ref_ratio = amrData.RefRatio()[iLevel]; int vol = 1; for (int i=0; i error(finestLevel+1); - + for (int iLevel = finestLevel; iLevel>=0; --iLevel) { const BoxArray& ba = amrData.boxArray(iLevel); - + error[iLevel] = new MultiFab(ba, nComp, 0); for (int iComp=0; iCompcopy(data,0,iComp+sComp,1); + MultiFab& data = amrData.GetGrids(iLevel,iComp); + error[iLevel]->copy(data,0,iComp+sComp,1); } // Zero out the error covered by fine grid long covered_volume = 0; - if (iLevel != finestLevel) + if (iLevel != finestLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; - BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); - for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) - { - for (int iGrid=0; iGrid 0.0) { - // Convert volume in numPts to volume in number of fine cells - total_volume += long(level_volume); - - // Get norms at this level - Vector n1(nComp,0.0), n2(nComp,0.0); - - for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) - { - FArrayBox& fab = (*error[iLevel])[mfi]; - const Box& fabbox = mfi.validbox(); - - FArrayBox vwFab(fabbox,nComp); - FArrayBox vwFabSqrd(fabbox,nComp); - - // sum - vwFab.copy(fab,0,0,nComp); - vwFab.mult(refMult[iLevel]); - - //sum-squared - vwFabSqrd.copy(fab,0,0,nComp); - vwFabSqrd.mult(fab,0,0,nComp); - vwFabSqrd.mult(refMult[iLevel]); - - for (int iComp=0; iComp n1(nComp,0.0), n2(nComp,0.0); + + for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) + { + FArrayBox& fab = (*error[iLevel])[mfi]; + const Box& fabbox = mfi.validbox(); + + FArrayBox vwFab(fabbox,nComp); + FArrayBox vwFabSqrd(fabbox,nComp); + + // sum + vwFab.copy(fab,0,0,nComp); + vwFab.mult(refMult[iLevel]); + + //sum-squared + vwFabSqrd.copy(fab,0,0,nComp); + vwFabSqrd.mult(fab,0,0,nComp); + vwFabSqrd.mult(refMult[iLevel]); + + for (int iComp=0; iComp mean, - Vector variance, - int sComp, - int nComp) + Vector mean, + Vector variance, + int sComp, + int nComp) { std::string oFile, iFileDir, oFileDir; - + int finestLevel = amrData.FinestLevel(); // @@ -177,41 +177,41 @@ ComputeAmrDataList (AmrData& amrData, { const BoxArray& ba = amrData.boxArray(iLevel); - MultiFab error(ba, nComp, 0); - for (int iComp=0; iComp cNames, - Vector bas, - Vector& mean, - Vector& variance) + Vector cNames, + Vector bas, + Vector& mean, + Vector& variance) { std::string oFile, iFileDir, oFileDir; @@ -221,10 +221,10 @@ ComputeAmrDataMeanVar (AmrData& amrData, Vector refMult(finestLevel + 1, 1); for (int iLevel=finestLevel-1; iLevel>=0; --iLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; - int area = ref_ratio; - for (int jLevel=0; jLevel<=iLevel; ++jLevel) - refMult[jLevel] *= area; + int ref_ratio = amrData.RefRatio()[iLevel]; + int area = ref_ratio; + for (int jLevel=0; jLevel<=iLevel; ++jLevel) + refMult[jLevel] *= area; } // @@ -235,7 +235,7 @@ ComputeAmrDataMeanVar (AmrData& amrData, for (int i=0; i=0; --iLevel) - { + { MultiFab mf(bas[iLevel],nComp,0,Fab_allocate); amrData.FillVar(mf,iLevel,cNames,destFillComps); @@ -243,87 +243,87 @@ ComputeAmrDataMeanVar (AmrData& amrData, long covered_volume = 0; if (iLevel != finestLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; - BoxArray baF = BoxArray(bas[iLevel+1]).coarsen(ref_ratio); - for (MFIter mfi(mf); mfi.isValid(); ++mfi) - { - for (int iGrid=0; iGrid 0.0) { - // Convert volume in numPts to volume in number of fine cells - total_volume += long(level_volume); - - // Get norms at this level - Vector n1(nComp,0.0), n2(nComp,0.0); - - for (MFIter mfi(mf); mfi.isValid(); ++mfi) - { - FArrayBox& fab = mf[mfi]; - const Box& fabbox = mfi.validbox(); - - FArrayBox vwFab(fabbox,nComp); - FArrayBox vwFabSqrd(fabbox,nComp); - - // sum - vwFab.copy(fab,0,0,nComp); - vwFab.mult(refMult[iLevel]); - - //sum-squared - vwFabSqrd.copy(fab,0,0,nComp); - vwFabSqrd.mult(fab,0,0,nComp); - vwFabSqrd.mult(refMult[iLevel]); - - for (int iComp=0; iComp n1(nComp,0.0), n2(nComp,0.0); + + for (MFIter mfi(mf); mfi.isValid(); ++mfi) + { + FArrayBox& fab = mf[mfi]; + const Box& fabbox = mfi.validbox(); + + FArrayBox vwFab(fabbox,nComp); + FArrayBox vwFabSqrd(fabbox,nComp); + + // sum + vwFab.copy(fab,0,0,nComp); + vwFab.mult(refMult[iLevel]); + + //sum-squared + vwFabSqrd.copy(fab,0,0,nComp); + vwFabSqrd.mult(fab,0,0,nComp); + vwFabSqrd.mult(refMult[iLevel]); + + for (int iComp=0; iComp cNames, - Vector barr, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile) + Vector cNames, + Vector barr, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile) { bool firsttime = true; @@ -1050,13 +1050,13 @@ VariogramUniform (AmrData& amrData, Vector dx(BL_SPACEDIM); for (int i=0; i cNames, - MultiFab& mf, - Vector barr, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile) + Vector cNames, + MultiFab& mf, + Vector barr, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile) { int finestLevel = amrData.FinestLevel(); int nComp = cNames.size(); @@ -1104,13 +1104,13 @@ VariogramCross(AmrData& amrData, Vector dx(BL_SPACEDIM); for (int i=0; i dx, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile) + Vector dx, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile) { Vector domloc(BL_SPACEDIM), domhic(BL_SPACEDIM); @@ -1144,7 +1144,7 @@ VariogramUniformMF (const MultiFab& mf, { const int* k_lo = mf[mfi].loVect(); const int* k_hi = mf[mfi].hiVect(); - + for (int i=0;i dx, - IntVect sm, - IntVect bg, - Vector< Vector > ivoption, - int nlag, - int isill, - Vector sills, - std::string oFile) + Vector dx, + IntVect sm, + IntVect bg, + Vector< Vector > ivoption, + int nlag, + int isill, + Vector sills, + std::string oFile) { bool firsttime = true; - for (int dir=0; dir np(nlag,0.0); - Vector gam(nlag,0.0); - Vector hm(nlag,0.0); - Vector tm(nlag,0.0); - Vector hv(nlag,0.0); - Vector tv(nlag,0.0); - - for (MFIter mfi(tmpx); mfi.isValid(); ++mfi) { - - const int* lo = tmpx[mfi].loVect(); - const int* hi = tmpx[mfi].hiVect(); - - Vector lod(BL_SPACEDIM),hid(BL_SPACEDIM); - if (dir == 0) { - lod[0] = lo[0]; - lod[1] = lo[1]; - hid[0] = hi[0]; - hid[1] = hi[1]; - } - else if (dir == 1){ - lod[0] = lo[1]; - lod[1] = lo[0]; - hid[0] = hi[1]; - hid[1] = hi[0]; - } - - for (int i=0; i 0.0) { - gam[i] = gam[i]/sills[ivtail]; - } - } - - if (ivtype == 1 || ivtype == 2 || ivtype == 6) { - gam[i] = 0.5*gam[i]; - } - else if (ivtype == 3) - gam[i] = gam[i] - hm[i]*hm[i]; - else if (ivtype == 4) { - hv[i] = hv[i] - hm[i]*hm[i]; - if (hv[i] <= 0.0) hv[i] = 0.0; - hv[i] = sqrt(hv[i]); - tv[i] = tv[i] - tm[i]*tm[i]; - if (tv[i] <= 0.0) tv[i] = 0.0; - tv[i] = sqrt(tv[i]); - if (hv[i]*tv[i] < 1.e-20) - gam[i] = 0.0; - else - gam[i] = (gam[i] - hm[i]*tm[i])/(hv[i]*tv[i]); - hv[i] = hv[i]*hv[i]; - tv[i] = tv[i]*tv[i]; - } - else if (ivtype == 5) { - Real htave = 0.5*hm[i]*tm[i]; - htave *= htave; - if (htave < 1.e-20) - gam[i] = 0.0; - else - gam[i] = gam[i]/htave; - } - } - std::cout << ivtail << " " << ivhead << " " - << ivtype << " " << dir << " " - << dx[dir] << " " << nlag << std::endl; - // write out to datafile - std::ofstream outputFile; - if (firsttime) { - outputFile.open(oFile.c_str(),std::ios::out); - if (outputFile.fail()) - amrex::Abort("Output file cannot be opened"); - firsttime = false; - } - else - outputFile.open(oFile.c_str(),std::ios::app); - - // ivtail,ivhead,ivtype,dir,nlag - outputFile << ivtail << " " << ivhead << " " - << ivtype << " " << dir << " " - << dx[dir] << " " << nlag << std::endl; - - for (int i=0; i np(nlag,0.0); + Vector gam(nlag,0.0); + Vector hm(nlag,0.0); + Vector tm(nlag,0.0); + Vector hv(nlag,0.0); + Vector tv(nlag,0.0); + + for (MFIter mfi(tmpx); mfi.isValid(); ++mfi) { + + const int* lo = tmpx[mfi].loVect(); + const int* hi = tmpx[mfi].hiVect(); + + Vector lod(BL_SPACEDIM),hid(BL_SPACEDIM); + if (dir == 0) { + lod[0] = lo[0]; + lod[1] = lo[1]; + hid[0] = hi[0]; + hid[1] = hi[1]; + } + else if (dir == 1){ + lod[0] = lo[1]; + lod[1] = lo[0]; + hid[0] = hi[1]; + hid[1] = hi[0]; + } + + for (int i=0; i 0.0) { + gam[i] = gam[i]/sills[ivtail]; + } + } + + if (ivtype == 1 || ivtype == 2 || ivtype == 6) { + gam[i] = 0.5*gam[i]; + } + else if (ivtype == 3) + gam[i] = gam[i] - hm[i]*hm[i]; + else if (ivtype == 4) { + hv[i] = hv[i] - hm[i]*hm[i]; + if (hv[i] <= 0.0) hv[i] = 0.0; + hv[i] = sqrt(hv[i]); + tv[i] = tv[i] - tm[i]*tm[i]; + if (tv[i] <= 0.0) tv[i] = 0.0; + tv[i] = sqrt(tv[i]); + if (hv[i]*tv[i] < 1.e-20) + gam[i] = 0.0; + else + gam[i] = (gam[i] - hm[i]*tm[i])/(hv[i]*tv[i]); + hv[i] = hv[i]*hv[i]; + tv[i] = tv[i]*tv[i]; + } + else if (ivtype == 5) { + Real htave = 0.5*hm[i]*tm[i]; + htave *= htave; + if (htave < 1.e-20) + gam[i] = 0.0; + else + gam[i] = gam[i]/htave; + } + } + std::cout << ivtail << " " << ivhead << " " + << ivtype << " " << dir << " " + << dx[dir] << " " << nlag << std::endl; + // write out to datafile + std::ofstream outputFile; + if (firsttime) { + outputFile.open(oFile.c_str(),std::ios::out); + if (outputFile.fail()) + amrex::Abort("Output file cannot be opened"); + firsttime = false; + } + else + outputFile.open(oFile.c_str(),std::ios::app); + + // ivtail,ivhead,ivtype,dir,nlag + outputFile << ivtail << " " << ivhead << " " + << ivtype << " " << dir << " " + << dx[dir] << " " << nlag << std::endl; + + for (int i=0; i cNames, - Vector barr, - std::string oFile) + AmrData& amrDatac, + Vector cNames, + Vector barr, + std::string oFile) { int nComp = cNames.size(); @@ -1411,7 +1411,7 @@ TakeDifferenceFine(AmrData& amrDataf, Box domainc = amrDatac.ProbDomain()[0]; for (int i=0;i bbll,bbur; @@ -1429,26 +1429,26 @@ TakeDifferenceFine(AmrData& amrDataf, Vector dxc(BL_SPACEDIM),dxf(BL_SPACEDIM); for (int i=0; i destFillComps(nComp); Vector destNames(nComp); - for (int i=0; i cNames, - Vector barr, - std::string oFile) + AmrData& amrDatac, + Vector cNames, + Vector barr, + std::string oFile) { int nComp = cNames.size(); @@ -1505,7 +1505,7 @@ TakeDifferenceCrse(AmrData& amrDataf, Box domainc = amrDatac.ProbDomain()[0]; for (int i=0;i bbll,bbur; @@ -1523,26 +1523,26 @@ TakeDifferenceCrse(AmrData& amrDataf, Vector dxc(BL_SPACEDIM),dxf(BL_SPACEDIM); for (int i=0; i destFillComps(nComp); Vector destNames(nComp); - for (int i=0; i cNames, - Vector barr, - std::string oFile) + AmrData& amrDatac, + Vector cNames, + Vector barr, + std::string oFile) { int nComp = cNames.size(); @@ -1604,7 +1604,7 @@ TakeDifferenceSum(AmrData& amrDataf, Box domainc = amrDatac.ProbDomain()[0]; for (int i=0;i bbll,bbur; @@ -1622,26 +1622,26 @@ TakeDifferenceSum(AmrData& amrDataf, Vector dxc(BL_SPACEDIM),dxf(BL_SPACEDIM); for (int i=0; i destFillComps(nComp); Vector destNames(nComp); - for (int i=0; i cNames, - Vector barr, - Vector rratio, - std::string oFile) + Vector cNames, + Vector barr, + Vector rratio, + std::string oFile) { int nComp = cNames.size(); @@ -1721,11 +1721,11 @@ TakeDifferenceMean(AmrData& amrDataf, Vector dxc(BL_SPACEDIM),dxf(BL_SPACEDIM); for (int i=0; i destFillComps(nComp); Vector destNames(nComp); - for (int i=0; i -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - int dir, - std::string iFile, - Real phi); + int nmax, + int nfac, + int dir, + std::string iFile, + Real phi); -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - int dir, - std::string iFile, - MultiFab& phidata); + int nmax, + int nfac, + int dir, + std::string iFile, + MultiFab& phidata); void -compute_flux(AmrData& amrData, - int dir, - Vector cNames, - Real dt, - Vector& xold, - Real& flux, - Real phi, - bool do_init, - Vector& barr); +compute_flux(AmrData& amrData, + int dir, + Vector cNames, + Real dt, + Vector& xold, + Real& flux, + Real phi, + bool do_init, + Vector& barr); diff --git a/Tools/C_util/Statistics/PltFileFluxAve.cpp b/Tools/C_util/Statistics/PltFileFluxAve.cpp index 9b9b365290f..908e3b67a6f 100644 --- a/Tools/C_util/Statistics/PltFileFluxAve.cpp +++ b/Tools/C_util/Statistics/PltFileFluxAve.cpp @@ -24,7 +24,7 @@ PrintUsage (const char* progName) std::cout << '\n'; std::cout << "This routine reads a set of pltfiles " << "and determines x/y fluxes. " << std::endl - << std::endl; + << std::endl; std::cout << "Usage:" << '\n'; std::cout << progName << '\n'; std::cout << " infile=inputFileName" << '\n'; @@ -41,7 +41,7 @@ SumThisComp(AmrData &amrData, int iComp) { Real sum(0.0); Real phi(1.0); - int finestLevel = amrData.FinestLevel(); + int finestLevel = amrData.FinestLevel(); int nComp = amrData.NComp(); Vector refMult(finestLevel + 1, 1); @@ -60,7 +60,7 @@ SumThisComp(AmrData &amrData, int iComp) // long total_volume = 0; Vector error(finestLevel+1); - + for (int iLevel = finestLevel; iLevel>=0; --iLevel) { const BoxArray& ba = amrData.boxArray(iLevel); @@ -76,19 +76,19 @@ SumThisComp(AmrData &amrData, int iComp) long covered_volume = 0; if (iLevel != finestLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; + int ref_ratio = amrData.RefRatio()[iLevel]; BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) { - for (int iGrid=0; iGrid 0.0) - n1 += fab(IntVect(ix,iy),iComp)*dx[0]*dx[1]; - } - } + FArrayBox& fab = (*error[iLevel])[mfi]; + const Box& fabbox = mfi.validbox(); + const int* lo = fab.loVect(); + const int* hi = fab.hiVect(); + +#if (BL_SPACEDIM == 2) + for (int iy=lo[1]; iy 0.0) + n1 += fab(IntVect(ix,iy),iComp)*dx[0]*dx[1]; + } + } #else - for (int iz=lo[2]; iz 0.0) - n1 += fab(IntVect(ix,iy,iz),iComp)*dx[0]*dx[1]*dx[2]; - } - } - } + for (int iz=lo[2]; iz 0.0) + n1 += fab(IntVect(ix,iy,iz),iComp)*dx[0]*dx[1]*dx[2]; + } + } + } #endif } // Do necessary communication, then blend this level's norms // in with the running global values ParallelDescriptor::ReduceRealSum(n1); - + sum += n1; } } - + // Clean up memory for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) @@ -159,7 +159,7 @@ main (int argc, { if (argc == 1) PrintUsage(argv[0]); - + amrex::Initialize(argc,argv); ParmParse pp; @@ -196,7 +196,7 @@ main (int argc, pp.query("dir", dir); - // limit to a smaller region + // limit to a smaller region Vector barr; bool do_bounds = 0; if (int nx=pp.countval("bounds")) @@ -218,17 +218,17 @@ main (int argc, DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); - + int analysis; pp.query("analysis",analysis); - if (analysis == 0) { + if (analysis == 0) { if (pfile.empty()) { - compute_flux_all(nstart, nmax, nfac, dir, iFile, phi); + compute_flux_all(nstart, nmax, nfac, dir, iFile, phi); } else - compute_flux_all(nstart, nmax, nfac, dir, iFile, phidata); + compute_flux_all(nstart, nmax, nfac, dir, iFile, phidata); } else if (analysis == 1) { @@ -242,47 +242,47 @@ main (int argc, for (int i = nstart; i < nmax; i++) { File = amrex::Concatenate(iFile, i*nfac, 5); - - DataServices dataServices(File, fileType); - - if (!dataServices.AmrDataOk()) - // - // This calls ParallelDescriptor::EndParallel() and exit() - // - DataServices::Dispatch(DataServices::ExitRequest, NULL); - - AmrData& amrData = dataServices.AmrDataRef(); - dtnew = amrData.Time(); - dt = dtnew - dtold; - dtold = dtnew; - - if (i == nstart) { - cNames[0] = amrData.PlotVarNames()[0]; - cNames[1] = amrData.PlotVarNames()[1]; - do_init = true; - } - else - do_init = false; - - compute_flux(amrData,dir,cNames,dt,xold,flux,phi,do_init,barr); - - std::cout << dtnew << " " << flux << std::endl; + + DataServices dataServices(File, fileType); + + if (!dataServices.AmrDataOk()) + // + // This calls ParallelDescriptor::EndParallel() and exit() + // + DataServices::Dispatch(DataServices::ExitRequest, NULL); + + AmrData& amrData = dataServices.AmrDataRef(); + dtnew = amrData.Time(); + dt = dtnew - dtold; + dtold = dtnew; + + if (i == nstart) { + cNames[0] = amrData.PlotVarNames()[0]; + cNames[1] = amrData.PlotVarNames()[1]; + do_init = true; + } + else + do_init = false; + + compute_flux(amrData,dir,cNames,dt,xold,flux,phi,do_init,barr); + + std::cout << dtnew << " " << flux << std::endl; } } - else + else amrex::Abort("Analysis type not defined"); amrex::Finalize(); } -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - int dir, - std::string iFile, - Real phi) + int nmax, + int nfac, + int dir, + std::string iFile, + Real phi) { DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); @@ -297,11 +297,11 @@ compute_flux_all(int nstart, MultiFab tmpmean; Real dtnew, dtold; dtold = 0.; - + for (int i = nstart; i < nmax; i++) { std::string File = amrex::Concatenate(iFile, i*nfac, 5); - + DataServices dataServices(File, fileType); if (!dataServices.AmrDataOk()) @@ -321,12 +321,12 @@ compute_flux_all(int nstart, names[1] = amrData.PlotVarNames()[1]; destcomp[0] = 0; destcomp[1] = 1; - + if (i == nstart) { finestLevel = 0; BoxArray ba = amrData.boxArray(finestLevel); - + tmpmean.define(ba,nComp,0,Fab_allocate); dmn = amrData.ProbDomain()[finestLevel]; @@ -338,22 +338,22 @@ compute_flux_all(int nstart, FLs.resize(dmn.bigEnd(dir)+1); for (int iy=0;iy& dx = amrData.DxLevel()[finestLevel]; - + amrData.FillVar(tmpmean,finestLevel,names,destcomp); - + for (int n = 0; n < nComp; n++) amrData.FlushGrids(destcomp[n]); - + Real dt = dtnew - dtold; dtold = dtnew; - + for (int ix = 0; ix < xnew.size(); ix++) xnew[ix] = 0.0; - + Real dmn_length; #if (BL_SPACEDIM == 2) if (dir == 0)dmn_length = dmn.length(1); @@ -368,34 +368,34 @@ compute_flux_all(int nstart, { const int* lo = tmpmean[mfi].loVect(); const int* hi = tmpmean[mfi].hiVect(); - + #if (BL_SPACEDIM == 2) if (dir == 0) { - for (int ix=lo[0]; ix<=hi[0]; ix++){ - for (int iy=lo[1]; iy<=hi[1]; iy++) { - xnew[ix] += tmpmean[mfi](IntVect(ix,iy),0)*phi; - } - } + for (int ix=lo[0]; ix<=hi[0]; ix++){ + for (int iy=lo[1]; iy<=hi[1]; iy++) { + xnew[ix] += tmpmean[mfi](IntVect(ix,iy),0)*phi; + } + } } if (dir == 1) { - for (int iy=lo[1]; iy<=hi[1]; iy++){ - for (int ix=lo[0]; ix<=hi[0]; ix++) { - xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; - } - } + for (int iy=lo[1]; iy<=hi[1]; iy++){ + for (int ix=lo[0]; ix<=hi[0]; ix++) { + xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; + } + } } #else if (dir == 2) { - for (int iz=lo[2]; iz<=hi[2]; iz++) - for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; + for (int iz=lo[2]; iz<=hi[2]; iz++) + for (int iy=lo[1]; iy<=hi[1]; iy++) + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; } #endif } const int IOProc = ParallelDescriptor::IOProcessorNumber(); - + ParallelDescriptor::ReduceRealSum(xnew.dataPtr(), xnew.size(), IOProc); if (ParallelDescriptor::IOProcessor()) @@ -404,11 +404,11 @@ compute_flux_all(int nstart, for (int iy = xnew.size()-1; iy >=0; iy--) { - xnew[iy] = xnew[iy]/dmn_length; - Real ct = (xnew[iy]-xold[iy])/dt; - FL += ct*dx[dir]; - FLs[iy] = FL; - xold[iy] = xnew[iy]; + xnew[iy] = xnew[iy]/dmn_length; + Real ct = (xnew[iy]-xold[iy])/dt; + FL += ct*dx[dir]; + FLs[iy] = FL; + xold[iy] = xnew[iy]; } std::cout << dtnew << " " << FL << std::endl; @@ -417,17 +417,17 @@ compute_flux_all(int nstart, //sumnew = SumThisComp(amrData, 0); //Real dF = phi*(sumnew - sumold)/dt; //sumold = sumnew; - - } + + } } -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - int dir, - std::string iFile, - MultiFab& phidata) + int nmax, + int nfac, + int dir, + std::string iFile, + MultiFab& phidata) { DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); @@ -442,14 +442,14 @@ compute_flux_all(int nstart, Vector destcomp(2); Real sumnew; MultiFab tmpmean; - MultiFab tmpphi; + MultiFab tmpphi; Real dtnew, dtold; dtold = 0.; - + for (int i = nstart; i < nmax; i++) { std::string File = amrex::Concatenate(iFile, i*nfac, 5); - + DataServices dataServices(File, fileType); if (!dataServices.AmrDataOk()) @@ -460,7 +460,7 @@ compute_flux_all(int nstart, AmrData& amrData = dataServices.AmrDataRef(); dtnew = amrData.Time(); - + if (i == nstart) { names[0] = amrData.PlotVarNames()[0]; @@ -475,27 +475,27 @@ compute_flux_all(int nstart, //int baseLevel = 0; //BoxArray ba = amrData.boxArray(baseLevel); - + int ng_twoexp = 1; for (int ii = 0; ii& dx = amrData.DxLevel()[finestLevel]; - + amrData.FillVar(tmpmean,finestLevel,names,destcomp); - + for (int n = 0; n < nComp; n++) amrData.FlushGrids(destcomp[n]); - + Real dt = dtnew - dtold; dtold = dtnew; - + for (int ix = 0; ix < xnew.size(); ix++) xnew[ix] = 0.0; @@ -530,42 +530,42 @@ compute_flux_all(int nstart, if (dir == 1)dmn_length = dmn.length(0)*dmn.length(2); if (dir == 2)dmn_length = dmn.length(0)*dmn.length(1); #endif - + for (MFIter mfi(tmpmean); mfi.isValid(); ++mfi) { const int* lo = tmpmean[mfi].loVect(); const int* hi = tmpmean[mfi].hiVect(); - + #if (BL_SPACEDIM == 2) if (dir == 0) { - for (int ix=lo[0]; ix<=hi[0]; ix++) { - for (int iy=lo[1]; iy<=hi[1]; iy++) { - xnew[ix] += tmpmean[mfi](IntVect(ix,iy),0)* - tmpphi[mfi](IntVect(ix,iy),0); - } - } + for (int ix=lo[0]; ix<=hi[0]; ix++) { + for (int iy=lo[1]; iy<=hi[1]; iy++) { + xnew[ix] += tmpmean[mfi](IntVect(ix,iy),0)* + tmpphi[mfi](IntVect(ix,iy),0); + } + } } if (dir == 1) { - for (int iy=lo[1]; iy<=hi[1]; iy++) { - for (int ix=lo[0]; ix<=hi[0]; ix++) { - xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)* - tmpphi[mfi](IntVect(ix,iy),0); - } - } + for (int iy=lo[1]; iy<=hi[1]; iy++) { + for (int ix=lo[0]; ix<=hi[0]; ix++) { + xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)* + tmpphi[mfi](IntVect(ix,iy),0); + } + } } #else if (dir == 2) { - for (int iz=lo[2]; iz<=hi[2]; iz++) - for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)* - tmpphi[mfi](IntVect(ix,iy),0); + for (int iz=lo[2]; iz<=hi[2]; iz++) + for (int iy=lo[1]; iy<=hi[1]; iy++) + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)* + tmpphi[mfi](IntVect(ix,iy),0); } #endif } const int IOProc = ParallelDescriptor::IOProcessorNumber(); - + ParallelDescriptor::ReduceRealSum(xnew.dataPtr(), xnew.size(), IOProc); if (ParallelDescriptor::IOProcessor()) @@ -574,31 +574,31 @@ compute_flux_all(int nstart, for (int iy = 0; iy < xnew.size(); iy++) { - xnew[iy] = xnew[iy]/dmn_length; - Real ct = (xnew[iy]-xold[iy])/dt; - FL += ct*dx[dir]; - FLs[iy] = FL; - xold[iy] = xnew[iy]; - std::cout << ct << " " < cNames, - Real dt, - Vector& xold, - Real& flux, - Real phi, - bool do_init, - Vector& barr) -{ +compute_flux(AmrData& amrData, + int dir, + Vector cNames, + Real dt, + Vector& xold, + Real& flux, + Real phi, + bool do_init, + Vector& barr) +{ Vector xnew; Vector FLs; MultiFab tmpmean; @@ -609,7 +609,7 @@ compute_flux(AmrData& amrData, Vector destFillComps(nComp); for (int i=0; i& dx = amrData.DxLevel()[finestLevel]; - + amrData.FillVar(tmpmean,finestLevel,cNames,destFillComps); for (int n = 0; n < nComp; n++) @@ -675,26 +675,26 @@ compute_flux(AmrData& amrData, { const int* lo = tmpmean[mfi].loVect(); const int* hi = tmpmean[mfi].hiVect(); - + #if (BL_SPACEDIM == 2) if (dir == 0) { for (int ix=lo[0]; ix<=hi[0]; ix++) { - for (int iy=lo[1]; iy<=hi[1]; iy++) { - xnew[ix] += tmpmean[mfi](IntVect(ix,iy),0)*phi; - } + for (int iy=lo[1]; iy<=hi[1]; iy++) { + xnew[ix] += tmpmean[mfi](IntVect(ix,iy),0)*phi; + } } } if (dir == 1) { for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; } #else if (dir == 2) { for (int iz=lo[2]; iz<=hi[2]; iz++) - for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; + for (int iy=lo[1]; iy<=hi[1]; iy++) + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; } #endif } @@ -715,7 +715,7 @@ compute_flux(AmrData& amrData, FLs[iy] = FL; xold[iy] = xtmp; } - flux = FL; + flux = FL; } } diff --git a/Tools/C_util/Statistics/PltFileList.cpp b/Tools/C_util/Statistics/PltFileList.cpp index 274983e39ef..5e4880ae9ec 100644 --- a/Tools/C_util/Statistics/PltFileList.cpp +++ b/Tools/C_util/Statistics/PltFileList.cpp @@ -24,7 +24,7 @@ PrintUsage (const char* progName) { std::cout << '\n'; std::cout << "This routine reads a pltfile and determines its statistics" << std::endl - << std::endl; + << std::endl; std::cout << "Usage:" << '\n'; std::cout << progName << '\n'; std::cout << " infile=inputFileName" << '\n'; @@ -42,7 +42,7 @@ main (int argc, { if (argc == 1) PrintUsage(argv[0]); - + amrex::Initialize(argc,argv); ParmParse pp; @@ -75,7 +75,7 @@ main (int argc, int iFile_type = 0; pp.query("infile_type", iFile_type); - + int nmax = 100; pp.query("nmax", nmax); @@ -93,59 +93,59 @@ main (int argc, bool first = true; - if (iFile_type == 0) + if (iFile_type == 0) { for (int i = 1; i <= nmax; i++) { File = amrex::Concatenate(hdrFile, i, 1); File += '/'; File += iFile; - - DataServices dataServices(File, fileType); - - if (!dataServices.AmrDataOk()) - // - // This calls ParallelDescriptor::EndParallel() and exit() - // - DataServices::Dispatch(DataServices::ExitRequest, NULL); - - AmrData& amrData = dataServices.AmrDataRef(); - - nComp = amrData.NComp(); - Vector names = amrData.PlotVarNames(); - Vector destcomp(names.size()); - - for (int j=0; j names = amrData.PlotVarNames(); + Vector destcomp(names.size()); + + for (int j=0; j varNames = amrData.PlotVarNames(); if (int nx=pp.countval("cNames")) { - Vector var_match(nx,0); - pp.getarr("cNames",cNames,0,nx); - for (int ic=0; ic var_match(nx,0); + pp.getarr("cNames",cNames,0,nx); + for (int ic=0; ic barr; Vector bas; @@ -140,36 +140,36 @@ main (int argc, bool do_bounds = 0; if (int nx=pp.countval("bounds")) { - do_bounds = 1; - pp.getarr("bounds",barr,0,nx); - int d=BL_SPACEDIM; - BL_ASSERT(barr.size()==2*d); - bbll.resize(d); - bbur.resize(d); - for (int i=0; i 0) { - bas.set(iLevel,baThisLev); - if (iLevel < finestLevel) { - levelDomain.refine(amrData.RefRatio()[iLevel]); + bas.set(iLevel,baThisLev); + if (iLevel < finestLevel) { + levelDomain.refine(amrData.RefRatio()[iLevel]); } } else - { - bas.resize(iLevel); + { + bas.resize(iLevel); } } - + // analysis options // 1: mean and variance of a plot file // 2: determine the pdf of a plot file - // 3: - + // 3: + if (analysis_type == 1) { // mean and variance of a plot file - Vector mean, variance; - AmrData& amrData = dataServices.AmrDataRef(); - ComputeAmrDataMeanVar(amrData, mean, variance, 0, nComp, verbose); - - for (int i=0;i > ivoption(nvarg); - for (int i=0; i mean(nComp), variance(nComp); - if (isill == 1) - ComputeAmrDataMeanVar (amrData,cNames,bas,mean,variance); - - VariogramUniform(amrData,cNames,barr, - ivoption,nBin,isill,variance,oFile); + std::cout << "GSLIB variogram calculations.\n"; + + std::string oFile(tmpFile + "_VAR"); + pp.query("outfile",oFile); + + int nvarg = pp.countname("varg"); + if (nvarg == 0) + amrex::Abort("No variogram is specified"); + + Vector< Vector > ivoption(nvarg); + for (int i=0; i mean(nComp), variance(nComp); + if (isill == 1) + ComputeAmrDataMeanVar (amrData,cNames,bas,mean,variance); + + VariogramUniform(amrData,cNames,barr, + ivoption,nBin,isill,variance,oFile); } - + else if (analysis_type == 6) // compare coarse and fine solution { - std::cout << "Analysis 6: take difference on fine grid.\n"; - - std::string crsefile; - pp.query("crsefile",crsefile); - if (crsefile.empty()) - amrex::Abort("You must specify `crsefile'"); - - std::string oFile = iFile + "_diffine"; - pp.query("outfile",oFile); - - DataServices crseDataServices(crsefile,fileType); - if (!crseDataServices.AmrDataOk()) - DataServices::Dispatch(DataServices::ExitRequest, NULL); - AmrData& amrDataCrse = crseDataServices.AmrDataRef(); - const Real dtcrse = amrDataCrse.Time(); - - // The I/O processor makes the directory if it doesn't already exist. - if (ParallelDescriptor::IOProcessor()) - if (!amrex::UtilCreateDirectory(oFile, 0755)) - amrex::CreateDirectoryFailed(oFile); - ParallelDescriptor::Barrier(); - std::string mFile = iFile + "_dif"; - TakeDifferenceFine(amrData,amrDataCrse,cNames,barr,mFile); + std::cout << "Analysis 6: take difference on fine grid.\n"; + + std::string crsefile; + pp.query("crsefile",crsefile); + if (crsefile.empty()) + amrex::Abort("You must specify `crsefile'"); + + std::string oFile = iFile + "_diffine"; + pp.query("outfile",oFile); + + DataServices crseDataServices(crsefile,fileType); + if (!crseDataServices.AmrDataOk()) + DataServices::Dispatch(DataServices::ExitRequest, NULL); + AmrData& amrDataCrse = crseDataServices.AmrDataRef(); + const Real dtcrse = amrDataCrse.Time(); + + // The I/O processor makes the directory if it doesn't already exist. + if (ParallelDescriptor::IOProcessor()) + if (!amrex::UtilCreateDirectory(oFile, 0755)) + amrex::CreateDirectoryFailed(oFile); + ParallelDescriptor::Barrier(); + std::string mFile = iFile + "_dif"; + TakeDifferenceFine(amrData,amrDataCrse,cNames,barr,mFile); } else if (analysis_type == 7) // compare coarse and fine solution { - std::cout << "Analysis 6: coarse-fine comparison.\n"; - - std::string crsefile; - pp.query("crsefile",crsefile); - if (crsefile.empty()) - amrex::Abort("You must specify `crsefile'"); - - std::string oFile; - pp.query("outfile",oFile); - - DataServices crseDataServices(crsefile,fileType); - if (!crseDataServices.AmrDataOk()) - DataServices::Dispatch(DataServices::ExitRequest, NULL); - AmrData& amrDataCrse = crseDataServices.AmrDataRef(); - const Real dtcrse = amrDataCrse.Time(); - - // The I/O processor makes the directory if it doesn't already exist. - if (ParallelDescriptor::IOProcessor()) - if (!amrex::UtilCreateDirectory(oFile, 0755)) - amrex::CreateDirectoryFailed(oFile); - ParallelDescriptor::Barrier(); - std::string mFile = iFile + "_dif"; - TakeDifferenceCrse(amrData,amrDataCrse,cNames,barr,mFile); - + std::cout << "Analysis 6: coarse-fine comparison.\n"; + + std::string crsefile; + pp.query("crsefile",crsefile); + if (crsefile.empty()) + amrex::Abort("You must specify `crsefile'"); + + std::string oFile; + pp.query("outfile",oFile); + + DataServices crseDataServices(crsefile,fileType); + if (!crseDataServices.AmrDataOk()) + DataServices::Dispatch(DataServices::ExitRequest, NULL); + AmrData& amrDataCrse = crseDataServices.AmrDataRef(); + const Real dtcrse = amrDataCrse.Time(); + + // The I/O processor makes the directory if it doesn't already exist. + if (ParallelDescriptor::IOProcessor()) + if (!amrex::UtilCreateDirectory(oFile, 0755)) + amrex::CreateDirectoryFailed(oFile); + ParallelDescriptor::Barrier(); + std::string mFile = iFile + "_dif"; + TakeDifferenceCrse(amrData,amrDataCrse,cNames,barr,mFile); + } else if (analysis_type == 8) // compare coarse and fine solution { - std::string crsefile; - pp.query("crsefile",crsefile); - if (crsefile.empty()) - amrex::Abort("You must specify `crsefile'"); - - std::string oFile; - pp.query("outfile",oFile); - - DataServices crseDataServices(crsefile,fileType); - if (!crseDataServices.AmrDataOk()) - DataServices::Dispatch(DataServices::ExitRequest, NULL); - AmrData& amrDataCrse = crseDataServices.AmrDataRef(); - const Real dtcrse = amrDataCrse.Time(); - - // The I/O processor makes the directory if it doesn't already exist. - if (ParallelDescriptor::IOProcessor()) - if (!amrex::UtilCreateDirectory(oFile, 0755)) - amrex::CreateDirectoryFailed(oFile); - ParallelDescriptor::Barrier(); - std::string mFile = iFile + "_dif"; - TakeDifferenceSum(amrData,amrDataCrse,cNames,barr,mFile); - + std::string crsefile; + pp.query("crsefile",crsefile); + if (crsefile.empty()) + amrex::Abort("You must specify `crsefile'"); + + std::string oFile; + pp.query("outfile",oFile); + + DataServices crseDataServices(crsefile,fileType); + if (!crseDataServices.AmrDataOk()) + DataServices::Dispatch(DataServices::ExitRequest, NULL); + AmrData& amrDataCrse = crseDataServices.AmrDataRef(); + const Real dtcrse = amrDataCrse.Time(); + + // The I/O processor makes the directory if it doesn't already exist. + if (ParallelDescriptor::IOProcessor()) + if (!amrex::UtilCreateDirectory(oFile, 0755)) + amrex::CreateDirectoryFailed(oFile); + ParallelDescriptor::Barrier(); + std::string mFile = iFile + "_dif"; + TakeDifferenceSum(amrData,amrDataCrse,cNames,barr,mFile); + } else if (analysis_type == 9) // determine the variogram based on GSLIB { - std::cout << "GSLIB cross-variogram calculations.\n"; - - std::string oFile(tmpFile + "_VAR"); - pp.query("outfile",oFile); - - std::string sFile; - pp.query("secfile",sFile); - - int nvarg = pp.countname("varg"); - if (nvarg == 0) - amrex::Abort("No variogram is specified"); - - Vector< Vector > ivoption(nvarg); - for (int i=0; i mean, variance; - Vector secmean, secvariance; - if (isill == 1) { - - mean.resize(nComp+secmf.nComp()); - variance.resize(nComp+secmf.nComp()); - secmean.resize(secmf.nComp()); - secvariance.resize(secmf.nComp()); - - ComputeAmrDataMeanVar(amrData,cNames,bas,mean,variance); - ComputeMeanVarMF(secmf,secmean,secvariance); - - for (int i=0;i < secmf.nComp(); i++) { - mean[nComp+i] = secmean[i]; - variance[nComp+i] = secvariance[i]; - } - - } - - VariogramCross(amrData,cNames,secmf,barr, - ivoption,nBin,isill,variance,oFile); + std::cout << "GSLIB cross-variogram calculations.\n"; + + std::string oFile(tmpFile + "_VAR"); + pp.query("outfile",oFile); + + std::string sFile; + pp.query("secfile",sFile); + + int nvarg = pp.countname("varg"); + if (nvarg == 0) + amrex::Abort("No variogram is specified"); + + Vector< Vector > ivoption(nvarg); + for (int i=0; i mean, variance; + Vector secmean, secvariance; + if (isill == 1) { + + mean.resize(nComp+secmf.nComp()); + variance.resize(nComp+secmf.nComp()); + secmean.resize(secmf.nComp()); + secvariance.resize(secmf.nComp()); + + ComputeAmrDataMeanVar(amrData,cNames,bas,mean,variance); + ComputeMeanVarMF(secmf,secmean,secvariance); + + for (int i=0;i < secmf.nComp(); i++) { + mean[nComp+i] = secmean[i]; + variance[nComp+i] = secvariance[i]; + } + + } + + VariogramCross(amrData,cNames,secmf,barr, + ivoption,nBin,isill,variance,oFile); } else if (analysis_type == 10) // compare coarse and fine solution { - std::cout << "Analysis 10: difference from mean.\n"; - - std::string oFile; - pp.query("outfile",oFile); - - Vector rratio(BL_SPACEDIM,0); - if (int nx=pp.countval("rratio")) - pp.getarr("rratio",rratio,0,BL_SPACEDIM); - for (int i=0;i rratio(BL_SPACEDIM,0); + if (int nx=pp.countval("rratio")) + pp.getarr("rratio",rratio,0,BL_SPACEDIM); + for (int i=0;i > ivoption(nvarg); - for (int i=0; i mean(mfIn.nComp(),0.0), variance(mfIn.nComp(),0.0); - Vector dx(BL_SPACEDIM); - dx[0] = 2; - dx[1] = 2; - - if (isill == 1) - ComputeMeanVarMF (mfIn,mean,variance); - VariogramUniformMF(mfIn,dx,ivoption,nBin,isill,variance,oFile); - } + std::cout << "GSLIB variogram calculations.\n"; + + std::string oFile(tmpFile + "_VAR"); + pp.query("outfile",oFile); + + int nvarg = pp.countname("varg"); + if (nvarg == 0) + amrex::Abort("No variogram is specified"); + + Vector< Vector > ivoption(nvarg); + for (int i=0; i mean(mfIn.nComp(),0.0), variance(mfIn.nComp(),0.0); + Vector dx(BL_SPACEDIM); + dx[0] = 2; + dx[1] = 2; + + if (isill == 1) + ComputeMeanVarMF (mfIn,mean,variance); + VariogramUniformMF(mfIn,dx,ivoption,nBin,isill,variance,oFile); + } } amrex::Finalize(); diff --git a/Tools/C_util/Statistics/PltFileXAve.H b/Tools/C_util/Statistics/PltFileXAve.H index 22b5c8b100e..a491c93d0e6 100644 --- a/Tools/C_util/Statistics/PltFileXAve.H +++ b/Tools/C_util/Statistics/PltFileXAve.H @@ -1,26 +1,26 @@ #include -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - std::string iFile, - Real phi); + int nmax, + int nfac, + std::string iFile, + Real phi); -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - std::string iFile, - MultiFab& phidata); + int nmax, + int nfac, + std::string iFile, + MultiFab& phidata); void -compute_flux(AmrData& amrData, - int dir, - Vector cNames, - Real dt, - Vector& xold, - Real& flux, - Real phi, - bool do_init, - Real* barr=0); +compute_flux(AmrData& amrData, + int dir, + Vector cNames, + Real dt, + Vector& xold, + Real& flux, + Real phi, + bool do_init, + Real* barr=0); diff --git a/Tools/C_util/Statistics/PltFileXAve.cpp b/Tools/C_util/Statistics/PltFileXAve.cpp index dc75ac46d37..8c80c1fe99a 100644 --- a/Tools/C_util/Statistics/PltFileXAve.cpp +++ b/Tools/C_util/Statistics/PltFileXAve.cpp @@ -24,7 +24,7 @@ PrintUsage (const char* progName) { std::cout << '\n'; std::cout << "This routine reads a pltfile and determines its statistics" << std::endl - << std::endl; + << std::endl; std::cout << "Usage:" << '\n'; std::cout << progName << '\n'; std::cout << " infile=inputFileName" << '\n'; @@ -42,7 +42,7 @@ SumThisComp(AmrData &amrData, int iComp) { Real sum(0.0); Real phi(1.0); - int finestLevel = amrData.FinestLevel(); + int finestLevel = amrData.FinestLevel(); int nComp = amrData.NComp(); Vector refMult(finestLevel + 1, 1); @@ -61,7 +61,7 @@ SumThisComp(AmrData &amrData, int iComp) // long total_volume = 0; Vector error(finestLevel+1); - + for (int iLevel = finestLevel; iLevel>=0; --iLevel) { const BoxArray& ba = amrData.boxArray(iLevel); @@ -77,19 +77,19 @@ SumThisComp(AmrData &amrData, int iComp) long covered_volume = 0; if (iLevel != finestLevel) { - int ref_ratio = amrData.RefRatio()[iLevel]; + int ref_ratio = amrData.RefRatio()[iLevel]; BoxArray baF = ::BoxArray(amrData.boxArray(iLevel+1)).coarsen(ref_ratio); for (MFIter mfi(*error[iLevel]); mfi.isValid(); ++mfi) { - for (int iGrid=0; iGrid 0.0) - n1 += fab(IntVect(ix,iy),iComp)*dx[0]*dx[1]; - } - } + FArrayBox& fab = (*error[iLevel])[mfi]; + const Box& fabbox = mfi.validbox(); + const int* lo = fab.loVect(); + const int* hi = fab.hiVect(); + +#if (BL_SPACEDIM == 2) + for (int iy=lo[1]; iy 0.0) + n1 += fab(IntVect(ix,iy),iComp)*dx[0]*dx[1]; + } + } #else - for (int iz=lo[2]; iz 0.0) - n1 += fab(IntVect(ix,iy,iz),iComp)*dx[0]*dx[1]*dx[2]; - } - } - } + for (int iz=lo[2]; iz 0.0) + n1 += fab(IntVect(ix,iy,iz),iComp)*dx[0]*dx[1]*dx[2]; + } + } + } #endif } // Do necessary communication, then blend this level's norms // in with the running global values ParallelDescriptor::ReduceRealSum(n1); - + sum += n1; } } - + // Clean up memory for (int iLevel = 0; iLevel <= finestLevel; ++iLevel) @@ -160,7 +160,7 @@ main (int argc, { if (argc == 1) PrintUsage(argv[0]); - + amrex::Initialize(argc,argv); ParmParse pp; @@ -207,17 +207,17 @@ main (int argc, DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); - + int analysis; pp.query("analysis",analysis); - if (analysis == 0) { + if (analysis == 0) { if (pfile.empty()) { - compute_flux_all(nstart, nmax, nfac, iFile, phi); + compute_flux_all(nstart, nmax, nfac, iFile, phi); } else - compute_flux_all(nstart, nmax, nfac, iFile, phidata); + compute_flux_all(nstart, nmax, nfac, iFile, phidata); } else if (analysis == 1) { @@ -231,46 +231,46 @@ main (int argc, for (int i = nstart; i < nmax; i++) { File = amrex::Concatenate(iFile, i*nfac, 5); - - DataServices dataServices(File, fileType); - - if (!dataServices.AmrDataOk()) - // - // This calls ParallelDescriptor::EndParallel() and exit() - // - DataServices::Dispatch(DataServices::ExitRequest, NULL); - - AmrData& amrData = dataServices.AmrDataRef(); - dtnew = amrData.Time(); - dt = dtnew - dtold; - dtold = dtnew; - - if (i == nstart) { - cNames[0] = amrData.PlotVarNames()[0]; - cNames[1] = amrData.PlotVarNames()[1]; - do_init = true; - } - else - do_init = false; - - compute_flux(amrData,0,cNames,dt,xold,flux,phi,do_init); - - std::cout << dtnew << " " << flux << std::endl; + + DataServices dataServices(File, fileType); + + if (!dataServices.AmrDataOk()) + // + // This calls ParallelDescriptor::EndParallel() and exit() + // + DataServices::Dispatch(DataServices::ExitRequest, NULL); + + AmrData& amrData = dataServices.AmrDataRef(); + dtnew = amrData.Time(); + dt = dtnew - dtold; + dtold = dtnew; + + if (i == nstart) { + cNames[0] = amrData.PlotVarNames()[0]; + cNames[1] = amrData.PlotVarNames()[1]; + do_init = true; + } + else + do_init = false; + + compute_flux(amrData,0,cNames,dt,xold,flux,phi,do_init); + + std::cout << dtnew << " " << flux << std::endl; } } - else + else amrex::Abort("Analysis type not defined"); amrex::Finalize(); } -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - std::string iFile, - Real phi) + int nmax, + int nfac, + std::string iFile, + Real phi) { DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); @@ -285,7 +285,7 @@ compute_flux_all(int nstart, MultiFab tmpmean; Real dtnew, dtold; dtold = 0.; - + for (int i = nstart; i < nmax; i++) { std::string File = amrex::Concatenate(iFile, i*nfac, 5); @@ -295,7 +295,7 @@ compute_flux_all(int nstart, //File = hdrFile + idxs; //if (i == 0) File +=idxs; //std::cout << File << std::endl; - + DataServices dataServices(File, fileType); if (!dataServices.AmrDataOk()) @@ -315,12 +315,12 @@ compute_flux_all(int nstart, names[1] = amrData.PlotVarNames()[1]; destcomp[0] = 0; destcomp[1] = 1; - + if (i == nstart) { finestLevel = 0; BoxArray ba = amrData.boxArray(finestLevel); - + tmpmean.define(ba,nComp,0,Fab_allocate); dmn = amrData.ProbDomain()[finestLevel]; @@ -332,47 +332,47 @@ compute_flux_all(int nstart, FLs.resize(dmn.bigEnd(BL_SPACEDIM-1)+1); for (int iy=0;iy& dx = amrData.DxLevel()[finestLevel]; - + // if (ParallelDescriptor::IOProcessor()) //std::cout << "Filling tmpmean ... " << std::flush; - + amrData.FillVar(tmpmean,finestLevel,names,destcomp); - + for (int n = 0; n < nComp; n++) amrData.FlushGrids(destcomp[n]); - + Real dt = dtnew - dtold; dtold = dtnew; - + for (int ix = 0; ix < xnew.size(); ix++) xnew[ix] = 0; - + for (MFIter mfi(tmpmean); mfi.isValid(); ++mfi) { const int* lo = tmpmean[mfi].loVect(); const int* hi = tmpmean[mfi].hiVect(); - + #if (BL_SPACEDIM == 2) for (int iy=lo[1]; iy<=hi[1]; iy++){ - for (int ix=lo[0]; ix<=hi[0]; ix++) { - xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; - std::cout << dx[1] << " " << dmn.length(0) << std::endl; - } + for (int ix=lo[0]; ix<=hi[0]; ix++) { + xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; + std::cout << dx[1] << " " << dmn.length(0) << std::endl; + } } #else for (int iz=lo[2]; iz<=hi[2]; iz++) - for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; + for (int iy=lo[1]; iy<=hi[1]; iy++) + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; #endif } const int IOProc = ParallelDescriptor::IOProcessorNumber(); - + ParallelDescriptor::ReduceRealSum(xnew.dataPtr(), xnew.size(), IOProc); if (ParallelDescriptor::IOProcessor()) @@ -382,20 +382,20 @@ compute_flux_all(int nstart, #if (BL_SPACEDIM == 2) for (int iy = 0; iy < xnew.size(); iy++) { - xnew[iy] = xnew[iy]/dmn.length(0); - Real ct = (xnew[iy]-xold[iy])/dt; - FL += ct*dx[1]; - FLs[iy] = FL; - xold[iy] = xnew[iy]; + xnew[iy] = xnew[iy]/dmn.length(0); + Real ct = (xnew[iy]-xold[iy])/dt; + FL += ct*dx[1]; + FLs[iy] = FL; + xold[iy] = xnew[iy]; } #else for (int iz = 0; iz < xnew.size(); iz++) { - xnew[iz] = xnew[iz]/(dmn.length(0)*dmn.length(1)); - Real ct = (xnew[iz]-xold[iz])/dt; - FL += ct*dx[2]; - FLs[iz] = FL; - xold[iz] = xnew[iz]; + xnew[iz] = xnew[iz]/(dmn.length(0)*dmn.length(1)); + Real ct = (xnew[iz]-xold[iz])/dt; + FL += ct*dx[2]; + FLs[iz] = FL; + xold[iz] = xnew[iz]; } #endif std::cout << dtnew << " " << FL << std::endl; @@ -404,16 +404,16 @@ compute_flux_all(int nstart, //sumnew = SumThisComp(amrData, 0); //Real dF = phi*(sumnew - sumold)/dt; //sumold = sumnew; - - } + + } } -void +void compute_flux_all(int nstart, - int nmax, - int nfac, - std::string iFile, - MultiFab& phidata) + int nmax, + int nfac, + std::string iFile, + MultiFab& phidata) { DataServices::SetBatchMode(); Amrvis::FileType fileType(Amrvis::NEWPLT); @@ -428,14 +428,14 @@ compute_flux_all(int nstart, Vector destcomp(2); Real sumnew; MultiFab tmpmean; - MultiFab tmpphi; + MultiFab tmpphi; Real dtnew, dtold; dtold = 0.; - + for (int i = nstart; i < nmax; i++) { std::string File = amrex::Concatenate(iFile, i*nfac, 5); - + DataServices dataServices(File, fileType); if (!dataServices.AmrDataOk()) @@ -446,7 +446,7 @@ compute_flux_all(int nstart, AmrData& amrData = dataServices.AmrDataRef(); dtnew = amrData.Time(); - + if (i == nstart) { names[0] = amrData.PlotVarNames()[0]; @@ -461,27 +461,27 @@ compute_flux_all(int nstart, //int baseLevel = 0; //BoxArray ba = amrData.boxArray(baseLevel); - + int ng_twoexp = 1; for (int ii = 0; ii& dx = amrData.DxLevel()[finestLevel]; - + amrData.FillVar(tmpmean,finestLevel,names,destcomp); - + for (int n = 0; n < nComp; n++) amrData.FlushGrids(destcomp[n]); - + Real dt = dtnew - dtold; dtold = dtnew; - + for (int ix = 0; ix < xnew.size(); ix++) xnew[ix] = 0; - + for (MFIter mfi(tmpmean); mfi.isValid(); ++mfi) { const int* lo = tmpmean[mfi].loVect(); const int* hi = tmpmean[mfi].hiVect(); - + #if (BL_SPACEDIM == 2) for (int iy=lo[1]; iy<=hi[1]; iy++) { - for (int ix=lo[0]; ix<=hi[0]; ix++) { - xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)* - tmpphi[mfi](IntVect(ix,iy),0); - } + for (int ix=lo[0]; ix<=hi[0]; ix++) { + xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)* + tmpphi[mfi](IntVect(ix,iy),0); + } } #else for (int iz=lo[2]; iz<=hi[2]; iz++) - for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)* - tmpphi[mfi](IntVect(ix,iy),0); + for (int iy=lo[1]; iy<=hi[1]; iy++) + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)* + tmpphi[mfi](IntVect(ix,iy),0); #endif } const int IOProc = ParallelDescriptor::IOProcessorNumber(); - + ParallelDescriptor::ReduceRealSum(xnew.dataPtr(), xnew.size(), IOProc); if (ParallelDescriptor::IOProcessor()) @@ -539,40 +539,40 @@ compute_flux_all(int nstart, #if (BL_SPACEDIM == 2) for (int iy = 0; iy < xnew.size(); iy++) { - xnew[iy] = xnew[iy]/dmn.length(0); - Real ct = (xnew[iy]-xold[iy])/dt; - FL += ct*dx[1]; - FLs[iy] = FL; - xold[iy] = xnew[iy]; + xnew[iy] = xnew[iy]/dmn.length(0); + Real ct = (xnew[iy]-xold[iy])/dt; + FL += ct*dx[1]; + FLs[iy] = FL; + xold[iy] = xnew[iy]; } #else for (int iz = 0; iz < xnew.size(); iz++) { - xnew[iz] = xnew[iz]/(dmn.length(0)*dmn.length(1)); - Real ct = (xnew[iz]-xold[iz])/dt; - FL += ct*dx[2]; - FLs[iz] = FL; - xold[iz] = xnew[iz]; + xnew[iz] = xnew[iz]/(dmn.length(0)*dmn.length(1)); + Real ct = (xnew[iz]-xold[iz])/dt; + FL += ct*dx[2]; + FLs[iz] = FL; + xold[iz] = xnew[iz]; } #endif std::cout << dtnew << " " << FL << std::endl; } - } + } } void -compute_flux(AmrData& amrData, - int dir, - Vector cNames, - Real dt, - Vector& xold, - Real& flux, - Real phi, - bool do_init, - Real* barr) -{ +compute_flux(AmrData& amrData, + int dir, + Vector cNames, + Real dt, + Vector& xold, + Real& flux, + Real phi, + bool do_init, + Real* barr) +{ Vector xnew; Vector FLs; MultiFab tmpmean; @@ -602,12 +602,12 @@ compute_flux(AmrData& amrData, // Find coarse-grid coordinates of bounding box, round outwardly for (int i=0; i& dx = amrData.DxLevel()[finestLevel]; - + amrData.FillVar(tmpmean,finestLevel,cNames,destFillComps); for (int n = 0; n < nComp; n++) @@ -639,16 +639,16 @@ compute_flux(AmrData& amrData, { const int* lo = tmpmean[mfi].loVect(); const int* hi = tmpmean[mfi].hiVect(); - + #if (BL_SPACEDIM == 2) for (int iy=lo[1]; iy<=hi[1]; iy++) for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; + xnew[iy] += tmpmean[mfi](IntVect(ix,iy),0)*phi; #else for (int iz=lo[2]; iz<=hi[2]; iz++) for (int iy=lo[1]; iy<=hi[1]; iy++) - for (int ix=lo[0]; ix<=hi[0]; ix++) - xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; + for (int ix=lo[0]; ix<=hi[0]; ix++) + xnew[iz] += tmpmean[mfi](IntVect(ix,iy,iz),0)*phi; #endif } @@ -680,7 +680,7 @@ compute_flux(AmrData& amrData, } #endif - flux = FL; + flux = FL; } } diff --git a/Tools/C_util/TV_TempWrite.H b/Tools/C_util/TV_TempWrite.H index fcc97067a78..624c48cc8a7 100644 --- a/Tools/C_util/TV_TempWrite.H +++ b/Tools/C_util/TV_TempWrite.H @@ -7,44 +7,44 @@ #include using namespace amrex; extern "C" { - + // dump multifab to file bool writeMF (const MultiFab* mf, - const char* file) - { - - std::string FullPath = file; - if (!FullPath.empty() && FullPath[FullPath.size()-1] != '/') - FullPath += '/'; - - if (ParallelDescriptor::IOProcessor()) - if (!amrex::UtilCreateDirectory(FullPath, 0755)) - amrex::CreateDirectoryFailed(FullPath); - - static const std::string MultiFabBaseName("MultiFab"); - FullPath += MultiFabBaseName; - - VisMF::Write(*mf,FullPath,VisMF::OneFilePerCPU); - - return true; - } - + const char* file) + { + + std::string FullPath = file; + if (!FullPath.empty() && FullPath[FullPath.size()-1] != '/') + FullPath += '/'; + + if (ParallelDescriptor::IOProcessor()) + if (!amrex::UtilCreateDirectory(FullPath, 0755)) + amrex::CreateDirectoryFailed(FullPath); + + static const std::string MultiFabBaseName("MultiFab"); + FullPath += MultiFabBaseName; + + VisMF::Write(*mf,FullPath,VisMF::OneFilePerCPU); + + return true; + } + // read multifab from file bool readMF (MultiFab& mf, - const char* file) - { - - std::string FullPath = file; - if (!FullPath.empty() && FullPath[FullPath.size()-1] != '/') - FullPath += '/'; - - static const std::string MultiFabBaseName("MultiFab"); - FullPath += MultiFabBaseName; - - VisMF::Read(mf,FullPath); - - return true; - } + const char* file) + { + + std::string FullPath = file; + if (!FullPath.empty() && FullPath[FullPath.size()-1] != '/') + FullPath += '/'; + + static const std::string MultiFabBaseName("MultiFab"); + FullPath += MultiFabBaseName; + + VisMF::Read(mf,FullPath); + + return true; + } } void NeverCalledEXT() diff --git a/Tools/C_util/ViewMF/MFNorm.cpp b/Tools/C_util/ViewMF/MFNorm.cpp index df0bfc1e165..8ac5f7812ca 100644 --- a/Tools/C_util/ViewMF/MFNorm.cpp +++ b/Tools/C_util/ViewMF/MFNorm.cpp @@ -9,7 +9,7 @@ // What's the slowest way I can think of to compute all the norms?? // Real -MFNorm (const MultiFab& mfab, +MFNorm (const MultiFab& mfab, const int exponent, const int srcComp, const int numComp, @@ -34,7 +34,7 @@ MFNorm (const MultiFab& mfab, mftmp[mftmpmfi].abs(boxes[mftmpmfi.index()], 0, numComp); myNorm = std::max(myNorm, mftmp[mftmpmfi].norm(0, 0, numComp)); } - ParallelDescriptor::ReduceRealMax(myNorm); + ParallelDescriptor::ReduceRealMax(myNorm); } else if ( exponent == 1 ) { @@ -44,7 +44,7 @@ MFNorm (const MultiFab& mfab, myNorm += mftmp[mftmpmfi].norm(1, 0, numComp); } - ParallelDescriptor::ReduceRealSum(myNorm); + ParallelDescriptor::ReduceRealSum(myNorm); } else if ( exponent == 2 ) { @@ -54,14 +54,14 @@ MFNorm (const MultiFab& mfab, myNorm += pow(mftmp[mftmpmfi].norm(2, 0, numComp), 2); } - ParallelDescriptor::ReduceRealSum(myNorm); + ParallelDescriptor::ReduceRealSum(myNorm); myNorm = sqrt( myNorm ); } else { amrex::Error("Invalid exponent to norm function"); } - + return myNorm; } diff --git a/Tools/C_util/ViewMF/checkMFghostcells.cpp b/Tools/C_util/ViewMF/checkMFghostcells.cpp index 73432e7d61f..8cd879557f8 100644 --- a/Tools/C_util/ViewMF/checkMFghostcells.cpp +++ b/Tools/C_util/ViewMF/checkMFghostcells.cpp @@ -14,7 +14,7 @@ static -void +void PrintUsage(int argc, char *argv[]) { cout << "Usage: " << endl; @@ -44,12 +44,12 @@ main (int argc, } ParmParse pp(argc-2,argv+2); - + if (pp.contains("help")) PrintUsage(argc, argv); - + aString iFile = argv[1]; - + bool ascii = false; if (pp.contains("ascii")) ascii = true; @@ -58,7 +58,7 @@ main (int argc, // MultiFab mf; readMF(mf,iFile.c_str()); - + int ngrow = mf.nGrow(); MultiFab tmp(mf.boxArray(),mf.nComp(),ngrow,Fab_allocate); @@ -80,7 +80,7 @@ main (int argc, { Box intersection = srcBox & dstBox; - mfiDst().copy(mfiSrc(), intersection, 0, + mfiDst().copy(mfiSrc(), intersection, 0, intersection, 0, mf.nComp()); } } @@ -105,6 +105,6 @@ main (int argc, } return true; } - + return ArrayViewMultiFab(&tmp); } diff --git a/Tools/C_util/ViewMF/main.cpp b/Tools/C_util/ViewMF/main.cpp index 3ee9bd2328c..3c1af106545 100644 --- a/Tools/C_util/ViewMF/main.cpp +++ b/Tools/C_util/ViewMF/main.cpp @@ -24,7 +24,7 @@ norm ( const MultiFab& mfab, #if (VIEW == 1) -void +void print_usage(int, char *argv[]) { cerr << "usage: "; @@ -41,7 +41,7 @@ main (int argc, { // Parse command line - ParmParse pp(argc-1,argv+1,NULL,NULL); + ParmParse pp(argc-1,argv+1,NULL,NULL); int nprocs = NPROCS; pp.query("nprocs", nprocs); @@ -70,13 +70,13 @@ main (int argc, tmp0.setVal(5); for (MultiFabIterator tmp0mfi(tmp0); tmp0mfi.isValid(); ++tmp0mfi) { - tmp0mfi().setVal(tmp0mfi.index()+1,tmp0mfi.validbox(),sComp,nComp); + tmp0mfi().setVal(tmp0mfi.index()+1,tmp0mfi.validbox(),sComp,nComp); } writeMF(&tmp0,"tmp0"); - + ParallelDescriptor::EndParallel(); } - + #else /* BUILD_TESTDATA */ @@ -105,7 +105,7 @@ main (int argc, MultiFab mf; readMF(mf,argv[1]); - + return ArrayViewMultiFab(&mf); } @@ -113,7 +113,7 @@ main (int argc, #elif (VIEW == 2) -void +void print_usage(int, char *argv[]) { cerr << "usage: "; @@ -133,7 +133,7 @@ main (int argc, set_new_handler(Utility::OutOfMemory); // Parse command line - ParmParse pp(argc-1,argv+1,NULL,NULL); + ParmParse pp(argc-1,argv+1,NULL,NULL); int nprocs = NPROCS; pp.query("nprocs", nprocs); #ifndef BL_USE_BSP @@ -161,7 +161,7 @@ main (int argc, tmp0.setVal(5); for (MultiFabIterator tmp0mfi(tmp0); tmp0mfi.isValid(); ++tmp0mfi) { - tmp0mfi().setVal(tmp0mfi.index()+1,tmp0mfi.validbox(),sComp,nComp); + tmp0mfi().setVal(tmp0mfi.index()+1,tmp0mfi.validbox(),sComp,nComp); } writeMF(&tmp0,"tmp0"); @@ -170,24 +170,24 @@ main (int argc, tmp1.setVal(7); for (MultiFabIterator tmp1mfi(tmp1); tmp1mfi.isValid(); ++tmp1mfi) { - tmp1mfi().setVal(tmp1mfi.index()+3,tmp1mfi.validbox(),sComp,nComp); + tmp1mfi().setVal(tmp1mfi.index()+3,tmp1mfi.validbox(),sComp,nComp); } writeMF(&tmp1,"tmp1"); - + ParallelDescriptor::EndParallel(); } #else /* BUILD_TESTDATA */ int main (int argc, - char* argv[]) + char* argv[]) { // // Make sure to catch new failures. // set_new_handler(Utility::OutOfMemory); - + if (argc < 6 || argc > 7) print_usage(argc,argv); int nprocs = NPROCS; @@ -207,14 +207,14 @@ int main (int argc, if (mf0.boxArray() != mf1.boxArray()) { - cerr << "BoxArray's incompatible" << endl; - return 0; + cerr << "BoxArray's incompatible" << endl; + return 0; } - + if (mf0.nGrow() != mf1.nGrow()) { - cerr << "nGrow's incompatible" << endl; - return 0; + cerr << "nGrow's incompatible" << endl; + return 0; } int comp0 = atoi(argv[3]); @@ -222,50 +222,50 @@ int main (int argc, int nComp = atoi(argv[5]); if (mf0.nComp() < comp0 + nComp || mf1.nComp() < comp1 + nComp) { - cerr << "nComp's incompatible" << endl; - cerr << "(need,have): (" << comp0 + nComp << "," << mf0.nComp() + cerr << "nComp's incompatible" << endl; + cerr << "(need,have): (" << comp0 + nComp << "," << mf0.nComp() << "), (" << comp1 + nComp << "," << mf1.nComp() << ")" << endl; - return 0; + return 0; } MultiFab diff(mf0.boxArray(), nComp, mf0.nGrow(), Fab_allocate); for (MultiFabIterator diffmfi(diff); diffmfi.isValid(); ++diffmfi) { - DependentMultiFabIterator mf0mfi(diffmfi, mf0); - DependentMultiFabIterator mf1mfi(diffmfi, mf1); + DependentMultiFabIterator mf0mfi(diffmfi, mf0); + DependentMultiFabIterator mf1mfi(diffmfi, mf1); - const Box& validBox = mf0mfi.validbox(); - diffmfi().setVal(0.0); - diffmfi().copy(mf0mfi(),validBox,comp0,validBox,0,nComp); - diffmfi().minus(mf1mfi(),validBox,validBox,comp1,0,nComp); + const Box& validBox = mf0mfi.validbox(); + diffmfi().setVal(0.0); + diffmfi().copy(mf0mfi(),validBox,comp0,validBox,0,nComp); + diffmfi().minus(mf1mfi(),validBox,validBox,comp1,0,nComp); } - + REAL norm0 = norm(diff,0); REAL norm1 = norm(diff,1); REAL norm2 = norm(diff,2); if(ParallelDescriptor::IOProcessor()) { - cout << "Norms of diff (0,1,2): " - << norm0 << ", " << norm1 << ", " << norm2 << endl; + cout << "Norms of diff (0,1,2): " + << norm0 << ", " << norm1 << ", " << norm2 << endl; } - + if (argc == 7) { - writeMF(&diff,argv[6]); - return 1; + writeMF(&diff,argv[6]); + return 1; } else { - if (norm0 == 0 && norm1 == 0 && norm2 == 0) - { - cout << "MultiFabs equal!" << endl; - return 1; + if (norm0 == 0 && norm1 == 0 && norm2 == 0) + { + cout << "MultiFabs equal!" << endl; + return 1; + + } else { - } else { - - return ArrayViewMultiFab(&diff); - } + return ArrayViewMultiFab(&diff); + } } } @@ -282,7 +282,7 @@ norm ( const MultiFab& mfab, int numComp = mfab.nComp(); int numGrow = 0; const BoxArray& grids = mfab.boxArray(); - + // Get a copy of the multifab, zero covered locations MultiFab mftmp( grids, numComp, numGrow, Fab_allocate ); mftmp.copy( mfab ); @@ -295,7 +295,7 @@ norm ( const MultiFab& mfab, mftmpmfi().abs(mftmpmfi.validbox()); } myNorm = mftmp.max( srcComp ); - ParallelDescriptor::ReduceRealMax(myNorm); + ParallelDescriptor::ReduceRealMax(myNorm); } else if ( exponent == 1 ) { @@ -304,7 +304,7 @@ norm ( const MultiFab& mfab, mftmpmfi().abs(); myNorm += mftmpmfi().sum( srcComp, numComp ); } - ParallelDescriptor::ReduceRealSum(myNorm); + ParallelDescriptor::ReduceRealSum(myNorm); } else if ( exponent == 2 ) { @@ -319,17 +319,17 @@ norm ( const MultiFab& mfab, int sum_exp = 1; for ( MultiFabIterator mftmpmfi(mftmp); mftmpmfi.isValid(); ++mftmpmfi) { - const Box& bx = mftmpmfi.validbox(); + const Box& bx = mftmpmfi.validbox(); myNorm += mftmpmfi().norm( bx, sum_exp, srcComp, numComp ); } - ParallelDescriptor::ReduceRealSum(myNorm); + ParallelDescriptor::ReduceRealSum(myNorm); myNorm = sqrt( myNorm ); } else { amrex::Error("Invalid exponent to norm function"); } - + return myNorm; } diff --git a/Tools/C_util/ViewMF/mfMinMax.cpp b/Tools/C_util/ViewMF/mfMinMax.cpp index 284c1939043..4e9689e584c 100644 --- a/Tools/C_util/ViewMF/mfMinMax.cpp +++ b/Tools/C_util/ViewMF/mfMinMax.cpp @@ -14,7 +14,7 @@ using std::cerr; using std::set_new_handler; static -void +void PrintUsage(int argc, char *argv[]) { cout << "Usage: " << endl; @@ -37,7 +37,7 @@ main (int argc, PrintUsage(argc,argv); ParmParse pp; - + std::string iFile; pp.get("iFile", iFile); // @@ -64,7 +64,7 @@ main (int argc, comps[i] = sComp + i; } - + int ngrow = mf.nGrow(); pp.query("ngrow",ngrow); ngrow = std::min(ngrow,mf.nGrow()); @@ -73,6 +73,6 @@ main (int argc, for (int n=0; n static -void +void PrintUsage(int argc, char *argv[]) { std::cout << "Usage: " << std::endl; @@ -28,15 +28,15 @@ PrintUsage(int argc, char *argv[]) int main (int argc, - char* argv[]) + char* argv[]) { amrex::Initialize(argc,argv); ParmParse pp; - + if (pp.contains("help")) PrintUsage(argc, argv); - + std::string iFile0; std::string iFile1; @@ -54,7 +54,7 @@ int main (int argc, int nComp = -1; pp.query("ncomp", nComp); - + int ngrow = 0; pp.query("ngrow",ngrow); BL_ASSERT(ngrow>=0); @@ -81,7 +81,7 @@ int main (int argc, common_bl.join(BoxList(amrex::intersect(mf1.boxArray(), mf0.boxArray()[i]))); compBoxes = BoxArray(common_bl); } - + if (ngrow != std::min(ngrow,mf0.nGrow())) { amrex::Warning("Shrinking ngrow to that available in mfab0"); @@ -103,10 +103,10 @@ int main (int argc, if (mf0.nComp() < comp0 + nComp || mf1.nComp() < comp1 + nComp) { - std::cerr << "nComp's incompatible" << std::endl; - std::cerr << "(need,have): (" << comp0 + nComp << "," << mf0.nComp() + std::cerr << "nComp's incompatible" << std::endl; + std::cerr << "(need,have): (" << comp0 + nComp << "," << mf0.nComp() << "), (" << comp1 + nComp << "," << mf1.nComp() << ")" << std::endl; - return 0; + return 0; } // @@ -118,7 +118,7 @@ int main (int argc, { fabs.set(i,new FArrayBox(compBoxes[i],nComp)); fabs[i].setVal(0.0); - + for (MFIter mf0_mfi(mf0); mf0_mfi.isValid(); ++mf0_mfi) { const Box& box = amrex::grow(mf0_mfi.validbox(),ngrow) & fabs[i].box(); @@ -147,26 +147,26 @@ int main (int argc, if(ParallelDescriptor::IOProcessor()) { - std::cout << "Norms of diff (0,1,2): " - << norm0 << ", " << norm1 << ", " << norm2 << std::endl; + std::cout << "Norms of diff (0,1,2): " + << norm0 << ", " << norm1 << ", " << norm2 << std::endl; } - + if (!outfile.empty()) { - writeMF(&diffmfab,outfile.c_str()); - return 1; + writeMF(&diffmfab,outfile.c_str()); + return 1; } else { - if (norm0 == 0 && norm1 == 0 && norm2 == 0) - { - std::cout << "MultiFabs equal!" << std::endl; - return 1; + if (norm0 == 0 && norm1 == 0 && norm2 == 0) + { + std::cout << "MultiFabs equal!" << std::endl; + return 1; + + } else { - } else { - - return ArrayViewMultiFab(&diffmfab); - } + return ArrayViewMultiFab(&diffmfab); + } } amrex::Finalize(); diff --git a/Tools/C_util/ViewMF/viewMFdiffHardWire.cpp b/Tools/C_util/ViewMF/viewMFdiffHardWire.cpp index 16cae395ea7..9571784018f 100644 --- a/Tools/C_util/ViewMF/viewMFdiffHardWire.cpp +++ b/Tools/C_util/ViewMF/viewMFdiffHardWire.cpp @@ -15,7 +15,7 @@ using std::setprecision; using std::set_new_handler; static -void +void PrintUsage(int argc, char *argv[]) { cout << "Usage: " << endl; @@ -33,7 +33,7 @@ PrintUsage(int argc, char *argv[]) int main (int argc, - char* argv[]) + char* argv[]) { // // Make sure to catch new failures. @@ -41,7 +41,7 @@ int main (int argc, set_new_handler(Utility::OutOfMemory); ParallelDescriptor::StartParallel(&argc, &argv); - + // // Parse the command line // @@ -55,10 +55,10 @@ int main (int argc, } ParmParse pp(argc-3,argv+3); - + if (pp.contains("help")) PrintUsage(argc, argv); - + const aString iFile0 = argv[1]; const aString iFile1 = argv[2]; @@ -70,7 +70,7 @@ int main (int argc, int nComp = -1; pp.query("ncomp", nComp); - + aString outfile; pp.query("outfile",outfile); @@ -137,10 +137,10 @@ int main (int argc, if (mf0.nComp() < comp0 + nComp || mf1.nComp() < comp1 + nComp) { - cerr << "nComp's incompatible" << endl; - cerr << "(need,have): (" << comp0 + nComp << "," << mf0.nComp() + cerr << "nComp's incompatible" << endl; + cerr << "(need,have): (" << comp0 + nComp << "," << mf0.nComp() << "), (" << comp1 + nComp << "," << mf1.nComp() << ")" << endl; - return 0; + return 0; } // @@ -155,7 +155,7 @@ cout << pedge0 << " " << pedge1 << endl; MultiFab datEdge1(pedge1BA, nComp, ngrow); datEdge0.setVal(1.0e30); datEdge1.setVal(1.0e30); - + for (MultiFabIterator mf0_mfi(mf0); mf0_mfi.isValid(); ++mf0_mfi) { const Box& box = ::grow(mf0_mfi.validbox(),mf0.nGrow()) & datEdge0[0].box(); @@ -189,26 +189,26 @@ cout << pedge0 << " " << pedge1 << endl; if(ParallelDescriptor::IOProcessor()) { - cout << "Norms of diff (0,1,2): " - << norm0 << ", " << norm1 << ", " << norm2 << endl; + cout << "Norms of diff (0,1,2): " + << norm0 << ", " << norm1 << ", " << norm2 << endl; } - + if (!outfile.isNull()) { - writeMF(&diffmfab,outfile.c_str()); - return 1; + writeMF(&diffmfab,outfile.c_str()); + return 1; } else { - if (norm0 == 0 && norm1 == 0 && norm2 == 0) - { - cout << "MultiFabs equal!" << endl; - return 1; + if (norm0 == 0 && norm1 == 0 && norm2 == 0) + { + cout << "MultiFabs equal!" << endl; + return 1; + + } else { - } else { - - return ArrayViewMultiFab(&diffmfab); - } + return ArrayViewMultiFab(&diffmfab); + } } } diff --git a/Tools/C_util/WritePlotFile.H b/Tools/C_util/WritePlotFile.H index ad0dbf58ea4..0916e3870b4 100644 --- a/Tools/C_util/WritePlotFile.H +++ b/Tools/C_util/WritePlotFile.H @@ -11,25 +11,25 @@ // plotfile data format void writePlotFile (const char* name, - const amrex::MultiFab& mf, - const amrex::Geometry& geom, - const amrex::IntVect& refRatio, - amrex::Real bgVal, + const amrex::MultiFab& mf, + const amrex::Geometry& geom, + const amrex::IntVect& refRatio, + amrex::Real bgVal, const amrex::Vector& names); // Write an array of multifabs pulling all the data from the AmrData void WritePlotFile(const amrex::Vector& mfa, const amrex::Vector& probDomain, - amrex::AmrData& amrdToMimic, - const std::string& oFile, - bool verbose = false, + amrex::AmrData& amrdToMimic, + const std::string& oFile, + bool verbose = false, const amrex::Vector& varNames = amrex::Vector()); // Write an array of multifabs pulling all the data from the AmrData void WritePlotFile(const amrex::Vector& mfa, - amrex::AmrData& amrdToMimic, - const std::string& oFile, - bool verbose = false, + amrex::AmrData& amrdToMimic, + const std::string& oFile, + bool verbose = false, const amrex::Vector& varNames = amrex::Vector()); #endif diff --git a/Tools/C_util/WritePlotFile.cpp b/Tools/C_util/WritePlotFile.cpp index a35ec90f0a1..894b74f904b 100644 --- a/Tools/C_util/WritePlotFile.cpp +++ b/Tools/C_util/WritePlotFile.cpp @@ -30,12 +30,12 @@ thePlotFileType () void writePlotFile (const std::string& dir, - std::ostream& os, - int level, - const MultiFab& mf, - const Geometry& geom, - const IntVect& refRatio, - Real bgVal, + std::ostream& os, + int level, + const MultiFab& mf, + const Geometry& geom, + const IntVect& refRatio, + Real bgVal, const Vector& names) { // @@ -48,16 +48,16 @@ writePlotFile (const std::string& dir, Vector< Box > domain(finestLevel+1); const IndexType& ixType = mf.boxArray().ixType(); if (ixType != IndexType::TheCellType()) - amrex::Error("writePlotfile unable to handle non cell-centered data for now"); + amrex::Error("writePlotfile unable to handle non cell-centered data for now"); Box tmpb = Box(geom.Domain()).convert(ixType); Vector corr(BL_SPACEDIM); for (int d = 0; d < BL_SPACEDIM; d++) { - corr[d] = (ixType.ixType(d) == IndexType::CELL ? 1 : 0); + corr[d] = (ixType.ixType(d) == IndexType::CELL ? 1 : 0); } for (int M = 0; M < finestLevel; M++) { - tmpb.coarsen(refRatio); + tmpb.coarsen(refRatio); } domain[0] = tmpb; const int levelSteps = 0; @@ -66,30 +66,30 @@ writePlotFile (const std::string& dir, const BoxArray& grids = mf.boxArray(); for (int j = 0; j<= finestLevel; j++) { - if (j!=0) - domain[j] = Box( domain[j-1] ).refine(refRatio); - dx[j].resize(BL_SPACEDIM); - for (int k = 0; k < BL_SPACEDIM; k++) - { - dx[j][k] = (geom.ProbHi(k) - geom.ProbLo(k))/domain[j].length(k); - } - if (j==0) - { - grid_loc[j].resize(1); - grid_loc[j][0] = RealBox(geom.ProbLo(),geom.ProbHi()); - } else { - grid_loc[j].resize(grids.size()); - for (int L=0; L < grids.size(); L++) - { - const Box& bx = grids[L]; - grid_loc[j][L] = RealBox(AMREX_D_DECL( dx[j][0]*bx.smallEnd(0), - dx[j][1]*bx.smallEnd(1), - dx[j][2]*bx.smallEnd(2) ), - AMREX_D_DECL( dx[j][0]*(bx.bigEnd(0)+corr[0]), - dx[j][1]*(bx.bigEnd(1)+corr[1]), - dx[j][2]*(bx.bigEnd(2)+corr[2]) )); - } - } + if (j!=0) + domain[j] = Box( domain[j-1] ).refine(refRatio); + dx[j].resize(BL_SPACEDIM); + for (int k = 0; k < BL_SPACEDIM; k++) + { + dx[j][k] = (geom.ProbHi(k) - geom.ProbLo(k))/domain[j].length(k); + } + if (j==0) + { + grid_loc[j].resize(1); + grid_loc[j][0] = RealBox(geom.ProbLo(),geom.ProbHi()); + } else { + grid_loc[j].resize(grids.size()); + for (int L=0; L < grids.size(); L++) + { + const Box& bx = grids[L]; + grid_loc[j][L] = RealBox(AMREX_D_DECL( dx[j][0]*bx.smallEnd(0), + dx[j][1]*bx.smallEnd(1), + dx[j][2]*bx.smallEnd(2) ), + AMREX_D_DECL( dx[j][0]*(bx.bigEnd(0)+corr[0]), + dx[j][1]*(bx.bigEnd(1)+corr[1]), + dx[j][2]*(bx.bigEnd(2)+corr[2]) )); + } + } } const int Coord = 0; BoxArray tba = BoxArray(&tmpb,1); @@ -98,7 +98,7 @@ writePlotFile (const std::string& dir, for (int j=0; j& names) { @@ -287,14 +287,14 @@ writePlotFile (const char* name, if(ParallelDescriptor::IOProcessor()) { std::cout << "Write plotfile time = " << dPlotFileTime << " seconds." << std::endl; } - + } void WritePlotFile(const Vector& mfa, const Vector& probDomain, - AmrData& amrdToMimic, - const std::string& oFile, - bool verbose, + AmrData& amrdToMimic, + const std::string& oFile, + bool verbose, const Vector& varNames) { // If varnames not provided, use names in original plotfile @@ -303,7 +303,7 @@ void WritePlotFile(const Vector& mfa, int ntype = derives.size(); int finestLevel = mfa.size() - 1; AMREX_ALWAYS_ASSERT(finestLevel >= 0); - + if (ParallelDescriptor::IOProcessor()) if (!amrex::UtilCreateDirectory(oFile,0755)) amrex::CreateDirectoryFailed(oFile); @@ -314,13 +314,13 @@ void WritePlotFile(const Vector& mfa, std::string oFileHeader(oFile); oFileHeader += "/Header"; - + VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); std::ofstream os; - + os.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - + if (verbose && ParallelDescriptor::IOProcessor()) std::cout << "Opening file = " << oFileHeader << '\n'; @@ -368,12 +368,12 @@ void WritePlotFile(const Vector& mfa, int nGrids = amrdToMimic.boxArray(iLevel).size(); std::string LevelStr = amrex::Concatenate("Level_", iLevel, 1); - + if (ParallelDescriptor::IOProcessor()) { os << iLevel << ' ' << nGrids << ' ' << amrdToMimic.Time() << '\n'; os << 0 << '\n'; - + for (i = 0; i < nGrids; ++i) { for (int n = 0; n < BL_SPACEDIM; n++) @@ -390,7 +390,7 @@ void WritePlotFile(const Vector& mfa, std::string Level(oFile); Level += '/'; Level += LevelStr; - + if (!amrex::UtilCreateDirectory(Level, 0755)) amrex::CreateDirectoryFailed(Level); } @@ -402,12 +402,12 @@ void WritePlotFile(const Vector& mfa, // Now build the full relative pathname of the MultiFab. // static const std::string MultiFabBaseName("/MultiFab"); - + std::string PathName(oFile); PathName += '/'; PathName += LevelStr; PathName += MultiFabBaseName; - + if (ParallelDescriptor::IOProcessor()) { // @@ -425,9 +425,9 @@ void WritePlotFile(const Vector& mfa, } void WritePlotFile(const Vector& mfa, - AmrData& amrdToMimic, - const std::string& oFile, - bool verbose, + AmrData& amrdToMimic, + const std::string& oFile, + bool verbose, const Vector& varNames) { WritePlotFile(mfa,amrdToMimic.ProbDomain(),amrdToMimic,oFile,verbose,varNames); diff --git a/Tools/C_util/dbgTools/crsGrids.cpp b/Tools/C_util/dbgTools/crsGrids.cpp index bb8c0a9f95c..c9a1c767e9a 100644 --- a/Tools/C_util/dbgTools/crsGrids.cpp +++ b/Tools/C_util/dbgTools/crsGrids.cpp @@ -82,7 +82,7 @@ main (int argc, amrex::Abort("nCrsRatio != nRefLevels"); std::cout << nRefLevels << std::endl; - + //----------------------------------------------------// // Loop Through Refined Levels and Generate MultiFabs // diff --git a/Tools/C_util/dbgTools/intersectGrids.cpp b/Tools/C_util/dbgTools/intersectGrids.cpp index 0387d42a226..10a59c9de9b 100644 --- a/Tools/C_util/dbgTools/intersectGrids.cpp +++ b/Tools/C_util/dbgTools/intersectGrids.cpp @@ -138,7 +138,7 @@ main (int argc, amrex::Abort("nCrsRatio != nRefLevels"); cout << nRefLevels << endl; - + //----------------------------------------------------// // Loop Through Refined Levels and Generate MultiFabs // @@ -156,7 +156,7 @@ main (int argc, for (int bx=0; bx < nGrds; bx++) { is >> inBox; - + if (intersectBox.intersects(inBox)) { oBoxes.set(nOBoxes, inBox); diff --git a/Tools/CompileTesting/compiletesting.py b/Tools/CompileTesting/compiletesting.py index 718bde713da..129e83ca960 100755 --- a/Tools/CompileTesting/compiletesting.py +++ b/Tools/CompileTesting/compiletesting.py @@ -78,16 +78,16 @@ def compiletesting(arg_string): for test in test_list: print("Compile", test) os.chdir(os.path.join(TOP,test)) - + command = "make realclean" outfile = "makerealclean.ou" run(command, outfile) - + command = "make -j4 " + args.make_flags if args.typecheck: command += " typecheck" outfile = "make.ou" - run(command, outfile) + run(command, outfile) test_success = False if args.typecheck: diff --git a/Tools/EBSurfaceTools/ConvertEBSurface.cpp b/Tools/EBSurfaceTools/ConvertEBSurface.cpp index 12f97e253b0..257707e0238 100644 --- a/Tools/EBSurfaceTools/ConvertEBSurface.cpp +++ b/Tools/EBSurfaceTools/ConvertEBSurface.cpp @@ -97,7 +97,7 @@ main (int argc, std::ifstream ifsd(FullDataPath.c_str(), std::ios::binary); if( ! ifsd.good()) { amrex::FileOpenFailed(FullDataPath); } - + Vector flattenedEdges(SizeOfEdgeData * nNodes[i]); Vector flattenedNodes(SizeOfNodeData * nNodes[i]); Vector flattenedFrags(SizeOfFragData * nElts[i]); @@ -162,7 +162,7 @@ main (int argc, Print() << sortedNodes.size() << " unique nodes and " << surfaceFragmentsG.size() << " surfaceFragmentsG found"; if (nNodesRedundant != sortedNodes.size()) { - Print() << " (" << nNodesRedundant - sortedNodes.size() << " redundant nodes removed)"; + Print() << " (" << nNodesRedundant - sortedNodes.size() << " redundant nodes removed)"; } Print() << '\n'; @@ -270,7 +270,7 @@ list> MakePolyLines(Vector& segVec) contourLines.push_back(list()); contourLines.back().push_back(segments.front()); segments.pop_front(); - + idx = contourLines.back().back()[1]; } } diff --git a/Tools/EBSurfaceTools/isoToVTK.py b/Tools/EBSurfaceTools/isoToVTK.py index 0564533ae1d..279713feab2 100644 --- a/Tools/EBSurfaceTools/isoToVTK.py +++ b/Tools/EBSurfaceTools/isoToVTK.py @@ -22,7 +22,7 @@ d = line.split(' ') id = Points.InsertNextPoint(float(d[0]),float(d[1]),float(d[2])) -print("Done") +print("Done") Triangles = vtk.vtkCellArray() Triangle = vtk.vtkTriangle() @@ -36,7 +36,7 @@ Triangle.GetPointIds().SetId(2,int(d[2])-1) Triangles.InsertNextCell(Triangle) -print("Done") +print("Done") f.close() @@ -46,7 +46,7 @@ polydata.Modified() if vtk.VTK_MAJOR_VERSION <= 5: polydata.Update() - + writer = vtk.vtkXMLPolyDataWriter(); writer.SetFileName(outfile); if vtk.VTK_MAJOR_VERSION <= 5: diff --git a/Tools/F_scripts/fcheck.py b/Tools/F_scripts/fcheck.py index 1526788c800..20033f85ac9 100755 --- a/Tools/F_scripts/fcheck.py +++ b/Tools/F_scripts/fcheck.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -# a simple routine to parse Fortran files and make sure that things are -# declared double precision, and constants are of the form 1.0_dp_t or +# a simple routine to parse Fortran files and make sure that things are +# declared double precision, and constants are of the form 1.0_dp_t or # 1.0d0. import os @@ -30,7 +30,7 @@ def __init__(self,extension=""): def visit(argFiles, dirname, files): # invoked by os.path.walk to find all the files - # matching our + # matching our for file in files: base, ext = os.path.splitext(file) if (ext == argFiles.extension): @@ -38,13 +38,13 @@ def visit(argFiles, dirname, files): #----------------------------------------------------------------------------- -# main program +# main program #----------------------------------------------------------------------------- # generate the regular expressions we will use to search # helpful site: http://re-try.appspot.com/ -# look for 'real' followed by any type of space character '\s', or +# look for 'real' followed by any type of space character '\s', or # an open parenthesis real_re = re.compile('real', re.IGNORECASE) @@ -75,7 +75,7 @@ def visit(argFiles, dirname, files): root = os.getcwd() os.path.walk(root, visit, currentFiles) - + for file in currentFiles.files: # open the file for parsing @@ -104,7 +104,7 @@ def visit(argFiles, dirname, files): print lineNum, ": ", line, - # if we find a floating point constant, make sure that it is + # if we find a floating point constant, make sure that it is # double precision if (not const_fp_re.search(line) == None and const_dp_re.search(line) == None): @@ -117,7 +117,7 @@ def visit(argFiles, dirname, files): - line = fin.readline() + line = fin.readline() lineNum += 1 if (badFile == 1): @@ -125,6 +125,6 @@ def visit(argFiles, dirname, files): - - - + + + diff --git a/Tools/GNUMake/README.md b/Tools/GNUMake/README.md index 6fcec95e264..61895344121 100644 --- a/Tools/GNUMake/README.md +++ b/Tools/GNUMake/README.md @@ -1,4 +1,4 @@ -The build system requires GNU make >= 3.81 and python >= 2.7. +The build system requires GNU make >= 3.81 and python >= 2.7. Typically an application will have its own `GNUmakefile`. (See `Tutorials/HelloWorld_C/` for a simple example, or @@ -84,7 +84,7 @@ The `Make.defs` includes the following files in the listed order: * `Make.local`: This file is not in the AMReX repository. It is the last file included in `Make.defs`, if it exists. It is meant for a purely local setup that can override the setup in previously - included files. See `Make.local.template` for examples. + included files. See `Make.local.template` for examples. The build system uses the `vpath` directive to specify a list of directories to be searched for files. The list of directories is @@ -108,4 +108,4 @@ variable `LIBRARY_LOCATIONS`. Each directory in `LIBRARY_LOCATIONS` will be prefixed with `-L` and passed to the compilers. See User's Guide for more information on various build options (e.g., -`USE_MPI`). +`USE_MPI`). diff --git a/Tools/Plotfile/AMReX_PPMUtil.cpp b/Tools/Plotfile/AMReX_PPMUtil.cpp index 0badf6bba31..776258a50d9 100644 --- a/Tools/Plotfile/AMReX_PPMUtil.cpp +++ b/Tools/Plotfile/AMReX_PPMUtil.cpp @@ -48,7 +48,7 @@ int loadPalette (const std::string& filename, amrex::Abort("loadPalette: fread() failed to read B"); } - if ( num_elements == 4 ) + if ( num_elements == 4 ) { if (std::fread(a.data(), 1, NCOLOR, fp) != NCOLOR) { diff --git a/Tools/Plotfile/fcompare.cpp b/Tools/Plotfile/fcompare.cpp index 8d98034351b..7003d00517f 100644 --- a/Tools/Plotfile/fcompare.cpp +++ b/Tools/Plotfile/fcompare.cpp @@ -25,7 +25,7 @@ int main_main() bool any_nans = false; ErrZone err_zone; bool all_variables_found = true; - + // defaults int norm = 0; std::string plotfile_a; @@ -61,7 +61,7 @@ int main_main() } else if (fname == "--abs_tol") { atol = std::stod(amrex::get_command_argument(++farg)); } else if (fname == "--abort_if_not_all_found") { - abort_if_not_all_found = true; + abort_if_not_all_found = true; } else { break; } @@ -148,7 +148,7 @@ int main_main() auto r = std::find(std::begin(names_a),std::end(names_a),names_b[n_b]); if (r == std::end(names_a)) { amrex::Print() << " WARNING: variable " << names_b[n_b] << " not found in plotfile 1\n"; - all_variables_found = false; + all_variables_found = false; } } diff --git a/Tools/Postprocessing/C_Src/HorizontalAvg.cpp b/Tools/Postprocessing/C_Src/HorizontalAvg.cpp index 636c44dee4c..c9b65320c9c 100644 --- a/Tools/Postprocessing/C_Src/HorizontalAvg.cpp +++ b/Tools/Postprocessing/C_Src/HorizontalAvg.cpp @@ -24,11 +24,11 @@ main (int argc, ParmParse pp; FArrayBox::setFormat(FABio::FAB_IEEE_32); - + int verbose; pp.query("verbose",verbose); if (verbose > 2) AmrData::SetVerbose(true); - + std::string infile; pp.get("infile",infile); std::string outdir = infile + std::string("_stats"); pp.query("outdir",outdir); std::string outfile("horizAvg.dat"); pp.query("outfile",outfile); @@ -37,7 +37,7 @@ main (int argc, Amrvis::FileType fileType(Amrvis::NEWPLT); DataServices dataServices(infile, fileType); if (!dataServices.AmrDataOk()) - DataServices::Dispatch(DataServices::ExitRequest, NULL); + DataServices::Dispatch(DataServices::ExitRequest, NULL); AmrData& amrData = dataServices.AmrDataRef(); Vector comps; @@ -70,7 +70,7 @@ main (int argc, comps[localWtComp] = weightComp; } } - + int nComp = comps.size(); Vector varNames(nComp); for (int i=0; i havg(ksize*nComp,0.); Vector wavg(ksize,0.); Vector area(ksize,0.); - + FArrayBox mask; for (MFIter mfi(mf); mfi.isValid(); ++mfi) { const Box& box = mfi.validbox(); FArrayBox& fab = mf[mfi]; mask.resize(box,1); mask.setVal(1); - + if (weightComp>=0) { for (int n=0; n1) @@ -161,7 +161,7 @@ main (int argc, } } } - + if (!amrex::UtilCreateDirectory(outdir, 0755)) amrex::CreateDirectoryFailed(outdir); diff --git a/Tools/Postprocessing/C_Src/IntegrateComp.cpp b/Tools/Postprocessing/C_Src/IntegrateComp.cpp index 280db23169d..291f0b33489 100644 --- a/Tools/Postprocessing/C_Src/IntegrateComp.cpp +++ b/Tools/Postprocessing/C_Src/IntegrateComp.cpp @@ -2,7 +2,7 @@ A very simple example of reading a plotfile and doing a simple analysis. Here, we want to do a volume integral of a component specified by name. - The twist here is to demonstrate what this might look like if the amr data were coming down a + The twist here is to demonstrate what this might look like if the amr data were coming down a pipe (such as SENSEI, e.g.). So, we read the plotfile as usual, but mine it for the required data, then put that data into a couple of structs that then are queried from that point forward. @@ -17,7 +17,7 @@ using namespace amrex; static -void +void print_usage (int, char* argv[]) { @@ -61,7 +61,7 @@ struct AMReXMeshHierarchy struct AMReXDataHierarchy { /* - Data on a AMReXMeshHierarchy, currently pointing to MultiFabs of + Data on a AMReXMeshHierarchy, currently pointing to MultiFabs of named variables managed by an AmrData object. */ public: @@ -173,7 +173,7 @@ main (int argc, // Zero out covered cells if (lev < finestLevel) { - std::vector< std::pair > isects = baf.intersections(box); + std::vector< std::pair > isects = baf.intersections(box); for (int ii = 0; ii < isects.size(); ii++) { myFab.setVal(0,isects[ii].second,0,1); } diff --git a/Tools/Postprocessing/C_Src/MultiFabToMatLab.cpp b/Tools/Postprocessing/C_Src/MultiFabToMatLab.cpp index 78eefa57c23..323d10a6da0 100644 --- a/Tools/Postprocessing/C_Src/MultiFabToMatLab.cpp +++ b/Tools/Postprocessing/C_Src/MultiFabToMatLab.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include #include #include @@ -13,7 +13,7 @@ #include #include -using namespace amrex; +using namespace amrex; // // This MUST be defined if don't have pubsetbuf() in I/O Streams Library. @@ -81,52 +81,52 @@ int main(int argc, char* argv[]) } ParmParse pp; - std::string name; - pp.get("infile", name); + std::string name; + pp.get("infile", name); // - // MatLab expects native floating-point format. + // MatLab expects native floating-point format. // - FArrayBox::setFormat(FABio::FAB_NATIVE); - - VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); - char buf[128]; - std::string file = name; - file += ".mat"; - std::ofstream os; - - os.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); - - os.open(file.c_str(), std::ios::out|std::ios::binary); - + FArrayBox::setFormat(FABio::FAB_NATIVE); + + VisMF::IO_Buffer io_buffer(VisMF::IO_Buffer_Size); + char buf[128]; + std::string file = name; + file += ".mat"; + std::ofstream os; + + os.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); + + os.open(file.c_str(), std::ios::out|std::ios::binary); + if(os.fail()) { - amrex::FileOpenFailed(file); + amrex::FileOpenFailed(file); } - MultiFab in; - VisMF::Read(in, name); - const BoxArray ba = in.boxArray(); + MultiFab in; + VisMF::Read(in, name); + const BoxArray ba = in.boxArray(); //Fake a xlo to xhi for(int i = 0; i < ba.size(); ++i) { - const Box& b = ba[i]; + const Box& b = ba[i]; Real xlo[BL_SPACEDIM], xhi[BL_SPACEDIM]; for(int d = 0; d #include -#include +#include #include #include #include @@ -42,16 +42,16 @@ using namespace amrex; /// This stores the metadata associated with an AMReX particle header file. /// struct ParticleHeader { - + /// - /// Initializes the struct to store the header information associated with + /// Initializes the struct to store the header information associated with /// the given particle header file. For example, if plt_file = "plt00000" /// and particle_type = "Tracer", this will read the Header file at /// "plt00000/Tracer/Header" /// - ParticleHeader(const std::string& plt_file, + ParticleHeader(const std::string& plt_file, const std::string& particle_type) { - + plt_file_name = plt_file; par_file_name = plt_file_name + "/" + particle_type; hdr_file_name = par_file_name + "/Header"; @@ -69,7 +69,7 @@ struct ParticleHeader { } #ifdef USE_MPI MPI_Finalize(); -#endif +#endif exit(1); } @@ -86,29 +86,29 @@ struct ParticleHeader { } #ifdef USE_MPI MPI_Finalize(); -#endif +#endif exit(1); } file.close(); - + num_real = ndim + num_real_extra; num_int = 2*is_checkpoint + num_int_extra; num_comp = num_real + num_int; - - std::string directions = "xyz"; + + std::string directions = "xyz"; for (int i = 0; i < ndim; ++i) { std::stringstream ss; ss << particle_type << "_position_" << directions[i]; comp_names.push_back(ss.str()); } - + for (int i = 0; i < num_real_extra; ++i) { std::stringstream ss; ss << particle_type << "_" << real_comps[i]; comp_names.push_back(ss.str()); } - + if (is_checkpoint) { { std::stringstream ss; @@ -121,14 +121,14 @@ struct ParticleHeader { comp_names.push_back(ss.str()); } } - + for (int i = 0; i < num_int_extra; ++i) { std::stringstream ss; ss << particle_type << "_" << int_comps[i]; comp_names.push_back(ss.str()); } } - + // This is the metadata actually stored in the header files std::string version; int ndim; @@ -153,7 +153,7 @@ struct ParticleHeader { std::string par_file_name; std::string hdr_file_name; std::vector comp_names; - + // These operators only use the data actually in the headers, not the derived stuff. friend std::ostream& operator<< (std::ostream& stream, const ParticleHeader& header); friend std::istream& operator>> (std::istream& stream, ParticleHeader& header); @@ -161,7 +161,7 @@ struct ParticleHeader { friend bool operator!=(const ParticleHeader& lhs, const ParticleHeader& rhs); }; -bool operator==(const ParticleHeader& lhs, const ParticleHeader& rhs) { +bool operator==(const ParticleHeader& lhs, const ParticleHeader& rhs) { return lhs.version == rhs.version && lhs.ndim == rhs.ndim && lhs.num_real_extra == rhs.num_real_extra && @@ -170,7 +170,7 @@ bool operator==(const ParticleHeader& lhs, const ParticleHeader& rhs) { lhs.int_comps == rhs.int_comps && lhs.is_checkpoint == rhs.is_checkpoint && lhs.nparticles == rhs.nparticles && - lhs.next_id == rhs.next_id && + lhs.next_id == rhs.next_id && lhs.finest_level == rhs.finest_level && lhs.num_grids == rhs.num_grids && lhs.particle_counts == rhs.particle_counts && @@ -204,7 +204,7 @@ std::ostream& operator<< (std::ostream& stream, const ParticleHeader& header) { for (int i = 0; i <= header.finest_level; ++i) { stream << header.num_grids[i] << std::endl; } - + for (int j = 0; j <= header.finest_level; ++j) { for (unsigned i = 0; i < header.file_nums.size(); ++i) { stream << header.file_nums[j][i] << " "; @@ -294,7 +294,7 @@ int sort_particles_ascending(const void *p, const void *q) int id1, id2; std::memcpy(&id1, iptr1, sizeof(int)); std::memcpy(&id2, iptr2, sizeof(int)); - + iptr1 += sizeof(int); iptr2 += sizeof(int); @@ -305,13 +305,13 @@ int sort_particles_ascending(const void *p, const void *q) if (cpu1 != cpu2) return (cpu1 - cpu2); if (id1 != id2 ) return (id1 - id2 ); return 0; -} +} void compare_particle_chunk(const ParticleHeader& header1, const ParticleHeader& header2, std::vector& norms, int level, int file_num, int np, int offset) { - + if (np == 0) return; std::string read_file1 = getDataFileName(header1.par_file_name, level, file_num); @@ -323,11 +323,11 @@ void compare_particle_chunk(const ParticleHeader& header1, single_precision = 1; } std::size_t rsize = single_precision ? sizeof(float) : sizeof(double); - + int idata_size = header1.num_int*sizeof(int); int rdata_size = header1.num_real*rsize; - int pdata_size = rdata_size + idata_size; + int pdata_size = rdata_size + idata_size; size_t buffer_size = pdata_size * np; std::vector read_data1(buffer_size); @@ -351,17 +351,17 @@ void compare_particle_chunk(const ParticleHeader& header1, src2 = read_data2.data() + idata_size*i; std::memcpy(dst1, src1, idata_size); dst1 += idata_size; std::memcpy(dst2, src2, idata_size); dst2 += idata_size; - + src1 = read_data1.data() + idata_size*np + rdata_size*i; src2 = read_data2.data() + idata_size*np + rdata_size*i; std::memcpy(dst1, src1, rdata_size); dst1 += rdata_size; std::memcpy(dst2, src2, rdata_size); dst2 += rdata_size; } } - - qsort(data1.data(), np, pdata_size, sort_particles_ascending); + + qsort(data1.data(), np, pdata_size, sort_particles_ascending); qsort(data2.data(), np, pdata_size, sort_particles_ascending); - + char* tmp1 = data1.data(); char* tmp2 = data2.data(); for (int i = 0; i < np; ++i) { @@ -369,13 +369,13 @@ void compare_particle_chunk(const ParticleHeader& header1, int val1, val2; std::memcpy(&val1, tmp1, sizeof(int)); std::memcpy(&val2, tmp2, sizeof(int)); - norms[j+header1.num_real] = std::max((double) std::abs(val2 - val1), + norms[j+header1.num_real] = std::max((double) std::abs(val2 - val1), norms[j+header1.num_real]); if (val1 == 0) { - norms[header1.num_comp+j+header1.num_real] = + norms[header1.num_comp+j+header1.num_real] = norms[j+header1.num_real]; } else { - norms[header1.num_comp+j+header1.num_real] = + norms[header1.num_comp+j+header1.num_real] = norms[j+header1.num_real] / std::abs(val1); } tmp1 += sizeof(int); @@ -424,7 +424,7 @@ int main(int argc, char* argv[]) amrex::Finalize(); #ifndef BL_USE_MPI return r; -#endif +#endif } int main_main() @@ -438,7 +438,7 @@ int main_main() std::string fn2; std::string pt; Real rtol = 0.0; - + int farg=1; while (farg <= narg) { const std::string fname = amrex::get_command_argument(farg); @@ -475,16 +475,16 @@ int main_main() << std::endl; return EXIT_SUCCESS; } - + ParticleHeader header1(fn1, pt); ParticleHeader header2(fn2, pt); - + if (header1 != header2) { amrex::Print() << "FAIL - Particle data headers do not agree. \n"; return EXIT_FAILURE; } - // for each grid, store the corresponding information about where to look up the + // for each grid, store the corresponding information about where to look up the // particle data std::vector levels; std::vector file_nums; @@ -516,12 +516,12 @@ int main_main() MPI_Barrier(MPI_COMM_WORLD); #endif - // Each proc computes the max norm of the particle data diff over its grids + // Each proc computes the max norm of the particle data diff over its grids // The first num_comp values are the abs norms, the second are the rel norms std::vector norms(2*header1.num_comp, 0.0); for (int i = ibegin; i < iend; ++i) { - compare_particle_chunk(header1, header2, norms, - levels[i], file_nums[i], + compare_particle_chunk(header1, header2, norms, + levels[i], file_nums[i], particle_counts[i], offsets[i]); } @@ -566,6 +566,6 @@ int main_main() { amrex::Print() << " PARTICLES DISAGREE to relative tolerance " << rtol << "\n"; } - + return exit_code; } diff --git a/Tools/Postprocessing/python/column_depth.py b/Tools/Postprocessing/python/column_depth.py index 57d2751ae25..3aff2ac4705 100755 --- a/Tools/Postprocessing/python/column_depth.py +++ b/Tools/Postprocessing/python/column_depth.py @@ -34,7 +34,7 @@ def columnDepth(inputFile,columnDepthStart): input file.""" if inputFile: - try: + try: fh = open(inputFile,'r') nLines = countValidLines(fh) data = fh.read() @@ -45,10 +45,10 @@ def columnDepth(inputFile,columnDepthStart): else: data = sys.stdin.readlines() nLines = countValidLines(data) - + print nLines print data[0] - + r = numpy.zeros(nLines,numpy.float64) density = numpy.zeros(nLines,numpy.float64) columnDepth = numpy.zeros(nLines,numpy.float64) diff --git a/Tools/Postprocessing/python/conv_slopes.py b/Tools/Postprocessing/python/conv_slopes.py index 7345dca911b..f2fe5404aae 100755 --- a/Tools/Postprocessing/python/conv_slopes.py +++ b/Tools/Postprocessing/python/conv_slopes.py @@ -8,10 +8,10 @@ EOSPath="/extern/EOS/helmeos/" def compileit(network,fParallel): - """ + """ This routine compiles the fconv_slopes.f90 routine with the specified network, -and then copies the executable to the current working directory. This assumes -that this script is in data_processing/python/ in order to find the location +and then copies the executable to the current working directory. This assumes +that this script is in data_processing/python/ in order to find the location of the fconv_slopes.f90 routine; if it can't be found, we complain.""" cwd = os.getcwd() @@ -30,7 +30,7 @@ def compileit(network,fParallel): cmd = "make programs=fconv_slopes NETWORK=%s NDEBUG=" % networkPath print cmd (status, out) = commands.getstatusoutput(cmd) - if status != 0: + if status != 0: print "Compilation Error:" print out sys.exit() @@ -52,10 +52,10 @@ def compileit(network,fParallel): sys.exit() os.chdir(cwd) - + def runit(network, fParallel, input, output,*args): - + compileit(network,fParallel) print "Running fconv_slopes..." @@ -64,22 +64,22 @@ def runit(network, fParallel, input, output,*args): (status,out) = commands.getstatusoutput(cmd) if status != 0: print out - + if __name__ == "__main__": import optparse usage=""" -This script compiles the data_processing/fconv_slopes.f90 routine with a -specified (via the -n or --network options) reaction network, then executes +This script compiles the data_processing/fconv_slopes.f90 routine with a +specified (via the -n or --network options) reaction network, then executes the fconv_slopes routine on the input file (-i or --input) and dumps the result to the output file (-o or --output; defaults to .out).""" - + parser = optparse.OptionParser(usage=usage) parser.add_option("-f", "--fParallel", dest="fParallel", default="~/MAESTRO/fParallel/", help="specify the location of the fParallel directory [default: %default]") - parser.add_option("-n", "--network", dest="network", + parser.add_option("-n", "--network", dest="network", help="specify which NETWORK to compile with") parser.add_option("-i", "--input", dest="inputFile", help="specify the input FILE", metavar="FILE") @@ -104,6 +104,6 @@ def runit(network, fParallel, input, output,*args): print "Change this with the -f option." sys.exit() - runit(options.network, + runit(options.network, options.fParallel, options.inputFile, options.outputFile) diff --git a/Tools/Postprocessing/python/dumpparthistory.py b/Tools/Postprocessing/python/dumpparthistory.py index ec2ec40b97f..044ec4b9cce 100755 --- a/Tools/Postprocessing/python/dumpparthistory.py +++ b/Tools/Postprocessing/python/dumpparthistory.py @@ -13,7 +13,7 @@ #----------------------------------------------------------------------------- def main(files): - # this returns a dict whose keys are a unique identifier (based on + # this returns a dict whose keys are a unique identifier (based on # id and CPU) and values are the actual particle objects particlesDict = parseparticles.parseParticleFile(files) @@ -41,7 +41,7 @@ def main(files): # get numpy arrays containing the time and coordinate # information for particle 0 coords, time = particles[n].flatten() - dim = particles[n].dim + dim = particles[n].dim # output to a flie of = open("particle_history.%03d" % (n), 'w') @@ -51,32 +51,32 @@ def main(files): idstuff = str(particles[n]) for line in idstuff.split("\n"): of.write("# %s\n" % (line)) - + # next the column labels if (dim == 1): - of.write("# %20s %20s\n" % ("time", "x")) + of.write("# %20s %20s\n" % ("time", "x")) elif (dim == 2): - of.write("# %20s %20s %20s\n" % ("time", "x", "y")) + of.write("# %20s %20s %20s\n" % ("time", "x", "y")) elif (dim == 3): of.write("# %20s %20s %20s %20s\n" % ("time", "x", "y", "z")) - - + + # t, x, [y, [z]] in columns i = 0 while (i < len(particles[n].history)): - + if (dim == 1): - of.write(" %20.10f %20.10f\n" % + of.write(" %20.10f %20.10f\n" % (time[i], coords[0,i])) elif (dim == 2): - of.write(" %20.10f %20.10f %20.10f\n" % + of.write(" %20.10f %20.10f %20.10f\n" % (time[i], coords[0,i], coords[1,i])) elif (dim == 3): - of.write(" %20.10f %20.10f %20.10f %20.10f\n" % + of.write(" %20.10f %20.10f %20.10f %20.10f\n" % (time[i], coords[0,i], coords[1,i], coords[2,i])) i += 1 - + of.close() @@ -85,7 +85,7 @@ def main(files): - + #----------------------------------------------------------------------------- @@ -100,4 +100,4 @@ def main(files): - + diff --git a/Tools/Postprocessing/python/eos_data.txt b/Tools/Postprocessing/python/eos_data.txt index d3ca40ffad4..cf68000bf6a 100644 --- a/Tools/Postprocessing/python/eos_data.txt +++ b/Tools/Postprocessing/python/eos_data.txt @@ -1,27 +1,27 @@ -Author: -Date: +Author: +Date: - # i j tfromp rho H He3 He4 C12 N14 O16 Ne20 Mg24 Ni56 - 206 751 7.74872038e8 6.91846534e5 4.76075271e-1 8.58266241e-15 1.53230648e-1 1.23344905e-10 1.56343074e-6 1.39298137e-7 2.24837861e-8 7.34074988e-12 3.70692356e-1 - 206 750 7.77671996e8 7.65567996e5 3.87128062e-1 6.40261151e-15 1.24602048e-1 9.11480288e-11 1.09232631e-6 1.13279820e-7 1.57026697e-8 5.11639030e-12 4.88268669e-1 - 207 751 7.80469498e8 - 207 750 7.77810448e8 - 208 751 7.78438875e8 - 208 750 7.83493311e8 - 209 751 7.73428231e8 - 209 750 7.79859124e8 - 210 751 7.69835251e8 - 210 750 7.77369911e8 - 206 747 7.13910806e8 1.10600645e6 1.62003355e-1 1.68466671e-15 5.21430503e-2 3.02140124e-11 1.81395524e-7 4.74388321e-8 1.72391228e-9 2.98640467e-13 7.85853364e-1 - 206 746 7.19849955e8 1.20888990e6 1.07773701e-1 8.69580284e-16 3.46885191e-2 1.45745042e-11 6.68329704e-8 3.15638276e-8 6.22651433e-10 9.76632348e-14 8.57537681e-1 - 207 747 7.13593445e8 - 207 746 7.17120476e8 - 208 747 7.14896049e8 - 208 746 7.15930636e8 - 209 747 7.18366841e8 - 209 746 7.19430129e8 - 210 747 7.16592755e8 - 210 746 7.18982423e8 + # i j tfromp rho H He3 He4 C12 N14 O16 Ne20 Mg24 Ni56 + 206 751 7.74872038e8 6.91846534e5 4.76075271e-1 8.58266241e-15 1.53230648e-1 1.23344905e-10 1.56343074e-6 1.39298137e-7 2.24837861e-8 7.34074988e-12 3.70692356e-1 + 206 750 7.77671996e8 7.65567996e5 3.87128062e-1 6.40261151e-15 1.24602048e-1 9.11480288e-11 1.09232631e-6 1.13279820e-7 1.57026697e-8 5.11639030e-12 4.88268669e-1 + 207 751 7.80469498e8 + 207 750 7.77810448e8 + 208 751 7.78438875e8 + 208 750 7.83493311e8 + 209 751 7.73428231e8 + 209 750 7.79859124e8 + 210 751 7.69835251e8 + 210 750 7.77369911e8 + 206 747 7.13910806e8 1.10600645e6 1.62003355e-1 1.68466671e-15 5.21430503e-2 3.02140124e-11 1.81395524e-7 4.74388321e-8 1.72391228e-9 2.98640467e-13 7.85853364e-1 + 206 746 7.19849955e8 1.20888990e6 1.07773701e-1 8.69580284e-16 3.46885191e-2 1.45745042e-11 6.68329704e-8 3.15638276e-8 6.22651433e-10 9.76632348e-14 8.57537681e-1 + 207 747 7.13593445e8 + 207 746 7.17120476e8 + 208 747 7.14896049e8 + 208 746 7.15930636e8 + 209 747 7.18366841e8 + 209 746 7.19430129e8 + 210 747 7.16592755e8 + 210 746 7.18982423e8 diff --git a/Tools/Postprocessing/python/helmeos.py b/Tools/Postprocessing/python/helmeos.py index e31a40c99c4..f2af98ffb91 100644 --- a/Tools/Postprocessing/python/helmeos.py +++ b/Tools/Postprocessing/python/helmeos.py @@ -2,7 +2,7 @@ # # it builds a python wrapper around the fortran source code helmeos.f90 using # the f2py framework. The easiest way to get this module to work is to make -# sure that 1) your PYTHONPATH can find this module for importing, and 2) +# sure that 1) your PYTHONPATH can find this module for importing, and 2) # setting a FPARALLEL environment variable # e.g. (for bash) # export FPARALLEL=~/MAESTRO/fParallel @@ -70,7 +70,7 @@ def _build_fhelmEOS(fParallel_path=None): helmEOS_path = fParallel_path + helmEOS_path # open and read the helmeos.f90 file if possible - try: + try: ifh = open(helmEOS_path+helmEOS) except IOError: print "Could not locate %s at expected location %s" % ( @@ -83,7 +83,7 @@ def _build_fhelmEOS(fParallel_path=None): # fix some things so we don't have to worry about modules for lineno, line in enumerate(fileContents): # remove the bl_error and bl_warn stuff - if line.find("use bl_error") >=0: + if line.find("use bl_error") >=0: fileContents[lineno] = '!' + line elif line.find("bl_error") >= 0: txt = line.split('(')[1].split(')')[0] @@ -91,7 +91,7 @@ def _build_fhelmEOS(fParallel_path=None): elif line.find("bl_warn") >= 0: txt = line.split('(')[1].split(')')[0] fileContents[lineno] = "print *, " + txt + "\n" - + # dump the updated helmeos.f90 to a temporary, local file try: ofh = open(helmEOS,'w') @@ -121,7 +121,7 @@ def _build_fhelmEOS(fParallel_path=None): sys.exit(1) # remove the un-needed local helmeos.f90 and include file - try: + try: subprocess.check_call(["rm", helmEOS, vectorEOS]) except subprocess.CalledProcessError: print "Couldn't cleanup the local %s and %s files!" % ( @@ -147,7 +147,7 @@ def fwrap(fParallel=None,rebuild=False): if rebuild: _built = False if not _built: _build_fhelmEOS(fParallel) - else: + else: print "" print "Great - wrapper already built." print "If you want to force a rebuild, call fwrap(rebuild=True)" @@ -157,17 +157,17 @@ def fwrap(fParallel=None,rebuild=False): _built = True -# this class is essentially a data storage container for all the interesting +# this class is essentially a data storage container for all the interesting # thermodynamic properties -# +# # a call to the eos will return an object of this type class EOS_dataType(object): # initialize some important variables # depending on the input_type for the EOS, some of these won't be needed # as initial data, but will be calculated - def __init__(self, - input_type, + def __init__(self, + input_type, abar, zbar, den, temp, h, p, e, s): @@ -197,7 +197,7 @@ def __str__(self): # this is the standard eos routine that calls the fortran helmeos code # # -def eos(input_type=None, +def eos(input_type=None, abar=None, zbar=None, den=ZERO, temp=ZERO, h=ZERO, p=ZERO, e=ZERO, s=ZERO): @@ -210,12 +210,12 @@ def eos(input_type=None, nRetQuant = 23 # initialize if this is the first call - if not _initialized: + if not _initialized: fhelmEOS.helmeos_init() _initialized = True # sanity checks - if not input_type: + if not input_type: print "eos: Must specify input type!" sys.exit(1) if not abar or not zbar: diff --git a/Tools/Postprocessing/python/parseparticles.py b/Tools/Postprocessing/python/parseparticles.py index b1acd3fe98d..e56e8177083 100755 --- a/Tools/Postprocessing/python/parseparticles.py +++ b/Tools/Postprocessing/python/parseparticles.py @@ -28,7 +28,7 @@ >>> particles = myparticles.values() - Here, each item in the list is a particle object. + Here, each item in the list is a particle object. Each particle object contains data describing the particle @@ -36,7 +36,7 @@ particleInstance objects. Each particleInstance object stores the state of the particle at an instance in time. For example: - >>> time = particles[n].history[i].t + >>> time = particles[n].history[i].t is the time associated with particleInstance i of particle n. There is a unique particleInstance for particle n for every @@ -65,7 +65,7 @@ The total number of particles can be found simply as len(particles), and the number of particleInstances for a given particle can be found as len(particles[n].history) -""" +""" import sys import string @@ -80,7 +80,7 @@ _particleDict = {} -# this works for < 1000 MPI processes, but should ultimately be something +# this works for < 1000 MPI processes, but should ultimately be something # dynamic to work for all cases _idFactor = 1000 @@ -106,15 +106,15 @@ def __str__(self): """ string = "particle pos: (%g, %g, %g) \n" % \ (self.xyz[0], self.xyz[1], self.xyz[2]) + \ - " time: %g \n" % (self.t) - + " time: %g \n" % (self.t) + return string - - + + def value(self): """ - return the value of a particleInstance for comparison + return the value of a particleInstance for comparison purposes. The value is simply the time. """ return self.t @@ -131,7 +131,7 @@ class particle(object): element of which is a particleInstance object. """ - __slots__ = ["pid", "originCPU", "dim", "finalized", + __slots__ = ["pid", "originCPU", "dim", "finalized", "history", "dataNames", "numInstances"] def __new__(cls, pid=None, originCPU=None, *args, **kwargs): @@ -148,13 +148,13 @@ def __new__(cls, pid=None, originCPU=None, *args, **kwargs): _particleDict[id] = obj return _particleDict[id] - + def __init__(self, pid, originCPU, dim, dataNames): """ initialize a particle object """ - + # a MAESTRO particle is identified by 2 numbers, the pid and # the CPU that it was created on. Together, these uniquely # identify the particle. @@ -166,7 +166,7 @@ def __init__(self, pid, originCPU, dim, dataNames): # finalized is 1 when we have finished adding data and sorted # the history in time-order - self.finalized = 0 + self.finalized = 0 # the history list will store instances of the particle at # different times. @@ -179,10 +179,10 @@ def __init__(self, pid, originCPU, dim, dataNames): # keep track of the number of particle history instances we've # stored self.numInstances = 0 - - def addInstance(self, xyz=[-1.0,-1.0,-1.0],t=0.0, + + def addInstance(self, xyz=[-1.0,-1.0,-1.0],t=0.0, dataValues=[]): """ add a particleInstance object to the particle history to @@ -200,7 +200,7 @@ def addInstance(self, xyz=[-1.0,-1.0,-1.0],t=0.0, # add this particle instance to the particle history self.history.append(particleInstance(xyz,t,dataValues)) - + self.numInstances += 1 @@ -209,7 +209,7 @@ def finalize(self): """ sort the particle histories in time order (since they may have been spread across multiple processors / files. - + finalize() should only be called after all particle data has been read in """ @@ -239,7 +239,7 @@ def flatten(self): n = 0 while (n < len(self.history)): coords[:,n] = self.history[n].xyz[:self.dim] - + time[n] = self.history[n].t n += 1 @@ -254,7 +254,7 @@ def getVarIndex(self, varname): """ index = self.dataNames.index(varname) - + return index @@ -268,7 +268,7 @@ def __str__(self): for item in self.dataNames: string += " %s" % (item) - + string += "\n" return string @@ -319,7 +319,7 @@ def parseParticleFile(maestroParticleFiles): if (haveHeader == 1): oldHeader = list(dataNames) # list() makes a copy - + fields = string.split(line[1:]) # make sure we know what we are doing -- the first 2 @@ -335,11 +335,11 @@ def parseParticleFile(maestroParticleFiles): # the next fields should be x, y, and z, depending on the # dimensionality - if (fields[2] == "x" and fields[3] == "y" and + if (fields[2] == "x" and fields[3] == "y" and fields[4] == "z"): dim = 3 ix = 2; iy = 3; iz = 4 - + elif (fields[2] == "x" and fields[3] == "y"): dim = 2 ix = 2; iy = 3 @@ -356,11 +356,11 @@ def parseParticleFile(maestroParticleFiles): # then comes time if (fields[2 + dim] == "time"): it = 2 + dim - + else: print "ERROR: particle file columns not in expected order" sys.exit(2) - + # everything else is associated data if (len(fields) > 3 + dim): @@ -401,7 +401,7 @@ def parseParticleFile(maestroParticleFiles): sys.exit(2) - if not id in _particleDict: + if not id in _particleDict: particle(pid, originCPU, dim, dataNames) _particleDict[id].addInstance(xyz=xyz, diff --git a/Tools/Postprocessing/python/test_helmeos.py b/Tools/Postprocessing/python/test_helmeos.py index b3f2299e341..890a66aef77 100755 --- a/Tools/Postprocessing/python/test_helmeos.py +++ b/Tools/Postprocessing/python/test_helmeos.py @@ -46,7 +46,7 @@ def main(inputFile): # make sure we have the python wrappers built around the fortran code # # this will build the shared object file, fhelmEOS.so, which contains - # links to the fortran helmeos.f90 code. + # links to the fortran helmeos.f90 code. # the helmeos python module (imported at the top of this script) uses # the fhelmEOS.so information helmeos.fwrap() @@ -55,13 +55,13 @@ def main(inputFile): fh = open(inputFile,'r') X = numpy.zeros(nspec,numpy.float64) - + lineno = 0 for line in fh: lineno += 1 if lineno < lineSkip: continue line = line.split() - + line.pop(0) line.pop(0) diff --git a/Tools/Postprocessing/python/test_parseparticles.py b/Tools/Postprocessing/python/test_parseparticles.py index 32b83776041..b9181af4d8a 100755 --- a/Tools/Postprocessing/python/test_parseparticles.py +++ b/Tools/Postprocessing/python/test_parseparticles.py @@ -14,7 +14,7 @@ def main(fileList): usage: ./test.py particleFile1 [particleFile2 particleFile3 ... ] """ - # this returns a dict whose keys are a unique identifier (based on + # this returns a dict whose keys are a unique identifier (based on # id and CPU) and values are the actual particle objects particlesDict = parseparticles.parseParticleFile(fileList) @@ -23,7 +23,7 @@ def main(fileList): print "there are %d unique particles" % len(particles) - # plots - this is based on the plotting done in the original + # plots - this is based on the plotting done in the original # parseparticles.py script, which has since become a module for importing pylab.clf() @@ -64,7 +64,7 @@ def main(fileList): # pylab.clf() # for particle in particles: - # pylab.scatter([particle.history[nstep].xyz[0]], + # pylab.scatter([particle.history[nstep].xyz[0]], # [particle.history[nstep].xyz[1]], # marker="o", s=1.0, edgecolor="None") @@ -83,7 +83,7 @@ def main(fileList): import sys # import cProfile - if len(sys.argv) == 1: + if len(sys.argv) == 1: print main.__doc__ sys.exit() diff --git a/Tools/Py_util/amrex_particles_to_vtp/amrex_binary_particles_to_vtp.py b/Tools/Py_util/amrex_particles_to_vtp/amrex_binary_particles_to_vtp.py index bd1c7a88095..ac862bef577 100644 --- a/Tools/Py_util/amrex_particles_to_vtp/amrex_binary_particles_to_vtp.py +++ b/Tools/Py_util/amrex_particles_to_vtp/amrex_binary_particles_to_vtp.py @@ -5,8 +5,8 @@ class AMReXParticleHeader(object): ''' - This class is designed to parse and store the information - contained in an AMReX particle header file. + This class is designed to parse and store the information + contained in an AMReX particle header file. Usage: @@ -67,11 +67,11 @@ def __init__(self, header_filename): entry = [int(val) for val in f.readline().strip().split()] self.grids[level_num].append(tuple(entry)) - + def read_amrex_binary_particle_file(fn, ptype="particle0"): ''' - This function returns the particle data stored in a particular + This function returns the particle data stored in a particular plot file and particle type. It returns two numpy arrays, the first containing the particle integer data, and the second the particle real data. For example, if a dataset has 3000 particles, @@ -80,22 +80,22 @@ def read_amrex_binary_particle_file(fn, ptype="particle0"): with the shape (3000, 5). Usage: - + idata, rdata = read_particle_data("plt00000", "particle0") ''' base_fn = fn + "/" + ptype header = AMReXParticleHeader(base_fn + "/Header") - - idtype = "(%d,)i4" % header.num_int + + idtype = "(%d,)i4" % header.num_int if header.real_type == np.float64: fdtype = "(%d,)f8" % header.num_real elif header.real_type == np.float32: fdtype = "(%d,)f4" % header.num_real - + idata = np.empty((header.num_particles, header.num_int )) rdata = np.empty((header.num_particles, header.num_real)) - + ip = 0 for lvl, level_grids in enumerate(header.grids): for (which, count, where) in level_grids: @@ -109,7 +109,7 @@ def read_amrex_binary_particle_file(fn, ptype="particle0"): idata[ip:ip+count] = ints floats = np.fromfile(f, dtype = fdtype, count=count) - rdata[ip:ip+count] = floats + rdata[ip:ip+count] = floats ip += count return idata, rdata diff --git a/Tools/Py_util/amrex_particles_to_vtp/write_pview_file.py b/Tools/Py_util/amrex_particles_to_vtp/write_pview_file.py index 6cb489195a4..9c26b8785c5 100644 --- a/Tools/Py_util/amrex_particles_to_vtp/write_pview_file.py +++ b/Tools/Py_util/amrex_particles_to_vtp/write_pview_file.py @@ -125,7 +125,7 @@ def write_paraview_file_unst_trimesh(fname,pts,conn,ccdata,ncdata): outfile.close() def write_paraview_file_cartmesh(fname,dx,prob_lo,N,ncdata,ccdata): - + zero=0 one=1 outfile=open(fname,'w') @@ -166,11 +166,11 @@ def write_paraview_file_cartmesh(fname,dx,prob_lo,N,ncdata,ccdata): for i in range(N[1]): outfile.write("%e\t"%(prob_lo[1]+i*dx[1])) outfile.write("\n\n") - + outfile.write("\n") outfile.write("%e\t"%(0.0)) outfile.write("\n\n") - + outfile.write("\n") outfile.write("\n") outfile.write("\n") diff --git a/Tools/Py_util/plotsinglevar.py b/Tools/Py_util/plotsinglevar.py index ba99a401d02..616c516c805 100755 --- a/Tools/Py_util/plotsinglevar.py +++ b/Tools/Py_util/plotsinglevar.py @@ -6,8 +6,8 @@ from __future__ import print_function -import matplotlib -matplotlib.use('agg') +import matplotlib +matplotlib.use('agg') import numpy import pylab @@ -31,8 +31,8 @@ #============================================================================== # do_plot #============================================================================== -def do_plot(plotfile, component, component2, outFile, log, - minval, maxval, minval2, maxval2, eps, dpi, origin, annotation, +def do_plot(plotfile, component, component2, outFile, log, + minval, maxval, minval2, maxval2, eps, dpi, origin, annotation, xmin_pass, ymin_pass, zmin_pass, xmax_pass, ymax_pass, zmax_pass): pylab.rc("font", size=9) @@ -143,12 +143,12 @@ def do_plot(plotfile, component, component2, outFile, log, else: data = numpy.log10(data) - if (not component2 == ""): + if (not component2 == ""): if (numpy.min(data2) < 0.0): data2 = numpy.log10(numpy.abs(data2)) else: data2 = numpy.log10(data2) - + if (not minval == None): minval = math.log10(minval) if (not maxval == None): maxval = math.log10(maxval) @@ -165,9 +165,9 @@ def do_plot(plotfile, component, component2, outFile, log, else: ax = pylab.subplot(1,1,1) - - divider = mpl_toolkits.axes_grid1.make_axes_locatable(ax) + + divider = mpl_toolkits.axes_grid1.make_axes_locatable(ax) im=pylab.imshow(data[iy0:iy,ix0:ix],origin='lower', extent=extent, vmin=minval, vmax=maxval) @@ -200,25 +200,25 @@ def do_plot(plotfile, component, component2, outFile, log, # y-axis. cl = pylab.getp(cb.ax, 'ymajorticklabels') pylab.setp(cl, fontsize=10) - + cb.ax.yaxis.offsetText.set_fontsize("small") # do a fixed offset in pixels from the (xmin,ymin) data point - trans=matplotlib.transforms.offset_copy(ax.transData, x=0, y=-0.5, + trans=matplotlib.transforms.offset_copy(ax.transData, x=0, y=-0.5, fig=fig1, units='inches') - pylab.text(xmin, ymin, "time = %7.3g s" % (time), - verticalalignment="bottom", transform = trans, + pylab.text(xmin, ymin, "time = %7.3g s" % (time), + verticalalignment="bottom", transform = trans, clip_on=False, fontsize=10) if (not annotation == ""): trans=matplotlib.transforms.offset_copy(ax.transData, x=0, y=-0.65, fig=fig1, units='inches') - pylab.text(xmin, ymin, "%s" % (annotation), - verticalalignment="bottom", transform = trans, - clip_on=False, fontsize=10) + pylab.text(xmin, ymin, "%s" % (annotation), + verticalalignment="bottom", transform = trans, + clip_on=False, fontsize=10) #---------------------------------------------------------------------- @@ -229,7 +229,7 @@ def do_plot(plotfile, component, component2, outFile, log, divider = mpl_toolkits.axes_grid1.make_axes_locatable(ax) - im = pylab.imshow(data2[iy0:iy,ix0:ix], origin='lower', extent=extent, + im = pylab.imshow(data2[iy0:iy,ix0:ix], origin='lower', extent=extent, vmin=minval2, vmax=maxval2) pylab.title(component2) @@ -259,7 +259,7 @@ def do_plot(plotfile, component, component2, outFile, log, # offsetText is the 10^N that appears at the top of the y-axis. cl = pylab.getp(cb.ax, 'ymajorticklabels') pylab.setp(cl, fontsize=10) - + cb.ax.yaxis.offsetText.set_fontsize("small") #ax_cb.yaxis.tick_right() @@ -280,7 +280,7 @@ def do_plot(plotfile, component, component2, outFile, log, # for the height, we will assume that the colorbar at the # bottom gets 0.15, and that we go until 0.95, leaving 0.8 of # height for the plots. - + pos1 = [0.05, 0.15, 0.3, 0.8] pos2 = [0.35, 0.15, 0.3, 0.8] pos3 = [0.65, 0.15, 0.3, 0.8] @@ -294,7 +294,7 @@ def do_plot(plotfile, component, component2, outFile, log, indir = 3 (data_xy, err) = \ - fsnapshot.fplotfile_get_data_3d(plotfile, component, indir, + fsnapshot.fplotfile_get_data_3d(plotfile, component, indir, origin, data_xy) if (not err == 0): sys.exit(2) @@ -311,7 +311,7 @@ def do_plot(plotfile, component, component2, outFile, log, # x-z data_xz = numpy.zeros( (nx, nz), dtype=numpy.float64) (data_xz, err) = \ - fsnapshot.fplotfile_get_data_3d(plotfile, component, 2, + fsnapshot.fplotfile_get_data_3d(plotfile, component, 2, origin, data_xz) if (not err == 0): sys.exit(2) @@ -323,12 +323,12 @@ def do_plot(plotfile, component, component2, outFile, log, data_xz = numpy.log10(numpy.abs(data_xz)) else: data_xz = numpy.log10(data_xz) - + # y-z data_yz = numpy.zeros( (ny, nz), dtype=numpy.float64) (data_yz, err) = \ - fsnapshot.fplotfile_get_data_3d(plotfile, component, 1, + fsnapshot.fplotfile_get_data_3d(plotfile, component, 1, origin, data_yz) if (not err == 0): sys.exit(2) @@ -340,11 +340,11 @@ def do_plot(plotfile, component, component2, outFile, log, data_yz = numpy.log10(numpy.abs(data_yz)) else: data_yz = numpy.log10(data_yz) - - - if (not minval == None): + + + if (not minval == None): if (log): minval = math.log10(minval) else: @@ -353,14 +353,14 @@ def do_plot(plotfile, component, component2, outFile, log, minval = min(minval,numpy.min(data_yz)) - if (not maxval == None): + if (not maxval == None): if (log): maxval = math.log10(maxval) else: maxval = numpy.max(data_xy) maxval = max(maxval,numpy.max(data_xz)) maxval = max(maxval,numpy.max(data_yz)) - + @@ -394,7 +394,7 @@ def do_plot(plotfile, component, component2, outFile, log, pylab.subplots_adjust(wspace=0.4) #fig.add_axes(pos1) - im=pylab.imshow(data_xy[iy0:iy,ix0:ix],origin='lower', extent=extent, + im=pylab.imshow(data_xy[iy0:iy,ix0:ix],origin='lower', extent=extent, vmin=minval, vmax=maxval)#, axes=pos1) pylab.xlabel("x") @@ -415,11 +415,11 @@ def do_plot(plotfile, component, component2, outFile, log, # do a fixed offset in pixels from the (xmin,ymin) data point fig1 = ax.get_figure() - trans=matplotlib.transforms.offset_copy(ax.transData, x=0, y=-0.5, + trans=matplotlib.transforms.offset_copy(ax.transData, x=0, y=-0.5, fig=fig1, units='inches') - # pylab.text(xmin_pass, ymin_pass, "time = %7.3g s" % (time), - # verticalalignment="bottom", transform = trans, + # pylab.text(xmin_pass, ymin_pass, "time = %7.3g s" % (time), + # verticalalignment="bottom", transform = trans, # clip_on=False, fontsize=10) @@ -452,7 +452,7 @@ def do_plot(plotfile, component, component2, outFile, log, ax = pylab.subplot(1,3,2) #fig.add_axes(pos2) - im=pylab.imshow(data_xz[iz0:iz,ix0:ix],origin='lower', extent=extent, + im=pylab.imshow(data_xz[iz0:iz,ix0:ix],origin='lower', extent=extent, vmin=minval, vmax=maxval) #, axes=pos2) pylab.xlabel("x") @@ -501,7 +501,7 @@ def do_plot(plotfile, component, component2, outFile, log, ax = pylab.subplot(1,3,3) #fig.add_axes(pos3) - im=pylab.imshow(data_yz[iz0:iz,iy0:iy],origin='lower', extent=extent, + im=pylab.imshow(data_yz[iz0:iz,iy0:iy],origin='lower', extent=extent, vmin=minval, vmax=maxval) #, axes=pos3) pylab.xlabel("y") @@ -525,7 +525,7 @@ def do_plot(plotfile, component, component2, outFile, log, pylab.subplots_adjust(bottom=0.1, left=0.05, right=0.95) formatter = matplotlib.ticker.ScalarFormatter(useMathText=True) - + cax = pylab.axes([0.05, 0.06, 0.9, 0.04]) pylab.colorbar(orientation="horizontal", cax=cax, format=formatter) @@ -595,7 +595,7 @@ def usage(): Note: this script requires the fsnapshot.so library, compiled with f2py using the GNUmakefile in AmrPostprocessing/python. - """ + """ print(usageStr) @@ -622,13 +622,13 @@ def usage(): ymax = None zmax = None - try: opts, next = getopt.getopt(sys.argv[1:], "o:m:M:n:N:x:y:z:X:Y:Z:", + try: opts, next = getopt.getopt(sys.argv[1:], "o:m:M:n:N:x:y:z:X:Y:Z:", ["log","eps","dpi=","origin","annotate="]) except getopt.GetoptError: print("invalid calling sequence") usage() - sys.exit(2) - + sys.exit(2) + for o, a in opts: @@ -663,38 +663,38 @@ def usage(): try: xmin = float(a) except ValueError: print("invalid value for -x") - sys.exit(2) + sys.exit(2) if o == "-y": try: ymin = float(a) except ValueError: print("invalid value for -y") - sys.exit(2) + sys.exit(2) if o == "-z": try: zmin = float(a) except ValueError: print("invalid value for -z") - sys.exit(2) + sys.exit(2) if o == "-X": try: xmax = float(a) except ValueError: print("invalid value for -X") - sys.exit(2) + sys.exit(2) if o == "-Y": try: ymax = float(a) except ValueError: print("invalid value for -Y") - sys.exit(2) + sys.exit(2) if o == "-Z": try: zmax = float(a) except ValueError: print("invalid value for -Z") - sys.exit(2) - + sys.exit(2) + if o == "--log": log = 1 @@ -725,12 +725,12 @@ def usage(): except IndexError: print("ERROR: no component specified") usage() - sys.exit(2) + sys.exit(2) try: component2 = next[2] except IndexError: component2 = "" - do_plot(plotfile, component, component2, outFile, + do_plot(plotfile, component, component2, outFile, log, minvar, maxvar, minvar2, maxvar2, eps, dpi, origin, annotation, xmin, ymin, zmin, xmax, ymax, zmax) diff --git a/Tools/Release/ppCleanup.py b/Tools/Release/ppCleanup.py index 3ec1603259b..00d3ffde7ac 100755 --- a/Tools/Release/ppCleanup.py +++ b/Tools/Release/ppCleanup.py @@ -13,7 +13,7 @@ def ppCleanup(cwfile, infile, oufile): except IOError: print "ERROR: ", cwfile, " doesn't appear to exist." sys.exit(2) - + cleanWords = [] for line in f.readlines(): word = line[:-1].strip() @@ -98,14 +98,14 @@ def ppCleanup(cwfile, infile, oufile): if status: # "ifdef" block is CLEANUP, so "else" block is KEEP cleanupstack.append(False) - else: # There are two possibilies: + else: # There are two possibilies: if dw in cleanWords: cleanupstack.append(True) else: - # we keep this "ELSE" line and the following block + # we keep this "ELSE" line and the following block # because the def word is in in our list for cleanup cleanupstack.append(False) - fou.write(line) + fou.write(line) defstack.append(dw) # do we need this???? @@ -134,13 +134,13 @@ def ppCleanup(cwfile, infile, oufile): else: if dw in cleanWords: cleanupstack.append(False) - else: + else: # Its word is not on out list cleanupstack.append(False) fou.write(line) else: - + if not cleanupstack[-1]: fou.write(line) @@ -160,7 +160,7 @@ def ppCleanup(cwfile, infile, oufile): try: opts, args = getopt.getopt(argv[1:], "c:o:", []) - + except getopt.GetoptError: print "invalid calling sequence" print usage diff --git a/Tools/Release/ppCleanupDir.py b/Tools/Release/ppCleanupDir.py index ded96506d9e..c43b8700d0a 100755 --- a/Tools/Release/ppCleanupDir.py +++ b/Tools/Release/ppCleanupDir.py @@ -7,7 +7,7 @@ import readline import subprocess import ppCleanup as ppc - + def ppCleanupDir(argv): usage = """ diff --git a/Tools/Release/release.py b/Tools/Release/release.py index 23627143c8c..ad633fc53c5 100755 --- a/Tools/Release/release.py +++ b/Tools/Release/release.py @@ -7,7 +7,7 @@ import readline import subprocess import ppCleanup as ppc - + def release(argv): usage = """ @@ -73,7 +73,7 @@ def release(argv): os.rename(f,ftmp) # systemCall("~/mygitrepo/BoxLib/Tools/ppCleanup/ppCleanup.py -c "+ # "~/mygitrepo/BoxLib/Tools/ppCleanup/cleanWords.txt "+ -# " -o "+f+" "+ftmp) +# " -o "+f+" "+ftmp) ppc.ppCleanup(cwfile, ftmp, f) os.remove(ftmp) if '.git' in dirs: @@ -107,7 +107,7 @@ def release(argv): break else: print "What did you type?", uin+"?" - + if uin == 'y': print "\nOK. The script will do it for you." doGitPush(my_public_git) @@ -145,7 +145,7 @@ def doGitCommit(d): stdout, stderr = p.communicate() p.stdout.close() p.stderr.close() - p.stdin.close() + p.stdin.close() ulist = [] @@ -163,7 +163,7 @@ def doGitCommit(d): stdout, stderr = p.communicate() p.stdout.close() p.stderr.close() - p.stdin.close() + p.stdin.close() print "\nCommiting changes" prog = ["git", "commit", "-a"] @@ -173,7 +173,7 @@ def doGitCommit(d): stdout, stderr = p.communicate() p.stdout.close() p.stderr.close() - p.stdin.close() + p.stdin.close() os.chdir(d0) return @@ -240,9 +240,9 @@ def doGitPushTag(d): p.stdin.close() os.chdir(d0) return - -def systemCall(string): + +def systemCall(string): status = os.system('bash -c "' + string + '"') return status diff --git a/Tools/typechecker/typechecker.py b/Tools/typechecker/typechecker.py index 50d9e454cd9..2086b22d1b5 100755 --- a/Tools/typechecker/typechecker.py +++ b/Tools/typechecker/typechecker.py @@ -94,7 +94,7 @@ def check_doit(node, workdir, func_src, fout, aux_info): error_msg.append(" C return type {0} does NOT match Fortran {1}.\n" .format(c_ret_type,f_ret_type)) fout.write(error_msg[-1]) - + if len(c_arg_type) == len(f_arg_type): fout.write(" number of arguments {0} matches {1}.\n" .format(len(c_arg_type),len(f_arg_type))) @@ -176,7 +176,7 @@ def findFortranSources(funcs, func_src, workdir): else: func_src[pname] = fname - + def getFortranArg(funcname, fortranfile): """given a function name and a gortran tree original file, return a tuple of function return type and list of arguments. For @@ -229,7 +229,7 @@ def getFortranArg(funcname, fortranfile): ws_proced_block = 0 if numblocks == 2: break - f.close() + f.close() if not this_func: print(fortranfile, "doesn't contain", function) @@ -237,7 +237,7 @@ def getFortranArg(funcname, fortranfile): print(funcname, "this_func...") for line in this_func: print(line,) - + return_type = '' arguments_type = [] found = False @@ -258,11 +258,11 @@ def parse_type_spec(line): for line in this_func: num_white_spaces = len(line) - len(line.lstrip()) # searching for blocks like - # symtree: 'fort_fab_copy'|| symbol: 'fort_fab_copy' + # symtree: 'fort_fab_copy'|| symbol: 'fort_fab_copy' # type spec : (UNKNOWN 0) # attributes: (PROCEDURE MODULE-PROC BIND(C) SUBROUTINE) # Formal arglist: lo hi dst dlo dhi src slo shi sblo ncomp - if "symtree: '"+funcname+"'" in line: # e.g. + if "symtree: '"+funcname+"'" in line: # e.g. found = True ws = num_white_spaces elif found: @@ -281,7 +281,7 @@ def parse_type_spec(line): argtype = '' for line in this_func: # search for blocks like - # symtree: 'dhi' || symbol: 'dhi' + # symtree: 'dhi' || symbol: 'dhi' # type spec : (INTEGER 4) # attributes: (VARIABLE DIMENSION DUMMY(IN)) # Array spec:(1 [0] AS_EXPLICIT 1 3 ) @@ -310,10 +310,10 @@ def parse_type_spec(line): print(arglist) print(func_args) print ("-------") - + for a in arglist: arguments_type.append(func_args[a]) - + return return_type, arguments_type if __name__ == "__main__": diff --git a/Tutorials/Amr/Advection_AmrCore/Exec/Prob.H b/Tutorials/Amr/Advection_AmrCore/Exec/Prob.H index a7c172468ba..81a7e65ad21 100644 --- a/Tutorials/Amr/Advection_AmrCore/Exec/Prob.H +++ b/Tutorials/Amr/Advection_AmrCore/Exec/Prob.H @@ -15,13 +15,13 @@ initdata(amrex::Box const& bx, amrex::Array4 const& phi, using namespace amrex; const auto lo = lbound(bx); - const auto hi = ubound(bx); + const auto hi = ubound(bx); for (int k = lo.z; k <= hi.z; ++k) { for (int j = lo.y; j <= hi.y; ++j) { Real y = prob_lo[1] + (0.5+j) * dx[1]; for (int i = lo.x; i <= hi.x; ++i) { - Real x = prob_lo[0] + (0.5+i) * dx[0]; + Real x = prob_lo[0] + (0.5+i) * dx[0]; Real r2 = (std::pow(x-0.5, 2) + std::pow((y-0.75),2)) / 0.01; phi(i,j,k) = 1.0 + std::exp(-r2); } diff --git a/Tutorials/Amr/Advection_AmrCore/Exec/paraview_amr101.py b/Tutorials/Amr/Advection_AmrCore/Exec/paraview_amr101.py index f77699bea34..7b98606d888 100644 --- a/Tutorials/Amr/Advection_AmrCore/Exec/paraview_amr101.py +++ b/Tutorials/Amr/Advection_AmrCore/Exec/paraview_amr101.py @@ -1,6 +1,6 @@ # based on traces generated using paraview version 5.8.0 # -# To ensure correct image size when batch processing, please search +# To ensure correct image size when batch processing, please search # for and uncomment the line `# renderView*.ViewSize = [*,*]` #### import the simple module from the paraview diff --git a/Tutorials/Amr/Advection_AmrCore/README.md b/Tutorials/Amr/Advection_AmrCore/README.md index 265740e0364..39111147b2d 100644 --- a/Tutorials/Amr/Advection_AmrCore/README.md +++ b/Tutorials/Amr/Advection_AmrCore/README.md @@ -4,18 +4,18 @@ ### What Features Are We Using -* Mesh data +* Mesh data * Dynamic AMR with and without subcycling ### The Problem -Consider a drop of dye (we'll define $$\phi$$ to be the concentration of dye) -in a thin incompressible fluid that is spinning -clock-wise then counter-clockwise with a prescribed motion. We consider the dye to be a +Consider a drop of dye (we'll define $$\phi$$ to be the concentration of dye) +in a thin incompressible fluid that is spinning +clock-wise then counter-clockwise with a prescribed motion. We consider the dye to be a passive tracer that is advected by the fluid velocity. The fluid is thin enough that we can model this as two-dimensional motion; here we have the option of solving in a 2D or 3D computational domain. -In other words, we want to solve for $$\phi(x,y,t)$$ by evolving +In other words, we want to solve for $$\phi(x,y,t)$$ by evolving $$\frac{\partial \phi}{\partial t} + \nabla \cdot (\bf{u^{spec}} \phi) = 0$$ @@ -48,7 +48,7 @@ in the code looks like } ``` -In this routine we use the macro AMREX_D_TERM so that we can write dimension-independent code; +In this routine we use the macro AMREX_D_TERM so that we can write dimension-independent code; in 3D this returns the flux differences in all three directions, but in 2D it does not include the z-fluxes. @@ -125,14 +125,14 @@ To build in 3d, type make DIM = 3 ``` -In this directory you'll see +In this directory you'll see ``` inputs -- an inputs file for both 2D and 3D ``` -To run in serial, +To run in serial, ``` ./main3d.gnu.MPI.ex inputs @@ -145,7 +145,7 @@ mpiexec -n 4 ./main3d.gnu.MPI.ex inputs ``` The following parameters can be set at run-time -- these are currently set in the inputs -file but you can also set them on the command line. +file but you can also set them on the command line. ``` stop_time = 2.0 # the final time (if we have not exceeded number of steps) @@ -163,20 +163,20 @@ adv.phierr = 1.01 1.1 1.5 # regridding criteria at each level ``` -The base grid here is a square of 64 x 64 x 8 cells, made up of 16 subgrids each of size 16x16x8 cells. +The base grid here is a square of 64 x 64 x 8 cells, made up of 16 subgrids each of size 16x16x8 cells. The problem is periodic in all directions. -We have hard-wired the code here to refine based on the magnitude of $$\phi$$. Here we set the +We have hard-wired the code here to refine based on the magnitude of $$\phi$$. Here we set the threshold level by level. If $$\phi > 1.01$$ then we want to refine at least once; if $$\phi > 1.1$$ we want to resolve $$\phi$$ with two levels of refinement, and if $$\phi > 1.5$$ we want even more refinement. Note that you can see the total runtime by looking at the line at the end of your run that says ``` -Total Time: +Total Time: ``` -and you can check conservation of $$\phi$$ by checking the line that prints, e.g. +and you can check conservation of $$\phi$$ by checking the line that prints, e.g. ``` Coarse STEP 8 ends. TIME = 0.007031485953 DT = 0.0008789650903 Sum(Phi) = 540755.0014 @@ -197,8 +197,8 @@ Questions to answer: a. If you set do_refluxing = 0 for the subcycling case, was phi still conserved? b. How in the algorithm is conservation enforced differently between subcycling and not? -4. How did the runtimes vary with 1 vs. 4 MPI processes? - We suggest you use a big enough problem here -- try running +4. How did the runtimes vary with 1 vs. 4 MPI processes? + We suggest you use a big enough problem here -- try running mpiexec -n 1 ./main3d.gnu.MPI.ex inputs_for_scaling @@ -262,4 +262,4 @@ You are now ready to play the movie! See the "VCR-like" controls at the top. Cl * What happens as you change the max grid size for decomposition? * What happens as you change the refinement criteria (i.e. use different values of $$\phi$$)? - (You can edit these in inputs) + (You can edit these in inputs) diff --git a/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAllLevels.cpp b/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAllLevels.cpp index 2176a7bf158..b5e48e6e409 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAllLevels.cpp +++ b/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAllLevels.cpp @@ -161,7 +161,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) velx, vely, phix_c, phiy_c, dtdy); - }); + }); // xz -------------------- Array4 phix_z = tmpfab.array(itmp); @@ -176,7 +176,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) velx, velz, phix, phiz, dtdz); - }); + }); // yx -------------------- Array4 phiy_x = tmpfab.array(itmp); @@ -191,7 +191,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) velx, vely, phix, phiy, dtdx); - }); + }); // yz -------------------- Array4 phiy_z = tmpfab.array(itmp); @@ -206,7 +206,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) vely, velz, phiy, phiz, dtdz); - }); + }); // zx -------------------- Array4 phiz_x = tmpfab.array(itmp); @@ -221,7 +221,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) velx, velz, phix, phiz, dtdx); - }); + }); // zy -------------------- Array4 phiz_y = tmpfab.array(itmp); @@ -236,10 +236,10 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) vely, velz, phiy, phiz, dtdy); - }); + }); #endif - // final edge states + // final edge states // =========================== amrex::ParallelFor(mfi.nodaltilebox(0), [=] AMREX_GPU_DEVICE (int i, int j, int k) @@ -285,14 +285,14 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) } // end lev // ======================================================= - // Average down the fluxes before using them to update phi + // Average down the fluxes before using them to update phi // ======================================================= for (int lev = finest_level; lev > 0; lev--) { average_down_faces(amrex::GetArrOfConstPtrs(fluxes[lev ]), amrex::GetArrOfPtrs (fluxes[lev-1]), refRatio(lev-1), Geom(lev-1)); - } + } for (int lev = 0; lev <= finest_level; lev++) { @@ -307,7 +307,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) Real dtdz = dt_lev/dx[2]); // =========================================== - // Compute phi_new using a conservative update + // Compute phi_new using a conservative update // =========================================== for (MFIter mfi(phi_new[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) { @@ -319,7 +319,7 @@ AmrCoreAdv::AdvancePhiAllLevels (Real time, Real dt_lev, int /*iteration*/) Array4 fluxz = fluxes[lev][2].const_array(mfi)); const Box& bx = mfi.tilebox(); - + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { diff --git a/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAtLevel.cpp b/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAtLevel.cpp index 9de01b03a83..3ddd055eda0 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAtLevel.cpp +++ b/Tutorials/Amr/Advection_AmrCore/Source/AdvancePhiAtLevel.cpp @@ -40,8 +40,8 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* #endif { FArrayBox tmpfab; - for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { + for (MFIter mfi(S_new,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { AMREX_ASSERT(S_new.nComp() == 1); // ======== GET FACE VELOCITY ========= @@ -52,7 +52,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* // ======== FLUX CALC AND UPDATE ========= - const Box& bx = mfi.tilebox(); + const Box& bx = mfi.tilebox(); const Box& gbx = amrex::grow(bx, 1); Array4 statein = Sborder.const_array(mfi); @@ -160,7 +160,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* velx, vely, phix_c, phiy_c, dtdy); - }); + }); // xz -------------------- Array4 phix_z = tmpfab.array(itmp++); @@ -174,7 +174,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* velx, velz, phix, phiz, dtdz); - }); + }); // yx -------------------- Array4 phiy_x = tmpfab.array(itmp++); @@ -188,7 +188,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* velx, vely, phix, phiy, dtdx); - }); + }); // yz -------------------- Array4 phiy_z = tmpfab.array(itmp++); @@ -202,7 +202,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* vely, velz, phiy, phiz, dtdz); - }); + }); // zx -------------------- Array4 phiz_x = tmpfab.array(itmp++); @@ -216,7 +216,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* velx, velz, phix, phiz, dtdx); - }); + }); // zy -------------------- Array4 phiz_y = tmpfab.array(itmp++); @@ -230,10 +230,10 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* vely, velz, phiy, phiz, dtdy); - }); + }); #endif - // final edge states + // final edge states // =========================== amrex::ParallelFor(amrex::surroundingNodes(bx,Direction::x), [=] AMREX_GPU_DEVICE (int i, int j, int k) @@ -282,7 +282,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* Array4 tfluxy_c = tfluxy;, Array4 tfluxz_c = tfluxz); - // Do a conservative update + // Do a conservative update amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { @@ -311,7 +311,7 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* { tfluxz(i,j,k) *= dt_lev*dx*dy; })); - + // Copy into Flux MultiFab AMREX_D_TERM(Array4 fluxx = fluxes[0].array(mfi);, Array4 fluxy = fluxes[1].array(mfi);, @@ -364,18 +364,18 @@ AmrCoreAdv::AdvancePhiAtLevel (int lev, Real time, Real dt_lev, int /*iteration* // the flux registers from the coarse or fine grid perspective // NOTE: the flux register associated with flux_reg[lev] is associated // with the lev/lev-1 interface (and has grid spacing associated with lev-1) - if (do_reflux) { - if (flux_reg[lev+1]) { - for (int i = 0; i < AMREX_SPACEDIM; ++i) { - // update the lev+1/lev flux register (index lev+1) - flux_reg[lev+1]->CrseInit(fluxes[i],i,0,0,fluxes[i].nComp(), -1.0); - } - } - if (flux_reg[lev]) { - for (int i = 0; i < AMREX_SPACEDIM; ++i) { - // update the lev/lev-1 flux register (index lev) - flux_reg[lev]->FineAdd(fluxes[i],i,0,0,fluxes[i].nComp(), 1.0); - } - } + if (do_reflux) { + if (flux_reg[lev+1]) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + // update the lev+1/lev flux register (index lev+1) + flux_reg[lev+1]->CrseInit(fluxes[i],i,0,0,fluxes[i].nComp(), -1.0); + } + } + if (flux_reg[lev]) { + for (int i = 0; i < AMREX_SPACEDIM; ++i) { + // update the lev/lev-1 flux register (index lev) + flux_reg[lev]->FineAdd(fluxes[i],i,0,0,fluxes[i].nComp(), 1.0); + } + } } } diff --git a/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.H b/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.H index a860f0731b2..4ead5e878ff 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.H +++ b/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.H @@ -32,17 +32,17 @@ public: // initializes multilevel data void InitData (); - // Make a new level using provided BoxArray and DistributionMapping and + // Make a new level using provided BoxArray and DistributionMapping and // fill with interpolated coarse level data. // overrides the pure virtual function in AmrCore virtual void MakeNewLevelFromCoarse (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) override; + const amrex::DistributionMapping& dm) override; - // Remake an existing level using provided BoxArray and DistributionMapping and + // Remake an existing level using provided BoxArray and DistributionMapping and // fill with existing fine and coarse data. // overrides the pure virtual function in AmrCore virtual void RemakeLevel (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) override; + const amrex::DistributionMapping& dm) override; // Delete level data // overrides the pure virtual function in AmrCore @@ -52,7 +52,7 @@ public: // Only used during initialization. // overrides the pure virtual function in AmrCore virtual void MakeNewLevelFromScratch (int lev, amrex::Real time, const amrex::BoxArray& ba, - const amrex::DistributionMapping& dm) override; + const amrex::DistributionMapping& dm) override; // tag all cells for refinement // overrides the pure virtual function in AmrCore @@ -132,7 +132,7 @@ private: amrex::Vector nsubsteps; // how many substeps on each level? // keep track of old time, new time, and time step at each level - amrex::Vector t_new; + amrex::Vector t_new; amrex::Vector t_old; amrex::Vector dt; @@ -149,13 +149,13 @@ private: // this will be sized "nlevs_max+1" // NOTE: the flux register associated with flux_reg[lev] is associated // with the lev/lev-1 interface (and has grid spacing associated with lev-1) - // therefore flux_reg[0] and flux_reg[nlevs_max] are never actually + // therefore flux_reg[0] and flux_reg[nlevs_max] are never actually // used in the reflux operation - amrex::Vector > flux_reg; + amrex::Vector > flux_reg; // Velocity on all faces at all levels amrex::Vector< amrex::Array > facevel; - + //////////////// // runtime parameters diff --git a/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.cpp b/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.cpp index b4256e543ad..c6d1cb51cdc 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.cpp +++ b/Tutorials/Amr/Advection_AmrCore/Source/AmrCoreAdv.cpp @@ -176,16 +176,16 @@ AmrCoreAdv::InitData () } } -// Make a new level using provided BoxArray and DistributionMapping and +// Make a new level using provided BoxArray and DistributionMapping and // fill with interpolated coarse level data. // overrides the pure virtual function in AmrCore void AmrCoreAdv::MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba, - const DistributionMapping& dm) + const DistributionMapping& dm) { const int ncomp = phi_new[lev-1].nComp(); const int nghost = phi_new[lev-1].nGrow(); - + phi_new[lev].define(ba, dm, ncomp, nghost); phi_old[lev].define(ba, dm, ncomp, nghost); @@ -195,22 +195,22 @@ AmrCoreAdv::MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba, // This clears the old MultiFab and allocates the new one for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); + facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); } if (lev > 0 && do_reflux) { - flux_reg[lev].reset(new FluxRegister(ba, dm, refRatio(lev-1), lev, ncomp)); + flux_reg[lev].reset(new FluxRegister(ba, dm, refRatio(lev-1), lev, ncomp)); } FillCoarsePatch(lev, time, phi_new[lev], 0, ncomp); } -// Remake an existing level using provided BoxArray and DistributionMapping and +// Remake an existing level using provided BoxArray and DistributionMapping and // fill with existing fine and coarse data. // overrides the pure virtual function in AmrCore void AmrCoreAdv::RemakeLevel (int lev, Real time, const BoxArray& ba, - const DistributionMapping& dm) + const DistributionMapping& dm) { const int ncomp = phi_new[lev].nComp(); const int nghost = phi_new[lev].nGrow(); @@ -229,12 +229,12 @@ AmrCoreAdv::RemakeLevel (int lev, Real time, const BoxArray& ba, // This clears the old MultiFab and allocates the new one for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); + facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); } if (lev > 0 && do_reflux) { - flux_reg[lev].reset(new FluxRegister(ba, dm, refRatio(lev-1), lev, ncomp)); - } + flux_reg[lev].reset(new FluxRegister(ba, dm, refRatio(lev-1), lev, ncomp)); + } } // Delete level data @@ -251,7 +251,7 @@ AmrCoreAdv::ClearLevel (int lev) // Only used during initialization. // overrides the pure virtual function in AmrCore void AmrCoreAdv::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba, - const DistributionMapping& dm) + const DistributionMapping& dm) { const int ncomp = 1; const int nghost = 0; @@ -265,11 +265,11 @@ void AmrCoreAdv::MakeNewLevelFromScratch (int lev, Real time, const BoxArray& ba // This clears the old MultiFab and allocates the new one for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); + facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); } if (lev > 0 && do_reflux) { - flux_reg[lev].reset(new FluxRegister(ba, dm, refRatio(lev-1), lev, ncomp)); + flux_reg[lev].reset(new FluxRegister(ba, dm, refRatio(lev-1), lev, ncomp)); } MultiFab& state = phi_new[lev]; @@ -304,17 +304,17 @@ AmrCoreAdv::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) // only do this during the first call to ErrorEst if (first) { - first = false; + first = false; // read in an array of "phierr", which is the tagging threshold // in this example, we tag values of "phi" which are greater than phierr // for that particular level // in subroutine state_error, you could use more elaborate tagging, such // as more advanced logical expressions, or gradients, etc. - ParmParse pp("adv"); - int n = pp.countval("phierr"); - if (n > 0) { - pp.getarr("phierr", phierr, 0, n); - } + ParmParse pp("adv"); + int n = pp.countval("phierr"); + if (n > 0) { + pp.getarr("phierr", phierr, 0, n); + } } if (lev >= phierr.size()) return; @@ -328,20 +328,20 @@ AmrCoreAdv::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) #pragma omp parallel if(Gpu::notInLaunchRegion()) #endif { - - for (MFIter mfi(state,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); + + for (MFIter mfi(state,TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); const auto statefab = state.array(mfi); const auto tagfab = tags.array(mfi); Real phierror = phierr[lev]; - + amrex::ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { state_error(i, j, k, tagfab, statefab, phierror, tagval); }); - } + } } } @@ -350,26 +350,26 @@ void AmrCoreAdv::ReadParameters () { { - ParmParse pp; // Traditionally, max_step and stop_time do not have prefix. - pp.query("max_step", max_step); - pp.query("stop_time", stop_time); + ParmParse pp; // Traditionally, max_step and stop_time do not have prefix. + pp.query("max_step", max_step); + pp.query("stop_time", stop_time); } { - ParmParse pp("amr"); // Traditionally, these have prefix, amr. + ParmParse pp("amr"); // Traditionally, these have prefix, amr. - pp.query("regrid_int", regrid_int); - pp.query("plot_file", plot_file); - pp.query("plot_int", plot_int); - pp.query("chk_file", chk_file); - pp.query("chk_int", chk_int); + pp.query("regrid_int", regrid_int); + pp.query("plot_file", plot_file); + pp.query("plot_int", plot_int); + pp.query("chk_file", chk_file); + pp.query("chk_int", chk_int); pp.query("restart",restart_chkfile); } { - ParmParse pp("adv"); - - pp.query("cfl", cfl); + ParmParse pp("adv"); + + pp.query("cfl", cfl); pp.query("do_reflux", do_reflux); pp.query("do_subcycle", do_subcycle); } @@ -381,7 +381,7 @@ AmrCoreAdv::AverageDown () { for (int lev = finest_level-1; lev >= 0; --lev) { - amrex::average_down(phi_new[lev+1], phi_new[lev], + amrex::average_down(phi_new[lev+1], phi_new[lev], geom[lev+1], geom[lev], 0, phi_new[lev].nComp(), refRatio(lev)); } @@ -403,33 +403,33 @@ AmrCoreAdv::FillPatch (int lev, Real time, MultiFab& mf, int icomp, int ncomp) { if (lev == 0) { - Vector smf; - Vector stime; - GetData(0, time, smf, stime); + Vector smf; + Vector stime; + GetData(0, time, smf, stime); if(Gpu::inLaunchRegion()) { GpuBndryFuncFab gpu_bndry_func(AmrCoreFill{}); PhysBCFunct > physbc(geom[lev],bcs,gpu_bndry_func); - amrex::FillPatchSingleLevel(mf, time, smf, stime, 0, icomp, ncomp, + amrex::FillPatchSingleLevel(mf, time, smf, stime, 0, icomp, ncomp, geom[lev], physbc, 0); } else { CpuBndryFuncFab bndry_func(nullptr); // Without EXT_DIR, we can pass a nullptr. PhysBCFunct physbc(geom[lev],bcs,bndry_func); - amrex::FillPatchSingleLevel(mf, time, smf, stime, 0, icomp, ncomp, + amrex::FillPatchSingleLevel(mf, time, smf, stime, 0, icomp, ncomp, geom[lev], physbc, 0); } } else { - Vector cmf, fmf; - Vector ctime, ftime; - GetData(lev-1, time, cmf, ctime); - GetData(lev , time, fmf, ftime); + Vector cmf, fmf; + Vector ctime, ftime; + GetData(lev-1, time, cmf, ctime); + GetData(lev , time, fmf, ftime); - Interpolater* mapper = &cell_cons_interp; + Interpolater* mapper = &cell_cons_interp; if(Gpu::inLaunchRegion()) { @@ -467,9 +467,9 @@ AmrCoreAdv::FillCoarsePatch (int lev, Real time, MultiFab& mf, int icomp, int nc Vector ctime; GetData(lev-1, time, cmf, ctime); Interpolater* mapper = &cell_cons_interp; - + if (cmf.size() != 1) { - amrex::Abort("FillCoarsePatch: how did this happen?"); + amrex::Abort("FillCoarsePatch: how did this happen?"); } if(Gpu::inLaunchRegion()) @@ -505,20 +505,20 @@ AmrCoreAdv::GetData (int lev, Real time, Vector& data, Vector& if (time > t_new[lev] - teps && time < t_new[lev] + teps) { - data.push_back(&phi_new[lev]); - datatime.push_back(t_new[lev]); + data.push_back(&phi_new[lev]); + datatime.push_back(t_new[lev]); } else if (time > t_old[lev] - teps && time < t_old[lev] + teps) { - data.push_back(&phi_old[lev]); - datatime.push_back(t_old[lev]); + data.push_back(&phi_old[lev]); + datatime.push_back(t_old[lev]); } else { - data.push_back(&phi_old[lev]); - data.push_back(&phi_new[lev]); - datatime.push_back(t_old[lev]); - datatime.push_back(t_new[lev]); + data.push_back(&phi_old[lev]); + data.push_back(&phi_new[lev]); + datatime.push_back(t_old[lev]); + datatime.push_back(t_new[lev]); } } @@ -536,15 +536,15 @@ AmrCoreAdv::timeStepWithSubcycling (int lev, Real time, int iteration) static Vector last_regrid_step(max_level+1, 0); // regrid changes level "lev+1" so we don't regrid on max_level - // also make sure we don't regrid fine levels again if + // also make sure we don't regrid fine levels again if // it was taken care of during a coarser regrid - if (lev < max_level && istep[lev] > last_regrid_step[lev]) + if (lev < max_level && istep[lev] > last_regrid_step[lev]) { if (istep[lev] % regrid_int == 0) { // regrid could add newly refine levels (if finest_level < max_level) // so we save the previous finest level index - int old_finest = finest_level; + int old_finest = finest_level; regrid(lev, time); // mark that we have regridded this level already @@ -562,7 +562,7 @@ AmrCoreAdv::timeStepWithSubcycling (int lev, Real time, int iteration) if (Verbose()) { amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] "; - amrex::Print() << "ADVANCE with time = " << t_new[lev] + amrex::Print() << "ADVANCE with time = " << t_new[lev] << " dt = " << dt[lev] << std::endl; } @@ -571,7 +571,7 @@ AmrCoreAdv::timeStepWithSubcycling (int lev, Real time, int iteration) t_old[lev] = t_new[lev]; t_new[lev] += dt[lev]; - Real t_nph = t_old[lev] + 0.5*dt[lev]; + Real t_nph = t_old[lev] + 0.5*dt[lev]; DefineVelocityAtLevel(lev, t_nph); AdvancePhiAtLevel(lev, time, dt[lev], iteration, nsubsteps[lev]); @@ -600,7 +600,7 @@ AmrCoreAdv::timeStepWithSubcycling (int lev, Real time, int iteration) AverageDownTo(lev); // average lev+1 down to lev } - + } // Advance all the levels with the same dt @@ -619,7 +619,7 @@ AmrCoreAdv::timeStepNoSubcycling (Real time, int iteration) for (int lev = 0; lev <= finest_level; lev++) { amrex::Print() << "[Level " << lev << " step " << istep[lev]+1 << "] "; - amrex::Print() << "ADVANCE with time = " << t_new[lev] + amrex::Print() << "ADVANCE with time = " << t_new[lev] << " dt = " << dt[0] << std::endl; } } @@ -721,7 +721,7 @@ AmrCoreAdv::PlotFileMF () const { Vector r; for (int i = 0; i <= finest_level; ++i) { - r.push_back(&phi_new[i]); + r.push_back(&phi_new[i]); } return r; } @@ -740,11 +740,11 @@ AmrCoreAdv::WritePlotFile () const const std::string& plotfilename = PlotFileName(istep[0]); const auto& mf = PlotFileMF(); const auto& varnames = PlotFileVarNames(); - + amrex::Print() << "Writing plotfile " << plotfilename << "\n"; amrex::WriteMultiLevelPlotfile(plotfilename, finest_level+1, mf, varnames, - Geom(), t_new[0], istep, refRatio()); + Geom(), t_new[0], istep, refRatio()); } void @@ -781,7 +781,7 @@ AmrCoreAdv::WriteCheckpointFile () const std::ofstream HeaderFile; HeaderFile.rdbuf()->pubsetbuf(io_buffer.dataPtr(), io_buffer.size()); HeaderFile.open(HeaderFileName.c_str(), std::ofstream::out | - std::ofstream::trunc | + std::ofstream::trunc | std::ofstream::binary); if( ! HeaderFile.good()) { amrex::FileOpenFailed(HeaderFileName); @@ -919,7 +919,7 @@ AmrCoreAdv::ReadCheckpointFile () // build face velocity MultiFabs for (int idim = 0; idim < AMREX_SPACEDIM; idim++) { - facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); + facevel[lev][idim] = MultiFab(amrex::convert(ba,IntVect::TheDimensionVector(idim)), dm, 1, 1); } } diff --git a/Tutorials/Amr/Advection_AmrCore/Source/DefineVelocity.cpp b/Tutorials/Amr/Advection_AmrCore/Source/DefineVelocity.cpp index 606bf1891d5..7a9539d0481 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/DefineVelocity.cpp +++ b/Tutorials/Amr/Advection_AmrCore/Source/DefineVelocity.cpp @@ -53,7 +53,7 @@ AmrCoreAdv::DefineVelocityAtLevel (int lev, Real time) amrex::launch(psibox, [=] AMREX_GPU_DEVICE (const Box& tbx) { - get_face_velocity_psi(tbx, time, psi, geomdata); + get_face_velocity_psi(tbx, time, psi, geomdata); }); amrex::ParallelFor diff --git a/Tutorials/Amr/Advection_AmrCore/Source/Src_K/Adv_K.H b/Tutorials/Amr/Advection_AmrCore/Source/Src_K/Adv_K.H index b6edf3b5590..5bb8fd6a432 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/Src_K/Adv_K.H +++ b/Tutorials/Amr/Advection_AmrCore/Source/Src_K/Adv_K.H @@ -6,7 +6,7 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE -void conservative(int i, int j, int k, +void conservative(int i, int j, int k, amrex::Array4 const& phi_out, amrex::Array4 const& phi_in, AMREX_D_DECL(amrex::Array4 const& flxx, @@ -14,7 +14,7 @@ void conservative(int i, int j, int k, amrex::Array4 const& flxz), AMREX_D_DECL(amrex::Real dtdx, amrex::Real dtdy, amrex::Real dtdz)) { - phi_out(i,j,k) = phi_in(i,j,k) + + phi_out(i,j,k) = phi_in(i,j,k) + ( AMREX_D_TERM( (flxx(i,j,k) - flxx(i+1,j,k)) * dtdx, + (flxy(i,j,k) - flxy(i,j+1,k)) * dtdy, + (flxz(i,j,k) - flxz(i,j,k+1)) * dtdz ) ); diff --git a/Tutorials/Amr/Advection_AmrCore/Source/Src_K/compute_flux_3D_K.H b/Tutorials/Amr/Advection_AmrCore/Source/Src_K/compute_flux_3D_K.H index 4579d752432..538bf8bd14e 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/Src_K/compute_flux_3D_K.H +++ b/Tutorials/Amr/Advection_AmrCore/Source/Src_K/compute_flux_3D_K.H @@ -44,7 +44,7 @@ void flux_z(int i, int j, int k, Array4 const& slope, Real dtdz) { - pz(i,j,k) = ( (vz(i,j,k) < 0) ? + pz(i,j,k) = ( (vz(i,j,k) < 0) ? phi(i,j,k ) - slope(i,j,k )*(0.5 + 0.5*dtdz*vz(i,j,k)) : phi(i,j,k-1) + slope(i,j,k-1)*(0.5 - 0.5*dtdz*vz(i,j,k)) ); } diff --git a/Tutorials/Amr/Advection_AmrCore/Source/face_velocity.H b/Tutorials/Amr/Advection_AmrCore/Source/face_velocity.H index 4803e1dbb7a..bac3d539b46 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/face_velocity.H +++ b/Tutorials/Amr/Advection_AmrCore/Source/face_velocity.H @@ -19,10 +19,10 @@ void get_face_velocity_psi(amrex::Box const& bx, const auto hi = ubound(bx); const Real* AMREX_RESTRICT prob_lo = geomdata.ProbLo(); - const Real* AMREX_RESTRICT dx = geomdata.CellSize(); + const Real* AMREX_RESTRICT dx = geomdata.CellSize(); for (int j = lo.y; j <= hi.y; ++j) { - Real y = dx[1]*(0.5+j) + prob_lo[1]; + Real y = dx[1]*(0.5+j) + prob_lo[1]; AMREX_PRAGMA_SIMD for (int i = lo.x; i <= hi.x; ++i) { Real x = dx[0]*(0.5+i) + prob_lo[0]; @@ -57,7 +57,7 @@ AMREX_FORCE_INLINE void get_face_velocity_z(int i, int j, int k, amrex::Array4 const& vz) { - vz(i,j,k) = 0.0; + vz(i,j,k) = 0.0; } #endif diff --git a/Tutorials/Amr/Advection_AmrCore/Source/main.cpp b/Tutorials/Amr/Advection_AmrCore/Source/main.cpp index df0fd0adae2..cd9f4bebf9c 100644 --- a/Tutorials/Amr/Advection_AmrCore/Source/main.cpp +++ b/Tutorials/Amr/Advection_AmrCore/Source/main.cpp @@ -23,21 +23,21 @@ int main(int argc, char* argv[]) // constructor - reads in parameters from inputs file // - sizes multilevel arrays and data structures AmrCoreAdv amr_core_adv; - + // initialize AMR data - amr_core_adv.InitData(); + amr_core_adv.InitData(); // advance solution to final time - amr_core_adv.Evolve(); - + amr_core_adv.Evolve(); + // wallclock time - auto end_total = amrex::second() - strt_total; - - if (amr_core_adv.Verbose()) { + auto end_total = amrex::second() - strt_total; + + if (amr_core_adv.Verbose()) { // print wallclock time ParallelDescriptor::ReduceRealMax(end_total ,ParallelDescriptor::IOProcessorNumber()); amrex::Print() << "\nTotal Time: " << end_total << '\n'; - } + } } amrex::Finalize(); diff --git a/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/Prob.f90 b/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/Prob.f90 index 93278c87712..4bce93a1bcc 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/Prob.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/Prob.f90 @@ -27,7 +27,7 @@ subroutine initdata(level, time, lo, hi, & integer :: dm integer :: i,j,k double precision :: x,y,z,r2 - + if (phi_lo(3) .eq. 0 .and. phi_hi(3) .eq. 0) then dm = 2 else @@ -41,7 +41,7 @@ subroutine initdata(level, time, lo, hi, & y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + if ( dm.eq. 2) then r2 = ((x-0.5d0)**2 + (y-0.75d0)**2) / 0.01d0 phi(i,j,k) = 1.d0 + exp(-r2) diff --git a/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_2d.f90 b/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_2d.f90 index 5ca92566473..91c510d9242 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_2d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_2d.f90 @@ -25,7 +25,7 @@ subroutine get_face_velocity(level, time, & plo(2) = min(vx_l2-1, vy_l2-1) phi(1) = max(vx_h1 , vy_h1+1) phi(2) = max(vx_h2+1, vy_h2 ) - + call bl_allocate(psi, plo(1), phi(1), plo(2), phi(2)) ! streamfunction psi @@ -36,7 +36,7 @@ subroutine get_face_velocity(level, time, & psi(i,j) = sin(M_PI*x)**2 * sin(M_PI*y)**2 * cos (M_PI*time/2.d0) * (1.d0 / M_PI) end do end do - + ! x velocity do j = vx_l2, vx_h2 y = (dble(j)+0.5d0) * dx(2) + prob_lo(2) @@ -56,6 +56,6 @@ subroutine get_face_velocity(level, time, & end do call bl_deallocate(psi) - + end subroutine get_face_velocity diff --git a/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_3d.f90 b/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_3d.f90 index d95a53e3d82..9005503c893 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_3d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Exec/SingleVortex/face_velocity_3d.f90 @@ -28,7 +28,7 @@ subroutine get_face_velocity(level, time, & plo(2) = min(vx_l2-1, vy_l2-1) phi(1) = max(vx_h1 , vy_h1+1) phi(2) = max(vx_h2+1, vy_h2 ) - + call bl_allocate(psi, plo(1), phi(1), plo(2), phi(2)) ! streamfunction psi @@ -39,7 +39,7 @@ subroutine get_face_velocity(level, time, & psi(i,j) = sin(M_PI*x)**2 * sin(M_PI*y)**2 * cos (M_PI*time/2.d0) * (1.d0 / M_PI) end do end do - + ! x velocity do k = vx_l3, vx_h3 do j = vx_l2, vx_h2 diff --git a/Tutorials/Amr/Advection_AmrLevel/Exec/UniformVelocity/Prob.f90 b/Tutorials/Amr/Advection_AmrLevel/Exec/UniformVelocity/Prob.f90 index aa7f946143e..3b7643d1a91 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Exec/UniformVelocity/Prob.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Exec/UniformVelocity/Prob.f90 @@ -12,23 +12,23 @@ subroutine amrex_probinit (init,name,namlen,problo,probhi) bind(c) integer untin,i namelist /fortin/ adv_vel - + ! ! Build "probin" filename -- the name of file containing fortin namelist. - ! + ! integer maxlen parameter (maxlen=256) character probin*(maxlen) - + if (namlen .gt. maxlen) then write(6,*) 'probin file name too long' stop end if - + do i = 1, namlen probin(i:i) = char(name(i)) end do - + ! set the namelist default adv_vel(:) = 1.d0 @@ -56,7 +56,7 @@ subroutine initdata(level, time, lo, hi, & integer :: dm integer :: i,j,k double precision :: x,y,z,r2 - + if (phi_lo(3) .eq. 0 .and. phi_hi(3) .eq. 0) then dm = 2 else @@ -70,7 +70,7 @@ subroutine initdata(level, time, lo, hi, & y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + if ( dm.eq. 2) then r2 = ((x-0.0d0)**2 + (y-0.0d0)**2) / 0.01d0 phi(i,j,k) = 1.d0 + exp(-r2) diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Adv_F.H b/Tutorials/Amr/Advection_AmrLevel/Source/Adv_F.H index 1c08189d996..ea7d245216b 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Adv_F.H +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Adv_F.H @@ -5,45 +5,45 @@ #include #ifdef __cplusplus -extern "C" +extern "C" { #endif void get_tagging_params(const int* name, const int* namlen); void nullfill(BL_FORT_FAB_ARG_3D(state), - const int* dlo, const int* dhi, - const amrex_real* dx, const amrex_real* glo, - const amrex_real* time, const int* bc); - + const int* dlo, const int* dhi, + const amrex_real* dx, const amrex_real* glo, + const amrex_real* time, const int* bc); + void state_error(int* tag, const int* tag_lo, const int* tag_hi, - BL_FORT_FAB_ARG_3D(state), - const int* tagval, const int* clearval, - const int* lo, const int* hi, - const amrex_real* dx, const amrex_real* problo, - const amrex_real* time, const int* level); - - void initdata(const int* level, const amrex_real* time, - const int* lo, const int* hi, - BL_FORT_FAB_ARG_3D(state), - const amrex_real* dx, const amrex_real* problo); - - void get_face_velocity(const int* level, const amrex_real* time, - AMREX_D_DECL(BL_FORT_FAB_ARG(xvel), - BL_FORT_FAB_ARG(yvel), - BL_FORT_FAB_ARG(zvel)), - const amrex_real* dx, const amrex_real* problo); - + BL_FORT_FAB_ARG_3D(state), + const int* tagval, const int* clearval, + const int* lo, const int* hi, + const amrex_real* dx, const amrex_real* problo, + const amrex_real* time, const int* level); + + void initdata(const int* level, const amrex_real* time, + const int* lo, const int* hi, + BL_FORT_FAB_ARG_3D(state), + const amrex_real* dx, const amrex_real* problo); + + void get_face_velocity(const int* level, const amrex_real* time, + AMREX_D_DECL(BL_FORT_FAB_ARG(xvel), + BL_FORT_FAB_ARG(yvel), + BL_FORT_FAB_ARG(zvel)), + const amrex_real* dx, const amrex_real* problo); + void advect(const amrex_real* time, const int* lo, const int* hi, - const BL_FORT_FAB_ARG_3D(statein), - BL_FORT_FAB_ARG_3D(stateout), - AMREX_D_DECL(const BL_FORT_FAB_ARG_3D(xvel), - const BL_FORT_FAB_ARG_3D(yvel), - const BL_FORT_FAB_ARG_3D(zvel)), - AMREX_D_DECL(BL_FORT_FAB_ARG_3D(fx), - BL_FORT_FAB_ARG_3D(fy), - BL_FORT_FAB_ARG_3D(fz)), - const amrex_real* dx, const amrex_real* dt); + const BL_FORT_FAB_ARG_3D(statein), + BL_FORT_FAB_ARG_3D(stateout), + AMREX_D_DECL(const BL_FORT_FAB_ARG_3D(xvel), + const BL_FORT_FAB_ARG_3D(yvel), + const BL_FORT_FAB_ARG_3D(zvel)), + AMREX_D_DECL(BL_FORT_FAB_ARG_3D(fx), + BL_FORT_FAB_ARG_3D(fy), + BL_FORT_FAB_ARG_3D(fz)), + const amrex_real* dx, const amrex_real* dt); #ifdef __cplusplus } diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.H b/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.H index 2062c77e714..f6d61486e73 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.H +++ b/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.H @@ -35,7 +35,7 @@ public: //The basic constructor. // AmrLevelAdv (amrex::Amr& papa, - int lev, + int lev, const amrex::Geometry& level_geom, const amrex::BoxArray& bl, const amrex::DistributionMapping& dm, @@ -49,12 +49,12 @@ public: // virtual void restart (amrex::Amr& papa, std::istream& is, - bool bReadSpecial = false) override; + bool bReadSpecial = false) override; virtual void checkPoint (const std::string& dir, - std::ostream& os, - amrex::VisMF::How how = amrex::VisMF::NFiles, - bool dump_old = true) override; + std::ostream& os, + amrex::VisMF::How how = amrex::VisMF::NFiles, + bool dump_old = true) override; // //Write a plotfile to specified directory. @@ -141,7 +141,7 @@ public: int clearval, int tagval, amrex::Real time, - int n_error_buf = 0, int ngrow = 0) override; + int n_error_buf = 0, int ngrow = 0) override; #ifdef AMREX_PARTICLES static amrex::AmrTracerParticleContainer* theTracerPC () { return TracerPC.get(); } @@ -181,7 +181,7 @@ protected: static int do_tracers; static std::unique_ptr TracerPC; #endif -}; +}; // // Inlines. diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.cpp b/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.cpp index 4060c680ce9..61ca8a005c8 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.cpp +++ b/Tutorials/Amr/Advection_AmrLevel/Source/AmrLevelAdv.cpp @@ -31,13 +31,13 @@ AmrLevelAdv::AmrLevelAdv () //The basic constructor. // AmrLevelAdv::AmrLevelAdv (Amr& papa, - int lev, + int lev, const Geometry& level_geom, const BoxArray& bl, const DistributionMapping& dm, Real time) : - AmrLevel(papa,lev,level_geom,bl,dm,time) + AmrLevel(papa,lev,level_geom,bl,dm,time) { flux_reg = 0; if (level > 0 && do_reflux) @@ -47,7 +47,7 @@ AmrLevelAdv::AmrLevelAdv (Amr& papa, // //The destructor. // -AmrLevelAdv::~AmrLevelAdv () +AmrLevelAdv::~AmrLevelAdv () { delete flux_reg; } @@ -57,7 +57,7 @@ AmrLevelAdv::~AmrLevelAdv () // void AmrLevelAdv::restart (Amr& papa, - std::istream& is, + std::istream& is, bool bReadSpecial) { AmrLevel::restart(papa,is,bReadSpecial); @@ -67,11 +67,11 @@ AmrLevelAdv::restart (Amr& papa, flux_reg = new FluxRegister(grids,dmap,crse_ratio,level,NUM_STATE); } -void +void AmrLevelAdv::checkPoint (const std::string& dir, - std::ostream& os, + std::ostream& os, VisMF::How how, - bool dump_old) + bool dump_old) { AmrLevel::checkPoint(dir, os, how, dump_old); #ifdef AMREX_PARTICLES @@ -86,7 +86,7 @@ AmrLevelAdv::checkPoint (const std::string& dir, // void AmrLevelAdv::writePlotFile (const std::string& dir, - std::ostream& os, + std::ostream& os, VisMF::How how) { @@ -112,25 +112,25 @@ AmrLevelAdv::variableSetUp () desc_lst.addDescriptor(Phi_Type,IndexType::TheCellType(), StateDescriptor::Point,0,NUM_STATE, - &cell_cons_interp); + &cell_cons_interp); int lo_bc[BL_SPACEDIM]; int hi_bc[BL_SPACEDIM]; for (int i = 0; i < BL_SPACEDIM; ++i) { - lo_bc[i] = hi_bc[i] = BCType::int_dir; // periodic boundaries + lo_bc[i] = hi_bc[i] = BCType::int_dir; // periodic boundaries } - + BCRec bc(lo_bc, hi_bc); - desc_lst.setComponent(Phi_Type, 0, "phi", bc, - StateDescriptor::BndryFunc(nullfill)); + desc_lst.setComponent(Phi_Type, 0, "phi", bc, + StateDescriptor::BndryFunc(nullfill)); } // //Cleanup data descriptors at end of run. // void -AmrLevelAdv::variableCleanUp () +AmrLevelAdv::variableCleanUp () { desc_lst.clear(); #ifdef AMREX_PARTICLES @@ -163,8 +163,8 @@ AmrLevelAdv::initData () const int* hi = box.hiVect(); initdata(&level, &cur_time, AMREX_ARLIM_3D(lo), AMREX_ARLIM_3D(hi), - BL_TO_FORTRAN_3D(S_new[mfi]), AMREX_ZFILL(dx), - AMREX_ZFILL(prob_lo)); + BL_TO_FORTRAN_3D(S_new[mfi]), AMREX_ZFILL(dx), + AMREX_ZFILL(prob_lo)); } #ifdef AMREX_PARTICLES @@ -172,7 +172,7 @@ AmrLevelAdv::initData () #endif if (verbose) { - amrex::Print() << "Done initializing the level " << level + amrex::Print() << "Done initializing the level " << level << " data " << std::endl; } } @@ -248,28 +248,28 @@ AmrLevelAdv::advance (Real time, // FluxRegister *fine = 0; FluxRegister *current = 0; - + int finest_level = parent->finestLevel(); if (do_reflux && level < finest_level) { - fine = &getFluxReg(level+1); - fine->setVal(0.0); + fine = &getFluxReg(level+1); + fine->setVal(0.0); } if (do_reflux && level > 0) { - current = &getFluxReg(level); + current = &getFluxReg(level); } MultiFab fluxes[BL_SPACEDIM]; if (do_reflux) { - for (int j = 0; j < BL_SPACEDIM; j++) - { - BoxArray ba = S_new.boxArray(); - ba.surroundingNodes(j); - fluxes[j].define(ba, dmap, NUM_STATE, 0); - } + for (int j = 0; j < BL_SPACEDIM; j++) + { + BoxArray ba = S_new.boxArray(); + ba.surroundingNodes(j); + fluxes[j].define(ba, dmap, NUM_STATE, 0); + } } // State with ghost cells @@ -288,59 +288,59 @@ AmrLevelAdv::advance (Real time, #pragma omp parallel #endif { - FArrayBox flux[BL_SPACEDIM], uface[BL_SPACEDIM]; + FArrayBox flux[BL_SPACEDIM], uface[BL_SPACEDIM]; - for (MFIter mfi(S_new, true); mfi.isValid(); ++mfi) - { - const Box& bx = mfi.tilebox(); + for (MFIter mfi(S_new, true); mfi.isValid(); ++mfi) + { + const Box& bx = mfi.tilebox(); - const FArrayBox& statein = Sborder[mfi]; - FArrayBox& stateout = S_new[mfi]; + const FArrayBox& statein = Sborder[mfi]; + FArrayBox& stateout = S_new[mfi]; - // Allocate fabs for fluxes and Godunov velocities. - for (int i = 0; i < BL_SPACEDIM ; i++) { - const Box& bxtmp = amrex::surroundingNodes(bx,i); - flux[i].resize(bxtmp,NUM_STATE); - uface[i].resize(amrex::grow(bxtmp, iteration), 1); - } + // Allocate fabs for fluxes and Godunov velocities. + for (int i = 0; i < BL_SPACEDIM ; i++) { + const Box& bxtmp = amrex::surroundingNodes(bx,i); + flux[i].resize(bxtmp,NUM_STATE); + uface[i].resize(amrex::grow(bxtmp, iteration), 1); + } - get_face_velocity(&level, &ctr_time, - AMREX_D_DECL(BL_TO_FORTRAN(uface[0]), - BL_TO_FORTRAN(uface[1]), - BL_TO_FORTRAN(uface[2])), - dx, prob_lo); + get_face_velocity(&level, &ctr_time, + AMREX_D_DECL(BL_TO_FORTRAN(uface[0]), + BL_TO_FORTRAN(uface[1]), + BL_TO_FORTRAN(uface[2])), + dx, prob_lo); - for (int i = 0; i < BL_SPACEDIM ; i++) { + for (int i = 0; i < BL_SPACEDIM ; i++) { const Box& bxtmp = mfi.grownnodaltilebox(i, iteration); Umac[i][mfi].copy(uface[i], bxtmp); - } + } advect(&time, bx.loVect(), bx.hiVect(), - BL_TO_FORTRAN_3D(statein), - BL_TO_FORTRAN_3D(stateout), - AMREX_D_DECL(BL_TO_FORTRAN_3D(uface[0]), - BL_TO_FORTRAN_3D(uface[1]), - BL_TO_FORTRAN_3D(uface[2])), - AMREX_D_DECL(BL_TO_FORTRAN_3D(flux[0]), - BL_TO_FORTRAN_3D(flux[1]), - BL_TO_FORTRAN_3D(flux[2])), - dx, &dt); - - if (do_reflux) { - for (int i = 0; i < BL_SPACEDIM ; i++) - fluxes[i][mfi].copy(flux[i],mfi.nodaltilebox(i)); - } - } + BL_TO_FORTRAN_3D(statein), + BL_TO_FORTRAN_3D(stateout), + AMREX_D_DECL(BL_TO_FORTRAN_3D(uface[0]), + BL_TO_FORTRAN_3D(uface[1]), + BL_TO_FORTRAN_3D(uface[2])), + AMREX_D_DECL(BL_TO_FORTRAN_3D(flux[0]), + BL_TO_FORTRAN_3D(flux[1]), + BL_TO_FORTRAN_3D(flux[2])), + dx, &dt); + + if (do_reflux) { + for (int i = 0; i < BL_SPACEDIM ; i++) + fluxes[i][mfi].copy(flux[i],mfi.nodaltilebox(i)); + } + } } if (do_reflux) { - if (current) { - for (int i = 0; i < BL_SPACEDIM ; i++) - current->FineAdd(fluxes[i],i,0,0,NUM_STATE,1.); - } - if (fine) { - for (int i = 0; i < BL_SPACEDIM ; i++) - fine->CrseInit(fluxes[i],i,0,0,NUM_STATE,-1.); - } + if (current) { + for (int i = 0; i < BL_SPACEDIM ; i++) + current->FineAdd(fluxes[i],i,0,0,NUM_STATE,1.); + } + if (fine) { + for (int i = 0; i < BL_SPACEDIM ; i++) + fine->CrseInit(fluxes[i],i,0,0,NUM_STATE,-1.); + } } #ifdef AMREX_PARTICLES @@ -358,7 +358,7 @@ AmrLevelAdv::advance (Real time, Real AmrLevelAdv::estTimeStep (Real) { - // This is just a dummy value to start with + // This is just a dummy value to start with Real dt_est = 1.0e+20; const Real* dx = geom.CellSize(); @@ -370,38 +370,38 @@ AmrLevelAdv::estTimeStep (Real) #pragma omp parallel reduction(min:dt_est) #endif { - FArrayBox uface[BL_SPACEDIM]; - - for (MFIter mfi(S_new, true); mfi.isValid(); ++mfi) - { - for (int i = 0; i < BL_SPACEDIM ; i++) { - const Box& bx = mfi.nodaltilebox(i); - uface[i].resize(bx,1); - } - - get_face_velocity(&level, &cur_time, - AMREX_D_DECL(BL_TO_FORTRAN(uface[0]), - BL_TO_FORTRAN(uface[1]), - BL_TO_FORTRAN(uface[2])), - dx, prob_lo); - - for (int i = 0; i < BL_SPACEDIM; ++i) { - Real umax = uface[i].norm(0); - if (umax > 1.e-100) { - dt_est = std::min(dt_est, dx[i] / umax); - } - } - } + FArrayBox uface[BL_SPACEDIM]; + + for (MFIter mfi(S_new, true); mfi.isValid(); ++mfi) + { + for (int i = 0; i < BL_SPACEDIM ; i++) { + const Box& bx = mfi.nodaltilebox(i); + uface[i].resize(bx,1); + } + + get_face_velocity(&level, &cur_time, + AMREX_D_DECL(BL_TO_FORTRAN(uface[0]), + BL_TO_FORTRAN(uface[1]), + BL_TO_FORTRAN(uface[2])), + dx, prob_lo); + + for (int i = 0; i < BL_SPACEDIM; ++i) { + Real umax = uface[i].norm(0); + if (umax > 1.e-100) { + dt_est = std::min(dt_est, dx[i] / umax); + } + } + } } ParallelDescriptor::ReduceRealMin(dt_est); dt_est *= cfl; if (verbose) { - amrex::Print() << "AmrLevelAdv::estTimeStep at level " << level + amrex::Print() << "AmrLevelAdv::estTimeStep at level " << level << ": dt_est = " << dt_est << std::endl; } - + return dt_est; } @@ -419,7 +419,7 @@ AmrLevelAdv::initialTimeStep () // void AmrLevelAdv::computeInitialDt (int finest_level, - int sub_cycle, + int sub_cycle, Vector& n_cycle, const Vector& ref_ratio, Vector& dt_level, @@ -463,7 +463,7 @@ AmrLevelAdv::computeInitialDt (int finest_level, // void AmrLevelAdv::computeNewDt (int finest_level, - int sub_cycle, + int sub_cycle, Vector& n_cycle, const Vector& ref_ratio, Vector& dt_min, @@ -484,28 +484,28 @@ AmrLevelAdv::computeNewDt (int finest_level, dt_min[i] = adv_level.estTimeStep(dt_level[i]); } - if (post_regrid_flag == 1) + if (post_regrid_flag == 1) { - // - // Limit dt's by pre-regrid dt - // - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],dt_level[i]); - } + // + // Limit dt's by pre-regrid dt + // + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],dt_level[i]); + } } - else + else { - // - // Limit dt's by change_max * old dt - // - static Real change_max = 1.1; - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); - } + // + // Limit dt's by change_max * old dt + // + static Real change_max = 1.1; + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); + } } - + // // Find the minimum over all levels // @@ -553,17 +553,17 @@ AmrLevelAdv::post_timestep (int iteration) if (level < finest_level) avgDown(); -#ifdef AMREX_PARTICLES +#ifdef AMREX_PARTICLES if (TracerPC) { const int ncycle = parent->nCycle(level); - + if (iteration < ncycle || level == 0) - { + { int ngrow = (level == 0) ? 0 : iteration; - TracerPC->Redistribute(level, TracerPC->finestLevel(), ngrow); - } + TracerPC->Redistribute(level, TracerPC->finestLevel(), ngrow); + } } #endif } @@ -584,7 +584,7 @@ AmrLevelAdv::post_regrid (int lbase, int new_finest) { //Do work after a restart(). // void -AmrLevelAdv::post_restart() +AmrLevelAdv::post_restart() { #ifdef AMREX_PARTICLES if (do_tracers && level == 0) { @@ -617,7 +617,7 @@ AmrLevelAdv::post_init (Real stop_time) // void AmrLevelAdv::errorEst (TagBoxArray& tags, - int clearval, + int clearval, int tagval, Real time, int n_error_buf, @@ -633,32 +633,32 @@ AmrLevelAdv::errorEst (TagBoxArray& tags, #endif { Vector itags; - - for (MFIter mfi(S_new,true); mfi.isValid(); ++mfi) - { - const Box& tilebx = mfi.tilebox(); + + for (MFIter mfi(S_new,true); mfi.isValid(); ++mfi) + { + const Box& tilebx = mfi.tilebox(); TagBox& tagfab = tags[mfi]; - - // We cannot pass tagfab to Fortran becuase it is BaseFab. - // So we are going to get a temporary integer array. - tagfab.get_itags(itags, tilebx); - + + // We cannot pass tagfab to Fortran becuase it is BaseFab. + // So we are going to get a temporary integer array. + tagfab.get_itags(itags, tilebx); + // data pointer and index space - int* tptr = itags.dataPtr(); - const int* tlo = tilebx.loVect(); - const int* thi = tilebx.hiVect(); - - state_error(tptr, AMREX_ARLIM_3D(tlo), AMREX_ARLIM_3D(thi), - BL_TO_FORTRAN_3D(S_new[mfi]), - &tagval, &clearval, - AMREX_ARLIM_3D(tilebx.loVect()), AMREX_ARLIM_3D(tilebx.hiVect()), - AMREX_ZFILL(dx), AMREX_ZFILL(prob_lo), &time, &level); - // - // Now update the tags in the TagBox. - // - tagfab.tags_and_untags(itags, tilebx); - } + int* tptr = itags.dataPtr(); + const int* tlo = tilebx.loVect(); + const int* thi = tilebx.hiVect(); + + state_error(tptr, AMREX_ARLIM_3D(tlo), AMREX_ARLIM_3D(thi), + BL_TO_FORTRAN_3D(S_new[mfi]), + &tagval, &clearval, + AMREX_ARLIM_3D(tilebx.loVect()), AMREX_ARLIM_3D(tilebx.hiVect()), + AMREX_ZFILL(dx), AMREX_ZFILL(prob_lo), &time, &level); + // + // Now update the tags in the TagBox. + // + tagfab.tags_and_untags(itags, tilebx); + } } } @@ -671,7 +671,7 @@ AmrLevelAdv::read_params () done = true; - ParmParse pp("adv"); + ParmParse pp("adv"); pp.query("v",verbose); pp.query("cfl",cfl); @@ -681,17 +681,17 @@ AmrLevelAdv::read_params () // This tutorial code only supports Cartesian coordinates. if (! gg->IsCartesian()) { - amrex::Abort("Please set geom.coord_sys = 0"); + amrex::Abort("Please set geom.coord_sys = 0"); } // This tutorial code only supports periodic boundaries. if (! gg->isAllPeriodic()) { - amrex::Abort("Please set geom.is_periodic = 1 1 1"); + amrex::Abort("Please set geom.is_periodic = 1 1 1"); } #ifdef AMREX_PARTICLES pp.query("do_tracers", do_tracers); -#endif +#endif // // read tagging parameters from probin file @@ -706,7 +706,7 @@ AmrLevelAdv::read_params () Vector probin_file_name(probin_file_length); for (int i = 0; i < probin_file_length; i++) - probin_file_name[i] = probin_file[i]; + probin_file_name[i] = probin_file[i]; // use a fortran routine to // read in tagging parameters from probin file @@ -722,15 +722,15 @@ AmrLevelAdv::reflux () const auto strt = amrex::second(); getFluxReg(level+1).Reflux(get_new_data(Phi_Type),1.0,0,0,NUM_STATE,geom); - + if (verbose) { const int IOProc = ParallelDescriptor::IOProcessorNumber(); auto end = amrex::second() - strt; - + ParallelDescriptor::ReduceRealMax(end,IOProc); - - amrex::Print() << "AmrLevelAdv::reflux() at level " << level + + amrex::Print() << "AmrLevelAdv::reflux() at level " << level << " : time = " << end << std::endl; } } @@ -750,7 +750,7 @@ AmrLevelAdv::avgDown (int state_indx) AmrLevelAdv& fine_lev = getLevel(level+1); MultiFab& S_fine = fine_lev.get_new_data(state_indx); MultiFab& S_crse = get_new_data(state_indx); - + amrex::average_down(S_fine,S_crse, fine_lev.geom,geom, 0,S_fine.nComp(),parent->refRatio(level)); @@ -763,7 +763,7 @@ AmrLevelAdv::init_particles () if (do_tracers && level == 0) { BL_ASSERT(TracerPC == nullptr); - + TracerPC.reset(new AmrTracerParticleContainer(parent)); AmrTracerParticleContainer::ParticleInitData pdata = {AMREX_D_DECL(0.0, 0.0, 0.0)}; diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/LevelBldAdv.cpp b/Tutorials/Amr/Advection_AmrLevel/Source/LevelBldAdv.cpp index fc3e077978b..d507d119650 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/LevelBldAdv.cpp +++ b/Tutorials/Amr/Advection_AmrLevel/Source/LevelBldAdv.cpp @@ -15,7 +15,7 @@ class LevelBldAdv int lev, const Geometry& level_geom, const BoxArray& ba, - const DistributionMapping& dm, + const DistributionMapping& dm, Real time) override; }; @@ -47,7 +47,7 @@ LevelBldAdv::operator() () AmrLevel* LevelBldAdv::operator() (Amr& papa, - int lev, + int lev, const Geometry& level_geom, const BoxArray& ba, const DistributionMapping& dm, diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/Adv_2d.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/Adv_2d.f90 index 6beaf0bbb79..8d9d980f380 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/Adv_2d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/Adv_2d.f90 @@ -7,7 +7,7 @@ subroutine advect(time, lo, hi, & & flxx, fx_lo, fx_hi, & & flxy, fy_lo, fy_hi, & & dx,dt) bind(C, name="advect") - + use amrex_mempool_module, only : bl_allocate, bl_deallocate use compute_flux_module, only : compute_flux_2d @@ -45,14 +45,14 @@ subroutine advect(time, lo, hi, & call bl_allocate(phiy_1d, glo(1), ghi(1), glo(2), ghi(2)) call bl_allocate(phix , glo(1), ghi(1), glo(2), ghi(2)) call bl_allocate(phiy , glo(1), ghi(1), glo(2), ghi(2)) - ! slope + ! slope call bl_allocate(slope , glo(1), ghi(1), glo(2), ghi(2)) ! We like to allocate these **pointers** here and then pass them to a function ! to remove their pointerness for performance, because normally pointers could ! be aliasing. We need to use pointers instead of allocatable arrays because ! we like to use AMReX's bl_allocate to allocate memeory instead of the intrinsic - ! allocate. Bl_allocate is much faster than allocate inside OMP. + ! allocate. Bl_allocate is much faster than allocate inside OMP. ! Note that one MUST CALL BL_DEALLOCATE. ! check if CFL condition is violated. @@ -88,9 +88,9 @@ subroutine advect(time, lo, hi, & flxx(i,j) = flxx(i,j) * ( dt * dx(2)) enddo enddo - + ! Scale by face area in order to correctly reflx - do j = lo(2), hi(2)+1 + do j = lo(2), hi(2)+1 do i = lo(1), hi(1) flxy(i,j) = flxy(i,j) * (dt * dx(1)) enddo diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/compute_flux_2d.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/compute_flux_2d.f90 index 72bc80cff76..05b7f7eaaa5 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/compute_flux_2d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/compute_flux_2d.f90 @@ -32,7 +32,7 @@ subroutine compute_flux_2d(lo, hi, dt, dx, & double precision, intent( out) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) double precision, dimension(glo(1):ghi(1),glo(2):ghi(2)) :: & phix_1d, phiy_1d, phix, phiy, slope - + integer :: i, j double precision :: hdtdx(2) diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/slope_2d.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/slope_2d.f90 index a032c8650b1..79f250edb64 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/slope_2d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_2d/slope_2d.f90 @@ -1,15 +1,15 @@ module slope_module - + implicit none double precision, parameter:: four3rd=4.d0/3.d0, sixth=1.d0/6.d0 - + private - + public :: slopex, slopey - + contains - + subroutine slopex(lo, hi, & q, qlo, qhi, & dq, dqlo, dqhi) @@ -123,4 +123,4 @@ subroutine slopey_doit(lo, hi, & end subroutine slopey_doit -end module slope_module +end module slope_module diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/Adv_3d.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/Adv_3d.f90 index 7cede23cf06..0acba602dc2 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/Adv_3d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/Adv_3d.f90 @@ -9,7 +9,7 @@ subroutine advect(time, lo, hi, & & flxy, fy_lo, fy_hi, & & flxz, fz_lo, fz_hi, & & dx,dt) bind(C, name="advect") - + use amrex_mempool_module, only : bl_allocate, bl_deallocate use compute_flux_module, only : compute_flux_3d @@ -58,13 +58,13 @@ subroutine advect(time, lo, hi, & call bl_allocate(phiz_x,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) call bl_allocate(phiz_y,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) ! slope - call bl_allocate(slope,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) - + call bl_allocate(slope,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) + ! We like to allocate these **pointers** here and then pass them to a function ! to remove their pointerness for performance, because normally pointers could ! be aliasing. We need to use pointers instead of allocatable arrays because ! we like to use AMReX's bl_allocate to allocate memeory instead of the intrinsic - ! allocate. Bl_allocate is much faster than allocate inside OMP. + ! allocate. Bl_allocate is much faster than allocate inside OMP. ! Note that one MUST CALL BL_DEALLOCATE. ! check if CFL condition is violated. @@ -113,7 +113,7 @@ subroutine advect(time, lo, hi, & enddo enddo do k = lo(3), hi(3) - do j = lo(2), hi(2)+1 + do j = lo(2), hi(2)+1 do i = lo(1), hi(1) flxy(i,j,k) = flxy(i,j,k) * (dt * dx(1)*dx(3)) enddo diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/compute_flux_3d.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/compute_flux_3d.f90 index fa959b8ac14..26bb127f02f 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/compute_flux_3d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/compute_flux_3d.f90 @@ -41,7 +41,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & double precision, intent( out) :: flxz(fz_lo(1):fz_hi(1),fz_lo(2):fz_hi(2),fz_lo(3):fz_hi(3)) double precision, dimension(glo(1):ghi(1),glo(2):ghi(2),glo(3):ghi(3)) :: & phix, phix_y, phix_z, phiy, phiy_x, phiy_z, phiz, phiz_x, phiz_y, slope - + integer :: i, j, k double precision :: hdtdx(3), tdtdx(3) @@ -51,7 +51,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & call slopex(glo, ghi, & phi, ph_lo, ph_hi, & slope, glo, ghi) - + ! compute phi on x faces using umac to upwind; ignore transverse terms do k = lo(3)-1, hi(3)+1 do j = lo(2)-1, hi(2)+1 @@ -70,7 +70,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & call slopey(glo, ghi, & phi, ph_lo, ph_hi, & slope, glo, ghi) - + ! compute phi on y faces using vmac to upwind; ignore transverse terms do k = lo(3)-1, hi(3)+1 do j = lo(2) , hi(2)+1 @@ -89,7 +89,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & call slopez(glo, ghi, & phi, ph_lo, ph_hi, & slope, glo, ghi) - + ! compute phi on z faces using wmac to upwind; ignore transverse terms do k = lo(3) , hi(3)+1 do j = lo(2)-1, hi(2)+1 diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/slope_3d.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/slope_3d.f90 index 418ac9d5ff3..7a543373b95 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/slope_3d.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_3d/slope_3d.f90 @@ -1,15 +1,15 @@ module slope_module - + implicit none double precision, parameter:: four3rd=4.d0/3.d0, sixth=1.d0/6.d0 - + private - + public :: slopex, slopey, slopez - + contains - + subroutine slopex(lo, hi, & q, qlo, qhi, & dq, dqlo, dqhi) @@ -40,7 +40,7 @@ subroutine slopex(lo, hi, & endif df(i) = dsgn(i)*min( dlim(i), abs(dcen(i)) ) end do - + ! Now limited fourth order slopes do i = lo(1), hi(1) dq1 = four3rd*dcen(i) - sixth*(df(i+1) + df(i-1)) @@ -116,7 +116,7 @@ subroutine slopey_doit(lo, hi, & df(i,j) = dsgn(i,j)*min( dlim(i,j),abs(dcen(i,j)) ) end do end do - + ! Now compute limited fourth order slopes do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -195,7 +195,7 @@ subroutine slopez_doit(lo, hi, & end do end do end do - + ! Now compute limited fourth order slopes do k = lo(3), hi(3) do j = lo(2), hi(2) @@ -208,4 +208,4 @@ subroutine slopez_doit(lo, hi, & end subroutine slopez_doit -end module slope_module +end module slope_module diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/Tagging_nd.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/Tagging_nd.f90 index fe12a0e4ce7..81af1aa8db9 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/Tagging_nd.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/Tagging_nd.f90 @@ -1,9 +1,9 @@ ! ::: ----------------------------------------------------------- ! ::: This routine will tag high error cells based on the state -! ::: +! ::: ! ::: INPUTS/OUTPUTS: -! ::: +! ::: ! ::: tag <= integer tag array ! ::: tag_lo,hi => index extent of tag array ! ::: state => state array @@ -25,7 +25,7 @@ subroutine state_error(tag,tag_lo,tag_hi, & use tagging_params_module, only : phierr, phigrad, max_phierr_lev, max_phigrad_lev implicit none - + integer :: lo(3),hi(3) integer :: state_lo(3),state_hi(3) integer :: tag_lo(3),tag_hi(3) @@ -80,6 +80,6 @@ subroutine state_error(tag,tag_lo,tag_hi, & enddo end do endif - + end subroutine state_error diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/tagging_params.f90 b/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/tagging_params.f90 index b0bebd02479..c561976b88f 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/tagging_params.f90 +++ b/Tutorials/Amr/Advection_AmrLevel/Source/Src_nd/tagging_params.f90 @@ -5,7 +5,7 @@ module tagging_params_module integer, save :: max_phierr_lev, max_phigrad_lev contains - + subroutine get_tagging_params(name, namlen) bind(C, name="get_tagging_params") ! Initialize the tagging parameters diff --git a/Tutorials/Amr/Advection_AmrLevel/Source/main.cpp b/Tutorials/Amr/Advection_AmrLevel/Source/main.cpp index 48cb3d93d99..f65df7ae738 100644 --- a/Tutorials/Amr/Advection_AmrLevel/Source/main.cpp +++ b/Tutorials/Amr/Advection_AmrLevel/Source/main.cpp @@ -39,33 +39,33 @@ main (int argc, } if (max_step < 0 && stop_time < 0.0) { - amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); + amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); } { - Amr amr; + Amr amr; - amr.init(strt_time,stop_time); + amr.init(strt_time,stop_time); - while ( amr.okToContinue() && - (amr.levelSteps(0) < max_step || max_step < 0) && - (amr.cumTime() < stop_time || stop_time < 0.0) ) + while ( amr.okToContinue() && + (amr.levelSteps(0) < max_step || max_step < 0) && + (amr.cumTime() < stop_time || stop_time < 0.0) ) - { - // - // Do a coarse timestep. Recursively calls timeStep() - // - amr.coarseTimeStep(stop_time); - } + { + // + // Do a coarse timestep. Recursively calls timeStep() + // + amr.coarseTimeStep(stop_time); + } - // Write final checkpoint and plotfile - if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { - amr.checkPoint(); - } + // Write final checkpoint and plotfile + if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { + amr.checkPoint(); + } - if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { - amr.writePlotFile(); - } + if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { + amr.writePlotFile(); + } } diff --git a/Tutorials/Basic/Build_with_libamrex/main.cpp b/Tutorials/Basic/Build_with_libamrex/main.cpp index 5a6a1e498ca..1b19db9a461 100644 --- a/Tutorials/Basic/Build_with_libamrex/main.cpp +++ b/Tutorials/Basic/Build_with_libamrex/main.cpp @@ -34,7 +34,7 @@ int main(int argc, char* argv[]) // We choose to call MPI_Init ourselves, instead of letting AMRex do it. // Then we are going to pass a communicatior, MPI_COMM_WORLD, (or another) to AMReX. MPI_Init(&argc, &argv); - + bool build_parm_parse_from_command_line = false; amrex::Initialize(argc, argv, build_parm_parse_from_command_line, MPI_COMM_WORLD, add_parameters); // last three arguments are optional @@ -63,14 +63,14 @@ void add_parameters () pp.add("an_int_scalar", 2); // integer scalar: an_int_scalar pp.add("a_bool_scalar", true); // logical scalar: a_bool_scalar pp.addarr("a_real_array", // real array: a_real_array - std::vector{1.,2.,3.}); + std::vector{1.,2.,3.}); } { // prefix "a_prefix" amrex::ParmParse pp("a_prefix"); pp.addarr("an_int_array", // integer array: a_prefix.an_int_array - std::vector{2, 3, 4}); + std::vector{2, 3, 4}); amrex::Real x = 3.14; pp.add("a_real_scalar", x); // real scalar : a_prefix.a_real_scalar pp.add("a_string", std::string{"vonNeumann"}); // string: a_prefix.a_string diff --git a/Tutorials/Basic/Build_with_libamrex/test_parameters.cpp b/Tutorials/Basic/Build_with_libamrex/test_parameters.cpp index 09715a5fd6f..4c46aa07d9c 100644 --- a/Tutorials/Basic/Build_with_libamrex/test_parameters.cpp +++ b/Tutorials/Basic/Build_with_libamrex/test_parameters.cpp @@ -25,7 +25,7 @@ #include #include -void +void MyParams::test_parameters () { { diff --git a/Tutorials/Basic/HeatEquation_EX1_C/Source/main.cpp b/Tutorials/Basic/HeatEquation_EX1_C/Source/main.cpp index 50a2b0f5928..e09f065082e 100644 --- a/Tutorials/Basic/HeatEquation_EX1_C/Source/main.cpp +++ b/Tutorials/Basic/HeatEquation_EX1_C/Source/main.cpp @@ -11,9 +11,9 @@ using namespace amrex; int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -31,7 +31,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -72,12 +72,12 @@ void main_main () geom.define(domain,real_box,CoordSys::cartesian,is_periodic); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 1; - + // Ncomp = number of components for each array int Ncomp = 1; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -122,9 +122,9 @@ void main_main () MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); // new_phi = old_phi + dt * (something) - advance(phi_old, phi_new, flux, dt, geom); + advance(phi_old, phi_new, flux, dt, geom); time = time + dt; - + // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; diff --git a/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.H b/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.H index c2cac7ddf0c..7f825217e57 100644 --- a/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.H +++ b/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.H @@ -8,8 +8,8 @@ void main_main (); void advance (amrex::MultiFab& phi_old, amrex::MultiFab& phi_new, - amrex::Array& flux, - amrex::Real dt, + amrex::Array& flux, + amrex::Real dt, amrex::Geometry const& geom); void init_phi (amrex::MultiFab& phi_new, amrex::Geometry const& geom); diff --git a/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.cpp b/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.cpp index ee02b6f1f8f..1eff8eec87a 100644 --- a/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.cpp +++ b/Tutorials/Basic/HeatEquation_EX1_C/Source/myfunc.cpp @@ -6,8 +6,8 @@ using namespace amrex; void advance (MultiFab& phi_old, MultiFab& phi_new, - Array& flux, - Real dt, + Array& flux, + Real dt, Geometry const& geom) { @@ -20,7 +20,7 @@ void advance (MultiFab& phi_old, // Note that this simple example is not optimized. // The following two MFIter loops could be merged // and we do not have to use flux MultiFab. - // + // // ======================================================= // This example supports both 2D and 3D. Otherwise, diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance.cpp b/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance.cpp index eaad43c02aa..9f896036a68 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance.cpp +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance.cpp @@ -6,8 +6,8 @@ using namespace amrex; void advance (MultiFab& phi_old, MultiFab& phi_new, - Array& flux, - Real dt, + Array& flux, + Real dt, const Geometry& geom) { // Fill the ghost cells of each grid from the other grids @@ -20,7 +20,7 @@ void advance (MultiFab& phi_old, // Note that this simple example is not optimized. // The following two MFIter loops could be merged // and we do not have to use flux MultiFab. - // + // const Box& domain_bx = geom.Domain(); @@ -34,23 +34,23 @@ void advance (MultiFab& phi_old, BL_TO_FORTRAN_ANYD(phi_old[mfi]), BL_TO_FORTRAN_ANYD(flux[0][mfi]), BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM == 3) BL_TO_FORTRAN_ANYD(flux[2][mfi]), #endif dx); } - + // Advance the solution one grid at a time for ( MFIter mfi(phi_old); mfi.isValid(); ++mfi ) { const Box& bx = mfi.validbox(); - + update_phi(BL_TO_FORTRAN_BOX(bx), BL_TO_FORTRAN_ANYD(phi_old[mfi]), BL_TO_FORTRAN_ANYD(phi_new[mfi]), BL_TO_FORTRAN_ANYD(flux[0][mfi]), BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM == 3) BL_TO_FORTRAN_ANYD(flux[2][mfi]), #endif dx, &dt); diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance_3d.f90 b/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance_3d.f90 index 6b50e26549e..03cb5c11e2c 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance_3d.f90 +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/advance_3d.f90 @@ -13,7 +13,7 @@ subroutine compute_flux (lo, hi, domlo, domhi, phi, philo, phihi, & real(amrex_real), intent(inout) :: fluxy( fylo(1): fyhi(1), fylo(2): fyhi(2), fylo(3): fyhi(3)) real(amrex_real), intent(inout) :: fluxz( fzlo(1): fzhi(1), fzlo(2): fzhi(2), fzlo(3): fzhi(3)) real(amrex_real), intent(in) :: dx(3) - + ! local variables integer i,j,k diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_2d.f90 b/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_2d.f90 index 02a4c036cff..ea934ded810 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_2d.f90 +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_2d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(2), hi(2), philo(2), phihi(2) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in ) :: dx(2) - real(amrex_real), intent(in ) :: prob_lo(2) - real(amrex_real), intent(in ) :: prob_hi(2) + real(amrex_real), intent(in ) :: dx(2) + real(amrex_real), intent(in ) :: prob_lo(2) + real(amrex_real), intent(in ) :: prob_hi(2) integer :: i,j double precision :: x,y,r2 diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_3d.f90 b/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_3d.f90 index ec281d22df5..9fc031ee221 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_3d.f90 +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/init_phi_3d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(3), hi(3), philo(3), phihi(3) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2),philo(3):phihi(3)) - real(amrex_real), intent(in ) :: dx(3) - real(amrex_real), intent(in ) :: prob_lo(3) - real(amrex_real), intent(in ) :: prob_hi(3) + real(amrex_real), intent(in ) :: dx(3) + real(amrex_real), intent(in ) :: prob_lo(3) + real(amrex_real), intent(in ) :: prob_hi(3) integer :: i,j,k double precision :: x,y,z,r2 diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/main.cpp b/Tutorials/Basic/HeatEquation_EX1_CF/Source/main.cpp index e3957416df9..1f8fee9ca0b 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/main.cpp +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/main.cpp @@ -11,9 +11,9 @@ using namespace amrex; int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -32,7 +32,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -72,12 +72,12 @@ void main_main () geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 1; - + // Ncomp = number of components for each array int Ncomp = 1; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -131,9 +131,9 @@ void main_main () MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); // new_phi = old_phi + dt * (something) - advance(phi_old, phi_new, flux, dt, geom); + advance(phi_old, phi_new, flux, dt, geom); time = time + dt; - + // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc.H b/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc.H index 6a5101842bf..6a760dc79a2 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc.H +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc.H @@ -9,8 +9,8 @@ void main_main (); void advance (amrex::MultiFab& phi_old, amrex::MultiFab& phi_new, - amrex::Array& flux, - amrex::Real dt, + amrex::Array& flux, + amrex::Real dt, const amrex::Geometry& geom); #endif diff --git a/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc_F.H b/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc_F.H index 7da2ca7e00b..897ad9697b9 100644 --- a/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc_F.H +++ b/Tutorials/Basic/HeatEquation_EX1_CF/Source/myfunc_F.H @@ -11,13 +11,13 @@ extern "C" void init_phi(const int* lo, const int* hi, amrex_real* data, const int* dlo, const int* dhi, const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi); - + void compute_flux(const int* lo, const int* hi, const int* domlo, const int* domhi, amrex_real* phi, const int* philo, const int* phihi, amrex_real* fluxx, const int* fxlo, const int* fxhi, amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) +#if (BL_SPACEDIM == 3) amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif const amrex_real* dx); @@ -27,7 +27,7 @@ extern "C" amrex_real* pnew, const int* pnlo, const int* pnhi, amrex_real* fluxx, const int* fxlo, const int* fxhi, amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) +#if (BL_SPACEDIM == 3) amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif const amrex_real* dx, const amrex_real* dt); diff --git a/Tutorials/Basic/HeatEquation_EX1_F/advance.f90 b/Tutorials/Basic/HeatEquation_EX1_F/advance.f90 index 5736b4cd6de..70ddeed6522 100644 --- a/Tutorials/Basic/HeatEquation_EX1_F/advance.f90 +++ b/Tutorials/Basic/HeatEquation_EX1_F/advance.f90 @@ -40,7 +40,7 @@ subroutine advance (old_phi, new_phi, geom, dt) plo = lbound(po) phi = ubound(po) - + select case (amrex_spacedim) case (2) call update_phi_2d(bx%lo, bx%hi, po, pn, plo, phi, dx, dt) @@ -64,17 +64,17 @@ subroutine update_phi_2d (lo, hi, pold, pnew, plo, phi, dx, dt) real(amrex_real) :: dxinv, dtdx real(amrex_real) :: fx(lo(1):hi(1)+1,lo(2):hi(2) ) real(amrex_real) :: fy(lo(1):hi(1) ,lo(2):hi(2)+1) - + dxinv = 1.d0/dx dtdx = dt*dxinv - + ! x-fluxes do j=lo(2),hi(2) do i=lo(1),hi(1)+1 fx(i,j) = ( pold(i-1,j) - pold(i,j) ) * dxinv end do end do - + ! y-fluxes do j=lo(2),hi(2)+1 do i=lo(1),hi(1) @@ -84,7 +84,7 @@ subroutine update_phi_2d (lo, hi, pold, pnew, plo, phi, dx, dt) do j=lo(2),hi(2) do i=lo(1),hi(1) - + pnew(i,j) = pold(i,j) - dtdx * & ( fx(i+1,j)-fx(i,j) & + fy(i,j+1)-fy(i,j) ) @@ -105,10 +105,10 @@ subroutine update_phi_3d (lo, hi, pold, pnew, plo, phi, dx, dt) real(amrex_real) :: fx(lo(1):hi(1)+1,lo(2):hi(2) ,lo(3):hi(3)) real(amrex_real) :: fy(lo(1):hi(1) ,lo(2):hi(2)+1,lo(3):hi(3)) real(amrex_real) :: fz(lo(1):hi(1) ,lo(2):hi(2) ,lo(3):hi(3)+1) - + dxinv = 1.d0/dx dtdx = dt*dxinv - + ! x-fluxes do k=lo(3),hi(3) do j=lo(2),hi(2) @@ -117,7 +117,7 @@ subroutine update_phi_3d (lo, hi, pold, pnew, plo, phi, dx, dt) end do end do end do - + ! y-fluxes do k=lo(3),hi(3) do j=lo(2),hi(2)+1 @@ -126,7 +126,7 @@ subroutine update_phi_3d (lo, hi, pold, pnew, plo, phi, dx, dt) end do end do end do - + ! z-fluxes do k=lo(3),hi(3)+1 do j=lo(2),hi(2) @@ -135,16 +135,16 @@ subroutine update_phi_3d (lo, hi, pold, pnew, plo, phi, dx, dt) end do end do end do - + do k=lo(3),hi(3) do j=lo(2),hi(2) do i=lo(1),hi(1) - + pnew(i,j,k) = pold(i,j,k) - dtdx * & ( fx(i+1,j,k)-fx(i,j,k) & + fy(i,j+1,k)-fy(i,j,k) & + fz(i,j,k+1)-fz(i,j,k) ) - + end do end do end do diff --git a/Tutorials/Basic/HeatEquation_EX1_F/fmain.F90 b/Tutorials/Basic/HeatEquation_EX1_F/fmain.F90 index 89ce2420244..7ff6e7bbdc9 100644 --- a/Tutorials/Basic/HeatEquation_EX1_F/fmain.F90 +++ b/Tutorials/Basic/HeatEquation_EX1_F/fmain.F90 @@ -34,7 +34,7 @@ program main plot_int = -1 ! default to no plotfiles call pp%query("plot_int", plot_int); - + call amrex_parmparse_destroy(pp) ! Define a single box covering the domain @@ -48,7 +48,7 @@ program main ! Build a DistributionMapping for the boxarray call amrex_distromap_build(dm, ba) - + ! This defines a amrex_geometry object. call amrex_geometry_build(geom, domain) diff --git a/Tutorials/Basic/HeatEquation_EX1_F/init_phi.f90 b/Tutorials/Basic/HeatEquation_EX1_F/init_phi.f90 index 98140894094..a6bdcc0d07a 100644 --- a/Tutorials/Basic/HeatEquation_EX1_F/init_phi.f90 +++ b/Tutorials/Basic/HeatEquation_EX1_F/init_phi.f90 @@ -10,9 +10,9 @@ module init_phi_module public :: init_phi contains - + subroutine init_phi(phi,geom) - + type(amrex_multifab), intent(inout) :: phi type(amrex_geometry), intent(in ) :: geom @@ -46,7 +46,7 @@ subroutine init_phi_2d(lo, hi, phi, dlo, dhi, prob_lo, dx) real(amrex_real) :: phi(dlo(1):dhi(1),dlo(2):dhi(2)) real(amrex_real) :: prob_lo(2) real(amrex_real) :: dx(2) - + ! local varables integer :: i,j real(amrex_real) :: x,y,r2 @@ -55,13 +55,13 @@ subroutine init_phi_2d(lo, hi, phi, dlo, dhi, prob_lo, dx) y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + r2 = ((x-0.25d0)**2 + (y-0.25d0)**2) / 0.01d0 phi(i,j) = 1.d0 + exp(-r2) - + end do end do - + end subroutine init_phi_2d subroutine init_phi_3d(lo, hi, phi, dlo, dhi, prob_lo, dx) @@ -89,5 +89,5 @@ subroutine init_phi_3d(lo, hi, phi, dlo, dhi, prob_lo, dx) end do end subroutine init_phi_3d - + end module init_phi_module diff --git a/Tutorials/Basic/HeatEquation_EX2_C/Source/main.cpp b/Tutorials/Basic/HeatEquation_EX2_C/Source/main.cpp index 8a1d1804084..85ac7555ad7 100644 --- a/Tutorials/Basic/HeatEquation_EX2_C/Source/main.cpp +++ b/Tutorials/Basic/HeatEquation_EX2_C/Source/main.cpp @@ -13,9 +13,9 @@ using namespace amrex; int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -35,7 +35,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -50,19 +50,19 @@ void main_main () // Default nsteps to 0, allow us to set it to something else in the inputs file nsteps = 10; pp.query("nsteps",nsteps); - - // By default, the boundary conditions will be set to periodic, or bc_lo = bc_hi = 0. - //Other options in this program include bc_lo, bc_hi = 2 for homogeneous Neumann, or - //bc_lo, bc_hi = 3 for external Dirichlet boundary conditions. + + // By default, the boundary conditions will be set to periodic, or bc_lo = bc_hi = 0. + //Other options in this program include bc_lo, bc_hi = 2 for homogeneous Neumann, or + //bc_lo, bc_hi = 3 for external Dirichlet boundary conditions. pp.queryarr("bc_lo", bc_lo); - pp.queryarr("bc_hi", bc_hi); + pp.queryarr("bc_hi", bc_hi); } Vector is_periodic(AMREX_SPACEDIM,0); for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { - if (bc_lo[idim] == INT_DIR && bc_hi[idim] == INT_DIR){ - is_periodic[idim] = 1; - } + if (bc_lo[idim] == INT_DIR && bc_hi[idim] == INT_DIR){ + is_periodic[idim] = 1; + } } // make BoxArray and Geometry @@ -86,12 +86,12 @@ void main_main () geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 1; - + // Ncomp = number of components for each array int Ncomp = 1; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -108,40 +108,40 @@ void main_main () Vector bc(phi_old.nComp()); for (int n = 0; n < phi_old.nComp(); ++n) { - for(int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - //Internal Dirichlet Periodic Boundary conditions, or bc_lo = bc_hi = 0 - if (bc_lo[idim] == INT_DIR) { - bc[n].setLo(idim, BCType::int_dir); - } - //First Order Extrapolation for Neumann boundary conditions or bc_lo, bc_hi = 2 - else if (bc_lo[idim] == FOEXTRAP) { - bc[n].setLo(idim, BCType::foextrap); - } - //External Dirichlet Boundary Condition, or bc_lo, bc_hi = 3 - else if(bc_lo[idim] == EXT_DIR) { - bc[n].setLo(idim, BCType::ext_dir); - } - else { - amrex::Abort("Invalid bc_lo"); - } - - //Internal Dirichlet Periodic Boundary conditions, or bc_lo = bc_hi = 0 - if (bc_hi[idim] == INT_DIR) { - bc[n].setHi(idim, BCType::int_dir); - } - //First Order Extrapolation for Neumann boundary conditions or bc_lo, bc_hi = 2 - else if (bc_hi[idim] == FOEXTRAP) { - bc[n].setHi(idim, BCType::foextrap); - } - //External Dirichlet Boundary Condition, or bc_lo, bc_hi = 3 - else if(bc_hi[idim] == EXT_DIR) { - bc[n].setHi(idim, BCType::ext_dir); - } - else { - amrex::Abort("Invalid bc_hi"); - } - } + for(int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + //Internal Dirichlet Periodic Boundary conditions, or bc_lo = bc_hi = 0 + if (bc_lo[idim] == INT_DIR) { + bc[n].setLo(idim, BCType::int_dir); + } + //First Order Extrapolation for Neumann boundary conditions or bc_lo, bc_hi = 2 + else if (bc_lo[idim] == FOEXTRAP) { + bc[n].setLo(idim, BCType::foextrap); + } + //External Dirichlet Boundary Condition, or bc_lo, bc_hi = 3 + else if(bc_lo[idim] == EXT_DIR) { + bc[n].setLo(idim, BCType::ext_dir); + } + else { + amrex::Abort("Invalid bc_lo"); + } + + //Internal Dirichlet Periodic Boundary conditions, or bc_lo = bc_hi = 0 + if (bc_hi[idim] == INT_DIR) { + bc[n].setHi(idim, BCType::int_dir); + } + //First Order Extrapolation for Neumann boundary conditions or bc_lo, bc_hi = 2 + else if (bc_hi[idim] == FOEXTRAP) { + bc[n].setHi(idim, BCType::foextrap); + } + //External Dirichlet Boundary Condition, or bc_lo, bc_hi = 3 + else if(bc_hi[idim] == EXT_DIR) { + bc[n].setHi(idim, BCType::ext_dir); + } + else { + amrex::Abort("Invalid bc_hi"); + } + } } Real cfl = 0.9; @@ -157,7 +157,7 @@ void main_main () if (plot_int > 0) { int n = 0; - const std::string& pltfile = amrex::Concatenate("plt",n,5); + const std::string& pltfile = amrex::Concatenate("plt",n,5); WriteSingleLevelPlotfile(pltfile, phi_new, {"phi"}, geom, time, 0); } @@ -178,7 +178,7 @@ void main_main () // new_phi = old_phi + dt * (something) advance(phi_old, phi_new, flux, dt, geom, bc); time = time + dt; - + // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; diff --git a/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.H b/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.H index 1ae852f4d08..28d975cad9c 100644 --- a/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.H +++ b/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.H @@ -11,10 +11,10 @@ void main_main (); void advance (amrex::MultiFab& phi_old, amrex::MultiFab& phi_new, - amrex::Array& flux, - amrex::Real dt, + amrex::Array& flux, + amrex::Real dt, amrex::Geometry const& geom, - Vector const& BoundaryCondition); + Vector const& BoundaryCondition); void init_phi (amrex::MultiFab& phi_new, amrex::Geometry const& geom); #endif diff --git a/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.cpp b/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.cpp index eea2782e193..ff154052f4b 100644 --- a/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.cpp +++ b/Tutorials/Basic/HeatEquation_EX2_C/Source/myfunc.cpp @@ -6,12 +6,12 @@ void advance (MultiFab& phi_old, MultiFab& phi_new, - Array& flux, - Real dt, + Array& flux, + Real dt, Geometry const& geom, - Vector const& BoundaryCondition) + Vector const& BoundaryCondition) { - + // Fill the ghost cells of each grid from the other grids // includes periodic domain boundaries phi_old.FillBoundary(geom.periodicity()); @@ -19,12 +19,12 @@ void advance (MultiFab& phi_old, FillDomainBoundary(phi_old, geom, BoundaryCondition); const BCRec& bc = BoundaryCondition[0]; - + // // Note that this simple example is not optimized. // The following two MFIter loops could be merged // and we do not have to use flux MultiFab. - // + // // ======================================================= const Real dxinv = geom.InvCellSize(0); @@ -39,39 +39,39 @@ void advance (MultiFab& phi_old, for ( MFIter mfi(phi_old); mfi.isValid(); ++mfi ) { const Box& xbx = mfi.nodaltilebox(0); - auto const& fluxx = flux[0].array(mfi); + auto const& fluxx = flux[0].array(mfi); const Box& ybx = mfi.nodaltilebox(1); - auto const& fluxy = flux[1].array(mfi); + auto const& fluxy = flux[1].array(mfi); #if (AMREX_SPACEDIM > 2) const Box& zbx = mfi.nodaltilebox(2); - auto const& fluxz = flux[2].array(mfi); + auto const& fluxz = flux[2].array(mfi); #endif - const Box& bx = mfi.validbox(); - const Dim3 lo = lbound(bx); - const Dim3 hi = ubound(bx); - + const Box& bx = mfi.validbox(); + const Dim3 lo = lbound(bx); + const Dim3 hi = ubound(bx); + auto const& phi = phi_old.array(mfi); - amrex::ParallelFor(xbx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) - { - compute_flux_x(i,j,k,fluxx,phi,dxinv, - lo.x, hi.x, dom_lo.x, dom_hi.x, bc.lo(0), bc.hi(0)); - }); - - amrex::ParallelFor(ybx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) - { - compute_flux_y(i,j,k,fluxy,phi,dyinv, - lo.y, hi.y, dom_lo.y, dom_hi.y, bc.lo(1), bc.hi(1)); - }); + amrex::ParallelFor(xbx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + compute_flux_x(i,j,k,fluxx,phi,dxinv, + lo.x, hi.x, dom_lo.x, dom_hi.x, bc.lo(0), bc.hi(0)); + }); + + amrex::ParallelFor(ybx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + compute_flux_y(i,j,k,fluxy,phi,dyinv, + lo.y, hi.y, dom_lo.y, dom_hi.y, bc.lo(1), bc.hi(1)); + }); #if (AMREX_SPACEDIM > 2) - amrex::ParallelFor(zbx, - [=] AMREX_GPU_DEVICE (int i, int j, int k) - { - compute_flux_z(i,j,k,fluxz,phi,dzinv, - lo.z, hi.z, dom_lo.z, dom_hi.z, bc.lo(2), bc.hi(2)); - }); + amrex::ParallelFor(zbx, + [=] AMREX_GPU_DEVICE (int i, int j, int k) + { + compute_flux_z(i,j,k,fluxz,phi,dzinv, + lo.z, hi.z, dom_lo.z, dom_hi.z, bc.lo(2), bc.hi(2)); + }); #endif } diff --git a/Tutorials/Basic/HeatEquation_EX2_C/Source/mykernel.H b/Tutorials/Basic/HeatEquation_EX2_C/Source/mykernel.H index f1cef4df381..9c40505c097 100644 --- a/Tutorials/Basic/HeatEquation_EX2_C/Source/mykernel.H +++ b/Tutorials/Basic/HeatEquation_EX2_C/Source/mykernel.H @@ -21,37 +21,37 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void compute_flux_x (int i, int j, int k, amrex::Array4 const& fluxx, amrex::Array4 const& phi, amrex::Real dxinv, - int lo, int hi, int dom_lo, int dom_hi, int bc_lo, int bc_hi) + int lo, int hi, int dom_lo, int dom_hi, int bc_lo, int bc_hi) { if (lo == dom_lo && - (bc_lo == BCType::foextrap || - bc_lo == BCType::ext_dir)) + (bc_lo == BCType::foextrap || + bc_lo == BCType::ext_dir)) { - if(i == lo) - { - fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv / 0.5; - } - else - { - fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; - } + if(i == lo) + { + fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv / 0.5; + } + else + { + fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; + } } else if (hi == dom_hi && - (bc_hi == BCType::foextrap || - bc_hi == BCType::ext_dir)) + (bc_hi == BCType::foextrap || + bc_hi == BCType::ext_dir)) { - if(i == hi+1) - { - fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv / 0.5; - } - else - { - fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; - } + if(i == hi+1) + { + fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv / 0.5; + } + else + { + fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; + } } else { - fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; + fluxx(i,j,k) = (phi(i,j,k)-phi(i-1,j,k)) * dxinv; } } @@ -59,37 +59,37 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void compute_flux_y (int i, int j, int k, amrex::Array4 const& fluxy, amrex::Array4 const& phi, amrex::Real dyinv, - int lo, int hi, int dom_lo, int dom_hi, int bc_lo, int bc_hi) + int lo, int hi, int dom_lo, int dom_hi, int bc_lo, int bc_hi) { if (lo == dom_lo && - (bc_lo == BCType::foextrap || - bc_lo == BCType::ext_dir)) + (bc_lo == BCType::foextrap || + bc_lo == BCType::ext_dir)) { - if(j == lo) - { - fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv / 0.5; - } - else - { - fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; - } + if(j == lo) + { + fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv / 0.5; + } + else + { + fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; + } } else if (hi == dom_hi && - (bc_hi == BCType::foextrap || - bc_hi == BCType::ext_dir)) + (bc_hi == BCType::foextrap || + bc_hi == BCType::ext_dir)) { - if(j == hi+1) - { - fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv / 0.5; - } - else - { - fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; - } + if(j == hi+1) + { + fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv / 0.5; + } + else + { + fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; + } } else { - fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; + fluxy(i,j,k) = (phi(i,j,k)-phi(i,j-1,k)) * dyinv; } } @@ -97,39 +97,39 @@ AMREX_GPU_DEVICE AMREX_FORCE_INLINE void compute_flux_z (int i, int j, int k, amrex::Array4 const& fluxz, amrex::Array4 const& phi, amrex::Real dzinv, - int lo, int hi, int dom_lo, int dom_hi, int bc_lo, int bc_hi) + int lo, int hi, int dom_lo, int dom_hi, int bc_lo, int bc_hi) { if (lo == dom_lo && - (bc_lo == BCType::foextrap || - bc_lo == BCType::ext_dir)) + (bc_lo == BCType::foextrap || + bc_lo == BCType::ext_dir)) { - if(k == lo) - { - fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv / 0.5; - } - else - { - fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; - } + if(k == lo) + { + fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv / 0.5; + } + else + { + fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; + } } else if (hi == dom_hi && - (bc_hi == BCType::foextrap || - bc_hi == BCType::ext_dir)) + (bc_hi == BCType::foextrap || + bc_hi == BCType::ext_dir)) { - if(k == hi+1) - { - fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv / 0.5; - } - else - { - fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; - } + if(k == hi+1) + { + fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv / 0.5; + } + else + { + fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; + } } else { - fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; + fluxz(i,j,k) = (phi(i,j,k)-phi(i,j,k-1)) * dzinv; } -} +} AMREX_GPU_DEVICE AMREX_FORCE_INLINE void update_phi (int i, int j, int k, diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance.cpp b/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance.cpp index da35a2ee645..a89cda5b1f1 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance.cpp +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance.cpp @@ -8,8 +8,8 @@ using namespace amrex; void advance (MultiFab& phi_old, MultiFab& phi_new, - Array& flux, - Real dt, + Array& flux, + Real dt, const Geometry& geom, const Vector& bc) { @@ -26,7 +26,7 @@ void advance (MultiFab& phi_old, // Note that this simple example is not optimized. // The following two MFIter loops could be merged // and we do not have to use flux MultiFab. - // + // const Box& domain_bx = geom.Domain(); @@ -40,23 +40,23 @@ void advance (MultiFab& phi_old, BL_TO_FORTRAN_ANYD(phi_old[mfi]), BL_TO_FORTRAN_ANYD(flux[0][mfi]), BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM == 3) BL_TO_FORTRAN_ANYD(flux[2][mfi]), #endif dx, bc[0].data()); } - + // Advance the solution one grid at a time for ( MFIter mfi(phi_old); mfi.isValid(); ++mfi ) { const Box& bx = mfi.validbox(); - + update_phi(BL_TO_FORTRAN_BOX(bx), BL_TO_FORTRAN_ANYD(phi_old[mfi]), BL_TO_FORTRAN_ANYD(phi_new[mfi]), BL_TO_FORTRAN_ANYD(flux[0][mfi]), BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM == 3) BL_TO_FORTRAN_ANYD(flux[2][mfi]), #endif dx, &dt); diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance_3d.f90 b/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance_3d.f90 index 189c76e9fef..86d17a59c39 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance_3d.f90 +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/advance_3d.f90 @@ -15,7 +15,7 @@ subroutine compute_flux (lo, hi, domlo, domhi, phi, philo, phihi, & real(amrex_real), intent(inout) :: fluxz( fzlo(1): fzhi(1), fzlo(2): fzhi(2), fzlo(3): fzhi(3)) real(amrex_real), intent(in) :: dx(3) integer, intent(in) :: bc(3,2,1) ! (dim,lohi,ncomp) - + ! local variables integer i,j,k diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_2d.f90 b/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_2d.f90 index 02a4c036cff..ea934ded810 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_2d.f90 +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_2d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(2), hi(2), philo(2), phihi(2) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in ) :: dx(2) - real(amrex_real), intent(in ) :: prob_lo(2) - real(amrex_real), intent(in ) :: prob_hi(2) + real(amrex_real), intent(in ) :: dx(2) + real(amrex_real), intent(in ) :: prob_lo(2) + real(amrex_real), intent(in ) :: prob_hi(2) integer :: i,j double precision :: x,y,r2 diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_3d.f90 b/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_3d.f90 index ec281d22df5..9fc031ee221 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_3d.f90 +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/init_phi_3d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(3), hi(3), philo(3), phihi(3) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2),philo(3):phihi(3)) - real(amrex_real), intent(in ) :: dx(3) - real(amrex_real), intent(in ) :: prob_lo(3) - real(amrex_real), intent(in ) :: prob_hi(3) + real(amrex_real), intent(in ) :: dx(3) + real(amrex_real), intent(in ) :: prob_lo(3) + real(amrex_real), intent(in ) :: prob_hi(3) integer :: i,j,k double precision :: x,y,z,r2 diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/main.cpp b/Tutorials/Basic/HeatEquation_EX2_CF/Source/main.cpp index 6dc1ecdcd07..7c31371ade1 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/main.cpp +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/main.cpp @@ -11,9 +11,9 @@ using namespace amrex; int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -33,7 +33,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -83,15 +83,15 @@ void main_main () geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 1; - + // Ncomp = number of components for each array int Ncomp = 1; // time = starting time in the simulation Real time = 0.0; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -116,7 +116,7 @@ void main_main () { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - + // lo-side BCs if (bc_lo[idim] == INT_DIR) { bc[n].setLo(idim, BCType::int_dir); // periodic uses "internal Dirichlet" @@ -179,9 +179,9 @@ void main_main () MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); // new_phi = old_phi + dt * (something) - advance(phi_old, phi_new, flux, dt, geom, bc); + advance(phi_old, phi_new, flux, dt, geom, bc); time = time + dt; - + // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc.H b/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc.H index c3ff470c901..c0b7b712f22 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc.H +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc.H @@ -10,9 +10,9 @@ void main_main (); void advance (amrex::MultiFab& phi_old, amrex::MultiFab& phi_new, - amrex::Array& flux, - amrex::Real dt, - const amrex::Geometry& geom, + amrex::Array& flux, + amrex::Real dt, + const amrex::Geometry& geom, const amrex::Vector& bc); #endif diff --git a/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc_F.H b/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc_F.H index 79960e0d080..33145dfc5fb 100644 --- a/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc_F.H +++ b/Tutorials/Basic/HeatEquation_EX2_CF/Source/myfunc_F.H @@ -11,13 +11,13 @@ extern "C" void init_phi(const int* lo, const int* hi, amrex_real* data, const int* dlo, const int* dhi, const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi); - + void compute_flux(const int* lo, const int* hi, const int* domlo, const int* domhi, amrex_real* phi, const int* philo, const int* phihi, amrex_real* fluxx, const int* fxlo, const int* fxhi, amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) +#if (BL_SPACEDIM == 3) amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif const amrex_real* dx, const int*); @@ -27,7 +27,7 @@ extern "C" amrex_real* pnew, const int* pnlo, const int* pnhi, amrex_real* fluxx, const int* fxlo, const int* fxhi, amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) +#if (BL_SPACEDIM == 3) amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif const amrex_real* dx, const amrex_real* dt); diff --git a/Tutorials/Basic/HeatEquation_EX3_C/Source/advance.cpp b/Tutorials/Basic/HeatEquation_EX3_C/Source/advance.cpp index 1fdbf7691c3..bf56e9e86b8 100644 --- a/Tutorials/Basic/HeatEquation_EX3_C/Source/advance.cpp +++ b/Tutorials/Basic/HeatEquation_EX3_C/Source/advance.cpp @@ -13,8 +13,8 @@ void advance (MultiFab& phi_old, MultiFab& phi_new, Real dt, const Geometry& geom, - const BoxArray& grids, - const DistributionMapping& dmap, + const BoxArray& grids, + const DistributionMapping& dmap, const Vector& bc) { /* @@ -34,14 +34,14 @@ void advance (MultiFab& phi_old, // Fill non-periodic physical boundaries FillDomainBoundary(phi_old, geom, bc); - + // assorment of solver and parallization options and parameters // see AMReX_MLLinOp.H for the defaults, accessors, and mutators LPInfo info; // Implicit solve using MLABecLaplacian class MLABecLaplacian mlabec({geom}, {grids}, {dmap}, info); - + // order of stencil int linop_maxorder = 2; mlabec.setMaxOrder(linop_maxorder); @@ -51,43 +51,43 @@ void advance (MultiFab& phi_old, std::array bc_lo; std::array bc_hi; - for (int n = 0; n < phi_old.nComp(); ++n) + for (int n = 0; n < phi_old.nComp(); ++n) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - // lo-side BCs - if (bc[n].lo(idim) == BCType::int_dir) { - bc_lo[idim] = LinOpBCType::Periodic; - } - else if (bc[n].lo(idim) == BCType::foextrap) { - bc_lo[idim] = LinOpBCType::Neumann; - } - else if (bc[n].lo(idim) == BCType::ext_dir) { - bc_lo[idim] = LinOpBCType::Dirichlet; - } - else { - amrex::Abort("Invalid bc_lo"); - } - - // hi-side BCs - if (bc[n].hi(idim) == BCType::int_dir) { - bc_hi[idim] = LinOpBCType::Periodic; - } - else if (bc[n].hi(idim) == BCType::foextrap) { - bc_hi[idim] = LinOpBCType::Neumann; - } - else if (bc[n].hi(idim) == BCType::ext_dir) { - bc_hi[idim] = LinOpBCType::Dirichlet; - } - else { - amrex::Abort("Invalid bc_hi"); - } - } + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + // lo-side BCs + if (bc[n].lo(idim) == BCType::int_dir) { + bc_lo[idim] = LinOpBCType::Periodic; + } + else if (bc[n].lo(idim) == BCType::foextrap) { + bc_lo[idim] = LinOpBCType::Neumann; + } + else if (bc[n].lo(idim) == BCType::ext_dir) { + bc_lo[idim] = LinOpBCType::Dirichlet; + } + else { + amrex::Abort("Invalid bc_lo"); + } + + // hi-side BCs + if (bc[n].hi(idim) == BCType::int_dir) { + bc_hi[idim] = LinOpBCType::Periodic; + } + else if (bc[n].hi(idim) == BCType::foextrap) { + bc_hi[idim] = LinOpBCType::Neumann; + } + else if (bc[n].hi(idim) == BCType::ext_dir) { + bc_hi[idim] = LinOpBCType::Dirichlet; + } + else { + amrex::Abort("Invalid bc_hi"); + } + } } // tell the solver what the domain boundary conditions are mlabec.setDomainBC(bc_lo, bc_hi); - + // set the boundary conditions mlabec.setLevelBC(0, &phi_old); @@ -102,16 +102,16 @@ void advance (MultiFab& phi_old, // fill in the acoef MultiFab and load this into the solver acoef.setVal(1.0); mlabec.setACoeffs(0, acoef); - + // bcoef lives on faces so we make an array of face-centered MultiFabs // then we will in face_bcoef MultiFabs and load them into the solver. std::array face_bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - const BoxArray& ba = amrex::convert(acoef.boxArray(), - IntVect::TheDimensionVector(idim)); - face_bcoef[idim].define(ba, acoef.DistributionMap(), 1, 0); - face_bcoef[idim].setVal(dt); + const BoxArray& ba = amrex::convert(acoef.boxArray(), + IntVect::TheDimensionVector(idim)); + face_bcoef[idim].define(ba, acoef.DistributionMap(), 1, 0); + face_bcoef[idim].setVal(dt); } mlabec.setBCoeffs(0, amrex::GetArrOfConstPtrs(face_bcoef)); diff --git a/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_2d.f90 b/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_2d.f90 index 02a4c036cff..ea934ded810 100644 --- a/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_2d.f90 +++ b/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_2d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(2), hi(2), philo(2), phihi(2) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in ) :: dx(2) - real(amrex_real), intent(in ) :: prob_lo(2) - real(amrex_real), intent(in ) :: prob_hi(2) + real(amrex_real), intent(in ) :: dx(2) + real(amrex_real), intent(in ) :: prob_lo(2) + real(amrex_real), intent(in ) :: prob_hi(2) integer :: i,j double precision :: x,y,r2 diff --git a/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_3d.f90 b/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_3d.f90 index ec281d22df5..9fc031ee221 100644 --- a/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_3d.f90 +++ b/Tutorials/Basic/HeatEquation_EX3_C/Source/init_phi_3d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(3), hi(3), philo(3), phihi(3) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2),philo(3):phihi(3)) - real(amrex_real), intent(in ) :: dx(3) - real(amrex_real), intent(in ) :: prob_lo(3) - real(amrex_real), intent(in ) :: prob_hi(3) + real(amrex_real), intent(in ) :: dx(3) + real(amrex_real), intent(in ) :: prob_lo(3) + real(amrex_real), intent(in ) :: prob_hi(3) integer :: i,j,k double precision :: x,y,z,r2 diff --git a/Tutorials/Basic/HeatEquation_EX3_C/Source/main.cpp b/Tutorials/Basic/HeatEquation_EX3_C/Source/main.cpp index 1d34aa7dc0d..5d3b0067c54 100644 --- a/Tutorials/Basic/HeatEquation_EX3_C/Source/main.cpp +++ b/Tutorials/Basic/HeatEquation_EX3_C/Source/main.cpp @@ -11,9 +11,9 @@ using namespace amrex; int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -33,7 +33,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -83,15 +83,15 @@ void main_main () geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 1; - + // Ncomp = number of components for each array int Ncomp = 1; // time = starting time in the simulation Real time = 0.0; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -116,13 +116,13 @@ void main_main () { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { - + // lo-side BCs if (bc_lo[idim] == INT_DIR) { bc[n].setLo(idim, BCType::int_dir); // periodic uses "internal Dirichlet" } else if (bc_lo[idim] == FOEXTRAP) { - bc[n].setLo(idim, BCType::foextrap); // first-order extrapolation + bc[n].setLo(idim, BCType::foextrap); // first-order extrapolation } else if (bc_lo[idim] == EXT_DIR) { bc[n].setLo(idim, BCType::ext_dir); // external Dirichlet @@ -171,10 +171,10 @@ void main_main () { MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); - // new_phi = (I-dt)^{-1} * old_phi + dt - advance(phi_old, phi_new, dt, geom, ba, dm, bc); + // new_phi = (I-dt)^{-1} * old_phi + dt + advance(phi_old, phi_new, dt, geom, ba, dm, bc); time = time + dt; - + // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; diff --git a/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc.H b/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc.H index b747dc76653..dab727cdcdb 100644 --- a/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc.H +++ b/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc.H @@ -10,10 +10,10 @@ void main_main (); void advance (amrex::MultiFab& phi_old, amrex::MultiFab& phi_new, - amrex::Real dt, - const amrex::Geometry& geom, - const amrex::BoxArray& grids, - const amrex::DistributionMapping& dmap, + amrex::Real dt, + const amrex::Geometry& geom, + const amrex::BoxArray& grids, + const amrex::DistributionMapping& dmap, const amrex::Vector& bc); #endif diff --git a/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc_F.H b/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc_F.H index b96073974c5..cbc3b6c83b4 100644 --- a/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc_F.H +++ b/Tutorials/Basic/HeatEquation_EX3_C/Source/myfunc_F.H @@ -11,7 +11,7 @@ extern "C" void init_phi(const int* lo, const int* hi, amrex_real* data, const int* dlo, const int* dhi, const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi); - + #ifdef __cplusplus } #endif diff --git a/Tutorials/Basic/main_C/main.cpp b/Tutorials/Basic/main_C/main.cpp index 31970b3d27b..c664a40aa6e 100644 --- a/Tutorials/Basic/main_C/main.cpp +++ b/Tutorials/Basic/main_C/main.cpp @@ -32,7 +32,7 @@ int main(int argc, char* argv[]) // We choose to call MPI_Init ourselves, instead of letting AMRex do it. // Then we are going to pass a communicatior, MPI_COMM_WORLD, (or another) to AMReX. MPI_Init(&argc, &argv); - + bool build_parm_parse_from_command_line = false; amrex::Initialize(argc, argv, build_parm_parse_from_command_line, MPI_COMM_WORLD, add_parameters); // last three arguments are optional @@ -58,14 +58,14 @@ void add_parameters () pp.add("an_int_scalar", 2); // integer scalar: an_int_scalar pp.add("a_bool_scalar", true); // logical scalar: a_bool_scalar pp.addarr("a_real_array", // real array: a_real_array - std::vector{1.,2.,3.}); + std::vector{1.,2.,3.}); } { // prefix "a_prefix" amrex::ParmParse pp("a_prefix"); pp.addarr("an_int_array", // integer array: a_prefix.an_int_array - std::vector{2, 3, 4}); + std::vector{2, 3, 4}); amrex::Real x = 3.14; pp.add("a_real_scalar", x); // real scalar : a_prefix.a_real_scalar pp.add("a_string", std::string{"vonNeumann"}); // string: a_prefix.a_string diff --git a/Tutorials/Basic/main_F/main.F90 b/Tutorials/Basic/main_F/main.F90 index e2630dadeac..9610db042e2 100644 --- a/Tutorials/Basic/main_F/main.F90 +++ b/Tutorials/Basic/main_F/main.F90 @@ -50,7 +50,7 @@ end subroutine add_parameters call test_parameters() ! ... - + call amrex_finalize() call mpi_finalize(ierr) ! We have to call this because we called MPI_Init. @@ -88,7 +88,7 @@ end subroutine add_parameters subroutine test_parameters () use amrex_base_module implicit none - + type(amrex_parmparse) :: pp integer :: i integer, allocatable :: ia(:) diff --git a/Tutorials/Blueprint/AssignMultiLevelDensity/main.cpp b/Tutorials/Blueprint/AssignMultiLevelDensity/main.cpp index 576b959c253..e2490c28f7d 100644 --- a/Tutorials/Blueprint/AssignMultiLevelDensity/main.cpp +++ b/Tutorials/Blueprint/AssignMultiLevelDensity/main.cpp @@ -27,9 +27,9 @@ struct TestParams { void test_assign_density(TestParams& parms) { - + int nlevs = parms.nlevs; - + RealBox real_box; for (int n = 0; n < BL_SPACEDIM; n++) { real_box.setLo(n, 0.0); @@ -42,9 +42,9 @@ void test_assign_density(TestParams& parms) fine_box.setLo(n,0.25); fine_box.setHi(n,0.75); } - - IntVect domain_lo(D_DECL(0 , 0, 0)); - IntVect domain_hi(D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); + + IntVect domain_lo(D_DECL(0 , 0, 0)); + IntVect domain_hi(D_DECL(parms.nx - 1, parms.ny - 1, parms.nz-1)); const Box domain(domain_lo, domain_hi); // Define the refinement ratio @@ -57,28 +57,28 @@ void test_assign_density(TestParams& parms) for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = 1; - // This defines a Geometry object which is useful for writing the plotfiles + // This defines a Geometry object which is useful for writing the plotfiles Vector geom(nlevs); geom[0].define(domain, &real_box, CoordSys::cartesian, is_per); for (int lev = 1; lev < nlevs; lev++) { - geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), - &real_box, CoordSys::cartesian, is_per); + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_per); } Vector ba(nlevs); ba[0].define(domain); - + // Now we make the refined level be the center eighth of the domain if (nlevs > 1) { int n_fine = parms.nx*rr[0]; - IntVect refined_lo(D_DECL(n_fine/4,n_fine/4,n_fine/4)); + IntVect refined_lo(D_DECL(n_fine/4,n_fine/4,n_fine/4)); IntVect refined_hi(D_DECL(3*n_fine/4-1,3*n_fine/4-1,3*n_fine/4-1)); // Build a box for the level 1 domain Box refined_patch(refined_lo, refined_hi); ba[1].define(refined_patch); } - + // break the BoxArrays at both levels into max_grid_size^3 boxes for (int lev = 0; lev < nlevs; lev++) { ba[lev].maxSize(parms.max_grid_size); @@ -137,16 +137,16 @@ void test_assign_density(TestParams& parms) outputMF[lev] = density[lev].get(); outputRR[lev] = IntVect(D_DECL(2, 2, 2)); } - - WriteMultiLevelPlotfile("plt00000", output_levs, outputMF, + + WriteMultiLevelPlotfile("plt00000", output_levs, outputMF, varnames, geom, 0.0, level_steps, outputRR); myPC.Checkpoint("plt00000", "particle0", true, particle_varnames); - + /////////////////////////////////////////////////////////////////////////// // Wrap our AMReX Mesh into a Conduit Mesh Blueprint Tree /////////////////////////////////////////////////////////////////////////// conduit::Node bp_mesh; - MultiLevelToBlueprint( output_levs, + MultiLevelToBlueprint( output_levs, outputMF, varnames, geom, @@ -161,16 +161,16 @@ void test_assign_density(TestParams& parms) particle_int_varnames, bp_particles); /////////////////////////////////////////////////////////////////////////// - // Save the Blueprint Mesh to a set of files that we can - // view in VisIt. + // Save the Blueprint Mesh to a set of files that we can + // view in VisIt. // (For debugging and to demonstrate how to do this w/o Ascent) /////////////////////////////////////////////////////////////////////////// WriteBlueprintFiles(bp_mesh,"bp_example_"); - + /////////////////////////////////////////////////////////////////// // Render with Ascent /////////////////////////////////////////////////////////////////// - + // add a scene with a pseudocolor plot Node scenes; scenes["s1/plots/p1/type"] = "pseudocolor"; @@ -189,10 +189,10 @@ void test_assign_density(TestParams& parms) Ascent ascent; ascent.open(); - + ascent.publish(bp_mesh); ascent.execute(actions); - + // add a scene with a pseudocolor plot scenes.reset(); scenes["s1/plots/p1/type"] = "pseudocolor"; @@ -207,21 +207,21 @@ void test_assign_density(TestParams& parms) add_act2["scenes"] = scenes; actions.append()["action"] = "execute"; actions.append()["action"] = "reset"; - + ascent.publish(bp_particles); ascent.execute(actions); - + ascent.close(); } int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + ParmParse pp; - + TestParams parms; - + pp.get("nx", parms.nx); pp.get("ny", parms.ny); pp.get("nz", parms.nz); @@ -230,10 +230,10 @@ int main(int argc, char* argv[]) pp.get("nppc", parms.nppc); if (parms.nppc < 1 && ParallelDescriptor::IOProcessor()) amrex::Abort("Must specify at least one particle per cell"); - + parms.verbose = false; pp.query("verbose", parms.verbose); - + if (parms.verbose && ParallelDescriptor::IOProcessor()) { std::cout << std::endl; std::cout << "Number of particles per cell : "; @@ -243,8 +243,8 @@ int main(int argc, char* argv[]) std::cout << parms.nlevs << std::endl; std::cout << parms.nx << " " << parms.ny << " " << parms.nz << std::endl; } - + test_assign_density(parms); - + amrex::Finalize(); } diff --git a/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.H b/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.H index fef695f0e1c..64af9f82c39 100644 --- a/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.H +++ b/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.H @@ -18,8 +18,8 @@ struct IntData enum { sorted, i, - j, - k, + j, + k, ncomps }; }; @@ -30,7 +30,7 @@ class CellSortedParticleContainer public: using MyParIter = amrex::ParIter; - + CellSortedParticleContainer (const amrex::Geometry & a_geom, const amrex::DistributionMapping & a_dmap, const amrex::BoxArray & a_ba); @@ -48,11 +48,11 @@ public: int numWrongCell(); void visitAllParticles(); - + protected: void UpdateCellVectors(); - + void UpdateFortranStructures(); // used to store vectors of particle indices on a cell-by-cell basis @@ -68,7 +68,7 @@ protected: amrex::DistributionMapping::RefID m_DMRef; private: - + void correctCellVectors(int old_index, int new_index, int grid, const ParticleType& p) override; }; diff --git a/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.cpp b/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.cpp index 720e3b1b88f..09ae4658def 100644 --- a/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.cpp +++ b/Tutorials/Blueprint/CellSortedParticles/CellSortedPC.cpp @@ -11,31 +11,31 @@ using namespace amrex; namespace { - + void get_position_unit_cell(Real* r, const IntVect& nppc, int i_part) { int nx = nppc[0]; int ny = nppc[1]; int nz = nppc[2]; - + int ix_part = i_part/(ny * nz); int iy_part = (i_part % (ny * nz)) % ny; int iz_part = (i_part % (ny * nz)) / ny; - + r[0] = (0.5+ix_part)/nx; r[1] = (0.5+iy_part)/ny; - r[2] = (0.5+iz_part)/nz; + r[2] = (0.5+iz_part)/nz; } - + void get_gaussian_random_momentum(Real* u, Real u_mean, Real u_std) { Real ux_th = amrex::RandomNormal(0.0, u_std); Real uy_th = amrex::RandomNormal(0.0, u_std); Real uz_th = amrex::RandomNormal(0.0, u_std); - + u[0] = u_mean + ux_th; u[1] = u_mean + uy_th; u[2] = u_mean + uz_th; - } + } } CellSortedParticleContainer:: @@ -51,14 +51,14 @@ CellSortedParticleContainer:: InitParticles(const IntVect& a_num_particles_per_cell) { BL_PROFILE("CellSortedParticleContainer::InitParticles"); - + const int lev = 0; const Geometry& geom = Geom(lev); const Real* dx = geom.CellSize(); const Real* plo = geom.ProbLo(); - const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], - *a_num_particles_per_cell[1], + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], + *a_num_particles_per_cell[1], *a_num_particles_per_cell[2]); for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) @@ -74,28 +74,28 @@ InitParticles(const IntVect& a_num_particles_per_cell) for (int i_part=0; i_partIndex(p, lev) == iv); - + particle_tile.push_back(p); } } @@ -106,7 +106,7 @@ void CellSortedParticleContainer::UpdateCellVectors() { BL_PROFILE("CellSortedParticleContainer::UpdateCellVectors"); - + const int lev = 0; bool needs_update = false; @@ -116,7 +116,7 @@ CellSortedParticleContainer::UpdateCellVectors() m_vectors_initialized = true; needs_update = true; } - else if ((m_BARef != this->ParticleBoxArray(lev).getRefID()) or + else if ((m_BARef != this->ParticleBoxArray(lev).getRefID()) or (m_DMRef != this->ParticleDistributionMap(lev).getRefID())) { // the grids have changed, so we must update @@ -124,14 +124,14 @@ CellSortedParticleContainer::UpdateCellVectors() m_DMRef = this->ParticleDistributionMap(lev).getRefID(); needs_update = true; } - + if (not needs_update) return; // clear old data m_cell_vectors.clear(); m_vector_size.clear(); m_vector_ptrs.clear(); - + // allocate storage for cell vectors. NOTE - do not tile this loop for(MFIter mfi = MakeMFIter(lev, false); mfi.isValid(); ++mfi) { @@ -145,7 +145,7 @@ CellSortedParticleContainer::UpdateCellVectors() // insert particles into vectors - this can be tiled #ifdef AMREX_USE_OMP #pragma omp parallel -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); @@ -161,7 +161,7 @@ CellSortedParticleContainer::UpdateCellVectors() m_cell_vectors[pti.index()](iv).push_back(pindex + 1); } } - + UpdateFortranStructures(); } @@ -169,7 +169,7 @@ void CellSortedParticleContainer::UpdateFortranStructures() { BL_PROFILE("CellSortedParticleContainer::UpdateFortranStructures"); - + const int lev = 0; #ifdef AMREX_USE_OMP @@ -193,7 +193,7 @@ CellSortedParticleContainer::MoveParticles() BL_PROFILE("CellSortedParticleContainer::MoveParticles()"); UpdateCellVectors(); - + const int lev = 0; const Real* dx = Geom(lev).CellSize(); const Real* plo = Geom(lev).ProbLo(); @@ -207,11 +207,11 @@ CellSortedParticleContainer::MoveParticles() const int grid_id = pti.index(); const int tile_id = pti.LocalTileIndex(); const Box& tile_box = pti.tilebox(); - + auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; auto& particles = particle_tile.GetArrayOfStructs(); const int np = particles.numParticles(); - + move_particles(particles.data(), &np, tile_box.loVect(), tile_box.hiVect(), m_vector_ptrs[grid_id].dataPtr(), @@ -234,7 +234,7 @@ void CellSortedParticleContainer::ReBin() { BL_PROFILE("CellSortedParticleContainer::ReBin()"); - + const int lev = 0; #ifdef AMREX_USE_OMP @@ -250,8 +250,8 @@ CellSortedParticleContainer::ReBin() const int np = particles.numParticles(); for(int pindex = 0; pindex < np; ++pindex) { - ParticleType& p = particles[pindex]; - if (p.idata(IntData::sorted)) continue; + ParticleType& p = particles[pindex]; + if (p.idata(IntData::sorted)) continue; const IntVect& iv = this->Index(p, lev); p.idata(IntData::sorted) = 1; p.idata(IntData::i) = iv[0]; @@ -266,8 +266,8 @@ CellSortedParticleContainer::ReBin() } void -CellSortedParticleContainer::correctCellVectors(int old_index, int new_index, - int grid, const ParticleType& p) +CellSortedParticleContainer::correctCellVectors(int old_index, int new_index, + int grid, const ParticleType& p) { if (not p.idata(IntData::sorted)) return; IntVect iv(p.idata(IntData::i), p.idata(IntData::j), p.idata(IntData::k)); @@ -287,19 +287,19 @@ CellSortedParticleContainer::SumCellVectors() const int lev = 0; int np = 0; - + #ifdef AMREX_USE_OMP #pragma omp parallel reduction(+:np) -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& tile_box = pti.tilebox(); for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) - { - np += m_vector_size[pti.index()](iv); + { + np += m_vector_size[pti.index()](iv); } } - + ParallelDescriptor::ReduceIntSum(np,ParallelDescriptor::IOProcessorNumber()); return np; } @@ -309,10 +309,10 @@ CellSortedParticleContainer::numUnsorted() { const int lev = 0; int num_unsorted = 0; - + #ifdef AMREX_USE_OMP #pragma omp parallel reduction(+:num_unsorted) -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); @@ -324,7 +324,7 @@ CellSortedParticleContainer::numUnsorted() } } } - + ParallelDescriptor::ReduceIntSum(num_unsorted, ParallelDescriptor::IOProcessorNumber()); return num_unsorted; } @@ -334,10 +334,10 @@ CellSortedParticleContainer::numWrongCell() { const int lev = 0; int num_wrong = 0; - + #ifdef AMREX_USE_OMP #pragma omp parallel reduction(+:num_wrong) -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); @@ -350,7 +350,7 @@ CellSortedParticleContainer::numWrongCell() } } } - + ParallelDescriptor::ReduceIntSum(num_wrong, ParallelDescriptor::IOProcessorNumber()); return num_wrong; } @@ -361,7 +361,7 @@ CellSortedParticleContainer::visitAllParticles() const int lev = 0; if (not m_vectors_initialized) return; - + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); @@ -369,14 +369,14 @@ CellSortedParticleContainer::visitAllParticles() Vector times_visited(np, 0); const Box& tile_box = pti.tilebox(); for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) - { + { auto& cell_vector = m_cell_vectors[grid](iv); for (int i = 0; i < static_cast(cell_vector.size()); ++i) { times_visited[cell_vector[i]-1] += 1; } } amrex::Print() << *std::min_element(times_visited.begin(), times_visited.end()) << std::endl; - amrex::Print() << *std::max_element(times_visited.begin(), times_visited.end()) << std::endl; + amrex::Print() << *std::max_element(times_visited.begin(), times_visited.end()) << std::endl; } } diff --git a/Tutorials/Blueprint/CellSortedParticles/cell_sorted_3d.F90 b/Tutorials/Blueprint/CellSortedParticles/cell_sorted_3d.F90 index 441c9a3228a..a54374623e3 100644 --- a/Tutorials/Blueprint/CellSortedParticles/cell_sorted_3d.F90 +++ b/Tutorials/Blueprint/CellSortedParticles/cell_sorted_3d.F90 @@ -1,12 +1,12 @@ module cell_sorted_particle_module use amrex_fort_module, only: amrex_real, amrex_particle_real use iso_c_binding , only: c_int - + implicit none private - + public particle_t, remove_particle_from_cell - + type, bind(C) :: particle_t real(amrex_particle_real) :: pos(3) !< Position real(amrex_particle_real) :: vel(3) !< Position @@ -19,33 +19,33 @@ module cell_sorted_particle_module end type particle_t contains - + subroutine remove_particle_from_cell(cell_parts, cell_np, new_np, i) - + use iso_c_binding, only: c_int - + implicit none - + integer(c_int), intent(in ) :: cell_np integer(c_int), intent(inout) :: cell_parts(cell_np) integer(c_int), intent(inout) :: new_np - integer(c_int), intent(in ) :: i + integer(c_int), intent(in ) :: i cell_parts(i) = cell_parts(new_np) new_np = new_np - 1 - + end subroutine remove_particle_from_cell - + end module cell_sorted_particle_module subroutine move_particles(particles, np, lo, hi, & cell_part_ids, cell_part_cnt, clo, chi, plo, dx, dt) & bind(c,name="move_particles") - + use amrex_fort_module, only: amrex_real use iso_c_binding, only: c_ptr, c_int, c_f_pointer use cell_sorted_particle_module, only: particle_t, remove_particle_from_cell - + implicit none type(particle_t), intent(inout), target :: particles(np) @@ -57,7 +57,7 @@ subroutine move_particles(particles, np, lo, hi, & real(amrex_real), intent(in) :: plo(3) real(amrex_real), intent(in) :: dx(3) real(amrex_real), intent(in) :: dt - + integer :: i, j, k, p, cell_np, new_np integer :: cell(3) integer(c_int), pointer :: cell_parts(:) @@ -65,7 +65,7 @@ subroutine move_particles(particles, np, lo, hi, & real(amrex_real) inv_dx(3) inv_dx = 1.d0/dx - + do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -76,25 +76,25 @@ subroutine move_particles(particles, np, lo, hi, & p = 1 do while (p <= new_np) part => particles(cell_parts(p)) - + ! move the particle in a straight line part%pos = part%pos + dt*part%vel ! if it has changed cells, remove from vector. ! otherwise continue - cell = floor((part%pos - plo)*inv_dx) + cell = floor((part%pos - plo)*inv_dx) if ((cell(1) /= i) .or. (cell(2) /= j) .or. (cell(3) /= k)) then part%sorted = 0 - call remove_particle_from_cell(cell_parts, cell_np, new_np, p) + call remove_particle_from_cell(cell_parts, cell_np, new_np, p) else p = p + 1 end if end do cell_part_cnt(i,j,k) = new_np - + end do end do end do - + end subroutine move_particles diff --git a/Tutorials/Blueprint/CellSortedParticles/cell_sorted_F.H b/Tutorials/Blueprint/CellSortedParticles/cell_sorted_F.H index 58158b3bbae..fc16e82b5f8 100644 --- a/Tutorials/Blueprint/CellSortedParticles/cell_sorted_F.H +++ b/Tutorials/Blueprint/CellSortedParticles/cell_sorted_F.H @@ -19,7 +19,7 @@ extern "C" const amrex_real* plo, const amrex_real* dx, const amrex_real* dt); - + #ifdef __cplusplus }; #endif diff --git a/Tutorials/Blueprint/CellSortedParticles/main.cpp b/Tutorials/Blueprint/CellSortedParticles/main.cpp index c4f2144c409..d7ea554a95c 100644 --- a/Tutorials/Blueprint/CellSortedParticles/main.cpp +++ b/Tutorials/Blueprint/CellSortedParticles/main.cpp @@ -37,23 +37,23 @@ void test_cell_sorted(const TestParams& parms) real_box.setLo(n, 0.0); real_box.setHi(n, 1.0); } - - IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); IntVect domain_hi(AMREX_D_DECL(parms.ncell[0]-1, parms.ncell[1]-1, - parms.ncell[2]-1)); + parms.ncell[2]-1)); const Box domain(domain_lo, domain_hi); - + int coord = 0; int is_per[BL_SPACEDIM]; - for (int i = 0; i < BL_SPACEDIM; i++) - is_per[i] = 1; + for (int i = 0; i < BL_SPACEDIM; i++) + is_per[i] = 1; Geometry geom(domain, &real_box, coord, is_per); - + BoxArray ba(domain); ba.maxSize(parms.max_grid_size); DistributionMapping dm(ba); - + amrex::Print() << "Initializing particles... "; CellSortedParticleContainer particles(geom, dm, ba); @@ -78,7 +78,7 @@ void test_cell_sorted(const TestParams& parms) int_comp_names.push_back("i"); int_comp_names.push_back("j"); int_comp_names.push_back("k"); - + // Create an instance of Ascent Ascent ascent; Node open_opts; @@ -115,7 +115,7 @@ void test_cell_sorted(const TestParams& parms) Node &rset = actions.append(); rset["action"] = "reset"; - /////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////// // Render the initial state /////////////////////////////////////////////////////////////////// ascent.publish(bp_mesh); @@ -123,7 +123,7 @@ void test_cell_sorted(const TestParams& parms) BL_PROFILE_VAR_STOP(blp_init); - + amrex::Print() << "Starting main loop... " << std::endl; BL_PROFILE_VAR_START(blp_evolve); @@ -151,7 +151,7 @@ void test_cell_sorted(const TestParams& parms) ascent.execute(actions); // shutdown ascent ascent.close(); - + amrex::Print() << "Done. " << std::endl; BL_PROFILE_VAR_STOP(blp_evolve); @@ -160,7 +160,7 @@ void test_cell_sorted(const TestParams& parms) int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + amrex::InitRandom(451); ParmParse pp; diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance.cpp b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance.cpp index e157d6dbaf9..294392725b0 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance.cpp +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance.cpp @@ -4,8 +4,8 @@ void advance (MultiFab& phi_old, MultiFab& phi_new, - std::array& flux, - Real dt, + std::array& flux, + Real dt, const Geometry& geom) { // Fill the ghost cells of each grid from the other grids @@ -22,7 +22,7 @@ void advance (MultiFab& phi_old, // Note that this simple example is not optimized. // The following two MFIter loops could be merged // and we do not have to use flux MultiFab. - // + // const Box& domain_bx = geom.Domain(); @@ -36,23 +36,23 @@ void advance (MultiFab& phi_old, BL_TO_FORTRAN_ANYD(phi_old[mfi]), BL_TO_FORTRAN_ANYD(flux[0][mfi]), BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM == 3) BL_TO_FORTRAN_ANYD(flux[2][mfi]), #endif dx); } - + // Advance the solution one grid at a time for ( MFIter mfi(phi_old); mfi.isValid(); ++mfi ) { const Box& bx = mfi.validbox(); - + update_phi(BL_TO_FORTRAN_BOX(bx), BL_TO_FORTRAN_ANYD(phi_old[mfi]), BL_TO_FORTRAN_ANYD(phi_new[mfi]), BL_TO_FORTRAN_ANYD(flux[0][mfi]), BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) +#if (AMREX_SPACEDIM == 3) BL_TO_FORTRAN_ANYD(flux[2][mfi]), #endif dx, &dt); diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance_3d.f90 b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance_3d.f90 index 6b50e26549e..03cb5c11e2c 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance_3d.f90 +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/advance_3d.f90 @@ -13,7 +13,7 @@ subroutine compute_flux (lo, hi, domlo, domhi, phi, philo, phihi, & real(amrex_real), intent(inout) :: fluxy( fylo(1): fyhi(1), fylo(2): fyhi(2), fylo(3): fyhi(3)) real(amrex_real), intent(inout) :: fluxz( fzlo(1): fzhi(1), fzlo(2): fzhi(2), fzlo(3): fzhi(3)) real(amrex_real), intent(in) :: dx(3) - + ! local variables integer i,j,k diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_2d.f90 b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_2d.f90 index 02a4c036cff..ea934ded810 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_2d.f90 +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_2d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(2), hi(2), philo(2), phihi(2) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in ) :: dx(2) - real(amrex_real), intent(in ) :: prob_lo(2) - real(amrex_real), intent(in ) :: prob_hi(2) + real(amrex_real), intent(in ) :: dx(2) + real(amrex_real), intent(in ) :: prob_lo(2) + real(amrex_real), intent(in ) :: prob_hi(2) integer :: i,j double precision :: x,y,r2 diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_3d.f90 b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_3d.f90 index ec281d22df5..9fc031ee221 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_3d.f90 +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/init_phi_3d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(3), hi(3), philo(3), phihi(3) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2),philo(3):phihi(3)) - real(amrex_real), intent(in ) :: dx(3) - real(amrex_real), intent(in ) :: prob_lo(3) - real(amrex_real), intent(in ) :: prob_hi(3) + real(amrex_real), intent(in ) :: dx(3) + real(amrex_real), intent(in ) :: prob_lo(3) + real(amrex_real), intent(in ) :: prob_hi(3) integer :: i,j,k double precision :: x,y,z,r2 diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/main.cpp b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/main.cpp index 66722eaa6c4..08ba92626bb 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/main.cpp +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/main.cpp @@ -1,8 +1,8 @@ // -// Demonstrates how to generate of a Conduit Mesh Blueprint +// Demonstrates how to generate of a Conduit Mesh Blueprint // description of an AMReX Single Level dataset and render this // data in situ using with ALPINE Ascent. -// +// #include #include @@ -24,9 +24,9 @@ using namespace ascent; int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -45,7 +45,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -85,12 +85,12 @@ void main_main () geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 1; - + // Ncomp = number of components for each array int Ncomp = 1; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -118,7 +118,7 @@ void main_main () int rank = ParallelDescriptor::MyProc(); // Return the rank int ntasks = ParallelDescriptor::NProcs(); // Return the number of processes - + ///////////////////////////// // Setup Ascent ///////////////////////////// @@ -147,10 +147,10 @@ void main_main () conduit::Node bp_mesh; SingleLevelToBlueprint( phi_new, {"phi"}, geom, time, n, bp_mesh); - + /////////////////////////////////////////////////////////////////// - // Save the Blueprint Mesh to a set of files that we can - // view in VisIt. + // Save the Blueprint Mesh to a set of files that we can + // view in VisIt. // (For debugging and to demonstrate how to do this w/o Ascent) /////////////////////////////////////////////////////////////////// WriteBlueprintFiles( bp_mesh, "bp_heateq_ex_", n); @@ -189,18 +189,18 @@ void main_main () flux[dir].define(edge_ba, dm, 1, 0); } - - + + for (int n = 1; n <= nsteps; ++n) { MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); // new_phi = old_phi + dt * (something) - advance(phi_old, phi_new, flux, dt, geom); + advance(phi_old, phi_new, flux, dt, geom); time = time + dt; - - + + // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; @@ -217,10 +217,10 @@ void main_main () conduit::Node bp_mesh; SingleLevelToBlueprint( phi_new, {"phi"}, geom, time, n, bp_mesh); - + /////////////////////////////////////////////////////////////////// - // Save the Blueprint Mesh to a set of files that we can - // view in VisIt. + // Save the Blueprint Mesh to a set of files that we can + // view in VisIt. // (For debugging and to demonstrate how to do this w/o Ascent) /////////////////////////////////////////////////////////////////// WriteBlueprintFiles( bp_mesh, "bp_heateq_ex_", n); @@ -228,7 +228,7 @@ void main_main () /////////////////////////////////////////////////////////////////// // Render with Ascent /////////////////////////////////////////////////////////////////// - + // add a scene with a pseudocolor plot Node scenes; scenes["s1/plots/p1/type"] = "pseudocolor"; diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc.H b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc.H index af0c1f885ac..038e68495a0 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc.H +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc.H @@ -10,8 +10,8 @@ void main_main (); void advance (MultiFab& phi_old, MultiFab& phi_new, - std::array& flux, - Real dt, + std::array& flux, + Real dt, const Geometry& geom); #endif diff --git a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc_F.H b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc_F.H index 146dea4b54a..547f22231cd 100644 --- a/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc_F.H +++ b/Tutorials/Blueprint/HeatEquation_EX1_C/Source/myfunc_F.H @@ -11,13 +11,13 @@ extern "C" void init_phi(const int* lo, const int* hi, amrex_real* data, const int* dlo, const int* dhi, const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi); - + void compute_flux(const int* lo, const int* hi, const int* domlo, const int* domhi, amrex_real* phi, const int* philo, const int* phihi, amrex_real* fluxx, const int* fxlo, const int* fxhi, amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) +#if (BL_SPACEDIM == 3) amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif const amrex_real* dx); @@ -27,7 +27,7 @@ extern "C" amrex_real* pnew, const int* pnlo, const int* pnhi, amrex_real* fluxx, const int* fxlo, const int* fxhi, amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) +#if (BL_SPACEDIM == 3) amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif const amrex_real* dx, const amrex_real* dt); diff --git a/Tutorials/Blueprint/README.txt b/Tutorials/Blueprint/README.txt index 435048c797f..0ead8392cd2 100644 --- a/Tutorials/Blueprint/README.txt +++ b/Tutorials/Blueprint/README.txt @@ -1,6 +1,6 @@ These tests demonstrate how to convert AMReX Mesh data into an in-memory Conduit Mesh Blueprint description for consumption by the ALPINE Ascent -in situ visualization and analysis tool. +in situ visualization and analysis tool. For details about what mesh features are currently supported, see: Src/Extern/Conduit/AMReX_Conduit_Blueprint.H @@ -12,10 +12,10 @@ The tests include: These tests use the interfaces in Src/Extern/Conduit/AMReX_Conduit_Blueprint.H, -which are built when USE_CONDUIT=TRUE. These tests' GNUmakefiles provide a -template of how to enable and link Conduit and Ascent. +which are built when USE_CONDUIT=TRUE. These tests' GNUmakefiles provide a +template of how to enable and link Conduit and Ascent. + - For more details about Conduit and Ascent, please see: Conduit: diff --git a/Tutorials/EB/CNS/Exec/Combustor/bc_fill_nd.F90 b/Tutorials/EB/CNS/Exec/Combustor/bc_fill_nd.F90 index 74bfcccfb72..24600ea1881 100644 --- a/Tutorials/EB/CNS/Exec/Combustor/bc_fill_nd.F90 +++ b/Tutorials/EB/CNS/Exec/Combustor/bc_fill_nd.F90 @@ -8,7 +8,7 @@ module bc_fill_module ! All subroutines in this file must be threadsafe because they are called ! inside OpenMP parallel regions. - + subroutine cns_hypfill(adv,adv_lo,adv_hi,domlo,domhi,delta,xlo,time,bc) & bind(C, name="cns_hypfill") diff --git a/Tutorials/EB/CNS/Exec/Pulse/cns_prob.F90 b/Tutorials/EB/CNS/Exec/Pulse/cns_prob.F90 index c7bf340b681..9c06802f3cf 100644 --- a/Tutorials/EB/CNS/Exec/Pulse/cns_prob.F90 +++ b/Tutorials/EB/CNS/Exec/Pulse/cns_prob.F90 @@ -36,7 +36,7 @@ subroutine cns_initdata(level, time, lo, hi, u, ulo, uhi, dx, prob_lo) bind(C, n real(rt), intent(in) :: time real(rt), intent(inout) :: u(ulo(1):uhi(1), ulo(2):uhi(2), ulo(3):uhi(3),nvar) real(rt), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(rt) :: x,y,z,r, Pt real(rt), parameter :: Pi = 4.d0*atan(1.d0) diff --git a/Tutorials/EB/CNS/Exec/ShockRef/cns_prob.F90 b/Tutorials/EB/CNS/Exec/ShockRef/cns_prob.F90 index 7e3035b3c47..ca97232b258 100644 --- a/Tutorials/EB/CNS/Exec/ShockRef/cns_prob.F90 +++ b/Tutorials/EB/CNS/Exec/ShockRef/cns_prob.F90 @@ -42,7 +42,7 @@ subroutine cns_initdata(level, time, lo, hi, u, ulo, uhi, dx, prob_lo) bind(C, n real(rt), intent(in) :: time real(rt), intent(inout) :: u(ulo(1):uhi(1), ulo(2):uhi(2), ulo(3):uhi(3),nvar) real(rt), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(rt) :: x diff --git a/Tutorials/EB/CNS/Exec/Sod/cns_prob.F90 b/Tutorials/EB/CNS/Exec/Sod/cns_prob.F90 index 8b03e3f00aa..ae6ebc42606 100644 --- a/Tutorials/EB/CNS/Exec/Sod/cns_prob.F90 +++ b/Tutorials/EB/CNS/Exec/Sod/cns_prob.F90 @@ -40,7 +40,7 @@ subroutine cns_initdata(level, time, lo, hi, u, ulo, uhi, dx, prob_lo) bind(C, n real(rt), intent(in) :: time real(rt), intent(inout) :: u(ulo(1):uhi(1), ulo(2):uhi(2), ulo(3):uhi(3),nvar) real(rt), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(rt) :: x, Pt, rhot, uxt diff --git a/Tutorials/EB/CNS/Source/CNS.H b/Tutorials/EB/CNS/Source/CNS.H index 5cb6f8d041a..1b8efa904f5 100644 --- a/Tutorials/EB/CNS/Source/CNS.H +++ b/Tutorials/EB/CNS/Source/CNS.H @@ -23,7 +23,7 @@ public: CNS (const CNS& rhs) = delete; CNS& operator= (const CNS& rhs) = delete; - + // Restart from a checkpoint file. virtual void restart (amrex::Amr& papa, std::istream& is, @@ -49,7 +49,7 @@ public: // Initialize data on this level after regridding if old level did not previously exist virtual void init () override; - + // Initialize grid data at problem start-up. virtual void initData () override; diff --git a/Tutorials/EB/CNS/Source/CNS.cpp b/Tutorials/EB/CNS/Source/CNS.cpp index ce96a3e1c14..f257a5a8539 100644 --- a/Tutorials/EB/CNS/Source/CNS.cpp +++ b/Tutorials/EB/CNS/Source/CNS.cpp @@ -179,28 +179,28 @@ CNS::computeNewDt (int finest_level, dt_min[i] = getLevel(i).estTimeStep(); } - if (post_regrid_flag == 1) + if (post_regrid_flag == 1) { - // - // Limit dt's by pre-regrid dt - // - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],dt_level[i]); - } + // + // Limit dt's by pre-regrid dt + // + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],dt_level[i]); + } } - else + else { - // - // Limit dt's by change_max * old dt - // - static Real change_max = 1.1; - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); - } + // + // Limit dt's by change_max * old dt + // + static Real change_max = 1.1; + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); + } } - + // // Find the minimum over all levels // @@ -380,13 +380,13 @@ CNS::read_params () ParmParse pp("cns"); pp.query("v", verbose); - + Vector tilesize(AMREX_SPACEDIM); if (pp.queryarr("hydro_tile_size", tilesize, 0, AMREX_SPACEDIM)) { - for (int i=0; i lo_bc(AMREX_SPACEDIM), hi_bc(AMREX_SPACEDIM); @@ -447,7 +447,7 @@ CNS::buildMetrics () } const auto& ebfactory = dynamic_cast(Factory()); - + volfrac = &(ebfactory.getVolFrac()); bndrycent = &(ebfactory.getBndryCent()); areafrac = ebfactory.getAreaFrac(); @@ -455,7 +455,7 @@ CNS::buildMetrics () level_mask.clear(); level_mask.define(grids,dmap,1,1); - level_mask.BuildMask(geom.Domain(), geom.periodicity(), + level_mask.BuildMask(geom.Domain(), geom.periodicity(), level_mask_covered, level_mask_notcovered, level_mask_physbnd, @@ -514,7 +514,7 @@ CNS::computeTemp (MultiFab& State, int ng) auto const& fact = dynamic_cast(State.Factory()); auto const& flags = fact.getMultiEBCellFlagFab(); - // This will reset Eint and compute Temperature + // This will reset Eint and compute Temperature #ifdef AMREX_USE_OMP #pragma omp parallel #endif diff --git a/Tutorials/EB/CNS/Source/CNS_F.H b/Tutorials/EB/CNS/Source/CNS_F.H index f5695dc8ceb..2e09ff313ec 100644 --- a/Tutorials/EB/CNS/Source/CNS_F.H +++ b/Tutorials/EB/CNS/Source/CNS_F.H @@ -18,17 +18,17 @@ extern "C" { void cns_hypfill (BL_FORT_FAB_ARG_3D(state), const int* dlo, const int* dhi, - const amrex_real* dx, const amrex_real* glo, + const amrex_real* dx, const amrex_real* glo, const amrex_real* time, const int* bc); - + void cns_denfill (BL_FORT_FAB_ARG_3D(state), const int* dlo, const int* dhi, - const amrex_real* dx, const amrex_real* glo, + const amrex_real* dx, const amrex_real* glo, const amrex_real* time, const int* bc); void cns_nullfill (BL_FORT_FAB_ARG_3D(state), const int* dlo, const int* dhi, - const amrex_real* dx, const amrex_real* glo, + const amrex_real* dx, const amrex_real* glo, const amrex_real* time, const int* bc); void cns_derpres (BL_FORT_FAB_ARG_3D(der),const int* nvar, @@ -36,7 +36,7 @@ extern "C" { const int* lo, const int* hi, const int* domain_lo, const int* domain_hi, const amrex_real* delta, const amrex_real* xlo, - const amrex_real* time, const amrex_real* dt, const int* bcrec, + const amrex_real* time, const amrex_real* dt, const int* bcrec, const int* level, const int* grid_no); void cns_dervel (BL_FORT_FAB_ARG_3D(der),const int* nvar, @@ -44,7 +44,7 @@ extern "C" { const int* lo, const int* hi, const int* domain_lo, const int* domain_hi, const amrex_real* delta, const amrex_real* xlo, - const amrex_real* time, const amrex_real* dt, const int* bcrec, + const amrex_real* time, const amrex_real* dt, const int* bcrec, const int* level, const int* grid_no); void cns_compute_temperature (const int* lo, const int* hi, @@ -53,7 +53,7 @@ extern "C" { void cns_estdt (const int* lo, const int* hi, const amrex_real* u, const int* ulo, const int* uhi, const amrex_real* dx, amrex_real* dt); - + void cns_compute_dudt (const int* lo, const int* hi, amrex_real* dudt, const int* utlo, const int* uthi, diff --git a/Tutorials/EB/CNS/Source/CNS_advance.cpp b/Tutorials/EB/CNS/Source/CNS_advance.cpp index d647c17a776..b36ff280366 100644 --- a/Tutorials/EB/CNS/Source/CNS_advance.cpp +++ b/Tutorials/EB/CNS/Source/CNS_advance.cpp @@ -11,7 +11,7 @@ Real CNS::advance (Real time, Real dt, int iteration, int ncycle) { BL_PROFILE("CNS::advance()"); - + for (int i = 0; i < num_state_data_types; ++i) { state[i].allocOldData(); state[i].swapTimeLevels(dt); @@ -21,7 +21,7 @@ CNS::advance (Real time, Real dt, int iteration, int ncycle) MultiFab& S_old = get_old_data(State_Type); MultiFab dSdt(grids,dmap,NUM_STATE,0,MFInfo(),Factory()); MultiFab Sborder(grids,dmap,NUM_STATE,NUM_GROW,MFInfo(),Factory()); - + MultiFab& C_new = get_new_data(Cost_Type); C_new.setVal(0.0); @@ -46,7 +46,7 @@ CNS::advance (Real time, Real dt, int iteration, int ncycle) // U^* = U^n + dt*dUdt^n MultiFab::LinComb(S_new, 1.0, Sborder, 0, dt, dSdt, 0, 0, NUM_STATE, 0); computeTemp(S_new,0); - + // RK2 stage 2 // After fillpatch Sborder = U^n+dt*dUdt^n FillPatch(*this, Sborder, NUM_GROW, time+dt, State_Type, 0, NUM_STATE); @@ -57,7 +57,7 @@ CNS::advance (Real time, Real dt, int iteration, int ncycle) MultiFab::Saxpy(S_new, 0.5*dt, dSdt, 0, 0, NUM_STATE, 0); // We now have S_new = U^{n+1} = (U^n+0.5*dt*dUdt^n) + 0.5*dt*dUdt^* computeTemp(S_new,0); - + return dt; } diff --git a/Tutorials/EB/CNS/Source/CNS_init_eb2.cpp b/Tutorials/EB/CNS/Source/CNS_init_eb2.cpp index 356ad7605b0..a2fc48c826b 100644 --- a/Tutorials/EB/CNS/Source/CNS_init_eb2.cpp +++ b/Tutorials/EB/CNS/Source/CNS_init_eb2.cpp @@ -22,14 +22,14 @@ initialize_EB2 (const Geometry& geom, const int required_coarsening_level, if (geom_type == "combustor") { ParmParse pp("combustor"); - - Real fwl; + + Real fwl; pp.get("far_wall_loc",fwl); EB2::PlaneIF farwall({AMREX_D_DECL(fwl,0.,0.)}, {AMREX_D_DECL(1. ,0.,0.)}); - - Vector pl1pt, pl2pt, pl2nm, pl3pt; + + Vector pl1pt, pl2pt, pl2nm, pl3pt; pp.getarr("ramp_plane1_point", pl1pt); pp.getarr("ramp_plane2_point", pl2pt); pp.getarr("ramp_plane2_normal", pl2nm); @@ -42,10 +42,10 @@ initialize_EB2 (const Geometry& geom, const int required_coarsening_level, EB2::PlaneIF({pl3pt[0], pl3pt[1], 0.}, { 1., 0., 0.})); - Vector pipelo, pipehi; + Vector pipelo, pipehi; pp.getarr("pipe_lo", pipelo); pp.getarr("pipe_hi", pipehi); - + EB2::BoxIF pipe({pipelo[0], pipelo[1], -1.}, {pipehi[0], pipehi[1], 1.}, false); // where does plane 1 and plane 2 intersect? @@ -55,13 +55,13 @@ initialize_EB2 (const Geometry& geom, const int required_coarsening_level, Real dx = geom.CellSize(0); Real dycut = 4.*(1.+max_coarsening_level)*std::min(dx, k2*dx); EB2::BoxIF flat_corner({pl3pt[0], 0., -1.}, {1.e10, secty+dycut, 1.}, false); - + auto polys = EB2::makeUnion(farwall, ramp, pipe, flat_corner); Real lenx = geom.ProbLength(0); Real leny = geom.ProbLength(1); auto pr = EB2::translate(EB2::lathe(polys), {lenx*0.5, leny*0.5, 0.}); - + auto gshop = EB2::makeShop(pr); EB2::Build(gshop, geom, max_coarsening_level, max_coarsening_level, 4); } diff --git a/Tutorials/EB/CNS/Source/CNS_io.cpp b/Tutorials/EB/CNS/Source/CNS_io.cpp index 9616b9d5aac..a6b38ffb845 100644 --- a/Tutorials/EB/CNS/Source/CNS_io.cpp +++ b/Tutorials/EB/CNS/Source/CNS_io.cpp @@ -19,8 +19,8 @@ CNS::restart (Amr& papa, std::istream& is, bool bReadSpecial) buildMetrics(); } -void -CNS::checkPoint (const std::string& dir, std::ostream& os, VisMF::How how, bool dump_old) +void +CNS::checkPoint (const std::string& dir, std::ostream& os, VisMF::How how, bool dump_old) { AmrLevel::checkPoint(dir, os, how, dump_old); } @@ -33,5 +33,5 @@ CNS::writePlotFile (const std::string& dir, std::ostream& os, VisMF::How how) MultiFab& C_new = get_new_data(Cost_Type); C_new.setVal(0.0); #endif - AmrLevel::writePlotFile(dir, os, how); + AmrLevel::writePlotFile(dir, os, how); } diff --git a/Tutorials/EB/CNS/Source/CNS_setup.cpp b/Tutorials/EB/CNS/Source/CNS_setup.cpp index 95fc72bb3d7..ca81589313b 100644 --- a/Tutorials/EB/CNS/Source/CNS_setup.cpp +++ b/Tutorials/EB/CNS/Source/CNS_setup.cpp @@ -38,8 +38,8 @@ set_scalar_bc (BCRec& bc, const BCRec& phys_bc) const int* hi_bc = phys_bc.hi(); for (int i = 0; i < AMREX_SPACEDIM; i++) { - bc.setLo(i,scalar_bc[lo_bc[i]]); - bc.setHi(i,scalar_bc[hi_bc[i]]); + bc.setLo(i,scalar_bc[lo_bc[i]]); + bc.setHi(i,scalar_bc[hi_bc[i]]); } } @@ -69,7 +69,7 @@ set_y_vel_bc(BCRec& bc, const BCRec& phys_bc) const int* hi_bc = phys_bc.hi(); bc.setLo(0,tang_vel_bc[lo_bc[0]]); bc.setHi(0,tang_vel_bc[hi_bc[0]]); -#if (AMREX_SPACEDIM >= 2) +#if (AMREX_SPACEDIM >= 2) bc.setLo(1,norm_vel_bc[lo_bc[1]]); bc.setHi(1,norm_vel_bc[hi_bc[1]]); #endif @@ -113,8 +113,8 @@ CNS::variableSetUp () bool state_data_extrap = false; bool store_in_checkpoint = true; desc_lst.addDescriptor(State_Type,IndexType::TheCellType(), - StateDescriptor::Point,NUM_GROW,NUM_STATE, - &eb_cell_cons_interp,state_data_extrap,store_in_checkpoint); + StateDescriptor::Point,NUM_GROW,NUM_STATE, + &eb_cell_cons_interp,state_data_extrap,store_in_checkpoint); Vector bcs(NUM_STATE); Vector name(NUM_STATE); @@ -129,10 +129,10 @@ CNS::variableSetUp () cnt++; set_scalar_bc(bc,phys_bc); bcs[cnt] = bc; name[cnt] = "Temp"; desc_lst.setComponent(State_Type, - Density, - name, - bcs, - BndryFunc(cns_denfill,cns_hypfill)); + Density, + name, + bcs, + BndryFunc(cns_denfill,cns_hypfill)); desc_lst.addDescriptor(Cost_Type, IndexType::TheCellType(), StateDescriptor::Point, 0,1, &pc_interp); diff --git a/Tutorials/EB/CNS/Source/diffusion/cns_diff_mod.F90 b/Tutorials/EB/CNS/Source/diffusion/cns_diff_mod.F90 index f0cc6360dde..517bec26663 100644 --- a/Tutorials/EB/CNS/Source/diffusion/cns_diff_mod.F90 +++ b/Tutorials/EB/CNS/Source/diffusion/cns_diff_mod.F90 @@ -24,7 +24,7 @@ subroutine diff_mol_3d (lo,hi, dx, & real(rt), intent( out) :: flux1(lo(1):hi(1)+1,lo(2):hi(2) ,lo(3):hi(3) ,5) real(rt), intent( out) :: flux2(lo(1):hi(1) ,lo(2):hi(2)+1,lo(3):hi(3) ,5) real(rt), intent( out) :: flux3(lo(1):hi(1) ,lo(2):hi(2) ,lo(3):hi(3)+1,5) - + integer :: i,j,k real(rt) :: dxinv(3) real(rt) :: tauxx, tauyy, tauzz, tauxy, tauxz, tauyz, muf, xif diff --git a/Tutorials/EB/CNS/Source/diffusion/cns_eb_diff_wall.F90 b/Tutorials/EB/CNS/Source/diffusion/cns_eb_diff_wall.F90 index 19da2380dfb..5be80ee4310 100644 --- a/Tutorials/EB/CNS/Source/diffusion/cns_eb_diff_wall.F90 +++ b/Tutorials/EB/CNS/Source/diffusion/cns_eb_diff_wall.F90 @@ -187,7 +187,7 @@ subroutine compute_diff_wallflux (divw, dxinv, i,j,k, & u1 = interp2d(cxm,cx0,cxp,cym,cy0,cyp, q(ixit-1:ixit+1,iyit-1:iyit+1,k+is,qu)) v1 = interp2d(cxm,cx0,cxp,cym,cy0,cyp, q(ixit-1:ixit+1,iyit-1:iyit+1,k+is,qv)) w1 = interp2d(cxm,cx0,cxp,cym,cy0,cyp, q(ixit-1:ixit+1,iyit-1:iyit+1,k+is,qw)) - + d2 = (bct(3) - 2.d0*s) * (1.0d0/anrmz) xit = bct(1) - d2*anrmx yit = bct(2) - d2*anrmy @@ -206,7 +206,7 @@ subroutine compute_diff_wallflux (divw, dxinv, i,j,k, & u2 = interp2d(cxm,cx0,cxp,cym,cy0,cyp, q(ixit-1:ixit+1,iyit-1:iyit+1,k+2*is,qu)) v2 = interp2d(cxm,cx0,cxp,cym,cy0,cyp, q(ixit-1:ixit+1,iyit-1:iyit+1,k+2*is,qv)) w2 = interp2d(cxm,cx0,cxp,cym,cy0,cyp, q(ixit-1:ixit+1,iyit-1:iyit+1,k+2*is,qw)) - + end if ! @@ -243,10 +243,10 @@ subroutine compute_diff_wallflux (divw, dxinv, i,j,k, & divw(2) = dxinv(1) * (dapx*tauxx + dapy*tauxy + dapz*tauxz) divw(3) = dxinv(1) * (dapx*tauxy + dapy*tauyy + dapz*tauyz) divw(4) = dxinv(1) * (dapx*tauxz + dapy*tauyz + dapz*tauzz) - + end subroutine compute_diff_wallflux - + real(rt) function interp2d(cym,cy0,cyp,czm,cz0,czp,v) real(rt), intent(in) :: cym,cy0,cyp,czm,cz0,czp,v(3,3) interp2d = czm*(cym*v(1,1) + cy0*v(2,1) + cyp*v(3,1)) & diff --git a/Tutorials/EB/CNS/Source/diffusion/diff_coef_mod.F90 b/Tutorials/EB/CNS/Source/diffusion/diff_coef_mod.F90 index ff5a7a177f2..056c4e54fcb 100644 --- a/Tutorials/EB/CNS/Source/diffusion/diff_coef_mod.F90 +++ b/Tutorials/EB/CNS/Source/diffusion/diff_coef_mod.F90 @@ -29,7 +29,7 @@ subroutine compute_diff_coef (q,qlo,qhi,lambda,mu,xi,clo,chi) end do else Prinv = 1.d0/Pr - + do k = clo(3), chi(3) do j = clo(2), chi(2) do i = clo(1), chi(1) diff --git a/Tutorials/EB/CNS/Source/fortran/CNS_divop.F90 b/Tutorials/EB/CNS/Source/fortran/CNS_divop.F90 index 83fc279bc9f..5c6e4fafce8 100644 --- a/Tutorials/EB/CNS/Source/fortran/CNS_divop.F90 +++ b/Tutorials/EB/CNS/Source/fortran/CNS_divop.F90 @@ -17,12 +17,12 @@ subroutine compute_divop (lo,hi,ncomp,dx,ut,ulo,uhi,fx,fxlo,fxhi,fy,fylo,fyhi,fz real(rt), intent(in ) :: fy(fylo(1):fyhi(1),fylo(2):fyhi(2),fylo(3):fyhi(3),ncomp) real(rt), intent(in ) :: fz(fzlo(1):fzhi(1),fzlo(2):fzhi(2),fzlo(3):fzhi(3),ncomp) real(rt), intent(in) :: dx(3) - + integer :: i,j,k,n real(rt) :: dxinv(3) - + dxinv = 1.d0/dx - + do n = 1, ncomp do k = lo(3),hi(3) do j = lo(2),hi(2) @@ -36,7 +36,7 @@ subroutine compute_divop (lo,hi,ncomp,dx,ut,ulo,uhi,fx,fxlo,fxhi,fy,fylo,fyhi,fz end do end subroutine compute_divop - + pure logical function is_inside (i,j,k,lo,hi) integer, intent(in) :: i,j,k,lo(3),hi(3) is_inside = i.ge.lo(1) .and. i.le.hi(1) & @@ -436,12 +436,12 @@ subroutine compute_eb_divop (lo,hi,ncomp, dx, dt, & end do elseif (use_mass_as_eb_weights) then do i = lo(1)-2, hi(1)+2 - rediswgt(i,j,k) = q(i,j,k,qrho) + rediswgt(i,j,k) = q(i,j,k,qrho) ! rediswgt(i,j,k) = max(smallr, q(i,j,k,qrho)+dt*divc(i,j,k)) end do elseif (use_volfrac_as_eb_weights) then do i = lo(1)-2, hi(1)+2 - rediswgt(i,j,k) = vfrac(i,j,k) + rediswgt(i,j,k) = vfrac(i,j,k) end do else do i = lo(1)-2, hi(1)+2 @@ -490,7 +490,7 @@ subroutine compute_eb_divop (lo,hi,ncomp, dx, dt, & do k = lo(3)-1, hi(3)+1 do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - if (is_single_valued_cell(cellflag(i,j,k))) then + if (is_single_valued_cell(cellflag(i,j,k))) then wtot = 0.d0 call get_neighbor_cells(cellflag(i,j,k),nbr) do kk = -1,1 diff --git a/Tutorials/EB/CNS/Source/fortran/CNS_dudt.F90 b/Tutorials/EB/CNS/Source/fortran/CNS_dudt.F90 index 56be5cb73a1..18648a7031f 100644 --- a/Tutorials/EB/CNS/Source/fortran/CNS_dudt.F90 +++ b/Tutorials/EB/CNS/Source/fortran/CNS_dudt.F90 @@ -52,9 +52,9 @@ subroutine cns_compute_dudt (lo,hi, dudt, utlo, uthi, & call amrex_allocate(lambda, clo, chi) call amrex_allocate(mu, clo, chi) call amrex_allocate(xi, clo, chi) - + call ctoprim(qlo, qhi, u, ulo, uhi, q, qlo, qhi) - + call hyp_mol_gam_3d(q, qlo, qhi, lo, hi, dx, fhx, fhy, fhz) call compute_diff_coef(q, qlo, qhi, lambda, mu, xi, clo, chi) @@ -157,11 +157,11 @@ subroutine cns_eb_compute_dudt (lo,hi, dudt, utlo, uthi, & qlo = lo - nghost qhi = hi + nghost call amrex_allocate(q, qlo(1),qhi(1), qlo(2),qhi(2), qlo(3),qhi(3), 1,qvar) - + dvlo = lo-2 dvhi = hi+2 call amrex_allocate(divc, dvlo, dvhi) - call amrex_allocate(optmp, dvlo, dvhi) + call amrex_allocate(optmp, dvlo, dvhi) call amrex_allocate(rediswgt, dvlo, dvhi) dmlo(1:3) = lo - 1 @@ -193,7 +193,7 @@ subroutine cns_eb_compute_dudt (lo,hi, dudt, utlo, uthi, & call ctoprim(qlo, qhi, u, ulo, uhi, q, qlo, qhi) call bl_proffortfuncstop_int(6) - + call bl_proffortfuncstart_int(7) call hyp_mol_gam_eb_3d(q, qlo, qhi, lo, hi, dx, & fhx, lfxlo, lfxhi, fhy, lfylo, lfyhi, fhz, lfzlo, lfzhi,& @@ -242,7 +242,7 @@ subroutine cns_eb_compute_dudt (lo,hi, dudt, utlo, uthi, & as_crse, rr_drho_crse, rdclo, rdchi, rr_flag_crse, rfclo, rfchi, & as_fine, dm_as_fine, dflo, dfhi, & levmsk, lmlo, lmhi) - + dudt(lo(1):hi(1),lo(2):hi(2),lo(3):hi(3),6:nvar) = 0.d0 call bl_proffortfuncstop_int(10) diff --git a/Tutorials/EB/CNS/Source/fortran/CNS_f.F90 b/Tutorials/EB/CNS/Source/fortran/CNS_f.F90 index 846e9c9160e..09a32d14776 100644 --- a/Tutorials/EB/CNS/Source/fortran/CNS_f.F90 +++ b/Tutorials/EB/CNS/Source/fortran/CNS_f.F90 @@ -5,7 +5,7 @@ module cns_module private ! these flags must be the same as in CNS.H - integer, parameter, public :: levmsk_interior = 0 ! valid cells + integer, parameter, public :: levmsk_interior = 0 ! valid cells integer, parameter, public :: levmsk_covered = 1 ! ghost cells covered by valid cells of this level integer, parameter, public :: levmsk_notcovered = 2 ! ghost cells not covered integer, parameter, public :: levmsk_physbnd = 3 ! outside domain @@ -28,7 +28,7 @@ module cns_module integer, parameter, public :: QEINT = 7 integer, parameter, public :: QTEMP = 8 integer, parameter, public :: QVAR = 8 - + real(rt), parameter, public :: smallp = 1.d-10 real(rt), parameter, public :: smallr = 1.d-19 diff --git a/Tutorials/EB/CNS/Source/fortran/CNS_nd.F90 b/Tutorials/EB/CNS/Source/fortran/CNS_nd.F90 index 126779ce2d2..5b89cbe5f34 100644 --- a/Tutorials/EB/CNS/Source/fortran/CNS_nd.F90 +++ b/Tutorials/EB/CNS/Source/fortran/CNS_nd.F90 @@ -62,10 +62,10 @@ subroutine ctoprim (lo,hi,u,ulo,uhi,q,qlo,qhi) integer, intent(in) :: lo(3),hi(3),ulo(3),uhi(3),qlo(3),qhi(3) real(rt), intent(in ) :: u(ulo(1):uhi(1),ulo(2):uhi(2),ulo(3):uhi(3),nvar) real(rt), intent(inout) :: q(qlo(1):qhi(1),qlo(2):qhi(2),qlo(3):qhi(3),qvar) - + integer :: i,j,k real(rt) :: rhoinv, kineng - + do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) diff --git a/Tutorials/EB/CNS/Source/fortran/bc_fill_nd.F90 b/Tutorials/EB/CNS/Source/fortran/bc_fill_nd.F90 index 830ed8d4991..54f41b3fdf3 100644 --- a/Tutorials/EB/CNS/Source/fortran/bc_fill_nd.F90 +++ b/Tutorials/EB/CNS/Source/fortran/bc_fill_nd.F90 @@ -8,7 +8,7 @@ module bc_fill_module ! All subroutines in this file must be threadsafe because they are called ! inside OpenMP parallel regions. - + subroutine cns_hypfill(adv,adv_lo,adv_hi,domlo,domhi,delta,xlo,time,bc) & bind(C, name="cns_hypfill") diff --git a/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL.F90 b/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL.F90 index aa791abb693..a38ed8e2b02 100644 --- a/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL.F90 +++ b/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL.F90 @@ -64,12 +64,12 @@ subroutine hyp_mol_gam_3d(q, qd_lo, qd_hi, & pl(i) = max(pl(i),smallp) ut1l(i) = q(i-1,j,k,QV) + 0.5d0 * dq(i-1,j,k,4) ut2l(i) = q(i-1,j,k,Qw) + 0.5d0 * dq(i-1,j,k,5) - + cspeed = q(i,j,k,QC) rr(i) = q(i,j,k,QRHO) - 0.5d0 * ( (dq(i,j,k,1)+dq(i,j,k,3))/cspeed + dq(i,j,k,2)) rr(i) = max(rr(i),smallr) ur(i) = q(i,j,k,QU) - 0.5d0 * ( (dq(i,j,k,3)-dq(i,j,k,1))/q(i,j,k,QRHO)) - pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed + pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed pr(i) = max(pr(i),smallp) ut1r(i) = q(i,j,k,QV) - 0.5d0 * dq(i,j,k,4) ut2r(i) = q(i,j,k,Qw) - 0.5d0 * dq(i,j,k,5) @@ -100,16 +100,16 @@ subroutine hyp_mol_gam_3d(q, qd_lo, qd_hi, & rl(i) = q(i,j-1,k,QRHO) + 0.5d0 * ( (dq(i,j-1,k,1)+dq(i,j-1,k,3))/cspeed + dq(i,j-1,k,2)) rl(i) = max(rl(i),smallr) ul(i) = q(i,j-1,k,QV) + 0.5d0 * ( (dq(i,j-1,k,3)-dq(i,j-1,k,1))/q(i,j-1,k,QRHO)) - pl(i) = q(i,j-1,k,QP) + 0.5d0 * (dq(i,j-1,k,1)+dq(i,j-1,k,3))*cspeed + pl(i) = q(i,j-1,k,QP) + 0.5d0 * (dq(i,j-1,k,1)+dq(i,j-1,k,3))*cspeed pl(i) = max(pl(i),smallp) ut1l(i) = q(i,j-1,k,QU) + 0.5d0 * dq(i,j-1,k,4) ut2l(i) = q(i,j-1,k,Qw) + 0.5d0 * dq(i,j-1,k,5) - + cspeed = q(i,j,k,QC) rr(i) = q(i,j,k,QRHO) - 0.5d0 * ( (dq(i,j,k,1)+dq(i,j,k,3))/cspeed + dq(i,j,k,2)) rr(i) = max(rr(i),smallr) ur(i) = q(i,j,k,QV) - 0.5d0 * ( (dq(i,j,k,3)-dq(i,j,k,1))/q(i,j,k,QRHO)) - pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed + pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed pr(i) = max(pr(i),smallp) ut1r(i) = q(i,j,k,QU) - 0.5d0 * dq(i,j,k,4) ut2r(i) = q(i,j,k,Qw) - 0.5d0 * dq(i,j,k,5) @@ -122,14 +122,14 @@ subroutine hyp_mol_gam_3d(q, qd_lo, qd_hi, & enddo enddo call bl_proffortfuncstop_int(3) - + call bl_proffortfuncstart_int(4) call slopez(q,qd_lo,qd_hi, & dq,qtlo,qthi, & lo(1),lo(2),lo(3), & hi(1),hi(2),hi(3),QVAR) call bl_proffortfuncstop_int(4) - + call bl_proffortfuncstart_int(5) do k = lo(3), hi(3)+1 do j = lo(2), hi(2) @@ -139,7 +139,7 @@ subroutine hyp_mol_gam_3d(q, qd_lo, qd_hi, & rl(i) = q(i,j,k-1,QRHO) + 0.5d0 * ( (dq(i,j,k-1,1)+dq(i,j,k-1,3))/cspeed + dq(i,j,k-1,2)) rl(i) = max(rl(i),smallr) ul(i) = q(i,j,k-1,QW) + 0.5d0 * ( (dq(i,j,k-1,3)-dq(i,j,k-1,1))/q(i,j,k-1,QRHO)) - pl(i) = q(i,j,k-1,QP) + 0.5d0 * (dq(i,j,k-1,1)+dq(i,j,k-1,3))*cspeed + pl(i) = q(i,j,k-1,QP) + 0.5d0 * (dq(i,j,k-1,1)+dq(i,j,k-1,3))*cspeed pl(i) = max(pl(i),smallp) ut1l(i) = q(i,j,k-1,QU) + 0.5d0 * dq(i,j,k-1,4) ut2l(i) = q(i,j,k-1,QV) + 0.5d0 * dq(i,j,k-1,5) @@ -148,12 +148,12 @@ subroutine hyp_mol_gam_3d(q, qd_lo, qd_hi, & rr(i) = q(i,j,k,QRHO) - 0.5d0 * ( (dq(i,j,k,1)+dq(i,j,k,3))/cspeed + dq(i,j,k,2)) rr(i) = max(rr(i),smallr) ur(i) = q(i,j,k,QW) - 0.5d0 * ( (dq(i,j,k,3)-dq(i,j,k,1))/q(i,j,k,QRHO)) - pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed + pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed pr(i) = max(pr(i),smallp) ut1r(i) = q(i,j,k,QU) - 0.5d0 * dq(i,j,k,4) ut2r(i) = q(i,j,k,QV) - 0.5d0 * dq(i,j,k,5) end do - + call analriem(gamma, smallp, smallr, lo(1), hi(1), j, k, & rl, ul, pl, ut1l, ut2l, & rr, ur, pr, ut1r, ut2r, & diff --git a/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL_EB.F90 b/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL_EB.F90 index 2de37eef54b..c19e992b175 100644 --- a/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL_EB.F90 +++ b/Tutorials/EB/CNS/Source/hydro/Hyp_gamma_MOL_EB.F90 @@ -62,16 +62,16 @@ subroutine hyp_mol_gam_eb_3d(q, qd_lo, qd_hi, & rl(i) = q(i-1,j,k,QRHO) + 0.5d0 * ( (dq(i-1,j,k,1)+dq(i-1,j,k,3))/cspeed + dq(i-1,j,k,2)) rl(i) = max(rl(i),smallr) ul(i) = q(i-1,j,k,QU) + 0.5d0 * ( (dq(i-1,j,k,3)-dq(i-1,j,k,1))/q(i-1,j,k,QRHO)) - pl(i)= q(i-1,j,k,QP) + 0.5d0 * (dq(i-1,j,k,1)+dq(i-1,j,k,3))*cspeed + pl(i)= q(i-1,j,k,QP) + 0.5d0 * (dq(i-1,j,k,1)+dq(i-1,j,k,3))*cspeed pl(i) = max(pl(i),smallp) ut1l(i) = q(i-1,j,k,QV) + 0.5d0 * dq(i-1,j,k,4) ut2l(i) = q(i-1,j,k,Qw) + 0.5d0 * dq(i-1,j,k,5) - + cspeed = q(i,j,k,QC) rr(i) = q(i,j,k,QRHO) - 0.5d0 * ( (dq(i,j,k,1)+dq(i,j,k,3))/cspeed + dq(i,j,k,2)) rr(i) = max(rr(i),smallr) ur(i) = q(i,j,k,QU) - 0.5d0 * ( (dq(i,j,k,3)-dq(i,j,k,1))/q(i,j,k,QRHO)) - pr(i)= q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed + pr(i)= q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed pr(i) = max(pr(i),smallp) ut1r(i) = q(i,j,k,QV) - 0.5d0 * dq(i,j,k,4) ut2r(i) = q(i,j,k,Qw) - 0.5d0 * dq(i,j,k,5) @@ -97,16 +97,16 @@ subroutine hyp_mol_gam_eb_3d(q, qd_lo, qd_hi, & rl(i) = q(i,j-1,k,QRHO) + 0.5d0 * ( (dq(i,j-1,k,1)+dq(i,j-1,k,3))/cspeed + dq(i,j-1,k,2)) rl(i) = max(rl(i),smallr) ul(i) = q(i,j-1,k,QV) + 0.5d0 * ( (dq(i,j-1,k,3)-dq(i,j-1,k,1))/q(i,j-1,k,QRHO)) - pl(i) = q(i,j-1,k,QP) + 0.5d0 * (dq(i,j-1,k,1)+dq(i,j-1,k,3))*cspeed + pl(i) = q(i,j-1,k,QP) + 0.5d0 * (dq(i,j-1,k,1)+dq(i,j-1,k,3))*cspeed pl(i) = max(pl(i),smallp) ut1l(i) = q(i,j-1,k,QU) + 0.5d0 * dq(i,j-1,k,4) ut2l(i) = q(i,j-1,k,Qw) + 0.5d0 * dq(i,j-1,k,5) - + cspeed = q(i,j,k,QC) rr(i) = q(i,j,k,QRHO) - 0.5d0 * ( (dq(i,j,k,1)+dq(i,j,k,3))/cspeed + dq(i,j,k,2)) rr(i) = max(rr(i),smallr) ur(i) = q(i,j,k,QV) - 0.5d0 * ( (dq(i,j,k,3)-dq(i,j,k,1))/q(i,j,k,QRHO)) - pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed + pr(i) = q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed pr(i) = max(pr(i),smallp) ut1r(i) = q(i,j,k,QU) - 0.5d0 * dq(i,j,k,4) ut2r(i) = q(i,j,k,Qw) - 0.5d0 * dq(i,j,k,5) @@ -118,12 +118,12 @@ subroutine hyp_mol_gam_eb_3d(q, qd_lo, qd_hi, & flux2, fd2_lo, fd2_hi, 3, 2, 4) enddo enddo - + call ebslopez(q,qd_lo,qd_hi, & dq,qtlo,qthi, flag, fg_lo, fg_hi, & lo(1)-nextra-1, lo(2)-nextra-1, lo(3)-nextra, & hi(1)+nextra+1, hi(2)+nextra+1, hi(3)+nextra, QVAR) - + do k = lo(3)-nextra , hi(3)+nextra+1 do j = lo(2)-nextra-1, hi(2)+nextra+1 do i = lo(1)-nextra-1, hi(1)+nextra+1 @@ -132,7 +132,7 @@ subroutine hyp_mol_gam_eb_3d(q, qd_lo, qd_hi, & rl(i) = q(i,j,k-1,QRHO) + 0.5d0 * ( (dq(i,j,k-1,1)+dq(i,j,k-1,3))/cspeed + dq(i,j,k-1,2)) rl(i) = max(rl(i),smallr) ul(i) = q(i,j,k-1,QW) + 0.5d0 * ( (dq(i,j,k-1,3)-dq(i,j,k-1,1))/q(i,j,k-1,QRHO)) - pl(i)= q(i,j,k-1,QP) + 0.5d0 * (dq(i,j,k-1,1)+dq(i,j,k-1,3))*cspeed + pl(i)= q(i,j,k-1,QP) + 0.5d0 * (dq(i,j,k-1,1)+dq(i,j,k-1,3))*cspeed pl(i) = max(pl(i),smallp) ut1l(i) = q(i,j,k-1,QU) + 0.5d0 * dq(i,j,k-1,4) ut2l(i) = q(i,j,k-1,QV) + 0.5d0 * dq(i,j,k-1,5) @@ -141,7 +141,7 @@ subroutine hyp_mol_gam_eb_3d(q, qd_lo, qd_hi, & rr(i) = q(i,j,k,QRHO) - 0.5d0 * ( (dq(i,j,k,1)+dq(i,j,k,3))/cspeed + dq(i,j,k,2)) rr(i) = max(rr(i),smallr) ur(i) = q(i,j,k,QW) - 0.5d0 * ( (dq(i,j,k,3)-dq(i,j,k,1))/q(i,j,k,QRHO)) - pr(i)= q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed + pr(i)= q(i,j,k,QP) - 0.5d0 * (dq(i,j,k,1)+dq(i,j,k,3))*cspeed pr(i) = max(pr(i),smallp) ut1r(i) = q(i,j,k,QU) - 0.5d0 * dq(i,j,k,4) ut2r(i) = q(i,j,k,QV) - 0.5d0 * dq(i,j,k,5) diff --git a/Tutorials/EB/CNS/Source/hydro/analriem3d.F90 b/Tutorials/EB/CNS/Source/hydro/analriem3d.F90 index 973c41efc16..b2fe8ded9d4 100644 --- a/Tutorials/EB/CNS/Source/hydro/analriem3d.F90 +++ b/Tutorials/EB/CNS/Source/hydro/analriem3d.F90 @@ -38,7 +38,7 @@ subroutine analriem (gamma, smallp, smallr, lo, hi, j, k, & wr = sqrt(clsqr) cleft(i) = wl/rl(i) cright(i) = wr/rr(i) - + pstar(i) = (wl*pr(i) + wr*pl(i) - wr*wl*(ur(i)-ul(i)))/(wl+wr) pstar(i) = max(pstar(i),smallp) pstnm1(i) = pstar(i) @@ -57,7 +57,7 @@ subroutine analriem (gamma, smallp, smallr, lo, hi, j, k, & do iter = 1, itno do i = lo, hi ! what changes in the iteration: ustarm, ustarp, pstnm1, pstar, ustar - + wlsq = (.5d0*(gamma-1.d0)*(pstar(i)+pl(i))+pstar(i))*rl(i) wrsq = (.5d0*(gamma-1.d0)*(pstar(i)+pr(i))+pstar(i))*rr(i) @@ -127,7 +127,7 @@ subroutine analriem (gamma, smallp, smallr, lo, hi, j, k, & frac = 0.5d0*(1.d0+(spin+spout)/max(spout-spin,spin+spout,small*(cleft(i)+cright(i)))) if(spout .lt. 0.d0)then - rgdnv = ro + rgdnv = ro ugdnv = uo pgdnv = po elseif(spin .ge. 0.d0)then @@ -139,7 +139,7 @@ subroutine analriem (gamma, smallp, smallr, lo, hi, j, k, & ugdnv = frac*ustar(i) + (1.d0 - frac)* uo pgdnv = frac*pstar(i) + (1.d0 - frac)* po endif - + flux(i,j,k,1) = rgdnv*ugdnv flux(i,j,k,iu) = rgdnv*ugdnv*ugdnv+pgdnv flux(i,j,k,iut1) = rgdnv*ugdnv*utrans1 diff --git a/Tutorials/EB/CNS/Source/hydro/cns_eb_hyp_wall.F90 b/Tutorials/EB/CNS/Source/hydro/cns_eb_hyp_wall.F90 index bf8d46c309f..cec1f3a0631 100644 --- a/Tutorials/EB/CNS/Source/hydro/cns_eb_hyp_wall.F90 +++ b/Tutorials/EB/CNS/Source/hydro/cns_eb_hyp_wall.F90 @@ -15,7 +15,7 @@ subroutine compute_hyp_wallflux (divw, i,j,k, rho, u, v, w, p, & integer, intent(in) :: i,j,k real(rt), intent(in) :: rho, u, v, w, p, axm, axp, aym, ayp, azm, azp real(rt), intent(out) :: divw(5) - + real(rt) :: apnorm, apnorminv, anrmx, anrmy, anrmz, un real(rt) :: flux(1,1,1,5) diff --git a/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma.F90 b/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma.F90 index a0c623ea2ba..f4b5016af33 100644 --- a/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma.F90 +++ b/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma.F90 @@ -1,5 +1,5 @@ module slope_module - + use amrex_fort_module, only : rt=>amrex_real use cns_module, only : qrho,qu,qv,qw,qp,qc,qeint,qtemp,qvar @@ -14,20 +14,20 @@ module slope_module contains -! ::: +! ::: ! ::: ------------------------------------------------------------------ -! ::: +! ::: subroutine slopex(q,qd_lo,qd_hi, & dqxal,qpd_lo,qpd_hi, & ilo1,ilo2,ilo3,ihi1,ihi2,ihi3,nv) - + integer, intent(in) :: qd_lo(3), qd_hi(3) integer, intent(in) :: qpd_lo(3),qpd_hi(3) integer, intent(in) :: ilo1, ilo2, ihi1, ihi2, ilo3, ihi3, nv real(rt), intent(in) :: q(qd_lo(1):qd_hi(1),qd_lo(2):qd_hi(2),qd_lo(3):qd_hi(3),nv) real(rt), intent(out) :: dqxal(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),5) - + integer i, j, k, n real(rt) :: slop, dsgn,dlim,dcen real(rt) :: dlft(ilo1-1:ihi1+1,5), drgt(ilo1-1:ihi1+1,5) @@ -43,7 +43,7 @@ subroutine slopex(q,qd_lo,qd_hi, & enddo enddo enddo - + else ! Compute slopes in first coordinate direction @@ -54,14 +54,14 @@ subroutine slopex(q,qd_lo,qd_hi, & dlft(i,1) = 0.5d0*(q(i,j,k,QP)-q(i-1,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QU) - q(i-1,j,k,QU)) dlft(i,2) = (q(i,j,k,QRHO)-q(i-1,j,k,QRHO))- (q(i,j,k,QP) - q(i-1,j,k,QP))/q(i,j,k,QC)**2 dlft(i,3) = 0.5d0*(q(i,j,k,QP)-q(i-1,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QU) - q(i-1,j,k,QU)) - dlft(i,4) = q(i,j,k,QV) - q(i-1,j,k,QV) - dlft(i,5) = q(i,j,k,QW) - q(i-1,j,k,QW) + dlft(i,4) = q(i,j,k,QV) - q(i-1,j,k,QV) + dlft(i,5) = q(i,j,k,QW) - q(i-1,j,k,QW) drgt(i,1) = 0.5d0*(q(i+1,j,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i+1,j,k,QU) - q(i,j,k,QU)) drgt(i,2) = (q(i+1,j,k,QRHO)-q(i,j,k,QRHO))- (q(i+1,j,k,QP) - q(i,j,k,QP))/q(i,j,k,QC)**2 drgt(i,3) = 0.5d0*(q(i+1,j,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i+1,j,k,QU) - q(i,j,k,QU)) - drgt(i,4) = q(i+1,j,k,QV) - q(i,j,k,QV) - drgt(i,5) = q(i+1,j,k,QW) - q(i,j,k,QW) + drgt(i,4) = q(i+1,j,k,QV) - q(i,j,k,QV) + drgt(i,5) = q(i+1,j,k,QW) - q(i,j,k,QW) enddo @@ -94,14 +94,14 @@ subroutine slopey(q,qd_lo,qd_hi, & integer, intent(in) :: ilo1, ilo2, ihi1, ihi2, ilo3, ihi3, nv real(rt), intent(in) :: q(qd_lo(1):qd_hi(1),qd_lo(2):qd_hi(2),qd_lo(3):qd_hi(3),nv) real(rt), intent(out) :: dqyal(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),5) - + integer i, j, k, n real(rt) slop real(rt)::dsgn,dlim,dcen real(rt) :: dlft(ilo1:ihi1,5), drgt(ilo1:ihi1,5) if(plm_iorder.eq.1) then - + do n = 1, 5 do k = ilo3, ihi3 do j = ilo2-1, ihi2+1 @@ -111,28 +111,28 @@ subroutine slopey(q,qd_lo,qd_hi, & enddo enddo enddo - + else - + do k = ilo3, ihi3 do j = ilo2-1, ihi2+1 - + do i = ilo1, ihi1 - + dlft(i,1) = 0.5d0*(q(i,j,k,QP)-q(i,j-1,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QV) - q(i,j-1,k,QV)) dlft(i,2) = (q(i,j,k,QRHO)-q(i,j-1,k,QRHO))- (q(i,j,k,QP) - q(i,j-1,k,QP))/q(i,j,k,QC)**2 dlft(i,3) = 0.5d0*(q(i,j,k,QP)-q(i,j-1,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QV) - q(i,j-1,k,QV)) - dlft(i,4) = q(i,j,k,QU) - q(i,j-1,k,QU) - dlft(i,5) = q(i,j,k,QW) - q(i,j-1,k,QW) - + dlft(i,4) = q(i,j,k,QU) - q(i,j-1,k,QU) + dlft(i,5) = q(i,j,k,QW) - q(i,j-1,k,QW) + drgt(i,1) = 0.5d0*(q(i,j+1,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j+1,k,QV) - q(i,j,k,QV)) drgt(i,2) = (q(i,j+1,k,QRHO)-q(i,j,k,QRHO))- (q(i,j+1,k,QP) - q(i,j,k,QP))/q(i,j,k,QC)**2 drgt(i,3) = 0.5d0*(q(i,j+1,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j+1,k,QV) - q(i,j,k,QV)) - drgt(i,4) = q(i,j+1,k,QU) - q(i,j,k,QU) - drgt(i,5) = q(i,j+1,k,QW) - q(i,j,k,QW) - + drgt(i,4) = q(i,j+1,k,QU) - q(i,j,k,QU) + drgt(i,5) = q(i,j+1,k,QW) - q(i,j,k,QW) + enddo - + do n=1,5 ! First compute Fromm slopes do i = ilo1, ihi1 @@ -147,29 +147,29 @@ subroutine slopey(q,qd_lo,qd_hi, & dqyal(i,j,k,n) = dsgn*min( dlim, abs(dcen) ) enddo enddo - + enddo enddo end if - + end subroutine slopey - + subroutine slopez(q,qd_lo,qd_hi, & dqzal,qpd_lo,qpd_hi, & ilo1,ilo2,ilo3,ihi1,ihi2,ihi3,nv) - + integer, intent(in) :: qd_lo(3), qd_hi(3) integer, intent(in) :: qpd_lo(3),qpd_hi(3) integer, intent(in) :: ilo1, ilo2, ihi1, ihi2, ilo3, ihi3, nv real(rt), intent(in) :: q(qd_lo(1):qd_hi(1),qd_lo(2):qd_hi(2),qd_lo(3):qd_hi(3),nv) real(rt), intent(out) :: dqzal(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),5) - + integer i, j, k, n real(rt) slop,dsgn,dlim,dcen real(rt) :: dlft(ilo1:ihi1,nv), drgt(ilo1:ihi1,nv) - + if(plm_iorder.eq.1) then - + do n = 1, 5 do k = ilo3-1, ihi3+1 do j = ilo2, ihi2 @@ -179,29 +179,29 @@ subroutine slopez(q,qd_lo,qd_hi, & enddo enddo enddo - + else - + do k = ilo3-1, ihi3+1 do j = ilo2, ihi2 do i = ilo1, ihi1 - + dlft(i,1) = 0.5d0*(q(i,j,k,QP)-q(i,j,k-1,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QW) - q(i,j,k-1,QW)) dlft(i,2) = (q(i,j,k,QRHO)-q(i,j,k-1,QRHO))- (q(i,j,k,QP) - q(i,j,k-1,QP))/q(i,j,k,QC)**2 dlft(i,3) = 0.5d0*(q(i,j,k,QP)-q(i,j,k-1,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QW) - q(i,j,k-1,QW)) - dlft(i,4) = q(i,j,k,QU) - q(i,j,k-1,QU) - dlft(i,5) = q(i,j,k,QV) - q(i,j,k-1,QV) - + dlft(i,4) = q(i,j,k,QU) - q(i,j,k-1,QU) + dlft(i,5) = q(i,j,k,QV) - q(i,j,k-1,QV) + drgt(i,1) = 0.5d0*(q(i,j,k+1,QP)-q(i,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k+1,QW) - q(i,j,k,QW)) drgt(i,2) = (q(i,j,k+1,QRHO)-q(i,j,k,QRHO))- (q(i,j,k+1,QP) - q(i,j,k,QP))/q(i,j,k,QC)**2 drgt(i,3) = 0.5d0*(q(i,j,k+1,QP)-q(i,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k+1,QW) - q(i,j,k,QW)) - drgt(i,4) = q(i,j,k+1,QU) - q(i,j,k,QU) - drgt(i,5) = q(i,j,k+1,QV) - q(i,j,k,QV) - + drgt(i,4) = q(i,j,k+1,QU) - q(i,j,k,QU) + drgt(i,5) = q(i,j,k+1,QV) - q(i,j,k,QV) + enddo - + do n=1,5 - + do i = ilo1, ihi1 dcen = 0.5d0 * (dlft(i,n)+drgt(i,n)) dsgn = sign(1.d0, dcen) @@ -213,12 +213,12 @@ subroutine slopez(q,qd_lo,qd_hi, & endif dqzal(i,j,k,n) = dsgn*min( dlim, abs(dcen) ) enddo - + enddo enddo enddo endif - + end subroutine slopez end module slope_module diff --git a/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma_EB.F90 b/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma_EB.F90 index e7e107d791e..3f48f84ca98 100644 --- a/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma_EB.F90 +++ b/Tutorials/EB/CNS/Source/hydro/slope_mol_3d_gamma_EB.F90 @@ -44,7 +44,7 @@ subroutine ebslopex(q,qd_lo,qd_hi, & enddo enddo enddo - + else ! Compute slopes in first coordinate direction @@ -58,8 +58,8 @@ subroutine ebslopex(q,qd_lo,qd_hi, & dlft(i,1) = 0.5d0*(q(i,j,k,QP)-q(i-1,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QU) - q(i-1,j,k,QU)) dlft(i,2) = (q(i,j,k,QRHO)-q(i-1,j,k,QRHO))- (q(i,j,k,QP) - q(i-1,j,k,QP))/q(i,j,k,QC)**2 dlft(i,3) = 0.5d0*(q(i,j,k,QP)-q(i-1,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QU) - q(i-1,j,k,QU)) - dlft(i,4) = q(i,j,k,QV) - q(i-1,j,k,QV) - dlft(i,5) = q(i,j,k,QW) - q(i-1,j,k,QW) + dlft(i,4) = q(i,j,k,QV) - q(i-1,j,k,QV) + dlft(i,5) = q(i,j,k,QW) - q(i-1,j,k,QW) else dlft(i,:) = 0.d0 end if @@ -68,8 +68,8 @@ subroutine ebslopex(q,qd_lo,qd_hi, & drgt(i,1) = 0.5d0*(q(i+1,j,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i+1,j,k,QU) - q(i,j,k,QU)) drgt(i,2) = (q(i+1,j,k,QRHO)-q(i,j,k,QRHO))- (q(i+1,j,k,QP) - q(i,j,k,QP))/q(i,j,k,QC)**2 drgt(i,3) = 0.5d0*(q(i+1,j,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i+1,j,k,QU) - q(i,j,k,QU)) - drgt(i,4) = q(i+1,j,k,QV) - q(i,j,k,QV) - drgt(i,5) = q(i+1,j,k,QW) - q(i,j,k,QW) + drgt(i,4) = q(i+1,j,k,QV) - q(i,j,k,QV) + drgt(i,5) = q(i+1,j,k,QW) - q(i,j,k,QW) else drgt(i,:) = 0.d0 end if @@ -108,14 +108,14 @@ subroutine ebslopey(q,qd_lo,qd_hi, & real(rt), intent(in) :: q(qd_lo(1):qd_hi(1),qd_lo(2):qd_hi(2),qd_lo(3):qd_hi(3),nv) real(rt), intent(out) :: dqyal(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),5) integer, intent(in) :: flag(fglo(1):fghi(1),fglo(2):fghi(2),fglo(3):fghi(3)) - + integer i, j, k, n real(rt) slop, dsgn,dlim,dcen real(rt) :: dlft(ilo1:ihi1,5), drgt(ilo1:ihi1,5) integer :: nbr(-1:1,-1:1,-1:1) - + if(plm_iorder.eq.1) then - + do n = 1, 5 do k = ilo3, ihi3 do j = ilo2-1, ihi2+1 @@ -125,9 +125,9 @@ subroutine ebslopey(q,qd_lo,qd_hi, & enddo enddo enddo - + else - + ! Compute slopes in first coordinate direction do k = ilo3, ihi3 do j = ilo2-1, ihi2+1 @@ -138,8 +138,8 @@ subroutine ebslopey(q,qd_lo,qd_hi, & dlft(i,1) = 0.5d0*(q(i,j,k,QP)-q(i,j-1,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QV) - q(i,j-1,k,QV)) dlft(i,2) = (q(i,j,k,QRHO)-q(i,j-1,k,QRHO))- (q(i,j,k,QP) - q(i,j-1,k,QP))/q(i,j,k,QC)**2 dlft(i,3) = 0.5d0*(q(i,j,k,QP)-q(i,j-1,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QV) - q(i,j-1,k,QV)) - dlft(i,4) = q(i,j,k,QU) - q(i,j-1,k,QU) - dlft(i,5) = q(i,j,k,QW) - q(i,j-1,k,QW) + dlft(i,4) = q(i,j,k,QU) - q(i,j-1,k,QU) + dlft(i,5) = q(i,j,k,QW) - q(i,j-1,k,QW) else dlft(i,:) = 0.d0 end if @@ -148,13 +148,13 @@ subroutine ebslopey(q,qd_lo,qd_hi, & drgt(i,1) = 0.5d0*(q(i,j+1,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j+1,k,QV) - q(i,j,k,QV)) drgt(i,2) = (q(i,j+1,k,QRHO)-q(i,j,k,QRHO))- (q(i,j+1,k,QP) - q(i,j,k,QP))/q(i,j,k,QC)**2 drgt(i,3) = 0.5d0*(q(i,j+1,k,QP)-q(i,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j+1,k,QV) - q(i,j,k,QV)) - drgt(i,4) = q(i,j+1,k,QU) - q(i,j,k,QU) + drgt(i,4) = q(i,j+1,k,QU) - q(i,j,k,QU) drgt(i,5) = q(i,j+1,k,QW) - q(i,j,k,QW) else drgt(i,:) = 0.d0 end if enddo - + do n=1,5 ! First compute Fromm slopes do i = ilo1, ihi1 @@ -172,14 +172,14 @@ subroutine ebslopey(q,qd_lo,qd_hi, & enddo enddo end if - + end subroutine ebslopey - + subroutine ebslopez(q,qd_lo,qd_hi, & dqzal,qpd_lo,qpd_hi, & flag, fglo, fghi, & ilo1,ilo2,ilo3,ihi1,ihi2,ihi3,nv) - + integer, intent(in) :: qd_lo(3), qd_hi(3) integer, intent(in) :: qpd_lo(3),qpd_hi(3) integer, intent(in) :: fglo(3), fghi(3) @@ -187,14 +187,14 @@ subroutine ebslopez(q,qd_lo,qd_hi, & real(rt), intent(in) :: q(qd_lo(1):qd_hi(1),qd_lo(2):qd_hi(2),qd_lo(3):qd_hi(3),nv) real(rt), intent(out) :: dqzal(qpd_lo(1):qpd_hi(1),qpd_lo(2):qpd_hi(2),qpd_lo(3):qpd_hi(3),5) integer, intent(in) :: flag(fglo(1):fghi(1),fglo(2):fghi(2),fglo(3):fghi(3)) - + integer i, j, k, n real(rt) slop,dsgn,dlim,dcen real(rt) :: dlft(ilo1:ihi1,nv), drgt(ilo1:ihi1,nv) integer :: nbr(-1:1,-1:1,-1:1) - + if(plm_iorder.eq.1) then - + do n = 1, 5 do k = ilo3-1, ihi3+1 do j = ilo2, ihi2 @@ -216,8 +216,8 @@ subroutine ebslopez(q,qd_lo,qd_hi, & dlft(i,1) = 0.5d0*(q(i,j,k,QP)-q(i,j,k-1,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QW) - q(i,j,k-1,QW)) dlft(i,2) = (q(i,j,k,QRHO)-q(i,j,k-1,QRHO))- (q(i,j,k,QP) - q(i,j,k-1,QP))/q(i,j,k,QC)**2 dlft(i,3) = 0.5d0*(q(i,j,k,QP)-q(i,j,k-1,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k,QW) - q(i,j,k-1,QW)) - dlft(i,4) = q(i,j,k,QU) - q(i,j,k-1,QU) - dlft(i,5) = q(i,j,k,QV) - q(i,j,k-1,QV) + dlft(i,4) = q(i,j,k,QU) - q(i,j,k-1,QU) + dlft(i,5) = q(i,j,k,QV) - q(i,j,k-1,QV) else dlft(i,:) = 0.d0 end if @@ -226,8 +226,8 @@ subroutine ebslopez(q,qd_lo,qd_hi, & drgt(i,1) = 0.5d0*(q(i,j,k+1,QP)-q(i,j,k,QP))/q(i,j,k,QC) - 0.5d0*q(i,j,k,QRHO)*(q(i,j,k+1,QW) - q(i,j,k,QW)) drgt(i,2) = (q(i,j,k+1,QRHO)-q(i,j,k,QRHO))- (q(i,j,k+1,QP) - q(i,j,k,QP))/q(i,j,k,QC)**2 drgt(i,3) = 0.5d0*(q(i,j,k+1,QP)-q(i,j,k,QP))/q(i,j,k,QC) + 0.5d0*q(i,j,k,QRHO)*(q(i,j,k+1,QW) - q(i,j,k,QW)) - drgt(i,4) = q(i,j,k+1,QU) - q(i,j,k,QU) - drgt(i,5) = q(i,j,k+1,QV) - q(i,j,k,QV) + drgt(i,4) = q(i,j,k+1,QU) - q(i,j,k,QU) + drgt(i,5) = q(i,j,k+1,QV) - q(i,j,k,QV) else drgt(i,:) = 0.d0 end if @@ -249,7 +249,7 @@ subroutine ebslopez(q,qd_lo,qd_hi, & enddo enddo endif - + end subroutine ebslopez end module ebslope_module diff --git a/Tutorials/EB/CNS/Source/main.cpp b/Tutorials/EB/CNS/Source/main.cpp index f4a24e07825..b61c205af54 100644 --- a/Tutorials/EB/CNS/Source/main.cpp +++ b/Tutorials/EB/CNS/Source/main.cpp @@ -26,7 +26,7 @@ int main (int argc, char* argv[]) Real stop_time; { - ParmParse pp; + ParmParse pp; max_step = -1; strt_time = 0.0; @@ -38,49 +38,49 @@ int main (int argc, char* argv[]) } if (strt_time < 0.0) { - amrex::Abort("MUST SPECIFY a non-negative strt_time"); + amrex::Abort("MUST SPECIFY a non-negative strt_time"); } if (max_step < 0 && stop_time < 0.0) { - amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); + amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); } { timer_init = amrex::second(); - Amr amr; + Amr amr; AmrLevel::SetEBSupportLevel(EBSupport::full); AmrLevel::SetEBMaxGrowCells(CNS::numGrow(),4,2); initialize_EB2(amr.Geom(amr.maxLevel()), amr.maxLevel(), amr.maxLevel()); - - amr.init(strt_time,stop_time); + + amr.init(strt_time,stop_time); timer_init = amrex::second() - timer_init; timer_advance = amrex::second(); - while ( amr.okToContinue() && - (amr.levelSteps(0) < max_step || max_step < 0) && - (amr.cumTime() < stop_time || stop_time < 0.0) ) - - { - // - // Do a coarse timestep. Recursively calls timeStep() - // - amr.coarseTimeStep(stop_time); - } - + while ( amr.okToContinue() && + (amr.levelSteps(0) < max_step || max_step < 0) && + (amr.cumTime() < stop_time || stop_time < 0.0) ) + + { + // + // Do a coarse timestep. Recursively calls timeStep() + // + amr.coarseTimeStep(stop_time); + } + timer_advance = amrex::second() - timer_advance; - // Write final checkpoint and plotfile - if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { - amr.checkPoint(); - } - - if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { - amr.writePlotFile(); - } + // Write final checkpoint and plotfile + if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { + amr.checkPoint(); + } + + if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { + amr.writePlotFile(); + } } timer_tot = amrex::second() - timer_tot; diff --git a/Tutorials/EB/GeometryGeneration/main.cpp b/Tutorials/EB/GeometryGeneration/main.cpp index 7610312d99b..cdec3a20e02 100644 --- a/Tutorials/EB/GeometryGeneration/main.cpp +++ b/Tutorials/EB/GeometryGeneration/main.cpp @@ -30,14 +30,14 @@ int main (int argc, char* argv[]) Array is_periodic{false, false, false}; Geometry::Setup(&rb, 0, is_periodic.data()); Box domain(IntVect(0), IntVect(n_cell-1)); - geom.define(domain); + geom.define(domain); } if (which_geom == 0) { EB2::SphereIF sphere(0.5, {0.0,0.0,0.0}, false); EB2::BoxIF cube({-0.4,-0.4,-0.4}, {0.4,0.4,0.4}, false); auto cubesphere = EB2::makeIntersection(sphere, cube); - + EB2::CylinderIF cylinder_x(0.25, 0, {0.0,0.0,0.0}, false); EB2::CylinderIF cylinder_y(0.25, 1, {0.0,0.0,0.0}, false); EB2::CylinderIF cylinder_z(0.25, 2, {0.0,0.0,0.0}, false); @@ -87,7 +87,7 @@ int main (int argc, char* argv[]) splpts.push_back(p); EB2::SplineIF Piston; Piston.addSplineElement(splpts); - + std::vector lnpts; p = amrex::RealVect(D_DECL(22.358*0.1*scaleFact, -7.6902*0.1*scaleFact, 0.0)); diff --git a/Tutorials/EB/MacProj/main.cpp b/Tutorials/EB/MacProj/main.cpp index 7f00e1f906f..9c8ad9efe0b 100644 --- a/Tutorials/EB/MacProj/main.cpp +++ b/Tutorials/EB/MacProj/main.cpp @@ -39,7 +39,7 @@ int main (int argc, char* argv[]) Box domain(IntVect{AMREX_D_DECL(0,0,0)}, IntVect{AMREX_D_DECL(n_cell-1,n_cell-1,n_cell-1)}); geom.define(domain); - + grids.define(domain); grids.maxSize(max_grid_size); @@ -83,13 +83,13 @@ int main (int argc, char* argv[]) // copy velocity into plotfile average_face_to_cellcenter(plotfile_mf,0,amrex::GetArrOfConstPtrs(vel)); - + // compute and output divergence, then copy into plofile MultiFab divu(grids, dmap, 1, 0, MFInfo(), factory); EB_computeDivergence(divu, amrex::GetArrOfConstPtrs(vel), geom, false); amrex::Print() << "\nmax-norm of divu before projection is " << divu.norm0() << "\n" << std::endl; plotfile_mf.copy(divu,0,AMREX_SPACEDIM,1); - + MacProjector macproj({amrex::GetArrOfPtrs(vel)}, // mac velocity MLMG::Location::FaceCenter, {amrex::GetArrOfConstPtrs(beta)}, // beta @@ -134,7 +134,7 @@ int main (int argc, char* argv[]) "divu-before", "after-vx", "after-vy", #if (AMREX_SPACEDIM == 3) - "after-vz", + "after-vz", #endif "divu-after"}, geom, 0.0, 0); diff --git a/Tutorials/EB/Poisson/main.cpp b/Tutorials/EB/Poisson/main.cpp index a2bda98ff7c..4a44a7a2836 100644 --- a/Tutorials/EB/Poisson/main.cpp +++ b/Tutorials/EB/Poisson/main.cpp @@ -77,7 +77,7 @@ int main (int argc, char* argv[]) // define array of LinOpBCType for domain boundary conditions std::array bc_lo; std::array bc_hi; - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { bc_lo[idim] = LinOpBCType::Periodic; bc_hi[idim] = LinOpBCType::Periodic; } @@ -88,14 +88,14 @@ int main (int argc, char* argv[]) // see AMReX_MLLinOp.H for an explanation mlebabec.setLevelBC(0, nullptr); - + // operator looks like (ACoef - div BCoef grad) phi = rhs // set ACoef to zero MultiFab acoef(grids, dmap, 1, 0, MFInfo(), factory); acoef.setVal(0.); mlebabec.setACoeffs(0, acoef); - + // set BCoef to 1.0 (and array of face-centered coefficients) Array bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { @@ -123,16 +123,16 @@ int main (int argc, char* argv[]) const Real tol_abs = 0.0; mlmg.setVerbose(verbose); - + // Solve linear system phi.setVal(0.0); // initial guess for phi mlmg.solve({&phi}, {&q}, tol_rel, tol_abs); - + // store plotfile variables; q and phi MultiFab plotfile_mf(grids, dmap, 2, 0, MFInfo(), factory); MultiFab::Copy(plotfile_mf, q,0,0,1,0); MultiFab::Copy(plotfile_mf,phi,0,1,1,0); - + EB_WriteSingleLevelPlotfile("plt", plotfile_mf, {"q", "phi"}, geom, 0.0, 0); } diff --git a/Tutorials/ForkJoin/MLMG/ff.f90 b/Tutorials/ForkJoin/MLMG/ff.f90 index 191700db7bf..5c75c43b97c 100644 --- a/Tutorials/ForkJoin/MLMG/ff.f90 +++ b/Tutorials/ForkJoin/MLMG/ff.f90 @@ -13,7 +13,7 @@ subroutine fort_set_rhs(rhs, lo, hi, nc, dx, a, b, sigma, w) bind(c) tpi = 2.0d0 * pi fpi = 4.0d0 * pi fac = 2.0d0 * tpi**2 - + theta = 0.5d0*log(3.0) / w do n = 1, nc @@ -21,14 +21,14 @@ subroutine fort_set_rhs(rhs, lo, hi, nc, dx, a, b, sigma, w) bind(c) y = (dble(j)+0.5d0)*dx(2) do i = lo(1), hi(1) x = (dble(i)+0.5d0)*dx(1) - + r = sqrt((x-0.5d0)**2+(y-0.5d0)**2) beta = (sigma-1.d0)/2.d0*tanh(theta*(r-0.25d0)) + (sigma+1.d0)/2.d0 beta = beta * b dbdrfac = (sigma-1.d0)/2.d0/(cosh(theta*(r-0.25d0)))**2 * theta/r dbdrfac = dbdrfac * b - + rhs(i,j,n) = beta*fac*(sin(tpi*x) * sin(tpi*y) & & + sin(fpi*x) * sin(fpi*y)) & & + dbdrfac*((x-0.5d0)*(-tpi*cos(tpi*x) * sin(tpi*y) & @@ -60,7 +60,7 @@ subroutine fort_set_coef(xcoef, xlo, xhi, ycoef, ylo, yhi, nc, dx, sigma, w) bin y = (dble(j)+0.5d0)*dx(2) do i = xlo(1), xhi(1) x = (dble(i))*dx(1) - + r = sqrt((x-0.5d0)**2 + (y-0.5d0)**2) xcoef(i,j,n) = (sigma-1.d0)/2.d0*tanh(theta*(r-0.25d0)) & @@ -72,7 +72,7 @@ subroutine fort_set_coef(xcoef, xlo, xhi, ycoef, ylo, yhi, nc, dx, sigma, w) bin y = (dble(j))*dx(2) do i = ylo(1), yhi(1) x = (dble(i)+0.5d0)*dx(1) - + r = sqrt((x-0.5d0)**2 + (y-0.5d0)**2) ycoef(i,j,n) = (sigma-1.d0)/2.d0*tanh(theta*(r-0.25d0)) & diff --git a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_2d.f90 b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_2d.f90 index 1b7d0a99c07..63bce7ea18e 100644 --- a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_2d.f90 +++ b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_2d.f90 @@ -13,7 +13,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & dx, prob_lo) use amrex_fort_module, only : amrex_spacedim, amrex_real - + implicit none integer, intent(in) :: level, lo(3), hi(3), phi_lo(3), phi_hi(3) real(amrex_real), intent(in) :: time @@ -21,10 +21,10 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & & phi_lo(2):phi_hi(2), & & phi_lo(3):phi_hi(3)) real(amrex_real), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(amrex_real) :: x,y,z,r2 - + !$omp parallel do private(i,j,k,x,y,z,r2) collapse(2) do k=lo(3),hi(3) do j=lo(2),hi(2) @@ -32,7 +32,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + if ( amrex_spacedim .eq. 2) then r2 = ((x-0.5d0)**2 + (y-0.75d0)**2) / 0.01d0 phi(i,j,k) = 1.d0 + exp(-r2) @@ -44,7 +44,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & end do end do !$omp end parallel do - + end subroutine init_prob_data subroutine init_part_data(pc, lev, mfi, lo, hi, dx, prob_lo) @@ -53,7 +53,7 @@ subroutine init_part_data(pc, lev, mfi, lo, hi, dx, prob_lo) use amrex_particlecontainer_module, only: amrex_particlecontainer, amrex_particle, & amrex_get_next_particle_id, amrex_get_cpu, amrex_set_particle_id, amrex_set_particle_cpu use amrex_multifab_module, only : amrex_mfiter - + implicit none type(amrex_particlecontainer), intent(inout) :: pc integer, intent(in) :: lev @@ -64,7 +64,7 @@ subroutine init_part_data(pc, lev, mfi, lo, hi, dx, prob_lo) integer :: i,j real(amrex_real) :: x,y type(amrex_particle) :: p - + do j=lo(2),hi(2) y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) @@ -82,8 +82,8 @@ subroutine init_part_data(pc, lev, mfi, lo, hi, dx, prob_lo) end do end do - + end subroutine init_part_data - + end module prob_module diff --git a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_3d.f90 b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_3d.f90 index 8eb0c211828..7af9a1cceb9 100644 --- a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_3d.f90 +++ b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/Prob_3d.f90 @@ -36,7 +36,7 @@ subroutine init_prob_data(level, time, lo, hi, & integer :: dm integer :: i,j,k double precision :: x,y,z,r2 - + if (phi_lo(3) .eq. 0 .and. phi_hi(3) .eq. 0) then dm = 2 else @@ -50,7 +50,7 @@ subroutine init_prob_data(level, time, lo, hi, & y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + if ( dm.eq. 2) then r2 = ((x-0.5d0)**2 + (y-0.75d0)**2) / 0.01d0 phi(i,j,k) = 1.d0 + exp(-r2) @@ -71,14 +71,14 @@ subroutine init_part_data(pc, lev, mfi, lo, hi, dx, prob_lo) use amrex_particlecontainer_module, only: amrex_particlecontainer, amrex_particle, & amrex_get_next_particle_id, amrex_get_cpu, amrex_set_particle_id, amrex_set_particle_cpu use amrex_multifab_module, only : amrex_mfiter - + implicit none type(amrex_particlecontainer), intent(inout) :: pc integer, intent(in) :: lev type(amrex_mfiter), intent(in) :: mfi integer, intent(in) :: lo(3), hi(3) real(amrex_real), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(amrex_real) :: x,y,z type(amrex_particle) :: p diff --git a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_2d.F90 b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_2d.F90 index cbce4d465c7..a64ab0f32a2 100644 --- a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_2d.F90 +++ b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_2d.F90 @@ -28,9 +28,9 @@ subroutine get_face_velocity(time, & plo(2) = min(vxlo(2)-1, vylo(2)-1) phi(1) = max(vxhi(1) , vyhi(1)+1) phi(2) = max(vxhi(2)+1, vyhi(2) ) - + call amrex_allocate(psi, plo, phi) - + ! streamfunction psi do j = plo(2), phi(2) y = (dble(j)+0.5d0)*dx(2) + prob_lo(2) @@ -39,7 +39,7 @@ subroutine get_face_velocity(time, & psi(i,j) = sin(M_PI*x)**2 * sin(M_PI*y)**2 * cos (M_PI*time/2.d0) * (1.d0 / M_PI) end do end do - + ! x velocity do j = vxlo(2), vxhi(2) y = (dble(j)+0.5d0) * dx(2) + prob_lo(2) @@ -48,7 +48,7 @@ subroutine get_face_velocity(time, & vx(i,j) = -( (psi(i,j+1)+psi(i-1,j+1)) - (psi(i,j-1)+psi(i-1,j-1)) ) * (0.25d0/dx(2)) end do end do - + ! y velocity do j = vylo(2), vyhi(2) y = dble(j) * dx(2) + prob_lo(2) @@ -57,9 +57,9 @@ subroutine get_face_velocity(time, & vy(i,j) = ( (psi(i+1,j)+psi(i+1,j-1)) - (psi(i-1,j)+psi(i-1,j-1)) ) * (0.25d0/dx(1)) end do end do - + call amrex_deallocate(psi) - + end subroutine get_face_velocity end module face_velocity_module diff --git a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_3d.F90 b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_3d.F90 index 4f385499f42..1c5c6e1b67c 100644 --- a/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_3d.F90 +++ b/Tutorials/FortranInterface/Advection_F/Exec/SingleVortex/face_velocity_3d.F90 @@ -36,7 +36,7 @@ subroutine get_face_velocity(time, & vx_l1=vxlo(1);vx_l2=vxlo(2);vx_l3=vxlo(3) vy_l1=vylo(1);vy_l2=vylo(2);vy_l3=vylo(3) - vz_l1=vzlo(1);vz_l2=vzlo(2);vz_l3=vzlo(3) + vz_l1=vzlo(1);vz_l2=vzlo(2);vz_l3=vzlo(3) vx_h1=vxhi(1);vx_h2=vxhi(2);vx_h3=vxhi(3) vy_h1=vyhi(1);vy_h2=vyhi(2);vy_h3=vyhi(3) vz_h1=vzhi(1);vz_h2=vzhi(2);vz_h3=vzhi(3) @@ -45,7 +45,7 @@ subroutine get_face_velocity(time, & plo(2) = min(vx_l2-1, vy_l2-1) phi(1) = max(vx_h1 , vy_h1+1) phi(2) = max(vx_h2+1, vy_h2 ) - + call bl_allocate(psi, plo(1), phi(1), plo(2), phi(2)) ! streamfunction psi @@ -56,7 +56,7 @@ subroutine get_face_velocity(time, & psi(i,j) = sin(M_PI*x)**2 * sin(M_PI*y)**2 * cos (M_PI*time/2.d0) * (1.d0 / M_PI) end do end do - + ! x velocity do k = vx_l3, vx_h3 do j = vx_l2, vx_h2 diff --git a/Tutorials/FortranInterface/Advection_F/Source/Src_2d/advect_2d_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/Src_2d/advect_2d_mod.F90 index 7832a876d4c..00bf308be08 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/Src_2d/advect_2d_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/Src_2d/advect_2d_mod.F90 @@ -5,7 +5,7 @@ module advect_module implicit none private - + public :: advect, advect_particles contains @@ -35,15 +35,15 @@ subroutine advect(time, lo, hi, & real(amrex_real), intent(in ) :: vy (vy_lo(1):vy_hi(1),vy_lo(2):vy_hi(2)) real(amrex_real), intent( out) :: flxx(fx_lo(1):fx_hi(1),fx_lo(2):fx_hi(2)) real(amrex_real), intent( out) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) - + integer :: i, j integer :: glo(2), ghi(2) real(amrex_real) :: dtdx(2), umax, vmax - + real(amrex_real), dimension(:,:), pointer, contiguous :: phix_1d, phiy_1d, phix, phiy, slope dtdx = dt/dx - + glo = lo - 1 ghi = hi + 1 @@ -59,9 +59,9 @@ subroutine advect(time, lo, hi, & ! to remove their pointerness for performance, because normally pointers could ! be aliasing. We need to use pointers instead of allocatable arrays because ! we like to use AMReX's amrex_allocate to allocate memeory instead of the intrinsic - ! allocate. Amrex_allocate is much faster than allocate inside OMP. + ! allocate. Amrex_allocate is much faster than allocate inside OMP. ! Note that one MUST CALL AMREX_DEALLOCATE. - + ! check if CFL condition is violated. umax = maxval(abs(vx)) vmax = maxval(abs(vy)) @@ -88,27 +88,27 @@ subroutine advect(time, lo, hi, & + (flxy(i,j) - flxy(i,j+1)) * dtdx(2) ) enddo enddo - + ! Scale by face area in order to correctly reflx do j = lo(2), hi(2) do i = lo(1), hi(1)+1 flxx(i,j) = flxx(i,j) * ( dt * dx(2)) enddo enddo - + ! Scale by face area in order to correctly reflx - do j = lo(2), hi(2)+1 + do j = lo(2), hi(2)+1 do i = lo(1), hi(1) flxy(i,j) = flxy(i,j) * (dt * dx(1)) enddo enddo - + call amrex_deallocate(phix_1d) call amrex_deallocate(phiy_1d) call amrex_deallocate(phix) call amrex_deallocate(phiy) call amrex_deallocate(slope) - + end subroutine advect subroutine advect_particles(particles, np, & @@ -126,11 +126,11 @@ subroutine advect_particles(particles, np, & real(amrex_real), intent(in) :: dt real(amrex_real), intent(in) :: plo(2) real(amrex_real), intent(in) :: dx(2) - + integer cell(2) integer cc_cell(2) integer e_cell(2) - + integer ipass, n, d real(amrex_real) w_lo(2), w_hi(2) real(amrex_real) e_lo(2), e_hi(2) @@ -147,32 +147,32 @@ subroutine advect_particles(particles, np, & if (np == 0) then return end if - + velocity(1)%p => ux velocity(2)%p => uy - + inv_dx = 1.0d0/dx do ipass = 1, 2 do n = 1, np - length = (particles(n)%pos - plo)*inv_dx + length = (particles(n)%pos - plo)*inv_dx cc_cell = floor(length) cell = floor(length + 0.5d0) - - w_hi = length + 0.5d0 - cell + + w_hi = length + 0.5d0 - cell w_lo = 1.d0 - w_hi do d = 1, 2 e_cell = cell e_cell(d) = cc_cell(d) + 1 - + e_hi = w_hi e_lo = w_lo e_hi(d) = length(d) - cc_cell(d) - e_hi = max(0.d0,min(1.d0,e_hi)) + e_hi = max(0.d0,min(1.d0,e_hi)) e_lo(d) = 1.d0 - e_hi(d) vel = e_lo(1)*e_lo(2)*velocity(d)%p(e_cell(1)-1, e_cell(2)-1) + & @@ -187,9 +187,9 @@ subroutine advect_particles(particles, np, & particles(n)%pos(d) = particles(n)%vel(d) + dt*vel particles(n)%vel(d) = vel end if - end do + end do end do end do end subroutine advect_particles - + end module advect_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/Src_2d/compute_flux_2d.f90 b/Tutorials/FortranInterface/Advection_F/Source/Src_2d/compute_flux_2d.f90 index 24fdbac7fbe..d78f419ebda 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/Src_2d/compute_flux_2d.f90 +++ b/Tutorials/FortranInterface/Advection_F/Source/Src_2d/compute_flux_2d.f90 @@ -34,7 +34,7 @@ subroutine compute_flux_2d(lo, hi, dt, dx, & real(amrex_real), intent( out) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) real(amrex_real), dimension(glo(1):ghi(1),glo(2):ghi(2)) :: & phix_1d, phiy_1d, phix, phiy, slope - + integer :: i, j, k real(amrex_real) :: hdtdx(2) diff --git a/Tutorials/FortranInterface/Advection_F/Source/Src_2d/slope_2d.f90 b/Tutorials/FortranInterface/Advection_F/Source/Src_2d/slope_2d.f90 index f32326e5e51..c0daa7e96d0 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/Src_2d/slope_2d.f90 +++ b/Tutorials/FortranInterface/Advection_F/Source/Src_2d/slope_2d.f90 @@ -1,18 +1,18 @@ module slope_module use amrex_base_module - + implicit none real(amrex_real), parameter:: four3rd=4._amrex_real/3._amrex_real, & sixth=1._amrex_real/6._amrex_real - + private - + public :: slopex, slopey - + contains - + subroutine slopex(lo, hi, & q, qlo, qhi, & dq, dqlo, dqhi) @@ -123,4 +123,4 @@ subroutine slopey_doit(lo, hi, & end subroutine slopey_doit -end module slope_module +end module slope_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/Src_3d/advect_3d_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/Src_3d/advect_3d_mod.F90 index 575ea309380..fe922fca485 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/Src_3d/advect_3d_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/Src_3d/advect_3d_mod.F90 @@ -19,7 +19,7 @@ subroutine advect(time, lo, hi, & & flxy, fy_lo, fy_hi, & & flxz, fz_lo, fz_hi, & & dx,dt) bind(C, name="advect") - + use amrex_mempool_module, only : bl_allocate, bl_deallocate use compute_flux_module, only : compute_flux_3d @@ -68,13 +68,13 @@ subroutine advect(time, lo, hi, & call bl_allocate(phiz_x,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) call bl_allocate(phiz_y,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) ! slope - call bl_allocate(slope,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) - + call bl_allocate(slope,glo(1), ghi(1), glo(2), ghi(2), glo(3), ghi(3)) + ! We like to allocate these **pointers** here and then pass them to a function ! to remove their pointerness for performance, because normally pointers could ! be aliasing. We need to use pointers instead of allocatable arrays because ! we like to use AMReX's bl_allocate to allocate memeory instead of the intrinsic - ! allocate. Bl_allocate is much faster than allocate inside OMP. + ! allocate. Bl_allocate is much faster than allocate inside OMP. ! Note that one MUST CALL BL_DEALLOCATE. ! check if CFL condition is violated. @@ -123,7 +123,7 @@ subroutine advect(time, lo, hi, & enddo enddo do k = lo(3), hi(3) - do j = lo(2), hi(2)+1 + do j = lo(2), hi(2)+1 do i = lo(1), hi(1) flxy(i,j,k) = flxy(i,j,k) * (dt * dx(1)*dx(3)) enddo @@ -155,7 +155,7 @@ subroutine advect_particles(particles, np, & use iso_c_binding use amrex_fort_module, only : amrex_real use amrex_particlecontainer_module, only : amrex_particle - + integer, intent(in) :: np type(amrex_particle), target, intent(inout) :: particles(np) integer, intent(in) :: uxlo(3), uxhi(3) @@ -171,7 +171,7 @@ subroutine advect_particles(particles, np, & integer cell(3) integer cc_cell(3) integer e_cell(3) - + integer ipass, n, d, j real(amrex_real) w_lo(3), w_hi(3) real(amrex_real) e_lo(3), e_hi(3) @@ -188,36 +188,36 @@ subroutine advect_particles(particles, np, & if (np == 0) then return end if - + velocity(1)%p => ux velocity(2)%p => uy velocity(3)%p => uy - + inv_dx = 1.0d0/dx - + do ipass = 1, 2 do n = 1, np - length = (particles(n)%pos - plo)*inv_dx + length = (particles(n)%pos - plo)*inv_dx cc_cell = floor(length) cell = floor(length + 0.5d0) - - w_hi = length + 0.5d0 - cell + + w_hi = length + 0.5d0 - cell w_lo = 1.d0 - w_hi - + ! x direction do d = 1, 3 e_cell = cell e_cell(d) = cc_cell(d) + 1 - + e_hi = w_hi e_lo = w_lo e_hi(d) = length(d) - cc_cell(d) e_hi = max(0.d0,min(1.d0,e_hi)) e_lo(d) = 1.d0 - e_hi(d) - + vel = e_lo(1)*e_lo(2)*e_lo(3)*velocity(d)%p(e_cell(1)-1, e_cell(2)-1, e_cell(3)-1) + & e_lo(1)*e_lo(2)*e_hi(3)*velocity(d)%p(e_cell(1)-1, e_cell(2)-1, e_cell(3) ) + & e_lo(1)*e_hi(2)*e_lo(3)*velocity(d)%p(e_cell(1)-1, e_cell(2) , e_cell(3)-1) + & diff --git a/Tutorials/FortranInterface/Advection_F/Source/Src_3d/compute_flux_3d.f90 b/Tutorials/FortranInterface/Advection_F/Source/Src_3d/compute_flux_3d.f90 index fa959b8ac14..26bb127f02f 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/Src_3d/compute_flux_3d.f90 +++ b/Tutorials/FortranInterface/Advection_F/Source/Src_3d/compute_flux_3d.f90 @@ -41,7 +41,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & double precision, intent( out) :: flxz(fz_lo(1):fz_hi(1),fz_lo(2):fz_hi(2),fz_lo(3):fz_hi(3)) double precision, dimension(glo(1):ghi(1),glo(2):ghi(2),glo(3):ghi(3)) :: & phix, phix_y, phix_z, phiy, phiy_x, phiy_z, phiz, phiz_x, phiz_y, slope - + integer :: i, j, k double precision :: hdtdx(3), tdtdx(3) @@ -51,7 +51,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & call slopex(glo, ghi, & phi, ph_lo, ph_hi, & slope, glo, ghi) - + ! compute phi on x faces using umac to upwind; ignore transverse terms do k = lo(3)-1, hi(3)+1 do j = lo(2)-1, hi(2)+1 @@ -70,7 +70,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & call slopey(glo, ghi, & phi, ph_lo, ph_hi, & slope, glo, ghi) - + ! compute phi on y faces using vmac to upwind; ignore transverse terms do k = lo(3)-1, hi(3)+1 do j = lo(2) , hi(2)+1 @@ -89,7 +89,7 @@ subroutine compute_flux_3d(lo, hi, dt, dx, & call slopez(glo, ghi, & phi, ph_lo, ph_hi, & slope, glo, ghi) - + ! compute phi on z faces using wmac to upwind; ignore transverse terms do k = lo(3) , hi(3)+1 do j = lo(2)-1, hi(2)+1 diff --git a/Tutorials/FortranInterface/Advection_F/Source/Src_3d/slope_3d.f90 b/Tutorials/FortranInterface/Advection_F/Source/Src_3d/slope_3d.f90 index 418ac9d5ff3..7a543373b95 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/Src_3d/slope_3d.f90 +++ b/Tutorials/FortranInterface/Advection_F/Source/Src_3d/slope_3d.f90 @@ -1,15 +1,15 @@ module slope_module - + implicit none double precision, parameter:: four3rd=4.d0/3.d0, sixth=1.d0/6.d0 - + private - + public :: slopex, slopey, slopez - + contains - + subroutine slopex(lo, hi, & q, qlo, qhi, & dq, dqlo, dqhi) @@ -40,7 +40,7 @@ subroutine slopex(lo, hi, & endif df(i) = dsgn(i)*min( dlim(i), abs(dcen(i)) ) end do - + ! Now limited fourth order slopes do i = lo(1), hi(1) dq1 = four3rd*dcen(i) - sixth*(df(i+1) + df(i-1)) @@ -116,7 +116,7 @@ subroutine slopey_doit(lo, hi, & df(i,j) = dsgn(i,j)*min( dlim(i,j),abs(dcen(i,j)) ) end do end do - + ! Now compute limited fourth order slopes do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -195,7 +195,7 @@ subroutine slopez_doit(lo, hi, & end do end do end do - + ! Now compute limited fourth order slopes do k = lo(3), hi(3) do j = lo(2), hi(2) @@ -208,4 +208,4 @@ subroutine slopez_doit(lo, hi, & end subroutine slopez_doit -end module slope_module +end module slope_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/amr_data_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/amr_data_mod.F90 index 4179aa8f769..b3baf9f15d4 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/amr_data_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/amr_data_mod.F90 @@ -6,7 +6,7 @@ module amr_data_module use amrex_fort_module, only : rt => amrex_real use amrex_particlecontainer_module, only: amrex_particlecontainer, & amrex_particlecontainer_destroy - + implicit none private @@ -22,7 +22,7 @@ module amr_data_module type(amrex_fluxregister), allocatable :: flux_reg(:) type(amrex_particlecontainer) :: pc - + contains subroutine amr_data_init () @@ -49,5 +49,5 @@ subroutine amr_data_finalize end do call amrex_particlecontainer_destroy(pc) end subroutine amr_data_finalize - + end module amr_data_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/averagedown_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/averagedown_mod.F90 index 997f6f917bf..d0fe70cbc97 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/averagedown_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/averagedown_mod.F90 @@ -6,7 +6,7 @@ module averagedown_module implicit none private - + public :: averagedown, averagedownto contains @@ -23,7 +23,7 @@ end subroutine averagedown subroutine averagedownto (clev) integer, intent(in) :: clev call amrex_average_down(phi_new(clev+1), phi_new(clev), amrex_geom(clev+1), amrex_geom(clev), & - 1, 1, amrex_ref_ratio(clev)) + 1, 1, amrex_ref_ratio(clev)) end subroutine averagedownto end module averagedown_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/bc_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/bc_mod.F90 index 33055cddc30..eceeac4cdb5 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/bc_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/bc_mod.F90 @@ -8,5 +8,5 @@ module bc_module ! periodic bc. See amrex_bc_types_module for a list of bc types. integer, save :: lo_bc(amrex_spacedim,1) = amrex_bc_int_dir ! the second dimension is the integer, save :: hi_bc(amrex_spacedim,1) = amrex_bc_int_dir ! number of components - + end module bc_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/compute_dt_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/compute_dt_mod.F90 index c15b7337b0e..3c1c63188ac 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/compute_dt_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/compute_dt_mod.F90 @@ -18,13 +18,13 @@ subroutine compute_dt () real(amrex_real), parameter :: change_max = 1.1_amrex_real nlevs = amrex_get_numlevels() - + allocate(dt_tmp(0:nlevs-1)) do lev = 0, nlevs-1 dt_tmp(lev) = est_timestep(lev, t_new(lev)) end do call amrex_parallel_reduce_min(dt_tmp, nlevs) - + dt_0 = dt_tmp(0) n_factor = 1 do lev = 0, nlevs-1 @@ -32,7 +32,7 @@ subroutine compute_dt () n_factor = n_factor * nsubsteps(lev) dt_0 = min(dt_0, n_factor*dt_tmp(lev)) end do - + ! Limit dt's by the value of stop_time. eps = 1.e-3_amrex_real * dt_0 if (t_new(0) + dt_0 .gt. stop_time - eps) then @@ -49,7 +49,7 @@ end subroutine compute_dt function est_timestep (lev, time) result(dt) use my_amr_module, only : phi_new, cfl use face_velocity_module, only : get_face_velocity - + real(amrex_real) :: dt integer, intent(in) :: lev real(amrex_real), intent(in) :: time @@ -99,7 +99,7 @@ function est_timestep (lev, time) result(dt) call amrex_mfiter_destroy(mfi) call amrex_fab_destroy(u) !$omp end parallel - + dt = dt_est * cfl end function est_timestep diff --git a/Tutorials/FortranInterface/Advection_F/Source/evolve_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/evolve_mod.F90 index 5208ce9a5ac..6b7b0df1af5 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/evolve_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/evolve_mod.F90 @@ -3,7 +3,7 @@ module evolve_module use amrex_amr_module use amrex_particlecontainer_module, only : amrex_particle - + implicit none private @@ -21,7 +21,7 @@ subroutine evolve () cur_time = t_new(0) last_plot_file_step = 0; - + do step = stepno(0), max_step-1 if (cur_time .ge. stop_time) exit @@ -35,7 +35,7 @@ subroutine evolve () lev = 0 substep = 1 call timestep(lev, cur_time, substep) - + cur_time = cur_time + dt(0) if (amrex_parallel_ioprocessor()) then @@ -68,11 +68,11 @@ recursive subroutine timestep (lev, time, substep) use averagedown_module, only : averagedownto integer, intent(in) :: lev, substep real(amrex_real), intent(in) :: time - + integer, allocatable, save :: last_regrid_step(:) integer :: k, old_finest_level, finest_level, fine_substep integer :: redistribute_ngrow - + if (regrid_int .gt. 0) then if (.not.allocated(last_regrid_step)) then allocate(last_regrid_step(0:amrex_max_level)) @@ -130,7 +130,7 @@ recursive subroutine timestep (lev, time, substep) end if call pc%redistribute(lev, amrex_get_finest_level(), redistribute_ngrow) end if - + end subroutine timestep ! update phi_new(lev) @@ -155,7 +155,7 @@ subroutine advance (lev, time, dt, step, substep, nsub) type(amrex_fab) :: flux(amrex_spacedim) type(amrex_multifab) :: fluxes(amrex_spacedim) type(amrex_particle), pointer :: particles(:) - + if (verbose .gt. 0 .and. amrex_parallel_ioprocessor()) then write(*,'(A, 1X, I0, 1X, A, 1X, I0, A, 1X, G0)') & "[Level", lev, "step", step, "] ADVANCE with dt =", dt @@ -226,7 +226,7 @@ subroutine advance (lev, time, dt, step, substep, nsub) pfz, lbound(pfz), ubound(pfz), & #endif amrex_geom(lev)%dx, dt) - + if (do_reflux) then do idim = 1, amrex_spacedim pf => fluxes(idim)%dataptr(mfi) @@ -246,7 +246,7 @@ subroutine advance (lev, time, dt, step, substep, nsub) puz, lbound(puz), ubound(puz), & #endif dt, amrex_geom(lev)%dx, amrex_problo) - + end do call amrex_mfiter_destroy(mfi) do idim = 1, amrex_spacedim diff --git a/Tutorials/FortranInterface/Advection_F/Source/fillpatch_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/fillpatch_mod.F90 index c20dc37e3f7..f337de6e7ff 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/fillpatch_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/fillpatch_mod.F90 @@ -18,7 +18,7 @@ subroutine fillpatch (lev, time, phi) integer, intent(in) :: lev real(amrex_real), intent(in) :: time type(amrex_multifab), intent(inout) :: phi - + integer, parameter :: src_comp=1, dst_comp=1, num_comp=1 ! for this test code if (lev .eq. 0) then @@ -48,7 +48,7 @@ subroutine fillcoarsepatch (lev, time, phi) type(amrex_multifab), intent(inout) :: phi integer, parameter :: src_comp=1, dst_comp=1, num_comp=1 ! for this test code - + call amrex_fillcoarsepatch(phi, t_old(lev-1), phi_old(lev-1), & & t_new(lev-1), phi_new(lev-1), & & amrex_geom(lev-1), fill_physbc, & @@ -97,7 +97,7 @@ subroutine fill_physbc (pmf, scomp, ncomp, time, pgeom) bind(c) !$omp end parallel end if - + end subroutine fill_physbc end module fillpatch_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/fmain.F90 b/Tutorials/FortranInterface/Advection_F/Source/fmain.F90 index d6e58293092..987b5788c15 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/fmain.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/fmain.F90 @@ -6,7 +6,7 @@ program main use my_amr_module use initdata_module use evolve_module - + implicit none call amrex_init() diff --git a/Tutorials/FortranInterface/Advection_F/Source/initdata.F90 b/Tutorials/FortranInterface/Advection_F/Source/initdata.F90 index 041104f6de0..e929a6cdae7 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/initdata.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/initdata.F90 @@ -20,7 +20,7 @@ subroutine initdata () call averagedown() call init_particle_data() - + if (plot_int .gt. 0) call writeplotfile else call amrex_abort("init from checkpoint not implemented yet") @@ -28,17 +28,17 @@ subroutine initdata () end subroutine initdata subroutine init_particle_data() - use amr_data_module, only : pc, phi_new + use amr_data_module, only : pc, phi_new use amrex_particlecontainer_module, only : amrex_particlecontainer_build use prob_module, only : init_part_data - - type(c_ptr) :: amrcore = c_null_ptr + + type(c_ptr) :: amrcore = c_null_ptr integer :: lev type(amrex_mfiter) :: mfi type(amrex_box) :: bx - + amrcore = amrex_get_amrcore() - call amrex_particlecontainer_build(pc, amrcore) + call amrex_particlecontainer_build(pc, amrcore) lev = 0 call amrex_mfiter_build(mfi, phi_new(lev), tiling=.false.) @@ -46,7 +46,7 @@ subroutine init_particle_data() bx = mfi%tilebox() call init_part_data(pc, lev, mfi, bx%lo, bx%hi, amrex_geom(lev)%dx, amrex_problo) end do - + call pc%redistribute() end subroutine init_particle_data diff --git a/Tutorials/FortranInterface/Advection_F/Source/my_amr_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/my_amr_mod.F90 index 1a35068870e..8df5117d77e 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/my_amr_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/my_amr_mod.F90 @@ -31,7 +31,7 @@ module my_amr_module real(rt), allocatable, save :: dt(:) integer, private, parameter :: ncomp = 1, nghost = 0 - + contains subroutine my_amr_init () @@ -40,7 +40,7 @@ subroutine my_amr_init () integer :: ilev if (.not.amrex_amrcore_initialized()) call amrex_amrcore_init() - + call amrex_init_virtual_functions (my_make_new_level_from_scratch, & & my_make_new_level_from_coarse, & & my_remake_level, & @@ -59,7 +59,7 @@ subroutine my_amr_init () call pp%query("max_step", max_step) call pp%query("stop_time", stop_time) call amrex_parmparse_destroy(pp) - + ! Parameters amr.* call amrex_parmparse_build(pp, "amr") call pp%query("regrid_int", regrid_int) @@ -69,7 +69,7 @@ subroutine my_amr_init () call pp%query("plot_file", plot_file) call pp%query("restart", restart) call amrex_parmparse_destroy(pp) - + ! Parameters myamr.* call amrex_parmparse_build(pp, "myamr") call pp%query("v", verbose) @@ -124,7 +124,7 @@ subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) t_old(lev) = time - 1.e200_amrex_real call my_clear_level(lev) - + call amrex_multifab_build(phi_new(lev), ba, dm, ncomp, nghost) call amrex_multifab_build(phi_old(lev), ba, dm, ncomp, nghost) @@ -179,7 +179,7 @@ subroutine my_remake_level (lev, time, pba, pdm) bind(c) integer, intent(in), value :: lev real(amrex_real), intent(in), value :: time type(c_ptr), intent(in), value :: pba, pdm - + type(amrex_boxarray) :: ba type(amrex_distromap) :: dm type(amrex_multifab) :: new_phi_new @@ -251,5 +251,5 @@ subroutine my_error_estimate (lev, cp, t, settag, cleartag) bind(c) !$omp end parallel end subroutine my_error_estimate - + end module my_amr_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/plotfile_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/plotfile_mod.F90 index dce9d5df907..506186d280f 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/plotfile_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/plotfile_mod.F90 @@ -18,7 +18,7 @@ subroutine writeplotfile () character(len=127) :: name character(len=16) :: current_step type(amrex_string) :: varname(1) - + if (stepno(0) .lt. 1000000) then write(current_step,fmt='(i5.5)') stepno(0) else if (stepno(0) .lt. 10000000) then @@ -40,7 +40,7 @@ subroutine writeplotfile () t_new(0), stepno, amrex_ref_ratio) call pc%write(name, "Tracer", .true.) - + end subroutine writeplotfile end module plotfile_module diff --git a/Tutorials/FortranInterface/Advection_F/Source/tagging_mod.F90 b/Tutorials/FortranInterface/Advection_F/Source/tagging_mod.F90 index 38c6f576aba..ca0530204e2 100644 --- a/Tutorials/FortranInterface/Advection_F/Source/tagging_mod.F90 +++ b/Tutorials/FortranInterface/Advection_F/Source/tagging_mod.F90 @@ -28,7 +28,7 @@ subroutine tag_phi_error (level, time, lo, hi, phi, philo, phihi, tag, taglo, ta if (level >= 1) then ! - ! This is here for testing only! + ! This is here for testing only! ! Remove this if you use this as a template. ! if (time > 0.75_amrex_real .and. time < 1.0_amrex_real) then diff --git a/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/Prob.f90 b/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/Prob.f90 index 14ceb17b946..bf378246935 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/Prob.f90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/Prob.f90 @@ -13,7 +13,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & dx, prob_lo) use amrex_fort_module, only : amrex_spacedim, amrex_real - + implicit none integer, intent(in) :: level, lo(3), hi(3), phi_lo(3), phi_hi(3) real(amrex_real), intent(in) :: time @@ -21,10 +21,10 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & & phi_lo(2):phi_hi(2), & & phi_lo(3):phi_hi(3)) real(amrex_real), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(amrex_real) :: x,y,z,r2 - + !$omp parallel do private(i,j,k,x,y,z,r2) collapse(2) do k=lo(3),hi(3) do j=lo(2),hi(2) @@ -32,7 +32,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + if ( amrex_spacedim .eq. 2) then r2 = ((x-0.5d0)**2 + (y-0.75d0)**2) / 0.01d0 phi(i,j,k) = 1.d0 + exp(-r2) @@ -44,7 +44,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & end do end do !$omp end parallel do - + end subroutine init_prob_data end module prob_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/face_velocity_2d.F90 b/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/face_velocity_2d.F90 index cbce4d465c7..a64ab0f32a2 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/face_velocity_2d.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Exec/SingleVortex/face_velocity_2d.F90 @@ -28,9 +28,9 @@ subroutine get_face_velocity(time, & plo(2) = min(vxlo(2)-1, vylo(2)-1) phi(1) = max(vxhi(1) , vyhi(1)+1) phi(2) = max(vxhi(2)+1, vyhi(2) ) - + call amrex_allocate(psi, plo, phi) - + ! streamfunction psi do j = plo(2), phi(2) y = (dble(j)+0.5d0)*dx(2) + prob_lo(2) @@ -39,7 +39,7 @@ subroutine get_face_velocity(time, & psi(i,j) = sin(M_PI*x)**2 * sin(M_PI*y)**2 * cos (M_PI*time/2.d0) * (1.d0 / M_PI) end do end do - + ! x velocity do j = vxlo(2), vxhi(2) y = (dble(j)+0.5d0) * dx(2) + prob_lo(2) @@ -48,7 +48,7 @@ subroutine get_face_velocity(time, & vx(i,j) = -( (psi(i,j+1)+psi(i-1,j+1)) - (psi(i,j-1)+psi(i-1,j-1)) ) * (0.25d0/dx(2)) end do end do - + ! y velocity do j = vylo(2), vyhi(2) y = dble(j) * dx(2) + prob_lo(2) @@ -57,9 +57,9 @@ subroutine get_face_velocity(time, & vy(i,j) = ( (psi(i+1,j)+psi(i+1,j-1)) - (psi(i-1,j)+psi(i-1,j-1)) ) * (0.25d0/dx(1)) end do end do - + call amrex_deallocate(psi) - + end subroutine get_face_velocity end module face_velocity_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/advect_2d_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/advect_2d_mod.F90 index f156191b9c7..63a9ca512e9 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/advect_2d_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/advect_2d_mod.F90 @@ -5,7 +5,7 @@ module advect_module implicit none private - + public :: advect contains @@ -35,15 +35,15 @@ subroutine advect(time, lo, hi, & real(amrex_real), intent(in ) :: vy (vy_lo(1):vy_hi(1),vy_lo(2):vy_hi(2)) real(amrex_real), intent( out) :: flxx(fx_lo(1):fx_hi(1),fx_lo(2):fx_hi(2)) real(amrex_real), intent( out) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) - + integer :: i, j integer :: glo(2), ghi(2) real(amrex_real) :: dtdx(2), umax, vmax - + real(amrex_real), dimension(:,:), pointer, contiguous :: phix_1d, phiy_1d, phix, phiy, slope dtdx = dt/dx - + glo = lo - 1 ghi = hi + 1 @@ -59,9 +59,9 @@ subroutine advect(time, lo, hi, & ! to remove their pointerness for performance, because normally pointers could ! be aliasing. We need to use pointers instead of allocatable arrays because ! we like to use AMReX's amrex_allocate to allocate memeory instead of the intrinsic - ! allocate. Amrex_allocate is much faster than allocate inside OMP. + ! allocate. Amrex_allocate is much faster than allocate inside OMP. ! Note that one MUST CALL AMREX_DEALLOCATE. - + ! check if CFL condition is violated. umax = maxval(abs(vx)) vmax = maxval(abs(vy)) @@ -88,27 +88,27 @@ subroutine advect(time, lo, hi, & + (flxy(i,j) - flxy(i,j+1)) * dtdx(2) ) enddo enddo - + ! Scale by face area in order to correctly reflx do j = lo(2), hi(2) do i = lo(1), hi(1)+1 flxx(i,j) = flxx(i,j) * ( dt * dx(2)) enddo enddo - + ! Scale by face area in order to correctly reflx - do j = lo(2), hi(2)+1 + do j = lo(2), hi(2)+1 do i = lo(1), hi(1) flxy(i,j) = flxy(i,j) * (dt * dx(1)) enddo enddo - + call amrex_deallocate(phix_1d) call amrex_deallocate(phiy_1d) call amrex_deallocate(phix) call amrex_deallocate(phiy) call amrex_deallocate(slope) - + end subroutine advect end module advect_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/compute_flux_2d.f90 b/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/compute_flux_2d.f90 index 24fdbac7fbe..d78f419ebda 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/compute_flux_2d.f90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/compute_flux_2d.f90 @@ -34,7 +34,7 @@ subroutine compute_flux_2d(lo, hi, dt, dx, & real(amrex_real), intent( out) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) real(amrex_real), dimension(glo(1):ghi(1),glo(2):ghi(2)) :: & phix_1d, phiy_1d, phix, phiy, slope - + integer :: i, j, k real(amrex_real) :: hdtdx(2) diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/slope_2d.f90 b/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/slope_2d.f90 index f32326e5e51..c0daa7e96d0 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/slope_2d.f90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/Src_2d/slope_2d.f90 @@ -1,18 +1,18 @@ module slope_module use amrex_base_module - + implicit none real(amrex_real), parameter:: four3rd=4._amrex_real/3._amrex_real, & sixth=1._amrex_real/6._amrex_real - + private - + public :: slopex, slopey - + contains - + subroutine slopex(lo, hi, & q, qlo, qhi, & dq, dqlo, dqhi) @@ -123,4 +123,4 @@ subroutine slopey_doit(lo, hi, & end subroutine slopey_doit -end module slope_module +end module slope_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/amr_data_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/amr_data_mod.F90 index 3f02b9bb92b..59adfddb473 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/amr_data_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/amr_data_mod.F90 @@ -44,5 +44,5 @@ subroutine amr_data_finalize call amrex_fluxregister_destroy(flux_reg(lev)) end do end subroutine amr_data_finalize - + end module amr_data_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/averagedown_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/averagedown_mod.F90 index 00cfd9971fc..40f23d41417 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/averagedown_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/averagedown_mod.F90 @@ -6,7 +6,7 @@ module averagedown_module implicit none private - + public :: averagedown contains diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/bc_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/bc_mod.F90 index ab2715896ed..2ad307b9ad4 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/bc_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/bc_mod.F90 @@ -8,7 +8,7 @@ module bc_module ! periodic bc. See amrex_bc_types_module for a list of bc types. integer, parameter :: lo_bc(amrex_spacedim,1) = amrex_bc_int_dir ! the second dimension is the integer, parameter :: hi_bc(amrex_spacedim,1) = amrex_bc_int_dir ! number of components - + contains end module bc_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/compute_dt_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/compute_dt_mod.F90 index 55978f85b7f..623e33be7a1 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/compute_dt_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/compute_dt_mod.F90 @@ -17,14 +17,14 @@ subroutine compute_dt () real(amrex_real), parameter :: change_max = 1.1_amrex_real nlevs = amrex_get_numlevels() - + dtstep= huge(1.0_amrex_real) do lev = 0, nlevs-1 dt_level = est_timestep(lev, t_new(lev)) dtstep= min(dtstep, dt_level) end do call amrex_parallel_reduce_min(dtstep) - + ! Limit dt by the value of stop_time. eps = 1.e-3_amrex_real * dtstep if (t_new(0) + dtstep.gt. stop_time - eps) then @@ -36,7 +36,7 @@ end subroutine compute_dt function est_timestep (lev, time) result(dt) use my_amr_module, only : phi_new, cfl use face_velocity_module, only : get_face_velocity - + real(amrex_real) :: dt integer, intent(in) :: lev real(amrex_real), intent(in) :: time @@ -86,7 +86,7 @@ function est_timestep (lev, time) result(dt) call amrex_mfiter_destroy(mfi) call amrex_fab_destroy(u) !$omp end parallel - + dt = dt_est * cfl end function est_timestep diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/evolve_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/evolve_mod.F90 index 7171195aae4..e2a449c4a0d 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/evolve_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/evolve_mod.F90 @@ -19,7 +19,7 @@ subroutine evolve () cur_time = t_new(0) last_plot_file_step = 0; - + do step = stepno, max_step-1 if (cur_time .ge. stop_time) exit @@ -29,7 +29,7 @@ subroutine evolve () end if call timestep(cur_time) - + cur_time = cur_time + dtstep if (amrex_parallel_ioprocessor()) then @@ -62,7 +62,7 @@ subroutine timestep (time) use averagedown_module, only : averagedown use compute_dt_module, only : compute_dt real(amrex_real), intent(in) :: time - + integer, save :: last_regrid_step = 0 integer :: lev, finest_level @@ -215,7 +215,7 @@ subroutine advance (time, dt, step) end do call amrex_multifab_destroy(phiborder(ilev)) end do - + end subroutine advance end module evolve_module diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/fillpatch_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/fillpatch_mod.F90 index fcc7068aca8..14f6198c5d4 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/fillpatch_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/fillpatch_mod.F90 @@ -18,7 +18,7 @@ subroutine fillpatch (lev, time, phi) integer, intent(in) :: lev real(amrex_real), intent(in) :: time type(amrex_multifab), intent(inout) :: phi - + integer, parameter :: src_comp=1, dst_comp=1, num_comp=1 ! for this test code if (lev .eq. 0) then @@ -48,7 +48,7 @@ subroutine fillcoarsepatch (lev, time, phi) type(amrex_multifab), intent(inout) :: phi integer, parameter :: src_comp=1, dst_comp=1, num_comp=1 ! for this test code - + call amrex_fillcoarsepatch(phi, t_old(lev-1), phi_old(lev-1), & & t_new(lev-1), phi_new(lev-1), & & amrex_geom(lev-1), fill_physbc, & @@ -65,10 +65,10 @@ subroutine fill_physbc (pmf, scomp, ncomp, time, pgeom) bind(c) real(amrex_real), value :: time ! In this test problem, we only have periodic boundaries. ! So there is noting to do. - + ! type(amrex_multifab) :: mf ! mf = pmf - + end subroutine fill_physbc subroutine pre_interp (lo, hi, d, dlo, dhi, nd, icomp, ncomp) bind(c) diff --git a/Tutorials/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 index 148ee3a367e..ea6e4513ff8 100644 --- a/Tutorials/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F/Source/my_amr_mod.F90 @@ -27,7 +27,7 @@ module my_amr_module real(rt), save :: dtstep integer, private, parameter :: ncomp = 1, nghost = 0 - + contains subroutine my_amr_init () @@ -35,7 +35,7 @@ subroutine my_amr_init () integer :: ilev if (.not.amrex_amrcore_initialized()) call amrex_amrcore_init() - + call amrex_init_virtual_functions (my_make_new_level_from_scratch, & & my_make_new_level_from_coarse, & & my_remake_level, & @@ -54,7 +54,7 @@ subroutine my_amr_init () call pp%query("max_step", max_step) call pp%query("stop_time", stop_time) call amrex_parmparse_destroy(pp) - + ! Parameters amr.* call amrex_parmparse_build(pp, "amr") call pp%query("regrid_int", regrid_int) @@ -64,7 +64,7 @@ subroutine my_amr_init () call pp%query("plot_file", plot_file) call pp%query("restart", restart) call amrex_parmparse_destroy(pp) - + ! Parameters myamr.* call amrex_parmparse_build(pp, "myamr") call pp%query("v", verbose) @@ -104,7 +104,7 @@ subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) t_old(lev) = time - 1.e200_amrex_real call my_clear_level(lev) - + call amrex_multifab_build(phi_new(lev), ba, dm, ncomp, nghost) call amrex_multifab_build(phi_old(lev), ba, dm, ncomp, nghost) @@ -163,7 +163,7 @@ subroutine my_remake_level (lev, time, pba, pdm) bind(c) integer, intent(in), value :: lev real(amrex_real), intent(in), value :: time type(c_ptr), intent(in), value :: pba, pdm - + type(amrex_boxarray) :: ba type(amrex_distromap) :: dm type(amrex_multifab) :: new_phi_new @@ -237,5 +237,5 @@ subroutine my_error_estimate (lev, cp, t, settag, cleartag) bind(c) !$omp end parallel end subroutine my_error_estimate - + end module my_amr_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/Prob.f90 b/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/Prob.f90 index 14ceb17b946..bf378246935 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/Prob.f90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/Prob.f90 @@ -13,7 +13,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & dx, prob_lo) use amrex_fort_module, only : amrex_spacedim, amrex_real - + implicit none integer, intent(in) :: level, lo(3), hi(3), phi_lo(3), phi_hi(3) real(amrex_real), intent(in) :: time @@ -21,10 +21,10 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & & phi_lo(2):phi_hi(2), & & phi_lo(3):phi_hi(3)) real(amrex_real), intent(in) :: dx(3), prob_lo(3) - + integer :: i,j,k real(amrex_real) :: x,y,z,r2 - + !$omp parallel do private(i,j,k,x,y,z,r2) collapse(2) do k=lo(3),hi(3) do j=lo(2),hi(2) @@ -32,7 +32,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & y = prob_lo(2) + (dble(j)+0.5d0) * dx(2) do i=lo(1),hi(1) x = prob_lo(1) + (dble(i)+0.5d0) * dx(1) - + if ( amrex_spacedim .eq. 2) then r2 = ((x-0.5d0)**2 + (y-0.75d0)**2) / 0.01d0 phi(i,j,k) = 1.d0 + exp(-r2) @@ -44,7 +44,7 @@ subroutine init_prob_data(level, time, lo, hi, phi, phi_lo, phi_hi, & end do end do !$omp end parallel do - + end subroutine init_prob_data end module prob_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/face_velocity_2d.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/face_velocity_2d.F90 index cbce4d465c7..a64ab0f32a2 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/face_velocity_2d.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Exec/SingleVortex/face_velocity_2d.F90 @@ -28,9 +28,9 @@ subroutine get_face_velocity(time, & plo(2) = min(vxlo(2)-1, vylo(2)-1) phi(1) = max(vxhi(1) , vyhi(1)+1) phi(2) = max(vxhi(2)+1, vyhi(2) ) - + call amrex_allocate(psi, plo, phi) - + ! streamfunction psi do j = plo(2), phi(2) y = (dble(j)+0.5d0)*dx(2) + prob_lo(2) @@ -39,7 +39,7 @@ subroutine get_face_velocity(time, & psi(i,j) = sin(M_PI*x)**2 * sin(M_PI*y)**2 * cos (M_PI*time/2.d0) * (1.d0 / M_PI) end do end do - + ! x velocity do j = vxlo(2), vxhi(2) y = (dble(j)+0.5d0) * dx(2) + prob_lo(2) @@ -48,7 +48,7 @@ subroutine get_face_velocity(time, & vx(i,j) = -( (psi(i,j+1)+psi(i-1,j+1)) - (psi(i,j-1)+psi(i-1,j-1)) ) * (0.25d0/dx(2)) end do end do - + ! y velocity do j = vylo(2), vyhi(2) y = dble(j) * dx(2) + prob_lo(2) @@ -57,9 +57,9 @@ subroutine get_face_velocity(time, & vy(i,j) = ( (psi(i+1,j)+psi(i+1,j-1)) - (psi(i-1,j)+psi(i-1,j-1)) ) * (0.25d0/dx(1)) end do end do - + call amrex_deallocate(psi) - + end subroutine get_face_velocity end module face_velocity_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/advect_2d_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/advect_2d_mod.F90 index 623535a2143..84ed890e08e 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/advect_2d_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/advect_2d_mod.F90 @@ -5,7 +5,7 @@ module advect_module implicit none private - + public :: advect contains @@ -29,7 +29,7 @@ subroutine advect(lo, hi, & real(amrex_real), intent(inout) :: uout(uo_lo(1):uo_hi(1),uo_lo(2):uo_hi(2)) real(amrex_real), intent(in ) :: flxx(fx_lo(1):fx_hi(1),fx_lo(2):fx_hi(2)) real(amrex_real), intent(in ) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) - + integer :: i, j integer :: glo(2), ghi(2) real(amrex_real) :: dtdx(2), umax, vmax @@ -44,7 +44,7 @@ subroutine advect(lo, hi, & + (flxy(i,j) - flxy(i,j+1)) * dtdx(2) ) enddo enddo - + end subroutine advect end module advect_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/compute_flux_2d.f90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/compute_flux_2d.f90 index 09d35514148..d8280c32d1c 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/compute_flux_2d.f90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/compute_flux_2d.f90 @@ -32,10 +32,10 @@ subroutine compute_flux(lo, hi, & real(amrex_real), intent(in ) :: vmac( v_lo(1): v_hi(1), v_lo(2): v_hi(2)) real(amrex_real), intent( out) :: flxx(fx_lo(1):fx_hi(1),fx_lo(2):fx_hi(2)) real(amrex_real), intent( out) :: flxy(fy_lo(1):fy_hi(1),fy_lo(2):fy_hi(2)) - + integer :: i, j, k, glo(2), ghi(2) real(amrex_real) :: hdtdx(2), umax, vmax - real(amrex_real), dimension(:,:), pointer, contiguous :: phix_1d, phiy_1d, phix, phiy, slope + real(amrex_real), dimension(:,:), pointer, contiguous :: phix_1d, phiy_1d, phix, phiy, slope ! check if CFL condition is violated. umax = maxval(abs(umac)) @@ -147,7 +147,7 @@ subroutine compute_flux(lo, hi, & call amrex_deallocate(phix) call amrex_deallocate(phiy) call amrex_deallocate(slope) - + end subroutine compute_flux end module compute_flux_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/slope_2d.f90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/slope_2d.f90 index f32326e5e51..c0daa7e96d0 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/slope_2d.f90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/Src_2d/slope_2d.f90 @@ -1,18 +1,18 @@ module slope_module use amrex_base_module - + implicit none real(amrex_real), parameter:: four3rd=4._amrex_real/3._amrex_real, & sixth=1._amrex_real/6._amrex_real - + private - + public :: slopex, slopey - + contains - + subroutine slopex(lo, hi, & q, qlo, qhi, & dq, dqlo, dqhi) @@ -123,4 +123,4 @@ subroutine slopey_doit(lo, hi, & end subroutine slopey_doit -end module slope_module +end module slope_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/amr_data_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/amr_data_mod.F90 index 3d7da4151d4..fe9b559eaed 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/amr_data_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/amr_data_mod.F90 @@ -44,5 +44,5 @@ subroutine amr_data_finalize call amrex_flash_fluxregister_destroy(flux_reg(lev)) end do end subroutine amr_data_finalize - + end module amr_data_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/averagedown_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/averagedown_mod.F90 index 00cfd9971fc..40f23d41417 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/averagedown_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/averagedown_mod.F90 @@ -6,7 +6,7 @@ module averagedown_module implicit none private - + public :: averagedown contains diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/bc_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/bc_mod.F90 index ab2715896ed..2ad307b9ad4 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/bc_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/bc_mod.F90 @@ -8,7 +8,7 @@ module bc_module ! periodic bc. See amrex_bc_types_module for a list of bc types. integer, parameter :: lo_bc(amrex_spacedim,1) = amrex_bc_int_dir ! the second dimension is the integer, parameter :: hi_bc(amrex_spacedim,1) = amrex_bc_int_dir ! number of components - + contains end module bc_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/compute_dt_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/compute_dt_mod.F90 index 55978f85b7f..623e33be7a1 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/compute_dt_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/compute_dt_mod.F90 @@ -17,14 +17,14 @@ subroutine compute_dt () real(amrex_real), parameter :: change_max = 1.1_amrex_real nlevs = amrex_get_numlevels() - + dtstep= huge(1.0_amrex_real) do lev = 0, nlevs-1 dt_level = est_timestep(lev, t_new(lev)) dtstep= min(dtstep, dt_level) end do call amrex_parallel_reduce_min(dtstep) - + ! Limit dt by the value of stop_time. eps = 1.e-3_amrex_real * dtstep if (t_new(0) + dtstep.gt. stop_time - eps) then @@ -36,7 +36,7 @@ end subroutine compute_dt function est_timestep (lev, time) result(dt) use my_amr_module, only : phi_new, cfl use face_velocity_module, only : get_face_velocity - + real(amrex_real) :: dt integer, intent(in) :: lev real(amrex_real), intent(in) :: time @@ -86,7 +86,7 @@ function est_timestep (lev, time) result(dt) call amrex_mfiter_destroy(mfi) call amrex_fab_destroy(u) !$omp end parallel - + dt = dt_est * cfl end function est_timestep diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/evolve_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/evolve_mod.F90 index 5c65dc025b2..7f40776ca2a 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/evolve_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/evolve_mod.F90 @@ -19,7 +19,7 @@ subroutine evolve () cur_time = t_new(0) last_plot_file_step = 0; - + do step = stepno, max_step-1 if (cur_time .ge. stop_time) exit @@ -29,7 +29,7 @@ subroutine evolve () end if call timestep(cur_time) - + cur_time = cur_time + dtstep if (amrex_parallel_ioprocessor()) then @@ -62,7 +62,7 @@ subroutine timestep (time) use averagedown_module, only : averagedown use compute_dt_module, only : compute_dt real(amrex_real), intent(in) :: time - + integer, save :: last_regrid_step = 0 integer :: lev, finest_level @@ -225,7 +225,7 @@ subroutine advance (time, dt, step) do ilev = 0, finest_level call amrex_multifab_destroy(phiborder(ilev)) end do - + end subroutine advance end module evolve_module diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/fillpatch_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/fillpatch_mod.F90 index fcc7068aca8..14f6198c5d4 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/fillpatch_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/fillpatch_mod.F90 @@ -18,7 +18,7 @@ subroutine fillpatch (lev, time, phi) integer, intent(in) :: lev real(amrex_real), intent(in) :: time type(amrex_multifab), intent(inout) :: phi - + integer, parameter :: src_comp=1, dst_comp=1, num_comp=1 ! for this test code if (lev .eq. 0) then @@ -48,7 +48,7 @@ subroutine fillcoarsepatch (lev, time, phi) type(amrex_multifab), intent(inout) :: phi integer, parameter :: src_comp=1, dst_comp=1, num_comp=1 ! for this test code - + call amrex_fillcoarsepatch(phi, t_old(lev-1), phi_old(lev-1), & & t_new(lev-1), phi_new(lev-1), & & amrex_geom(lev-1), fill_physbc, & @@ -65,10 +65,10 @@ subroutine fill_physbc (pmf, scomp, ncomp, time, pgeom) bind(c) real(amrex_real), value :: time ! In this test problem, we only have periodic boundaries. ! So there is noting to do. - + ! type(amrex_multifab) :: mf ! mf = pmf - + end subroutine fill_physbc subroutine pre_interp (lo, hi, d, dlo, dhi, nd, icomp, ncomp) bind(c) diff --git a/Tutorials/FortranInterface/Advection_octree_F2/Source/my_amr_mod.F90 b/Tutorials/FortranInterface/Advection_octree_F2/Source/my_amr_mod.F90 index 21d0495df64..bf9f7b1cb25 100644 --- a/Tutorials/FortranInterface/Advection_octree_F2/Source/my_amr_mod.F90 +++ b/Tutorials/FortranInterface/Advection_octree_F2/Source/my_amr_mod.F90 @@ -27,7 +27,7 @@ module my_amr_module real(rt), save :: dtstep integer, private, parameter :: ncomp = 1, nghost = 0 - + contains subroutine my_amr_init () @@ -35,7 +35,7 @@ subroutine my_amr_init () integer :: ilev if (.not.amrex_amrcore_initialized()) call amrex_amrcore_init() - + call amrex_init_virtual_functions (my_make_new_level_from_scratch, & & my_make_new_level_from_coarse, & & my_remake_level, & @@ -54,7 +54,7 @@ subroutine my_amr_init () call pp%query("max_step", max_step) call pp%query("stop_time", stop_time) call amrex_parmparse_destroy(pp) - + ! Parameters amr.* call amrex_parmparse_build(pp, "amr") call pp%query("regrid_int", regrid_int) @@ -64,7 +64,7 @@ subroutine my_amr_init () call pp%query("plot_file", plot_file) call pp%query("restart", restart) call amrex_parmparse_destroy(pp) - + ! Parameters myamr.* call amrex_parmparse_build(pp, "myamr") call pp%query("v", verbose) @@ -104,7 +104,7 @@ subroutine my_make_new_level_from_scratch (lev, time, pba, pdm) bind(c) t_old(lev) = time - 1.e200_amrex_real call my_clear_level(lev) - + call amrex_multifab_build(phi_new(lev), ba, dm, ncomp, nghost) call amrex_multifab_build(phi_old(lev), ba, dm, ncomp, nghost) @@ -165,7 +165,7 @@ subroutine my_remake_level (lev, time, pba, pdm) bind(c) integer, intent(in), value :: lev real(amrex_real), intent(in), value :: time type(c_ptr), intent(in), value :: pba, pdm - + type(amrex_boxarray) :: ba type(amrex_distromap) :: dm type(amrex_multifab) :: new_phi_new @@ -240,5 +240,5 @@ subroutine my_error_estimate (lev, cp, t, settag, cleartag) bind(c) !$omp end parallel end subroutine my_error_estimate - + end module my_amr_module diff --git a/Tutorials/GPU/CNS/Source/CNS.H b/Tutorials/GPU/CNS/Source/CNS.H index 99ab41a1839..69bb07b6287 100644 --- a/Tutorials/GPU/CNS/Source/CNS.H +++ b/Tutorials/GPU/CNS/Source/CNS.H @@ -25,7 +25,7 @@ public: CNS (const CNS& rhs) = delete; CNS& operator= (const CNS& rhs) = delete; - + // Restart from a checkpoint file. virtual void restart (amrex::Amr& papa, std::istream& is, @@ -51,7 +51,7 @@ public: // Initialize data on this level after regridding if old level did not previously exist virtual void init () override; - + // Initialize grid data at problem start-up. virtual void initData () override; diff --git a/Tutorials/GPU/CNS/Source/CNS.cpp b/Tutorials/GPU/CNS/Source/CNS.cpp index ab0cf577889..207bbfa9983 100644 --- a/Tutorials/GPU/CNS/Source/CNS.cpp +++ b/Tutorials/GPU/CNS/Source/CNS.cpp @@ -114,7 +114,7 @@ CNS::computeInitialDt (int finest_level, if (level > 0) { return; } - + Real dt_0 = std::numeric_limits::max(); int n_factor = 1; for (int i = 0; i <= finest_level; i++) @@ -123,7 +123,7 @@ CNS::computeInitialDt (int finest_level, n_factor *= n_cycle[i]; dt_0 = std::min(dt_0,n_factor*dt_level[i]); } - + // // Limit dt's by the value of stop_time. // @@ -133,7 +133,7 @@ CNS::computeInitialDt (int finest_level, if ((cur_time + dt_0) > (stop_time - eps)) dt_0 = stop_time - cur_time; } - + n_factor = 1; for (int i = 0; i <= finest_level; i++) { @@ -165,28 +165,28 @@ CNS::computeNewDt (int finest_level, dt_min[i] = getLevel(i).estTimeStep(); } - if (post_regrid_flag == 1) + if (post_regrid_flag == 1) { - // - // Limit dt's by pre-regrid dt - // - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],dt_level[i]); - } + // + // Limit dt's by pre-regrid dt + // + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],dt_level[i]); + } } - else + else { - // - // Limit dt's by change_max * old dt - // - static Real change_max = 1.1; - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); - } + // + // Limit dt's by change_max * old dt + // + static Real change_max = 1.1; + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); + } } - + // // Find the minimum over all levels // @@ -330,13 +330,13 @@ CNS::read_params () ParmParse pp("cns"); pp.query("v", verbose); - + Vector tilesize(AMREX_SPACEDIM); if (pp.queryarr("hydro_tile_size", tilesize, 0, AMREX_SPACEDIM)) { - for (int i=0; i lo_bc(AMREX_SPACEDIM), hi_bc(AMREX_SPACEDIM); @@ -423,7 +423,7 @@ CNS::computeTemp (MultiFab& State, int ng) Parm const* lparm = d_parm; - // This will reset Eint and compute Temperature + // This will reset Eint and compute Temperature #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif diff --git a/Tutorials/GPU/CNS/Source/CNS_K.H b/Tutorials/GPU/CNS/Source/CNS_K.H index b59526d6090..7222df7b9f4 100644 --- a/Tutorials/GPU/CNS/Source/CNS_K.H +++ b/Tutorials/GPU/CNS/Source/CNS_K.H @@ -47,7 +47,7 @@ cns_estdt (amrex::Box const& bx, amrex::Array4 const& state, } } } - + return dt; } diff --git a/Tutorials/GPU/CNS/Source/CNS_advance.cpp b/Tutorials/GPU/CNS/Source/CNS_advance.cpp index e3905809f8a..f25083f4e28 100644 --- a/Tutorials/GPU/CNS/Source/CNS_advance.cpp +++ b/Tutorials/GPU/CNS/Source/CNS_advance.cpp @@ -52,7 +52,7 @@ CNS::advance (Real time, Real dt, int /*iteration*/, int /*ncycle*/) MultiFab::Saxpy(S_new, Real(0.5)*dt, dSdt, 0, 0, NUM_STATE, 0); // We now have S_new = U^{n+1} = (U^n+0.5*dt*dUdt^n) + 0.5*dt*dUdt^* computeTemp(S_new,0); - + return dt; } diff --git a/Tutorials/GPU/CNS/Source/CNS_io.cpp b/Tutorials/GPU/CNS/Source/CNS_io.cpp index 0715d21b547..284c8a5371b 100644 --- a/Tutorials/GPU/CNS/Source/CNS_io.cpp +++ b/Tutorials/GPU/CNS/Source/CNS_io.cpp @@ -15,8 +15,8 @@ CNS::restart (Amr& papa, std::istream& is, bool bReadSpecial) buildMetrics(); } -void -CNS::checkPoint (const std::string& dir, std::ostream& os, VisMF::How how, bool dump_old) +void +CNS::checkPoint (const std::string& dir, std::ostream& os, VisMF::How how, bool dump_old) { AmrLevel::checkPoint(dir, os, how, dump_old); } diff --git a/Tutorials/GPU/CNS/Source/CNS_setup.cpp b/Tutorials/GPU/CNS/Source/CNS_setup.cpp index ff52a1cad6c..4acd2dc2591 100644 --- a/Tutorials/GPU/CNS/Source/CNS_setup.cpp +++ b/Tutorials/GPU/CNS/Source/CNS_setup.cpp @@ -40,8 +40,8 @@ set_scalar_bc (BCRec& bc, const BCRec& phys_bc) const int* hi_bc = phys_bc.hi(); for (int i = 0; i < AMREX_SPACEDIM; i++) { - bc.setLo(i,scalar_bc[lo_bc[i]]); - bc.setHi(i,scalar_bc[hi_bc[i]]); + bc.setLo(i,scalar_bc[lo_bc[i]]); + bc.setHi(i,scalar_bc[hi_bc[i]]); } } @@ -71,7 +71,7 @@ set_y_vel_bc(BCRec& bc, const BCRec& phys_bc) const int* hi_bc = phys_bc.hi(); bc.setLo(0,tang_vel_bc[lo_bc[0]]); bc.setHi(0,tang_vel_bc[hi_bc[0]]); -#if (AMREX_SPACEDIM >= 2) +#if (AMREX_SPACEDIM >= 2) bc.setLo(1,norm_vel_bc[lo_bc[1]]); bc.setHi(1,norm_vel_bc[hi_bc[1]]); #endif @@ -112,8 +112,8 @@ CNS::variableSetUp () bool state_data_extrap = false; bool store_in_checkpoint = true; desc_lst.addDescriptor(State_Type,IndexType::TheCellType(), - StateDescriptor::Point,NUM_GROW,NUM_STATE, - &cell_cons_interp,state_data_extrap,store_in_checkpoint); + StateDescriptor::Point,NUM_GROW,NUM_STATE, + &cell_cons_interp,state_data_extrap,store_in_checkpoint); Vector bcs(NUM_STATE); Vector name(NUM_STATE); @@ -131,9 +131,9 @@ CNS::variableSetUp () bndryfunc.setRunOnGPU(true); // I promise the bc function will launch gpu kernels. desc_lst.setComponent(State_Type, - Density, - name, - bcs, + Density, + name, + bcs, bndryfunc); num_state_data_types = desc_lst.size(); diff --git a/Tutorials/GPU/CNS/Source/hydro/CNS_hydro_K.H b/Tutorials/GPU/CNS/Source/hydro/CNS_hydro_K.H index 7892a756c60..ff81a9a4325 100644 --- a/Tutorials/GPU/CNS/Source/hydro/CNS_hydro_K.H +++ b/Tutorials/GPU/CNS/Source/hydro/CNS_hydro_K.H @@ -305,7 +305,7 @@ riemann (const amrex::Real gamma, const amrex::Real smallp, const amrex::Real /* ugdnv = frac*ustar + (Real(1.) - frac)* uo; pgdnv = frac*pstar + (Real(1.) - frac)* po; } - + flxrho = rgdnv*ugdnv; flxu = rgdnv*ugdnv*ugdnv+pgdnv; flxut = rgdnv*ugdnv*utrans1; diff --git a/Tutorials/GPU/CNS/Source/main.cpp b/Tutorials/GPU/CNS/Source/main.cpp index a7004713dc2..aec42ccd531 100644 --- a/Tutorials/GPU/CNS/Source/main.cpp +++ b/Tutorials/GPU/CNS/Source/main.cpp @@ -23,7 +23,7 @@ int main (int argc, char* argv[]) Real stop_time; { - ParmParse pp; + ParmParse pp; max_step = -1; strt_time = Real( 0.0); @@ -35,44 +35,44 @@ int main (int argc, char* argv[]) } if (strt_time < Real(0.0)) { - amrex::Abort("MUST SPECIFY a non-negative strt_time"); + amrex::Abort("MUST SPECIFY a non-negative strt_time"); } if (max_step < 0 && stop_time < Real(0.0)) { - amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); + amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); } { timer_init = amrex::second(); - Amr amr; - amr.init(strt_time,stop_time); + Amr amr; + amr.init(strt_time,stop_time); timer_init = amrex::second() - timer_init; timer_advance = amrex::second(); - while ( amr.okToContinue() && - (amr.levelSteps(0) < max_step || max_step < 0) && - (amr.cumTime() < stop_time || stop_time < Real(0.0)) ) - - { - // - // Do a coarse timestep. Recursively calls timeStep() - // - amr.coarseTimeStep(stop_time); - } - + while ( amr.okToContinue() && + (amr.levelSteps(0) < max_step || max_step < 0) && + (amr.cumTime() < stop_time || stop_time < Real(0.0)) ) + + { + // + // Do a coarse timestep. Recursively calls timeStep() + // + amr.coarseTimeStep(stop_time); + } + timer_advance = amrex::second() - timer_advance; - // Write final checkpoint and plotfile - if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { - amr.checkPoint(); - } - - if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { - amr.writePlotFile(); - } + // Write final checkpoint and plotfile + if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { + amr.checkPoint(); + } + + if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { + amr.writePlotFile(); + } } timer_tot = amrex::second() - timer_tot; diff --git a/Tutorials/GPU/EBCNS/Source/CNS.H b/Tutorials/GPU/EBCNS/Source/CNS.H index e17042f6794..d6e0517357e 100644 --- a/Tutorials/GPU/EBCNS/Source/CNS.H +++ b/Tutorials/GPU/EBCNS/Source/CNS.H @@ -23,7 +23,7 @@ public: CNS (const CNS& rhs) = delete; CNS& operator= (const CNS& rhs) = delete; - + // Restart from a checkpoint file. virtual void restart (amrex::Amr& papa, std::istream& is, @@ -49,7 +49,7 @@ public: // Initialize data on this level after regridding if old level did not previously exist virtual void init () override; - + // Initialize grid data at problem start-up. virtual void initData () override; diff --git a/Tutorials/GPU/EBCNS/Source/CNS.cpp b/Tutorials/GPU/EBCNS/Source/CNS.cpp index c05726e4337..af9cee20d2f 100644 --- a/Tutorials/GPU/EBCNS/Source/CNS.cpp +++ b/Tutorials/GPU/EBCNS/Source/CNS.cpp @@ -170,24 +170,24 @@ CNS::computeNewDt (int finest_level, if (post_regrid_flag == 1) { - // - // Limit dt's by pre-regrid dt - // - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],dt_level[i]); - } + // + // Limit dt's by pre-regrid dt + // + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],dt_level[i]); + } } else { - // - // Limit dt's by change_max * old dt - // - static Real change_max = 1.1; - for (int i = 0; i <= finest_level; i++) - { - dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); - } + // + // Limit dt's by change_max * old dt + // + static Real change_max = 1.1; + for (int i = 0; i <= finest_level; i++) + { + dt_min[i] = std::min(dt_min[i],change_max*dt_level[i]); + } } // @@ -337,7 +337,7 @@ CNS::read_params () Vector tilesize(AMREX_SPACEDIM); if (pp.queryarr("hydro_tile_size", tilesize, 0, AMREX_SPACEDIM)) { - for (int i=0; i pl1pt, pl2pt, pl2nm, pl3pt; + + Vector pl1pt, pl2pt, pl2nm, pl3pt; pp.getarr("ramp_plane1_point", pl1pt); pp.getarr("ramp_plane2_point", pl2pt); pp.getarr("ramp_plane2_normal", pl2nm); @@ -42,10 +42,10 @@ initialize_EB2 (const Geometry& geom, const int required_coarsening_level, EB2::PlaneIF({pl3pt[0], pl3pt[1], 0.}, { 1., 0., 0.})); - Vector pipelo, pipehi; + Vector pipelo, pipehi; pp.getarr("pipe_lo", pipelo); pp.getarr("pipe_hi", pipehi); - + EB2::BoxIF pipe({pipelo[0], pipelo[1], -1.}, {pipehi[0], pipehi[1], 1.}, false); // where does plane 1 and plane 2 intersect? @@ -55,13 +55,13 @@ initialize_EB2 (const Geometry& geom, const int required_coarsening_level, Real dx = geom.CellSize(0); Real dycut = 4.*(1.+max_coarsening_level)*std::min(dx, k2*dx); EB2::BoxIF flat_corner({pl3pt[0], 0., -1.}, {1.e10, secty+dycut, 1.}, false); - + auto polys = EB2::makeUnion(farwall, ramp, pipe, flat_corner); Real lenx = geom.ProbLength(0); Real leny = geom.ProbLength(1); auto pr = EB2::translate(EB2::lathe(polys), {lenx*0.5, leny*0.5, 0.}); - + auto gshop = EB2::makeShop(pr); EB2::Build(gshop, geom, max_coarsening_level, max_coarsening_level, 4); } diff --git a/Tutorials/GPU/EBCNS/Source/CNS_io.cpp b/Tutorials/GPU/EBCNS/Source/CNS_io.cpp index 563bd63f234..d0d83b8f7db 100644 --- a/Tutorials/GPU/EBCNS/Source/CNS_io.cpp +++ b/Tutorials/GPU/EBCNS/Source/CNS_io.cpp @@ -15,8 +15,8 @@ CNS::restart (Amr& papa, std::istream& is, bool bReadSpecial) buildMetrics(); } -void -CNS::checkPoint (const std::string& dir, std::ostream& os, VisMF::How how, bool dump_old) +void +CNS::checkPoint (const std::string& dir, std::ostream& os, VisMF::How how, bool dump_old) { AmrLevel::checkPoint(dir, os, how, dump_old); } @@ -47,14 +47,14 @@ CNS::writePlotFile (const std::string& dir, std::ostream& os, VisMF::How how) const std::list& dlist = derive_lst.dlist(); for (std::list::const_iterator it = dlist.begin(); - it != dlist.end(); - ++it) + it != dlist.end(); + ++it) { if (parent->isDerivePlotVar(it->name())) { derive_names.push_back(it->name()); num_derive++; - } + } } int n_data_items = plot_var_map.size() + num_derive + 1; @@ -73,20 +73,20 @@ CNS::writePlotFile (const std::string& dir, std::ostream& os, VisMF::How how) os << n_data_items << '\n'; - // - // Names of variables -- first state, then derived - // - for (int i = 0; i < plot_var_map.size(); i++) + // + // Names of variables -- first state, then derived + // + for (int i = 0; i < plot_var_map.size(); i++) { - int typ = plot_var_map[i].first; - int comp = plot_var_map[i].second; - os << desc_lst[typ].name(comp) << '\n'; + int typ = plot_var_map[i].first; + int comp = plot_var_map[i].second; + os << desc_lst[typ].name(comp) << '\n'; } - for ( std::list::iterator it = derive_names.begin(); - it != derive_names.end(); ++it) + for ( std::list::iterator it = derive_names.begin(); + it != derive_names.end(); ++it) { - const DeriveRec* rec = derive_lst.get(*it); + const DeriveRec* rec = derive_lst.get(*it); os << rec->variableName(0) << '\n'; } @@ -191,30 +191,30 @@ CNS::writePlotFile (const std::string& dir, std::ostream& os, VisMF::How how) // for (int i = 0; i < plot_var_map.size(); i++) { - int typ = plot_var_map[i].first; - int comp = plot_var_map[i].second; - this_dat = &state[typ].newData(); - MultiFab::Copy(plotMF,*this_dat,comp,cnt,1,nGrow); + int typ = plot_var_map[i].first; + int comp = plot_var_map[i].second; + this_dat = &state[typ].newData(); + MultiFab::Copy(plotMF,*this_dat,comp,cnt,1,nGrow); #ifdef AMREX_TESTING // to avoid fcompare failure if (typ == Cost_Type) { plotMF.setVal(0.0, cnt, 1, nGrow); } #endif - cnt++; + cnt++; } // // Cull data from derived variables. // if (derive_names.size() > 0) { - for (std::list::iterator it = derive_names.begin(); - it != derive_names.end(); ++it) - { + for (std::list::iterator it = derive_names.begin(); + it != derive_names.end(); ++it) + { auto derive_dat = derive(*it,cur_time,nGrow); MultiFab::Copy(plotMF,*derive_dat,0,cnt,1,nGrow); - cnt++; - } + cnt++; + } } plotMF.setVal(0.0, cnt, 1, nGrow); diff --git a/Tutorials/GPU/EBCNS/Source/CNS_setup.cpp b/Tutorials/GPU/EBCNS/Source/CNS_setup.cpp index 853e06ada1f..4c1cd9e46b6 100644 --- a/Tutorials/GPU/EBCNS/Source/CNS_setup.cpp +++ b/Tutorials/GPU/EBCNS/Source/CNS_setup.cpp @@ -36,8 +36,8 @@ set_scalar_bc (BCRec& bc, const BCRec& phys_bc) const int* hi_bc = phys_bc.hi(); for (int i = 0; i < AMREX_SPACEDIM; i++) { - bc.setLo(i,scalar_bc[lo_bc[i]]); - bc.setHi(i,scalar_bc[hi_bc[i]]); + bc.setLo(i,scalar_bc[lo_bc[i]]); + bc.setHi(i,scalar_bc[hi_bc[i]]); } } @@ -67,7 +67,7 @@ set_y_vel_bc(BCRec& bc, const BCRec& phys_bc) const int* hi_bc = phys_bc.hi(); bc.setLo(0,tang_vel_bc[lo_bc[0]]); bc.setHi(0,tang_vel_bc[hi_bc[0]]); -#if (AMREX_SPACEDIM >= 2) +#if (AMREX_SPACEDIM >= 2) bc.setLo(1,norm_vel_bc[lo_bc[1]]); bc.setHi(1,norm_vel_bc[hi_bc[1]]); #endif @@ -103,8 +103,8 @@ CNS::variableSetUp () bool state_data_extrap = false; bool store_in_checkpoint = true; desc_lst.addDescriptor(State_Type,IndexType::TheCellType(), - StateDescriptor::Point,NUM_GROW,NUM_STATE, - &cell_cons_interp,state_data_extrap,store_in_checkpoint); + StateDescriptor::Point,NUM_GROW,NUM_STATE, + &cell_cons_interp,state_data_extrap,store_in_checkpoint); Vector bcs(NUM_STATE); Vector name(NUM_STATE); @@ -122,9 +122,9 @@ CNS::variableSetUp () bndryfunc.setRunOnGPU(true); // I promise the bc function will launch gpu kernels. desc_lst.setComponent(State_Type, - Density, - name, - bcs, + Density, + name, + bcs, bndryfunc); num_state_data_types = desc_lst.size(); diff --git a/Tutorials/GPU/EBCNS/Source/hydro/CNS_hydro_K.H b/Tutorials/GPU/EBCNS/Source/hydro/CNS_hydro_K.H index eb7c9e0f1bb..e8b1ed5c915 100644 --- a/Tutorials/GPU/EBCNS/Source/hydro/CNS_hydro_K.H +++ b/Tutorials/GPU/EBCNS/Source/hydro/CNS_hydro_K.H @@ -291,7 +291,7 @@ riemann (const amrex::Real gamma, const amrex::Real smallp, const amrex::Real sm ugdnv = frac*ustar + (1. - frac)* uo; pgdnv = frac*pstar + (1. - frac)* po; } - + flxrho = rgdnv*ugdnv; flxu = rgdnv*ugdnv*ugdnv+pgdnv; flxut = rgdnv*ugdnv*utrans1; diff --git a/Tutorials/GPU/EBCNS/Source/main.cpp b/Tutorials/GPU/EBCNS/Source/main.cpp index 77b35a8b3e8..b61c205af54 100644 --- a/Tutorials/GPU/EBCNS/Source/main.cpp +++ b/Tutorials/GPU/EBCNS/Source/main.cpp @@ -26,7 +26,7 @@ int main (int argc, char* argv[]) Real stop_time; { - ParmParse pp; + ParmParse pp; max_step = -1; strt_time = 0.0; @@ -38,49 +38,49 @@ int main (int argc, char* argv[]) } if (strt_time < 0.0) { - amrex::Abort("MUST SPECIFY a non-negative strt_time"); + amrex::Abort("MUST SPECIFY a non-negative strt_time"); } if (max_step < 0 && stop_time < 0.0) { - amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); + amrex::Abort("Exiting because neither max_step nor stop_time is non-negative."); } { timer_init = amrex::second(); - Amr amr; + Amr amr; AmrLevel::SetEBSupportLevel(EBSupport::full); AmrLevel::SetEBMaxGrowCells(CNS::numGrow(),4,2); initialize_EB2(amr.Geom(amr.maxLevel()), amr.maxLevel(), amr.maxLevel()); - amr.init(strt_time,stop_time); + amr.init(strt_time,stop_time); timer_init = amrex::second() - timer_init; timer_advance = amrex::second(); - while ( amr.okToContinue() && - (amr.levelSteps(0) < max_step || max_step < 0) && - (amr.cumTime() < stop_time || stop_time < 0.0) ) - - { - // - // Do a coarse timestep. Recursively calls timeStep() - // - amr.coarseTimeStep(stop_time); - } - + while ( amr.okToContinue() && + (amr.levelSteps(0) < max_step || max_step < 0) && + (amr.cumTime() < stop_time || stop_time < 0.0) ) + + { + // + // Do a coarse timestep. Recursively calls timeStep() + // + amr.coarseTimeStep(stop_time); + } + timer_advance = amrex::second() - timer_advance; - // Write final checkpoint and plotfile - if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { - amr.checkPoint(); - } - - if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { - amr.writePlotFile(); - } + // Write final checkpoint and plotfile + if (amr.stepOfLastCheckPoint() < amr.levelSteps(0)) { + amr.checkPoint(); + } + + if (amr.stepOfLastPlotFile() < amr.levelSteps(0)) { + amr.writePlotFile(); + } } timer_tot = amrex::second() - timer_tot; diff --git a/Tutorials/GPU/Launch/MyKernel_F.F90 b/Tutorials/GPU/Launch/MyKernel_F.F90 index 7118872a6b0..86ec260b77f 100644 --- a/Tutorials/GPU/Launch/MyKernel_F.F90 +++ b/Tutorials/GPU/Launch/MyKernel_F.F90 @@ -25,7 +25,7 @@ subroutine plusone_acc (lo, hi, dat, dlo, dhi) & !$acc end kernels end subroutine plusone_acc #endif - + #ifdef AMREX_USE_OMP_OFFLOAD subroutine plusone_omp (lo, hi, dat, dlo, dhi) & bind(c,name='plusone_omp') diff --git a/Tutorials/GPU/ParallelScan/main.cpp b/Tutorials/GPU/ParallelScan/main.cpp index 181197b49fd..16ea33f662a 100644 --- a/Tutorials/GPU/ParallelScan/main.cpp +++ b/Tutorials/GPU/ParallelScan/main.cpp @@ -18,7 +18,7 @@ namespace amrex { public: using value_type = T; - + typedef thrust::device_ptr pointer; inline pointer allocate(size_t n) { @@ -26,7 +26,7 @@ namespace amrex result = (value_type*) The_Arena()->alloc(n * sizeof(T)); return thrust::device_pointer_cast(result); } - + inline void deallocate(pointer ptr, size_t) { The_Arena()->free(thrust::raw_pointer_cast(ptr)); @@ -41,7 +41,7 @@ namespace amrex namespace Gpu { ThrustManagedAllocator& The_ThrustCachedAllocator () { return g_cached_allocator; }; - + AMREX_FORCE_INLINE auto The_ThrustCachedPolicy() -> decltype (thrust::cuda::par(Gpu::The_ThrustCachedAllocator())) { return thrust::cuda::par(Gpu::The_ThrustCachedAllocator()); diff --git a/Tutorials/MUI/Source_01/init_phi_3d.f90 b/Tutorials/MUI/Source_01/init_phi_3d.f90 index 7789d132472..ca12da9232c 100644 --- a/Tutorials/MUI/Source_01/init_phi_3d.f90 +++ b/Tutorials/MUI/Source_01/init_phi_3d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(3), hi(3), philo(3), phihi(3) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2),philo(3):phihi(3)) - real(amrex_real), intent(in ) :: dx(3) - real(amrex_real), intent(in ) :: prob_lo(3) - real(amrex_real), intent(in ) :: prob_hi(3) + real(amrex_real), intent(in ) :: dx(3) + real(amrex_real), intent(in ) :: prob_lo(3) + real(amrex_real), intent(in ) :: prob_hi(3) integer :: i,j,k double precision :: x,y,z,r2 @@ -44,7 +44,7 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam endif r2 = ((x_p-0.0d0)**2 + (y_p-0.0d0)**2 + (z_p-0.0d0)**2) / 0.10d0 phi(i,j,k) = 1.d0 + exp(-r2) - + end do end do end do diff --git a/Tutorials/MUI/Source_01/main_01.cpp b/Tutorials/MUI/Source_01/main_01.cpp index c5096eef2ab..a80732f8b63 100644 --- a/Tutorials/MUI/Source_01/main_01.cpp +++ b/Tutorials/MUI/Source_01/main_01.cpp @@ -19,9 +19,9 @@ int main (int argc, char* argv[]) { MPI_Comm comm = mui::mpi_split_by_app( argc, argv ); amrex::Initialize(argc,argv,true,comm); - + main_main(); - + amrex::Finalize(); return 0; } @@ -42,7 +42,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -81,10 +81,10 @@ void main_main () // This defines a Geometry object geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); - + if (verbosity > 0) { Print() << "3D:" << std::endl << ba << std::endl; Print() << "3D:" << std::endl << dm << std::endl; @@ -119,7 +119,7 @@ void main_main () } // define an interface. Note on the MUI side, the string "FHD-KMC-coupling" must match - // note you could use an uniface2d, however, uniface3d allows targeted receive of data on 3D side, + // note you could use an uniface2d, however, uniface3d allows targeted receive of data on 3D side, // because multiple 3D grids contain the same (x,y) coordinates, where (x,y,z) is unique mui::uniface3d uniface( "mpi://FHD-side/FHD-KMC-coupling" ); mui::sampler_exact3d r; @@ -132,67 +132,67 @@ void main_main () for ( MFIter mfi(phi); mfi.isValid(); ++mfi ) { - // Pull box from MFIter loop of MultiFab - const Box& bx = mfi.validbox(); - - // Determine box dimensions & low indices - int nx = bx.size()[0]; - int ny = bx.size()[1]; - int nz = bx.size()[2]; - int low_x = bx.smallEnd(0); - int low_y = bx.smallEnd(1); - int low_z = bx.smallEnd(2); - - if (verbosity > 0) - Print() << "3D: nx, ny = " << nx << ", " << ny << std::endl; - - // Declare single index to iterate through each box - int local_indx = 0; - - // Integer position vector used as unique identifier in MUI send & receive - Vector< int > pos; - - // Temporary variable to store data to send - double x_send; - - // z-index of slice can be specified, in this case set to "lower most" face + // Pull box from MFIter loop of MultiFab + const Box& bx = mfi.validbox(); + + // Determine box dimensions & low indices + int nx = bx.size()[0]; + int ny = bx.size()[1]; + int nz = bx.size()[2]; + int low_x = bx.smallEnd(0); + int low_y = bx.smallEnd(1); + int low_z = bx.smallEnd(2); + + if (verbosity > 0) + Print() << "3D: nx, ny = " << nx << ", " << ny << std::endl; + + // Declare single index to iterate through each box + int local_indx = 0; + + // Integer position vector used as unique identifier in MUI send & receive + Vector< int > pos; + + // Temporary variable to store data to send + double x_send; + + // z-index of slice can be specified, in this case set to "lower most" face int k = 0; - if (k >= low_z && k <= low_z+nz) { // Only send if on correct face - - // annouce send span - geometry::box3d send_region( {(double)low_x, (double)low_y, (double)k}, - {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); - printf( "send region for rank %d: %d %d - %d %d\n", myrank, low_x, low_y, - low_x+nx, low_y+ny ); - uniface.announce_send_span( 0, 1, send_region ); - - for(int j=0; j 0) { - printf("SENDER %d, step %d: channel1 at (%d,%d) is %f\n", - ParallelDescriptor::MyProc(), n, pos[0], pos[1], x_send); - } - - local_indx++; - } - } - } + if (k >= low_z && k <= low_z+nz) { // Only send if on correct face + + // annouce send span + geometry::box3d send_region( {(double)low_x, (double)low_y, (double)k}, + {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); + printf( "send region for rank %d: %d %d - %d %d\n", myrank, low_x, low_y, + low_x+nx, low_y+ny ); + uniface.announce_send_span( 0, 1, send_region ); + + for(int j=0; j 0) { + printf("SENDER %d, step %d: channel1 at (%d,%d) is %f\n", + ParallelDescriptor::MyProc(), n, pos[0], pos[1], x_send); + } + + local_indx++; + } + } + } } - + // Make sure to "commit" pushed data uniface.commit( n ); @@ -208,66 +208,66 @@ void main_main () for ( MFIter mfi(phi); mfi.isValid(); ++mfi ) { - - // Pull box from MFIter loop of MultiFab - const Box& bx = mfi.validbox(); - - // Determine box dimensions & low indices - int nx = bx.size()[0]; - int ny = bx.size()[1]; - int nz = bx.size()[2]; - int low_x = bx.smallEnd(0); - int low_y = bx.smallEnd(1); - int low_z = bx.smallEnd(2); - - if (verbosity > 0) - Print() << "3D: nx, ny = " << nx << ", " << ny << std::endl; - - // Declare single index to iterate through each box - int local_indx = 0; - - // Integer position vector used as unique identifier in MUI send & receive - Vector< int > pos; - - // Temporary variable to store incoming data - double x_recv; - - // z-index of slice can be specified, in this case set to "lower most" face + + // Pull box from MFIter loop of MultiFab + const Box& bx = mfi.validbox(); + + // Determine box dimensions & low indices + int nx = bx.size()[0]; + int ny = bx.size()[1]; + int nz = bx.size()[2]; + int low_x = bx.smallEnd(0); + int low_y = bx.smallEnd(1); + int low_z = bx.smallEnd(2); + + if (verbosity > 0) + Print() << "3D: nx, ny = " << nx << ", " << ny << std::endl; + + // Declare single index to iterate through each box + int local_indx = 0; + + // Integer position vector used as unique identifier in MUI send & receive + Vector< int > pos; + + // Temporary variable to store incoming data + double x_recv; + + // z-index of slice can be specified, in this case set to "lower most" face int k = 0; - if (k >= low_z && k <= low_z+nz) { // Only recv if on correct face - - // announce 'span' for smart sending - geometry::box3d recv_region( {(double)low_x, (double)low_y, (double)k}, - {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); - uniface.announce_recv_span( 0, 1, recv_region); - - for(int j=0; j 0) { - printf("RETURNED %d, step %d: channel2 at (%d,%d) is %f\n", - myrank, n, pos[0], pos[1], x_recv); - } - - // Access MultiFab values directly (0th component), belonging to MFI grid - // Note: make sure no ghost cells - phi[mfi].dataPtr(0)[k*nx*ny + local_indx] = x_recv; - - local_indx++; - } - } - } + if (k >= low_z && k <= low_z+nz) { // Only recv if on correct face + + // announce 'span' for smart sending + geometry::box3d recv_region( {(double)low_x, (double)low_y, (double)k}, + {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); + uniface.announce_recv_span( 0, 1, recv_region); + + for(int j=0; j 0) { + printf("RETURNED %d, step %d: channel2 at (%d,%d) is %f\n", + myrank, n, pos[0], pos[1], x_recv); + } + + // Access MultiFab values directly (0th component), belonging to MFI grid + // Note: make sure no ghost cells + phi[mfi].dataPtr(0)[k*nx*ny + local_indx] = x_recv; + + local_indx++; + } + } + } } - + // Clear data after it is received to free-up storage uniface.forget( n ); @@ -276,8 +276,8 @@ void main_main () // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && n%plot_int == 0) { - const std::string& pltfile = amrex::Concatenate("plt3D_mod_",n,5); - WriteSingleLevelPlotfile(pltfile, phi, {"phi_modified"}, geom, time, n); + const std::string& pltfile = amrex::Concatenate("plt3D_mod_",n,5); + WriteSingleLevelPlotfile(pltfile, phi, {"phi_modified"}, geom, time, n); } // Call the timer again and compute the maximum difference between the start time and stop time diff --git a/Tutorials/MUI/Source_02/main_02.cpp b/Tutorials/MUI/Source_02/main_02.cpp index 4f9646cebcd..de71e91824d 100644 --- a/Tutorials/MUI/Source_02/main_02.cpp +++ b/Tutorials/MUI/Source_02/main_02.cpp @@ -18,9 +18,9 @@ int main (int argc, char* argv[]) { MPI_Comm comm = mui::mpi_split_by_app( argc, argv ); amrex::Initialize(argc,argv,true,comm); - + main_main(); - + amrex::Finalize(); return 0; } @@ -42,7 +42,7 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - // We need to get n_cell from the inputs file - this is the number of cells on each side of + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); @@ -76,15 +76,15 @@ void main_main () // This defines the physical box, [-1,1] in each direction. RealBox real_box({AMREX_D_DECL(-1.0,-1.0,-1.0)}, - {AMREX_D_DECL( 1.0, 1.0,-1.0)}); + {AMREX_D_DECL( 1.0, 1.0,-1.0)}); // This defines a Geometry object geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); - + if (verbosity > 0) { Print() << "2D:" << std::endl << ba << std::endl; Print() << "2D:" << std::endl << dm << std::endl; @@ -102,7 +102,7 @@ void main_main () int root_proc = 0; // define an interface. Note on the MUI side, the string "FHD-KMC-coupling" must match - // note you could use an uniface2d, however, uniface3d allows targeted receive of data on 3D side, + // note you could use an uniface2d, however, uniface3d allows targeted receive of data on 3D side, // because multiple 3D grids contain the same (x,y) coordinates, where (x,y,z) is unique mui::uniface3d uniface( "mpi://KMC-side/FHD-KMC-coupling" ); mui::sampler_exact3d r; @@ -111,7 +111,7 @@ void main_main () ////////////////////////////////////////////////////////////// // Recieve 2D slice from executable - + // Wait for other executable // "uniface.commit( n )" command in other code signals current process to continue uniface.barrier( n ); @@ -119,60 +119,60 @@ void main_main () for ( MFIter mfi(phi); mfi.isValid(); ++mfi ) { - // Pull box from MFIter loop of MultiFab - const Box& bx = mfi.validbox(); + // Pull box from MFIter loop of MultiFab + const Box& bx = mfi.validbox(); + + // Determine box dimensions & low indices + int nx = bx.size()[0]; + int ny = bx.size()[1]; + int low_x = bx.smallEnd(0); + int low_y = bx.smallEnd(1); - // Determine box dimensions & low indices - int nx = bx.size()[0]; - int ny = bx.size()[1]; - int low_x = bx.smallEnd(0); - int low_y = bx.smallEnd(1); - - if (verbosity > 0) - Print() << "2D: nx, ny = " << nx << ", " << ny << std::endl; + if (verbosity > 0) + Print() << "2D: nx, ny = " << nx << ", " << ny << std::endl; - // Declare single index to iterate through each box - int local_indx = 0; + // Declare single index to iterate through each box + int local_indx = 0; - // Integer position vector used as unique identifier in MUI send & receive - Vector< int > pos; + // Integer position vector used as unique identifier in MUI send & receive + Vector< int > pos; - // Temporary variable to store incoming data - double x_recv; + // Temporary variable to store incoming data + double x_recv; - // z-index of slice must be specified for receiving span on 3D side (uniface3d only) + // z-index of slice must be specified for receiving span on 3D side (uniface3d only) int k = 0; - + // announce 'span' for smart sending - geometry::box3d recv_region( {(double)low_x, (double)low_y, (double)k}, - {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); - uniface.announce_recv_span( 0, 1, recv_region); - - for(int j=0; j 0) { - printf("RECEIVER %d, step %d: channel1 at (%d,%d) is %f\n", - myrank, n, pos[0], pos[1], x_recv); - } - - // Access MultiFab values directly (0th component), belonging to MFI grid - // Note: make sure no ghost cells - phi[mfi].dataPtr(0)[local_indx] = x_recv; - - local_indx++; - } - } + geometry::box3d recv_region( {(double)low_x, (double)low_y, (double)k}, + {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); + uniface.announce_recv_span( 0, 1, recv_region); + + for(int j=0; j 0) { + printf("RECEIVER %d, step %d: channel1 at (%d,%d) is %f\n", + myrank, n, pos[0], pos[1], x_recv); + } + + // Access MultiFab values directly (0th component), belonging to MFI grid + // Note: make sure no ghost cells + phi[mfi].dataPtr(0)[local_indx] = x_recv; + + local_indx++; + } + } } - + // Clear data after it is received to free-up storage uniface.forget( n ); @@ -181,8 +181,8 @@ void main_main () // Write a plotfile of the current data (plot_int was defined in the inputs file) if (plot_int > 0 && n%plot_int == 0) { - const std::string& pltfile = amrex::Concatenate("plt2D_",n,5); - WriteSingleLevelPlotfile(pltfile, phi, {"phi"}, geom, time, n); + const std::string& pltfile = amrex::Concatenate("plt2D_",n,5); + WriteSingleLevelPlotfile(pltfile, phi, {"phi"}, geom, time, n); } // Modify 2D phi: in this case, multiply by a constant @@ -194,65 +194,65 @@ void main_main () for ( MFIter mfi(phi); mfi.isValid(); ++mfi ) { - // Pull box from MFIter loop of MultiFab - const Box& bx = mfi.validbox(); + // Pull box from MFIter loop of MultiFab + const Box& bx = mfi.validbox(); - // Determine box dimensions & low indices - int nx = bx.size()[0]; - int ny = bx.size()[1]; - int low_x = bx.smallEnd(0); - int low_y = bx.smallEnd(1); - - if (verbosity > 0) - Print() << "3D: nx, ny = " << nx << ", " << ny << std::endl; + // Determine box dimensions & low indices + int nx = bx.size()[0]; + int ny = bx.size()[1]; + int low_x = bx.smallEnd(0); + int low_y = bx.smallEnd(1); - // Declare single index to iterate through each box - int local_indx = 0; + if (verbosity > 0) + Print() << "3D: nx, ny = " << nx << ", " << ny << std::endl; - // Integer position vector used as unique identifier in MUI send & receive - Vector< int > pos; + // Declare single index to iterate through each box + int local_indx = 0; - // Temporary variable to store data to send - double x_send; + // Integer position vector used as unique identifier in MUI send & receive + Vector< int > pos; - // z-index of slice must be specified for receiving span on 3D side (uniface3d only) + // Temporary variable to store data to send + double x_send; + + // z-index of slice must be specified for receiving span on 3D side (uniface3d only) int k = 0; - // annouce send span - geometry::box3d send_region( {(double)low_x, (double)low_y, (double)k}, - {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); - printf( "send region for rank %d: %d %d - %d %d\n", myrank, low_x, low_y, - low_x+nx, low_y+ny ); - uniface.announce_send_span( 0, 1, send_region ); - - for(int j=0; j 0) { - printf("SENDER %d, step %d: channel1 at (%d,%d) is %f\n", - ParallelDescriptor::MyProc(), n, pos[0], pos[1], x_send); - } - - local_indx++; - } - } + // annouce send span + geometry::box3d send_region( {(double)low_x, (double)low_y, (double)k}, + {(double)(low_x + nx-1), (double)(low_y + ny-1), (double)k} ); + printf( "send region for rank %d: %d %d - %d %d\n", myrank, low_x, low_y, + low_x+nx, low_y+ny ); + uniface.announce_send_span( 0, 1, send_region ); + + for(int j=0; j 0) { + printf("SENDER %d, step %d: channel1 at (%d,%d) is %f\n", + ParallelDescriptor::MyProc(), n, pos[0], pos[1], x_send); + } + + local_indx++; + } + } } // Make sure to "commit" pushed data uniface.commit( n ); - + ////////////////////////////////////////////////////////////// } diff --git a/Tutorials/MUI/doc/MUIcouplingNotes.tex b/Tutorials/MUI/doc/MUIcouplingNotes.tex index e8de74b255d..e25c3b0fd43 100644 --- a/Tutorials/MUI/doc/MUIcouplingNotes.tex +++ b/Tutorials/MUI/doc/MUIcouplingNotes.tex @@ -28,7 +28,7 @@ % text layout - change as needed \raggedright \setlength{\parindent}{0.75cm} -\textwidth 6.50in +\textwidth 6.50in \textheight 8.75in \newcommand{\natwidthpng}{610} @@ -88,7 +88,7 @@ \section*{Operation \& Test Case} \noindent In this tutorial, two different C++ codes are built separately. Each has different spatial dimensions: one is built in 3D ({\fontfamily{qcr}\selectfont AMREX\_SPACEDIM = 3}), and the other in 2D ({\fontfamily{qcr}\selectfont AMREX\_SPACEDIM = 2}). Each code is compiled separately within their respective ``exec'' directories {\fontfamily{qcr}\selectfont Exec\_01 \& Exec\_02}, after which the two executables are run together using the following command, specifying the number of MPI processes to designate to each executable: \begin{verbatim} -$ mpirun -np N1 ../Exec_01/main3d.gnu.MPI.ex inputs +$ mpirun -np N1 ../Exec_01/main3d.gnu.MPI.ex inputs : -np n2 ../Exec_02/main2d.gnu.MPI.ex inputs \end{verbatim} diff --git a/Tutorials/Particles/CellSortedParticles/CellSortedPC.H b/Tutorials/Particles/CellSortedParticles/CellSortedPC.H index fef695f0e1c..64af9f82c39 100644 --- a/Tutorials/Particles/CellSortedParticles/CellSortedPC.H +++ b/Tutorials/Particles/CellSortedParticles/CellSortedPC.H @@ -18,8 +18,8 @@ struct IntData enum { sorted, i, - j, - k, + j, + k, ncomps }; }; @@ -30,7 +30,7 @@ class CellSortedParticleContainer public: using MyParIter = amrex::ParIter; - + CellSortedParticleContainer (const amrex::Geometry & a_geom, const amrex::DistributionMapping & a_dmap, const amrex::BoxArray & a_ba); @@ -48,11 +48,11 @@ public: int numWrongCell(); void visitAllParticles(); - + protected: void UpdateCellVectors(); - + void UpdateFortranStructures(); // used to store vectors of particle indices on a cell-by-cell basis @@ -68,7 +68,7 @@ protected: amrex::DistributionMapping::RefID m_DMRef; private: - + void correctCellVectors(int old_index, int new_index, int grid, const ParticleType& p) override; }; diff --git a/Tutorials/Particles/CellSortedParticles/CellSortedPC.cpp b/Tutorials/Particles/CellSortedParticles/CellSortedPC.cpp index 66957d8705f..8ebf5fcb366 100644 --- a/Tutorials/Particles/CellSortedParticles/CellSortedPC.cpp +++ b/Tutorials/Particles/CellSortedParticles/CellSortedPC.cpp @@ -11,31 +11,31 @@ using namespace amrex; namespace { - + void get_position_unit_cell(Real* r, const IntVect& nppc, int i_part) { int nx = nppc[0]; int ny = nppc[1]; int nz = nppc[2]; - + int ix_part = i_part/(ny * nz); int iy_part = (i_part % (ny * nz)) % ny; int iz_part = (i_part % (ny * nz)) / ny; - + r[0] = (0.5+ix_part)/nx; r[1] = (0.5+iy_part)/ny; - r[2] = (0.5+iz_part)/nz; + r[2] = (0.5+iz_part)/nz; } - + void get_gaussian_random_momentum(Real* u, Real u_mean, Real u_std) { Real ux_th = amrex::RandomNormal(0.0, u_std); Real uy_th = amrex::RandomNormal(0.0, u_std); Real uz_th = amrex::RandomNormal(0.0, u_std); - + u[0] = u_mean + ux_th; u[1] = u_mean + uy_th; u[2] = u_mean + uz_th; - } + } } CellSortedParticleContainer:: @@ -51,14 +51,14 @@ CellSortedParticleContainer:: InitParticles(const IntVect& a_num_particles_per_cell) { BL_PROFILE("CellSortedParticleContainer::InitParticles"); - + const int lev = 0; const Geometry& geom = Geom(lev); const Real* dx = geom.CellSize(); const Real* plo = geom.ProbLo(); - const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], - *a_num_particles_per_cell[1], + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], + *a_num_particles_per_cell[1], *a_num_particles_per_cell[2]); for (MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) @@ -74,28 +74,28 @@ InitParticles(const IntVect& a_num_particles_per_cell) for (int i_part=0; i_partIndex(p, lev) == iv); - + particle_tile.push_back(p); } } @@ -106,7 +106,7 @@ void CellSortedParticleContainer::UpdateCellVectors() { BL_PROFILE("CellSortedParticleContainer::UpdateCellVectors"); - + const int lev = 0; bool needs_update = false; @@ -124,14 +124,14 @@ CellSortedParticleContainer::UpdateCellVectors() m_DMRef = this->ParticleDistributionMap(lev).getRefID(); needs_update = true; } - + if (! needs_update) return; // clear old data m_cell_vectors.clear(); m_vector_size.clear(); m_vector_ptrs.clear(); - + // allocate storage for cell vectors. NOTE - do not tile this loop for(MFIter mfi = MakeMFIter(lev, false); mfi.isValid(); ++mfi) { @@ -145,7 +145,7 @@ CellSortedParticleContainer::UpdateCellVectors() // insert particles into vectors - this can be tiled #ifdef AMREX_USE_OMP #pragma omp parallel -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); @@ -161,7 +161,7 @@ CellSortedParticleContainer::UpdateCellVectors() m_cell_vectors[pti.index()](iv).push_back(pindex + 1); } } - + UpdateFortranStructures(); } @@ -169,7 +169,7 @@ void CellSortedParticleContainer::UpdateFortranStructures() { BL_PROFILE("CellSortedParticleContainer::UpdateFortranStructures"); - + const int lev = 0; #ifdef AMREX_USE_OMP @@ -193,7 +193,7 @@ CellSortedParticleContainer::MoveParticles() BL_PROFILE("CellSortedParticleContainer::MoveParticles()"); UpdateCellVectors(); - + const int lev = 0; const Real* dx = Geom(lev).CellSize(); const Real* plo = Geom(lev).ProbLo(); @@ -207,11 +207,11 @@ CellSortedParticleContainer::MoveParticles() const int grid_id = pti.index(); const int tile_id = pti.LocalTileIndex(); const Box& tile_box = pti.tilebox(); - + auto& particle_tile = GetParticles(lev)[std::make_pair(grid_id,tile_id)]; auto& particles = particle_tile.GetArrayOfStructs(); const int np = particles.numParticles(); - + move_particles(particles.data(), &np, tile_box.loVect(), tile_box.hiVect(), m_vector_ptrs[grid_id].dataPtr(), @@ -234,7 +234,7 @@ void CellSortedParticleContainer::ReBin() { BL_PROFILE("CellSortedParticleContainer::ReBin()"); - + const int lev = 0; #ifdef AMREX_USE_OMP @@ -250,8 +250,8 @@ CellSortedParticleContainer::ReBin() const int np = particles.numParticles(); for(int pindex = 0; pindex < np; ++pindex) { - ParticleType& p = particles[pindex]; - if (p.idata(IntData::sorted)) continue; + ParticleType& p = particles[pindex]; + if (p.idata(IntData::sorted)) continue; const IntVect& iv = this->Index(p, lev); p.idata(IntData::sorted) = 1; p.idata(IntData::i) = iv[0]; @@ -266,8 +266,8 @@ CellSortedParticleContainer::ReBin() } void -CellSortedParticleContainer::correctCellVectors(int old_index, int new_index, - int grid, const ParticleType& p) +CellSortedParticleContainer::correctCellVectors(int old_index, int new_index, + int grid, const ParticleType& p) { if (! p.idata(IntData::sorted)) return; IntVect iv(p.idata(IntData::i), p.idata(IntData::j), p.idata(IntData::k)); @@ -287,19 +287,19 @@ CellSortedParticleContainer::SumCellVectors() const int lev = 0; int np = 0; - + #ifdef AMREX_USE_OMP #pragma omp parallel reduction(+:np) -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& tile_box = pti.tilebox(); for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) - { - np += m_vector_size[pti.index()](iv); + { + np += m_vector_size[pti.index()](iv); } } - + ParallelDescriptor::ReduceIntSum(np,ParallelDescriptor::IOProcessorNumber()); return np; } @@ -309,10 +309,10 @@ CellSortedParticleContainer::numUnsorted() { const int lev = 0; int num_unsorted = 0; - + #ifdef AMREX_USE_OMP #pragma omp parallel reduction(+:num_unsorted) -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); @@ -324,7 +324,7 @@ CellSortedParticleContainer::numUnsorted() } } } - + ParallelDescriptor::ReduceIntSum(num_unsorted, ParallelDescriptor::IOProcessorNumber()); return num_unsorted; } @@ -334,10 +334,10 @@ CellSortedParticleContainer::numWrongCell() { const int lev = 0; int num_wrong = 0; - + #ifdef AMREX_USE_OMP #pragma omp parallel reduction(+:num_wrong) -#endif +#endif for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); @@ -350,7 +350,7 @@ CellSortedParticleContainer::numWrongCell() } } } - + ParallelDescriptor::ReduceIntSum(num_wrong, ParallelDescriptor::IOProcessorNumber()); return num_wrong; } @@ -361,7 +361,7 @@ CellSortedParticleContainer::visitAllParticles() const int lev = 0; if (! m_vectors_initialized) return; - + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); @@ -369,14 +369,14 @@ CellSortedParticleContainer::visitAllParticles() Vector times_visited(np, 0); const Box& tile_box = pti.tilebox(); for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) - { + { auto& cell_vector = m_cell_vectors[grid](iv); for (int i = 0; i < static_cast(cell_vector.size()); ++i) { times_visited[cell_vector[i]-1] += 1; } } amrex::Print() << *std::min_element(times_visited.begin(), times_visited.end()) << std::endl; - amrex::Print() << *std::max_element(times_visited.begin(), times_visited.end()) << std::endl; + amrex::Print() << *std::max_element(times_visited.begin(), times_visited.end()) << std::endl; } } diff --git a/Tutorials/Particles/CellSortedParticles/cell_sorted_3d.F90 b/Tutorials/Particles/CellSortedParticles/cell_sorted_3d.F90 index 441c9a3228a..a54374623e3 100644 --- a/Tutorials/Particles/CellSortedParticles/cell_sorted_3d.F90 +++ b/Tutorials/Particles/CellSortedParticles/cell_sorted_3d.F90 @@ -1,12 +1,12 @@ module cell_sorted_particle_module use amrex_fort_module, only: amrex_real, amrex_particle_real use iso_c_binding , only: c_int - + implicit none private - + public particle_t, remove_particle_from_cell - + type, bind(C) :: particle_t real(amrex_particle_real) :: pos(3) !< Position real(amrex_particle_real) :: vel(3) !< Position @@ -19,33 +19,33 @@ module cell_sorted_particle_module end type particle_t contains - + subroutine remove_particle_from_cell(cell_parts, cell_np, new_np, i) - + use iso_c_binding, only: c_int - + implicit none - + integer(c_int), intent(in ) :: cell_np integer(c_int), intent(inout) :: cell_parts(cell_np) integer(c_int), intent(inout) :: new_np - integer(c_int), intent(in ) :: i + integer(c_int), intent(in ) :: i cell_parts(i) = cell_parts(new_np) new_np = new_np - 1 - + end subroutine remove_particle_from_cell - + end module cell_sorted_particle_module subroutine move_particles(particles, np, lo, hi, & cell_part_ids, cell_part_cnt, clo, chi, plo, dx, dt) & bind(c,name="move_particles") - + use amrex_fort_module, only: amrex_real use iso_c_binding, only: c_ptr, c_int, c_f_pointer use cell_sorted_particle_module, only: particle_t, remove_particle_from_cell - + implicit none type(particle_t), intent(inout), target :: particles(np) @@ -57,7 +57,7 @@ subroutine move_particles(particles, np, lo, hi, & real(amrex_real), intent(in) :: plo(3) real(amrex_real), intent(in) :: dx(3) real(amrex_real), intent(in) :: dt - + integer :: i, j, k, p, cell_np, new_np integer :: cell(3) integer(c_int), pointer :: cell_parts(:) @@ -65,7 +65,7 @@ subroutine move_particles(particles, np, lo, hi, & real(amrex_real) inv_dx(3) inv_dx = 1.d0/dx - + do k = lo(3), hi(3) do j = lo(2), hi(2) do i = lo(1), hi(1) @@ -76,25 +76,25 @@ subroutine move_particles(particles, np, lo, hi, & p = 1 do while (p <= new_np) part => particles(cell_parts(p)) - + ! move the particle in a straight line part%pos = part%pos + dt*part%vel ! if it has changed cells, remove from vector. ! otherwise continue - cell = floor((part%pos - plo)*inv_dx) + cell = floor((part%pos - plo)*inv_dx) if ((cell(1) /= i) .or. (cell(2) /= j) .or. (cell(3) /= k)) then part%sorted = 0 - call remove_particle_from_cell(cell_parts, cell_np, new_np, p) + call remove_particle_from_cell(cell_parts, cell_np, new_np, p) else p = p + 1 end if end do cell_part_cnt(i,j,k) = new_np - + end do end do end do - + end subroutine move_particles diff --git a/Tutorials/Particles/CellSortedParticles/cell_sorted_F.H b/Tutorials/Particles/CellSortedParticles/cell_sorted_F.H index 39f30abcd1a..e10134daba6 100644 --- a/Tutorials/Particles/CellSortedParticles/cell_sorted_F.H +++ b/Tutorials/Particles/CellSortedParticles/cell_sorted_F.H @@ -19,7 +19,7 @@ extern "C" const amrex_real* plo, const amrex_real* dx, const amrex_real* dt); - + #ifdef __cplusplus } #endif diff --git a/Tutorials/Particles/CellSortedParticles/main.cpp b/Tutorials/Particles/CellSortedParticles/main.cpp index b9d74a4ef5d..d454c79cde0 100644 --- a/Tutorials/Particles/CellSortedParticles/main.cpp +++ b/Tutorials/Particles/CellSortedParticles/main.cpp @@ -29,32 +29,32 @@ void test_cell_sorted(const TestParams& parms) real_box.setLo(n, 0.0); real_box.setHi(n, 1.0); } - - IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); + + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); IntVect domain_hi(AMREX_D_DECL(parms.ncell[0]-1, parms.ncell[1]-1, - parms.ncell[2]-1)); + parms.ncell[2]-1)); const Box domain(domain_lo, domain_hi); - + int coord = 0; int is_per[BL_SPACEDIM]; - for (int i = 0; i < BL_SPACEDIM; i++) - is_per[i] = 1; + for (int i = 0; i < BL_SPACEDIM; i++) + is_per[i] = 1; Geometry geom(domain, &real_box, coord, is_per); - + BoxArray ba(domain); ba.maxSize(parms.max_grid_size); DistributionMapping dm(ba); - + amrex::Print() << "Initializing particles... "; CellSortedParticleContainer particles(geom, dm, ba); particles.InitParticles(parms.nppc); - + amrex::Print() << "Done. " << std::endl; BL_PROFILE_VAR_STOP(blp_init); - + amrex::Print() << "Starting main loop... " << std::endl; BL_PROFILE_VAR_START(blp_evolve); @@ -66,7 +66,7 @@ void test_cell_sorted(const TestParams& parms) particles.MoveParticles(); particles.Redistribute(); particles.ReBin(); - amrex::Print() << " Number of particles in cell vectors is " << particles.SumCellVectors() << std::endl; + amrex::Print() << " Number of particles in cell vectors is " << particles.SumCellVectors() << std::endl; } amrex::Print() << "Done. " << std::endl; @@ -77,7 +77,7 @@ void test_cell_sorted(const TestParams& parms) int main(int argc, char* argv[]) { amrex::Initialize(argc,argv); - + amrex::InitRandom(451); ParmParse pp; diff --git a/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/EMParticleContainer.cpp b/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/EMParticleContainer.cpp index b2b4e95108b..5af94ae64e3 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/EMParticleContainer.cpp +++ b/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/EMParticleContainer.cpp @@ -71,25 +71,25 @@ PushAndDeposeParticles(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& E deposit_current(jxarr, jyarr, jzarr, pstruct[i], uxp[i], uyp[i], uzp[i], ginv, wp[i], q, dt, plo, dxi); - }); + }); } } void EMParticleContainer:: PushParticleMomenta(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, Real dt) -{ +{ BL_PROFILE("EMParticleContainer::PushParticleMomenta"); const int lev = 0; - + const auto dxi = Geom(lev).InvCellSizeArray(); const auto plo = Geom(lev).ProbLoArray(); for (EMParIter pti(*this, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); - + ParticleType const* AMREX_RESTRICT pstruct = &(pti.GetArrayOfStructs()[0]); auto& attribs = pti.GetAttribs(); diff --git a/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/em_pic_K.H b/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/em_pic_K.H index a20a1ea4fdb..5d6f1d801ec 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/em_pic_K.H +++ b/Tutorials/Particles/ElectromagneticPIC/Exec/CUDA/em_pic_K.H @@ -45,7 +45,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, constexpr amrex::Real sx0[] = {1.,0.}; constexpr amrex::Real sy0[] = {1.,0.}; constexpr amrex::Real sz0[] = {1.,0.}; - + constexpr int ixmin = 0; constexpr int ixmax = 0; constexpr int iymin = 0; @@ -67,7 +67,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, Exp += sx0[jj]*sy[kk]*sz[ll]*Exarr(j0+jj,k+kk,l+ll); } } - } + } Eyp = 0.; for (int ll = izmin ; ll <= izmax +1; ++ll) { @@ -76,7 +76,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, Eyp += sx[jj]*sy0[kk]*sz[ll]*Eyarr(j+jj,k0+kk,l+ll); } } - } + } Ezp = 0.; for (int ll = izmin0; ll <= izmax0 ; ++ll) { @@ -85,7 +85,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, Ezp += sx[jj]*sy[kk]*sz0[ll]*Ezarr(j+jj,k+kk,l0+ll); } } - } + } Bxp = 0.; for (int ll = izmin0 ; ll <= izmax0 ; ++ll) { @@ -94,7 +94,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, Bxp += sx[jj]*sy0[kk]*sz0[ll]*Bxarr(j+jj,k0+kk,l0+ll); } } - } + } Byp = 0.; for (int ll = izmin0; ll <= izmax0 ; ++ll) { @@ -103,7 +103,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, Byp += sx0[jj]*sy[kk]*sz0[ll]*Byarr(j0+jj,k+kk,l0+ll); } } - } + } Bzp = 0.; for (int ll = izmin ; ll <= izmax +1; ++ll) { @@ -112,7 +112,7 @@ void gather_fields (EMParticleContainer::ParticleType const& p, Bzp += sx0[jj]*sy0[kk]*sz[ll]*Bzarr(j0+jj,k0+kk,l+ll); } } - } + } } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -124,7 +124,7 @@ void push_momentum_boris(amrex::Real& uxp, amrex::Real& uyp, amrex::Real& uzp, a amrex::Real cst = q*dt*0.5/m; constexpr amrex::Real clight = 2.99792458e8; constexpr amrex::Real clightisq = 1./(clight*clight); - + uxp += Exp * cst; uyp += Eyp * cst; uzp += Ezp * cst; @@ -151,7 +151,7 @@ void push_momentum_boris(amrex::Real& uxp, amrex::Real& uyp, amrex::Real& uzp, a uzp += Ezp*cst; usq = (uxp*uxp + uyp*uyp + uzp*uzp) * clightisq; - gaminv = 1.0/std::sqrt(1.0+usq); + gaminv = 1.0/std::sqrt(1.0+usq); } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE diff --git a/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/EMParticleContainer.cpp b/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/EMParticleContainer.cpp index 77eb119e0d8..1f0bced789a 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/EMParticleContainer.cpp +++ b/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/EMParticleContainer.cpp @@ -13,14 +13,14 @@ PushAndDeposeParticles(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& E BL_PROFILE("EMParticleContainer::PushAndDeposeParticles"); const int lev = 0; - + const Real* dx = Geom(lev).CellSize(); const Real* plo = Geom(lev).ProbLo(); for (EMParIter pti(*this, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); - + auto& structs = pti.GetArrayOfStructs(); auto& attribs = pti.GetAttribs(); @@ -35,29 +35,29 @@ PushAndDeposeParticles(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& E auto& Byp = attribs[PIdx::By]; auto& Bzp = attribs[PIdx::Bz]; auto& ginv = attribs[PIdx::ginv]; - - gather_magnetic_field(np, structs.dataPtr(), + + gather_magnetic_field(np, structs.dataPtr(), Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), BL_TO_FORTRAN_3D(Bx[pti]), BL_TO_FORTRAN_3D(By[pti]), BL_TO_FORTRAN_3D(Bz[pti]), plo, dx); - - gather_electric_field(np, structs.dataPtr(), + + gather_electric_field(np, structs.dataPtr(), Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), BL_TO_FORTRAN_3D(Ex[pti]), BL_TO_FORTRAN_3D(Ey[pti]), BL_TO_FORTRAN_3D(Ez[pti]), plo, dx); - + push_momentum_boris(np, uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), ginv.dataPtr(), Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), m_charge, m_mass, dt); - + push_position_boris(np, structs.dataPtr(), uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), ginv.dataPtr(), dt); - + deposit_current(BL_TO_FORTRAN_3D(jx[pti]), BL_TO_FORTRAN_3D(jy[pti]), BL_TO_FORTRAN_3D(jz[pti]), @@ -71,18 +71,18 @@ PushAndDeposeParticles(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& E void EMParticleContainer:: PushParticleMomenta(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, const MultiFab& Bx, const MultiFab& By, const MultiFab& Bz, Real dt) -{ +{ BL_PROFILE("EMParticleContainer::PushParticleMomenta"); const int lev = 0; - + const Real* dx = Geom(lev).CellSize(); const Real* plo = Geom(lev).ProbLo(); for (EMParIter pti(*this, lev); pti.isValid(); ++pti) { const int np = pti.numParticles(); - + auto& structs = pti.GetArrayOfStructs(); auto& attribs = pti.GetAttribs(); @@ -97,20 +97,20 @@ PushParticleMomenta(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, auto& Bzp = attribs[PIdx::Bz]; auto& ginv = attribs[PIdx::ginv]; - gather_magnetic_field(np, structs.dataPtr(), + gather_magnetic_field(np, structs.dataPtr(), Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), BL_TO_FORTRAN_3D(Bx[pti]), BL_TO_FORTRAN_3D(By[pti]), BL_TO_FORTRAN_3D(Bz[pti]), plo, dx); - - gather_electric_field(np, structs.dataPtr(), + + gather_electric_field(np, structs.dataPtr(), Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), BL_TO_FORTRAN_3D(Ex[pti]), BL_TO_FORTRAN_3D(Ey[pti]), BL_TO_FORTRAN_3D(Ez[pti]), plo, dx); - + push_momentum_boris(np, uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), ginv.dataPtr(), Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), diff --git a/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/em_pic_3d.F90 b/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/em_pic_3d.F90 index 3b8bd43806c..060ffe71909 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/em_pic_3d.F90 +++ b/Tutorials/Particles/ElectromagneticPIC/Exec/OpenACC/em_pic_3d.F90 @@ -98,14 +98,14 @@ subroutine push_position_boris(np, structs, uxp, uyp, uzp, gaminv, dt) & use em_particle_module, only : particle_t use amrex_fort_module, only : amrex_real implicit none - + integer, intent(in), value :: np type(particle_t), intent(inout) :: structs(np) real(amrex_real), intent(in) :: uxp(np), uyp(np), uzp(np), gaminv(np) real(amrex_real), intent(in), value :: dt - + integer :: ip - + !$acc parallel deviceptr(structs, uxp, uyp, uzp, gaminv) !$acc loop gang vector do ip = 1, np @@ -115,7 +115,7 @@ subroutine push_position_boris(np, structs, uxp, uyp, uzp, gaminv, dt) & end do !$acc end loop !$acc end parallel - + end subroutine push_position_boris subroutine set_gamma(np, uxp, uyp, uzp, gaminv) & @@ -132,16 +132,16 @@ subroutine set_gamma(np, uxp, uyp, uzp, gaminv) & integer :: ip real(amrex_real) :: clghtisq, usq clghtisq = 1.d0/clight**2 - + !$acc parallel deviceptr(uxp, uyp, uzp, gaminv) !$acc loop gang vector - do ip = 1, np + do ip = 1, np usq = (uxp(ip)**2 + uyp(ip)**2+ uzp(ip)**2)*clghtisq - gaminv(ip) = 1.d0/sqrt(1.d0 + usq) + gaminv(ip) = 1.d0/sqrt(1.d0 + usq) end do !$acc end loop !$acc end parallel - + end subroutine set_gamma subroutine deposit_current(jx, jxlo, jxhi, jy, jylo, jyhi, jz, jzlo, jzhi, np, structs, & diff --git a/Tutorials/Particles/ElectromagneticPIC/Exec/OpenMP/EMParticleContainer.cpp b/Tutorials/Particles/ElectromagneticPIC/Exec/OpenMP/EMParticleContainer.cpp index 8fcac3cc2c9..1f0bced789a 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Exec/OpenMP/EMParticleContainer.cpp +++ b/Tutorials/Particles/ElectromagneticPIC/Exec/OpenMP/EMParticleContainer.cpp @@ -36,14 +36,14 @@ PushAndDeposeParticles(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& E auto& Bzp = attribs[PIdx::Bz]; auto& ginv = attribs[PIdx::ginv]; - gather_magnetic_field(np, structs.dataPtr(), + gather_magnetic_field(np, structs.dataPtr(), Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), BL_TO_FORTRAN_3D(Bx[pti]), BL_TO_FORTRAN_3D(By[pti]), BL_TO_FORTRAN_3D(Bz[pti]), plo, dx); - gather_electric_field(np, structs.dataPtr(), + gather_electric_field(np, structs.dataPtr(), Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), BL_TO_FORTRAN_3D(Ex[pti]), BL_TO_FORTRAN_3D(Ey[pti]), @@ -97,14 +97,14 @@ PushParticleMomenta(const MultiFab& Ex, const MultiFab& Ey, const MultiFab& Ez, auto& Bzp = attribs[PIdx::Bz]; auto& ginv = attribs[PIdx::ginv]; - gather_magnetic_field(np, structs.dataPtr(), + gather_magnetic_field(np, structs.dataPtr(), Bxp.dataPtr(), Byp.dataPtr(), Bzp.dataPtr(), BL_TO_FORTRAN_3D(Bx[pti]), BL_TO_FORTRAN_3D(By[pti]), BL_TO_FORTRAN_3D(Bz[pti]), plo, dx); - gather_electric_field(np, structs.dataPtr(), + gather_electric_field(np, structs.dataPtr(), Exp.dataPtr(), Eyp.dataPtr(), Ezp.dataPtr(), BL_TO_FORTRAN_3D(Ex[pti]), BL_TO_FORTRAN_3D(Ey[pti]), diff --git a/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainer.H b/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainer.H index c06296aff4f..ea4c2ded391 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainer.H +++ b/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainer.H @@ -20,19 +20,19 @@ public: // EMParIter (ContainerType& pc, int level); - const std::array& GetAttribs () const { - return GetStructOfArrays().GetRealData(); + const std::array& GetAttribs () const { + return GetStructOfArrays().GetRealData(); } - std::array& GetAttribs () { - return GetStructOfArrays().GetRealData(); + std::array& GetAttribs () { + return GetStructOfArrays().GetRealData(); } - const RealVector& GetAttribs (int comp) const { + const RealVector& GetAttribs (int comp) const { return GetStructOfArrays().GetRealData(comp); } - RealVector& GetAttribs (int comp) { + RealVector& GetAttribs (int comp) { return GetStructOfArrays().GetRealData(comp); } }; diff --git a/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainerInit.cpp b/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainerInit.cpp index fc36b7d83bd..8175267ef1a 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainerInit.cpp +++ b/Tutorials/Particles/ElectromagneticPIC/Source/EMParticleContainerInit.cpp @@ -4,7 +4,7 @@ using namespace amrex; namespace -{ +{ AMREX_GPU_HOST_DEVICE void get_position_unit_cell(Real* r, const IntVect& nppc, int i_part) { int nx = nppc[0]; @@ -24,7 +24,7 @@ namespace Real ux_th = amrex::RandomNormal(0.0, u_std); Real uy_th = amrex::RandomNormal(0.0, u_std); Real uz_th = amrex::RandomNormal(0.0, u_std); - + u[0] = u_mean + ux_th; u[1] = u_mean + uy_th; u[2] = u_mean + uz_th; @@ -53,15 +53,15 @@ InitParticles(const IntVect& a_num_particles_per_cell, { BL_PROFILE("EMParticleContainer::InitParticles"); - const int lev = 0; + const int lev = 0; const auto dx = Geom(lev).CellSizeArray(); const auto plo = Geom(lev).ProbLoArray(); - + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], *a_num_particles_per_cell[1], *a_num_particles_per_cell[2]); const Real scale_fac = dx[0]*dx[1]*dx[2]/num_ppc; - + for(MFIter mfi = MakeMFIter(lev); mfi.isValid(); ++mfi) { const Box& tile_box = mfi.tilebox(); @@ -71,33 +71,33 @@ InitParticles(const IntVect& a_num_particles_per_cell, Gpu::ManagedVector counts(tile_box.numPts(), 0); unsigned int* pcount = counts.dataPtr(); - + Gpu::ManagedVector offsets(tile_box.numPts()); unsigned int* poffset = offsets.dataPtr(); - + amrex::ParallelFor(tile_box, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { for (int i_part=0; i_part= a_bounds.hi(0) || x < a_bounds.lo(0) || y >= a_bounds.hi(1) || y < a_bounds.lo(1) || z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue; - + int ix = i - lo.x; int iy = j - lo.y; int iz = k - lo.z; int nx = hi.x-lo.x+1; int ny = hi.y-lo.y+1; - int nz = hi.z-lo.z+1; + int nz = hi.z-lo.z+1; unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); @@ -118,11 +118,11 @@ InitParticles(const IntVect& a_num_particles_per_cell, particle_tile.resize(new_size); if (num_to_add == 0) continue; - + ParticleType* pstruct = particle_tile.GetArrayOfStructs()().data(); auto arrdata = particle_tile.GetStructOfArrays().realarray(); - + int procID = ParallelDescriptor::MyProc(); amrex::ParallelFor(tile_box, @@ -133,7 +133,7 @@ InitParticles(const IntVect& a_num_particles_per_cell, int iz = k - lo.z; int nx = hi.x-lo.x+1; int ny = hi.y-lo.y+1; - int nz = hi.z-lo.z+1; + int nz = hi.z-lo.z+1; unsigned int uix = amrex::min(nx-1,amrex::max(0,ix)); unsigned int uiy = amrex::min(ny-1,amrex::max(0,iy)); unsigned int uiz = amrex::min(nz-1,amrex::max(0,iz)); @@ -145,13 +145,13 @@ InitParticles(const IntVect& a_num_particles_per_cell, { Real r[3]; Real u[3]; - + get_position_unit_cell(r, a_num_particles_per_cell, i_part); - + Real x = plo[0] + (i + r[0])*dx[0]; Real y = plo[1] + (j + r[1])*dx[1]; Real z = plo[2] + (k + r[2])*dx[2]; - + if (a_problem == 0) { get_gaussian_random_momentum(u, a_thermal_momentum_mean, a_thermal_momentum_std); @@ -163,18 +163,18 @@ InitParticles(const IntVect& a_num_particles_per_cell, } else { amrex::Abort("problem type not valid"); } - + if (x >= a_bounds.hi(0) || x < a_bounds.lo(0) || y >= a_bounds.hi(1) || y < a_bounds.lo(1) || z >= a_bounds.hi(2) || z < a_bounds.lo(2) ) continue; - + ParticleType& p = pstruct[pidx]; p.id() = 0; p.cpu() = procID; p.pos(0) = x; p.pos(1) = y; p.pos(2) = z; - + arrdata[PIdx::ux ][pidx] = u[0] * PhysConst::c; arrdata[PIdx::uy ][pidx] = u[1] * PhysConst::c; arrdata[PIdx::uz ][pidx] = u[2] * PhysConst::c; diff --git a/Tutorials/Particles/ElectromagneticPIC/Source/main.cpp b/Tutorials/Particles/ElectromagneticPIC/Source/main.cpp index 2d519246fb6..f824019513e 100644 --- a/Tutorials/Particles/ElectromagneticPIC/Source/main.cpp +++ b/Tutorials/Particles/ElectromagneticPIC/Source/main.cpp @@ -31,9 +31,9 @@ void test_em_pic(const TestParams& parms) BL_PROFILE("test_em_pic"); BL_PROFILE_VAR_NS("evolve_time", blp_evolve); BL_PROFILE_VAR_NS("init_time", blp_init); - + BL_PROFILE_VAR_START(blp_init); - + RealBox real_box; for (int n = 0; n < BL_SPACEDIM; n++) { @@ -50,11 +50,11 @@ void test_em_pic(const TestParams& parms) for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = 1; Geometry geom(domain, &real_box, coord, is_per); - + BoxArray ba(domain); ba.maxSize(parms.max_grid_size); DistributionMapping dm(ba); - + const int ng = 1; MultiFab Bx(amrex::convert(ba, YeeGrid::Bx_nodal_flag), dm, 1, ng); @@ -157,9 +157,9 @@ void test_em_pic(const TestParams& parms) evolve_magnetic_field(Ex, Ey, Ez, Bx, By, Bz, geom, 0.5*dt); fill_boundary_magnetic_field(Bx, By, Bz, geom); - + evolve_electric_field(Ex, Ey, Ez, Bx, By, Bz, jx, jy, jz, geom, dt); - + if (step == nsteps - 1) { evolve_magnetic_field(Ex, Ey, Ez, Bx, By, Bz, geom, 0.5*dt); @@ -174,14 +174,14 @@ void test_em_pic(const TestParams& parms) { particles[i]->RedistributeLocal(); } - + time += dt; } - + amrex::Print() << "Done. " << std::endl; - + BL_PROFILE_VAR_STOP(blp_evolve); - + if (parms.problem_type == Langmuir) { check_solution(jx, geom, time); diff --git a/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.H b/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.H index 5a9e44a053b..e297878320a 100644 --- a/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.H +++ b/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.H @@ -17,7 +17,7 @@ struct PIdx #if BL_SPACEDIM == 3 vz, #endif - Ex, Ey, + Ex, Ey, #if BL_SPACEDIM == 3 Ez, #endif @@ -31,7 +31,7 @@ class MyParIter public: using amrex::ParIter<0, 0, PIdx::nattribs>::ParIter; using RealVector = ParIter::ContainerType::RealVector; - + const std::array& GetAttribs () const { return GetStructOfArrays().GetRealData(); } @@ -43,7 +43,7 @@ public: const RealVector& GetAttribs (int comp) const { return GetStructOfArrays().GetRealData(comp); } - + RealVector& GetAttribs (int comp) { return GetStructOfArrays().GetRealData(comp); } @@ -58,16 +58,16 @@ class ElectrostaticParticleContainer { public: - ElectrostaticParticleContainer(const Vector & geom, + ElectrostaticParticleContainer(const Vector & geom, const Vector & dmap, const Vector & ba, const Vector & rr) - : ParticleContainer<0, 0, PIdx::nattribs> (geom, dmap, ba, rr) + : ParticleContainer<0, 0, PIdx::nattribs> (geom, dmap, ba, rr) { charge = -PhysConst::q_e; mass = PhysConst::m_e; } - + void InitParticles(); void pushX(const Real& dt); @@ -82,7 +82,7 @@ public: void writeParticles(int n); private: - + Real charge, mass; }; diff --git a/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.cpp b/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.cpp index df7e427533f..48225eb1b36 100644 --- a/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.cpp +++ b/Tutorials/Particles/ElectrostaticPIC/ElectrostaticParticleContainer.cpp @@ -14,14 +14,14 @@ void ElectrostaticParticleContainer::InitParticles() { ParticleType p; p.id() = ParticleType::NextID(); - p.cpu() = ParallelDescriptor::MyProc(); - - p.pos(0) = -2.5e-6; + p.cpu() = ParallelDescriptor::MyProc(); + + p.pos(0) = -2.5e-6; p.pos(1) = 0.0; #if BL_SPACEDIM == 3 p.pos(2) = 0.0; #endif - + std::array attribs; attribs[PIdx::w] = 1.0; attribs[PIdx::vx] = 0.0; @@ -34,12 +34,12 @@ void ElectrostaticParticleContainer::InitParticles() { #if BL_SPACEDIM == 3 attribs[PIdx::Ez] = 0.0; #endif - + // Add to level 0, grid 0, and tile 0 // Redistribute() will move it to the proper place. std::pair key {0,0}; auto& particle_tile = GetParticles(0)[key]; - + particle_tile.push_back(p); particle_tile.push_back_real(attribs); @@ -50,37 +50,37 @@ void ElectrostaticParticleContainer::InitParticles() { void ElectrostaticParticleContainer::DepositCharge(ScalarMeshData& rho) { - + int num_levels = rho.size(); int finest_level = num_levels - 1; // each level deposits it's own particles const int ng = rho[0]->nGrow(); - for (int lev = 0; lev < num_levels; ++lev) { + for (int lev = 0; lev < num_levels; ++lev) { rho[lev]->setVal(0.0, ng); const auto& gm = m_gdb->Geom(lev); const auto& ba = m_gdb->ParticleBoxArray(lev); - + const Real* dx = gm.CellSize(); const Real* plo = gm.ProbLo(); BoxArray nba = ba; nba.surroundingNodes(); - + for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { const Box& box = nba[pti]; - + auto& wp = pti.GetAttribs(PIdx::w); const auto& particles = pti.GetArrayOfStructs(); int nstride = particles.dataShape().first; const Long np = pti.numParticles(); - + FArrayBox& rhofab = (*rho[lev])[pti]; - + deposit_cic(particles.data(), nstride, np, wp.data(), &this->charge, - rhofab.dataPtr(), box.loVect(), box.hiVect(), + rhofab.dataPtr(), box.loVect(), box.hiVect(), plo, dx, &ng); } @@ -94,12 +94,12 @@ ElectrostaticParticleContainer::DepositCharge(ScalarMeshData& rho) { const DistributionMapping& fine_dm = rho[lev+1]->DistributionMap(); BoxArray coarsened_fine_BA = fine_BA; coarsened_fine_BA.coarsen(m_gdb->refRatio(lev)); - + MultiFab coarsened_fine_data(coarsened_fine_BA, fine_dm, 1, 0); coarsened_fine_data.setVal(0.0); - + IntVect ratio(D_DECL(2, 2, 2)); // FIXME - + for (MFIter mfi(coarsened_fine_data); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); const Box& crse_box = coarsened_fine_data[mfi].box(); @@ -108,10 +108,10 @@ ElectrostaticParticleContainer::DepositCharge(ScalarMeshData& rho) { coarsened_fine_data[mfi].dataPtr(), crse_box.loVect(), crse_box.hiVect(), (*rho[lev+1])[mfi].dataPtr(), fine_box.loVect(), fine_box.hiVect()); } - + rho[lev]->copy(coarsened_fine_data, m_gdb->Geom(lev).periodicity(), FabArrayBase::ADD); } - + for (int lev = 0; lev < num_levels; ++lev) { rho[lev]->mult(-1.0/PhysConst::ep0, ng); } @@ -164,11 +164,11 @@ FieldGather(const VectorMeshData& E, #endif interpolate_cic(particles.data(), nstride, np, - Exp.data(), Eyp.data(), -#if BL_SPACEDIM == 3 + Exp.data(), Eyp.data(), +#if BL_SPACEDIM == 3 Ezp.data(), #endif - exfab.dataPtr(), eyfab.dataPtr(), + exfab.dataPtr(), eyfab.dataPtr(), #if BL_SPACEDIM == 3 ezfab.dataPtr(), #endif @@ -188,7 +188,7 @@ FieldGather(const VectorMeshData& E, #if BL_SPACEDIM == 3 MultiFab coarse_Ez(coarsened_fine_BA, fine_dm, 1, 1); #endif - + coarse_Ex.copy(*E[0][0], 0, 0, 1, 1, 1); coarse_Ey.copy(*E[0][1], 0, 0, 1, 1, 1); #if BL_SPACEDIM == 3 @@ -234,35 +234,35 @@ FieldGather(const VectorMeshData& E, if (lev == 0) { interpolate_cic(particles.data(), nstride, np, - Exp.data(), Eyp.data(), -#if BL_SPACEDIM == 3 + Exp.data(), Eyp.data(), +#if BL_SPACEDIM == 3 Ezp.data(), #endif - exfab.dataPtr(), eyfab.dataPtr(), + exfab.dataPtr(), eyfab.dataPtr(), #if BL_SPACEDIM == 3 ezfab.dataPtr(), #endif - box.loVect(), box.hiVect(), plo, dx, &ng); + box.loVect(), box.hiVect(), plo, dx, &ng); } else { - + const FArrayBox& exfab_coarse = coarse_Ex[pti]; const FArrayBox& eyfab_coarse = coarse_Ey[pti]; #if BL_SPACEDIM == 3 const FArrayBox& ezfab_coarse = coarse_Ez[pti]; -#endif +#endif const Box& coarse_box = coarsened_fine_BA[pti]; const Real* coarse_dx = Geom(0).CellSize(); - + interpolate_cic_two_levels(particles.data(), nstride, np, - Exp.data(), Eyp.data(), -#if BL_SPACEDIM == 3 + Exp.data(), Eyp.data(), +#if BL_SPACEDIM == 3 Ezp.data(), #endif - exfab.dataPtr(), eyfab.dataPtr(), + exfab.dataPtr(), eyfab.dataPtr(), #if BL_SPACEDIM == 3 ezfab.dataPtr(), #endif - box.loVect(), box.hiVect(), dx, + box.loVect(), box.hiVect(), dx, exfab_coarse.dataPtr(), eyfab_coarse.dataPtr(), #if BL_SPACEDIM == 3 ezfab_coarse.dataPtr(), @@ -275,25 +275,25 @@ FieldGather(const VectorMeshData& E, } } -void +void ElectrostaticParticleContainer:: Evolve(const VectorMeshData& E, ScalarMeshData& rho, const Real& dt) { - + const int num_levels = E.size(); for (int lev = 0; lev < num_levels; ++lev) { - + const auto& gm = m_gdb->Geom(lev); const RealBox& prob_domain = gm.ProbDomain(); - + BL_ASSERT(OnSameGrids(lev, *rho[lev])); for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { - + // Particle structs auto& particles = pti.GetArrayOfStructs(); - int nstride = particles.dataShape().first; + int nstride = particles.dataShape().first; const Long np = pti.numParticles(); - + // Particle attributes auto& attribs = pti.GetAttribs(); auto& vxp = attribs[PIdx::vx]; @@ -309,16 +309,16 @@ Evolve(const VectorMeshData& E, ScalarMeshData& rho, const Real& dt) { #if BL_SPACEDIM == 3 auto& Ezp = attribs[PIdx::Ez]; #endif - + // // Particle Push // push_leapfrog(particles.data(), nstride, np, - vxp.data(), vyp.data(), + vxp.data(), vyp.data(), #if BL_SPACEDIM == 3 vzp.data(), #endif - Exp.data(), Eyp.data(), + Exp.data(), Eyp.data(), #if BL_SPACEDIM == 3 Ezp.data(), #endif @@ -329,24 +329,24 @@ Evolve(const VectorMeshData& E, ScalarMeshData& rho, const Real& dt) { } void ElectrostaticParticleContainer::pushX(const Real& dt) { - for (int lev = 0; lev <= finestLevel(); ++lev) { + for (int lev = 0; lev <= finestLevel(); ++lev) { const auto& gm = m_gdb->Geom(lev); const RealBox& prob_domain = gm.ProbDomain(); for (MyParIter pti(*this, lev); pti.isValid(); ++pti) { auto& particles = pti.GetArrayOfStructs(); int nstride = particles.dataShape().first; const Long np = pti.numParticles(); - + auto& attribs = pti.GetAttribs(); auto& vxp = attribs[PIdx::vx]; auto& vyp = attribs[PIdx::vy]; #if BL_SPACEDIM == 3 auto& vzp = attribs[PIdx::vz]; -#endif +#endif push_leapfrog_positions(particles.data(), nstride, np, - vxp.data(), vyp.data(), + vxp.data(), vyp.data(), #if BL_SPACEDIM == 3 - vzp.data(), + vzp.data(), #endif &dt, prob_domain.lo(), prob_domain.hi()); diff --git a/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_2d.f90 b/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_2d.f90 index f8222d99d66..a06462f03de 100644 --- a/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_2d.f90 +++ b/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_2d.f90 @@ -16,16 +16,16 @@ subroutine sum_fine_to_crse_nodal (lo, hi, lrat, crse, clo, chi, fine, flo, fhi) integer, intent(in) :: lrat(2) real(amrex_real), intent(inout) :: crse(clo(1):chi(1),clo(2):chi(2)) real(amrex_real), intent(in) :: fine(flo(1):fhi(1),flo(2):fhi(2)) - + integer :: i, j, ii, jj - + do j = lo(2), hi(2) jj = j * lrat(2) do i = lo(1), hi(1) ii = i * lrat(1) crse(i,j) = fine(ii,jj) + & ! These four fine nodes are shared by two coarse nodes... - 0.5d0 * (fine(ii-1,jj) + fine(ii+1,jj) + & + 0.5d0 * (fine(ii-1,jj) + fine(ii+1,jj) + & fine(ii,jj-1) + fine(ii,jj+1)) + & ! ... and these four are shared by four... 0.25d0 * (fine(ii-1,jj-1) + fine(ii-1,jj+1) + & @@ -44,11 +44,11 @@ subroutine zero_out_bndry (lo, hi, input_data, bndry_data, mask) & double precision, intent(inout) :: input_data(lo(1):hi(1),lo(2):hi(2)) double precision, intent(inout) :: bndry_data(lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1) integer(c_int), intent(in ) :: mask (lo(1):hi(1),lo(2):hi(2)) - + integer :: i, j - + do j = lo(2), hi(2) - do i = lo(1), hi(1) + do i = lo(1), hi(1) if (mask(i,j) .eq. 1) then bndry_data(i,j) = input_data(i,j) input_data(i,j) = 0.d0 @@ -69,14 +69,14 @@ subroutine build_mask (lo, hi, tmp_mask, mask, ncells) & do j = lo(2), hi(2) do i = lo(1), hi(1) - + total = 0 do ii = i-ncells, i+ncells do jj = j-ncells, j+ncells total = total + tmp_mask(ii, jj) end do end do - + if (total .gt. 0) then mask(i,j) = 1 else @@ -89,7 +89,7 @@ subroutine build_mask (lo, hi, tmp_mask, mask, ncells) & end subroutine build_mask ! This routine computes the node-centered electric field given a node-centered phi. -! The gradient is computed using 2nd-order centered differences. It assumes the +! The gradient is computed using 2nd-order centered differences. It assumes the ! Boundary conditions have already been set and that you have one row of ghost cells. ! Note that this routine includes the minus sign in E = - grad phi. ! @@ -113,7 +113,7 @@ subroutine compute_E_nodal (lo, hi, phi, Ex, Ey, dx) & do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - + Ex(i,j) = fac(1) * (phi(i-1,j) - phi(i+1,j)) Ey(i,j) = fac(2) * (phi(i,j-1) - phi(i,j+1)) @@ -122,11 +122,11 @@ subroutine compute_E_nodal (lo, hi, phi, Ex, Ey, dx) & end subroutine compute_E_nodal -! This routine computes the charge density due to the particles using cloud-in-cell +! This routine computes the charge density due to the particles using cloud-in-cell ! deposition. The Fab rho is assumed to be node-centered. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! weights : the particle weights @@ -193,7 +193,7 @@ end subroutine deposit_cic ! node-centered. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! Ex_p : the electric field in the x-direction at the particle positions (output) @@ -289,7 +289,7 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & lx = (particles(1, n) - plo(1))*inv_dx(1) ly = (particles(2, n) - plo(2))*inv_dx(2) - + i = floor(lx) j = floor(ly) @@ -304,10 +304,10 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi = lx - i wy_hi = ly - j - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi - + Ex_p(n) = wx_lo*wy_lo*cEx(i, j ) + & wx_lo*wy_hi*cEx(i, j+1) + & wx_hi*wy_lo*cEx(i+1, j ) + & @@ -317,13 +317,13 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_lo*wy_hi*cEy(i, j+1) + & wx_hi*wy_lo*cEy(i+1, j ) + & wx_hi*wy_hi*cEy(i+1, j+1) - + ! otherwise use the fine else wx_hi = lx - i wy_hi = ly - j - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi @@ -331,12 +331,12 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_lo*wy_hi*Ex(i, j+1) + & wx_hi*wy_lo*Ex(i+1, j ) + & wx_hi*wy_hi*Ex(i+1, j+1) - + Ey_p(n) = wx_lo*wy_lo*Ey(i, j ) + & wx_lo*wy_hi*Ey(i, j+1) + & wx_hi*wy_lo*Ey(i+1, j ) + & wx_hi*wy_hi*Ey(i+1, j+1) - + end if end do @@ -344,13 +344,13 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & end subroutine interpolate_cic_two_levels ! -! This routine updates the particle positions and velocities using the +! This routine updates the particle positions and velocities using the ! leapfrog time integration algorithm, given the electric fields at the ! particle positions. It also enforces specular reflection off the domain ! walls. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! vx_p : the particle x-velocities @@ -379,7 +379,7 @@ subroutine push_leapfrog(particles, ns, np, & real(amrex_real), intent(in) :: mass real(amrex_real), intent(in) :: dt real(amrex_real), intent(in) :: prob_lo(2), prob_hi(2) - + integer n real(amrex_real) fac @@ -403,7 +403,7 @@ subroutine push_leapfrog(particles, ns, np, & vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) @@ -423,7 +423,7 @@ end subroutine push_leapfrog ! from the velocities after particle initialization. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! xx_p : the electric field in the x-direction at the particle positions @@ -460,7 +460,7 @@ subroutine push_leapfrog_positions(particles, ns, np, & vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) diff --git a/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_3d.f90 b/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_3d.f90 index cbdb1579486..bfd3d2efc5c 100644 --- a/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_3d.f90 +++ b/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_3d.f90 @@ -16,9 +16,9 @@ subroutine sum_fine_to_crse_nodal (lo, hi, lrat, crse, clo, chi, fine, flo, fhi) integer, intent(in) :: lrat(3) real(amrex_real), intent(inout) :: crse(clo(1):chi(1),clo(2):chi(2),clo(3):chi(3)) real(amrex_real), intent(in) :: fine(flo(1):fhi(1),flo(2):fhi(2),flo(3):fhi(3)) - + integer :: i, j, k, ii, jj, kk - + do k = lo(3), hi(3) kk = k * lrat(3) do j = lo(2), hi(2) @@ -27,7 +27,7 @@ subroutine sum_fine_to_crse_nodal (lo, hi, lrat, crse, clo, chi, fine, flo, fhi) ii = i * lrat(1) crse(i,j,k) = fine(ii,jj,kk) + & ! These six fine nodes are shared by two coarse nodes... - 0.5d0 * (fine(ii-1,jj,kk) + fine(ii+1,jj,kk) + & + 0.5d0 * (fine(ii-1,jj,kk) + fine(ii+1,jj,kk) + & fine(ii,jj-1,kk) + fine(ii,jj+1,kk) + & fine(ii,jj,kk-1) + fine(ii,jj,kk+1)) + & ! ... these twelve are shared by four... @@ -57,12 +57,12 @@ subroutine zero_out_bndry (lo, hi, input_data, bndry_data, mask) & double precision, intent(inout) :: input_data(lo(1):hi(1),lo(2):hi(2),lo(3):hi(3)) double precision, intent(inout) :: bndry_data(lo(1)-1:hi(1)+1,lo(2)-1:hi(2)+1,lo(3)-1:hi(3)+1) integer(c_int), intent(in ) :: mask (lo(1):hi(1),lo(2):hi(2),lo(3):hi(3)) - + integer :: i, j, k - + do k = lo(3), hi(3) do j = lo(2), hi(2) - do i = lo(1), hi(1) + do i = lo(1), hi(1) if (mask(i,j,k) .eq. 1) then bndry_data(i,j,k) = input_data(i,j,k) input_data(i,j,k) = 0.d0 @@ -94,7 +94,7 @@ subroutine build_mask (lo, hi, tmp_mask, mask, ncells) & end do end do end do - + if (total .gt. 0) then mask(i,j,k) = 1 else @@ -108,7 +108,7 @@ subroutine build_mask (lo, hi, tmp_mask, mask, ncells) & end subroutine build_mask ! This routine computes the node-centered electric field given a node-centered phi. -! The gradient is computed using 2nd-order centered differences. It assumes the +! The gradient is computed using 2nd-order centered differences. It assumes the ! Boundary conditions have already been set and that you have one row of ghost cells. ! Note that this routine includes the minus sign in E = - grad phi. ! @@ -134,7 +134,7 @@ subroutine compute_E_nodal (lo, hi, phi, Ex, Ey, Ez, dx) & do k = lo(3)-1, hi(3)+1 do j = lo(2)-1, hi(2)+1 do i = lo(1)-1, hi(1)+1 - + Ex(i,j,k) = fac(1) * (phi(i-1,j,k) - phi(i+1,j,k)) Ey(i,j,k) = fac(2) * (phi(i,j-1,k) - phi(i,j+1,k)) Ez(i,j,k) = fac(3) * (phi(i,j,k-1) - phi(i,j,k+1)) @@ -145,11 +145,11 @@ subroutine compute_E_nodal (lo, hi, phi, Ex, Ey, Ez, dx) & end subroutine compute_E_nodal -! This routine computes the charge density due to the particles using cloud-in-cell +! This routine computes the charge density due to the particles using cloud-in-cell ! deposition. The Fab rho is assumed to be node-centered. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! weights : the particle weights @@ -224,7 +224,7 @@ end subroutine deposit_cic ! node-centered. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! Ex_p : the electric field in the x-direction at the particle positions (output) @@ -303,7 +303,7 @@ subroutine interpolate_cic(particles, ns, np, & wx_hi*wy_lo*wz_hi*Ez(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ez(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ez(i+1, j+1, k+1) - + end do end subroutine interpolate_cic @@ -345,7 +345,7 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & lx = (particles(1, n) - plo(1))*inv_dx(1) ly = (particles(2, n) - plo(2))*inv_dx(2) lz = (particles(3, n) - plo(3))*inv_dx(3) - + i = floor(lx) j = floor(ly) k = floor(lz) @@ -364,11 +364,11 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi = lx - i wy_hi = ly - j wz_hi = lz - k - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi wz_lo = 1.0d0 - wz_hi - + Ex_p(n) = wx_lo*wy_lo*wz_lo*cEx(i, j, k ) + & wx_lo*wy_lo*wz_hi*cEx(i, j, k+1) + & wx_lo*wy_hi*wz_lo*cEx(i, j+1, k ) + & @@ -386,7 +386,7 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi*wy_lo*wz_hi*cEy(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*cEy(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*cEy(i+1, j+1, k+1) - + Ez_p(n) = wx_lo*wy_lo*wz_lo*cEz(i, j, k ) + & wx_lo*wy_lo*wz_hi*cEz(i, j, k+1) + & wx_lo*wy_hi*wz_lo*cEz(i, j+1, k ) + & @@ -395,14 +395,14 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi*wy_lo*wz_hi*cEz(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*cEz(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*cEz(i+1, j+1, k+1) - + ! otherwise use the fine else wx_hi = lx - i wy_hi = ly - j wz_hi = lz - k - + wx_lo = 1.0d0 - wx_hi wy_lo = 1.0d0 - wy_hi wz_lo = 1.0d0 - wz_hi @@ -415,7 +415,7 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi*wy_lo*wz_hi*Ex(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ex(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ex(i+1, j+1, k+1) - + Ey_p(n) = wx_lo*wy_lo*wz_lo*Ey(i, j, k ) + & wx_lo*wy_lo*wz_hi*Ey(i, j, k+1) + & wx_lo*wy_hi*wz_lo*Ey(i, j+1, k ) + & @@ -424,7 +424,7 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi*wy_lo*wz_hi*Ey(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ey(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ey(i+1, j+1, k+1) - + Ez_p(n) = wx_lo*wy_lo*wz_lo*Ez(i, j, k ) + & wx_lo*wy_lo*wz_hi*Ez(i, j, k+1) + & wx_lo*wy_hi*wz_lo*Ez(i, j+1, k ) + & @@ -433,7 +433,7 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & wx_hi*wy_lo*wz_hi*Ez(i+1, j, k+1) + & wx_hi*wy_hi*wz_lo*Ez(i+1, j+1, k ) + & wx_hi*wy_hi*wz_hi*Ez(i+1, j+1, k+1) - + end if end do @@ -441,13 +441,13 @@ subroutine interpolate_cic_two_levels(particles, ns, np, & end subroutine interpolate_cic_two_levels ! -! This routine updates the particle positions and velocities using the +! This routine updates the particle positions and velocities using the ! leapfrog time integration algorithm, given the electric fields at the ! particle positions. It also enforces specular reflection off the domain ! walls. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! vx_p : the particle x-velocities @@ -463,7 +463,7 @@ end subroutine interpolate_cic_two_levels ! prob_hi : the right-hand corner of the problem domain ! subroutine push_leapfrog(particles, ns, np, & - vx_p, vy_p, vz_p, & + vx_p, vy_p, vz_p, & Ex_p, Ey_p, Ez_p, & charge, mass, dt, & prob_lo, prob_hi) & @@ -476,7 +476,7 @@ subroutine push_leapfrog(particles, ns, np, & real(amrex_real), intent(in) :: mass real(amrex_real), intent(in) :: dt real(amrex_real), intent(in) :: prob_lo(3), prob_hi(3) - + integer n real(amrex_real) fac @@ -502,7 +502,7 @@ subroutine push_leapfrog(particles, ns, np, & vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) @@ -532,7 +532,7 @@ end subroutine push_leapfrog ! from the velocities after particle initialization. ! ! Arguments: -! particles : a pointer to the particle array-of-structs +! particles : a pointer to the particle array-of-structs ! ns : the stride length of particle struct (the size of the struct in number of reals) ! np : the number of particles ! xx_p : the electric field in the x-direction at the particle positions @@ -570,7 +570,7 @@ subroutine push_leapfrog_positions(particles, ns, np, & vx_p(n) = -vx_p(n) end do -! ... y... +! ... y... do while (particles(2, n) .lt. prob_lo(2) .or. particles(2, n) .gt. prob_hi(2)) if (particles(2, n) .lt. prob_lo(2)) then particles(2, n) = 2.d0*prob_lo(2) - particles(2, n) @@ -589,7 +589,7 @@ subroutine push_leapfrog_positions(particles, ns, np, & end if vz_p(n) = -vz_p(n) end do - + end do end subroutine push_leapfrog_positions diff --git a/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_F.H b/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_F.H index a0e7bf46bf5..3645545e116 100644 --- a/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_F.H +++ b/Tutorials/Particles/ElectrostaticPIC/electrostatic_pic_F.H @@ -13,7 +13,7 @@ extern "C" const amrex::Real* fine, const int* flo, const int* fhi); void zero_out_bndry(const int* lo, const int* hi, - amrex::Real* input_data, amrex::Real* bndry_data, + amrex::Real* input_data, amrex::Real* bndry_data, const int* mask); void build_mask(const int* lo, const int* hi, @@ -21,31 +21,31 @@ extern "C" void compute_E_nodal (const int* lo, const int* hi, const amrex::Real* phi, - amrex::Real* Ex, amrex::Real* Ey, -#if BL_SPACEDIM == 3 - amrex::Real* Ez, + amrex::Real* Ex, amrex::Real* Ey, +#if BL_SPACEDIM == 3 + amrex::Real* Ez, #endif const amrex::Real* dx); - + void deposit_cic(const amrex::Real* particles, int ns, int np, - const amrex::Real* weights, + const amrex::Real* weights, const amrex::Real* charge, amrex::Real* rho, const int* lo, const int* hi, const amrex::Real* plo, const amrex::Real* dx, const int* ng); - + void interpolate_cic_two_levels(const amrex::Real* particles, int ns, int np, - amrex::Real* Ex_p, amrex::Real* Ey_p, -#if BL_SPACEDIM == 3 + amrex::Real* Ex_p, amrex::Real* Ey_p, +#if BL_SPACEDIM == 3 amrex::Real* Ez_p, #endif - const amrex::Real* Ex, const amrex::Real* Ey, -#if BL_SPACEDIM == 3 + const amrex::Real* Ex, const amrex::Real* Ey, +#if BL_SPACEDIM == 3 const amrex::Real* Ez, #endif const int* lo, const int* hi, const amrex::Real* dx, - const amrex::Real* cEx, const amrex::Real* cEy, -#if BL_SPACEDIM == 3 + const amrex::Real* cEx, const amrex::Real* cEy, +#if BL_SPACEDIM == 3 const amrex::Real* cEz, #endif const int* mask, @@ -53,33 +53,33 @@ extern "C" const amrex::Real* plo, const int* ng, const int* lev); void interpolate_cic(const amrex::Real* particles, int ns, int np, - amrex::Real* Ex_p, amrex::Real* Ey_p, -#if BL_SPACEDIM == 3 + amrex::Real* Ex_p, amrex::Real* Ey_p, +#if BL_SPACEDIM == 3 amrex::Real* Ez_p, #endif - const amrex::Real* Ex, const amrex::Real* Ey, -#if BL_SPACEDIM == 3 + const amrex::Real* Ex, const amrex::Real* Ey, +#if BL_SPACEDIM == 3 const amrex::Real* Ez, #endif const int* lo, const int* hi, const amrex::Real* plo, const amrex::Real* dx, const int* ng); - void push_leapfrog(amrex::Real* particles, int ns, int np, - amrex::Real* vx_p, amrex::Real* vy_p, -#if BL_SPACEDIM == 3 + void push_leapfrog(amrex::Real* particles, int ns, int np, + amrex::Real* vx_p, amrex::Real* vy_p, +#if BL_SPACEDIM == 3 amrex::Real* vz_p, #endif - const amrex::Real* Ex_p, const amrex::Real* Ey_p, -#if BL_SPACEDIM == 3 + const amrex::Real* Ex_p, const amrex::Real* Ey_p, +#if BL_SPACEDIM == 3 const amrex::Real* Ez_p, #endif const amrex::Real* charge, const amrex::Real* mass, const amrex::Real* dt, const amrex::Real* prob_lo, const amrex::Real* prob_hi); void push_leapfrog_positions(amrex::Real* particles, int ns, int np, - amrex::Real* vx_p, amrex::Real* vy_p, -#if BL_SPACEDIM == 3 + amrex::Real* vx_p, amrex::Real* vy_p, +#if BL_SPACEDIM == 3 amrex::Real* vz_p, #endif const amrex::Real* dt, const amrex::Real* prob_lo, diff --git a/Tutorials/Particles/ElectrostaticPIC/main.cpp b/Tutorials/Particles/ElectrostaticPIC/main.cpp index 605f3d37a0e..a91b7fdb606 100644 --- a/Tutorials/Particles/ElectrostaticPIC/main.cpp +++ b/Tutorials/Particles/ElectrostaticPIC/main.cpp @@ -31,7 +31,7 @@ void WritePlotFile(const ScalarMeshData& rhs, Vector > output_cc(num_levels); for (int lev = 0; lev < num_levels; ++lev) { const BoxArray& nodal_ba = rhs[lev]->boxArray(); - output_cc[lev].reset(new MultiFab(amrex::convert(nodal_ba, cc_flag), + output_cc[lev].reset(new MultiFab(amrex::convert(nodal_ba, cc_flag), rhs[lev]->DistributionMap(), num_output_comp, 0)); amrex::average_node_to_cellcenter(*output_cc[lev], 0, *rhs[lev], 0, 1); amrex::average_node_to_cellcenter(*output_cc[lev], 1, *phi[lev], 0, 1); @@ -39,7 +39,7 @@ void WritePlotFile(const ScalarMeshData& rhs, amrex::average_node_to_cellcenter(*output_cc[lev], 2+i, *E[lev][i], 0, 1); } } - + Vector varnames; varnames.push_back("rhs"); varnames.push_back("phi"); @@ -61,19 +61,19 @@ void WritePlotFile(const ScalarMeshData& rhs, #if BL_SPACEDIM == 3 particle_varnames.push_back("Ez"); #endif - + Vector level_steps; level_steps.push_back(0); level_steps.push_back(0); - + int output_levs = num_levels; - + Vector outputRR(output_levs); for (int lev = 0; lev < output_levs; ++lev) { outputRR[lev] = IntVect(D_DECL(2, 2, 2)); } - const std::string& pltfile = amrex::Concatenate("plt", nstep, 5); + const std::string& pltfile = amrex::Concatenate("plt", nstep, 5); WriteMultiLevelPlotfile(pltfile, output_levs, GetVecOfConstPtrs(output_cc), varnames, geom, 0.0, level_steps, outputRR); @@ -82,7 +82,7 @@ void WritePlotFile(const ScalarMeshData& rhs, } void computeE( VectorMeshData& E, - const ScalarMeshData& phi, + const ScalarMeshData& phi, const Vector& geom) { const int num_levels = E.size(); @@ -108,12 +108,12 @@ void computeE( VectorMeshData& E, E[lev][0]->FillBoundary(gm.periodicity()); E[lev][1]->FillBoundary(gm.periodicity()); #if BL_SPACEDIM == 3 - E[lev][2]->FillBoundary(gm.periodicity()); + E[lev][2]->FillBoundary(gm.periodicity()); #endif // VisMF::Write(*E[lev][0], amrex::MultiFabFileFullPrefix(lev, "tmp", "Level_", "Ex")); } } - + void zeroOutBoundary(MultiFab& input_data, MultiFab& bndry_data, const FabArray >& mask) { @@ -128,17 +128,17 @@ void zeroOutBoundary(MultiFab& input_data, bndry_data.FillBoundary(); } -void sumFineToCrseNodal(const MultiFab& fine, MultiFab& crse, +void sumFineToCrseNodal(const MultiFab& fine, MultiFab& crse, const Geometry& cgeom, const IntVect& ratio) { - + const BoxArray& fine_BA = fine.boxArray(); const DistributionMapping& fine_dm = fine.DistributionMap(); BoxArray coarsened_fine_BA = fine_BA; coarsened_fine_BA.coarsen(ratio); - + MultiFab coarsened_fine_data(coarsened_fine_BA, fine_dm, 1, 0); coarsened_fine_data.setVal(0.0); - + for (MFIter mfi(coarsened_fine_data); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); const Box& crse_box = coarsened_fine_data[mfi].box(); @@ -147,8 +147,8 @@ void sumFineToCrseNodal(const MultiFab& fine, MultiFab& crse, coarsened_fine_data[mfi].dataPtr(), crse_box.loVect(), crse_box.hiVect(), fine[mfi].dataPtr(), fine_box.loVect(), fine_box.hiVect()); } - - crse.copy(coarsened_fine_data, cgeom.periodicity(), FabArrayBase::ADD); + + crse.copy(coarsened_fine_data, cgeom.periodicity(), FabArrayBase::ADD); } void fixRHSForSolve(Vector >& rhs, @@ -157,7 +157,7 @@ void fixRHSForSolve(Vector >& rhs, int num_levels = rhs.size(); for (int lev = 0; lev < num_levels; ++lev) { MultiFab& fine_rhs = *rhs[lev]; - const FabArray >& mask = *masks[lev]; + const FabArray >& mask = *masks[lev]; const BoxArray& fine_ba = fine_rhs.boxArray(); const DistributionMapping& fine_dm = fine_rhs.DistributionMap(); MultiFab fine_bndry_data(fine_ba, fine_dm, 1, 1); @@ -173,12 +173,12 @@ void computePhi(ScalarMeshData& rhs, ScalarMeshData& phi, int num_levels = rhs.size(); - Vector > tmp_rhs(num_levels); + Vector > tmp_rhs(num_levels); for (int lev = 0; lev < num_levels; ++lev) { tmp_rhs[lev].reset(new MultiFab(rhs[lev]->boxArray(), dm[lev], 1, 0)); MultiFab::Copy(*tmp_rhs[lev], *rhs[lev], 0, 0, 1, 0); } - + IntVect ratio(D_DECL(2, 2, 2)); fixRHSForSolve(tmp_rhs, masks, geom, ratio); @@ -191,7 +191,7 @@ void computePhi(ScalarMeshData& rhs, ScalarMeshData& phi, Vector level_dm(1); Vector level_phi(1); Vector level_rhs(1); - + for (int lev = 0; lev < num_levels; ++lev) { level_phi[0] = phi[lev].get(); level_rhs[0] = tmp_rhs[lev].get(); @@ -213,7 +213,7 @@ void computePhi(ScalarMeshData& rhs, ScalarMeshData& phi, MultiFab sigma(level_grids[0], level_dm[0], 1, 0); sigma.setVal(1.0); linop.setSigma(0, sigma); - + MLMG mlmg(linop); mlmg.setMaxIter(100); mlmg.setMaxFmgIter(0); @@ -244,11 +244,11 @@ void computePhi(ScalarMeshData& rhs, ScalarMeshData& phi, // VisMF::Write(*phi[lev], amrex::MultiFabFileFullPrefix(lev, "tmp", "Level_", "phi")); // VisMF::Write(*rhs[lev], amrex::MultiFabFileFullPrefix(lev, "tmp", "Level_", "rhs")); } - + for (int lev = 0; lev < num_levels; ++lev) { const Geometry& gm = geom[lev]; phi[lev]->FillBoundary(gm.periodicity()); - } + } } void @@ -264,11 +264,11 @@ getLevelMasks(Vector > > >& masks, int notcovered = 1; int physbnd = 1; int interior = 0; - + for (int lev = 0; lev < num_levels; ++lev) { BoxArray nba = grids[lev]; nba.surroundingNodes(); - + FabArray > tmp_mask(nba, dmap[lev], 1, ncells); tmp_mask.BuildMask(geom[lev].Domain(), geom[lev].periodicity(), covered, notcovered, physbnd, interior); @@ -291,16 +291,16 @@ void main_main () { pp.get("max_level", max_level); pp.get("n_cell", n_cell); pp.get("n_buffer", n_buffer); - pp.get("max_grid_size", max_grid_size); + pp.get("max_grid_size", max_grid_size); pp.get("max_step", max_step); pp.get("particle_output_int", particle_output_int); pp.get("dt", dt); } - + assert(max_level < 2); int num_levels = max_level + 1; - + Vector rr(num_levels-1); for (int lev = 1; lev < num_levels; lev++) rr[lev-1] = 2; @@ -310,7 +310,7 @@ void main_main () { real_box.setLo(n,-20.0e-6); real_box.setHi(n, 20.0e-6); } - + // This sets the boundary conditions to be doubly or triply periodic for (int i = 0; i < BL_SPACEDIM; i++) { is_periodic[i] = 0; @@ -319,34 +319,34 @@ void main_main () { IntVect dom_lo(IntVect(D_DECL(0,0,0))); IntVect dom_hi(IntVect(D_DECL(n_cell-1, n_cell-1, n_cell-1))); Box domain(dom_lo, dom_hi); - + // make Geometry for each level Vector geom(num_levels); geom[0].define(domain,&real_box,CoordSys::cartesian,is_periodic); for (int lev = 1; lev < num_levels; ++lev) { - geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), - &real_box, CoordSys::cartesian, is_periodic); + geom[lev].define(amrex::refine(geom[lev-1].Domain(), rr[lev-1]), + &real_box, CoordSys::cartesian, is_periodic); } - + // make grids for each level Vector grids(num_levels); grids[0].define(domain); if (num_levels > 1) { int n_fine = n_cell*rr[0]; - IntVect refined_lo(D_DECL(3*n_fine/8,3*n_fine/8,3*n_fine/8)); + IntVect refined_lo(D_DECL(3*n_fine/8,3*n_fine/8,3*n_fine/8)); IntVect refined_hi(D_DECL(5*n_fine/8-1,5*n_fine/8-1,5*n_fine/8-1)); // Build a box for the level 1 domain Box refined_patch(refined_lo, refined_hi); grids[1].define(refined_patch); } - + for (int lev = 0; lev < num_levels; lev++) { grids[lev].maxSize(max_grid_size); } - + int Ncomp = 1; - + Vector dm(num_levels); Vector > phi(num_levels); Vector > rhs(num_levels); @@ -359,22 +359,22 @@ void main_main () { rhs[lev].reset(new MultiFab(nba, dm[lev], Ncomp, 1)); phi[lev].reset(new MultiFab(nba, dm[lev], Ncomp, 2)); - + for (int idim = 0; idim < BL_SPACEDIM; ++idim) { eField[lev][idim].reset(new MultiFab(nba, dm[lev], Ncomp, 1)); eField[lev][idim]->setVal(0.0); } rhs[lev]->setVal(0.0); - phi[lev]->setVal(0.0); + phi[lev]->setVal(0.0); } - Vector > > > masks(num_levels); + Vector > > > masks(num_levels); getLevelMasks(masks, grids, dm, geom); Vector > > > gather_masks(num_levels); getLevelMasks(gather_masks, grids, dm, geom, n_buffer + 1); // convert from num nodes to num cells - + ElectrostaticParticleContainer myPC(geom, dm, grids, rr); myPC.InitParticles(); @@ -384,7 +384,7 @@ void main_main () { myPC.DepositCharge(rhs); computePhi(rhs, phi, grids, dm, geom, masks); - + computeE(eField, phi, geom); myPC.FieldGather(eField, gather_masks); @@ -392,10 +392,10 @@ void main_main () { if (step % particle_output_int == 0) myPC.writeParticles(step); // WritePlotFile(rhs, phi, eField, myPC, geom, step); - + myPC.Evolve(eField, rhs, dt); - myPC.Redistribute(); + myPC.Redistribute(); } } diff --git a/Tutorials/Particles/NeighborList/CheckPair.H b/Tutorials/Particles/NeighborList/CheckPair.H index 5edc5cb4b9c..8018e9c49e9 100644 --- a/Tutorials/Particles/NeighborList/CheckPair.H +++ b/Tutorials/Particles/NeighborList/CheckPair.H @@ -11,8 +11,8 @@ struct CheckPair { amrex::Real d0 = (p1.pos(0) - p2.pos(0)); amrex::Real d1 = (p1.pos(1) - p2.pos(1)); - amrex::Real d2 = (p1.pos(2) - p2.pos(2)); - amrex::Real dsquared = d0*d0 + d1*d1 + d2*d2; + amrex::Real d2 = (p1.pos(2) - p2.pos(2)); + amrex::Real dsquared = d0*d0 + d1*d1 + d2*d2; return (dsquared <= 25.0*Params::cutoff*Params::cutoff); } }; diff --git a/Tutorials/Particles/NeighborList/Constants.H b/Tutorials/Particles/NeighborList/Constants.H index 2e26b1939b9..18c94dbe255 100644 --- a/Tutorials/Particles/NeighborList/Constants.H +++ b/Tutorials/Particles/NeighborList/Constants.H @@ -5,7 +5,7 @@ namespace Params { - // This is designed to represent MFiX-like conditions where the grid spacing is + // This is designed to represent MFiX-like conditions where the grid spacing is // roughly 2.5 times the particle diameter. In main.cpp we set grid spacing to 1 // so here we set cutoff to diameter = 1/2.5 --> cutoff = 0.2 static constexpr amrex::Real cutoff = 0.2 ; diff --git a/Tutorials/Particles/NeighborList/MDParticleContainer.H b/Tutorials/Particles/NeighborList/MDParticleContainer.H index 40dc10ad579..dfef2cbe4c8 100644 --- a/Tutorials/Particles/NeighborList/MDParticleContainer.H +++ b/Tutorials/Particles/NeighborList/MDParticleContainer.H @@ -36,7 +36,7 @@ public: void moveParticles (const amrex::Real& dt); - void writeParticles (const int n); + void writeParticles (const int n); amrex::Real computeStepSize(amrex::Real& cfl); }; diff --git a/Tutorials/Particles/NeighborList/MDParticleContainer.cpp b/Tutorials/Particles/NeighborList/MDParticleContainer.cpp index 3d0cea65513..0a0f79ac0dc 100644 --- a/Tutorials/Particles/NeighborList/MDParticleContainer.cpp +++ b/Tutorials/Particles/NeighborList/MDParticleContainer.cpp @@ -6,31 +6,31 @@ using namespace amrex; namespace -{ +{ void get_position_unit_cell(Real* r, const IntVect& nppc, int i_part) { int nx = nppc[0]; int ny = nppc[1]; int nz = nppc[2]; - + int ix_part = i_part/(ny * nz); int iy_part = (i_part % (ny * nz)) % ny; int iz_part = (i_part % (ny * nz)) / ny; - + r[0] = (0.5+ix_part)/nx; r[1] = (0.5+iy_part)/ny; r[2] = (0.5+iz_part)/nz; } - + void get_gaussian_random_momentum(Real* u, Real u_mean, Real u_std) { Real ux_th = amrex::RandomNormal(0.0, u_std); Real uy_th = amrex::RandomNormal(0.0, u_std); Real uz_th = amrex::RandomNormal(0.0, u_std); - + u[0] = u_mean + ux_th; u[1] = u_mean + uy_th; u[2] = u_mean + uz_th; - } + } } void @@ -41,10 +41,10 @@ InitParticles(const IntVect& a_num_particles_per_cell, { BL_PROFILE("MDParticleContainer::InitParticles"); - const int lev = 0; + const int lev = 0; const Real* dx = Geom(lev).CellSize(); const Real* plo = Geom(lev).ProbLo(); - + const int num_ppc = AMREX_D_TERM( a_num_particles_per_cell[0], *a_num_particles_per_cell[1], *a_num_particles_per_cell[2]); @@ -54,28 +54,28 @@ InitParticles(const IntVect& a_num_particles_per_cell, const Box& tile_box = mfi.tilebox(); Gpu::HostVector host_particles; - + for (IntVect iv = tile_box.smallEnd(); iv <= tile_box.bigEnd(); tile_box.next(iv)) { for (int i_part=0; i_part Params::cutoff*Params::cutoff) return; + if (r2 > Params::cutoff*Params::cutoff) return; Real r = sqrt(r2); - + Real coef = (1.0 - Params::cutoff / r) / r2; p1.rdata(PIdx::ax) += coef * dx; p1.rdata(PIdx::ay) += coef * dy; @@ -175,30 +175,30 @@ Real MDParticleContainer::minDistance() auto nbor_data = m_neighbor_list[lev][index].data(); ParticleType* pstruct = aos().dataPtr(); - Gpu::DeviceScalar min_d_gpu(min_d); - Real* pmin_d = min_d_gpu.dataPtr(); + Gpu::DeviceScalar min_d_gpu(min_d); + Real* pmin_d = min_d_gpu.dataPtr(); AMREX_FOR_1D ( np, i, { ParticleType& p1 = pstruct[i]; for (const auto& p2 : nbor_data.getNeighbors(i)) - { + { Real dx = p1.pos(0) - p2.pos(0); Real dy = p1.pos(1) - p2.pos(1); Real dz = p1.pos(2) - p2.pos(2); - + Real r2 = dx*dx + dy*dy + dz*dz; r2 = amrex::max(r2, Params::min_r*Params::min_r); Real r = sqrt(r2); - - Gpu::Atomic::Min(pmin_d, r); + + Gpu::Atomic::Min(pmin_d, r); } }); - Gpu::Device::streamSynchronize(); + Gpu::Device::streamSynchronize(); - min_d = std::min(min_d, min_d_gpu.dataValue()); + min_d = std::min(min_d, min_d_gpu.dataValue()); } ParallelDescriptor::ReduceRealMin(min_d, ParallelDescriptor::IOProcessorNumber()); @@ -220,17 +220,17 @@ void MDParticleContainer::moveParticles(const amrex::Real& dt) { int gid = mfi.index(); int tid = mfi.LocalTileIndex(); - + auto& ptile = plev[std::make_pair(gid, tid)]; auto& aos = ptile.GetArrayOfStructs(); ParticleType* pstruct = &(aos[0]); const size_t np = aos.numParticles(); - + // now we move the particles AMREX_FOR_1D ( np, i, { - ParticleType& p = pstruct[i]; + ParticleType& p = pstruct[i]; p.rdata(PIdx::vx) += p.rdata(PIdx::ax) * dt; p.rdata(PIdx::vy) += p.rdata(PIdx::ay) * dt; p.rdata(PIdx::vz) += p.rdata(PIdx::az) * dt; @@ -275,9 +275,9 @@ Real MDParticleContainer::computeStepSize(amrex::Real& cfl) }); ParallelDescriptor::ReduceRealMax(maxVel); - + // This would compute dt based on the grid spacing dx - // const int lev = 0; + // const int lev = 0; // const Real* dx = Geom(lev).CellSize(); // return cfl*dx[0]/maxVel; diff --git a/Tutorials/Particles/NeighborList/README.md b/Tutorials/Particles/NeighborList/README.md index 3c8faaf279f..13bc9fe5be0 100644 --- a/Tutorials/Particles/NeighborList/README.md +++ b/Tutorials/Particles/NeighborList/README.md @@ -1,9 +1,9 @@ -This tutorial initializes particles, and steps through "nsteps" time steps where in each time step -we +This tutorial initializes particles, and steps through "nsteps" time steps where in each time step +we * compute the timestep = cfl * "cutoff" (particle radius) / max_particle_vel - * compute or update the grid neighbors + * compute or update the grid neighbors * calculate particle neighbor lists * compute forces due to particle-particle collisions - * update the particle velocities then particle positions. + * update the particle velocities then particle positions. At every time step we print out dt and the number of particles. diff --git a/Tutorials/Particles/NeighborList/main.cpp b/Tutorials/Particles/NeighborList/main.cpp index b63b623b296..b31b80a4667 100644 --- a/Tutorials/Particles/NeighborList/main.cpp +++ b/Tutorials/Particles/NeighborList/main.cpp @@ -71,7 +71,7 @@ void main_main () for (int i = 0; i < BL_SPACEDIM; i++) is_per[i] = 0; Geometry geom(domain, &real_box, coord, is_per); - + BoxArray ba(domain); ba.maxSize(params.max_grid_size); DistributionMapping dm(ba); @@ -84,41 +84,41 @@ void main_main () pc.InitParticles(nppc, 1.0, 0.0); - if (params.print_num_particles) + if (params.print_num_particles) amrex::Print() << "Num particles after init is " << pc.TotalNumberOfParticles() << "\n"; int num_rebuild = params.num_rebuild; Real cfl = params.cfl; - + Real min_d = std::numeric_limits::max(); for (int step = 0; step < params.nsteps; ++step) { - Real dt = pc.computeStepSize(cfl); + Real dt = pc.computeStepSize(cfl); - if (step % num_rebuild == 0) - { - if (step > 0) pc.RedistributeLocal(); + if (step % num_rebuild == 0) + { + if (step > 0) pc.RedistributeLocal(); - pc.fillNeighbors(); + pc.fillNeighbors(); - pc.buildNeighborList(CheckPair()); - } - else - { - pc.updateNeighbors(); - } + pc.buildNeighborList(CheckPair()); + } + else + { + pc.updateNeighbors(); + } - if (params.print_min_dist) - min_d = std::min(min_d, pc.minDistance()); + if (params.print_min_dist) + min_d = std::min(min_d, pc.minDistance()); - if (params.print_neighbor_list) + if (params.print_neighbor_list) pc.printNeighborList(); - pc.computeForces(); + pc.computeForces(); - pc.moveParticles(dt); + pc.moveParticles(dt); } pc.RedistributeLocal(); diff --git a/Tutorials/SDC/MISDC_ADR_2d/Source/SDC_sweeper.cpp b/Tutorials/SDC/MISDC_ADR_2d/Source/SDC_sweeper.cpp index 699df6320ec..d1977558657 100644 --- a/Tutorials/SDC/MISDC_ADR_2d/Source/SDC_sweeper.cpp +++ b/Tutorials/SDC/MISDC_ADR_2d/Source/SDC_sweeper.cpp @@ -5,21 +5,21 @@ #include void SDC_advance(MultiFab& phi_old, - MultiFab& phi_new, - std::array& flux, - Real dt, - const Geometry& geom, - const Vector& bc, - MLMG& mlmg, - MLABecLaplacian& mlabec, - SDCstruct &SDC, Real a, Real d, Real r) + MultiFab& phi_new, + std::array& flux, + Real dt, + const Geometry& geom, + const Vector& bc, + MLMG& mlmg, + MLABecLaplacian& mlabec, + SDCstruct &SDC, Real a, Real d, Real r) { - /* This is a multi-implicit SDC example time step for an + /* This is a multi-implicit SDC example time step for an advection-diffusion-reaction equation of the form phi_t = A(phi)+D(phi)+R(phi) - + The advection is treated explicilty and the diffusion and reaction implicitly and uncoupled @@ -36,10 +36,10 @@ void SDC_advance(MultiFab& phi_old, // Fill the ghost cells of each grid from the other grids // includes periodic domain boundaries SDC.sol[0].FillBoundary(geom.periodicity()); - + // Fill non-periodic physical boundaries FillDomainBoundary(SDC.sol[0], geom, bc); - + // Compute the first function value int sdc_m=0; SDC_feval(flux,geom,bc,SDC,a,d,r,sdc_m,-1); @@ -50,7 +50,7 @@ void SDC_advance(MultiFab& phi_old, MultiFab::Copy(SDC.f[0][sdc_n],SDC.f[0][0], 0, 0, 1, 0); MultiFab::Copy(SDC.f[1][sdc_n],SDC.f[1][0], 0, 0, 1, 0); if (SDC.Npieces==3) - MultiFab::Copy(SDC.f[2][sdc_n],SDC.f[2][0], 0, 0, 1, 0); + MultiFab::Copy(SDC.f[2][sdc_n],SDC.f[2][0], 0, 0, 1, 0); } @@ -61,56 +61,56 @@ void SDC_advance(MultiFab& phi_old, // Compute RHS integrals SDC.SDC_rhs_integrals(dt); - + // Substep over SDC nodes for (int sdc_m = 0; sdc_m < SDC.Nnodes-1; sdc_m++) - { - // use phi_new as rhs and fill it with terms at this iteration - SDC.SDC_rhs_k_plus_one(phi_new,dt,sdc_m); - - // get the best initial guess for implicit solve - MultiFab::Copy(SDC.sol[sdc_m+1],phi_new, 0, 0, 1, 0); - for ( MFIter mfi(SDC.sol[sdc_m+1]); mfi.isValid(); ++mfi ) - { - // const Box& bx = mfi.validbox(); - qij = dt*SDC.Qimp[sdc_m][sdc_m+1]; - SDC.sol[sdc_m+1][mfi].saxpy(qij,SDC.f[1][sdc_m+1][mfi]); - } - - // Solve for the first implicit part - SDC_fcomp(phi_new, geom, bc, SDC, mlmg, mlabec,dt,d,r,sdc_m+1,1); - - if (SDC.Npieces==3) - { - // Build rhs for 2nd solve - MultiFab::Copy(phi_new, SDC.sol[sdc_m+1],0, 0, 1, 0); - - // Add in the part for the 2nd implicit term to rhs - SDC.SDC_rhs_misdc(phi_new,dt,sdc_m); - - // Solve for the second implicit part - SDC_fcomp(phi_new, geom, bc, SDC, mlmg, mlabec,dt,d,r,sdc_m+1,2); - } - // Compute the function values at node sdc_m+1 - SDC_feval(flux,geom,bc,SDC,a,d,r,sdc_m+1,-1); - - } // end SDC substep loop + { + // use phi_new as rhs and fill it with terms at this iteration + SDC.SDC_rhs_k_plus_one(phi_new,dt,sdc_m); + + // get the best initial guess for implicit solve + MultiFab::Copy(SDC.sol[sdc_m+1],phi_new, 0, 0, 1, 0); + for ( MFIter mfi(SDC.sol[sdc_m+1]); mfi.isValid(); ++mfi ) + { + // const Box& bx = mfi.validbox(); + qij = dt*SDC.Qimp[sdc_m][sdc_m+1]; + SDC.sol[sdc_m+1][mfi].saxpy(qij,SDC.f[1][sdc_m+1][mfi]); + } + + // Solve for the first implicit part + SDC_fcomp(phi_new, geom, bc, SDC, mlmg, mlabec,dt,d,r,sdc_m+1,1); + + if (SDC.Npieces==3) + { + // Build rhs for 2nd solve + MultiFab::Copy(phi_new, SDC.sol[sdc_m+1],0, 0, 1, 0); + + // Add in the part for the 2nd implicit term to rhs + SDC.SDC_rhs_misdc(phi_new,dt,sdc_m); + + // Solve for the second implicit part + SDC_fcomp(phi_new, geom, bc, SDC, mlmg, mlabec,dt,d,r,sdc_m+1,2); + } + // Compute the function values at node sdc_m+1 + SDC_feval(flux,geom,bc,SDC,a,d,r,sdc_m+1,-1); + + } // end SDC substep loop } // end sweeps loop - + // Return the last node in phi_new MultiFab::Copy(phi_new, SDC.sol[SDC.Nnodes-1], 0, 0, 1, 0); } void SDC_feval(std::array& flux, - const Geometry& geom, - const Vector& bc, - SDCstruct &SDC, - Real a,Real d,Real r, - int sdc_m,int npiece) + const Geometry& geom, + const Vector& bc, + SDCstruct &SDC, + Real a,Real d,Real r, + int sdc_m,int npiece) { /* Evaluate explicitly the rhs terms of the equation at the SDC node "sdc_m". - The input parameter "npiece" describes which term to do. + The input parameter "npiece" describes which term to do. If npiece = -1, do all the pieces */ const BoxArray &ba=SDC.sol[0].boxArray(); const DistributionMapping &dm=SDC.sol[0].DistributionMap(); @@ -133,30 +133,30 @@ void SDC_feval(std::array& flux, for ( MFIter mfi(SDC.sol[sdc_m]); mfi.isValid(); ++mfi ) { const Box& bx = mfi.validbox(); - for (int n = nlo; n < nhi; n++) - { - SDC_feval_F(BL_TO_FORTRAN_BOX(bx), - BL_TO_FORTRAN_BOX(domain_bx), - BL_TO_FORTRAN_ANYD(SDC.sol[sdc_m][mfi]), - BL_TO_FORTRAN_ANYD(flux[0][mfi]), - BL_TO_FORTRAN_ANYD(flux[1][mfi]), -#if (AMREX_SPACEDIM == 3) - BL_TO_FORTRAN_ANYD(flux[2][mfi]), -#endif - BL_TO_FORTRAN_ANYD(SDC.f[n][sdc_m][mfi]), - dx,&a,&d,&r,&n); - } - + for (int n = nlo; n < nhi; n++) + { + SDC_feval_F(BL_TO_FORTRAN_BOX(bx), + BL_TO_FORTRAN_BOX(domain_bx), + BL_TO_FORTRAN_ANYD(SDC.sol[sdc_m][mfi]), + BL_TO_FORTRAN_ANYD(flux[0][mfi]), + BL_TO_FORTRAN_ANYD(flux[1][mfi]), +#if (AMREX_SPACEDIM == 3) + BL_TO_FORTRAN_ANYD(flux[2][mfi]), +#endif + BL_TO_FORTRAN_ANYD(SDC.f[n][sdc_m][mfi]), + dx,&a,&d,&r,&n); + } + } } void SDC_fcomp(MultiFab& rhs, - const Geometry& geom, - const Vector& bc, - SDCstruct &SDC, - MLMG &mlmg, - MLABecLaplacian& mlabec, - Real dt,Real d,Real r, - int sdc_m,int npiece) + const Geometry& geom, + const Vector& bc, + SDCstruct &SDC, + MLMG &mlmg, + MLABecLaplacian& mlabec, + Real dt,Real d,Real r, + int sdc_m,int npiece) { /* Solve implicitly for the implicit terms of the equation at the SDC node "sdc_m". The input parameter "npiece" describes which term to do. */ @@ -172,28 +172,28 @@ void SDC_fcomp(MultiFab& rhs, const Real tol_rel = 1.e-10; const Real tol_abs = 0.0; - if (npiece == 1) + if (npiece == 1) { // Do diffusion solve - + // Fill the ghost cells of each grid from the other grids // includes periodic domain boundaries rhs.FillBoundary(geom.periodicity()); - + // Fill non-periodic physical boundaries FillDomainBoundary(rhs, geom, bc); // Set diffusion scalar in solve - qij = d*dt*SDC.Qimp[sdc_m-1][sdc_m]; + qij = d*dt*SDC.Qimp[sdc_m-1][sdc_m]; Real ascalar = 1.0; mlabec.setScalars(ascalar, qij); // set the boundary conditions mlabec.setLevelBC(0, &rhs); - mlabec.setLevelBC(0, &SDC.sol[sdc_m]); + mlabec.setLevelBC(0, &SDC.sol[sdc_m]); // Do the multigrid solve mlmg.solve({&SDC.sol[sdc_m]}, {&rhs}, tol_rel, tol_abs); - + } else { // Do reaction solve y - qij*y*(1-y)*(y-1/2) = rhs @@ -201,18 +201,18 @@ void SDC_fcomp(MultiFab& rhs, // make a flag to change how the reaction is done int nflag=1; // Lazy approximation - qij = r*dt*SDC.Qimp[sdc_m-1][sdc_m]; + qij = r*dt*SDC.Qimp[sdc_m-1][sdc_m]; for ( MFIter mfi(SDC.sol[sdc_m]); mfi.isValid(); ++mfi ) - { - const Box& bx = mfi.validbox(); - SDC_fcomp_reaction_F(BL_TO_FORTRAN_BOX(bx), - BL_TO_FORTRAN_BOX(domain_bx), - BL_TO_FORTRAN_ANYD(SDC.sol[sdc_m][mfi]), - BL_TO_FORTRAN_ANYD(rhs[mfi]), - BL_TO_FORTRAN_ANYD(SDC.f[2][sdc_m][mfi]), - &qij,&nflag); + { + const Box& bx = mfi.validbox(); + SDC_fcomp_reaction_F(BL_TO_FORTRAN_BOX(bx), + BL_TO_FORTRAN_BOX(domain_bx), + BL_TO_FORTRAN_ANYD(SDC.sol[sdc_m][mfi]), + BL_TO_FORTRAN_ANYD(rhs[mfi]), + BL_TO_FORTRAN_ANYD(SDC.f[2][sdc_m][mfi]), + &qij,&nflag); } - + } } diff --git a/Tutorials/SDC/MISDC_ADR_2d/Source/functions_2d.f90 b/Tutorials/SDC/MISDC_ADR_2d/Source/functions_2d.f90 index c22e0534d46..7b186ab0e94 100644 --- a/Tutorials/SDC/MISDC_ADR_2d/Source/functions_2d.f90 +++ b/Tutorials/SDC/MISDC_ADR_2d/Source/functions_2d.f90 @@ -17,7 +17,7 @@ subroutine SDC_feval_F(lo, hi, domlo, domhi, phi, philo, phihi, & real(amrex_real), intent(in) :: a,d,r integer, intent(in) :: n - + ! local variables integer i,j @@ -26,11 +26,11 @@ subroutine SDC_feval_F(lo, hi, domlo, domhi, phi, philo, phihi, & ! x-fluxes do j = lo(2), hi(2) do i = lo(1), hi(1)+1 - fluxx(i,j) = ( phi(i,j) + phi(i-1,j) ) / 2.0d0 + fluxx(i,j) = ( phi(i,j) + phi(i-1,j) ) / 2.0d0 fluxx(i,j) = (-phi(i+1,j)+ 7.0d0*( phi(i,j) + phi(i-1,j) )-phi(i-2,j)) / 12.0d0 end do end do - + ! y-fluxes do j = lo(2), hi(2)+1 do i = lo(1), hi(1) @@ -44,7 +44,7 @@ subroutine SDC_feval_F(lo, hi, domlo, domhi, phi, philo, phihi, & f(i,j) = a*((fluxx(i+1,j ) - fluxx(i,j))/dx(1) & + (fluxy(i ,j+1) - fluxy(i,j))/dx(2)) end do - + end do case (1) ! First implicit piece (here it is diffusion) ! x-fluxes @@ -54,7 +54,7 @@ subroutine SDC_feval_F(lo, hi, domlo, domhi, phi, philo, phihi, & ! fluxx(i,j) = ( -phi(i+1,j) +15.0d0*(phi(i,j) - phi(i-1,j)) + phi(i-2,j) ) /(12.0d0*dx(1)) end do end do - + ! y-fluxes do j = lo(2), hi(2)+1 do i = lo(1), hi(1) @@ -81,9 +81,9 @@ subroutine SDC_feval_F(lo, hi, domlo, domhi, phi, philo, phihi, & case default print *, 'bad case in advance_2d' end select - + end subroutine SDC_feval_F - + subroutine SDC_fcomp_reaction_F (lo, hi, domlo, domhi, phi, philo, phihi, & rhs, rhslo, rhshi, & @@ -98,16 +98,16 @@ subroutine SDC_fcomp_reaction_F (lo, hi, domlo, domhi, phi, philo, phihi, & integer rhslo(2), rhshi(2) integer flo(2), fhi(2) real(amrex_real), intent(inout) :: phi (philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in) :: rhs (rhslo(1):rhshi(1),rhslo(2):rhshi(2)) + real(amrex_real), intent(in) :: rhs (rhslo(1):rhshi(1),rhslo(2):rhshi(2)) real(amrex_real), intent(in) :: f (flo(1):fhi(1),flo(2):fhi(2)) real(amrex_real), intent(in) :: dtq integer, intent(in) :: n - + ! local variables integer i,j real(amrex_real) c - ! Function + ! Function do j = lo(2), hi(2) do i = lo(1), hi(1) c = 1.0d0-dtq*(1.0d0-phi(i,j))*(0.5d0-phi(i,j)) diff --git a/Tutorials/SDC/MISDC_ADR_2d/Source/init_phi_2d.f90 b/Tutorials/SDC/MISDC_ADR_2d/Source/init_phi_2d.f90 index 3bf1c1fa989..241911bf20d 100644 --- a/Tutorials/SDC/MISDC_ADR_2d/Source/init_phi_2d.f90 +++ b/Tutorials/SDC/MISDC_ADR_2d/Source/init_phi_2d.f90 @@ -6,9 +6,9 @@ subroutine init_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi) bind(C, nam integer, intent(in) :: lo(2), hi(2), philo(2), phihi(2) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in ) :: dx(2) - real(amrex_real), intent(in ) :: prob_lo(2) - real(amrex_real), intent(in ) :: prob_hi(2) + real(amrex_real), intent(in ) :: dx(2) + real(amrex_real), intent(in ) :: prob_lo(2) + real(amrex_real), intent(in ) :: prob_hi(2) integer :: i,j double precision :: x,y,tupi @@ -34,9 +34,9 @@ subroutine err_phi(lo, hi, phi, philo, phihi, dx, prob_lo, prob_hi,a,d,r,time) b integer, intent(in) :: lo(2), hi(2), philo(2), phihi(2) real(amrex_real), intent(inout) :: phi(philo(1):phihi(1),philo(2):phihi(2)) - real(amrex_real), intent(in ) :: dx(2) - real(amrex_real), intent(in ) :: prob_lo(2) - real(amrex_real), intent(in ) :: prob_hi(2) + real(amrex_real), intent(in ) :: dx(2) + real(amrex_real), intent(in ) :: prob_lo(2) + real(amrex_real), intent(in ) :: prob_hi(2) real(amrex_real), intent(in ) :: a,d,r real(amrex_real), intent(in ) :: time diff --git a/Tutorials/SDC/MISDC_ADR_2d/Source/main.cpp b/Tutorials/SDC/MISDC_ADR_2d/Source/main.cpp index 69b97d6d027..be60723f9d7 100644 --- a/Tutorials/SDC/MISDC_ADR_2d/Source/main.cpp +++ b/Tutorials/SDC/MISDC_ADR_2d/Source/main.cpp @@ -8,9 +8,9 @@ int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + main_main(); - + amrex::Finalize(); return 0; } @@ -19,7 +19,7 @@ void main_main () { Real a; // advection coef. Real d; // diffusion coef. - Real r; // reaction coef. + Real r; // reaction coef. // AMREX_SPACEDIM: number of dimensions int n_cell, max_grid_size, Nsteps, plot_int; @@ -31,14 +31,14 @@ void main_main () // ParmParse is way of reading inputs from the inputs file ParmParse pp; - - // We need to get n_cell from the inputs file - this is the number of cells on each side of + + // We need to get n_cell from the inputs file - this is the number of cells on each side of // a square (or cubic) domain. pp.get("n_cell",n_cell); - + // The domain is broken into boxes of size max_grid_size pp.get("max_grid_size",max_grid_size); - + // Default plot_int to -1, allow us to set it to something else in the inputs file // If plot_int < 0 then no plot files will be writtenq plot_int = -1; @@ -46,22 +46,22 @@ void main_main () // Set plot_err = 1 to output the error to plot files instead of the solution int plot_err = 1; - + // Read in number of steps and final time pp.query("Nsteps",Nsteps); Real Tfin=0.0; - pp.query("Tfin",Tfin); + pp.query("Tfin",Tfin); Real dt = Tfin/Nsteps; // Set the time step // read in BC; see Src/Base/AMReX_BC_TYPES.H for supported types pp.queryarr("bc_lo", bc_lo); pp.queryarr("bc_hi", bc_hi); - + // Read in the coefficients for A-D-R pp.query("a",a); pp.query("d",d); pp.query("r",r); - + // determine whether boundary conditions are periodic Vector is_periodic(AMREX_SPACEDIM,0); for (int idim=0; idim < AMREX_SPACEDIM; ++idim) { @@ -91,15 +91,15 @@ void main_main () geom.define(domain,&real_box,CoordSys::cartesian,is_periodic.data()); } - // Nghost = number of ghost cells for each array + // Nghost = number of ghost cells for each array int Nghost = 2; - + // Ncomp = number of components for each array int Ncomp = 1; // time = starting time in the simulation Real time = 0.0; - + // How Boxes are distrubuted among MPI processes DistributionMapping dm(ba); @@ -116,29 +116,29 @@ void main_main () BL_TO_FORTRAN_ANYD(phi_new[mfi]), geom.CellSize(), geom.ProbLo(), geom.ProbHi()); } - + // Set up BCRec; see Src/Base/AMReX_BC_TYPES.H for supported types Vector bc(phi_old.nComp()); for (int n = 0; n < phi_old.nComp(); ++n) { - for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - // lo-side BCs - if (bc_lo[idim] == INT_DIR) { - bc[n].setLo(idim, BCType::int_dir); // periodic uses "internal Dirichlet" - } - else { - amrex::Abort("Invalid bc_lo"); - } - - // hi-side BCs - if (bc_hi[idim] == INT_DIR) { - bc[n].setHi(idim, BCType::int_dir); // periodic uses "internal Dirichlet" - } - else { - amrex::Abort("Invalid bc_hi"); - } - } + for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) + { + // lo-side BCs + if (bc_lo[idim] == INT_DIR) { + bc[n].setLo(idim, BCType::int_dir); // periodic uses "internal Dirichlet" + } + else { + amrex::Abort("Invalid bc_lo"); + } + + // hi-side BCs + if (bc_hi[idim] == INT_DIR) { + bc[n].setHi(idim, BCType::int_dir); // periodic uses "internal Dirichlet" + } + else { + amrex::Abort("Invalid bc_hi"); + } + } } // Build the flux multifabs @@ -157,103 +157,103 @@ void main_main () int Nsweeps=2*Nnodes-2; // This will give highest formal accuracy for Lobatto nodes pp.get("Nnodes",Nnodes); pp.get("Npieces",Npieces); - // pp.get("Nsweeps",Nsweeps); // Uncomment to adjust Nsweeps + // pp.get("Nsweeps",Nsweeps); // Uncomment to adjust Nsweeps // Build the structure SDCstruct SDCmats(Nnodes,Npieces,phi_old); SDCmats.Nsweeps =Nsweeps; // Number of SDC sweeps per time step - + const Real* dx = geom.CellSize(); - + // Write a plotfile of the initial data if plot_int > 0 (plot_int was defined in the inputs file) if (plot_int > 0) { - if (plot_err == 1) // Turn the solution into the error - { - MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); - for ( MFIter mfi(phi_new); mfi.isValid(); ++mfi ) - { - const Box& bx = mfi.validbox(); - err_phi(BL_TO_FORTRAN_BOX(bx), - BL_TO_FORTRAN_ANYD(phi_new[mfi]), - geom.CellSize(), geom.ProbLo(), geom.ProbHi(),&a,&d,&r,&time); - } - } - int n = 0; - const std::string& pltfile = amrex::Concatenate("plt",n,5); - WriteSingleLevelPlotfile(pltfile, phi_new, {"phi"}, geom, time, 0); - if (plot_err == 1) // Put the solution back - MultiFab::Copy(phi_new, phi_old, 0, 0, 1, 0); + if (plot_err == 1) // Turn the solution into the error + { + MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); + for ( MFIter mfi(phi_new); mfi.isValid(); ++mfi ) + { + const Box& bx = mfi.validbox(); + err_phi(BL_TO_FORTRAN_BOX(bx), + BL_TO_FORTRAN_ANYD(phi_new[mfi]), + geom.CellSize(), geom.ProbLo(), geom.ProbHi(),&a,&d,&r,&time); + } + } + int n = 0; + const std::string& pltfile = amrex::Concatenate("plt",n,5); + WriteSingleLevelPlotfile(pltfile, phi_new, {"phi"}, geom, time, 0); + if (plot_err == 1) // Put the solution back + MultiFab::Copy(phi_new, phi_old, 0, 0, 1, 0); } // Set an assorment of solver and parallization options and parameters // see AMReX_MLLinOp.H for the defaults, accessors, and mutators LPInfo info; - + // Implicit solve using MLABecLaplacian class MLABecLaplacian mlabec({geom}, {ba}, {dm}, info); - + // order of stencil int linop_maxorder = 2; mlabec.setMaxOrder(linop_maxorder); - + // build array of boundary conditions needed by MLABecLaplacian // see Src/Boundary/AMReX_LO_BCTYPES.H for supported types std::array mgbc_lo; std::array mgbc_hi; - - for (int n = 0; n < phi_old.nComp(); ++n) + + for (int n = 0; n < phi_old.nComp(); ++n) { for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) - { - // lo-side BCs - if (bc[n].lo(idim) == BCType::int_dir) { - mgbc_lo[idim] = LinOpBCType::Periodic; - } - else { - amrex::Abort("Invalid bc_lo"); - } - - // hi-side BCs - if (bc[n].hi(idim) == BCType::int_dir) { - mgbc_hi[idim] = LinOpBCType::Periodic; - } - else { - amrex::Abort("Invalid bc_hi"); - } - } + { + // lo-side BCs + if (bc[n].lo(idim) == BCType::int_dir) { + mgbc_lo[idim] = LinOpBCType::Periodic; + } + else { + amrex::Abort("Invalid bc_lo"); + } + + // hi-side BCs + if (bc[n].hi(idim) == BCType::int_dir) { + mgbc_hi[idim] = LinOpBCType::Periodic; + } + else { + amrex::Abort("Invalid bc_hi"); + } + } } - + // tell the solver what the domain boundary conditions are mlabec.setDomainBC(mgbc_lo, mgbc_hi); - + // scaling factors Real ascalar = 1.0; Real bscalar = 1.0; mlabec.setScalars(ascalar, bscalar); - + // Set up coefficient matrices MultiFab acoef(ba, dm, 1, 0); - + // fill in the acoef MultiFab and load this into the solver acoef.setVal(1.0); mlabec.setACoeffs(0, acoef); - + // bcoef lives on faces so we make an array of face-centered MultiFabs // then we will in face_bcoef MultiFabs and load them into the solver. std::array face_bcoef; for (int idim = 0; idim < AMREX_SPACEDIM; ++idim) { const BoxArray& bamg = amrex::convert(acoef.boxArray(), - IntVect::TheDimensionVector(idim)); + IntVect::TheDimensionVector(idim)); face_bcoef[idim].define(bamg, acoef.DistributionMap(), 1, 0); - face_bcoef[idim].setVal(1.0); + face_bcoef[idim].setVal(1.0); } mlabec.setBCoeffs(0, amrex::GetArrOfConstPtrs(face_bcoef)); - + // build an MLMG solver MLMG mlmg(mlabec); - + // set solver parameters int max_iter = 100; mlmg.setMaxIter(max_iter); @@ -263,28 +263,28 @@ void main_main () mlmg.setVerbose(verbose); int bottom_verbose = 0; mlmg.setBottomVerbose(bottom_verbose); - + // Do the time stepp[ing MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); for (int n = 1; n <= Nsteps; ++n) { - + // Do an SDC step - SDC_advance(phi_old, phi_new,flux, dt, geom, bc, mlmg,mlabec,SDCmats,a,d,r); + SDC_advance(phi_old, phi_new,flux, dt, geom, bc, mlmg,mlabec,SDCmats,a,d,r); time = time + dt; - MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); - - + MultiFab::Copy(phi_old, phi_new, 0, 0, 1, 0); + + if (plot_err == 1) // Turn the solution into the error - for ( MFIter mfi(phi_new); mfi.isValid(); ++mfi ) - { - const Box& bx = mfi.validbox(); - err_phi(BL_TO_FORTRAN_BOX(bx), - BL_TO_FORTRAN_ANYD(phi_new[mfi]), - geom.CellSize(), geom.ProbLo(), geom.ProbHi(),&a,&d,&r,&time); - } + for ( MFIter mfi(phi_new); mfi.isValid(); ++mfi ) + { + const Box& bx = mfi.validbox(); + err_phi(BL_TO_FORTRAN_BOX(bx), + BL_TO_FORTRAN_ANYD(phi_new[mfi]), + geom.CellSize(), geom.ProbLo(), geom.ProbHi(),&a,&d,&r,&time); + } // Tell the I/O Processor to write out which step we're doing amrex::Print() << "Advanced step " << n << "\n"; diff --git a/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc.H b/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc.H index 359b4b1ff25..992419c5d4d 100644 --- a/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc.H +++ b/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc.H @@ -15,27 +15,27 @@ using namespace amrex; void main_main (); void SDC_advance (MultiFab& phi_old, - MultiFab& phi_new, - std::array& flux, - Real dt, - const Geometry& geom, - const Vector& bc, - MLMG& mlmg, - MLABecLaplacian& mlabec, - SDCstruct &sdcmats,Real a,Real d,Real r); + MultiFab& phi_new, + std::array& flux, + Real dt, + const Geometry& geom, + const Vector& bc, + MLMG& mlmg, + MLABecLaplacian& mlabec, + SDCstruct &sdcmats,Real a,Real d,Real r); void SDC_feval (std::array& flux, - const Geometry& geom, - const Vector& bc, - SDCstruct &sdcmats, - Real a,Real d,Real r, - int sdc_m,int npiece); + const Geometry& geom, + const Vector& bc, + SDCstruct &sdcmats, + Real a,Real d,Real r, + int sdc_m,int npiece); void SDC_fcomp(MultiFab& rhs, - const Geometry& geom, - const Vector& bc, - SDCstruct &sdcmats, - MLMG &mlmg, - MLABecLaplacian& mlabec, - Real dt,Real d,Real r, - int sdc_m,int npiece); + const Geometry& geom, + const Vector& bc, + SDCstruct &sdcmats, + MLMG &mlmg, + MLABecLaplacian& mlabec, + Real dt,Real d,Real r, + int sdc_m,int npiece); #endif diff --git a/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc_F.H b/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc_F.H index 06ccd5bfd6f..f66a9d2cc62 100644 --- a/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc_F.H +++ b/Tutorials/SDC/MISDC_ADR_2d/Source/myfunc_F.H @@ -9,33 +9,33 @@ extern "C" #endif void init_phi(const int* lo, const int* hi, - amrex_real* data, const int* dlo, const int* dhi, - const amrex_real* dx, const amrex_real* prob_lo, - const amrex_real* prob_hi); - + amrex_real* data, const int* dlo, const int* dhi, + const amrex_real* dx, const amrex_real* prob_lo, + const amrex_real* prob_hi); + void err_phi(const int* lo, const int* hi, - amrex_real* data, const int* dlo, const int* dhi, - const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi, - amrex_real* a,amrex_real* d,amrex_real* r,amrex_real* time); + amrex_real* data, const int* dlo, const int* dhi, + const amrex_real* dx, const amrex_real* prob_lo, const amrex_real* prob_hi, + amrex_real* a,amrex_real* d,amrex_real* r,amrex_real* time); void SDC_feval_F(const int* lo, const int* hi, - const int* domlo, const int* domhi, - amrex_real* phi, const int* philo, const int* phihi, - amrex_real* fluxx, const int* fxlo, const int* fxhi, - amrex_real* fluxy, const int* fylo, const int* fyhi, -#if (BL_SPACEDIM == 3) - amrex_real* fluxz, const int* fzlo, const int* fzhi, + const int* domlo, const int* domhi, + amrex_real* phi, const int* philo, const int* phihi, + amrex_real* fluxx, const int* fxlo, const int* fxhi, + amrex_real* fluxy, const int* fylo, const int* fyhi, +#if (BL_SPACEDIM == 3) + amrex_real* fluxz, const int* fzlo, const int* fzhi, #endif - amrex_real* f, const int* flo, const int* fhi, - const amrex_real* dx,const amrex_real* a, - const amrex_real* d,const amrex_real* r, int *n); + amrex_real* f, const int* flo, const int* fhi, + const amrex_real* dx,const amrex_real* a, + const amrex_real* d,const amrex_real* r, int *n); void SDC_fcomp_reaction_F(const int* lo, const int* hi, const int* domlo, const int* domhi, amrex_real* phi, const int* philo, const int* phihi, - amrex_real* rhs, const int* rhslo, const int* rhshi, + amrex_real* rhs, const int* rhslo, const int* rhshi, amrex_real* f, const int* flo, const int* fhi, - const amrex_real* dtq, int *n); - + const amrex_real* dtq, int *n); + #ifdef __cplusplus } #endif diff --git a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.H b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.H index e26d01bd2c6..535cd7f2923 100644 --- a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.H +++ b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.H @@ -25,7 +25,7 @@ private: amrex::MultiFab soln; amrex::MultiFab the_soln; amrex::Geometry geom; - + void init_rhs (); void comp_the_solution (); }; diff --git a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.cpp b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.cpp index 310b81e8089..45c3b7df113 100644 --- a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.cpp +++ b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test.cpp @@ -57,7 +57,7 @@ SWFFT_Test::SWFFT_Test () pp.query("verbose", verbose); } - + BoxArray ba; { // Make up a dx that is not 1 @@ -81,7 +81,7 @@ SWFFT_Test::SWFFT_Test () geom.define(domain, &real_box, CoordSys::cartesian, is_periodic.data()); } - // Make sure we define both the soln and the rhs with the same DistributionMapping + // Make sure we define both the soln and the rhs with the same DistributionMapping DistributionMapping dmap{ba}; // Note that we are defining rhs with NO ghost cells @@ -154,9 +154,9 @@ SWFFT_Test::solve () if (verbose) { - amrex::Print() << "MAX / MIN VALUE OF COMP SOLN " << soln.max(0) << " " + amrex::Print() << "MAX / MIN VALUE OF COMP SOLN " << soln.max(0) << " " << soln.min(0) << std::endl; - amrex::Print() << "MAX / MIN VALUE OF EXACT SOLN " << the_soln.max(0) << " " + amrex::Print() << "MAX / MIN VALUE OF EXACT SOLN " << the_soln.max(0) << " " << the_soln.min(0) << std::endl; } diff --git a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.F90 b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.F90 index 83d9182bf49..8db9c361220 100644 --- a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.F90 +++ b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.F90 @@ -32,13 +32,13 @@ subroutine fort_init_rhs (lo, hi, rhs, rlo, rhi, domlo, domhi, problo, probhi, d do k = lo(3), hi(3) z = (dble(k)+0.5d0)*dx(3)/Lz - + do j = lo(2), hi(2) y = (dble(j)+0.5d0)*dx(2)/Ly - + do i = lo(1), hi(1) x = (dble(i)+0.5d0)*dx(1)/Lx - + rhs(i,j,k) = -fac * (sin(tpi*x) * sin(tpi*y) * sin(tpi*z)) & & -fac * (sin(fpi*x) * sin(fpi*y) * sin(fpi*z)) end do @@ -74,14 +74,14 @@ subroutine fort_comp_asol (lo, hi, soln, slo, shi, domlo, domhi, problo, probhi, y = (dble(j)+0.5d0)*dx(2) do i = lo(1), hi(1) x = (dble(i)+0.5d0)*dx(1) - + soln(i,j,k) = 1.d0 * (sin(tpi*x/Lx) * sin(tpi*y/Ly) * sin(tpi*z/Lz)) & & + .25d0 * (sin(fpi*x/Lx) * sin(fpi*y/Ly) * sin(fpi*z/Lz)) end do end do end do - + end subroutine fort_comp_asol end module abl_module diff --git a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.H b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.H index 6f5f9582fd5..df5387f5aa4 100644 --- a/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.H +++ b/Tutorials/SWFFT/SWFFT_poisson/SWFFT_Test_F.H @@ -4,15 +4,15 @@ extern "C" { #endif - void fort_init_rhs (const int* lo, const int* hi, + void fort_init_rhs (const int* lo, const int* hi, amrex_real* rhs, const int* rlo, const int* rhi, - const int* dom_lo, const int* dom_hi, + const int* dom_lo, const int* dom_hi, const amrex_real* problo, const amrex_real* probhi, const amrex_real* dx); - void fort_comp_asol (const int* lo, const int* hi, + void fort_comp_asol (const int* lo, const int* hi, const amrex_real* soln, const int* slo, const int* shi, - const int* dom_lo, const int* dom_hi, + const int* dom_lo, const int* dom_hi, const amrex_real* problo, const amrex_real* probhi, const amrex_real* dx); diff --git a/Tutorials/SWFFT/SWFFT_poisson/main.cpp b/Tutorials/SWFFT/SWFFT_poisson/main.cpp index 7457e01582f..64828f10037 100644 --- a/Tutorials/SWFFT/SWFFT_poisson/main.cpp +++ b/Tutorials/SWFFT/SWFFT_poisson/main.cpp @@ -4,13 +4,13 @@ int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + { BL_PROFILE("main()"); SWFFT_Test sw_test; sw_test.solve(); } - + amrex::Finalize(); return 0; } diff --git a/Tutorials/SWFFT/SWFFT_poisson/swfft_solver.cpp b/Tutorials/SWFFT/SWFFT_poisson/swfft_solver.cpp index ed0456cfd96..a12f4c871d8 100644 --- a/Tutorials/SWFFT/SWFFT_poisson/swfft_solver.cpp +++ b/Tutorials/SWFFT/SWFFT_poisson/swfft_solver.cpp @@ -21,14 +21,14 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) amrex::Print() << "BA " << ba << std::endl; const DistributionMapping& dm = soln.DistributionMap(); - if (rhs.nGrow() != 0 || soln.nGrow() != 0) + if (rhs.nGrow() != 0 || soln.nGrow() != 0) amrex::Error("Current implementation requires that both rhs and soln have no ghost cells"); // Define pi and (two pi) here Real pi = 4 * std::atan(1.0); Real tpi = 2 * pi; - // We assume that all grids have the same size hence + // We assume that all grids have the same size hence // we have the same nx,ny,nz on all ranks int nx = ba[0].size()[0]; int ny = ba[0].size()[1]; @@ -40,7 +40,7 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) int nby = domain.length(1) / ny; int nbz = domain.length(2) / nz; int nboxes = nbx * nby * nbz; - if (nboxes != ba.size()) + if (nboxes != ba.size()) amrex::Error("NBOXES NOT COMPUTED CORRECTLY"); amrex::Print() << "Number of boxes:\t" << nboxes << std::endl; @@ -64,7 +64,7 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) rank_mapping[local_index] = dmap[ib]; if (verbose) - amrex::Print() << "LOADING RANK NUMBER " << dmap[ib] << " FOR GRID NUMBER " << ib + amrex::Print() << "LOADING RANK NUMBER " << dmap[ib] << " FOR GRID NUMBER " << ib << " WHICH IS LOCAL NUMBER " << local_index << std::endl; } @@ -79,13 +79,13 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) int n[3] = {domain.length(2), domain.length(1), domain.length(0)}; hacc::Distribution d(MPI_COMM_WORLD,n,Ndims,&rank_mapping[0]); hacc::Dfft dfft(d); - + for (MFIter mfi(rhs,false); mfi.isValid(); ++mfi) { int gid = mfi.index(); size_t local_size = dfft.local_size(); - + std::vector > a; std::vector > b; @@ -106,7 +106,7 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) complex_t temp(rhs[mfi].dataPtr()[local_indx],0.); a[local_indx] = temp; - local_indx++; + local_indx++; } } @@ -126,9 +126,9 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) const int *global_ng = dfft.global_ng(); for(size_t i=0; i<(size_t)local_ng[0]; i++) { - size_t global_i = local_ng[0]*self[0] + i; + size_t global_i = local_ng[0]*self[0] + i; - for(size_t j=0; j<(size_t)local_ng[1]; j++) { + for(size_t j=0; j<(size_t)local_ng[1]; j++) { size_t global_j = local_ng[1]*self[1] + j; for(size_t k=0; k<(size_t)local_ng[2]; k++) { @@ -144,7 +144,7 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) a[local_indx] = a[local_indx] / fac; } - local_indx++; + local_indx++; } } @@ -165,7 +165,7 @@ swfft_solver(MultiFab& rhs, MultiFab& soln, Geometry& geom, int verbose) // Divide by 2 pi N soln[mfi].dataPtr()[local_indx] = fac * std::real(a[local_indx]); - local_indx++; + local_indx++; } } diff --git a/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.H b/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.H index 217d26c3aea..879080502a8 100644 --- a/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.H +++ b/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.H @@ -25,7 +25,7 @@ private: amrex::MultiFab phi_spatial; amrex::MultiFab phi_dft; amrex::Geometry geom; - + void init_phi_spatial (); void WritePlotFile(const int step=0, const amrex::Real time=0.0); }; diff --git a/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.cpp b/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.cpp index 34cb7bbbce2..60d0872c885 100644 --- a/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.cpp +++ b/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test.cpp @@ -34,7 +34,7 @@ SWFFT_Test::SWFFT_Test () Vector ncs; pp.getarr("n_cell",ncs); #if (AMREX_SPACEDIM == 2) - n_cell = IntVect{ncs[0],ncs[1]}; + n_cell = IntVect{ncs[0],ncs[1]}; #elif (AMREX_SPACEDIM == 3) n_cell = IntVect{ncs[0],ncs[1],ncs[2]}; #endif @@ -42,15 +42,15 @@ SWFFT_Test::SWFFT_Test () int ncs; pp.get("n_cell",ncs); #if (AMREX_SPACEDIM == 2) - n_cell = IntVect{ncs,ncs}; + n_cell = IntVect{ncs,ncs}; #elif (AMREX_SPACEDIM == 3) - n_cell = IntVect{ncs,ncs,ncs}; + n_cell = IntVect{ncs,ncs,ncs}; #endif } else { #if (AMREX_SPACEDIM == 2) - n_cell = IntVect{32,32}; + n_cell = IntVect{32,32}; #elif (AMREX_SPACEDIM == 3) - n_cell = IntVect{32,32,32}; + n_cell = IntVect{32,32,32}; #endif } @@ -60,9 +60,9 @@ SWFFT_Test::SWFFT_Test () Vector mgs; pp.getarr("max_grid_size",mgs); #if (AMREX_SPACEDIM == 2) - max_grid_size = IntVect{mgs[0],mgs[1]}; + max_grid_size = IntVect{mgs[0],mgs[1]}; #elif (AMREX_SPACEDIM == 3) - max_grid_size = IntVect{mgs[0],mgs[1],mgs[2]}; + max_grid_size = IntVect{mgs[0],mgs[1],mgs[2]}; #endif } else if (cnt > 0) { int mgs; @@ -81,9 +81,9 @@ SWFFT_Test::SWFFT_Test () } pp.query("verbose", verbose); - pp.query("prob_type", prob_type); + pp.query("prob_type", prob_type); } - + BoxArray ba; { // Make up a dx that is not 1 @@ -112,14 +112,14 @@ SWFFT_Test::SWFFT_Test () // The FFT assumes fully periodic boundaries #if (AMREX_SPACEDIM == 2) - std::array is_periodic {1,1}; + std::array is_periodic {1,1}; #elif (AMREX_SPACEDIM == 3) - std::array is_periodic {1,1,1}; + std::array is_periodic {1,1,1}; #endif geom.define(domain, &real_box, CoordSys::cartesian, is_periodic.data()); } - // Make sure we define both the phi_dft and the phi_spatial with the same DistributionMapping + // Make sure we define both the phi_dft and the phi_spatial with the same DistributionMapping DistributionMapping dmap{ba}; // Note that we are defining phi_spatial with NO ghost cells @@ -165,7 +165,7 @@ SWFFT_Test::computeFFT () if (verbose) { - amrex::Print() << "MAX / MIN VALUE OF DFT " << phi_dft.max(0) << " " + amrex::Print() << "MAX / MIN VALUE OF DFT " << phi_dft.max(0) << " " << phi_dft.min(0) << std::endl; } @@ -175,7 +175,7 @@ SWFFT_Test::computeFFT () void SWFFT_Test::WritePlotFile (const int step, const amrex::Real time) { - + MultiFab plotfile; Vector varNames; int nPlot; diff --git a/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test_F.H b/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test_F.H index 072e75c13e2..932ba4857fa 100644 --- a/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test_F.H +++ b/Tutorials/SWFFT/SWFFT_simple/SWFFT_Test_F.H @@ -4,15 +4,15 @@ extern "C" { #endif - void fort_init_phi_spatial (const int* lo, const int* hi, + void fort_init_phi_spatial (const int* lo, const int* hi, amrex_real* phi_spatial, const int* rlo, const int* rhi, - const int* dom_lo, const int* dom_hi, + const int* dom_lo, const int* dom_hi, const amrex_real* problo, const amrex_real* probhi, const amrex_real* dx, int* prob_type); - void fort_comp_asol (const int* lo, const int* hi, + void fort_comp_asol (const int* lo, const int* hi, const amrex_real* phi_dft, const int* slo, const int* shi, - const int* dom_lo, const int* dom_hi, + const int* dom_lo, const int* dom_hi, const amrex_real* problo, const amrex_real* probhi, const amrex_real* dx); diff --git a/Tutorials/SWFFT/SWFFT_simple/main.cpp b/Tutorials/SWFFT/SWFFT_simple/main.cpp index 62d9e1e128f..3285fb5e3f1 100644 --- a/Tutorials/SWFFT/SWFFT_simple/main.cpp +++ b/Tutorials/SWFFT/SWFFT_simple/main.cpp @@ -4,13 +4,13 @@ int main (int argc, char* argv[]) { amrex::Initialize(argc,argv); - + { BL_PROFILE("main()"); SWFFT_Test sw_test; sw_test.computeFFT(); } - + amrex::Finalize(); return 0; } diff --git a/Tutorials/SWFFT/SWFFT_simple/swfft_compute.cpp b/Tutorials/SWFFT/SWFFT_simple/swfft_compute.cpp index ad2aa087f2d..948a4317e9a 100644 --- a/Tutorials/SWFFT/SWFFT_simple/swfft_compute.cpp +++ b/Tutorials/SWFFT/SWFFT_simple/swfft_compute.cpp @@ -27,10 +27,10 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb exit(0); } - if (phi_spatial.nGrow() != 0 || phi_dft.nGrow() != 0) + if (phi_spatial.nGrow() != 0 || phi_dft.nGrow() != 0) amrex::Error("Current implementation requires that both phi_spatial and phi_dft have no ghost cells"); - // We assume that all grids have the same size hence + // We assume that all grids have the same size hence // we have the same nx,ny,nz on all ranks int nx = ba[0].size()[0]; int ny = ba[0].size()[1]; @@ -50,7 +50,7 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb int nbz = domain.length(2) / nz; #endif int nboxes = nbx * nby * nbz; - if (nboxes != ba.size()) + if (nboxes != ba.size()) amrex::Error("NBOXES NOT COMPUTED CORRECTLY"); amrex::Print() << "Number of boxes:\t" << nboxes << std::endl; @@ -64,9 +64,9 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb int i = ba[ib].smallEnd(0) / nx; int j = ba[ib].smallEnd(1) / ny; #if (AMREX_SPACEDIM == 2) - int k = 0; + int k = 0; #elif (AMREX_SPACEDIM == 3) - int k = ba[ib].smallEnd(2) / nz; + int k = ba[ib].smallEnd(2) / nz; #endif // This would be the "correct" local index if the data wasn't being transformed @@ -79,7 +79,7 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb rank_mapping[local_index] = dmap[ib]; if (verbose) - amrex::Print() << "LOADING RANK NUMBER " << dmap[ib] << " FOR GRID NUMBER " << ib + amrex::Print() << "LOADING RANK NUMBER " << dmap[ib] << " FOR GRID NUMBER " << ib << " WHICH IS LOCAL NUMBER " << local_index << std::endl; } @@ -95,13 +95,13 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb #endif hacc::Distribution d(MPI_COMM_WORLD,n,Ndims,&rank_mapping[0]); hacc::Dfft dfft(d); - + for (MFIter mfi(phi_spatial,false); mfi.isValid(); ++mfi) { int gid = mfi.index(); size_t local_size = dfft.local_size(); - + std::vector > a; std::vector > b; @@ -122,7 +122,7 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb complex_t temp(phi_spatial[mfi].dataPtr()[local_indx],0.); a[local_indx] = temp; - local_indx++; + local_indx++; } } @@ -132,16 +132,16 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb // Compute the forward transform // ******************************************* dfft.forward(&a[0]); - + // ******************************************* // Redistribute from z-pencils to blocks // ******************************************* d.redistribute_2_to_3(&a[0],&b[0],2); - + size_t global_size = dfft.global_size(); double fac; - // fac = sqrt(1.0 / (double)global_size); + // fac = sqrt(1.0 / (double)global_size); fac = 1.0; // Overwrite fac local_indx = 0; @@ -152,7 +152,7 @@ swfft_compute(MultiFab& phi_spatial, MultiFab& phi_dft, Geometry& geom, int verb // Divide by 2 pi N phi_dft[mfi].dataPtr()[local_indx] = fac * std::abs(b[local_indx]); - local_indx++; + local_indx++; } }