Buffer cleanup

CMakeLists.txt

@@ -49,6 +49,19 @@ option(IPPL_USE_ALTERNATIVE_VARIANT
 option(IPPL_USE_STANDARD_FOLDERS "Put all generated binaries in bin/lib folders" OFF)
 option(IPPL_SKIP_FAILING_TESTS "Do not build/test tests that are currently marked as failing" OFF)
 option(IPPL_ENABLE_SCRIPTS "Generate job script templates for some benchmarks/tests" OFF)
+option(IPPL_GPU_AWARE_MPI "Allow MPI to/from from device memory buffers" OFF)
+# logging options
+set(IPPL_LOG_LEVEL "off" CACHE STRING "Enable logging for messages of >=level")
+set_property(
+  CACHE IPPL_LOG_LEVEL
+  PROPERTY STRINGS
+           off
+           trace
+           debug
+           info
+           warn
+           error
+           critical)
 
 # "Build IPPL as a shared library (ON) or static library (OFF)" OFF) if(IPPL_DYL)
 # set(BUILD_SHARED_LIBS ON CACHE BOOL "" FORCE) message(WARNING "IPPL_DYL is deprecated; use
@@ -97,21 +110,6 @@ if(DEFINED USE_ALTERNATIVE_VARIANT)
   set(IPPL_USE_ALTERNATIVE_VARIANT ${USE_ALTERNATIVE_VARIANT} CACHE BOOL "" FORCE)
 endif()
 
-# ------------------------------------------------------------------------------
-# Debug: This tells the compiler to replace occurrences of ${<DIR>} with <x> in debug info and error
-# messages.
-# ------------------------------------------------------------------------------
-add_compile_options(
-  $<$<COMPILE_LANG_AND_ID:C,Clang,AppleClang,GNU>:-ffile-prefix-map=${CMAKE_SOURCE_DIR}=.>
-  $<$<COMPILE_LANG_AND_ID:CXX,Clang,AppleClang,GNU>:-ffile-prefix-map=${CMAKE_SOURCE_DIR}=.>)
-
-if(DEFINED FETCHCONTENT_BASE_DIR)
-  add_compile_options(
-    $<$<COMPILE_LANG_AND_ID:C,Clang,AppleClang,GNU>:-ffile-prefix-map=${FETCHCONTENT_BASE_DIR}=.3p>
-    $<$<COMPILE_LANG_AND_ID:CXX,Clang,AppleClang,GNU>:-ffile-prefix-map=${FETCHCONTENT_BASE_DIR}=.3p>
-  )
-endif()
-
 # ------------------------------------------------------------------------------
 # Define sources for project
 # ------------------------------------------------------------------------------

alpine/LandauDamping.cpp

@@ -37,17 +37,22 @@ const char* TestName   = "LandauDamping";
 #include "Manager/datatypes.h"
 
 #include "Utility/IpplTimings.h"
+#include "Utility/Logging.h"
 
 #include "LandauDampingManager.h"
 #include "Manager/PicManager.h"
 
 int main(int argc, char* argv[]) {
+#if defined(SPDLOG_ACTIVE_LEVEL) && (SPDLOG_ACTIVE_LEVEL != SPDLOG_LEVEL_OFF)
+    spdlog::set_pattern("[%^%-8l%$]%t| %v");
+    spdlog::set_level(spdlog::level::trace);
+#endif
     ippl::initialize(argc, argv);
     {
         Inform msg(TestName);
         Inform msg2all(TestName, INFORM_ALL_NODES);
 
-        static IpplTimings::TimerRef mainTimer = IpplTimings::getTimer("total");
+        static IpplTimings::TimerRef mainTimer       = IpplTimings::getTimer("total");
         static IpplTimings::TimerRef initializeTimer = IpplTimings::getTimer("initialize");
         IpplTimings::startTimer(mainTimer);
         IpplTimings::startTimer(initializeTimer);
@@ -82,7 +87,7 @@ int main(int argc, char* argv[]) {
         manager.pre_run();
 
         IpplTimings::stopTimer(initializeTimer);
-        
+
         manager.setTime(0.0);
 
         msg << "Starting iterations ..." << endl;

cmake/Dependencies.cmake

@@ -44,6 +44,23 @@ if("OPENMP" IN_LIST IPPL_PLATFORMS)
   colour_message(STATUS ${Green} "✅ OpenMP platform requested OpenMP found ${OPENMP_VERSION}")
 endif()
 
+# ------------------------------------------------------------------------------
+# spdlog logging library
+# ------------------------------------------------------------------------------
+string(TOUPPER ${IPPL_LOG_LEVEL} IPPL_LOG_LEVEL_UPPERCASE)
+if(NOT "${IPPL_LOG_LEVEL_UPPERCASE}" MATCHES "OFF")
+  find_package(spdlog REQUIRED)
+  colour_message(STATUS ${Green} "✅ spdlog found ${spdlog_VERSION}")
+endif()
+
+# ------------------------------------------------------------------------------
+# fmt library (for formatting nice log messages)
+# ------------------------------------------------------------------------------
+if(NOT "${IPPL_LOG_LEVEL_UPPERCASE}" MATCHES "OFF")
+  find_package(fmt REQUIRED)
+  colour_message(STATUS ${Green} "✅ fmt found ${fmt_VERSION}")
+endif()
+
 # ------------------------------------------------------------------------------
 # Utility function to clear a list of vars one by one
 # ------------------------------------------------------------------------------

scripts/CMakeLists.txt

@@ -9,6 +9,20 @@ message(STATUS "IPPL_MACHINENAME for script generation is: ${IPPL_MACHINENAME}")
 # ------------------------------------------------------------------------------
 set(IPPL_JOB_SUBMISSION_ACCOUNT "c41" CACHE STRING "Account to use for job submission templates")
 
+# populate IPPL_SCRIPTS_UENV from environment variable UENV_MOUNT_LIST (if present)
+if(DEFINED ENV{UENV_MOUNT_LIST})
+  set(IPPL_SCRIPTS_UENV "$ENV{UENV_MOUNT_LIST}"
+      CACHE STRING "UENV to use on alps when running scripts mount")
+  set(IPPL_SCRIPTS_UENV_VIEW "default" CACHE STRING "View to set in uenv")
+  colour_message(STATUS ${LightBlue}
+                 "IPPL_SCRIPTS_UENV set from UENV_MOUNT_LIST: ${IPPL_SCRIPTS_UENV}")
+else()
+  # Remove any cached and normal definitions so the variable is completely unset
+  unset(IPPL_SCRIPTS_UENV CACHE)
+  unset(IPPL_SCRIPTS_UENV_VIEW CACHE)
+  colour_message(STATUS ${LightBlue} "No UENV detected")
+endif()
+
 # ------------------------------------------------------------------------------
 # utility function to get target path/name since we can't use generator expressions to set variables
 # directly

scripts/landau/strong-scaling-alps/jobscript-gh200.slurm

@@ -12,8 +12,8 @@
 #SBATCH --cpus-per-task=72
 #SBATCH --exclusive
 
-#SBATCH --uenv=/capstor/store/cscs/cscs/public/uenvs/opal-x-gh200-mpich-gcc-2025-09-28.squashfs
-#SBATCH --view=develop
+#SBATCH --uenv=@IPPL_SCRIPTS_UENV@
+#SBATCH --view=@IPPL_SCRIPTS_UENV_VIEW@
 
 #SBATCH --output=landau__n_.out
 #SBATCH --error=landau__n_.error

scripts/landau/strong-scaling-alps/jobscript-mi300.slurm

@@ -13,8 +13,8 @@
 #SBATCH --cpus-per-task=48
 #SBATCH --exclusive
 
-#SBATCH --uenv=/capstor/scratch/cscs/biddisco/opal-x-mi300-mpich-gcc-2025-10-27.squashfs
-#SBATCH --view=default
+#SBATCH --uenv=@IPPL_SCRIPTS_UENV@
+#SBATCH --view=@IPPL_SCRIPTS_UENV_VIEW@
 
 #SBATCH --output=landau__n_.out
 #SBATCH --error=landau__n_.error

src/CMakeLists.txt

@@ -35,6 +35,14 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/IpplVersions.h.in
 
 add_library(ippl)
 
+if(NOT "${IPPL_LOG_LEVEL_UPPERCASE}" MATCHES "OFF")
+  target_compile_definitions(ippl PUBLIC IPPL_LOGGING_ENABLED=1)
+  target_compile_definitions(ippl
+                             PUBLIC SPDLOG_ACTIVE_LEVEL=SPDLOG_LEVEL_${IPPL_LOG_LEVEL_UPPERCASE})
+  target_link_libraries(ippl PUBLIC spdlog::spdlog $<$<BOOL:${MINGW}>:ws2_32>)
+  target_link_libraries(ippl PUBLIC fmt::fmt)
+endif()
+
 target_compile_features(ippl PUBLIC cxx_std_20)
 
 target_compile_options(

src/Communicate/Archive.h

@@ -16,19 +16,20 @@
 #include "Types/ViewTypes.h"
 
 #include "Types/Vector.h"
+#undef IPPL_SIMPLE_VIEW_STORAGE
 
 namespace ippl {
     namespace detail {
         /*!
          * @file Archive.h
-         * Serialize and desesrialize particle attributes.
+         * Serialize and deserialize particle attributes.
          * @tparam Properties variadic template for Kokkos::View
          */
 
-        template <class... Properties>
+        template <typename BufferType>
         class Archive {
         public:
-            using buffer_type  = typename ViewType<char, 1, Properties...>::view_type;
+            using buffer_type  = BufferType;
             using pointer_type = typename buffer_type::pointer_type;
 
             Archive(size_type size = 0);
@@ -73,7 +74,7 @@ namespace ippl {
             /*!
              * @returns a pointer to the data of the buffer
              */
-            pointer_type getBuffer() { return buffer_m.data(); }
+            pointer_type getData() { return buffer_m.data(); }
 
             /*!
              * @returns the size of the buffer
@@ -82,10 +83,18 @@ namespace ippl {
 
             size_type getBufferSize() const { return buffer_m.size(); }
 
-            void resizeBuffer(size_type size) { Kokkos::resize(buffer_m, size); }
-
-            void reallocBuffer(size_type size) { Kokkos::realloc(buffer_m, size); }
+            void reallocBuffer(size_type size) {
+#ifdef IPPL_SIMPLE_VIEW_STORAGE
+                Kokkos::realloc(buffer_m, size);
+#else
+                buffer_m.reallocBuffer(size);
+#endif
+            }
 
+            void resetReadWritePos() {
+                readpos_m  = 0;
+                writepos_m = 0;
+            }
             void resetWritePos() { writepos_m = 0; }
             void resetReadPos() { readpos_m = 0; }
 
@@ -97,7 +106,7 @@ namespace ippl {
             //! read position for deserialization
             size_type readpos_m;
             //! serialized data
-            buffer_type buffer_m;
+            BufferType buffer_m;
         };
     }  // namespace detail
 }  // namespace ippl

src/Communicate/Archive.hpp

@@ -4,99 +4,116 @@
 //
 #include <cstring>
 
+#include "Utility/Logging.h"
+
 #include "Archive.h"
 
 namespace ippl {
     namespace detail {
 
-        template <class... Properties>
-        Archive<Properties...>::Archive(size_type size)
+        template <typename BufferType>
+        Archive<BufferType>::Archive(size_type size)
             : writepos_m(0)
             , readpos_m(0)
             , buffer_m("buffer", size) {}
 
-        template <class... Properties>
+        // -----------------------------------
+        // Scalar serialize
+        template <typename BufferType>
         template <typename T, class... ViewArgs>
-        void Archive<Properties...>::serialize(const Kokkos::View<T*, ViewArgs...>& view,
-                                               size_type nsends) {
-            using exec_space  = typename Kokkos::View<T*, ViewArgs...>::execution_space;
-            using policy_type = Kokkos::RangePolicy<exec_space>;
-
-            size_t size = sizeof(T);
-            Kokkos::parallel_for(
-                "Archive::serialize()", policy_type(0, nsends),
-                KOKKOS_CLASS_LAMBDA(const size_type i) {
-                    std::memcpy(buffer_m.data() + i * size + writepos_m, view.data() + i, size);
-                });
+        void Archive<BufferType>::serialize(const Kokkos::View<T*, ViewArgs...>& view,
+                                            size_type nsends) {
+            constexpr size_t size = sizeof(T);
+            char* dst_ptr         = (char*)(buffer_m.data()) + writepos_m;
+            char* src_ptr         = (char*)(view.data());
+            assert(writepos_m + (nsends * size) <= buffer_m.size());
+            // construct temp views of the src/dst buffers of the correct size (bytes)
+            Kokkos::View<char*, Kokkos::MemoryUnmanaged> src_view(src_ptr, size * nsends);
+            Kokkos::View<char*, Kokkos::MemoryUnmanaged> dst_view(dst_ptr, size * nsends);
+            Kokkos::deep_copy(dst_view, src_view);
             Kokkos::fence();
-            writepos_m += size * nsends;
+            SPDLOG_TRACE("Incrementing writepos: {}, from {}, to {}", (void*)dst_view.data(),
+                         writepos_m, writepos_m + (nsends * size));
+            writepos_m += (nsends * size);
         }
 
-        template <class... Properties>
+        // -----------------------------------
+        // Vector serialize
+        template <typename BufferType>
         template <typename T, unsigned Dim, class... ViewArgs>
-        void Archive<Properties...>::serialize(
-            const Kokkos::View<Vector<T, Dim>*, ViewArgs...>& view, size_type nsends) {
-            using exec_space = typename Kokkos::View<T*, ViewArgs...>::execution_space;
-
-            size_t size = sizeof(T);
-            // Default index type for range policies is int64,
+        void Archive<BufferType>::serialize(const Kokkos::View<Vector<T, Dim>*, ViewArgs...>& view,
+                                            size_type nsends) {
+            constexpr size_t size         = sizeof(T);
+            char* dst_ptr                 = (char*)(buffer_m.data());
+            ippl::Vector<T, Dim>* src_ptr = view.data();
+            auto wp                       = writepos_m;
+            // The Kokkos range policies expect int64
             // so we have to explicitly specify size_type (uint64)
+            using exec_space = typename Kokkos::View<T*, ViewArgs...>::execution_space;
             using mdrange_t =
                 Kokkos::MDRangePolicy<Kokkos::Rank<2>, Kokkos::IndexType<size_type>, exec_space>;
             Kokkos::parallel_for(
-                "Archive::serialize()",
-                // The constructor for Kokkos range policies always
-                // expects int64 regardless of index type provided
-                // by template parameters, so the typecast is necessary
-                // to avoid compiler warnings
-                mdrange_t({0, 0}, {(long int)nsends, Dim}),
-                KOKKOS_CLASS_LAMBDA(const size_type i, const size_t d) {
-                    std::memcpy(buffer_m.data() + (Dim * i + d) * size + writepos_m,
-                                &(*(view.data() + i))[d], size);
+                "Archive::serialize()", mdrange_t({0, 0}, {(long int)nsends, Dim}),
+                KOKKOS_LAMBDA(const size_type i, const size_t d) {
+                    std::memcpy(dst_ptr + (Dim * i + d) * size + wp, &(*(src_ptr + i))[d], size);
                 });
+
             Kokkos::fence();
             writepos_m += Dim * size * nsends;
         }
 
-        template <class... Properties>
+        // -----------------------------------
+        // Scalar Deserialize
+        template <typename BufferType>
         template <typename T, class... ViewArgs>
-        void Archive<Properties...>::deserialize(Kokkos::View<T*, ViewArgs...>& view,
-                                                 size_type nrecvs) {
-            using exec_space  = typename Kokkos::View<T*, ViewArgs...>::execution_space;
-            using policy_type = Kokkos::RangePolicy<exec_space>;
-
-            size_t size = sizeof(T);
+        void Archive<BufferType>::deserialize(Kokkos::View<T*, ViewArgs...>& view,
+                                              size_type nrecvs) {
+            // if we have to enlarge the destination view
             if (nrecvs > view.extent(0)) {
+                SPDLOG_WARN("DeSerialization realloc: {}, from {}, to {}", (void*)view.data(),
+                            view.extent(0), nrecvs);
                 Kokkos::realloc(view, nrecvs);
             }
-            Kokkos::parallel_for(
-                "Archive::deserialize()", policy_type(0, nrecvs),
-                KOKKOS_CLASS_LAMBDA(const size_type i) {
-                    std::memcpy(view.data() + i, buffer_m.data() + i * size + readpos_m, size);
-                });
-            // Wait for deserialization kernel to complete
-            // (as with serialization kernels)
+            //
+            constexpr size_t size = sizeof(T);
+            char* src_ptr         = (char*)(buffer_m.data()) + readpos_m;
+            char* dst_ptr         = (char*)(view.data());
+            assert(readpos_m + (nrecvs * size) <= buffer_m.size());
+            // construct temp views of the src/dst buffers of the correct size (bytes)
+            Kokkos::View<char*, Kokkos::MemoryUnmanaged> src_view(src_ptr, size * nrecvs);
+            Kokkos::View<char*, Kokkos::MemoryUnmanaged> dst_view(dst_ptr, size * nrecvs);
+            Kokkos::deep_copy(dst_view, src_view);
             Kokkos::fence();
-            readpos_m += size * nrecvs;
+            SPDLOG_TRACE("Incrementing readpos: {}, from {}, to {}", (void*)buffer_m.data(),
+                         readpos_m, readpos_m + (nrecvs * size));
+            readpos_m += (nrecvs * size);
         }
 
-        template <class... Properties>
+        // -----------------------------------
+        // Vecto Deserialize
+        template <typename BufferType>
         template <typename T, unsigned Dim, class... ViewArgs>
-        void Archive<Properties...>::deserialize(Kokkos::View<Vector<T, Dim>*, ViewArgs...>& view,
-                                                 size_type nrecvs) {
-            using exec_space = typename Kokkos::View<T*, ViewArgs...>::execution_space;
-
-            size_t size = sizeof(T);
+        void Archive<BufferType>::deserialize(Kokkos::View<Vector<T, Dim>*, ViewArgs...>& view,
+                                              size_type nrecvs)  //
+        {
+            // if we have to enlarge the destination view
             if (nrecvs > view.extent(0)) {
+                SPDLOG_WARN("DeSerialization realloc: {}, from {}, to {}", (void*)view.data(),
+                            view.extent(0), nrecvs);
                 Kokkos::realloc(view, nrecvs);
             }
+            //
+            constexpr size_t size         = sizeof(T);
+            char* src_ptr                 = (char*)(buffer_m.data());
+            ippl::Vector<T, Dim>* dst_ptr = view.data();
+            auto rp                       = readpos_m;
+            using exec_space              = typename Kokkos::View<T*, ViewArgs...>::execution_space;
             using mdrange_t =
                 Kokkos::MDRangePolicy<Kokkos::Rank<2>, Kokkos::IndexType<size_type>, exec_space>;
             Kokkos::parallel_for(
                 "Archive::deserialize()", mdrange_t({0, 0}, {(long int)nrecvs, Dim}),
-                KOKKOS_CLASS_LAMBDA(const size_type i, const size_t d) {
-                    std::memcpy(&(*(view.data() + i))[d],
-                                buffer_m.data() + (Dim * i + d) * size + readpos_m, size);
+                KOKKOS_LAMBDA(const size_type i, const size_t d) {
+                    std::memcpy(&(*(dst_ptr + i))[d], src_ptr + (Dim * i + d) * size + rp, size);
                 });
             Kokkos::fence();
             readpos_m += Dim * size * nrecvs;