New Particle Send/Recv Routine

.taplo.toml

@@ -0,0 +1,6 @@
+[formatting]
+  align_entries    = true
+  indent_tables    = true
+  indent_entries   = true
+  trailing_newline = true
+  align_comments   = true

CMakeLists.txt

@@ -101,23 +101,12 @@ if(${output})
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/adios2Config.cmake)
   find_or_fetch_dependency(adios2 FALSE)
   if(NOT DEFINED ENV{HDF5_ROOT})
-    set(USE_CUSTOM_HDF5 OFF)
     if(DEFINED ENV{CONDA_PREFIX})
       execute_process(COMMAND bash -c "conda list | grep \"hdf5\" -q"
                       RESULT_VARIABLE HDF5_INSTALLED)
       if(HDF5_INSTALLED EQUAL 0)
         set(HDF5_ROOT $ENV{CONDA_PREFIX})
-      else()
-        set(USE_CUSTOM_HDF5 ON)
       endif()
-    else()
-      set(USE_CUSTOM_HDF5 ON)
-    endif()
-    if(USE_CUSTOM_HDF5)
-      message(
-        FATAL_ERROR
-          "HDF5_ROOT is not set. Please set it to the root of the HDF5 installation"
-      )
     endif()
   endif()
   find_package(HDF5 REQUIRED)
@@ -134,6 +123,9 @@ link_libraries(${DEPENDENCIES})
 if(TESTS)
   # ---------------------------------- Tests --------------------------------- #
   include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/tests.cmake)
+elseif(BENCHMARK)
+  # ------------------------------ Benchmark --------------------------------- #
+  include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/benchmark.cmake)
 else()
   # ----------------------------------- GUI ---------------------------------- #
   if(${gui})

TASKLIST.md

@@ -3,3 +3,7 @@
 - [ ] removing temporary variables in interpolation
 - [ ] passing by value vs const ref in metric
 - [ ] return physical coords one-by-one instead of by passing full vector
+
+### Things to look into
+
+1. _h fields in mpi communication

benchmark/benchmark.cpp

@@ -0,0 +1,17 @@
+#include "global.h"
+
+#include <iostream>
+#include <stdexcept>
+
+auto main(int argc, char* argv[]) -> int {
+  ntt::GlobalInitialize(argc, argv);
+  try {
+    // ...
+  } catch (const std::exception& e) {
+    std::cerr << "Error: " << e.what() << std::endl;
+    GlobalFinalize();
+    return 1;
+  }
+  GlobalFinalize();
+  return 0;
+}

cmake/benchmark.cmake

@@ -0,0 +1,24 @@
+set(SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src)
+
+add_subdirectory(${SRC_DIR}/global ${CMAKE_CURRENT_BINARY_DIR}/global)
+add_subdirectory(${SRC_DIR}/metrics ${CMAKE_CURRENT_BINARY_DIR}/metrics)
+add_subdirectory(${SRC_DIR}/kernels ${CMAKE_CURRENT_BINARY_DIR}/kernels)
+add_subdirectory(${SRC_DIR}/archetypes ${CMAKE_CURRENT_BINARY_DIR}/archetypes)
+add_subdirectory(${SRC_DIR}/framework ${CMAKE_CURRENT_BINARY_DIR}/framework)
+
+if(${output})
+  add_subdirectory(${SRC_DIR}/output ${CMAKE_CURRENT_BINARY_DIR}/output)
+  add_subdirectory(${SRC_DIR}/checkpoint ${CMAKE_CURRENT_BINARY_DIR}/checkpoint)
+endif()
+
+set(exec benchmark.xc)
+set(src ${CMAKE_CURRENT_SOURCE_DIR}/benchmark/benchmark.cpp)
+
+add_executable(${exec} ${src})
+
+set(libs ntt_global ntt_metrics ntt_kernels ntt_archetypes ntt_framework)
+if(${output})
+  list(APPEND libs ntt_output ntt_checkpoint)
+endif()
+add_dependencies(${exec} ${libs})
+target_link_libraries(${exec} PRIVATE ${libs} stdc++fs)

dev/nix/adios2.nix

@@ -0,0 +1,67 @@
+{
+  pkgs ? import <nixpkgs> { },
+  hdf5 ? false,
+  mpi ? false,
+}:
+
+let
+  name = "adios2";
+  version = "2.10.2";
+  cmakeFlags = {
+    CMAKE_CXX_STANDARD = "17";
+    CMAKE_CXX_EXTENSIONS = "OFF";
+    CMAKE_POSITION_INDEPENDENT_CODE = "TRUE";
+    BUILD_SHARED_LIBS = "ON";
+    ADIOS2_USE_HDF5 = if hdf5 then "ON" else "OFF";
+    ADIOS2_USE_Python = "OFF";
+    ADIOS2_USE_Fortran = "OFF";
+    ADIOS2_USE_ZeroMQ = "OFF";
+    BUILD_TESTING = "OFF";
+    ADIOS2_BUILD_EXAMPLES = "OFF";
+    ADIOS2_USE_MPI = if mpi then "ON" else "OFF";
+    CMAKE_BUILD_TYPE = "Release";
+  } // (if !mpi then { ADIOS2_HAVE_HDF5_VOL = "OFF"; } else { });
+in
+pkgs.stdenv.mkDerivation {
+  pname = "${name}${if hdf5 then "-hdf5" else ""}${if mpi then "-mpi" else ""}";
+  version = "${version}";
+  src = pkgs.fetchgit {
+    url = "https://github.com/ornladios/ADIOS2/";
+    rev = "v${version}";
+    sha256 = "sha256-NVyw7xoPutXeUS87jjVv1YxJnwNGZAT4QfkBLzvQbwg=";
+  };
+
+  nativeBuildInputs = with pkgs; [
+    cmake
+    perl
+  ];
+
+  propagatedBuildInputs =
+    [
+      pkgs.gcc13
+    ]
+    ++ (if hdf5 then (if mpi then [ pkgs.hdf5-mpi ] else [ pkgs.hdf5 ]) else [ ])
+    ++ (if mpi then [ pkgs.openmpi ] else [ ]);
+
+  configurePhase = ''
+    cmake -B build $src ${
+      pkgs.lib.attrsets.foldlAttrs (
+        acc: key: value:
+        acc + " -D ${key}=${value}"
+      ) "" cmakeFlags
+    }
+  '';
+
+  buildPhase = ''
+    cmake --build build -j
+  '';
+
+  installPhase = ''
+    sed -i '/if(CMAKE_INSTALL_COMPONENT/,/^[[:space:]]&endif()$/d' build/cmake/install/post/cmake_install.cmake
+    cmake --install build --prefix $out
+    chmod +x build/cmake/install/post/generate-adios2-config.sh
+    sh build/cmake/install/post/generate-adios2-config.sh $out
+  '';
+
+  enableParallelBuilding = true;
+}

dev/nix/kokkos.nix

@@ -0,0 +1,64 @@
+{
+  pkgs ? import <nixpkgs> { },
+  arch ? "native",
+  gpu ? "none",
+}:
+
+let
+  gpuUpper = pkgs.lib.toUpper gpu;
+  name = "kokkos";
+  version = "4.5.01";
+  compilerPkgs = {
+    "HIP" = with pkgs.rocmPackages; [
+      rocm-core
+      clr
+      rocthrust
+      rocprim
+      rocminfo
+      rocm-smi
+    ];
+    "NONE" = [
+      pkgs.gcc13
+    ];
+  };
+  cmakeFlags = {
+    "HIP" = [
+      "-D CMAKE_C_COMPILER=hipcc"
+      "-D CMAKE_CXX_COMPILER=hipcc"
+    ];
+    "NONE" = [ ];
+  };
+  getArch =
+    _:
+    if gpu != "none" && arch == "native" then
+      throw "Please specify an architecture when the GPU support is enabled. Available architectures: https://kokkos.org/kokkos-core-wiki/keywords.html#architectures"
+    else
+      pkgs.lib.toUpper arch;
+
+in
+pkgs.stdenv.mkDerivation {
+  pname = "${name}";
+  version = "${version}";
+  src = pkgs.fetchgit {
+    url = "https://github.com/kokkos/kokkos/";
+    rev = "v${version}";
+    sha256 = "sha256-cI2p+6J+8BRV5fXTDxxHTfh6P5PeeLUiF73o5zVysHQ=";
+  };
+
+  nativeBuildInputs = with pkgs; [
+    cmake
+  ];
+
+  propagatedBuildInputs = compilerPkgs.${gpuUpper};
+
+  cmakeFlags = [
+    "-D CMAKE_CXX_STANDARD=17"
+    "-D CMAKE_CXX_EXTENSIONS=OFF"
+    "-D CMAKE_POSITION_INDEPENDENT_CODE=TRUE"
+    "-D Kokkos_ARCH_${getArch { }}=ON"
+    (if gpu != "none" then "-D Kokkos_ENABLE_${gpuUpper}=ON" else "")
+    "-D CMAKE_BUILD_TYPE=Release"
+  ] ++ cmakeFlags.${gpuUpper};
+
+  enableParallelBuilding = true;
+}

dev/nix/shell.nix

@@ -0,0 +1,48 @@
+{
+  pkgs ? import <nixpkgs> { },
+  mpi ? false,
+  hdf5 ? false,
+  gpu ? "none",
+  arch ? "native",
+}:
+
+let
+  name = "entity-dev";
+  adios2Pkg = (pkgs.callPackage ./adios2.nix { inherit pkgs mpi hdf5; });
+  kokkosPkg = (pkgs.callPackage ./kokkos.nix { inherit pkgs arch gpu; });
+in
+pkgs.mkShell {
+  name = "${name}-env";
+  nativeBuildInputs = with pkgs; [
+    zlib
+    cmake
+
+    clang-tools
+
+    adios2Pkg
+    kokkosPkg
+
+    python312
+    python312Packages.jupyter
+
+    cmake-format
+    neocmakelsp
+    black
+    pyright
+    taplo
+    vscode-langservers-extracted
+  ];
+
+  LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath ([
+    pkgs.stdenv.cc.cc
+    pkgs.zlib
+  ]);
+
+  shellHook = ''
+    BLUE='\033[0;34m'
+    NC='\033[0m'
+
+    echo ""
+    echo -e "${name} nix-shell activated"
+  '';
+}

input.example.toml

@@ -105,7 +105,7 @@
     #   @note: In spherical, bondaries in theta/phi are set automatically (only specify bc @ [rmin, rmax]) [["ATMOSPHERE", "ABSORB"]]
     #   @note: In GR, the horizon boundary is set automatically (only specify bc @ rmax): [["ABSORB"]]
     particles = ""
-    
+
     [grid.boundaries.absorb]
       # Size of the absorption layer in physical (code) units:
       #   @type: float
@@ -119,7 +119,7 @@
       coeff = ""
 
     [grid.boundaries.atmosphere]
-    # @required: if ATMOSPHERE is one of the boundaries
+      # @required: if ATMOSPHERE is one of the boundaries
       # Temperature of the atmosphere in units of m0 c^2
       #   @type: float
       temperature = ""
@@ -210,7 +210,7 @@
     #   @type: float: ~1
     #   @default: 1.0
     correction = ""
-    
+
     # @inferred:
     # - dt [= CFL * dx0]
     #     @brief: timestep duration
@@ -252,12 +252,11 @@
   #   @type: bool
   #   @default: false
   use_weights = ""
-  # Timesteps between particle re-sorting:
+  # Timesteps between particle re-sorting (removing dead particles):
   #   @type: unsigned int
   #   @default: 100
-  #   @note: When MPI is enable, particles are sorted every step.
-  #   @note: When `sort_interval` == 0, the sorting is disabled.
-  sort_interval = ""
+  #   @note: set to 0 to disable re-sorting
+  clear_interval = ""
 
   # @inferred:
   # - nspec