Release 0.14.3 backports

.appveyor.yml

@@ -1,33 +1,19 @@
 environment:
   matrix:
-    - TARGET_ARCH: x86
-      CONDA_PY: 36
-      CONDA_INSTALL_LOCN: C:\\Miniconda36
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      platform: x86
-      SHARED: ON
-
     - TARGET_ARCH: x64
-      CONDA_PY: 37
+      CONDA_PY: 3.7
       CONDA_INSTALL_LOCN: C:\\Miniconda37-x64
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       platform: x64
-      SHARED: ON
+      SHARED: OFF
 
     - TARGET_ARCH: x86
-      CONDA_PY: 37
+      CONDA_PY: 3.7
       CONDA_INSTALL_LOCN: C:\\Miniconda37
       APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
       platform: x86
       SHARED: ON
 
-    - TARGET_ARCH: x64
-      CONDA_PY: 36
-      CONDA_INSTALL_LOCN: C:\\Miniconda36-x64
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      platform: x64
-      SHARED: OFF
-
 configuration:
 #  - Debug
   - Release
@@ -91,7 +77,6 @@ install:
 
   # Add path, activate `conda` and update conda.
   - cmd: call %CONDA_INSTALL_LOCN%\Scripts\activate.bat
-  - cmd: conda update --yes --quiet conda
 
   - cmd: set PYTHONUNBUFFERED=1
 
@@ -101,9 +86,9 @@ install:
   - cmd: conda config --append channels conda-forge
 
   # Configure the VM.
-  - cmd: conda install -n root --quiet --yes numpy>=1.15 cmake hdf5
-  # ADIOS2 build only for 64bit Windows and Python 3.6+
-  - cmd: if "%TARGET_ARCH%"=="x64" if %CONDA_PY% GEQ 36 conda install -n root --quiet --yes adios2
+  - cmd: conda install -n root --quiet --yes numpy cmake hdf5 python=%CONDA_PY%
+  # ADIOS2 build only for 64bit Windows
+  - cmd: if "%TARGET_ARCH%"=="x64" conda install -n root --quiet --yes adios2 python=%CONDA_PY%
 
 before_build:
   - cmd: cd C:\projects\openpmd-api
@@ -121,12 +106,12 @@ before_build:
 #  - cmd: if "%TARGET_ARCH%"=="x64" "C:\Program Files (x86)\Microsoft Visual Studio 15.9\VC\vcvarsall.bat" amd64
 
   # CMake configure
-  - cmd: cmake -G "%OPENPMD_CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".."
+  - cmd: cmake -G "%OPENPMD_CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".."
 
 build_script:
-  - cmd: cmake --build . --config %CONFIGURATION%
+  - cmd: cmake --build . --config %CONFIGURATION% -j 2
   - cmd: cmake --build . --config %CONFIGURATION% --target install
 
 test_script:
   - cmd: ctest -V -C %CONFIGURATION%
-  - cmd: python -c "import openpmd_api; print(openpmd_api.__version__); print(openpmd_api.variants)"
+  - cmd: call %CONDA_INSTALL_LOCN%\python.exe -c "import openpmd_api; print(openpmd_api.__version__); print(openpmd_api.variants)"

.github/workflows/windows.yml

@@ -0,0 +1,57 @@
+name: 🪟 Windows
+
+on: [push, pull_request]
+
+concurrency:
+  group: ${{ github.ref }}-${{ github.head_ref }}-windows
+  cancel-in-progress: true
+
+jobs:
+  build_win_msvc:
+    name: MSVC w/o MPI
+    runs-on: windows-latest
+    if: github.event.pull_request.draft == false
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build & Install
+      run: |
+        python3.exe -m pip install --upgrade pip
+        python3.exe -m pip install --upgrade numpy
+
+        pwsh "share\openPMD\download_samples.ps1" build
+        cmake -S . -B build               `
+              -DCMAKE_BUILD_TYPE=Debug    `
+              -DopenPMD_USE_MPI=OFF
+        cmake --build build --config Debug --parallel 2
+        cmake --build build --config Debug --target install
+
+# add before install, and fix Python path:
+# ctest --test-dir build -C Debug --output-on-failure
+
+
+  build_win_clang:
+    name: Clang w/o MPI
+    runs-on: windows-2019
+    if: github.event.pull_request.draft == false
+    steps:
+    - uses: actions/checkout@v2
+    - uses: seanmiddleditch/gha-setup-ninja@master
+    - name: Build & Install
+      shell: cmd
+      run: |
+        python3.exe -m pip install --upgrade pip
+        python3.exe -m pip install --upgrade numpy
+
+        call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\vc\Auxiliary\build\vcvarsall.bat" x64
+        pwsh "share\openPMD\download_samples.ps1" build
+        cmake -S . -B build   ^
+              -G "Ninja"      ^
+              -DCMAKE_C_COMPILER=clang-cl   ^
+              -DCMAKE_CXX_COMPILER=clang-cl ^
+              -DCMAKE_BUILD_TYPE=Release    ^
+              -DopenPMD_USE_MPI=OFF
+        cmake --build build --config Release --parallel 2
+        cmake --build build --config Debug --target install
+
+# add before install, and fix Python path:
+# ctest --test-dir build -C Debug --output-on-failure

.rodare.json

@@ -108,6 +108,12 @@
             "name": "Schnetter, Erik",
             "orcid": "0000-0002-4518-9017",
             "type": "Other"
+        },
+        {
+            "affiliation": "Lawrence Berkeley National Laboratory",
+            "name": "Bez, Jean Luca",
+            "orcid": "0000-0002-3915-1135",
+            "type": "Other"          
         }
     ],
     "title": "C++ & Python API for Scientific I/O with openPMD",

CHANGELOG.rst

@@ -3,6 +3,49 @@
 Changelog
 =========
 
+0.14.2
+------
+**Date:** 2021-11-03
+
+Read Bugs, C++17 Mixing and HDF5 Performance
+
+This release makes reads more robust by fixing small API, file-based parsing and test bugs.
+Building the library in C++14 and using it in C++17 will not result in incompatible ABIs anymore.
+HDF5 1.10.1+ performance was improved significantly.
+
+Changes to "0.14.2"
+^^^^^^^^^^^^^^^^^^^
+
+Bug Fixes
+"""""""""
+
+- read:
+
+  - allow inconsistent zero pads #1118
+  - time/dt also in long double #1096
+- test 8b - bench read parallel:
+
+  - support variable encoding #1131
+  - block located at top left corner was mistaken to read a block in the center #1131
+- CI (AppVeyor): Python executable #1127
+- C++17 mixing: remember ``<variant>`` implementation #1128
+- support NVCC + C++17 #1103
+- avoid object slicing when deriving from ``Series`` class #1107
+- executables: ``CXX_STANDARD``/``EXTENSIONS`` #1102
+
+Other
+"""""
+
+- HDF5 I/O optimizations #1129 #1132 #1133
+- libfabric 1.6+: Document SST Work-Arounds #1134
+- OpenMPI: Document ``OMPI_MCA_io`` Control #1114
+- HDF5: Document ``HDF5_USE_FILE_LOCKING`` #1106
+- Lazy parsing: Make findable in docs and use in ``openpmd-ls`` #1111
+- Docs: More Locations ``-DPython_EXECUTABLE`` #1104
+- Spack: No More ``load -r`` #1125
+- ``openPMD.hpp``: include auxiliary ``StringManip`` #1124
+
+
 0.14.2
 ------
 **Date:** 2021-08-17

CITATION.cff

@@ -25,7 +25,7 @@ contact:
   orcid: https://orcid.org/0000-0003-1943-7141
   email: axelhuebl@lbl.gov
 title: "openPMD-api: C++ & Python API for Scientific I/O with openPMD"
-version: 0.14.2
+version: 0.14.3
 repository-code: https://github.com/openPMD/openPMD-api
 doi: 10.14278/rodare.27
 license: LGPL-3.0-or-later

CMakeLists.txt

@@ -2,7 +2,7 @@
 #
 cmake_minimum_required(VERSION 3.15.0)
 
-project(openPMD VERSION 0.14.2) # LANGUAGES CXX
+project(openPMD VERSION 0.14.3) # LANGUAGES CXX
 
 # the openPMD "markup"/"schema" standard version
 set(openPMD_STANDARD_VERSION 1.1.0)
@@ -453,26 +453,36 @@ target_include_directories(openPMD PUBLIC
     $<INSTALL_INTERFACE:include>
 )
 
+# is this a C++17 compile? If yes, skip MPark.Variant
+try_compile(openPMD_HAS_CXX17
+    ${openPMD_BINARY_DIR}/try_variant
+    ${openPMD_SOURCE_DIR}/cmake/try_variant.cpp
+    CXX_STANDARD_REQUIRED ON
+    CXX_EXTENSIONS OFF
+)
+message(STATUS "<variant> supported (C++17 or newer): ${openPMD_HAS_CXX17}")
+
 # C++11 std::variant (C++17 stdlib preview)
-# TODO not needed with C++17 compiler
-add_library(openPMD::thirdparty::mpark_variant INTERFACE IMPORTED)
-set(openPMD_PC_EXTRA_INCLUDE "")
-if(openPMD_USE_INTERNAL_VARIANT)
-    target_include_directories(openPMD::thirdparty::mpark_variant SYSTEM INTERFACE
-        $<BUILD_INTERFACE:${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/variant/include>
-    )
-    message(STATUS "MPark.Variant: Using INTERNAL version '1.4.0'")
-else()
-    find_package(mpark_variant 1.3.0 REQUIRED) # TODO: we want 1.4.1+
-    target_link_libraries(openPMD::thirdparty::mpark_variant
-        INTERFACE mpark_variant)
-    get_target_property(EXTERNAL_MPARK_INCLUDE mpark_variant INTERFACE_INCLUDE_DIRECTORIES)
-    if(openPMD_HAVE_PKGCONFIG AND EXTERNAL_MPARK_INCLUDE)
-        set(openPMD_PC_EXTRA_INCLUDE "-I${EXTERNAL_MPARK_INCLUDE}")
+if(NOT openPMD_HAS_CXX17)
+    add_library(openPMD::thirdparty::mpark_variant INTERFACE IMPORTED)
+    set(openPMD_PC_EXTRA_INCLUDE "")
+    if(openPMD_USE_INTERNAL_VARIANT)
+        target_include_directories(openPMD::thirdparty::mpark_variant SYSTEM INTERFACE
+            $<BUILD_INTERFACE:${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/variant/include>
+        )
+        message(STATUS "MPark.Variant: Using INTERNAL version '1.4.0'")
+    else()
+        find_package(mpark_variant 1.3.0 REQUIRED) # TODO: we want 1.4.1+
+        target_link_libraries(openPMD::thirdparty::mpark_variant
+            INTERFACE mpark_variant)
+        get_target_property(EXTERNAL_MPARK_INCLUDE mpark_variant INTERFACE_INCLUDE_DIRECTORIES)
+        if(openPMD_HAVE_PKGCONFIG AND EXTERNAL_MPARK_INCLUDE)
+            set(openPMD_PC_EXTRA_INCLUDE "-I${EXTERNAL_MPARK_INCLUDE}")
+        endif()
+        message(STATUS "MPark.Variant: Found version '${mpark_variant_VERSION}'")
     endif()
-    message(STATUS "MPark.Variant: Found version '${mpark_variant_VERSION}'")
+    target_link_libraries(openPMD PUBLIC openPMD::thirdparty::mpark_variant)
 endif()
-target_link_libraries(openPMD PUBLIC openPMD::thirdparty::mpark_variant)
 
 # Catch2 for unit tests
 if(openPMD_BUILD_TESTING)
@@ -518,8 +528,10 @@ if(openPMD_HAVE_ADIOS1)
     )
     target_compile_options(openPMD.ADIOS1.Serial PUBLIC ${_msvc_options})
     target_compile_options(openPMD.ADIOS1.Parallel PUBLIC ${_msvc_options})
-    target_link_libraries(openPMD.ADIOS1.Serial PUBLIC openPMD::thirdparty::mpark_variant)
-    target_link_libraries(openPMD.ADIOS1.Parallel PUBLIC openPMD::thirdparty::mpark_variant)
+    if(NOT openPMD_HAS_CXX17)
+        target_link_libraries(openPMD.ADIOS1.Serial PUBLIC openPMD::thirdparty::mpark_variant)
+        target_link_libraries(openPMD.ADIOS1.Parallel PUBLIC openPMD::thirdparty::mpark_variant)
+    endif()
 
     target_include_directories(openPMD.ADIOS1.Serial PRIVATE
         ${openPMD_SOURCE_DIR}/include ${openPMD_BINARY_DIR}/include)
@@ -811,13 +823,22 @@ if(openPMD_BUILD_TESTING)
         else()
             target_link_libraries(${testname}Tests PRIVATE CatchMain)
         endif()
+
+        set_target_properties(${testname}Tests PROPERTIES
+            CXX_EXTENSIONS OFF
+            CXX_STANDARD_REQUIRED ON
+        )
     endforeach()
 endif()
 
 if(openPMD_BUILD_CLI_TOOLS)
     foreach(toolname ${openPMD_CLI_TOOL_NAMES})
         add_executable(openpmd-${toolname} src/cli/${toolname}.cpp)
         target_link_libraries(openpmd-${toolname} PRIVATE openPMD)
+        set_target_properties(openpmd-${toolname} PROPERTIES
+            CXX_EXTENSIONS OFF
+            CXX_STANDARD_REQUIRED ON
+        )
     endforeach()
 endif()
 
@@ -827,10 +848,18 @@ if(openPMD_BUILD_EXAMPLES)
             if(openPMD_HAVE_MPI)
                 add_executable(${examplename} examples/${examplename}.cpp)
                 target_link_libraries(${examplename} PRIVATE openPMD)
+                set_target_properties(${examplename} PROPERTIES
+                    CXX_EXTENSIONS OFF
+                    CXX_STANDARD_REQUIRED ON
+                )
             endif()
         else()
             add_executable(${examplename} examples/${examplename}.cpp)
             target_link_libraries(${examplename} PRIVATE openPMD)
+            set_target_properties(${examplename} PROPERTIES
+                CXX_EXTENSIONS OFF
+                CXX_STANDARD_REQUIRED ON
+            )
         endif()
     endforeach()
 endif()
@@ -983,7 +1012,7 @@ if(openPMD_INSTALL)
     )
     # install third-party libraries
     # TODO not needed with C++17 compiler
-    if(openPMD_USE_INTERNAL_VARIANT)
+    if(NOT openPMD_HAS_CXX17 AND openPMD_USE_INTERNAL_VARIANT)
         install(DIRECTORY "${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/variant/include/mpark"
             DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
         )
@@ -1216,7 +1245,8 @@ if(openPMD_BUILD_TESTING)
         endforeach()
 
         # openpmd-pipe (python) test
-        if( openPMD_HAVE_HDF5
+        if( NOT WIN32
+            AND openPMD_HAVE_HDF5
             AND (openPMD_HAVE_ADIOS2 OR openPMD_HAVE_ADIOS1)
             AND EXAMPLE_DATA_FOUND
         )
@@ -1345,7 +1375,11 @@ if(openPMD_INSTALL)
     endif()
     message("")
     message("  Additionally, install following third party libraries:")
-    message("    MPark.Variant: ${openPMD_USE_INTERNAL_VARIANT}")
+    if(openPMD_HAS_CXX17)
+        message("    MPark.Variant: OFF")
+    else()
+        message("    MPark.Variant: ${openPMD_USE_INTERNAL_VARIANT}")
+    endif()
 else()
     message("  Installation: OFF")
 endif()

README.md

@@ -148,7 +148,7 @@ Choose *one* of the install methods below to get started:
 ```bash
 # optional:               +python +adios1 -adios2 -hdf5 -mpi
 spack install openpmd-api
-spack load -r openpmd-api
+spack load openpmd-api
 ```
 
 ### [Conda](https://conda.io)
@@ -232,7 +232,7 @@ cd openPMD-api-build
 # for options append:
 #   -DopenPMD_USE_...=...
 # e.g. for python support add:
-#   -DopenPMD_USE_PYTHON=ON
+#   -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE=$(which python3)
 cmake ../openPMD-api
 
 cmake --build .
@@ -412,6 +412,8 @@ Further thanks go to improvements and contributions from:
   Dask guidance & reviews
 * [Erik Schnetter (PITP)](https://github.com/eschnett):
   C++ API bug fixes
+* [Jean Luca Bez (LBNL)](https://github.com/jeanbez):
+  HDF5 performance tuning
 
 ### Grants