Python packaging for PyNEST-NG

.github/workflows/build_nest_wheel.yml

@@ -0,0 +1,100 @@
+name: Build NEST simulator wheels
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [macos-13, macos-14, ubuntu-24.04]        
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.22.0
+        with:
+          package-dir: .
+          output-dir: wheelhouse
+          config-file: "{package}/pyproject.toml"
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Build sdist
+        run: pipx run build --sdist
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-sdist
+          path: dist/*.tar.gz
+
+  upload_pypi:
+    name: Upload wheels to pypi
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    environment: testpypi
+    permissions:
+      id-token: write
+    # if: github.event_name == 'release' && github.event.action == 'published'
+    # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this)
+    # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          # unpacks all CIBW artifacts into dist/
+          pattern: cibw-*
+          path: dist
+          merge-multiple: true
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          attestations: false
+          verbose: true
+          # only for test pypi
+          repository-url: https://test.pypi.org/legacy/
+
+  test_uploaded_package:
+    name: Test uploaded package in virtualenv
+    needs: [upload_pypi]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install system dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y python3 python3-pip python3-virtualenv time openmpi-bin openmpi-common libopenmpi-dev
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Create and activate virtualenv
+        run: |
+          python -m venv .env
+          source .env/bin/activate
+          python -m pip install --upgrade pip
+
+      - name: Install package from TestPyPI
+        run: |
+          source .env/bin/activate
+          python3 -m pip --no-cache-dir install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple pynest-ng
+
+      - name: Test import
+        run: |
+          source .env/bin/activate
+          python -c "import nest; print(nest.__version__)"
+          pip install pytest pytest-timeout pytest-xdist pytest-pylint pytest-mypy pytest-cov data-science-types terminaltables pycodestyle pydocstyle rstcheck mypy>=0.8 junitparser>=2 clang-format==17.0.4 attrs
+          # bash $($VIRTUAL_ENV/bin/python3 -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')/share/nest/testsuite/do_tests.sh --with-python=$VIRTUAL_ENV/bin/python3 --prefix=$($VIRTUAL_ENV/bin/python3 -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')
+
+

CMakeLists.txt

@@ -101,7 +101,11 @@ set( with-full-logging OFF CACHE STRING "Write debug output to 'dump_<num_ranks>
 # GNUInstallDirs (included after calling nest_process_with_python()), but the
 # CMAKE_INSTALL_DATADIR is usually just CMAKE_INSTALL_DATAROOTDIR
 # and we want it to be CMAKE_INSTALL_DATAROOTDIR/PROJECT_NAME
-set( CMAKE_INSTALL_DATADIR "share/${PROJECT_NAME}" CACHE STRING "Relative directory, where NEST installs its data (share/nest)" )
+if(NOT "${SKBUILD}" STREQUAL "") 
+  set( CMAKE_INSTALL_DATADIR "nest/share/${PROJECT_NAME}" CACHE STRING "Relative directory, where NEST installs its data (share/nest)" )
+else()
+  set( CMAKE_INSTALL_DATADIR "share/${PROJECT_NAME}" CACHE STRING "Relative directory, where NEST installs its data (share/nest)" )
+endif()
 
 ################################################################################
 ##################           Find utility programs            ##################
@@ -348,6 +352,9 @@ configure_file(
 ################################################################################
 ##################            Install Extra Files             ##################
 ################################################################################
+if(NOT "${SKBUILD}" STREQUAL "") 
+  set( CMAKE_INSTALL_DOCDIR "nest/doc")
+endif()
 
 install( FILES LICENSE README.md
     DESTINATION ${CMAKE_INSTALL_DOCDIR}

bin/CMakeLists.txt

@@ -17,10 +17,22 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-install( PROGRAMS
-    nest-server
-    nest-server-mpi
-    ${PROJECT_BINARY_DIR}/bin/nest-config
-    ${PROJECT_BINARY_DIR}/bin/nest_vars.sh
-    DESTINATION ${CMAKE_INSTALL_BINDIR}
-    )
+if(NOT "${SKBUILD}" STREQUAL "") 
+  install( PROGRAMS
+      nest-server
+      nest-server-mpi
+      ${PROJECT_BINARY_DIR}/bin/nest-config
+      ${PROJECT_BINARY_DIR}/bin/nest_vars.sh
+      DESTINATION "../bin"
+      )
+else()
+  install( PROGRAMS
+      nest-server
+      nest-server-mpi
+      ${PROJECT_BINARY_DIR}/bin/nest-config
+      ${PROJECT_BINARY_DIR}/bin/nest_vars.sh
+      DESTINATION ${CMAKE_INSTALL_BINDIR}
+      )
+endif()
+
+

libnestutil/CMakeLists.txt

@@ -57,6 +57,10 @@ target_include_directories(nestutil PRIVATE
 
 FILTER_HEADERS("${nestutil_sources}" install_headers)
 
+if(NOT "${SKBUILD}" STREQUAL "") 
+  set( CMAKE_INSTALL_INCLUDEDIR "nest/include")
+endif()
+
 install(FILES ${install_headers} ${PROJECT_BINARY_DIR}/libnestutil/config.h
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nest
 )

models/CMakeLists.txt

@@ -48,6 +48,12 @@ target_include_directories(models PRIVATE
     ${PROJECT_SOURCE_DIR}/nestkernel
 )
 
+# Install the models headers
+if(NOT "${SKBUILD}" STREQUAL "") 
+  set( CMAKE_INSTALL_INCLUDEDIR "nest/include")
+endif()
+
+
 FILTER_HEADERS("${models_sources}" install_headers)
 install(FILES ${install_headers}
     DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nest)

nestkernel/CMakeLists.txt

@@ -161,6 +161,11 @@ target_include_directories( nestkernel PRIVATE
 )
 
 FILTER_HEADERS( "${nestkernel_sources}" install_headers )
+
+if(NOT "${SKBUILD}" STREQUAL "") 
+  set( CMAKE_INSTALL_INCLUDEDIR "nest/include")
+endif()
+
 install( FILES ${install_headers}
   DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/nest
 )

pynest/CMakeLists.txt

@@ -63,15 +63,24 @@ if ( HAVE_PYTHON )
   target_compile_definitions( nestkernel_api PRIVATE
     -D_IS_PYNEST
   )
-
-  install( DIRECTORY nest/ ${PROJECT_BINARY_DIR}/pynest/nest/
-    DESTINATION ${CMAKE_INSTALL_PREFIX}/${PYEXECDIR}/nest
-    PATTERN "versionchecker.py.in" EXCLUDE
-  )
-
-  install( DIRECTORY examples/
+  # TODO PYNEST NG: Do changes only for the new pip installation
+  if(NOT "${SKBUILD}" STREQUAL "")    
+    install( DIRECTORY nest/ ${PROJECT_BINARY_DIR}/pynest/nest/
+      DESTINATION ./nest
+      PATTERN "versionchecker.py.in" EXCLUDE
+    )
+    install( DIRECTORY examples/
+      DESTINATION ./nest/doc/examples/pynest
+    )
+    install( TARGETS nestkernel_api DESTINATION ./nest )
+  else()
+    install( DIRECTORY nest/ ${PROJECT_BINARY_DIR}/pynest/nest/
+      DESTINATION ${CMAKE_INSTALL_PREFIX}/${PYEXECDIR}/nest
+      PATTERN "versionchecker.py.in" EXCLUDE
+    )
+    install( DIRECTORY examples/
       DESTINATION ${CMAKE_INSTALL_DOCDIR}/examples/pynest
-  )
-
-  install( TARGETS nestkernel_api DESTINATION ${PYEXECDIR}/nest/ )
+    )
+    install( TARGETS nestkernel_api DESTINATION ${PYEXECDIR}/nest/ )
+  endif()
 endif ()

pynest/examples/run_examples.sh

@@ -19,12 +19,6 @@
 # You should have received a copy of the GNU General Public License
 # along with NEST.  If not, see <http://www.gnu.org/licenses/>.
 
-NEST_CMD="$(which nest)"
-if [ $? != 0 ] ; then
-    echo "ERROR: command 'nest' not found. Please make sure PATH is set correctly"
-    echo "       by sourcing the script nest_vars.sh from your NEST installation."
-    exit 1
-fi
 
 python3 -c "import nest" >/dev/null 2>&1
 if [ $? != 0 ] ; then
@@ -43,11 +37,6 @@ if [ "${#}" -eq 0 ]; then
     # Find all examples that have a line containing "autorun=true"
     # The examples can be found in subdirectory nest and in the
     # examples installation path.
-    if [ -d "nest/" ] ; then
-        EXAMPLES="$(grep -rl --include=\*\.sli 'autorun=true' nest/)"
-    else
-        EXAMPLES="$(grep -rl --include=\*\.sli 'autorun=true' examples/)"
-    fi
     EXAMPLES+=" $(find . -name '*.py')"
 else
     EXAMPLES+=${@}
@@ -77,9 +66,7 @@ for i in $EXAMPLES; do
 
     ext="$(echo "$example" | cut -d. -f2)"
 
-    if [ $ext = sli ] ; then
-        runner=nest
-    elif [ $ext = py ] ; then
+    if [ $ext = py ] ; then
         runner=python3
     fi