setup.py: Transition to scikit-build

Use scikit-build instead of vanilla low-level setuptools for
building. This should enable us shipping generated binaries.

CMakeLists.txt

@@ -1042,8 +1042,9 @@ endif()
 #
 message("")
 message("openPMD build configuration:")
-message("  library Version: ${openPMD_VERSION}")
+message("  Library Version: ${openPMD_VERSION}")
 message("  openPMD Standard: ${openPMD_STANDARD_VERSION}")
+message("  Generator: ${CMAKE_GENERATOR}")
 message("  C++ Compiler: ${CMAKE_CXX_COMPILER_ID} "
                         "${CMAKE_CXX_COMPILER_VERSION} "
                         "${CMAKE_CXX_COMPILER_WRAPPER}")

Dockerfile

@@ -26,7 +26,7 @@ RUN        yum check-update -y \
 #ENV        PATH=/opt/cmake/bin:${PATH}
 RUN        for PY_TARGET in ${PY_VERSIONS}; do \
                PY_BIN=/opt/python/cp${PY_TARGET:0:2}-cp${PY_TARGET}/bin/python \
-               && ${PY_BIN} -m pip install setuptools cmake; \
+               && ${PY_BIN} -m pip install setuptools cmake scikit-build ninja; \
            done;
 
 RUN        curl -sLo hdf5-1.10.5.tar.gz https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.10/hdf5-1.10.5/src/hdf5-1.10.5.tar.gz \
@@ -94,9 +94,10 @@ RUN        cd /opt/src; \
            for PY_TARGET in ${PY_VERSIONS}; do \
                PY_BIN=/opt/python/cp${PY_TARGET:0:2}-cp${PY_TARGET}/bin/python \
                && CMAKE_BIN="$(${PY_BIN} -m pip show cmake 2>/dev/null | grep Location | cut -d' ' -f2)/cmake/data/bin/" \
-               && PATH=${CMAKE_BIN}:${PATH} ${PY_BIN} setup.py bdist_wheel \
+               && PATH=${CMAKE_BIN}:${PATH} ${PY_BIN} \
+                  setup.py bdist_wheel -- -DCMAKE_INSTALL_LIBDIR=lib -- -j2 \
                && ${PY_BIN} setup.py clean --all \
-               && rm -rf openPMD_api.egg-info/; \
+               && rm -rf openPMD_api.egg-info/ _skbuild/; \
            done; \
            ls dist/
 
@@ -139,7 +140,7 @@ RUN        ls -hal /usr/local/lib/python3.7/dist-packages/
 # RUN        objdump -x /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so | grep RPATH
 RUN        ldd /usr/local/lib/python3.7/dist-packages/openpmd_api.cpython-37m-x86_64-linux-gnu.so
 RUN        python3 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
-#RUN        openpmd-ls --help
+RUN        /root/.local/bin/openpmd-ls --help
 #RUN        echo "* soft core 100000" >> /etc/security/limits.conf && \
 #           python3 -c "import openpmd_api as io"; \
 #           gdb -ex bt -c core
@@ -155,7 +156,9 @@ RUN        python3.8 --version \
            && curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py \
            && python3.8 get-pip.py \
            && python3.8 -m pip install openPMD_api-*-cp38-cp38-manylinux2010_x86_64.whl
+RUN        find / -name "openpmd*"
 RUN        python3.8 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
+RUN        /root/.local/bin/openpmd-ls --help
 
 # test in fresh env: Ubuntu:18.04 + Python 3.6
 FROM       ubuntu:18.04
@@ -181,8 +184,7 @@ RUN        python3 --version \
            && python3 -m pip install -U pip \
            && python3 -m pip install openPMD_api-*-cp35-cp35m-manylinux2010_x86_64.whl
 RUN        python3 -c "import openpmd_api as io; print(io.__version__); print(io.variants)"
-#RUN        openpmd-ls --help
-
+RUN        /root/.local/bin/openpmd-ls --help
 
 # copy binary artifacts (wheels)
 FROM       quay.io/pypa/manylinux2010_x86_64

pyproject.toml

@@ -1,2 +1,2 @@
 [build-system]
-requires = ["setuptools>38.6", "wheel", "cmake>=3.11.0,<4.0.0", "pybind11>=2.3.0,<3.0.0"]
+requires = ["setuptools>38.6", "wheel", "scikit-build>=0.8.0", "cmake>=3.11.0,<4.0.0", "pybind11>=2.3.0,<3.0.0", "ninja>=1.9.0"]

setup.py

@@ -4,128 +4,14 @@
 import platform
 import subprocess
 
-from setuptools import setup, Extension
-from setuptools.command.build_ext import build_ext
-from distutils.version import LooseVersion
-
-
-class CMakeExtension(Extension):
-    def __init__(self, name, sourcedir=''):
-        Extension.__init__(self, name, sources=[])
-        self.sourcedir = os.path.abspath(sourcedir)
-
-
-class CMakeBuild(build_ext):
-    def run(self):
-        try:
-            out = subprocess.check_output(['cmake', '--version'])
-        except OSError:
-            raise RuntimeError(
-                "CMake 3.11.0+ must be installed to build the following " +
-                "extensions: " +
-                ", ".join(e.name for e in self.extensions))
-
-        cmake_version = LooseVersion(re.search(
-            r'version\s*([\d.]+)',
-            out.decode()
-        ).group(1))
-        if cmake_version < '3.11.0':
-            raise RuntimeError("CMake >= 3.11.0 is required")
-
-        for ext in self.extensions:
-            self.build_extension(ext)
-
-    def build_extension(self, ext):
-        extdir = os.path.abspath(os.path.dirname(
-            self.get_ext_fullpath(ext.name)
-        ))
-        # required for auto-detection of auxiliary "native" libs
-        if not extdir.endswith(os.path.sep):
-            extdir += os.path.sep
-
-        cmake_args = [
-            '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
-            # '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=' + extdir,
-            '-DBUILD_CLI_TOOLS:BOOL=OFF',  # TODO: how to install properly?
-            '-DCMAKE_PYTHON_OUTPUT_DIRECTORY=' + extdir,
-            '-DPYTHON_EXECUTABLE=' + sys.executable,
-            # variants
-            '-DopenPMD_USE_MPI:BOOL=' + openPMD_USE_MPI,
-            # skip building tests & examples
-            '-DBUILD_TESTING:BOOL=' + BUILD_TESTING,
-            '-DBUILD_EXAMPLES:BOOL=' + BUILD_EXAMPLES,
-            # static/shared libs
-            '-DBUILD_SHARED_LIBS:BOOL=' + BUILD_SHARED_LIBS,
-            '-DHDF5_USE_STATIC_LIBRARIES:BOOL=' + HDF5_USE_STATIC_LIBRARIES,
-            '-DADIOS_USE_STATIC_LIBS:BOOL=' + ADIOS_USE_STATIC_LIBS,
-            # Unix: rpath to current dir when packaged
-            #       needed for shared (here non-default) builds and ADIOS1
-            #       wrapper libraries
-            '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON',
-            '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF',
-            # Windows: has no RPath concept, all `.dll`s must be in %PATH%
-            #          or same dir as calling executable
-        ]
-        if sys.platform == "darwin":
-            cmake_args.append('-DCMAKE_INSTALL_RPATH=@loader_path')
-        else:
-            # values: linux*, aix, freebsd, ...
-            #   just as well win32 & cygwin (although Windows has no RPaths)
-            cmake_args.append('-DCMAKE_INSTALL_RPATH=$ORIGIN')
-
-        cfg = 'Debug' if self.debug else 'Release'
-        build_args = ['--config', cfg]
-
-        if platform.system() == "Windows":
-            cmake_args += [
-                '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{}={}'.format(
-                    cfg.upper(),
-                    extdir
-                )
-                # TODO: how to install properly?
-                # '-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_{}={}'.format(
-                #     cfg.upper(),
-                #     extdir
-                # )
-            ]
-            if sys.maxsize > 2**32:
-                cmake_args += ['-A', 'x64']
-            build_args += ['--', '/m']
-        else:
-            cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
-            build_args += ['--', '-j2']
-
-        env = os.environ.copy()
-        env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(
-            env.get('CXXFLAGS', ''),
-            self.distribution.get_version()
-        )
-        if not os.path.exists(self.build_temp):
-            os.makedirs(self.build_temp)
-        subprocess.check_call(
-            ['cmake', ext.sourcedir] + cmake_args,
-            cwd=self.build_temp,
-            env=env
-        )
-        subprocess.check_call(
-            ['cmake', '--build', '.'] + build_args,
-            cwd=self.build_temp
-        )
+from skbuild import setup
 
 
 with open('./README.md', encoding='utf-8') as f:
     long_description = f.read()
 
-# Allow to control options via environment vars.
 # Work-around for https://github.com/pypa/setuptools/issues/1712
-# note: changed default for SHARED, MPI, TESTING and EXAMPLES
 openPMD_USE_MPI = os.environ.get('openPMD_USE_MPI', 'OFF')
-HDF5_USE_STATIC_LIBRARIES = os.environ.get('HDF5_USE_STATIC_LIBRARIES', 'OFF')
-ADIOS_USE_STATIC_LIBS = os.environ.get('ADIOS_USE_STATIC_LIBS', 'OFF')
-BUILD_SHARED_LIBS = os.environ.get('BUILD_SHARED_LIBS', 'OFF')
-BUILD_TESTING = os.environ.get('BUILD_TESTING', 'OFF')
-BUILD_EXAMPLES = os.environ.get('BUILD_EXAMPLES', 'OFF')
-
 # https://cmake.org/cmake/help/v3.0/command/if.html
 if openPMD_USE_MPI.upper() in ['1', 'ON', 'YES', 'TRUE', 'YES']:
     openPMD_USE_MPI = "ON"
@@ -139,7 +25,7 @@ def build_extension(self, ext):
         install_requires.append('mpi4py>=2.1.0')
 
 # keyword reference:
-#   https://packaging.python.org/guides/distributing-packages-using-setuptools
+#   https://scikit-build.readthedocs.io/en/latest/usage.html#setup-options
 setup(
     name='openPMD-api',
     # note PEP-440 syntax: x.y.zaN but x.y.z.devN
@@ -161,13 +47,44 @@ def build_extension(self, ext):
         'Source': 'https://github.com/openPMD/openPMD-api',
         'Tracker': 'https://github.com/openPMD/openPMD-api/issues',
     },
-    ext_modules=[CMakeExtension('openpmd_api')],
-    cmdclass=dict(build_ext=CMakeBuild),
-    # scripts=['openpmd-ls'],
     zip_safe=False,
     python_requires='>=3.5, <3.9',
     # tests_require=['pytest'],
     install_requires=install_requires,
+    cmake_minimum_required_version="3.11.0",
+    cmake_args=[
+        # note: changed default for MPI, TESTING and EXAMPLES
+        #    '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
+        #    '-DCMAKE_PYTHON_OUTPUT_DIRECTORY=' + extdir,
+        '-DPYTHON_EXECUTABLE=%s' % sys.executable,
+        # change variant search default
+        '-DopenPMD_USE_MPI:BOOL=%s' % openPMD_USE_MPI,
+        # skip building CLI tools, tests & examples
+        '-DBUILD_CLI_TOOLS:BOOL=OFF',  # FIXME: solve via entry points instead?
+        '-DBUILD_TESTING:BOOL=%s' % os.environ.get('BUILD_TESTING', 'OFF'),
+        '-DBUILD_EXAMPLES:BOOL=%s' % os.environ.get('BUILD_EXAMPLES', 'OFF'),
+        # static/shared libs
+        '-DBUILD_SHARED_LIBS:BOOL=%s' % os.environ.get('BUILD_SHARED_LIBS', 'OFF'),
+        '-DHDF5_USE_STATIC_LIBRARIES:BOOL=%s' % os.environ.get('HDF5_USE_STATIC_LIBRARIES', 'OFF'),
+        '-DADIOS_USE_STATIC_LIBS:BOOL=%s' % os.environ.get('ADIOS_USE_STATIC_LIBS', 'OFF'),
+        # Unix: rpath to current dir when packaged
+        #       needed for shared (here non-default) builds and ADIOS1
+        #       wrapper libraries
+        '-DCMAKE_BUILD_WITH_INSTALL_RPATH:BOOL=ON',
+        '-DCMAKE_INSTALL_RPATH_USE_LINK_PATH:BOOL=OFF',
+        # Windows: has no RPath concept, all `.dll`s must be in %PATH%
+        #          or same dir as calling executable
+        '-DCMAKE_INSTALL_RPATH={0}'.format(
+            #"@loader_path" if sys.platform == "darwin" else "$ORIGIN"),
+            # rpath to lib/: bin/../lib and pythonX.Z/site-packages/../..
+            "@loader_path/../lib;@loader_path/../.." if sys.platform == "darwin" else "$ORIGIN/../lib:$ORIGIN/../.."),
+    ],
+#    entry_points={
+#        'console_scripts': [
+#            'openpmd-ls = openpmd_api:ls'
+#        ]
+#    },
+    # cmake_languages=('C', 'CXX'),
     # we would like to use this mechanism, but pip / setuptools do not
     # influence the build and build_ext with it.
     # therefore, we use environment vars to control.
@@ -176,7 +93,7 @@ def build_extension(self, ext):
     #     'mpi': ['mpi4py>=2.1.0'],
     # },
     # cmdclass={'test': PyTest},
-    # platforms='any',
+    # platforms=['any'],
     classifiers=[
         'Development Status :: 3 - Alpha',
         'Natural Language :: English',