Merge pull request #1660 from IntelPython/feature/pyproject

Metadata improvements

Author: oleksandr-pavlyk

.github/workflows/generate-coverage.yaml

@@ -92,7 +92,7 @@ jobs:
       - name: Install dpctl dependencies
         shell: bash -l {0}
         run: |
-          pip install numpy"<1.26.0" cython setuptools pytest pytest-cov scikit-build cmake coverage[toml]
+          pip install numpy"<1.26.0" cython setuptools pytest pytest-cov scikit-build cmake coverage[toml] versioneer[toml]==0.29
 
       - name: Build dpctl with coverage
         shell: bash -l {0}

.github/workflows/generate-docs.yml

@@ -56,7 +56,8 @@ jobs:
         run: |
           pip install numpy cython setuptools scikit-build cmake sphinx"<7.2" pydot graphviz furo \
                       sphinxcontrib-programoutput sphinxcontrib-googleanalytics sphinx-design \
-                      sphinxcontrib-jsmath sphinx-copybutton sphinxcontrib-spelling
+                      sphinxcontrib-jsmath sphinx-copybutton sphinxcontrib-spelling \
+                      versioneer[toml]==0.29
       - name: Checkout repo
         uses: actions/checkout@v4.1.7
         with:

.github/workflows/os-llvm-sycl-build.yml

@@ -110,7 +110,7 @@ jobs:
       - name: Install dpctl dependencies
         shell: bash -l {0}
         run: |
-          pip install numpy"<1.26.0" cython setuptools pytest scikit-build cmake ninja
+          pip install numpy"<1.26.0" cython setuptools pytest scikit-build cmake ninja versioneer[toml]==0.29
 
       - name: Checkout repo
         uses: actions/checkout@v4.1.7
@@ -164,4 +164,6 @@ jobs:
           SYCL_CACHE_PERSISTENT: 1
         run: |
           source set_allvars.sh
-          python -m pytest -v dpctl/tests
+          # Skip the test that checks if there is only one hard
+          # copy of DPCTLSyclInterface library
+          python -m pytest -v dpctl/tests --no-sycl-interface-test

.pre-commit-config.yaml

@@ -37,3 +37,8 @@ repos:
     hooks:
     -   id: clang-format
         args: ["-i"]
+-   repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks
+    rev: v2.12.0
+    hooks:
+    -   id: pretty-format-toml
+        args: [--autofix]

CMakeLists.txt

@@ -71,6 +71,7 @@ FetchContent_Declare(
     pybind11
     URL https://github.com/pybind/pybind11/archive/refs/tags/v2.12.0.tar.gz
     URL_HASH SHA256=bf8f242abd1abcd375d516a7067490fb71abd79519a282d22b6e4d19282185a7
+    FIND_PACKAGE_ARGS NAMES pybind11
 )
 FetchContent_MakeAvailable(pybind11)
 

conda-recipe/bld.bat

@@ -4,11 +4,10 @@ set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%"
 set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%"
 
 "%PYTHON%" setup.py clean --all
-set "SKBUILD_ARGS=-G Ninja -- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icx -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
 
 REM Overriding IPO is useful for building in resources constrained VMs (public CI)
 if DEFINED OVERRIDE_INTEL_IPO (
-   set "SKBUILD_ARGS=%SKBUILD_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE"
+  set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE"
 )
 
 FOR %%V IN (14.0.0 14 15.0.0 15 16.0.0 16 17.0.0 17) DO @(
@@ -32,20 +31,39 @@ if EXIST "%PLATFORM_DIR%" (
   if errorlevel 1 exit 1
 )
 
+set "CC=icx"
+set "CXX=icx"
+
+set "CMAKE_GENERATOR=Ninja"
+:: Make CMake verbose
+set "VERBOSE=1"
+
+set "CMAKE_ARGS=%CMAKE_ARGS% -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=%PREFIX:\=/%/Library/include/level_zero"
+
+%PYTHON% -m build -w -n -x
+if %ERRORLEVEL% neq 0 exit 1
+
+:: `pip install dist\numpy*.whl` does not work on windows,
+:: so use a loop; there's only one wheel in dist/ anyway
+for /f %%f in ('dir /b /S .\dist') do (
+    %PYTHON% -m wheel tags --remove --build %GIT_DESCRIBE_NUMBER% %%f
+    if %ERRORLEVEL% neq 0 exit 1
+)
+
+:: wheel file was renamed
+for /f %%f in ('dir /b /S .\dist') do (
+    %PYTHON% -m pip install %%f ^
+      --no-build-isolation ^
+      --no-deps ^
+      --only-binary :all: ^
+      --no-index ^
+      --prefix %PREFIX% ^
+      -vv
+    if %ERRORLEVEL% neq 0 exit 1
+)
+
+:: Copy wheel package
 if NOT "%WHEELS_OUTPUT_FOLDER%"=="" (
-    rem Install and assemble wheel package from the build bits
-    "%PYTHON%" setup.py install bdist_wheel --build-number %GIT_DESCRIBE_NUMBER% %SKBUILD_ARGS%
-    if errorlevel 1 exit 1
     copy dist\dpctl*.whl %WHEELS_OUTPUT_FOLDER%
     if errorlevel 1 exit 1
-) ELSE (
-    rem Only install
-    "%PYTHON%" setup.py install %SKBUILD_ARGS%
-    if errorlevel 1 exit 1
-)
-
-if EXIST "%PLATFORM_DIR%" (
-  rem copy back
-  copy /Y "%FN%" "%PLATFORM_DIR%\%FN%"
-  if errorlevel 1 exit 1
 )

conda-recipe/build.sh

@@ -8,21 +8,69 @@ echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot
 export ICPXCFG="$(pwd)/icpx_for_conda.cfg"
 export ICXCFG="$(pwd)/icpx_for_conda.cfg"
 
+read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \
+    | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')"
+
 if [ -e "_skbuild" ]; then
     ${PYTHON} setup.py clean --all
 fi
-export CMAKE_GENERATOR="Ninja"
-SKBUILD_ARGS="-- -DCMAKE_C_COMPILER:PATH=icx -DCMAKE_CXX_COMPILER:PATH=icpx -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON"
-echo "${PYTHON} setup.py install ${SKBUILD_ARGS}"
-
-if [ -n "${WHEELS_OUTPUT_FOLDER}" ]; then
-    # Install packages and assemble wheel package from built bits
-    WHEELS_BUILD_ARGS="-p manylinux_2_28_x86_64 --build-number ${GIT_DESCRIBE_NUMBER}"
-    ${PYTHON} setup.py install bdist_wheel ${WHEELS_BUILD_ARGS} ${SKBUILD_ARGS}
-    cp dist/dpctl*.whl ${WHEELS_OUTPUT_FOLDER}
-else
-    # Perform regular install
-    ${PYTHON} setup.py install ${SKBUILD_ARGS}
+
+export CC=icx
+export CXX=icpx
+
+export CMAKE_GENERATOR=Ninja
+# Make CMake verbose
+export VERBOSE=1
+
+CMAKE_ARGS="${CMAKE_ARGS} -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=${PREFIX}/include/level_zero"
+
+# -wnx flags mean: --wheel --no-isolation --skip-dependency-check
+${PYTHON} -m build -w -n -x
+
+pushd dist
+${PYTHON} -m wheel unpack -d dpctl_wheel dpctl*.whl
+export lib_name=libDPCTLSyclInterface
+export so_full_path=$(find dpctl_wheel -regextype posix-extended -regex "^.*${lib_name}\.so")
+export sox_full_path=$(find dpctl_wheel -regextype posix-extended -regex "^.*${lib_name}\.so\.[0-9]*$")
+export soxxx_full_path=$(find dpctl_wheel -regextype posix-extended -regex "^.*${lib_name}\.so\.[0-9]*\.[0-9]*$")
+
+rm -rf ${so_full_path} ${soxxx_full_path}
+
+export so_name=$(basename ${so_full_path})
+export sox_name=$(basename ${sox_full_path})
+export soxxx_name=$(basename ${soxxx_full_path})
+export wheel_path=$(dirname $(dirname ${so_full_path}))
+
+# deal with hard copies
+${PYTHON} -m wheel pack ${wheel_path}
+
+rm -rf dpctl_wheel
+popd
+
+${PYTHON} -m wheel tags --remove --build "$GIT_DESCRIBE_NUMBER" \
+    --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \
+    dist/dpctl*.whl
+
+${PYTHON} -m pip install dist/dpctl*.whl \
+    --no-build-isolation \
+    --no-deps \
+    --only-binary :all: \
+    --no-index \
+    --prefix "${PREFIX}" \
+    -vv
+
+export libdir=$(find $PREFIX -name 'libDPCTLSyclInterface*' -exec dirname \{\} \;)
+
+# Recover symbolic links
+# libDPCTLSyclInterface.so.0 -> libDPCTLSyclInterface.so.0.17
+# libDPCTLSyclInterface.so -> libDPCTLSyclInterface.so.0
+mv ${libdir}/${sox_name} ${libdir}/${soxxx_name}
+ln -s ${libdir}/${soxxx_name} ${libdir}/${sox_name}
+ln -s ${libdir}/${sox_name} ${libdir}/${so_name}
+
+# Copy wheel package
+if [[ -d "${WHEELS_OUTPUT_FOLDER}" ]]; then
+    cp dist/dpctl*.whl "${WHEELS_OUTPUT_FOLDER[@]}"
 fi
 
 # need to create this folder so ensure that .dpctl-post-link.sh can work correctly

conda-recipe/meta.yaml

@@ -3,6 +3,9 @@
 {% set excluded_compiler_version2 = "2024.0.2" %}
 {% set excluded_compiler_version3 = "2024.0.3" %}
 
+{% set pyproject = load_file_data('pyproject.toml') %}
+{% set py_build_deps = pyproject.get('build-system', {}).get('requires', []) %}
+
 package:
     name: dpctl
     version: {{ GIT_DESCRIBE_TAG }}
@@ -17,26 +20,43 @@ build:
         - OVERRIDE_INTEL_IPO  # [win]
 
 requirements:
+    # TODO: keep in sync with /pyproject.toml
     build:
         - {{ compiler('cxx') }}
         - {{ compiler('dpcpp') }} >={{ required_compiler_version }},!={{ excluded_compiler_version1 }},!={{ excluded_compiler_version2 }},!={{ excluded_compiler_version3 }}  # [win]
         - {{ compiler('dpcpp') }} >={{ required_compiler_version }},!={{ excluded_compiler_version1 }},!={{ excluded_compiler_version2 }}  # [linux]
         - sysroot_linux-64 >=2.28  # [linux]
     host:
-        - setuptools
-        - cmake  >=3.21
-        - ninja
-        - git
-        - cython
         - python
-        - scikit-build
-        - numpy
-        - wheel
+        - pip >=24.0
+        - level-zero-devel >=1.16
+        - pybind11 >=2.12
+        - {{ pin_compatible('dpcpp-cpp-rt', min_pin='x.x', max_pin='x') }}
+        # Ensure we are using latest version of setuptools, since we don't need
+        # editable environments for release.
+        - setuptools >=69
+        {% for dep in py_build_deps %}
+        {% if dep.startswith('ninja') %}
+        - {{ dep.split(';')[0] }} # [not win]
+        {% elif dep.startswith('cmake') %}
+        # We need to pin cmake version on Windows, because we are patching
+        # cmake files distributed with it.
+        - cmake=3.26 # [win]
+        - {{ dep }} # [not win]
+        {% elif dep.startswith('build>=') %}
+        - {{ 'python-' ~ dep }}
+        {% else %}
+        - {{ dep|replace('_','-') }}
+        {% endif %}
+        {% endfor %}
+        # versioneer dependency
+        - tomli # [py<311]
     run:
         - python
+        - {{ pin_compatible('dpcpp-cpp-rt', min_pin='x.x', max_pin='x') }}
+        - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }}
         - {{ pin_compatible('numpy', min_pin='x.x', max_pin='x') }}
-        - dpcpp-cpp-rt >={{ required_compiler_version }}
-        - level-zero  # [linux]
+        - {{ pin_compatible('level-zero', min_pin='x.x', max_pin='x') }} # [linux]
 
 test:
     requires:
@@ -67,3 +87,4 @@ extra:
         - diptorupd
         - oleksandr-pavlyk
         - ndgrigorian
+        - ZzEeKkAa