[Build] Migrate to scikit-build-core (#939)

* cleanup

* init

* build first wheel that may not work

* build cython ext

* fix tvm build

* use sabi

* update rpath to support auditwheel

* pass editible build

* update ci

* fix warnings

* do not use ccache in self host runner

* test local uv cache

* test pip index

* update lib search to respect new lib location

* fix

* update ci

* enable cuda by default

* update src map

* fix

* fix

* fix

* Generate version with backend and git information at build time

* copy tvm_cython to wheels

* fix tvm lib search

* fmt

* remove unused

* auto detect ccache

* add back backend-related files

* remove jit cython adaptor to simplify code

* fmt

* fix ci

* ci fix 2

* ci fix 3

* workaround metal

* ci fix 4

* fmt

* fmt

* Revert "ci fix 4"

This reverts commit d1de829.

* tmp

* fix metal

* trivial cleanup

* add detailed build-time version for cuda

* add back mlc

* Restore wheel info and other trivial updates

* update

* fix cuda

* upd

* fix metal ci

* test for ga build

* test for nvidia/cuda

* test ubuntu 20

* fix

* fix

* Do not use `uv build`

* fix

* fix

* log toolchain version

* merge wheel

* update

* debug

* fix

* update

* skip rocm

* update artifacts each

* fix

* fix

* add mac

* fix cache

* fix cache

* fix cache

* reset and add comment

* upd

* fix git version

* update deps

* trivial update

* use in-tree build dir and install to src to speedup editable build

* Revert "use in-tree build dir and install to src to speedup editable build"

This reverts commit 6ab87b0.

* add build-dir

* update docs

* remove old scrips

* [1/n] cleanup scripts

* [Lint]: [pre-commit.ci] auto fixes [...]

* fix and update

* wait for tvm fix

* revert some tmp fix

* fix

* fix

* spell

* doc update

* test cibuildwheel

* fix and test macos on ci

* Update .github/workflows/dist.yml

Co-authored-by: Xuehai Pan <XuehaiPan@outlook.com>

* fix

* test ga event

* cleanup

* bump tvm to support api3

* test final version

* add cron

* Update .github/workflows/dist.yml

Co-authored-by: Xuehai Pan <XuehaiPan@outlook.com>

* fix

* test ccache for metal cibuildwheel

* test newer macos

* finish

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Xuehai Pan <XuehaiPan@outlook.com>

Author: 3 people

.github/workflows/cuda-ci.yml

@@ -12,49 +12,39 @@ env:
 jobs:
   format-check:
     runs-on: [self-hosted, nvidia]
-
     permissions:
       contents: write
+    env:
+      UV_INDEX_URL: https://mirrors.bfsu.edu.cn/pypi/web/simple
 
     steps:
     - name: Checkout repository
       uses: actions/checkout@v5
       with:
         fetch-depth: 0
+        submodules: recursive
 
-    - name: Set up Python
-      uses: actions/setup-python@v6
+    - name: Install python via uv
+      uses: astral-sh/setup-uv@v6
       with:
+        enable-cache: false
+        cache-local-path: ${{ runner.tool_cache }}/uv
+        activate-environment: true
         python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Ensure venv (local & persistent)
       run: |
-        set -e
-        REQS_HASH=$(sha256sum requirements-test.txt 2>/dev/null | awk '{print $1}' || echo "no_requirements")
-        MARKER="${{ runner.tool_cache }}/.venv_marker_${{ env.PYTHON_VERSION }}_${REQS_HASH:0:8}"
-
-        if [[ -f "$MARKER" ]] && [[ -f "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate" ]]; then
-          echo "venv exists and hash matches – reuse it"
-        else
-          echo "venv stale or missing – recreating"
-          rm -rf "${{ runner.tool_cache }}/${{ env.VENV_DIR }}" "$MARKER"
-          python -m venv "${{ runner.tool_cache }}/${{ env.VENV_DIR }}"
-          # shellcheck source=/dev/null
-          source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
-          python -m pip install --upgrade pip --no-user
-          [[ -f requirements-test.txt ]] && \
-            PIP_NO_BUILD_ISOLATION=1 pip install -r requirements-test.txt --no-user
-          pip install flash_attn==2.5.8 --no-user --no-build-isolation
-          touch "$MARKER"
-        fi
+        [[ -f requirements-test.txt ]] && \
+          uv pip install -r requirements-test.txt --no-build-isolation
+        uv pip install flash_attn==2.5.8 --no-build-isolation
 
     - name: Run format check
       run: |
-        source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
-        git submodule update --init --recursive
+        set -ex
         mkdir -p build
         # run cmake to create the build directory with compile_commands.json
-        cd build; cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_CUDA=ON; cd ..
+        uv pip install cmake
+        cd build; USE_CUDA=1 cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON; cd ..
         if ! output=$(./format.sh 2>&1); then
           echo "------------------------------------"
           echo "message:"
@@ -70,56 +60,41 @@ jobs:
     needs: format-check
     permissions:
       contents: read
+    env:
+      UV_INDEX_URL: https://mirrors.bfsu.edu.cn/pypi/web/simple
     steps:
     - name: Checkout repository
       uses: actions/checkout@v5
       with:
         fetch-depth: 0
+        submodules: recursive
         repository: ${{ github.event.pull_request.head.repo.full_name }}
         ref: ${{ github.event.pull_request.head.ref }}
 
-    - name: Set up Python
-      uses: actions/setup-python@v6
+    - name: Install python via uv
+      uses: astral-sh/setup-uv@v6
       with:
+        enable-cache: false
+        cache-local-path: ${{ runner.tool_cache }}/uv
+        activate-environment: true
         python-version: ${{ env.PYTHON_VERSION }}
 
-    - name: Ensure venv (local & persistent)
+    - name: Setup venv
       run: |
-        set -e
-        REQS_HASH=$(cat requirements-test.txt 2>/dev/null || true)
-        MARKER="${{ runner.tool_cache }}/.venv_marker_${{ env.PYTHON_VERSION }}_${REQS_HASH:0:8}"
-
-        if [[ -f "$MARKER" ]] && [[ -f "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate" ]]; then
-          echo "venv exists and hash matches – reuse it"
-        else
-          echo "venv stale or missing – recreating"
-          rm -rf "${{ runner.tool_cache }}/${{ env.VENV_DIR }}" "$MARKER"
-          python -m venv "${{ runner.tool_cache }}/${{ env.VENV_DIR }}"
-          source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
-          python -m pip install --upgrade pip --no-user
-          [[ -f requirements-test.txt ]] && \
-            PIP_NO_BUILD_ISOLATION=1 pip install -r requirements-test.txt --no-user
-          # flash attention usually requires no isolation build
-          pip install flash_attn==2.5.8 --no-user --no-build-isolation
-          pip install . --no-user
-          touch "$MARKER"
-        fi
+        [[ -f requirements-test.txt ]] && \
+          uv pip install -r requirements-test.txt --no-build-isolation
+        uv pip install flash_attn==2.5.8 --no-build-isolation
 
     - name: Install project (wheel form)
       run: |
-        source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
-        pip install . --no-user -v
+        uv pip install .
 
     - name: Run examples
       run: |
-        source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
         cd examples
-        unset PYTHONPATH
         python -m pytest -n 4 **/test*.py -v -r fE --durations=0 --cache-clear
 
     - name: Run tests
       run: |
-        source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
         cd testing/python
-        unset PYTHONPATH
         python -m pytest -n 4 -v -r fE --durations=0 --cache-clear --timeout=3600

.github/workflows/dist.yml

@@ -0,0 +1,61 @@
+name: Dist
+on:
+  schedule:
+    # gemini said this is 6:00 china time
+    - cron: '0 22 * * *'
+  release:
+    types: [ published ]
+
+env:
+  PYTHON_VERSION: '3.12'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-wheels:
+    strategy:
+      matrix:
+        os: [ubuntu-22.04, ubuntu-22.04-arm, macos-16]
+        include:
+        - os: ubuntu-22.04
+          cuda_version: "12.1"
+        - os: ubuntu-22.04-arm
+          cuda_version: "12.8"
+      fail-fast: true
+    runs-on: ${{ matrix.os }}
+    env:
+      CUDA_VERSION: ${{ matrix.cuda_version }}
+      NO_VERSION_LABEL: ${{ github.event_name != 'release' }}
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+      with:
+        fetch-depth: 1
+        submodules: recursive
+
+    - name: ccache
+      uses: hendrikmuhs/ccache-action@v1.2
+      if: startsWith(matrix.os, 'macos')
+      with:
+        create-symlink: true
+        key: ${{ github.job }}-${{ matrix.os }}
+
+    - name: Build wheels
+      uses: pypa/cibuildwheel@v3.2
+      with:
+        output-dir: wheelhouse
+        config-file: "{package}/pyproject.toml"
+
+    # just for now to list all files
+    - name: List wheels
+      id: ls-whl
+      run: echo "whl_name=$(ls wheelhouse | head -n1)" >> $GITHUB_OUTPUT
+
+    - uses: actions/upload-artifact@v4
+      with:
+        name: ${{ steps.ls-whl.outputs.whl_name }}.zip
+        path: wheelhouse/${{ steps.ls-whl.outputs.whl_name }}
+        compression-level: 0

.github/workflows/metal-ci.yml

@@ -81,12 +81,12 @@ jobs:
         python-version: ${{ env.PYTHON_VERSION }}
 
     - name: Ensure venv (local & persistent)
-      run: uv pip install -r requirements-test.txt -r requirements-build.txt
+      run: uv pip install -r requirements-test.txt
 
     - name: Build wheel
       run: |
         source .venv/bin/activate
-        uv pip install -v --no-build-isolation .
+        uv pip install -v .
 
     - name: Run metal test
       run: |

.github/workflows/rocm-ci.yml

@@ -51,9 +51,9 @@ jobs:
     - name: Run format check
       run: |
         source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
-        git submodule update --init --recursive
+        git submodule update --init --recursive --checkout
         mkdir -p build
-        cd build; cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DUSE_ROCM=ON; cd ..
+        cd build; USE_ROCM=1 cmake .. -DCMAKE_EXPORT_COMPILE_COMMANDS=ON; cd ..
         if ! output=$(./format.sh 2>&1); then
           echo "------------------------------------"
           echo "message:"
@@ -73,7 +73,7 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v5
       with:
-        fetch-depth: 0
+        fetch-depth: 1
         repository: ${{ github.event.pull_request.head.repo.full_name }}
         ref: ${{ github.event.pull_request.head.ref }}
 
@@ -111,6 +111,7 @@ jobs:
       run: |
         echo "Installing project (wheel form)"
         source "${{ runner.tool_cache }}/${{ env.VENV_DIR }}/bin/activate"
+        git submodule update --init --recursive --checkout --recommend-shallow
         USE_ROCM=True pip install . --no-user
 
     - name: Run tests

.gitignore

@@ -2,6 +2,7 @@
 *.slo
 *.lo
 *.o
+*.so
 *.obj
 *.pyc
 
@@ -18,7 +19,7 @@
 
 debug/
 build/
-dist/
+*dist/
 wheelhouse/
 __pycache__
 nnfusion.tar.gz
@@ -82,18 +83,12 @@ models/frozenmodels/
 # .ruff_cache
 .ruff_cache
 
-# build sdist
-build_sdist/
-
 # exclude debug testing folder
 !testing/python/debug
 
 # ignore lib with develop mode
 tilelang/lib
 
-# tox
-.tox/
-
 # cython
 tilelang/jit/adapter/cython/.cycache