More architectures supported, add testing

santosh-d3vpl3x · Aug 29, 2024 · 9706393 · 9706393
1 parent c12f488
commit 9706393
Show file tree

Hide file tree

Showing 7 changed files with 216 additions and 11 deletions.
diff --git a/.github/workflows/publish-httpfs-to-pypi.yml b/.github/workflows/publish-httpfs-to-pypi.yml
@@ -8,6 +8,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+    - name: Install Act dependencies
+      if: ${{ env.ACT }}
+      run: |
+        apt-get update && apt-get install sudo -y
     - name: Install uv
       run: |
         sudo apt update
@@ -19,10 +23,22 @@ jobs:
       working-directory: ./extensions/duckdb_extension_httpfs
       run: |
         uv tool run hatch build -t wheel osx_arm64
-    - name: build duckdb_extension_httpfs for linux_amd64
+    - name: build duckdb_extension_httpfs for linux_amd64_gcc4
+      working-directory: ./extensions/duckdb_extension_httpfs
+      run: |
+        uv tool run hatch build -t wheel linux_amd64_gcc4
+    - name: build duckdb_extension_httpfs for linux_arm64
+      working-directory: ./extensions/duckdb_extension_httpfs
+      run: |
+        uv tool run hatch build -t wheel linux_arm64
+    - name: build duckdb_extension_httpfs for osx_amd64
+      working-directory: ./extensions/duckdb_extension_httpfs
+      run: |
+        uv tool run hatch build -t wheel osx_amd64
+    - name: build duckdb_extension_httpfs for windows_amd64
       working-directory: ./extensions/duckdb_extension_httpfs
       run: |
-        uv tool run hatch build -t wheel linux_amd64
+        uv tool run hatch build -t wheel windows_amd64
     - name: Copy wheels to dist directory
       run: |
         mkdir dist
@@ -32,6 +48,50 @@ jobs:
       with:
         name: duckdb-extension-httpfs-distributions
         path: dist/*.whl
+  test:
+    needs:
+      - build
+    strategy:
+      matrix:
+        spec:
+          - { os: macos-latest, platform: macos, arch: x64 }
+          - { os: macos-latest, platform: macos, arch: arm64 }
+          - { os: ubuntu-latest, platform: linux, arch: x64 }
+          - { os: ubuntu-latest, platform: linux, arch: aarch64 }
+          - { os: windows-latest, platform: windows, arch: x64 }
+    runs-on: ${{ matrix.spec.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Act dependencies
+        if: ${{ env.ACT }}
+        run: |
+          apt-get update && apt-get install sudo -y
+      - name: Install curl
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y curl
+      - name: Install uv via curl
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install uv via powershell
+        if: runner.os == 'Windows'
+        run: powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
+      - name: Install python
+        run: uv venv
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: duckdb-extension-httpfs-distributions
+          path: dist/
+      - name: Install packages
+        env:
+          EXTENSION_NAME: httpfs
+        run: |
+          uv sync --verbose
+          uv pip install pytest
+          uv pip install --no-index --find-links dist --verbose duckdb-extension-httpfs
+          uv run pytest
   publish-to-pypi:
     name: Publish 🦆duckdb_extension_httpfs 🐍📦 to PyPI
     if: startsWith(github.ref, 'refs/tags/')

diff --git a/.github/workflows/publish-parquet-to-pypi.yml b/.github/workflows/publish-parquet-to-pypi.yml
@@ -8,6 +8,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+    - name: Install Act dependencies
+      if: ${{ env.ACT }}
+      run: |
+        apt-get update && apt-get install sudo -y
     - name: Install uv
       run: |
         sudo apt update
@@ -19,10 +23,22 @@ jobs:
       working-directory: ./extensions/duckdb_extension_parquet
       run: |
         uv tool run hatch build -t wheel osx_arm64
-    - name: build duckdb_extension_parquet for linux_amd64
+    - name: build duckdb_extension_parquet for linux_amd64_gcc4
+      working-directory: ./extensions/duckdb_extension_parquet
+      run: |
+        uv tool run hatch build -t wheel linux_amd64_gcc4
+    - name: build duckdb_extension_parquet for linux_arm64
+      working-directory: ./extensions/duckdb_extension_parquet
+      run: |
+        uv tool run hatch build -t wheel linux_arm64
+    - name: build duckdb_extension_parquet for osx_amd64
+      working-directory: ./extensions/duckdb_extension_parquet
+      run: |
+        uv tool run hatch build -t wheel osx_amd64
+    - name: build duckdb_extension_parquet for windows_amd64
       working-directory: ./extensions/duckdb_extension_parquet
       run: |
-        uv tool run hatch build -t wheel linux_amd64
+        uv tool run hatch build -t wheel windows_amd64
     - name: Copy wheels to dist directory
       run: |
         mkdir dist
@@ -32,6 +48,50 @@ jobs:
       with:
         name: duckdb-extension-parquet-distributions
         path: dist/*.whl
+  test:
+    needs:
+      - build
+    strategy:
+      matrix:
+        spec:
+          - { os: macos-latest, platform: macos, arch: x64 }
+          - { os: macos-latest, platform: macos, arch: arm64 }
+          - { os: ubuntu-latest, platform: linux, arch: x64 }
+          - { os: ubuntu-latest, platform: linux, arch: aarch64 }
+          - { os: windows-latest, platform: windows, arch: x64 }
+    runs-on: ${{ matrix.spec.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Act dependencies
+        if: ${{ env.ACT }}
+        run: |
+          apt-get update && apt-get install sudo -y
+      - name: Install curl
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y curl
+      - name: Install uv via curl
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install uv via powershell
+        if: runner.os == 'Windows'
+        run: powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
+      - name: Install python
+        run: uv venv
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: duckdb-extension-parquet-distributions
+          path: dist/
+      - name: Install packages
+        env:
+          EXTENSION_NAME: parquet
+        run: |
+          uv sync --verbose
+          uv pip install pytest
+          uv pip install --no-index --find-links dist --verbose duckdb-extension-parquet
+          uv run pytest
   publish-to-pypi:
     name: Publish 🦆duckdb_extension_parquet 🐍📦 to PyPI
     if: startsWith(github.ref, 'refs/tags/')

diff --git a/.github/workflows/publish-spatial-to-pypi.yml b/.github/workflows/publish-spatial-to-pypi.yml
@@ -8,6 +8,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+    - name: Install Act dependencies
+      if: ${{ env.ACT }}
+      run: |
+        apt-get update && apt-get install sudo -y
     - name: Install uv
       run: |
         sudo apt update
@@ -19,10 +23,22 @@ jobs:
       working-directory: ./extensions/duckdb_extension_spatial
       run: |
         uv tool run hatch build -t wheel osx_arm64
-    - name: build duckdb_extension_spatial for linux_amd64
+    - name: build duckdb_extension_spatial for linux_amd64_gcc4
+      working-directory: ./extensions/duckdb_extension_spatial
+      run: |
+        uv tool run hatch build -t wheel linux_amd64_gcc4
+    - name: build duckdb_extension_spatial for linux_arm64
+      working-directory: ./extensions/duckdb_extension_spatial
+      run: |
+        uv tool run hatch build -t wheel linux_arm64
+    - name: build duckdb_extension_spatial for osx_amd64
+      working-directory: ./extensions/duckdb_extension_spatial
+      run: |
+        uv tool run hatch build -t wheel osx_amd64
+    - name: build duckdb_extension_spatial for windows_amd64
       working-directory: ./extensions/duckdb_extension_spatial
       run: |
-        uv tool run hatch build -t wheel linux_amd64
+        uv tool run hatch build -t wheel windows_amd64
     - name: Copy wheels to dist directory
       run: |
         mkdir dist
@@ -32,6 +48,50 @@ jobs:
       with:
         name: duckdb-extension-spatial-distributions
         path: dist/*.whl
+  test:
+    needs:
+      - build
+    strategy:
+      matrix:
+        spec:
+          - { os: macos-latest, platform: macos, arch: x64 }
+          - { os: macos-latest, platform: macos, arch: arm64 }
+          - { os: ubuntu-latest, platform: linux, arch: x64 }
+          - { os: ubuntu-latest, platform: linux, arch: aarch64 }
+          - { os: windows-latest, platform: windows, arch: x64 }
+    runs-on: ${{ matrix.spec.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Install Act dependencies
+        if: ${{ env.ACT }}
+        run: |
+          apt-get update && apt-get install sudo -y
+      - name: Install curl
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt update
+          sudo apt install -y curl
+      - name: Install uv via curl
+        if: runner.os == 'Linux' || runner.os == 'macOS'
+        run: curl -LsSf https://astral.sh/uv/install.sh | sh
+      - name: Install uv via powershell
+        if: runner.os == 'Windows'
+        run: powershell -c "irm https://astral.sh/uv/install.ps1 | iex"
+      - name: Install python
+        run: uv venv
+      - name: Download all the dists
+        uses: actions/download-artifact@v4
+        with:
+          name: duckdb-extension-spatial-distributions
+          path: dist/
+      - name: Install packages
+        env:
+          EXTENSION_NAME: spatial
+        run: |
+          uv sync --verbose
+          uv pip install pytest
+          uv pip install --no-index --find-links dist --verbose duckdb-extension-spatial
+          uv run pytest
   publish-to-pypi:
     name: Publish 🦆duckdb_extension_spatial 🐍📦 to PyPI
     if: startsWith(github.ref, 'refs/tags/')

diff --git a/README.md b/README.md
@@ -16,8 +16,11 @@
 - duckdb-extension-parquet
 
 **The architectures supported:**
-- linux_amd64
+- linux_amd64_gcc4
+- linux_arm64
 - osx_arm64
+- osx_amd64
+- windows_amd64
 
 ## Installation
 First install `duckdb-extensions`.

diff --git a/add_extension_files.py b/add_extension_files.py
@@ -27,14 +27,20 @@ def include_files(self, build_data, duckdb_arch, duckdb_version, download_dir):
         file_path = download_dir / duckdb_version / duckdb_arch
         for file in file_path.glob("*.duckdb_extension.gz"):
             root_name = self.metadata.name.replace("-", "_")
-            build_data["force_include"][file] = f"{root_name}/extensions/{duckdb_version}/{duckdb_arch}/{file.name}"
+            build_data["force_include"][file] = f"{root_name}/extensions/{duckdb_version}/{file.name}"
 
     @staticmethod
     def add_tag(build_data, duckdb_arch):
-        if duckdb_arch == "linux_amd64":
+        if duckdb_arch == "linux_amd64_gcc4":
             build_data["tag"] = "py3-none-manylinux2014_x86_64"
+        elif duckdb_arch == "linux_arm64":
+            build_data["tag"] = "py3-none-manylinux2014_aarch64"
         elif duckdb_arch == "osx_arm64":
             build_data["tag"] = "py3-none-macosx_11_0_arm64"
+        elif duckdb_arch == "osx_amd64":
+            build_data["tag"] = "py3-none-macosx_11_0_x86_64"
+        elif duckdb_arch == "windows_amd64":
+            build_data["tag"] = "py3-none-win_amd64"
         else:
             raise Exception("Not supported platform")
 

diff --git a/src/duckdb_extensions/extension_importer.py b/src/duckdb_extensions/extension_importer.py
@@ -16,8 +16,7 @@ def import_extension(name: str):
     quack_module = importlib.import_module(f"duckdb_extension_{name}")
     module_path = pathlib.Path(str(ilr.files(quack_module)))
     duckdb_version = duckdb.sql("PRAGMA version;").fetchone()[0]
-    duckdb_arch = duckdb.sql("PRAGMA platform;").fetchone()[0]
-    extension_dir = module_path / "extensions" / duckdb_version / duckdb_arch
+    extension_dir = module_path / "extensions" / duckdb_version
     extension_file_gz = extension_dir / f"{name}.duckdb_extension.gz"
     extension_file = extension_dir / f"{name}.duckdb_extension"
 

diff --git a/test_artifact.py b/test_artifact.py
@@ -0,0 +1,17 @@
+import os
+
+import duckdb
+
+from duckdb_extensions import extension_importer
+
+
+def test_extensions():
+    extension = os.environ.get("EXTENSION_NAME")
+    extension_importer.import_extension(extension)
+
+    assert (
+        duckdb.sql(f"""SELECT count(*)
+                FROM duckdb_extensions() where extension_name = '{extension}' 
+                AND installed=true""").fetchone()[0]
+        == 1
+    )