Build platform-specific wheels containing libmagic

ddelange · ddelange · commit 14f7dbbd65dd · 2023-09-07T12:03:47.000+02:00
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,134 @@
+name: GH
+
+permissions:
+  contents: write
+
+on:
+  pull_request:
+  push:
+    branches: master
+  release:
+    types: [released, prereleased]
+  workflow_dispatch:  # allows running workflow manually from the Actions tab
+
+jobs:
+
+  build-sdist:
+    runs-on: ubuntu-latest
+
+    env:
+      PIP_DISABLE_PIP_VERSION_CHECK: 1
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+
+    - run: sudo apt-get install -y libmagic1
+
+    - name: Build source distribution
+      run: |
+        pip install -U setuptools wheel pip
+        python setup.py sdist
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: dist
+        path: dist/*.tar.*
+
+
+  build-wheels-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      include: ${{ steps.set-matrix.outputs.include }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+      - run: pip install cibuildwheel==2.15.0
+      - id: set-matrix
+        env:
+          CIBW_PROJECT_REQUIRES_PYTHON: '==3.8.*'
+        run: |
+          MATRIX_INCLUDE=$(
+            {
+              cibuildwheel --print-build-identifiers --platform linux --arch x86_64,aarch64,i686 | grep cp | jq -nRc '{"only": inputs, "os": "ubuntu-latest"}' \
+              && cibuildwheel --print-build-identifiers --platform macos --arch x86_64,arm64 | grep cp |  jq -nRc '{"only": inputs, "os": "macos-11"}' \
+              && cibuildwheel --print-build-identifiers --platform windows --arch x86,AMD64 | grep cp |  jq -nRc '{"only": inputs, "os": "windows-latest"}'
+            } | jq -sc
+          )
+          echo "include=$MATRIX_INCLUDE" >> $GITHUB_OUTPUT
+
+
+  build-wheels:
+    needs: build-wheels-matrix
+    runs-on: ${{ matrix.os }}
+    name: Build ${{ matrix.only }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include: ${{ fromJson(needs.build-wheels-matrix.outputs.include) }}
+
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v2
+
+    - uses: pypa/cibuildwheel@v2.15.0
+      timeout-minutes: 10
+      with:
+        only: ${{ matrix.only }}
+      env:
+        CIBW_BUILD_VERBOSITY: 1
+        CIBW_BEFORE_BUILD: 'bash -c "make install_libmagic"'
+
+    - uses: actions/upload-artifact@v3
+      with:
+        name: dist
+        path: wheelhouse/*.whl
+
+
+  publish:
+    needs: [build-sdist, build-wheels]
+    if: github.event_name == 'release'
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/setup-python@v4
+      with:
+        python-version: 3.x
+
+    - uses: actions/download-artifact@v3
+      with:
+        name: dist
+        path: dist/
+
+    - run: ls -ltra dist/
+
+    - run: pip install -U twine python-magic --find-links ./dist
+
+    - name: Smoketest
+      run: python -c "import magic; magic.Magic()"
+
+    - name: Upload release assets
+      uses: softprops/action-gh-release@v0.1.15
+      with:
+        files: dist/*
+
+    - name: Upload to PyPI
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
+      run: |
+        twine upload dist/*
diff --git a/Makefile b/Makefile
@@ -0,0 +1,20 @@
+SHELL := /bin/bash
+
+.PHONY: install_libmagic
+## Install libmagic
+install_libmagic:
+	# Debian https://packages.ubuntu.com/libmagic1
+	# RHEL https://git.almalinux.org/rpms/file
+	# Mac https://formulae.brew.sh/formula/libmagic
+	# Windows https://github.com/julian-r/file-windows
+	( ( ( brew install libmagic || ( apt-get update && apt-get install -y libmagic1 ) ) || apk add --update libmagic ) || yum install file-libs ) || ( python -c 'import platform, sysconfig, io, zipfile, urllib.request; assert platform.system() == "Windows"; machine = "x86" if sysconfig.get_platform() == "win32" else "x64"; print(machine); zipfile.ZipFile(io.BytesIO(urllib.request.urlopen(f"https://github.com/julian-r/file-windows/releases/download/v5.44/file_5.44-build104-vs2022-{machine}.zip").read())).extractall(".")' && ls -ltra )
+	# on cibuildwheel, the lib needs to exist in the project before running setup.py
+	python -c "import subprocess; from magic.loader import load_lib; lib = load_lib()._name; print(f'linking {lib}'); subprocess.check_call(['cp', lib, 'magic'])"
+	cp /usr/share/misc/magic /usr/share/misc/magic.mgc magic || true # only on linux
+	ls -ltra magic
+
+.DEFAULT_GOAL := help
+.PHONY: help
+## Print Makefile documentation
+help:
+	@perl -0 -nle 'printf("\033[36m  %-15s\033[0m %s\n", "$$2", "$$1") while m/^##\s*([^\r\n]+)\n^([\w.-]+):[^=]/gm' $(MAKEFILE_LIST) | sort
diff --git a/README.md b/README.md
@@ -30,8 +30,7 @@ will fail throw if this is attempted.
 ```python
 >>> f = magic.Magic(uncompress=True)
 >>> f.from_file('testdata/test.gz')
-'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28
-21:32:52 2008, from Unix)'
+'ASCII text (gzip compressed data, was "test", last modified: Sat Jun 28 21:32:52 2008, from Unix)'
 ```
 
 You can also combine the flag options:
@@ -53,26 +52,40 @@ Other sources:
 - GitHub: https://github.com/ahupp/python-magic
 
 This module is a simple wrapper around the libmagic C library, and
-that must be installed as well:
+comes bundled in the wheels on PyPI. For systems not supported by the wheels, libmagic
+needs to be installed before installing this library:
 
-### Debian/Ubuntu
+### Linux
 
-```
-sudo apt-get install libmagic1
+The Linux wheels should run on most systems out of the box.
+
+Depending on your system and CPU architecture, there might be no compatible wheel uploaded. However, precompiled libmagic might still be available for your system:
+
+```sh
+# Debian/Ubuntu
+apt-get update && apt-get install -y libmagic1
+# Alpine
+apk add --update libmagic
+# RHEL
+yum install file-libs
 ```
 
 ### Windows
 
-You'll need DLLs for libmagic.  @julian-r maintains a pypi package with the DLLs, you can fetch it with:
+The DLLs that are bundled in the Windows wheels are compiled by @julian-r and hosted at https://github.com/julian-r/file-windows/releases.
 
-```
-pip install python-magic-bin
-```
+For ARM64 Windows, you'll need to compile libmagic from source.
 
 ### OSX
 
-- When using Homebrew: `brew install libmagic`
-- When using macports: `port install file`
+The Mac wheels are compiled on GitHub Actions using `macos-11` runners. For older Macs, you'll need to install libmagic from source:
+
+```sh
+# homebrew
+brew install libmagic
+# macports
+port install file
+```
 
 ### Troubleshooting
 
diff --git a/magic/__init__.py b/magic/__init__.py
@@ -330,7 +330,12 @@ def magic_descriptor(cookie, fd):
 
 
 def magic_load(cookie, filename):
-    return _magic_load(cookie, coerce_filename(filename))
+    try:
+        return _magic_load(cookie, coerce_filename(filename))
+    except MagicException:
+        # wheels package the mime database in this directory
+        filename = os.path.join(os.path.dirname(__file__), 'magic.mgc')
+        return _magic_load(cookie, coerce_filename(filename))
 
 
 magic_setflags = libmagic.magic_setflags
diff --git a/magic/loader.py b/magic/loader.py
@@ -1,50 +1,61 @@
 from ctypes.util import find_library
 import ctypes
-import sys
 import glob
 import os.path
+import subprocess
+import sys
 
 def _lib_candidates():
+    here = os.path.dirname(__file__)
 
-  yield find_library('magic')
+    if sys.platform == 'darwin':
 
-  if sys.platform == 'darwin':
+        paths = [
+            here,
+            os.path.abspath("."),
+            '/opt/local/lib',
+            '/usr/local/lib',
+            '/opt/homebrew/lib',
+        ] + glob.glob('/usr/local/Cellar/libmagic/*/lib')
 
-    paths = [
-      '/opt/local/lib',
-      '/usr/local/lib',
-      '/opt/homebrew/lib',
-    ] + glob.glob('/usr/local/Cellar/libmagic/*/lib')
+        for i in paths:
+            yield os.path.join(i, 'libmagic.dylib')
 
-    for i in paths:
-      yield os.path.join(i, 'libmagic.dylib')
+    elif sys.platform in ('win32', 'cygwin'):
 
-  elif sys.platform in ('win32', 'cygwin'):
+        prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']
 
-    prefixes = ['libmagic', 'magic1', 'magic-1', 'cygmagic-1', 'libmagic-1', 'msys-magic-1']
+        for i in prefixes:
+            # find_library searches in %PATH% but not the current directory,
+            # so look for both
+            yield os.path.join(here, '%s.dll' % i)
+            yield os.path.join(os.path.abspath("."), '%s.dll' % i)
+            yield find_library(i)
 
-    for i in prefixes:
-      # find_library searches in %PATH% but not the current directory,
-      # so look for both
-      yield './%s.dll' % (i,)
-      yield find_library(i)
+    elif sys.platform == 'linux':
+        # on some linux systems (musl/alpine), find_library('magic') returns None
+        yield subprocess.check_output(
+            "( ldconfig -p | grep 'libmagic.so.1' | grep -o '/.*' ) || echo '/usr/lib/libmagic.so.1'",
+            shell=True,
+            universal_newlines=True,
+        ).strip()
+        yield os.path.join(here, 'libmagic.so.1')
+        yield os.path.join(os.path.abspath("."), 'libmagic.so.1')
 
-  elif sys.platform == 'linux':
-    # This is necessary because alpine is bad
-    yield 'libmagic.so.1'
+    yield find_library('magic')
 
 
 def load_lib():
 
-  for lib in _lib_candidates():
-    # find_library returns None when lib not found
-    if lib is None:
-      continue
-    try:
-      return ctypes.CDLL(lib)
-    except OSError:
-      pass
-  else:
-    # It is better to raise an ImportError since we are importing magic module
-    raise ImportError('failed to find libmagic.  Check your installation')
+    for lib in _lib_candidates():
+        # find_library returns None when lib not found
+        if lib is None:
+            continue
+        try:
+            return ctypes.CDLL(lib)
+        except OSError as exc:
+            pass
+    else:
+        # It is better to raise an ImportError since we are importing magic module
+        raise ImportError('failed to find libmagic. Check your installation')
 
diff --git a/setup.py b/setup.py
@@ -4,14 +4,40 @@
 import setuptools
 import io
 import os
+import sys
 
+# python packages should not install succesfully if libraries are missing
+from magic.loader import load_lib
+lib = load_lib()._name
 
 def read(file_name):
     """Read a text file and return the content as a string."""
     with io.open(os.path.join(os.path.dirname(__file__), file_name),
                  encoding='utf-8') as f:
         return f.read()
 
+def get_cmdclass():
+    """Build a forward compatible ABI3 wheel when `setup.py bdist_wheel` is called."""
+    if sys.version_info[0] == 2:
+        return {}
+
+    try:
+        from wheel.bdist_wheel import bdist_wheel
+    except ImportError:
+        return {}
+
+    class bdist_wheel_abi3(bdist_wheel):
+        def get_tag(self):
+            python, abi, _ = super().get_tag()
+            # get the platform tag based on libmagic included in this wheel
+            self.root_is_pure = False
+            _, _, plat = super().get_tag()
+            return python, abi, plat
+
+    return {"bdist_wheel": bdist_wheel_abi3}
+
+cmdclass = get_cmdclass()
+
 setuptools.setup(
     name='python-magic',
     description='File type identification using libmagic',
@@ -23,8 +49,11 @@ def read(file_name):
     long_description_content_type='text/markdown',
     packages=['magic'],
     package_data={
-        'magic': ['py.typed', '*.pyi', '**/*.pyi'],
+        'magic': [
+            'py.typed', '*.pyi', '*.dylib*', '*.dll', '*.so*', 'magic', 'magic.mgc'
+        ]
     },
+    cmdclass=cmdclass,
     keywords="mime magic file",
     license="MIT",
     python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
