Pip packaging new attempt

.gitignore

@@ -136,6 +136,12 @@ a.out
 *.d
 *.dSYM
 
+# Package files
+*.deb
+*.tar.gz
+*.tgz
+*.zip
+
 ################################################################################
 ## Temporary and swap files
 
@@ -162,6 +168,11 @@ venv/
 __pycache__
 *.py[cod]
 
+# Python package build artifacts
+*.egg-info/
+*.whl
+MANIFEST.in
+
 ################################################################################
 ## CMake
 

CMakeLists.txt

@@ -94,20 +94,13 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
         message(STATUS "Building tests disabled")
     endif ()
 
-    option(WITH_PYTHON_BINDINGS "Build Python bindings" ON)
+    cmake_dependent_option(
+        WITH_PYTHON_BINDINGS "Build Python bindings" ON
+        "Halide_ENABLE_RTTI;Halide_ENABLE_EXCEPTIONS" OFF
+    )
     if (WITH_PYTHON_BINDINGS)
-        if (Halide_ENABLE_RTTI AND Halide_ENABLE_EXCEPTIONS)
-            message(STATUS "Building Python bindings enabled")
-            add_subdirectory(python_bindings)
-        else ()
-            if (NOT Halide_ENABLE_RTTI)
-                message(WARNING "Building Python bindings disabled: must compile with RTTI")
-            endif ()
-            if (NOT Halide_ENABLE_EXCEPTIONS)
-                message(WARNING "Building Python bindings disabled: must compile with exceptions")
-            endif ()
-            set(WITH_PYTHON_BINDINGS OFF CACHE BOOL "Build Python bindings" FORCE)
-        endif ()
+        message(STATUS "Building Python bindings enabled")
+        add_subdirectory(python_bindings)
     else ()
         message(STATUS "Building Python bindings disabled")
     endif ()

cmake/FindHalide.cmake

@@ -0,0 +1,6 @@
+# This file should NOT be installed.
+# It is used by python_bindings (and future externalizable projects) to satisfy
+# calls to `find_package(Halide)` when used in-tree.
+
+message(VERBOSE "Spoofing find_package(Halide) since in-tree builds already have Halide available.")
+set(Halide_FOUND 1)

packaging/CMakeLists.txt

@@ -45,28 +45,6 @@ foreach (dep IN ITEMS Halide_LLVM Halide_wabt)
     endif ()
 endforeach ()
 
-##
-# Python bindings
-##
-
-if (WITH_PYTHON_BINDINGS)
-    set(Halide_INSTALL_PYTHONDIR "${CMAKE_INSTALL_LIBDIR}/python3/site-packages"
-        CACHE STRING "Path to Halide Python bindings folder")
-
-    install(DIRECTORY "${Halide_SOURCE_DIR}/python_bindings/src/halide"
-            DESTINATION "${Halide_INSTALL_PYTHONDIR}"
-            COMPONENT Halide_Python
-            FILES_MATCHING
-            PATTERN "*.py"
-            PATTERN "*/halide_" EXCLUDE
-            PATTERN "*/__pycache__" EXCLUDE)
-
-    install(TARGETS Halide_Python
-            LIBRARY DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide"
-            COMPONENT Halide_Python
-            NAMELINK_COMPONENT Halide_Python)
-endif ()
-
 ##
 # Library-type-agnostic interface targets
 ##
@@ -111,13 +89,6 @@ if (TARGET Halide_Adams2019)
             COMPONENT Halide_Development)
 endif ()
 
-if (TARGET Halide_Python AND NOT CMAKE_INSTALL_RPATH)
-    file(RELATIVE_PATH lib_dir
-        "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_PYTHONDIR}/halide"
-        "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
-    set_target_properties(Halide_Python PROPERTIES INSTALL_RPATH "${rbase}/${lib_dir}")
-endif ()
-
 ##
 # READMEs and other top-level documentation
 ##
@@ -170,13 +141,6 @@ if (WITH_TUTORIALS)
             PATTERN "*.jpg"
             PATTERN "*.mp4"
             PATTERN "*.png")
-
-    if (WITH_PYTHON_BINDINGS)
-      install(DIRECTORY ${Halide_SOURCE_DIR}/python_bindings/tutorial/
-              DESTINATION ${CMAKE_INSTALL_DOCDIR}/tutorial-python
-              COMPONENT Halide_Documentation
-              FILES_MATCHING PATTERN "*.py")
-    endif ()
 endif ()
 
 ##

python_bindings/CMakeLists.txt

@@ -1,17 +1,52 @@
+cmake_minimum_required(VERSION 3.16...3.23)
+project(Halide_Python)
+
+include(CMakeDependentOption)
+
 ##
-# Load Python dependencies, including external pybind11
+# Project options
+##
+
+# Preferred defaults for built-in options
+set(CMAKE_CXX_STANDARD 17 CACHE STRING "The minimum C++ standard to use")
+option(CMAKE_CXX_STANDARD_REQUIRED "Prevent CMake C++ standard selection decay" ON)
+option(CMAKE_CXX_EXTENSIONS "Enable C++ vendor extensions (e.g. GNU)" OFF)
+
+# Duplicated options from parent project
+option(WITH_TESTS "Build tests" ON)
+option(WITH_TUTORIALS "Build tutorials" ON)
+
+# Enable/disable testing
+cmake_dependent_option(
+    WITH_TEST_PYTHON "Build Python tests" ON
+    WITH_TESTS OFF
+)
+
+# Set the expected (downloaded) version of pybind11
+option(PYBIND11_USE_FETCHCONTENT "Enable to download pybind11 via FetchContent" ON)
+set(PYBIND11_VER 2.6.2 CACHE STRING "The pybind11 version to use (or download)")
+
+##
+# Dependencies
 ##
 
 find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
 
-set(PYBIND11_VER 2.6.2)
-find_package(pybind11 ${PYBIND11_VER} QUIET)
-if (NOT pybind11_FOUND)
+if (PYBIND11_USE_FETCHCONTENT)
     include(FetchContent)
-    FetchContent_Declare(pybind11
-                         GIT_REPOSITORY https://github.com/pybind/pybind11.git
-                         GIT_TAG v${PYBIND11_VER})
+    FetchContent_Declare(
+        pybind11
+        GIT_REPOSITORY https://github.com/pybind/pybind11.git
+        GIT_TAG v${PYBIND11_VER}
+    )
     FetchContent_MakeAvailable(pybind11)
+else ()
+    find_package(pybind11 ${PYBIND11_VER} REQUIRED)
+endif ()
+
+find_package(Halide REQUIRED)
+if (NOT Halide_ENABLE_RTTI OR NOT Halide_ENABLE_EXCEPTIONS)
+    message(FATAL_ERROR "Python bindings require RTTI and exceptions to be enabled.")
 endif ()
 
 ##
@@ -21,8 +56,10 @@ endif ()
 add_subdirectory(src/halide)
 add_subdirectory(stub)
 
-option(WITH_TEST_PYTHON "Build Python tests" ON)
-if (WITH_TESTS AND WITH_TEST_PYTHON)
+if (WITH_TEST_PYTHON)
     add_subdirectory(test)
+endif ()
+
+if (WITH_TUTORIALS)
     add_subdirectory(tutorial)
 endif ()

python_bindings/pyproject.toml

@@ -0,0 +1,10 @@
+[build-system]
+requires = [
+  "setuptools>=42",
+  "wheel",
+  "scikit-build",
+  "pybind11==2.6.2",
+  "cmake>=3.22",
+  "ninja; platform_system!='Windows'"
+]
+build-backend = "setuptools.build_meta"

python_bindings/setup.py

@@ -0,0 +1,53 @@
+from skbuild import setup, cmaker, utils
+from setuptools import find_packages
+from pathlib import Path
+import pybind11
+from tempfile import TemporaryDirectory as mkdtemp_ctx
+import textwrap
+
+
+def get_version():
+    """
+    Builds a dummy project that prints the found Halide version. The "version"
+    of these Halide bindings is whatever version of Halide they're building
+    against.
+    """
+
+    cmakelists_txt = textwrap.dedent(
+        """
+        cmake_minimum_required(VERSION 3.22)
+        project(dummy)
+        find_package(Halide REQUIRED)
+        file(WRITE halide_version.txt "${Halide_VERSION}")
+        """
+    )
+
+    with mkdtemp_ctx() as srcdir, mkdtemp_ctx() as dstdir:
+        src, dst = Path(srcdir), Path(dstdir)
+        (src / "CMakeLists.txt").write_text(cmakelists_txt)
+        with utils.push_dir(dst):
+            cmkr = cmaker.CMaker()
+            cmkr.configure(cmake_source_dir=src, clargs=("--no-warn-unused-cli",))
+            version = (src / "halide_version.txt").read_text().strip()
+            return version
+
+
+setup(
+    name="halide",
+    version=get_version(),
+    author="The Halide team",
+    author_email="",
+    description="",
+    long_description=Path("readme.md").read_text(),
+    python_requires=">=3.6",
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    cmake_args=[
+        f"-Dpybind11_ROOT={pybind11.get_cmake_dir()}",
+        "-DCMAKE_REQUIRE_FIND_PACKAGE_pybind11=YES",
+        "-DHalide_INSTALL_PYTHONDIR=src",
+        "-DCMAKE_INSTALL_RPATH=$ORIGIN",
+        "-DHalide_Python_INSTALL_IMPORTED_DEPS=ON",
+        "--no-warn-unused-cli",
+    ],
+)

python_bindings/src/halide/CMakeLists.txt

@@ -49,3 +49,79 @@ if (WIN32 AND BUILD_SHARED_LIBS)
                        COMMAND ${CMAKE_COMMAND} -E copy_if_different $<TARGET_FILE:Halide::Halide> $<TARGET_FILE_DIR:Halide::Python>
                        VERBATIM)
 endif ()
+
+##
+# Packaging
+##
+
+include(CMakeDependentOption)
+include(GNUInstallDirs)
+
+set(Halide_INSTALL_PYTHONDIR "${CMAKE_INSTALL_LIBDIR}/python3/site-packages"
+    CACHE STRING "Path to the Python site-packages folder")
+
+install(DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/halide"
+        DESTINATION "${Halide_INSTALL_PYTHONDIR}"
+        COMPONENT Halide_Python
+        FILES_MATCHING
+        PATTERN "*.py"
+        PATTERN "*/halide_" EXCLUDE
+        PATTERN "*/__pycache__" EXCLUDE)
+
+install(TARGETS Halide_Python
+        LIBRARY DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide"
+        COMPONENT Halide_Python
+        NAMELINK_COMPONENT Halide_Python)
+
+get_property(halide_is_imported TARGET Halide::Halide PROPERTY IMPORTED)
+get_property(halide_type TARGET Halide::Halide PROPERTY TYPE)
+cmake_dependent_option(
+    Halide_Python_INSTALL_IMPORTED_DEPS "" OFF
+    "halide_is_imported;halide_type STREQUAL \"SHARED_LIBRARY\"" OFF
+)
+
+if (Halide_Python_INSTALL_IMPORTED_DEPS)
+    # The following might be a bit confusing, but installing both libHalide
+    # and its SONAME symbolic link causes the following bad behavior:
+    #   1. CMake does the right thing and installs libHalide.so.X.Y.Z
+    #      (TARGET_FILE) as a real file and libHalide.so.X
+    #      (TARGET_SONAME_FILE_NAME) as a symbolic link to the former.
+    #   2. Setuptools dutifully packs both of these into a Python wheel, which
+    #      is a structured zip file. Zip files do not support symbolic links.
+    #      Thus, two independent copies of libHalide are inserted, bloating the
+    #      package.
+    # The Python module (on Unix systems) links to the SONAME file, and
+    # installing the symbolic link directly results in a broken link. Hence,
+    # the renaming dance here.
+
+    if (NOT MSVC)
+        set(rename_arg RENAME "$<TARGET_SONAME_FILE_NAME:Halide::Halide>")
+    else ()
+        # DLL systems do not have sonames.
+        set(rename_arg "")
+    endif ()
+
+    # TODO: when we upgrade to CMake 3.22, replace with RUNTIME_DEPENDENCY_SET?
+    install(FILES "$<TARGET_FILE:Halide::Halide>"
+            DESTINATION "${Halide_INSTALL_PYTHONDIR}/halide"
+            COMPONENT Halide_Python
+            ${rename_arg})
+endif ()
+
+if (
+    NOT CMAKE_INSTALL_RPATH  # Honor user overrides
+    AND NOT halide_is_imported  # Imported Halide means user is responsible for RPATH
+    AND halide_type STREQUAL "SHARED_LIBRARY"  # No need to set RPATH if statically linked
+)
+    if (APPLE)
+        set(rbase @loader_path)
+    else ()
+        set(rbase $ORIGIN)
+    endif ()
+
+    file(RELATIVE_PATH lib_dir
+        "${CMAKE_CURRENT_BINARY_DIR}/${Halide_INSTALL_PYTHONDIR}/halide"
+        "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_INSTALL_LIBDIR}")
+
+    set_target_properties(Halide_Python PROPERTIES INSTALL_RPATH "${rbase}/${lib_dir}")
+endif ()

python_bindings/tutorial/CMakeLists.txt

@@ -71,3 +71,14 @@ set_tests_properties(python_tutorial_lesson_10_compile PROPERTIES
 
 set_tests_properties(python_tutorial_lesson_10_aot_compilation_run PROPERTIES
                      FIXTURES_REQUIRED py_lesson_10)
+
+##
+# Packaging
+##
+
+include(GNUInstallDirs)
+
+install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/
+        DESTINATION ${CMAKE_INSTALL_DOCDIR}/tutorial-python
+        COMPONENT Halide_Documentation
+        FILES_MATCHING PATTERN "*.py")