fix(cmake): improved cross-compilation support

CMakeLists.txt

@@ -116,6 +116,7 @@ option(PYBIND11_NUMPY_1_ONLY
 set(PYBIND11_INTERNALS_VERSION
     ""
     CACHE STRING "Override the ABI version, may be used to enable the unstable ABI.")
+option(PYBIND11_USE_CROSSCOMPILING "Respect CMAKE_CROSSCOMPILING" OFF)
 
 if(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
   add_compile_definitions(PYBIND11_DISABLE_HANDLE_TYPE_NAME_DEFAULT_IMPLEMENTATION)
@@ -299,6 +300,7 @@ if(PYBIND11_INSTALL)
           tools/pybind11Common.cmake
           tools/pybind11Tools.cmake
           tools/pybind11NewTools.cmake
+          tools/pybind11GuessPythonExtSuffix.cmake
     DESTINATION ${PYBIND11_CMAKECONFIG_INSTALL_DIR})
 
   if(NOT PYBIND11_EXPORT_NAME)

tests/extra_python_package/test_files.py

@@ -73,6 +73,7 @@
     "share/cmake/pybind11/pybind11Common.cmake",
     "share/cmake/pybind11/pybind11Config.cmake",
     "share/cmake/pybind11/pybind11ConfigVersion.cmake",
+    "share/cmake/pybind11/pybind11GuessPythonExtSuffix.cmake",
     "share/cmake/pybind11/pybind11NewTools.cmake",
     "share/cmake/pybind11/pybind11Targets.cmake",
     "share/cmake/pybind11/pybind11Tools.cmake",

tools/FindPythonLibsNew.cmake

@@ -205,7 +205,7 @@ endif()
 # Make sure the Python has the same pointer-size as the chosen compiler
 # Skip if CMAKE_SIZEOF_VOID_P is not defined
 # This should be skipped for (non-Apple) cross-compiles (like EMSCRIPTEN)
-if(NOT CMAKE_CROSSCOMPILING
+if(NOT _PYBIND11_CROSSCOMPILING
    AND CMAKE_SIZEOF_VOID_P
    AND (NOT "${PYTHON_SIZEOF_VOID_P}" STREQUAL "${CMAKE_SIZEOF_VOID_P}"))
   if(PythonLibsNew_FIND_REQUIRED)

tools/pybind11Common.cmake

@@ -42,6 +42,16 @@ set(pybind11_INCLUDE_DIRS
     "${pybind11_INCLUDE_DIR}"
     CACHE INTERNAL "Include directory for pybind11 (Python not requested)")
 
+if(CMAKE_CROSSCOMPILING AND PYBIND11_USE_CROSSCOMPILING)
+  set(_PYBIND11_CROSSCOMPILING
+      ON
+      CACHE INTERNAL "")
+else()
+  set(_PYBIND11_CROSSCOMPILING
+      OFF
+      CACHE INTERNAL "")
+endif()
+
 # --------------------- Shared targets ----------------------------
 
 # Build an interface library target:
@@ -195,7 +205,7 @@ endif()
 
 # --------------------- pybind11_find_import -------------------------------
 
-if(NOT _pybind11_nopython)
+if(NOT _pybind11_nopython AND NOT _PYBIND11_CROSSCOMPILING)
   # Check to see if modules are importable. Use REQUIRED to force an error if
   # one of the modules is not found. <package_name>_FOUND will be set if the
   # package was found (underscores replace dashes if present). QUIET will hide

tools/pybind11GuessPythonExtSuffix.cmake

@@ -0,0 +1,86 @@
+cmake_minimum_required(VERSION 3.5)
+
+function(pybind11_guess_python_module_extension python)
+
+  # The SETUPTOOLS_EXT_SUFFIX environment variable takes precedence:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ENV{SETUPTOOLS_EXT_SUFFIX})
+    message(
+      STATUS
+        "Getting Python extension suffix from ENV{SETUPTOOLS_EXT_SUFFIX}: $ENV{SETUPTOOLS_EXT_SUFFIX}"
+    )
+    set(PYTHON_MODULE_EXT_SUFFIX
+        "$ENV{SETUPTOOLS_EXT_SUFFIX}"
+        CACHE
+          STRING
+          "Extension suffix for Python extension modules (Initialized from SETUPTOOLS_EXT_SUFFIX)")
+  endif()
+  # If that didn't work, use the Python_SOABI variable:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX AND DEFINED ${python}_SOABI)
+    message(
+      STATUS "Determining Python extension suffix based on ${python}_SOABI: ${${python}_SOABI}")
+    # The final extension depends on the system
+    set(_PY_BUILD_EXTENSION "${CMAKE_SHARED_MODULE_SUFFIX}")
+    if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+      set(_PY_BUILD_EXTENSION ".pyd")
+    endif()
+    # If the SOABI already has an extension, use it as the full suffix
+    # (used for debug versions of Python on Windows)
+    if(${python}_SOABI MATCHES "\\.")
+      set(PYTHON_MODULE_EXT_SUFFIX "${${python}_SOABI}")
+      # If the SOABI is empty, this is usually a bug, but we generate a
+      # correct extension anyway, which is the best we can do
+    elseif("${${python}_SOABI}" STREQUAL "")
+      message(
+        WARNING
+          "${python}_SOABI is defined but empty. You may want to set PYTHON_MODULE_EXT_SUFFIX explicitly."
+      )
+      set(PYTHON_MODULE_EXT_SUFFIX "${_PY_BUILD_EXTENSION}")
+      # Otherwise, add the system-dependent extension to it
+    else()
+      set(PYTHON_MODULE_EXT_SUFFIX ".${${python}_SOABI}${_PY_BUILD_EXTENSION}")
+    endif()
+  endif()
+
+  # If we could not deduce the extension suffix, unset the results:
+  if(NOT DEFINED PYTHON_MODULE_EXT_SUFFIX)
+    unset(PYTHON_MODULE_DEBUG_POSTFIX PARENT_SCOPE)
+    unset(PYTHON_MODULE_EXTENSION PARENT_SCOPE)
+    unset(PYTHON_IS_DEBUG PARENT_SCOPE)
+    return()
+  endif()
+
+  # Sanity checks:
+  if(${python}_SOABI AND NOT (PYTHON_MODULE_EXT_SUFFIX STREQUAL ${python}_SOABI
+                              OR PYTHON_MODULE_EXT_SUFFIX MATCHES "\\.${${python}_SOABI}\\."))
+    message(
+      WARNING
+        "Python extension suffix (${PYTHON_MODULE_EXT_SUFFIX}) does not match ${python}_SOABI (${${python}_SOABI})."
+    )
+  endif()
+
+  # Separate file name postfix from extension: (https://github.com/pybind/pybind11/issues/4699)
+  get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+  get_filename_component(_PYTHON_MODULE_EXTENSION "${PYTHON_MODULE_EXT_SUFFIX}" EXT)
+
+  # Try to deduce the debug ABI from the extension suffix:
+  if(NOT DEFINED _PYTHON_IS_DEBUG)
+    if(_PYTHON_MODULE_EXTENSION MATCHES "^\\.(cpython-|cp|pypy)[0-9]+dm?-"
+       OR _PYTHON_MODULE_DEBUG_POSTFIX MATCHES "^_d")
+      set(_PYTHON_IS_DEBUG On)
+    else()
+      set(_PYTHON_IS_DEBUG Off)
+    endif()
+  endif()
+
+  # Return results
+  set(PYTHON_MODULE_DEBUG_POSTFIX
+      "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+      PARENT_SCOPE)
+  set(PYTHON_MODULE_EXTENSION
+      "${_PYTHON_MODULE_EXTENSION}"
+      PARENT_SCOPE)
+  set(PYTHON_IS_DEBUG
+      "${_PYTHON_IS_DEBUG}"
+      PARENT_SCOPE)
+
+endfunction()

tools/pybind11NewTools.cmake

@@ -32,6 +32,13 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     set(Python_ROOT_DIR "$ENV{pythonLocation}")
   endif()
 
+  # Interpreter should not be found when cross-compiling
+  if(_PYBIND11_CROSSCOMPILING)
+    set(_pybind11_interp_component "")
+  else()
+    set(_pybind11_interp_component Interpreter)
+  endif()
+
   # Development.Module support (required for manylinux) started in 3.18
   if(CMAKE_VERSION VERSION_LESS 3.18)
     set(_pybind11_dev_component Development)
@@ -48,8 +55,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     endif()
   endif()
 
-  find_package(Python 3.6 REQUIRED COMPONENTS Interpreter ${_pybind11_dev_component}
-                                              ${_pybind11_quiet} ${_pybind11_global_keyword})
+  find_package(
+    Python 3.6 REQUIRED COMPONENTS ${_pybind11_interp_component} ${_pybind11_dev_component}
+                                   ${_pybind11_quiet} ${_pybind11_global_keyword})
 
   # If we are in submodule mode, export the Python targets to global targets.
   # If this behavior is not desired, FindPython _before_ pybind11.
@@ -59,7 +67,9 @@ if(NOT Python_FOUND AND NOT Python3_FOUND)
     if(TARGET Python::Python)
       set_property(TARGET Python::Python PROPERTY IMPORTED_GLOBAL TRUE)
     endif()
-    set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
+    if(TARGET Python::Interpreter)
+      set_property(TARGET Python::Interpreter PROPERTY IMPORTED_GLOBAL TRUE)
+    endif()
     if(TARGET Python::Module)
       set_property(TARGET Python::Module PROPERTY IMPORTED_GLOBAL TRUE)
     endif()
@@ -100,69 +110,89 @@ if(PYBIND11_MASTER_PROJECT)
   endif()
 endif()
 
-# If a user finds Python, they may forget to include the Interpreter component
-# and the following two steps require it. It is highly recommended by CMake
-# when finding development libraries anyway, so we will require it.
-if(NOT DEFINED ${_Python}_EXECUTABLE)
-  message(
-    FATAL_ERROR
-      "${_Python} was found without the Interpreter component. Pybind11 requires this component.")
-
-endif()
-
-if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
-                                               PYBIND11_PYTHON_EXECUTABLE_LAST)
-  # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
-  unset(PYTHON_IS_DEBUG CACHE)
-  unset(PYTHON_MODULE_EXTENSION CACHE)
-endif()
-
-set(PYBIND11_PYTHON_EXECUTABLE_LAST
-    "${${_Python}_EXECUTABLE}"
-    CACHE INTERNAL "Python executable during the last CMake run")
-
-if(NOT DEFINED PYTHON_IS_DEBUG)
-  # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
-  execute_process(
-    COMMAND "${${_Python}_EXECUTABLE}" "-c"
-            "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
-    RESULT_VARIABLE _PYTHON_IS_DEBUG)
-  set(PYTHON_IS_DEBUG
-      "${_PYTHON_IS_DEBUG}"
-      CACHE INTERNAL "Python debug status")
-endif()
-
-# Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
-# required for PyPy3 (as of 7.3.1)
-if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
-  execute_process(
-    COMMAND
-      "${${_Python}_EXECUTABLE}" "-c"
-      "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
-    OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
-    ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
-    OUTPUT_STRIP_TRAILING_WHITESPACE)
-
-  if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
+if(NOT _PYBIND11_CROSSCOMPILING)
+  # If a user finds Python, they may forget to include the Interpreter component
+  # and the following two steps require it. It is highly recommended by CMake
+  # when finding development libraries anyway, so we will require it.
+  if(NOT DEFINED ${_Python}_EXECUTABLE)
     message(
-      FATAL_ERROR "pybind11 could not query the module file extension, likely the 'distutils'"
-                  "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}"
+      FATAL_ERROR
+        "${_Python} was found without the Interpreter component. Pybind11 requires this component."
     )
+
   endif()
 
-  # This needs to be available for the pybind11_extension function
-  if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
-    get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
-    set(PYTHON_MODULE_DEBUG_POSTFIX
-        "${_PYTHON_MODULE_DEBUG_POSTFIX}"
-        CACHE INTERNAL "")
+  if(DEFINED PYBIND11_PYTHON_EXECUTABLE_LAST AND NOT ${_Python}_EXECUTABLE STREQUAL
+                                                 PYBIND11_PYTHON_EXECUTABLE_LAST)
+    # Detect changes to the Python version/binary in subsequent CMake runs, and refresh config if needed
+    unset(PYTHON_IS_DEBUG CACHE)
+    unset(PYTHON_MODULE_EXTENSION CACHE)
   endif()
 
-  if(NOT DEFINED PYTHON_MODULE_EXTENSION)
-    get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
-    set(PYTHON_MODULE_EXTENSION
-        "${_PYTHON_MODULE_EXTENSION}"
-        CACHE INTERNAL "")
+  set(PYBIND11_PYTHON_EXECUTABLE_LAST
+      "${${_Python}_EXECUTABLE}"
+      CACHE INTERNAL "Python executable during the last CMake run")
+
+  if(NOT DEFINED PYTHON_IS_DEBUG)
+    # Debug check - see https://stackoverflow.com/questions/646518/python-how-to-detect-debug-Interpreter
+    execute_process(
+      COMMAND "${${_Python}_EXECUTABLE}" "-c"
+              "import sys; sys.exit(hasattr(sys, 'gettotalrefcount'))"
+      RESULT_VARIABLE _PYTHON_IS_DEBUG)
+    set(PYTHON_IS_DEBUG
+        "${_PYTHON_IS_DEBUG}"
+        CACHE INTERNAL "Python debug status")
+  endif()
+
+  # Get the suffix - SO is deprecated, should use EXT_SUFFIX, but this is
+  # required for PyPy3 (as of 7.3.1)
+  if(NOT DEFINED PYTHON_MODULE_EXTENSION OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+    execute_process(
+      COMMAND
+        "${${_Python}_EXECUTABLE}" "-c"
+        "import sys, importlib; s = importlib.import_module('distutils.sysconfig' if sys.version_info < (3, 10) else 'sysconfig'); print(s.get_config_var('EXT_SUFFIX') or s.get_config_var('SO'))"
+      OUTPUT_VARIABLE _PYTHON_MODULE_EXT_SUFFIX
+      ERROR_VARIABLE _PYTHON_MODULE_EXT_SUFFIX_ERR
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+    if(_PYTHON_MODULE_EXT_SUFFIX STREQUAL "")
+      message(
+        FATAL_ERROR
+          "pybind11 could not query the module file extension, likely the 'distutils'"
+          "package is not installed. Full error message:\n${_PYTHON_MODULE_EXT_SUFFIX_ERR}")
+    endif()
+
+    # This needs to be available for the pybind11_extension function
+    if(NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+      get_filename_component(_PYTHON_MODULE_DEBUG_POSTFIX "${_PYTHON_MODULE_EXT_SUFFIX}" NAME_WE)
+      set(PYTHON_MODULE_DEBUG_POSTFIX
+          "${_PYTHON_MODULE_DEBUG_POSTFIX}"
+          CACHE INTERNAL "")
+    endif()
+
+    if(NOT DEFINED PYTHON_MODULE_EXTENSION)
+      get_filename_component(_PYTHON_MODULE_EXTENSION "${_PYTHON_MODULE_EXT_SUFFIX}" EXT)
+      set(PYTHON_MODULE_EXTENSION
+          "${_PYTHON_MODULE_EXTENSION}"
+          CACHE INTERNAL "")
+    endif()
+  endif()
+else()
+  if(NOT DEFINED PYTHON_IS_DEBUG
+     OR NOT DEFINED PYTHON_MODULE_EXTENSION
+     OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+    include("${CMAKE_CURRENT_LIST_DIR}/pybind11GuessPythonExtSuffix.cmake")
+    pybind11_guess_python_module_extension("${_Python}")
+  endif()
+  # When cross-compiling, we cannot query the Python interpreter, so we require
+  # the user to set these variables explicitly.
+  if(NOT DEFINED PYTHON_IS_DEBUG
+     OR NOT DEFINED PYTHON_MODULE_EXTENSION
+     OR NOT DEFINED PYTHON_MODULE_DEBUG_POSTFIX)
+    message(
+      FATAL_ERROR
+        "When cross-compiling, you should set the PYTHON_IS_DEBUG, PYTHON_MODULE_EXTENSION and PYTHON_MODULE_DEBUG_POSTFIX \
+        variables appropriately before loading pybind11 (e.g. in your CMake toolchain file)")
   endif()
 endif()