Demo of issue #442

pyproject.toml

@@ -270,9 +270,10 @@ exclude = []
 [tool.ruff.per-file-ignores]
 "tests/**" = ["T20"]
 "noxfile.py" = ["T20", "TID251"]
-"src/scikit_build_core/resources/*.py" = ["PTH", "ARG002", "FBT"]
+"src/scikit_build_core/resources/*.py" = ["PTH", "ARG002", "FBT", "TID251"]
 "src/scikit_build_core/_compat/**.py" = ["TID251"]
 "tests/conftest.py" = ["TID251"]
+"tests/packages/**.py" = ["TID251"]
 "docs/conf.py" = ["TID251"]
 
 

src/scikit_build_core/resources/_editable_redirect.py

@@ -1,12 +1,27 @@
 from __future__ import annotations
 
 import importlib.abc
-import importlib.machinery
 import importlib.util
 import os
 import subprocess
 import sys
 
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+    import importlib.machinery
+
+    if sys.version_info < (3, 8):
+        from typing_extensions import TypedDict
+    else:
+        from typing import TypedDict
+
+    class KWDict_1(TypedDict, total=False):
+        submodule_search_locations: list[str]
+
+else:
+    KWDict_1 = dict
+
+
 DIR = os.path.abspath(os.path.dirname(__file__))
 MARKER = "SKBUILD_EDITABLE_SKIP"
 VERBOSE = "SKBUILD_EDITABLE_VERBOSE"
@@ -36,24 +51,66 @@ def __init__(
         self.verbose = verbose
         self.build_options = build_options
         self.install_options = install_options
+        # Construct the __path__ of all resource files
+        # I.e. the paths of all package-like objects
+        submodule_search_locations: dict[str, set[str]] = {}
+        pkgs: list[str] = []
+        # Loop over both python native source files and cmake installed ones
+        for tree in (known_source_files, known_wheel_files):
+            for module, file in tree.items():
+                # Strip the last element of the module
+                parent = ".".join(module.split(".")[:-1])
+                # Check if it is a package
+                if "__init__.py" in file:
+                    parent = module
+                    pkgs.append(parent)
+                # Skip if it's a root module (there are no search paths for these)
+                if not parent:
+                    continue
+                # Initialize the tree element if needed
+                submodule_search_locations.setdefault(parent, set())
+                # Add the parent path to the dictionary values
+                parent_path, _ = os.path.split(file)
+                if not parent_path:
+                    # root modules are skipped so all files should be in a parent package
+                    msg = f"Unexpected path to source file: {file} [{module}]"
+                    raise ImportError(msg)
+                if not os.path.isabs(parent_path):
+                    parent_path = os.path.join(str(DIR), parent_path)
+                submodule_search_locations[parent].add(parent_path)
+        self.submodule_search_locations = submodule_search_locations
+        self.pkgs = pkgs
 
     def find_spec(
         self,
         fullname: str,
         path: object = None,
         target: object = None,
     ) -> importlib.machinery.ModuleSpec | None:
+        # If current item is a know package use its search locations, otherwise if it's a module use the parent's
+        parent = (
+            fullname if fullname in self.pkgs else ".".join(fullname.split(".")[:-1])
+        )
+        # If no known submodule_search_locations is found, it means it is a root module. Do not populate its kwargs
+        # in that case
+        kwargs = KWDict_1()
+        if parent in self.submodule_search_locations:
+            kwargs["submodule_search_locations"] = list(
+                self.submodule_search_locations[parent]
+            )
         if fullname in self.known_wheel_files:
             redir = self.known_wheel_files[fullname]
             if self.rebuild_flag:
                 self.rebuild()
+            # Note: MyPy reports wrong type for `submodule_search_locations`
             return importlib.util.spec_from_file_location(
-                fullname, os.path.join(DIR, redir)
+                fullname,
+                os.path.join(DIR, redir),
+                **kwargs,
             )
         if fullname in self.known_source_files:
             redir = self.known_source_files[fullname]
-            return importlib.util.spec_from_file_location(fullname, redir)
-
+            return importlib.util.spec_from_file_location(fullname, redir, **kwargs)
         return None
 
     def rebuild(self) -> None:

tests/conftest.py

@@ -106,6 +106,12 @@ def __init__(self, env_dir: Path, *, wheelhouse: Path | None = None) -> None:
         self.wheelhouse = None
         self.install("pip>=21.3.1")
         self.wheelhouse = wheelhouse
+        self.platlib = Path(
+            self.execute("import sysconfig; print(sysconfig.get_path('platlib'))")
+        )
+        self.purelib = Path(
+            self.execute("import sysconfig; print(sysconfig.get_path('purelib'))")
+        )
 
     def ensure_directories(
         self, env_dir: str | bytes | os.PathLike[str] | os.PathLike[bytes]
@@ -300,6 +306,15 @@ def package_simplest_c(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Packa
     return package
 
 
+@pytest.fixture()
+def navigate_editable(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> PackageInfo:
+    package = PackageInfo(
+        "navigate_editable",
+    )
+    process_package(package, tmp_path, monkeypatch)
+    return package
+
+
 @pytest.fixture()
 def package_sdist_config(
     tmp_path: Path, monkeypatch: pytest.MonkeyPatch

tests/packages/cython_pxd_editable/pkg1/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.15)
+project(${SKBUILD_PROJECT_NAME} LANGUAGES C)
+
+find_package(
+  Python
+  COMPONENTS Interpreter Development.Module
+  REQUIRED)
+
+find_program(CYTHON "cython")
+
+add_custom_command(
+  OUTPUT src/pkg1/one.c
+  DEPENDS src/pkg1/one.pyx
+  VERBATIM
+  COMMAND "${CYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/src/pkg1/one.pyx"
+          --output-file "${CMAKE_CURRENT_BINARY_DIR}/src/pkg1/one.c")
+
+python_add_library(one MODULE "${CMAKE_CURRENT_BINARY_DIR}/src/pkg1/one.c"
+                   WITH_SOABI)
+
+install(TARGETS one DESTINATION src/pkg1/)

tests/packages/cython_pxd_editable/pkg1/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "cython>=3.0.0",
+    "scikit-build-core",
+]
+
+[project]
+name = "pkg1"
+version = "1.0.0"
+
+[tool.scikit-build]
+wheel.packages = ["src/pkg1"]

tests/packages/cython_pxd_editable/pkg1/src/pkg1/one.pxd

@@ -0,0 +1 @@
+cdef int one()

tests/packages/cython_pxd_editable/pkg1/src/pkg1/one.pyx

@@ -0,0 +1,2 @@
+cdef int one():
+    return 1

tests/packages/cython_pxd_editable/pkg2/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.15)
+project(${SKBUILD_PROJECT_NAME} LANGUAGES C)
+
+find_package(
+  Python
+  COMPONENTS Interpreter Development.Module
+  REQUIRED)
+
+find_program(CYTHON "cython")
+
+add_custom_command(
+  OUTPUT src/pkg2/two.c
+  DEPENDS src/pkg2/two.pyx
+  VERBATIM
+  COMMAND "${CYTHON}" "${CMAKE_CURRENT_SOURCE_DIR}/src/pkg2/two.pyx"
+          --output-file "${CMAKE_CURRENT_BINARY_DIR}/src/pkg2/two.c")
+
+python_add_library(two MODULE "${CMAKE_CURRENT_BINARY_DIR}/src/pkg2/two.c"
+                   WITH_SOABI)
+
+install(TARGETS two DESTINATION src/pkg2/)

tests/packages/cython_pxd_editable/pkg2/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+build-backend = "scikit_build_core.build"
+requires = [
+    "cython>=3.0.0",
+    "scikit-build-core",
+]
+
+[project]
+name = "pkg2"
+version = "1.0.0"
+
+[tool.scikit-build]
+wheel.packages = ["src/pkg2"]

tests/packages/cython_pxd_editable/pkg2/src/pkg2/two.pyx

@@ -0,0 +1,4 @@
+from pkg1.one cimport one
+
+cdef int two():
+    return one() + one()

tests/packages/navigate_editable/CMakeLists.txt

@@ -0,0 +1,21 @@
+cmake_minimum_required(VERSION 3.15...3.26)
+
+project(
+  ${SKBUILD_PROJECT_NAME}
+  LANGUAGES C
+  VERSION ${SKBUILD_PROJECT_VERSION})
+
+find_package(Python COMPONENTS Interpreter Development.Module)
+
+python_add_library(c_module MODULE src/shared_pkg/c_module.c WITH_SOABI)
+
+set(CMakeVar "Some_value_C")
+configure_file(src/shared_pkg/data/generated.txt.in
+               shared_pkg/data/c_generated.txt)
+
+install(
+  TARGETS c_module
+  DESTINATION shared_pkg/
+  COMPONENT PythonModule)
+install(FILES ${PROJECT_BINARY_DIR}/shared_pkg/data/c_generated.txt
+        DESTINATION shared_pkg/data/)

tests/packages/navigate_editable/pyproject.toml

@@ -0,0 +1,13 @@
+[build-system]
+requires = ["scikit-build-core"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "navigate_editable"
+version = "0.0.1"
+dependencies = [
+    "importlib-resources; python_version<'3.9'"
+]
+
+[tool.scikit-build]
+wheel.packages = ["python/shared_pkg"]

tests/packages/navigate_editable/python/shared_pkg/__init__.py

@@ -0,0 +1,9 @@
+from .c_module import call_py_method
+from .py_module import call_c_method, read_c_generated_txt, read_py_data_txt
+
+__all__ = [
+    "call_py_method",
+    "call_c_method",
+    "read_py_data_txt",
+    "read_c_generated_txt",
+]

tests/packages/navigate_editable/python/shared_pkg/c_module.pyi

@@ -0,0 +1,2 @@
+def c_method() -> str: ...
+def call_py_method() -> None: ...

tests/packages/navigate_editable/python/shared_pkg/data/py_data.txt

@@ -0,0 +1 @@
+Some_value_Py

tests/packages/navigate_editable/python/shared_pkg/py_module.py

@@ -0,0 +1,28 @@
+import sys
+
+if sys.version_info < (3, 9):
+    from importlib_resources import files
+else:
+    from importlib.resources import files
+
+from .c_module import c_method
+
+
+def call_c_method():
+    print(c_method())
+
+
+def py_method():
+    print("py_method")
+
+
+def read_py_data_txt():
+    root = files("shared_pkg.data")
+    py_data = root / "py_data.txt"
+    print(py_data.read_text())
+
+
+def read_c_generated_txt():
+    root = files("shared_pkg.data")
+    c_generated_txt = root / "c_generated.txt"
+    print(c_generated_txt.read_text())

tests/packages/navigate_editable/src/shared_pkg/c_module.c

@@ -0,0 +1,54 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+const char* c_method() { return "c_method"; }
+
+static PyObject *c_method_wrapper(PyObject *self, PyObject *args) {
+  return PyUnicode_FromString(c_method());
+}
+
+static PyObject *py_method_wrapper(PyObject *self, PyObject *args) {
+  PyObject *py_module = PyImport_ImportModule("shared_pkg.py_module");
+  if (py_module == NULL) {
+    PyErr_Print();
+    fprintf(stderr, "Failed to load shared_pkg.py_module\n");
+    exit(1);
+  }
+  PyObject *py_method = PyObject_GetAttrString(py_module,(char*)"py_method");
+  if (py_method == NULL) {
+    PyErr_Print();
+    fprintf(stderr, "Failed to load shared_pkg.py_module.py_method\n");
+    exit(1);
+  }
+
+#if PY_MAJOR_VERSION == 3 && PY_MINOR_VERSION > 8
+  PyObject *res = PyObject_CallNoArgs(py_method);
+#else
+  PyObject *res = PyObject_CallObject(py_method, NULL);
+#endif
+
+  if (res == NULL) {
+    PyErr_Print();
+    fprintf(stderr, "Failed to execute shared_pkg.py_module.py_method\n");
+    exit(1);
+  }
+
+  Py_DECREF(py_module);
+  Py_DECREF(py_method);
+  Py_DECREF(res);
+  Py_RETURN_NONE;
+}
+
+static PyMethodDef c_module_methods[] = {
+    {"c_method", c_method_wrapper, METH_NOARGS, "C native method"},
+    {"call_py_method", py_method_wrapper, METH_NOARGS, "Call python native method"},
+    {NULL, NULL, 0, NULL}};
+
+static struct PyModuleDef c_module = {PyModuleDef_HEAD_INIT, "c_module",
+                                             NULL, -1, c_module_methods};
+
+PyMODINIT_FUNC PyInit_c_module(void) {
+  return PyModule_Create(&c_module);
+}

tests/packages/navigate_editable/src/shared_pkg/data/generated.txt.in

@@ -0,0 +1 @@
+@CMakeVar@