Skip to content

Commit

Permalink
Implement declarative ext-modules in pyproject.toml ("experimenta…
Browse files Browse the repository at this point in the history
…l") (#4568)
  • Loading branch information
abravalheri authored Sep 2, 2024
2 parents 61f2906 + 592d089 commit 5d4473e
Show file tree
Hide file tree
Showing 9 changed files with 420 additions and 48 deletions.
53 changes: 41 additions & 12 deletions docs/userguide/ext_modules.rst
Original file line number Diff line number Diff line change
Expand Up @@ -27,22 +27,41 @@ and all project metadata configuration in the ``pyproject.toml`` file:
version = "0.42"
To instruct setuptools to compile the ``foo.c`` file into the extension module
``mylib.foo``, we need to add a ``setup.py`` file similar to the following:
``mylib.foo``, we need to define an appropriate configuration in either
``pyproject.toml`` [#pyproject.toml]_ or ``setup.py`` file ,
similar to the following:

.. code-block:: python
.. tab:: pyproject.toml

from setuptools import Extension, setup
.. code-block:: toml
setup(
ext_modules=[
Extension(
name="mylib.foo", # as it would be imported
# may include packages/namespaces separated by `.`
sources=["foo.c"], # all sources are compiled into a single binary file
),
[tool.setuptools]
ext-modules = [
{name = "mylib.foo", sources = ["foo.c"]}
]
)
.. tab:: setup.py

.. code-block:: python
from setuptools import Extension, setup
setup(
ext_modules=[
Extension(
name="mylib.foo",
sources=["foo.c"],
),
]
)
The ``name`` value corresponds to how the extension module would be
imported and may include packages/namespaces separated by ``.``.
The ``sources`` value is a list of all source files that are compiled
into a single binary file.
Optionally any other parameter of :class:`setuptools.Extension` can be defined
in the configuration file (but in the case of ``pyproject.toml`` they must be
written using :wiki:`kebab-case` convention).

.. seealso::
You can find more information on the `Python docs about C/C++ extensions`_.
Expand Down Expand Up @@ -168,6 +187,16 @@ Extension API Reference
.. autoclass:: setuptools.Extension


----

.. rubric:: Notes

.. [#pyproject.toml]
Declarative configuration of extension modules via ``pyproject.toml`` was
introduced recently and is still considered experimental.
Therefore it might change in future versions of ``setuptools``.
.. _Python docs about C/C++ extensions: https://docs.python.org/3/extending/extending.html
.. _Cython: https://cython.readthedocs.io/en/stable/index.html
.. _directory options: https://gcc.gnu.org/onlinedocs/gcc/Directory-Options.html
Expand Down
3 changes: 3 additions & 0 deletions docs/userguide/pyproject_config.rst
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,9 @@ file, and can be set via the ``tool.setuptools`` table:
Key Value Type (TOML) Notes
========================= =========================== =========================
``py-modules`` array See tip below.
``ext-modules`` array of **Experimental** - Each item corresponds to a
tables/inline-tables :class:`setuptools.Extension` object and may define
the associated parameters in :wiki:`kebab-case`.
``packages`` array or ``find`` directive See tip below.
``package-dir`` table/inline-table Used when explicitly/manually listing ``packages``.
------------------------- --------------------------- -------------------------
Expand Down
2 changes: 2 additions & 0 deletions newsfragments/4568.feature.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
Added support for defining ``ext-modules`` via ``pyproject.toml``
(**EXPERIMENTAL**, may change in future releases).
26 changes: 22 additions & 4 deletions setuptools/config/_apply_pyprojecttoml.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,11 @@
from inspect import cleandoc
from itertools import chain
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, Union
from typing import TYPE_CHECKING, Any, Callable, Dict, Mapping, TypeVar, Union

from .._path import StrPath
from ..errors import RemovedConfigError
from ..extension import Extension
from ..warnings import SetuptoolsWarning

if TYPE_CHECKING:
Expand All @@ -35,6 +36,7 @@
_ProjectReadmeValue: TypeAlias = Union[str, Dict[str, str]]
_CorrespFn: TypeAlias = Callable[["Distribution", Any, StrPath], None]
_Correspondence: TypeAlias = Union[str, _CorrespFn]
_T = TypeVar("_T")

_logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -117,13 +119,14 @@ def json_compatible_key(key: str) -> str:


def _set_config(dist: Distribution, field: str, value: Any):
val = _PREPROCESS.get(field, _noop)(dist, value)
setter = getattr(dist.metadata, f"set_{field}", None)
if setter:
setter(value)
setter(val)
elif hasattr(dist.metadata, field) or field in SETUPTOOLS_PATCHES:
setattr(dist.metadata, field, value)
setattr(dist.metadata, field, val)
else:
setattr(dist, field, value)
setattr(dist, field, val)


_CONTENT_TYPES = {
Expand Down Expand Up @@ -218,6 +221,17 @@ def _optional_dependencies(dist: Distribution, val: dict, _root_dir):
dist.extras_require = {**existing, **val}


def _ext_modules(dist: Distribution, val: list[dict]) -> list[Extension]:
existing = dist.ext_modules or []
args = ({k.replace("-", "_"): v for k, v in x.items()} for x in val)
new = [Extension(**kw) for kw in args]
return [*existing, *new]


def _noop(_dist: Distribution, val: _T) -> _T:
return val


def _unify_entry_points(project_table: dict):
project = project_table
entry_points = project.pop("entry-points", project.pop("entry_points", {}))
Expand Down Expand Up @@ -376,6 +390,10 @@ def _acessor(obj):
"license_files",
}

_PREPROCESS = {
"ext_modules": _ext_modules,
}

_PREVIOUSLY_DEFINED = {
"name": _attrgetter("metadata.name"),
"version": _attrgetter("metadata.version"),
Expand Down
277 changes: 246 additions & 31 deletions setuptools/config/_validate_pyproject/fastjsonschema_validations.py

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions setuptools/config/pyprojecttoml.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,9 @@ def read_configuration(
asdict["tool"] = tool_table
tool_table["setuptools"] = setuptools_table

if "ext-modules" in setuptools_table:
_ExperimentalConfiguration.emit(subject="[tool.setuptools.ext-modules]")

with _ignore_errors(ignore_option_errors):
# Don't complain about unrelated errors (e.g. tools not using the "tool" table)
subset = {"project": project_table, "tool": {"setuptools": setuptools_table}}
Expand Down
81 changes: 81 additions & 0 deletions setuptools/config/setuptools.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,11 @@
"items": {"type": "string", "format": "python-module-name-relaxed"},
"$comment": "TODO: clarify the relationship with ``packages``"
},
"ext-modules": {
"description": "Extension modules to be compiled by setuptools",
"type": "array",
"items": {"$ref": "#/definitions/ext-module"}
},
"data-files": {
"$$description": [
"``dict``-like structure where each key represents a directory and",
Expand Down Expand Up @@ -254,6 +259,82 @@
{"type": "string", "format": "pep561-stub-name"}
]
},
"ext-module": {
"$id": "#/definitions/ext-module",
"title": "Extension module",
"description": "Parameters to construct a :class:`setuptools.Extension` object",
"type": "object",
"required": ["name", "sources"],
"additionalProperties": false,
"properties": {
"name": {
"type": "string",
"format": "python-module-name-relaxed"
},
"sources": {
"type": "array",
"items": {"type": "string"}
},
"include-dirs":{
"type": "array",
"items": {"type": "string"}
},
"define-macros": {
"type": "array",
"items": {
"type": "array",
"items": [
{"description": "macro name", "type": "string"},
{"description": "macro value", "oneOf": [{"type": "string"}, {"type": "null"}]}
],
"additionalItems": false
}
},
"undef-macros": {
"type": "array",
"items": {"type": "string"}
},
"library-dirs": {
"type": "array",
"items": {"type": "string"}
},
"libraries": {
"type": "array",
"items": {"type": "string"}
},
"runtime-library-dirs": {
"type": "array",
"items": {"type": "string"}
},
"extra-objects": {
"type": "array",
"items": {"type": "string"}
},
"extra-compile-args": {
"type": "array",
"items": {"type": "string"}
},
"extra-link-args": {
"type": "array",
"items": {"type": "string"}
},
"export-symbols": {
"type": "array",
"items": {"type": "string"}
},
"swig-opts": {
"type": "array",
"items": {"type": "string"}
},
"depends": {
"type": "array",
"items": {"type": "string"}
},
"language": {"type": "string"},
"optional": {"type": "boolean"},
"py-limited-api": {"type": "boolean"}
}
},
"file-directive": {
"$id": "#/definitions/file-directive",
"title": "'file:' directive",
Expand Down
2 changes: 1 addition & 1 deletion setuptools/extension.py
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ class Extension(_Extension):
:keyword bool py_limited_api:
opt-in flag for the usage of :doc:`Python's limited API <python:c-api/stable>`.
:raises setuptools.errors.PlatformError: if 'runtime_library_dirs' is
:raises setuptools.errors.PlatformError: if ``runtime_library_dirs`` is
specified on Windows. (since v63)
"""

Expand Down
21 changes: 21 additions & 0 deletions setuptools/tests/config/test_apply_pyprojecttoml.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,27 @@ def test_invalid_module_name(self, tmp_path, monkeypatch, module):
self.dist(module).py_modules


class TestExtModules:
def test_pyproject_sets_attribute(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
pyproject = Path("pyproject.toml")
toml_config = """
[project]
name = "test"
version = "42.0"
[tool.setuptools]
ext-modules = [
{name = "my.ext", sources = ["hello.c", "world.c"]}
]
"""
pyproject.write_text(cleandoc(toml_config), encoding="utf-8")
with pytest.warns(pyprojecttoml._ExperimentalConfiguration):
dist = pyprojecttoml.apply_configuration(Distribution({}), pyproject)
assert len(dist.ext_modules) == 1
assert dist.ext_modules[0].name == "my.ext"
assert set(dist.ext_modules[0].sources) == {"hello.c", "world.c"}


class TestDeprecatedFields:
def test_namespace_packages(self, tmp_path):
pyproject = tmp_path / "pyproject.toml"
Expand Down

0 comments on commit 5d4473e

Please sign in to comment.