Skip to content

Commit

Permalink
feature: add musllinux_1_2 support (#1561)
Browse files Browse the repository at this point in the history
* feature: add musllinux_1_2 support

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix[test]: clean-up docker images in test_*linux*_only.py

We are reaching disk space limits (14 GB) when running tests.
Add a fixture to clean-up docker images after tests.
This fixture is applied on tests that pull a specific image for one test only in order not to take too much time pulling that image many times.

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
mayeut and pre-commit-ci[bot] authored Aug 7, 2023
1 parent 229f857 commit 36049d8
Show file tree
Hide file tree
Showing 9 changed files with 147 additions and 32 deletions.
6 changes: 6 additions & 0 deletions bin/update_docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,12 @@ class Image:
Image("musllinux_1_1", "aarch64", "quay.io/pypa/musllinux_1_1_aarch64", None),
Image("musllinux_1_1", "ppc64le", "quay.io/pypa/musllinux_1_1_ppc64le", None),
Image("musllinux_1_1", "s390x", "quay.io/pypa/musllinux_1_1_s390x", None),
# musllinux_1_2 images
Image("musllinux_1_2", "x86_64", "quay.io/pypa/musllinux_1_2_x86_64", None),
Image("musllinux_1_2", "i686", "quay.io/pypa/musllinux_1_2_i686", None),
Image("musllinux_1_2", "aarch64", "quay.io/pypa/musllinux_1_2_aarch64", None),
Image("musllinux_1_2", "ppc64le", "quay.io/pypa/musllinux_1_2_ppc64le", None),
Image("musllinux_1_2", "s390x", "quay.io/pypa/musllinux_1_2_s390x", None),
]

config = configparser.ConfigParser()
Expand Down
43 changes: 24 additions & 19 deletions cibuildwheel/resources/pinned_docker_images.cfg
Original file line number Diff line number Diff line change
@@ -1,49 +1,54 @@
[x86_64]
manylinux1 = quay.io/pypa/manylinux1_x86_64:2023-06-25-d2e0575
manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-14-55e4124
musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2023-07-14-55e4124
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-29-8793e83
musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2023-07-29-8793e83
musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2023-07-29-8793e83

[i686]
manylinux1 = quay.io/pypa/manylinux1_i686:2023-06-25-d2e0575
manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177
manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463
musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2023-07-14-55e4124
musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2023-07-29-8793e83
musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2023-07-29-8793e83

[pypy_x86_64]
manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-14-55e4124
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2023-07-29-8793e83

[pypy_i686]
manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177
manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_i686:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463

[aarch64]
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-12-26-0d38463
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-14-55e4124
musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2023-07-14-55e4124
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-29-8793e83
musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2023-07-29-8793e83
musllinux_1_2 = quay.io/pypa/musllinux_1_2_aarch64:2023-07-29-8793e83

[ppc64le]
manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2022-12-26-0d38463
manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2023-07-14-55e4124
musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2023-07-14-55e4124
manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2023-07-29-8793e83
musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2023-07-29-8793e83
musllinux_1_2 = quay.io/pypa/musllinux_1_2_ppc64le:2023-07-29-8793e83

[s390x]
manylinux2014 = quay.io/pypa/manylinux2014_s390x:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_s390x:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2022-12-26-0d38463
manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2023-07-14-55e4124
musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2023-07-14-55e4124
manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2023-07-29-8793e83
musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2023-07-29-8793e83
musllinux_1_2 = quay.io/pypa/musllinux_1_2_s390x:2023-07-29-8793e83

[pypy_aarch64]
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-14-55e4124
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2023-07-29-8793e83
manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-12-26-0d38463
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-14-55e4124
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2023-07-29-8793e83

2 changes: 1 addition & 1 deletion docs/options.md
Original file line number Diff line number Diff line change
Expand Up @@ -981,7 +981,7 @@ Set an alternative Docker image to be used for building [manylinux / musllinux](
For `CIBW_MANYLINUX_*_IMAGE`, the value of this option can either be set to `manylinux1`, `manylinux2010`, `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` to use a pinned version of the [official manylinux images](https://github.com/pypa/manylinux). Alternatively, set these options to any other valid Docker image name. For PyPy, the `manylinux1` image is not available. For architectures other
than x86 (x86\_64 and i686) `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` must be used, because the first version of the manylinux specification that supports additional architectures is `manylinux2014`. `manylinux_2_28` is not supported for `i686` architecture.

For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name.
For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` or `musllinux_1_2` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name.

If this option is blank, it will fall though to the next available definition (environment variable -> pyproject.toml -> default).

Expand Down
34 changes: 34 additions & 0 deletions test/conftest.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,15 @@
from __future__ import annotations

import json
import subprocess
from typing import Generator

import pytest

from cibuildwheel.util import detect_ci_provider

from .utils import platform


def pytest_addoption(parser) -> None:
parser.addoption(
Expand All @@ -21,3 +29,29 @@ def pytest_addoption(parser) -> None:
)
def build_frontend_env(request) -> dict[str, str]:
return request.param # type: ignore[no-any-return]


@pytest.fixture()
def docker_cleanup() -> Generator[None, None, None]:
def get_images() -> set[str]:
images = subprocess.run(
["docker", "image", "ls", "--format", "{{json .ID}}"],
text=True,
check=True,
stdout=subprocess.PIPE,
).stdout
return {json.loads(image.strip()) for image in images.splitlines() if image.strip()}

if detect_ci_provider() is None or platform != "linux":
try:
yield
finally:
pass
return
images_before = get_images()
try:
yield
finally:
images_after = get_images()
for image in images_after - images_before:
subprocess.run(["docker", "rmi", image], check=False)
1 change: 1 addition & 0 deletions test/test_container_images.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
)


@pytest.mark.usefixtures("docker_cleanup")
def test(tmp_path):
if utils.platform != "linux":
pytest.skip("the test is only relevant to the linux build")
Expand Down
1 change: 1 addition & 0 deletions test/test_manylinuxXXXX_only.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
"manylinux_image",
["manylinux1", "manylinux2010", "manylinux2014", "manylinux_2_24", "manylinux_2_28"],
)
@pytest.mark.usefixtures("docker_cleanup")
def test(manylinux_image, tmp_path):
if utils.platform != "linux":
pytest.skip("the container image test is only relevant to the linux build")
Expand Down
56 changes: 56 additions & 0 deletions test/test_musllinux_X_Y_only.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
from __future__ import annotations

import textwrap

import pytest

from . import test_projects, utils

project_with_manylinux_symbols = test_projects.new_c_project(
spam_c_top_level_add=textwrap.dedent(
r"""
#include <stdlib.h>
#if defined(__GLIBC_PREREQ)
#error "Must not run on a glibc linux environment"
#endif
"""
),
spam_c_function_add=textwrap.dedent(
r"""
sts = 0;
"""
),
)


@pytest.mark.parametrize(
"musllinux_image",
["musllinux_1_1", "musllinux_1_2"],
)
@pytest.mark.usefixtures("docker_cleanup")
def test(musllinux_image, tmp_path):
if utils.platform != "linux":
pytest.skip("the container image test is only relevant to the linux build")

project_dir = tmp_path / "project"
project_with_manylinux_symbols.generate(project_dir)

# build the wheels
add_env = {
"CIBW_BUILD": "*-musllinux*",
"CIBW_MUSLLINUX_X86_64_IMAGE": musllinux_image,
"CIBW_MUSLLINUX_I686_IMAGE": musllinux_image,
"CIBW_MUSLLINUX_AARCH64_IMAGE": musllinux_image,
"CIBW_MUSLLINUX_PPC64LE_IMAGE": musllinux_image,
"CIBW_MUSLLINUX_S390X_IMAGE": musllinux_image,
}

actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
expected_wheels = utils.expected_wheels(
"spam",
"0.1.0",
manylinux_versions=[],
musllinux_versions=[musllinux_image],
)
assert set(actual_wheels) == set(expected_wheels)
15 changes: 8 additions & 7 deletions test/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -205,13 +205,14 @@ def expected_wheels(
if machine_arch == "x86_64":
architectures.append("i686")

platform_tags = [
".".join(
f"{manylinux_version}_{architecture}"
for manylinux_version in manylinux_versions
)
for architecture in architectures
]
if len(manylinux_versions) > 0:
platform_tags = [
".".join(
f"{manylinux_version}_{architecture}"
for manylinux_version in manylinux_versions
)
for architecture in architectures
]
if len(musllinux_versions) > 0 and not python_abi_tag.startswith("pp"):
platform_tags.extend(
[
Expand Down
21 changes: 16 additions & 5 deletions unit_test/option_prepare_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,13 +101,15 @@ def test_build_with_override_launches(monkeypatch, tmp_path):
cibw_toml.write_text(
"""
[tool.cibuildwheel]
manylinux-x86_64-image = "manylinux_2_24"
manylinux-x86_64-image = "manylinux_2_28"
musllinux-x86_64-image = "musllinux_1_2"
# Before Python 3.10, use manylinux2014
# Before Python 3.10, use manylinux2014, musllinux_1_1
[[tool.cibuildwheel.overrides]]
select = "cp3?-*"
manylinux-x86_64-image = "manylinux2014"
manylinux-i686-image = "manylinux2014"
musllinux-x86_64-image = "musllinux_1_1"
[[tool.cibuildwheel.overrides]]
select = "cp36-manylinux_x86_64"
Expand All @@ -122,7 +124,7 @@ def test_build_with_override_launches(monkeypatch, tmp_path):

build_in_container = typing.cast(mock.Mock, linux.build_in_container)

assert build_in_container.call_count == 6
assert build_in_container.call_count == 7

kwargs = build_in_container.call_args_list[0][1]
assert "quay.io/pypa/manylinux2014_x86_64" in kwargs["container"]["image"]
Expand All @@ -146,7 +148,7 @@ def test_build_with_override_launches(monkeypatch, tmp_path):
assert kwargs["options"].build_options("cp37-manylinux_x86_64").before_all == ""

kwargs = build_in_container.call_args_list[2][1]
assert "quay.io/pypa/manylinux_2_24_x86_64" in kwargs["container"]["image"]
assert "quay.io/pypa/manylinux_2_28_x86_64" in kwargs["container"]["image"]
assert kwargs["container"]["cwd"] == PurePosixPath("/project")
assert not kwargs["container"]["simulate_32_bit"]
identifiers = {x.identifier for x in kwargs["platform_configs"]}
Expand All @@ -169,10 +171,19 @@ def test_build_with_override_launches(monkeypatch, tmp_path):

identifiers = {x.identifier for x in kwargs["platform_configs"]}
assert identifiers == {
f"{x}-musllinux_x86_64" for x in ALL_IDS for x in ALL_IDS if "pp" not in x
f"{x}-musllinux_x86_64" for x in ALL_IDS & {"cp36", "cp37", "cp38", "cp39"} if "pp" not in x
}

kwargs = build_in_container.call_args_list[5][1]
assert "quay.io/pypa/musllinux_1_2_x86_64" in kwargs["container"]["image"]
assert kwargs["container"]["cwd"] == PurePosixPath("/project")
assert not kwargs["container"]["simulate_32_bit"]
identifiers = {x.identifier for x in kwargs["platform_configs"]}
assert identifiers == {
f"{x}-musllinux_x86_64" for x in ALL_IDS - {"cp36", "cp37", "cp38", "cp39"} if "pp" not in x
}

kwargs = build_in_container.call_args_list[6][1]
assert "quay.io/pypa/musllinux_1_1_i686" in kwargs["container"]["image"]
assert kwargs["container"]["cwd"] == PurePosixPath("/project")
assert kwargs["container"]["simulate_32_bit"]
Expand Down

0 comments on commit 36049d8

Please sign in to comment.