Replace system python with hermetic python from rules_python

Closes #15850 PiperOrigin-RevId: 633147621
protocolbuffers · May 13, 2024 · 13ae13c · 13ae13c
1 parent fad4736
commit 13ae13c
Show file tree

Hide file tree

Showing 14 changed files with 94 additions and 409 deletions.
diff --git a/.readthedocs.yml b/.readthedocs.yml
@@ -16,7 +16,7 @@ conda:
   environment: python/docs/environment.yml
 
 python:
-  version: 3.7
+  version: 3.8
   install:
     - method: setuptools
       path: python
diff --git a/MODULE.bazel b/MODULE.bazel
@@ -20,9 +20,29 @@ bazel_dep(name = "rules_cc", version = "0.0.9")
 bazel_dep(name = "rules_java", version = "5.3.5")
 bazel_dep(name = "rules_jvm_external", version = "5.1")
 bazel_dep(name = "rules_pkg", version = "0.7.0")
-bazel_dep(name = "rules_python", version = "0.10.2")
+bazel_dep(name = "rules_python", version = "0.28.0")
 bazel_dep(name = "platforms", version = "0.0.8")
 bazel_dep(name = "zlib", version = "1.2.11")
 
 # TODO: remove after toolchain types are moved to protobuf
-bazel_dep(name = "rules_proto", version = "4.0.0")
+bazel_dep(name = "rules_proto", version = "4.0.0")
+
+SUPPORTED_PYTHON_VERSIONS = [
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+    "3.12",
+]
+
+python = use_extension("@rules_python//python/extensions:python.bzl", "python")
+
+[
+    python.toolchain(
+        # Required to avoid an error when running as root in OSS-Fuzz.
+        ignore_root_user_error = True,
+        is_default = python_version == SUPPORTED_PYTHON_VERSIONS[-1],
+        python_version = python_version,
+    )
+    for python_version in SUPPORTED_PYTHON_VERSIONS
+]
diff --git a/WORKSPACE b/WORKSPACE
@@ -25,6 +25,10 @@ load("@rules_python//python:repositories.bzl", "py_repositories")
 
 py_repositories()
 
+load("@rules_python//python/pip_install:repositories.bzl", "pip_install_dependencies")
+
+pip_install_dependencies()
+
 # Bazel platform rules.
 http_archive(
     name = "platforms",
@@ -164,11 +168,31 @@ http_archive(
     patch_cmds = ["find google -type f -name BUILD.bazel -delete"],
 )
 
-load("@system_python//:pip.bzl", "pip_parse")
+load("@rules_python//python:repositories.bzl", "python_register_multi_toolchains")
+
+SUPPORTED_PYTHON_VERSIONS = [
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+    "3.12",
+]
+
+python_register_multi_toolchains(
+    name = "python",
+    default_version = SUPPORTED_PYTHON_VERSIONS[-1],
+    python_versions = SUPPORTED_PYTHON_VERSIONS,
+    ignore_root_user_error = True,
+)
+
+load("@python_{}//:defs.bzl".format(SUPPORTED_PYTHON_VERSIONS[-1].replace(".", "_")), "interpreter")
+
+load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
     name = "pip_deps",
-    requirements = "//python:requirements.txt",
+    requirements_lock = "//python:requirements.txt",
+    python_interpreter_target = interpreter,
 )
 
 load("@pip_deps//:requirements.bzl", "install_deps")
@@ -177,9 +201,9 @@ install_deps()
 
 http_archive(
     name = "rules_fuzzing",
-    sha256 = "ff52ef4845ab00e95d29c02a9e32e9eff4e0a4c9c8a6bcf8407a2f19eb3f9190",
-    strip_prefix = "rules_fuzzing-0.4.1",
-    urls = ["https://github.com/bazelbuild/rules_fuzzing/releases/download/v0.4.1/rules_fuzzing-0.4.1.zip"],
+    sha256 = "77206c54b71f4dd5335123a6ff2a8ea688eca5378d34b4838114dff71652cf26",
+    strip_prefix = "rules_fuzzing-0.5.1",
+    urls = ["https://github.com/bazelbuild/rules_fuzzing/releases/download/v0.5.1/rules_fuzzing-0.5.1.zip"],
     patches = ["//third_party:rules_fuzzing.patch"],
     patch_args = ["-p1"],
 )
@@ -192,10 +216,6 @@ load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init")
 
 rules_fuzzing_init()
 
-load("@fuzzing_py_deps//:requirements.bzl", fuzzing_py_deps_install_deps = "install_deps")
-
-fuzzing_py_deps_install_deps()
-
 http_archive(
     name = "rules_rust",
     sha256 = "9ecd0f2144f0a24e6bc71ebcc50a1ee5128cedeceb32187004532c9710cb2334",

diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl
@@ -2,7 +2,6 @@
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("//python/dist:python_downloads.bzl", "python_nuget_package", "python_source_archive")
-load("//python/dist:system_python.bzl", "system_python")
 
 PROTOBUF_MAVEN_ARTIFACTS = [
     "com.google.caliper:caliper:1.0-beta-3",
@@ -100,15 +99,9 @@ def protobuf_deps():
     if not native.existing_rule("rules_python"):
         http_archive(
             name = "rules_python",
-            sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
-            strip_prefix = "rules_python-0.26.0",
-            url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz",
-        )
-
-    if not native.existing_rule("system_python"):
-        system_python(
-            name = "system_python",
-            minimum_python_version = "3.7",
+            sha256 = "d70cd72a7a4880f0000a6346253414825c19cdd40a28289bdf67b8e6480edff8",
+            strip_prefix = "rules_python-0.28.0",
+            url = "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz",
         )
 
     if not native.existing_rule("rules_jvm_external"):
@@ -149,33 +142,3 @@ def protobuf_deps():
             urls = ["https://github.com/bazelbuild/rules_kotlin/releases/download/v1.8.1/rules_kotlin_release.tgz"],
             sha256 = "a630cda9fdb4f56cf2dc20a4bf873765c41cf00e9379e8d59cd07b24730f4fde",
         )
-
-    # Python Downloads
-    python_source_archive(
-        name = "python-3.8.0",
-        sha256 = "f1069ad3cae8e7ec467aa98a6565a62a48ef196cb8f1455a245a08db5e1792df",
-    )
-    python_nuget_package(
-        name = "nuget_python_i686_3.8.0",
-        sha256 = "87a6481f5eef30b42ac12c93f06f73bd0b8692f26313b76a6615d1641c4e7bca",
-    )
-    python_nuget_package(
-        name = "nuget_python_x86-64_3.8.0",
-        sha256 = "96c61321ce90dd053c8a04f305a5f6cc6d91350b862db34440e4a4f069b708a0",
-    )
-    python_nuget_package(
-        name = "nuget_python_i686_3.9.0",
-        sha256 = "229abecbe49dc08fe5709e0b31e70edfb3b88f23335ebfc2904c44f940fd59b6",
-    )
-    python_nuget_package(
-        name = "nuget_python_x86-64_3.9.0",
-        sha256 = "6af58a733e7dfbfcdd50d55788134393d6ffe7ab8270effbf724bdb786558832",
-    )
-    python_nuget_package(
-        name = "nuget_python_i686_3.10.0",
-        sha256 = "e115e102eb90ce160ab0ef7506b750a8d7ecc385bde0a496f02a54337a8bc333",
-    )
-    python_nuget_package(
-        name = "nuget_python_x86-64_3.10.0",
-        sha256 = "4474c83c25625d93e772e926f95f4cd398a0abbb52793625fa30f39af3d2cc00",
-    )
diff --git a/python/BUILD.bazel b/python/BUILD.bazel
@@ -138,8 +138,6 @@ selects.config_setting_group(
 # begin:github_only
 _message_target_compatible_with = {
     "@platforms//os:windows": ["@platforms//:incompatible"],
-    "@system_python//:none": ["@platforms//:incompatible"],
-    "@system_python//:unsupported": ["@platforms//:incompatible"],
     "//conditions:default": [],
 }
 

diff --git a/python/build_targets.bzl b/python/build_targets.bzl
@@ -87,7 +87,7 @@ def build_targets(name):
         ],
         deps = select({
             "//conditions:default": [],
-            ":use_fast_cpp_protos": ["@system_python//:python_headers"],
+            ":use_fast_cpp_protos": ["@rules_python//python/cc:current_py_cc_headers"],
         }),
     )
 
@@ -136,7 +136,7 @@ def build_targets(name):
             "@com_google_absl//absl/strings",
         ] + select({
             "//conditions:default": [],
-            ":use_fast_cpp_protos": ["@system_python//:python_headers"],
+            ":use_fast_cpp_protos": ["@rules_python//python/cc:current_py_cc_headers"],
         }),
     )
 
@@ -426,7 +426,7 @@ def build_targets(name):
         hdrs = ["google/protobuf/proto_api.h"],
         visibility = ["//visibility:public"],
         deps = [
-            "@system_python//:python_headers",
+            "@rules_python//python/cc:current_py_cc_headers",
         ],
     )
 
@@ -440,7 +440,6 @@ def build_targets(name):
         env = {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "python"},
         failure_list = "//conformance:failure_list_python.txt",
         target_compatible_with = select({
-            "@system_python//:none": ["@platforms//:incompatible"],
             ":use_fast_cpp_protos": ["@platforms//:incompatible"],
             "//conditions:default": [],
         }),
@@ -455,7 +454,6 @@ def build_targets(name):
         env = {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "cpp"},
         failure_list = "//conformance:failure_list_python.txt",
         target_compatible_with = select({
-            "@system_python//:none": ["@platforms//:incompatible"],
             ":use_fast_cpp_protos": [],
             "//conditions:default": ["@platforms//:incompatible"],
         }),
@@ -469,7 +467,6 @@ def build_targets(name):
         env = {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "upb"},
         failure_list = "//conformance:failure_list_python_upb.txt",
         target_compatible_with = select({
-            "@system_python//:none": ["@platforms//:incompatible"],
             ":use_fast_cpp_protos": ["@platforms//:incompatible"],
             "//conditions:default": [],
         }),

diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel
@@ -11,7 +11,6 @@ load("@pip_deps//:requirements.bzl", "requirement")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
 load("@rules_python//python:packaging.bzl", "py_wheel")
-load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION")
 load("//:protobuf_version.bzl", "PROTOBUF_PYTHON_VERSION")
 load(":dist.bzl", "py_dist", "py_dist_module")
 load(":py_proto_library.bzl", "py_proto_library")
@@ -272,7 +271,6 @@ pkg_tar(
     package_file_name = "protobuf.tar.gz",
     strip_prefix = ".",
     target_compatible_with = select({
-        "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
 )
@@ -291,7 +289,6 @@ genrule(
         mv protobuf/dist/*.tar.gz $@
     """,
     target_compatible_with = select({
-        "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
     tools = [requirement("setuptools")],
@@ -338,7 +335,7 @@ py_wheel(
         ("//python:limited_api_3.8", "//python:full_api_3.8"): "cp38",
         "//python:full_api_3.9": "cp39",
         "//python:limited_api_3.10": "cp310",
-        "//conditions:default": "cp" + SYSTEM_PYTHON_VERSION,
+        "//conditions:default": "cp38",  # TODO: Make more formal
     }),
     # LINT.ThenChange(
     #    :full_api_version,
@@ -350,7 +347,6 @@ py_wheel(
         "src/",
     ],
     target_compatible_with = select({
-        "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
     version = PROTOBUF_PYTHON_VERSION,
@@ -390,7 +386,6 @@ py_wheel(
         "src/",
     ],
     target_compatible_with = select({
-        "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
     version = PROTOBUF_PYTHON_VERSION,
@@ -416,7 +411,6 @@ py_wheel(
         "src/",
     ],
     target_compatible_with = select({
-        "@system_python//:none": ["@platforms//:incompatible"],
         "//conditions:default": [],
     }),
     version = PROTOBUF_PYTHON_VERSION,

diff --git a/python/dist/dist.bzl b/python/dist/dist.bzl
@@ -1,7 +1,6 @@
 """Rules to create python distribution files and properly name them"""
 
 load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo")
-load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION")
 
 def _get_suffix(limited_api, python_version, cpu):
     """Computes an ABI version tag for an extension module per PEP 3149."""
@@ -15,30 +14,29 @@ def _get_suffix(limited_api, python_version, cpu):
         else:
             fail("Unsupported CPU: " + cpu)
         return ".cp{}-{}.{}".format(python_version, abi, "pyd")
-
-    if python_version == "system":
-        python_version = SYSTEM_PYTHON_VERSION
-        if int(python_version) < 38:
-            python_version += "m"
-        abis = {
-            "darwin_arm64": "darwin",
-            "darwin_x86_64": "darwin",
-            "darwin": "darwin",
-            "osx-x86_64": "darwin",
-            "osx-aarch_64": "darwin",
-            "linux-aarch_64": "aarch64-linux-gnu",
-            "linux-x86_64": "x86_64-linux-gnu",
-            "k8": "x86_64-linux-gnu",
-        }
-
-        return ".cpython-{}-{}.{}".format(
-            python_version,
-            abis[cpu],
-            "so" if limited_api else "abi3.so",
-        )
     elif limited_api:
         return ".abi3.so"
 
+    python_version = runtime.interpreter_version_info
+    if int(python_version) < 38:
+        python_version += "m"
+    abis = {
+        "darwin_arm64": "darwin",
+        "darwin_x86_64": "darwin",
+        "darwin": "darwin",
+        "osx-x86_64": "darwin",
+        "osx-aarch_64": "darwin",
+        "linux-aarch_64": "aarch64-linux-gnu",
+        "linux-x86_64": "x86_64-linux-gnu",
+        "k8": "x86_64-linux-gnu",
+    }
+
+    return ".cpython-{}-{}.{}".format(
+        python_version,
+        abis[cpu],
+        "so" if limited_api else "abi3.so",
+    )
+
     fail("Unsupported combination of flags")
 
 def _declare_module_file(ctx, module_name, python_version, limited_api):
@@ -114,6 +112,7 @@ def _py_dist_module_impl(ctx):
 
 py_dist_module = rule(
     implementation = _py_dist_module_impl,
+    toolchains = ["@rules_python//python:toolchain_type"],
     attrs = {
         "module_name": attr.string(mandatory = True),
         "extension": attr.label(