From 86512e67a12b86b531ecaa7d502d14d54d0cd751 Mon Sep 17 00:00:00 2001 From: maleo Date: Thu, 15 Feb 2024 08:58:07 +0000 Subject: [PATCH] Replace system python with hermetic Python from rules_python --- WORKSPACE | 4 +- protobuf_deps.bzl | 7 - python/BUILD.bazel | 2 - python/build_targets.bzl | 9 +- python/dist/BUILD.bazel | 7 - python/dist/dist.bzl | 22 -- python/dist/system_python.bzl | 271 ------------------ python/internal.bzl | 1 - python/pb_unit_tests/BUILD | 1 - python/pb_unit_tests/pyproto_test_wrapper.bzl | 1 - python/py_extension.bzl | 2 +- 11 files changed, 6 insertions(+), 321 deletions(-) delete mode 100644 python/dist/system_python.bzl diff --git a/WORKSPACE b/WORKSPACE index 97a24f590b419..9cc5fe3c35231 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -164,11 +164,11 @@ http_archive( patch_cmds = ["find google -type f -name BUILD.bazel -delete"], ) -load("@system_python//:pip.bzl", "pip_parse") +load("@rules_python//python:pip.bzl", "pip_parse") pip_parse( name = "pip_deps", - requirements = "//python:requirements.txt", + requirements_lock = "//python:requirements.txt", ) load("@pip_deps//:requirements.bzl", "install_deps") diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index e77d596109bf7..59682784d574a 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -2,7 +2,6 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//python/dist:python_downloads.bzl", "python_nuget_package", "python_source_archive") -load("//python/dist:system_python.bzl", "system_python") PROTOBUF_MAVEN_ARTIFACTS = [ "com.google.caliper:caliper:1.0-beta-3", @@ -105,12 +104,6 @@ def protobuf_deps(): url = "https://github.com/bazelbuild/rules_python/releases/download/0.28.0/rules_python-0.28.0.tar.gz", ) - if not native.existing_rule("system_python"): - system_python( - name = "system_python", - minimum_python_version = "3.7", - ) - if not native.existing_rule("rules_jvm_external"): _github_archive( name = "rules_jvm_external", diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 3e34bbdbc48b8..94e0fd102bb46 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -138,8 +138,6 @@ selects.config_setting_group( # begin:github_only _message_target_compatible_with = { "@platforms//os:windows": ["@platforms//:incompatible"], - "@system_python//:none": ["@platforms//:incompatible"], - "@system_python//:unsupported": ["@platforms//:incompatible"], "//conditions:default": [], } diff --git a/python/build_targets.bzl b/python/build_targets.bzl index 0b18fe19b9ed6..dadc5bb375464 100644 --- a/python/build_targets.bzl +++ b/python/build_targets.bzl @@ -87,7 +87,7 @@ def build_targets(name): ], deps = select({ "//conditions:default": [], - ":use_fast_cpp_protos": ["@system_python//:python_headers"], + ":use_fast_cpp_protos": ["@rules_python//python/cc:current_py_cc_headers"], }), ) @@ -136,7 +136,7 @@ def build_targets(name): "@com_google_absl//absl/strings", ] + select({ "//conditions:default": [], - ":use_fast_cpp_protos": ["@system_python//:python_headers"], + ":use_fast_cpp_protos": ["@rules_python//python/cc:current_py_cc_headers"], }), ) @@ -421,7 +421,7 @@ def build_targets(name): hdrs = ["google/protobuf/proto_api.h"], visibility = ["//visibility:public"], deps = [ - "@system_python//:python_headers", + "@rules_python//python/cc:current_py_cc_headers", ], ) @@ -435,7 +435,6 @@ def build_targets(name): env = {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "python"}, failure_list = "//conformance:failure_list_python.txt", target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], ":use_fast_cpp_protos": ["@platforms//:incompatible"], "//conditions:default": [], }), @@ -450,7 +449,6 @@ def build_targets(name): env = {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "cpp"}, failure_list = "//conformance:failure_list_python.txt", target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], ":use_fast_cpp_protos": [], "//conditions:default": ["@platforms//:incompatible"], }), @@ -464,7 +462,6 @@ def build_targets(name): env = {"PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": "upb"}, failure_list = "//conformance:failure_list_python_upb.txt", target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], ":use_fast_cpp_protos": ["@platforms//:incompatible"], "//conditions:default": [], }), diff --git a/python/dist/BUILD.bazel b/python/dist/BUILD.bazel index 265d9b8851a70..7bbee941f3c70 100644 --- a/python/dist/BUILD.bazel +++ b/python/dist/BUILD.bazel @@ -11,7 +11,6 @@ load("@pip_deps//:requirements.bzl", "requirement") load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("@rules_python//python:packaging.bzl", "py_wheel") -load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") load("//:protobuf_version.bzl", "PROTOBUF_PYTHON_VERSION") load(":dist.bzl", "py_dist", "py_dist_module") load(":py_proto_library.bzl", "py_proto_library") @@ -272,7 +271,6 @@ pkg_tar( package_file_name = "protobuf.tar.gz", strip_prefix = ".", target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], "//conditions:default": [], }), ) @@ -291,7 +289,6 @@ genrule( mv protobuf/dist/*.tar.gz $@ """, target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], "//conditions:default": [], }), tools = [requirement("setuptools")], @@ -338,7 +335,6 @@ py_wheel( ("//python:limited_api_3.8", "//python:full_api_3.8"): "cp38", "//python:full_api_3.9": "cp39", "//python:limited_api_3.10": "cp310", - "//conditions:default": "cp" + SYSTEM_PYTHON_VERSION, }), # LINT.ThenChange( # :full_api_version, @@ -350,7 +346,6 @@ py_wheel( "src/", ], target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], "//conditions:default": [], }), version = PROTOBUF_PYTHON_VERSION, @@ -390,7 +385,6 @@ py_wheel( "src/", ], target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], "//conditions:default": [], }), version = PROTOBUF_PYTHON_VERSION, @@ -416,7 +410,6 @@ py_wheel( "src/", ], target_compatible_with = select({ - "@system_python//:none": ["@platforms//:incompatible"], "//conditions:default": [], }), version = PROTOBUF_PYTHON_VERSION, diff --git a/python/dist/dist.bzl b/python/dist/dist.bzl index 061125ede6376..f357140bda7f6 100644 --- a/python/dist/dist.bzl +++ b/python/dist/dist.bzl @@ -1,7 +1,6 @@ """Rules to create python distribution files and properly name them""" load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") -load("@system_python//:version.bzl", "SYSTEM_PYTHON_VERSION") def _get_suffix(limited_api, python_version, cpu): """Computes an ABI version tag for an extension module per PEP 3149.""" @@ -15,27 +14,6 @@ def _get_suffix(limited_api, python_version, cpu): else: fail("Unsupported CPU: " + cpu) return ".cp{}-{}.{}".format(python_version, abi, "pyd") - - if python_version == "system": - python_version = SYSTEM_PYTHON_VERSION - if int(python_version) < 38: - python_version += "m" - abis = { - "darwin_arm64": "darwin", - "darwin_x86_64": "darwin", - "darwin": "darwin", - "osx-x86_64": "darwin", - "osx-aarch_64": "darwin", - "linux-aarch_64": "aarch64-linux-gnu", - "linux-x86_64": "x86_64-linux-gnu", - "k8": "x86_64-linux-gnu", - } - - return ".cpython-{}-{}.{}".format( - python_version, - abis[cpu], - "so" if limited_api else "abi3.so", - ) elif limited_api: return ".abi3.so" diff --git a/python/dist/system_python.bzl b/python/dist/system_python.bzl deleted file mode 100644 index db5b30e0b61b0..0000000000000 --- a/python/dist/system_python.bzl +++ /dev/null @@ -1,271 +0,0 @@ -# Copyright (c) 2009-2021, Google LLC -# All rights reserved. -# -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file or at -# https://developers.google.com/open-source/licenses/bsd - -"""Repository rule for using Python 3.x headers from the system.""" - -# Mock out rules_python's pip.bzl for cases where no system python is found. -_mock_pip = """ -def _pip_install_impl(repository_ctx): - repository_ctx.file("BUILD.bazel", ''' -py_library( - name = "noop", - visibility = ["//visibility:public"], -) -''') - repository_ctx.file("requirements.bzl", ''' -def install_deps(*args, **kwargs): - print("WARNING: could not install pip dependencies") - -def requirement(*args, **kwargs): - return "@{}//:noop" -'''.format(repository_ctx.attr.name)) -pip_install = repository_rule( - implementation = _pip_install_impl, - attrs = { - "requirements": attr.string(), - "requirements_overrides": attr.string_dict(), - "python_interpreter_target": attr.string(), - }, -) -pip_parse = pip_install -""" - -# Alias rules_python's pip.bzl for cases where a system python is found. -_alias_pip = """ -load("@rules_python//python:pip.bzl", _pip_parse = "pip_parse") - -def _get_requirements(requirements, requirements_overrides): - for version, override in requirements_overrides.items(): - if version in "{python_version}": - requirements = override - break - return requirements - -def pip_parse(requirements, requirements_overrides={{}}, **kwargs): - _pip_parse( - python_interpreter_target = "@{repo}//:interpreter", - requirements_lock = _get_requirements(requirements, requirements_overrides), - **kwargs, - ) - -pip_install = pip_parse -""" - -_mock_fuzzing_py = """ -def fuzzing_py_install_deps(): - print("WARNING: could not install fuzzing_py dependencies") -""" - -# Alias rules_fuzzing's requirements.bzl for cases where a system python is found. -_alias_fuzzing_py = """ -load("@fuzzing_py_deps//:requirements.bzl", _fuzzing_py_install_deps = "install_deps") - -def fuzzing_py_install_deps(): - _fuzzing_py_install_deps() -""" - -_build_file = """ -load("@bazel_skylib//lib:selects.bzl", "selects") -load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") - -cc_library( - name = "python_headers", - hdrs = glob(["python/**/*.h"], allow_empty = True), - includes = ["python"], - visibility = ["//visibility:public"], -) - -string_flag( - name = "internal_python_support", - build_setting_default = "{support}", - values = [ - "None", - "Supported", - "Unsupported", - ] -) - -config_setting( - name = "none", - flag_values = {{ - ":internal_python_support": "None", - }}, - visibility = ["//visibility:public"], -) - -config_setting( - name = "supported", - flag_values = {{ - ":internal_python_support": "Supported", - }}, - visibility = ["//visibility:public"], -) - -config_setting( - name = "unsupported", - flag_values = {{ - ":internal_python_support": "Unsupported", - }}, - visibility = ["//visibility:public"], -) - -selects.config_setting_group( - name = "exists", - match_any = [":supported", ":unsupported"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "interpreter", - srcs = ["interpreter"], - visibility = ["//visibility:public"], -) - -py_runtime( - name = "py3_runtime", - interpreter_path = "{interpreter}", - python_version = "PY3", -) - -py_runtime_pair( - name = "runtime_pair", - py3_runtime = ":py3_runtime", -) - -toolchain( - name = "python_toolchain", - toolchain = ":runtime_pair", - toolchain_type = "@rules_python//python:toolchain_type", -) -""" - -_register = """ -def register_system_python(): - native.register_toolchains("@{}//:python_toolchain") -""" - -_mock_register = """ -def register_system_python(): - pass -""" - -def _get_python_version(repository_ctx): - py_program = "import sys; print(str(sys.version_info.major) + '.' + str(sys.version_info.minor) + '.' + str(sys.version_info.micro))" - result = repository_ctx.execute(["python3", "-c", py_program]) - return (result.stdout).strip().split(".") - -def _get_python_path(repository_ctx): - py_program = "import sysconfig; print(sysconfig.get_config_var('%s'), end='')" - result = repository_ctx.execute(["python3", "-c", py_program % ("INCLUDEPY")]) - if result.return_code != 0: - return None - return result.stdout - -def _populate_package(ctx, path, python3, python_version): - ctx.symlink(path, "python") - supported = True - for idx, v in enumerate(ctx.attr.minimum_python_version.split(".")): - if int(python_version[idx]) < int(v): - supported = False - break - if "win" in ctx.os.name: - # buildifier: disable=print - print("WARNING: python is not supported on Windows") - supported = False - - build_file = _build_file.format( - interpreter = python3, - support = "Supported" if supported else "Unsupported", - ) - - ctx.file("interpreter", "#!/bin/sh\nexec {} \"$@\"".format(python3)) - ctx.file("BUILD.bazel", build_file) - ctx.file("version.bzl", "SYSTEM_PYTHON_VERSION = '{}{}'".format(python_version[0], python_version[1])) - ctx.file("register.bzl", _register.format(ctx.attr.name)) - if supported: - ctx.file("pip.bzl", _alias_pip.format( - python_version = ".".join(python_version), - repo = ctx.attr.name, - )) - ctx.file("fuzzing_py.bzl", _alias_fuzzing_py) - else: - # Dependencies are unlikely to be satisfiable for unsupported versions of python. - ctx.file("pip.bzl", _mock_pip) - ctx.file("fuzzing_py.bzl", _mock_fuzzing_py) - -def _populate_empty_package(ctx): - # Mock out all the entrypoints we need to run from WORKSPACE. Targets that - # actually need python should use `target_compatible_with` and the generated - # @system_python//:exists or @system_python//:supported constraints. - ctx.file( - "BUILD.bazel", - _build_file.format( - interpreter = "", - support = "None", - ), - ) - ctx.file("version.bzl", "SYSTEM_PYTHON_VERSION = 'None'") - ctx.file("register.bzl", _mock_register) - ctx.file("pip.bzl", _mock_pip) - ctx.file("fuzzing_py.bzl", _mock_fuzzing_py) - -def _system_python_impl(repository_ctx): - path = _get_python_path(repository_ctx) - python3 = repository_ctx.which("python3") - python_version = _get_python_version(repository_ctx) - - if path and python_version[0] == "3": - _populate_package(repository_ctx, path, python3, python_version) - else: - # buildifier: disable=print - print("WARNING: no system python available, builds against system python will fail") - _populate_empty_package(repository_ctx) - -# The system_python() repository rule exposes information from the version of python installed in the current system. -# -# In WORKSPACE: -# system_python( -# name = "system_python_repo", -# minimum_python_version = "3.7", -# ) -# -# This repository exposes some repository rules for configuring python in Bazel. The python toolchain -# *must* be registered in your WORKSPACE: -# load("@system_python_repo//:register.bzl", "register_system_python") -# register_system_python() -# -# Pip dependencies can optionally be specified using a wrapper around rules_python's repository rules: -# load("@system_python//:pip.bzl", "pip_install") -# pip_install( -# name="pip_deps", -# requirements = "@com_google_protobuf//python:requirements.txt", -# ) -# An optional argument `requirements_overrides` takes a dictionary mapping python versions to alternate -# requirements files. This works around the requirement for fully pinned dependencies in python_rules. -# -# Four config settings are exposed from this repository to help declare target compatibility in Bazel. -# For example, `@system_python_repo//:exists` will be true if a system python version has been found. -# The `none` setting will be true only if no python version was found, and `supported`/`unsupported` -# correspond to whether or not the system version is compatible with `minimum_python_version`. -# -# This repository also exposes a header rule that you can depend on from BUILD files: -# cc_library( -# name = "foobar", -# srcs = ["foobar.cc"], -# deps = ["@system_python_repo//:python_headers"], -# ) -# -# The headers should correspond to the version of python obtained by running -# the `python3` command on the system. -system_python = repository_rule( - implementation = _system_python_impl, - local = True, - attrs = { - "minimum_python_version": attr.string(default = "3.7"), - }, -) diff --git a/python/internal.bzl b/python/internal.bzl index 8f99becec4ce6..6fec5861069b5 100644 --- a/python/internal.bzl +++ b/python/internal.bzl @@ -127,7 +127,6 @@ def internal_py_test(deps = [], **kwargs): imports = ["."], deps = deps + ["//python:python_test_lib"], target_compatible_with = select({ - "@system_python//:supported": [], "//conditions:default": ["@platforms//:incompatible"], }), **kwargs diff --git a/python/pb_unit_tests/BUILD b/python/pb_unit_tests/BUILD index 74b01c8eefda9..32207ffd010d9 100644 --- a/python/pb_unit_tests/BUILD +++ b/python/pb_unit_tests/BUILD @@ -41,7 +41,6 @@ py_test( srcs = ["numpy_test_wrapper.py"], main = "numpy_test_wrapper.py", target_compatible_with = select({ - "@system_python//:supported": [], "//conditions:default": ["@platforms//:incompatible"], }), deps = [ diff --git a/python/pb_unit_tests/pyproto_test_wrapper.bzl b/python/pb_unit_tests/pyproto_test_wrapper.bzl index 3ddf6302954d2..fb46a1f365a35 100644 --- a/python/pb_unit_tests/pyproto_test_wrapper.bzl +++ b/python/pb_unit_tests/pyproto_test_wrapper.bzl @@ -16,7 +16,6 @@ def pyproto_test_wrapper(name, deps = []): "//:python_srcs", ] + deps, target_compatible_with = select({ - "@system_python//:supported": [], "//conditions:default": ["@platforms//:incompatible"], }), ) diff --git a/python/py_extension.bzl b/python/py_extension.bzl index 9a2828b950d30..94ca359058c0b 100644 --- a/python/py_extension.bzl +++ b/python/py_extension.bzl @@ -34,7 +34,7 @@ def py_extension(name, srcs, copts, deps = [], **kwargs): "//python:full_api_3.9_win64": ["@nuget_python_x86-64_3.9.0//:python_full_api"], "//python:limited_api_3.10_win32": ["@nuget_python_i686_3.10.0//:python_limited_api"], "//python:limited_api_3.10_win64": ["@nuget_python_x86-64_3.10.0//:python_limited_api"], - "//conditions:default": ["@system_python//:python_headers"], + "//conditions:default": ["@rules_python//python/cc:current_py_cc_headers"], }), **kwargs )