feat: cpython toolchains

README.md

@@ -54,6 +54,28 @@ http_archive(
 )
 ```
 
+To register a hermetic Python toolchain, you can add to the `WORKSPACE` file:
+
+```python
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+# Multiple versions are available in the @rules_python//python:versions.bzl file.
+python_register_toolchains(
+    name = "python310",
+    python_version = "3.10",
+)
+
+load("@python310_host//:defs.bzl", "interpreter")
+
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+    ...
+    python_interpreter_target = interpreter,
+    ...
+)
+```
+
 Once you've imported the rule set into your `WORKSPACE` using any of these
 methods, you can then load the core rules in your `BUILD` files with:
 

WORKSPACE

@@ -25,6 +25,14 @@ load("//:internal_setup.bzl", "rules_python_internal_setup")
 
 rules_python_internal_setup()
 
+load("//python:repositories.bzl", "python_register_toolchains")
+load("//python:versions.bzl", "MINOR_MAPPING")
+
+python_register_toolchains(
+    name = "python_toolchain",
+    python_version = MINOR_MAPPING.values()[-1],
+)
+
 load("//gazelle:deps.bzl", "gazelle_deps")
 
 # gazelle:repository_macro gazelle/deps.bzl%gazelle_deps

examples/pip_install/WORKSPACE

@@ -16,6 +16,14 @@ local_repository(
     path = "../..",
 )
 
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python39",
+    python_version = "3.9",
+)
+
+load("@python39_host//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_install")
 
 pip_install(
@@ -32,7 +40,9 @@ pip_install(
     # 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
     # 2. Pre-compiled python interpreter included with http_archive
     # 3. Wrapper script, like in the autodetecting python toolchain.
-    #python_interpreter_target = "@python_interpreter//:python_bin",
+    #
+    # Here, we use the interpreter constant that resolves to the host interpreter from the default Python toolchain.
+    python_interpreter_target = interpreter,
 
     # (Optional) You can set quiet to False if you want to see pip output.
     #quiet = False,

examples/pip_parse/WORKSPACE

@@ -5,9 +5,23 @@ local_repository(
     path = "../..",
 )
 
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python39",
+    python_version = "3.9",
+)
+
+load("@python39_host//:defs.bzl", "interpreter")
 load("@rules_python//python:pip.bzl", "pip_parse")
 
 pip_parse(
+    # (Optional) You can set an environment in the pip process to control its
+    # behavior. Note that pip is run in "isolated" mode so no PIP_<VAR>_<NAME>
+    # style env vars are read, but env vars that control requests and urllib3
+    # can be passed
+    # environment = {"HTTPS_PROXY": "http://my.proxy.fun/"},
+    name = "pypi",
     # (Optional) You can provide extra parameters to pip.
     # Here, make pip output verbose (this is usable with `quiet = False`).
     # extra_pip_args = ["-v"],
@@ -21,17 +35,12 @@ pip_parse(
     # 1. Python interpreter that you compile in the build file (as above in @python_interpreter).
     # 2. Pre-compiled python interpreter included with http_archive
     # 3. Wrapper script, like in the autodetecting python toolchain.
-    #python_interpreter_target = "@python_interpreter//:python_bin",
+    #
+    # Here, we use the interpreter constant that resolves to the host interpreter from the default Python toolchain.
+    python_interpreter_target = interpreter,
 
     # (Optional) You can set quiet to False if you want to see pip output.
     #quiet = False,
-
-    # (Optional) You can set an environment in the pip process to control its
-    # behavior. Note that pip is run in "isolated" mode so no PIP_<VAR>_<NAME>
-    # style env vars are read, but env vars that control requests and urllib3
-    # can be passed
-    # environment = {"HTTPS_PROXY": "http://my.proxy.fun/"},
-    name = "pypi",
     requirements_lock = "//:requirements_lock.txt",
 )
 

internal_setup.bzl

@@ -1,6 +1,21 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 """Setup for rules_python tests and tools."""
 
 load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies")
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@build_bazel_integration_testing//tools:repositories.bzl", "bazel_binaries")
 load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
 load("//:version.bzl", "SUPPORTED_BAZEL_VERSIONS")
@@ -16,6 +31,8 @@ def rules_python_internal_setup():
     # Depend on the Bazel binaries for running bazel-in-bazel tests
     bazel_binaries(versions = SUPPORTED_BAZEL_VERSIONS)
 
+    bazel_skylib_workspace()
+
     # gazelle:repository_macro gazelle/deps.bzl%gazelle_deps
     _go_repositories()
 

python/private/BUILD

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+load("//python:versions.bzl", "print_toolchains_checksums")
 load(":stamp.bzl", "stamp_build_setting")
 
 licenses(["notice"])  # Apache 2.0
@@ -43,3 +44,5 @@ exports_files(
 
 # Used to determine the use of `--stamp` in Starlark rules
 stamp_build_setting(name = "stamp")
+
+print_toolchains_checksums(name = "print_toolchains_checksums")

python/private/toolchains_repo.bzl

@@ -0,0 +1,158 @@
+# Copyright 2022 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Create a repository to hold the toolchains.
+
+This follows guidance here:
+https://docs.bazel.build/versions/main/skylark/deploying.html#registering-toolchains
+
+The "complex computation" in our case is simply downloading large artifacts.
+This guidance tells us how to avoid that: we put the toolchain targets in the
+alias repository with only the toolchain attribute pointing into the
+platform-specific repositories.
+"""
+
+load(
+    "//python:versions.bzl",
+    "LINUX_NAME",
+    "MACOS_NAME",
+    "PLATFORMS",
+    "WINDOWS_NAME",
+)
+
+def _toolchains_repo_impl(rctx):
+    build_content = """\
+# Generated by toolchains_repo.bzl
+#
+# These can be registered in the workspace file or passed to --extra_toolchains
+# flag. By default all these toolchains are registered by the
+# python_register_toolchains macro so you don't normally need to interact with
+# these targets.
+
+"""
+
+    for [platform, meta] in PLATFORMS.items():
+        build_content += """\
+toolchain(
+    name = "{platform}_toolchain",
+    exec_compatible_with = {compatible_with},
+    target_compatible_with = {compatible_with},
+    toolchain = "@{user_repository_name}_{platform}//:python_runtimes",
+    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
+)
+""".format(
+            platform = platform,
+            name = rctx.attr.name,
+            user_repository_name = rctx.attr.user_repository_name,
+            compatible_with = meta.compatible_with,
+        )
+
+    rctx.file("BUILD.bazel", build_content)
+
+toolchains_repo = repository_rule(
+    _toolchains_repo_impl,
+    doc = "Creates a repository with toolchain definitions for all known platforms " +
+          "which can be registered or selected.",
+    attrs = {
+        "user_repository_name": attr.string(doc = "what the user chose for the base name"),
+    },
+)
+
+def _host_os_alias_impl(rctx):
+    (os_name, arch) = _host_os_arch(rctx)
+
+    host_platform = None
+    for platform, meta in PLATFORMS.items():
+        if meta.os_name == os_name and meta.arch == arch:
+            host_platform = platform
+    if not host_platform:
+        fail("No platform declared for host OS {} on arch {}".format(os_name, arch))
+
+    is_windows = (os_name == WINDOWS_NAME)
+    python3_binary_path = "python.exe" if is_windows else "bin/python3"
+
+    # Base BUILD file for this repository.
+    build_contents = """\
+# Generated by python/repositories.bzl
+package(default_visibility = ["//visibility:public"])
+alias(name = "files",           actual = "@{py_repository}_{host_platform}//:files")
+alias(name = "py3_runtime",     actual = "@{py_repository}_{host_platform}//:py3_runtime")
+alias(name = "python_runtimes", actual = "@{py_repository}_{host_platform}//:python_runtimes")
+alias(name = "python3",         actual = "@{py_repository}_{host_platform}//:{python3_binary_path}")
+""".format(
+        py_repository = rctx.attr.user_repository_name,
+        host_platform = host_platform,
+        python3_binary_path = python3_binary_path,
+    )
+    if not is_windows:
+        build_contents += """\
+alias(name = "pip",             actual = "@{py_repository}_{host_platform}//:bin/pip")
+""".format(
+            py_repository = rctx.attr.user_repository_name,
+            host_platform = host_platform,
+        )
+    rctx.file("BUILD.bazel", build_contents)
+
+    # Expose a Starlark file so rules can know what host platform we used and where to find an interpreter
+    # when using repository_ctx.path, which doesn't understand aliases.
+    rctx.file("defs.bzl", content = """\
+# Generated by python/repositories.bzl
+host_platform = "{host_platform}"
+interpreter = "@{py_repository}_{host_platform}//:{python3_binary_path}"
+""".format(
+        py_repository = rctx.attr.user_repository_name,
+        host_platform = host_platform,
+        python3_binary_path = python3_binary_path,
+    ))
+
+host_os_alias = repository_rule(
+    _host_os_alias_impl,
+    doc = """Creates a repository with a shorter name meant for the host platform, which contains
+    a BUILD.bazel file declaring aliases to the host platform's targets.
+    """,
+    attrs = {
+        "user_repository_name": attr.string(
+            mandatory = True,
+            doc = "The base name for all created repositories, like 'python38'.",
+        ),
+    },
+)
+
+def _host_os_arch(rctx):
+    """Infer the host OS name and arch from a repository context.
+
+    Args:
+        rctx: Bazel's repository_ctx.
+    Returns:
+        A tuple with the host OS name and arch.
+    """
+    os_name = rctx.os.name
+
+    # We assume the arch for Windows is always x86_64.
+    if "windows" in os_name.lower():
+        arch = "x86_64"
+
+        # Normalize the os_name. E.g. os_name could be "OS windows server 2019".
+        os_name = WINDOWS_NAME
+    else:
+        # This is not ideal, but bazel doesn't directly expose arch.
+        arch = rctx.execute(["uname", "-m"]).stdout.strip()
+
+        # Normalize the os_name.
+        if "mac" in os_name.lower():
+            os_name = MACOS_NAME
+        elif "linux" in os_name.lower():
+            os_name = LINUX_NAME
+
+    return (os_name, arch)