From 92bf7ddafcdba8728bfaaca314ca0b678c1b165a Mon Sep 17 00:00:00 2001 From: UebelAndre Date: Thu, 24 Feb 2022 08:57:52 -0800 Subject: [PATCH] Added windows support to hermetic toolchains --- .../tests/toolchains/python_version_test.py | 8 +- .../tests/toolchains/versions_test.bzl | 1 + python/private/toolchains_repo.bzl | 6 + python/private/versions.bzl | 33 ++--- python/repositories.bzl | 132 +++++++++--------- 5 files changed, 93 insertions(+), 87 deletions(-) diff --git a/python/private/tests/toolchains/python_version_test.py b/python/private/tests/toolchains/python_version_test.py index ad3596f467..700b6ec5e2 100644 --- a/python/private/tests/toolchains/python_version_test.py +++ b/python/private/tests/toolchains/python_version_test.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest import platform +import unittest + class TestPythonVersion(unittest.TestCase): def test_match_toolchain(self): - self.assertEqual(platform.python_version(), "3.9.7") + self.assertEqual(platform.python_version(), "3.9.10") + -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/python/private/tests/toolchains/versions_test.bzl b/python/private/tests/toolchains/versions_test.bzl index 8650c89565..866249b5c0 100644 --- a/python/private/tests/toolchains/versions_test.bzl +++ b/python/private/tests/toolchains/versions_test.bzl @@ -21,6 +21,7 @@ load("//python/private:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS") required_platforms = [ "x86_64-apple-darwin", + "x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu", ] diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl index 297024f291..c9ad35c6e5 100644 --- a/python/private/toolchains_repo.bzl +++ b/python/private/toolchains_repo.bzl @@ -36,6 +36,12 @@ PLATFORMS = { "@platforms//cpu:x86_64", ], ), + "x86_64-pc-windows-msvc": struct( + compatible_with = [ + "@platforms//os:windows", + "@platforms//cpu:x86_64", + ], + ), "x86_64-unknown-linux-gnu": struct( compatible_with = [ "@platforms//os:linux", diff --git a/python/private/versions.bzl b/python/private/versions.bzl index 2bd32f3da1..be1237c168 100644 --- a/python/private/versions.bzl +++ b/python/private/versions.bzl @@ -15,32 +15,33 @@ """The Python versions we use for the toolchains. """ -RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20211017" -RELEASE_DATE = "20211017T1616" +RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20220222" +RELEASE_DATE = "20220220T1113" -# The integrity hashes can be computed with: -# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64 # buildifier: disable=unsorted-dict-items TOOL_VERSIONS = { "3.8.12": { - "x86_64-apple-darwin": "sha384-es0kCVBb4q5xSC09lOw83TKXtR6qdt0NeU56JtK7Y5M5V784k9MM2q8leE3QWGH6", - "x86_64-unknown-linux-gnu": "sha384-61BjosOUm+Uo2yDtkLcRqbSIAXIPiHKmvySGO2J/GSOHXkj5QBf4cwzrDuOWR0H2", + "x86_64-apple-darwin": "", + "x86_64-pc-windows-msvc": "", + "x86_64-unknown-linux-gnu": "", }, - "3.9.7": { - "x86_64-apple-darwin": "sha384-Ix7lxb+niRWcDCOsK1q53dAkOp0CIw40kM8wHv//WQs2O/il8SRJ60cP3R67fEnm", - "aarch64-apple-darwin": "sha384-JSi3XZdHVZ+6DMybPYKgEYSQuBPpjXgBlj1uGqB9f/r3Wi6P0+CnYRG12TEzgcs6", - "x86_64-unknown-linux-gnu": "sha384-+PNcmKXJ+ZiyKyZ2WR1XedDXJ05ujC2w9TdXl80vloYMqfIOpcVPOWUgre+btI+3", + "3.9.10": { + "aarch64-apple-darwin": "sha256-Ilc5Vg5UC18nPWhc19MJ2LDOVE+OZdAFqd0Tj9lzMQc=", + "x86_64-apple-darwin": "sha256-PFq5e36vnxxgwAQBq4h6xI6tkszPzltsRxHO+BP++Cc=", + "x86_64-pc-windows-msvc": "sha256-zsHiI/F5K5F94SuMjyL5hzc2XdgDEc8G9TP3W6+PJUg=", + "x86_64-unknown-linux-gnu": "sha256-JA7qlmgyWsb+oNkhf3JW6drYLjNVqb3S323RPoE3Tjs=", }, - "3.10.0": { - "x86_64-apple-darwin": "sha384-eVTq704heZyKW6SB/DzarWB9S5bRH1LTQ4rFHzlKTE9jDjHDCBKyQhSgYy8a62lt", - "aarch64-apple-darwin": "sha384-NbhxnZL0pBTKpzEfoCYWl6s2GYdfiI9HOSSHn1iCMZnIY6htt/KhzjwIgCP+Nj2u", - "x86_64-unknown-linux-gnu": "sha384-iYJF9Y9o2Ain3YkuuF7ZGrGuJ+MyiD/xnrjJSap0TF2DR+I9XDx4stunNgx17gSn", + "3.10.2": { + "aarch64-apple-darwin": "sha256-i0dZe0vEh18aqLoV4XB+kq2c5TLmiI4mk6Bcb3rgtAY=", + "x86_64-apple-darwin": "sha256-fxE5lvtHnSJ7T7lLpHcJix+nKfaiGmVSgnp0+rpyeIs=", + "x86_64-pc-windows-msvc": "sha256-0bssoAg5kEOVy9sJFeWW5HZLmlR86Pu3bnsMYC+07Js=", + "x86_64-unknown-linux-gnu": "sha256-TJGxUpHdLEC119ZevCzOOBH3OQ4hZWAM0pucFIrVsG0=", }, } # buildifier: disable=unsorted-dict-items MINOR_MAPPING = { "3.8": "3.8.12", - "3.9": "3.9.7", - "3.10": "3.10.0", + "3.9": "3.9.10", + "3.10": "3.10.2", } diff --git a/python/repositories.bzl b/python/repositories.bzl index 2ec6233cec..3e4f2daf7e 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -27,89 +27,78 @@ def py_repositories(): ######## # Remaining content of the file is only used to support toolchains. ######## -_DOC = "Fetch external tools needed for python toolchain." -_ATTRS = { - "integrity": attr.string(mandatory = True), - "platform": attr.string(mandatory = True, values = PLATFORMS.keys()), - "python_version": attr.string(mandatory = True, values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys()), -} def _python_repository_impl(rctx): python_version = rctx.attr.python_version platform = rctx.attr.platform - release_filename = "cpython-{version}-{platform}-pgo+lto-{release_date}.tar.zst".format( + is_windows = "windows" in platform + build = "shared-install_only" if is_windows else "install_only" + release_filename = "cpython-{version}-{platform}-{build}-{release_date}.tar.gz".format( + build = build, + platform = platform, release_date = RELEASE_DATE, version = python_version, - platform = platform, ) url = "{release_url}/{release_filename}".format( release_url = RELEASE_URL, release_filename = release_filename, ) - integrity = rctx.attr.integrity - rctx.download( + download_result = rctx.download_and_extract( url = url, - integrity = integrity, - output = release_filename, + integrity = rctx.attr.integrity, + stripPrefix = "python", ) - unzstd = rctx.which("unzstd") - if not unzstd: - url = rctx.attr._zstd_url.format(version = rctx.attr._zstd_version) - rctx.download_and_extract( - url = url, - sha256 = rctx.attr._zstd_sha256, - ) - working_directory = "zstd-{version}".format(version = rctx.attr._zstd_version) - rctx.execute( - ["make", "--jobs=4"], - timeout = 600, - quiet = True, - working_directory = working_directory, - ) - zstd = "{working_directory}/zstd".format(working_directory = working_directory) - unzstd = "./unzstd" - rctx.symlink(zstd, unzstd) - - exec_result = rctx.execute([ - "tar", - "--extract", - "--strip-components=2", - "--use-compress-program={unzstd}".format(unzstd = unzstd), - "--file={}".format(release_filename), - ]) - if exec_result.return_code: - fail(exec_result.stderr) # Remove files with spaces. - exec_result = rctx.execute([ - "find", - ".", - "-type", - "f", - "-name", - "*[[:space:]]*", - "-delete", - ]) + if is_windows: + arguments = [ + "powershell.exe", + "-c", + """Get-ChildItem -File -Path "$(Get-Location)" -Include "* *" -Recurse | Remove-Item -Force -Verbose""", + ] + else: + arguments = [ + "find", + ".", + "-type", + "f", + "-name", + "*[[:space:]]*", + "-delete", + ] + exec_result = rctx.execute(arguments) if exec_result.return_code: fail(exec_result.stderr) + python_bin = "python.exe" if is_windows else "bin/python3" + build_content = """\ # Generated by python/repositories.bzl load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") +package(default_visibility = ["//visibility:public"]) + filegroup( name = "files", - srcs = glob([ - "bin/**", - "include/**", - "lib/**", - "share/**", - ]), - visibility = ["//visibility:public"], + srcs = glob( + include = [ + "bin/**", + "DLLs/**", + "extensions/**", + "include/**", + "lib/**", + "libs/**", + "Scripts/**", + "share/**", + ], + exclude = [ + "**/__pycache__/**", + ], + ), ) -exports_files(["bin/python3"]) +exports_files(["{python_path}"]) py_runtime( name = "py2_runtime", @@ -120,7 +109,7 @@ py_runtime( py_runtime( name = "py3_runtime", files = [":files"], - interpreter = "bin/python3", + interpreter = "{python_path}", python_version = "PY3", ) @@ -129,11 +118,13 @@ py_runtime_pair( py2_runtime = ":py2_runtime", py3_runtime = ":py3_runtime", ) -""" +""".format( + python_path = python_bin, + ) rctx.file("BUILD.bazel", build_content) return { - "integrity": integrity, + "integrity": download_result.integrity, "name": rctx.attr.name, "platform": platform, "python_version": python_version, @@ -141,18 +132,23 @@ py_runtime_pair( python_repository = repository_rule( _python_repository_impl, - doc = _DOC, - attrs = dict({ - "_zstd_sha256": attr.string( - default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0", + doc = "Fetch external tools needed for python toolchain.", + attrs = { + "integrity": attr.string( + doc = "TODO", + mandatory = True, ), - "_zstd_url": attr.string( - default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz", + "platform": attr.string( + doc = "TODO", + mandatory = True, + values = PLATFORMS.keys(), ), - "_zstd_version": attr.string( - default = "1.5.2", + "python_version": attr.string( + doc = "TODO", + mandatory = True, + values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys(), ), - }, **_ATTRS), + }, ) # Wrapper macro around everything above, this is the primary API.