Added windows support to hermetic toolchains

bazelbuild · Feb 26, 2022 · 92bf7dd · 92bf7dd
1 parent 66f46dd
commit 92bf7dd
Show file tree

Hide file tree

Showing 5 changed files with 93 additions and 87 deletions.
diff --git a/python/private/tests/toolchains/python_version_test.py b/python/private/tests/toolchains/python_version_test.py
@@ -12,12 +12,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import unittest
 import platform
+import unittest
+
 
 class TestPythonVersion(unittest.TestCase):
     def test_match_toolchain(self):
-        self.assertEqual(platform.python_version(), "3.9.7")
+        self.assertEqual(platform.python_version(), "3.9.10")
+
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     unittest.main()
diff --git a/python/private/tests/toolchains/versions_test.bzl b/python/private/tests/toolchains/versions_test.bzl
@@ -21,6 +21,7 @@ load("//python/private:versions.bzl", "MINOR_MAPPING", "TOOL_VERSIONS")
 
 required_platforms = [
     "x86_64-apple-darwin",
+    "x86_64-pc-windows-msvc",
     "x86_64-unknown-linux-gnu",
 ]
 

diff --git a/python/private/toolchains_repo.bzl b/python/private/toolchains_repo.bzl
@@ -36,6 +36,12 @@ PLATFORMS = {
             "@platforms//cpu:x86_64",
         ],
     ),
+    "x86_64-pc-windows-msvc": struct(
+        compatible_with = [
+            "@platforms//os:windows",
+            "@platforms//cpu:x86_64",
+        ],
+    ),
     "x86_64-unknown-linux-gnu": struct(
         compatible_with = [
             "@platforms//os:linux",

diff --git a/python/private/versions.bzl b/python/private/versions.bzl
@@ -15,32 +15,33 @@
 """The Python versions we use for the toolchains.
 """
 
-RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20211017"
-RELEASE_DATE = "20211017T1616"
+RELEASE_URL = "https://github.com/indygreg/python-build-standalone/releases/download/20220222"
+RELEASE_DATE = "20220220T1113"
 
-# The integrity hashes can be computed with:
-# shasum -b -a 384 [downloaded file] | awk '{ print $1 }' | xxd -r -p | base64
 # buildifier: disable=unsorted-dict-items
 TOOL_VERSIONS = {
     "3.8.12": {
-        "x86_64-apple-darwin": "sha384-es0kCVBb4q5xSC09lOw83TKXtR6qdt0NeU56JtK7Y5M5V784k9MM2q8leE3QWGH6",
-        "x86_64-unknown-linux-gnu": "sha384-61BjosOUm+Uo2yDtkLcRqbSIAXIPiHKmvySGO2J/GSOHXkj5QBf4cwzrDuOWR0H2",
+        "x86_64-apple-darwin": "",
+        "x86_64-pc-windows-msvc": "",
+        "x86_64-unknown-linux-gnu": "",
     },
-    "3.9.7": {
-        "x86_64-apple-darwin": "sha384-Ix7lxb+niRWcDCOsK1q53dAkOp0CIw40kM8wHv//WQs2O/il8SRJ60cP3R67fEnm",
-        "aarch64-apple-darwin": "sha384-JSi3XZdHVZ+6DMybPYKgEYSQuBPpjXgBlj1uGqB9f/r3Wi6P0+CnYRG12TEzgcs6",
-        "x86_64-unknown-linux-gnu": "sha384-+PNcmKXJ+ZiyKyZ2WR1XedDXJ05ujC2w9TdXl80vloYMqfIOpcVPOWUgre+btI+3",
+    "3.9.10": {
+        "aarch64-apple-darwin": "sha256-Ilc5Vg5UC18nPWhc19MJ2LDOVE+OZdAFqd0Tj9lzMQc=",
+        "x86_64-apple-darwin": "sha256-PFq5e36vnxxgwAQBq4h6xI6tkszPzltsRxHO+BP++Cc=",
+        "x86_64-pc-windows-msvc": "sha256-zsHiI/F5K5F94SuMjyL5hzc2XdgDEc8G9TP3W6+PJUg=",
+        "x86_64-unknown-linux-gnu": "sha256-JA7qlmgyWsb+oNkhf3JW6drYLjNVqb3S323RPoE3Tjs=",
     },
-    "3.10.0": {
-        "x86_64-apple-darwin": "sha384-eVTq704heZyKW6SB/DzarWB9S5bRH1LTQ4rFHzlKTE9jDjHDCBKyQhSgYy8a62lt",
-        "aarch64-apple-darwin": "sha384-NbhxnZL0pBTKpzEfoCYWl6s2GYdfiI9HOSSHn1iCMZnIY6htt/KhzjwIgCP+Nj2u",
-        "x86_64-unknown-linux-gnu": "sha384-iYJF9Y9o2Ain3YkuuF7ZGrGuJ+MyiD/xnrjJSap0TF2DR+I9XDx4stunNgx17gSn",
+    "3.10.2": {
+        "aarch64-apple-darwin": "sha256-i0dZe0vEh18aqLoV4XB+kq2c5TLmiI4mk6Bcb3rgtAY=",
+        "x86_64-apple-darwin": "sha256-fxE5lvtHnSJ7T7lLpHcJix+nKfaiGmVSgnp0+rpyeIs=",
+        "x86_64-pc-windows-msvc": "sha256-0bssoAg5kEOVy9sJFeWW5HZLmlR86Pu3bnsMYC+07Js=",
+        "x86_64-unknown-linux-gnu": "sha256-TJGxUpHdLEC119ZevCzOOBH3OQ4hZWAM0pucFIrVsG0=",
     },
 }
 
 # buildifier: disable=unsorted-dict-items
 MINOR_MAPPING = {
     "3.8": "3.8.12",
-    "3.9": "3.9.7",
-    "3.10": "3.10.0",
+    "3.9": "3.9.10",
+    "3.10": "3.10.2",
 }
diff --git a/python/repositories.bzl b/python/repositories.bzl
@@ -27,89 +27,78 @@ def py_repositories():
 ########
 # Remaining content of the file is only used to support toolchains.
 ########
-_DOC = "Fetch external tools needed for python toolchain."
-_ATTRS = {
-    "integrity": attr.string(mandatory = True),
-    "platform": attr.string(mandatory = True, values = PLATFORMS.keys()),
-    "python_version": attr.string(mandatory = True, values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys()),
-}
 
 def _python_repository_impl(rctx):
     python_version = rctx.attr.python_version
     platform = rctx.attr.platform
-    release_filename = "cpython-{version}-{platform}-pgo+lto-{release_date}.tar.zst".format(
+    is_windows = "windows" in platform
+    build = "shared-install_only" if is_windows else "install_only"
+    release_filename = "cpython-{version}-{platform}-{build}-{release_date}.tar.gz".format(
+        build = build,
+        platform = platform,
         release_date = RELEASE_DATE,
         version = python_version,
-        platform = platform,
     )
     url = "{release_url}/{release_filename}".format(
         release_url = RELEASE_URL,
         release_filename = release_filename,
     )
-    integrity = rctx.attr.integrity
-    rctx.download(
+    download_result = rctx.download_and_extract(
         url = url,
-        integrity = integrity,
-        output = release_filename,
+        integrity = rctx.attr.integrity,
+        stripPrefix = "python",
     )
-    unzstd = rctx.which("unzstd")
-    if not unzstd:
-        url = rctx.attr._zstd_url.format(version = rctx.attr._zstd_version)
-        rctx.download_and_extract(
-            url = url,
-            sha256 = rctx.attr._zstd_sha256,
-        )
-        working_directory = "zstd-{version}".format(version = rctx.attr._zstd_version)
-        rctx.execute(
-            ["make", "--jobs=4"],
-            timeout = 600,
-            quiet = True,
-            working_directory = working_directory,
-        )
-        zstd = "{working_directory}/zstd".format(working_directory = working_directory)
-        unzstd = "./unzstd"
-        rctx.symlink(zstd, unzstd)
-
-    exec_result = rctx.execute([
-        "tar",
-        "--extract",
-        "--strip-components=2",
-        "--use-compress-program={unzstd}".format(unzstd = unzstd),
-        "--file={}".format(release_filename),
-    ])
-    if exec_result.return_code:
-        fail(exec_result.stderr)
 
     # Remove files with spaces.
-    exec_result = rctx.execute([
-        "find",
-        ".",
-        "-type",
-        "f",
-        "-name",
-        "*[[:space:]]*",
-        "-delete",
-    ])
+    if is_windows:
+        arguments = [
+            "powershell.exe",
+            "-c",
+            """Get-ChildItem -File -Path "$(Get-Location)" -Include "* *" -Recurse | Remove-Item -Force -Verbose""",
+        ]
+    else:
+        arguments = [
+            "find",
+            ".",
+            "-type",
+            "f",
+            "-name",
+            "*[[:space:]]*",
+            "-delete",
+        ]
+    exec_result = rctx.execute(arguments)
     if exec_result.return_code:
         fail(exec_result.stderr)
 
+    python_bin = "python.exe" if is_windows else "bin/python3"
+
     build_content = """\
 # Generated by python/repositories.bzl
 
 load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair")
 
+package(default_visibility = ["//visibility:public"])
+
 filegroup(
     name = "files",
-    srcs = glob([
-        "bin/**",
-        "include/**",
-        "lib/**",
-        "share/**",
-    ]),
-    visibility = ["//visibility:public"],
+    srcs = glob(
+        include = [
+            "bin/**",
+            "DLLs/**",
+            "extensions/**",
+            "include/**",
+            "lib/**",
+            "libs/**",
+            "Scripts/**",
+            "share/**",
+        ],
+        exclude = [
+            "**/__pycache__/**",
+        ],
+    ),
 )
 
-exports_files(["bin/python3"])
+exports_files(["{python_path}"])
 
 py_runtime(
     name = "py2_runtime",
@@ -120,7 +109,7 @@ py_runtime(
 py_runtime(
     name = "py3_runtime",
     files = [":files"],
-    interpreter = "bin/python3",
+    interpreter = "{python_path}",
     python_version = "PY3",
 )
 
@@ -129,30 +118,37 @@ py_runtime_pair(
     py2_runtime = ":py2_runtime",
     py3_runtime = ":py3_runtime",
 )
-"""
+""".format(
+        python_path = python_bin,
+    )
     rctx.file("BUILD.bazel", build_content)
 
     return {
-        "integrity": integrity,
+        "integrity": download_result.integrity,
         "name": rctx.attr.name,
         "platform": platform,
         "python_version": python_version,
     }
 
 python_repository = repository_rule(
     _python_repository_impl,
-    doc = _DOC,
-    attrs = dict({
-        "_zstd_sha256": attr.string(
-            default = "7c42d56fac126929a6a85dbc73ff1db2411d04f104fae9bdea51305663a83fd0",
+    doc = "Fetch external tools needed for python toolchain.",
+    attrs = {
+        "integrity": attr.string(
+            doc = "TODO",
+            mandatory = True,
         ),
-        "_zstd_url": attr.string(
-            default = "https://github.com/facebook/zstd/releases/download/v{version}/zstd-{version}.tar.gz",
+        "platform": attr.string(
+            doc = "TODO",
+            mandatory = True,
+            values = PLATFORMS.keys(),
         ),
-        "_zstd_version": attr.string(
-            default = "1.5.2",
+        "python_version": attr.string(
+            doc = "TODO",
+            mandatory = True,
+            values = TOOL_VERSIONS.keys() + MINOR_MAPPING.keys(),
         ),
-    }, **_ATTRS),
+    },
 )
 
 # Wrapper macro around everything above, this is the primary API.