Use a Python toolchain (Python 3.9)

Should improve reproducibility, and removes the need to customize the requirements lock file (which of course was defeating the point of a lock file in the first place). PiperOrigin-RevId: 571970687
JoshVarga · Oct 9, 2023 · eb2fb6d · eb2fb6d
1 parent 838bff5
commit eb2fb6d
Show file tree

Hide file tree

Showing 14 changed files with 36 additions and 35 deletions.
diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml
@@ -45,10 +45,6 @@ jobs:
       - name: Install dependencies via apt
         run: sudo apt-get install python3-dev python3-distutils python3-dev python-is-python3 libtinfo5
 
-      - name: Bazel update PIP requirements lock
-        run: |
-          "${GITHUB_WORKSPACE}/bin/bazel" run //dependency_support:pip_requirements.update
-
       - name: Bazel fetch
         # Let's fetch all WORKSPACE artifacts to see if they can resolve fully
         run: |

diff --git a/.github/workflows/nightly-ubuntu-22.04.yml b/.github/workflows/nightly-ubuntu-22.04.yml
@@ -32,9 +32,6 @@ jobs:
           mkdir -p "${GITHUB_WORKSPACE}/bin/"
           mv bazelisk-linux-amd64 "${GITHUB_WORKSPACE}/bin/bazel"
           chmod +x "${GITHUB_WORKSPACE}/bin/bazel"
-      - name: Bazel update PIP requirements lock
-        run: |
-          "${GITHUB_WORKSPACE}/bin/bazel" run //dependency_support:pip_requirements.update
       # Note: xlscc is excluded because it builds Clang, and that can make
       # builds time out.
       - name: Bazel Test All (opt)

diff --git a/.github/workflows/one-bazel-test-ubuntu-20.04.yml b/.github/workflows/one-bazel-test-ubuntu-20.04.yml
@@ -44,9 +44,6 @@ jobs:
           chmod +x "${GITHUB_WORKSPACE}/bin/bazel"
       - name: Install dependencies via apt
         run: sudo apt-get install python3-dev python3-distutils python3-dev python-is-python3 libtinfo5
-      - name: Bazel update PIP requirements lock
-        run: |
-          "${GITHUB_WORKSPACE}/bin/bazel" run //dependency_support:pip_requirements.update
       - name: Bazel Test Target (opt)
         run: |
           "${GITHUB_WORKSPACE}/bin/bazel" test -c opt --test_output=all --runs_per_test=$BAZEL_RUNS_PER_TEST -- $BAZEL_TARGET
diff --git a/.github/workflows/one-bazel-test-ubuntu-22.04.yml b/.github/workflows/one-bazel-test-ubuntu-22.04.yml
@@ -44,9 +44,6 @@ jobs:
           chmod +x "${GITHUB_WORKSPACE}/bin/bazel"
       - name: Install dependencies via apt
         run: sudo apt-get install python3-distutils python3-dev python-is-python3 libtinfo5  build-essential liblapack-dev libblas-dev gfortran
-      - name: Bazel update PIP requirements lock
-        run: |
-          "${GITHUB_WORKSPACE}/bin/bazel" run //dependency_support:pip_requirements.update
       - name: Bazel Test Target (opt)
         run: |
           "${GITHUB_WORKSPACE}/bin/bazel" test -c opt --test_output=all --runs_per_test=$BAZEL_RUNS_PER_TEST -- $BAZEL_TARGET
diff --git a/Dockerfile-ubuntu-20.04 b/Dockerfile-ubuntu-20.04
@@ -28,9 +28,6 @@ USER xls-developer
 ADD --chown=xls-developer . /home/xls-developer/xls/
 WORKDIR /home/xls-developer/xls/
 
-# Update pip dependencies
-RUN bazel run -c opt //dependency_support:pip_requirements.update
-
 # Build everything (opt)
 RUN bazel build -c opt ...
 

diff --git a/Dockerfile-ubuntu-20.10 b/Dockerfile-ubuntu-20.10
@@ -31,9 +31,6 @@ USER xls-developer
 ADD --chown=xls-developer . /home/xls-developer/xls/
 WORKDIR /home/xls-developer/xls/
 
-# Update pip dependencies
-RUN bazel run -c opt //dependency_support:pip_requirements.update
-
 
 # Test everything (opt), exclude xlscc for now due to increased build time when
 # we add Clang.

diff --git a/Dockerfile-ubuntu-22.04 b/Dockerfile-ubuntu-22.04
@@ -31,9 +31,6 @@ USER xls-developer
 ADD --chown=xls-developer . /home/xls-developer/xls/
 WORKDIR /home/xls-developer/xls/
 
-# Update pip dependencies
-RUN bazel run -c opt //dependency_support:pip_requirements.update
-
 
 # Test everything (opt), exclude xlscc for now due to increased build time when
 # we add Clang.

diff --git a/README.md b/README.md
@@ -100,11 +100,6 @@ bazel 5.2.0
 ~/xls$ # This is important. Without this step, you may experience cryptic error messages:
 ~/xls$ sudo apt install python3-distutils python3-dev libtinfo5 python-is-python3
 
-~/xls$ # We're also going to update the Python/pip dependency lock file for your Python
-~/xls$ # version, since it's locked to one version by default. Without this, you may
-~/xls$ # also experience cryptic error messages.
-~/xls$ bazel run -c opt -- //xls/dependency_support:pip_requirements.update
-
 ~/xls$ # Now build/test in optimized build mode.
 ~/xls$ # If you don't plan on using the C++ frontend, which is not needed to get started,
 ~/xls$ # use this command line:

diff --git a/WORKSPACE b/WORKSPACE
@@ -49,6 +49,23 @@ load("//dependency_support:load_external.bzl", "load_external_repositories")
 
 load_external_repositories()
 
+load(
+  "@rules_python//python:repositories.bzl",
+  "py_repositories",
+  "python_register_toolchains",
+)
+
+py_repositories()
+
+python_register_toolchains(
+    name = "python39",
+    python_version = "3.9",
+
+    # Required for our containerized CI environments; we do not recommend
+    # building XLS as root normally.
+    ignore_root_user_error = True,
+)
+
 # gRPC deps should be loaded before initializing other repos. Otherwise, various
 # errors occur during repo loading and initialization.
 load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")

diff --git a/dependency_support/initialize_external.bzl b/dependency_support/initialize_external.bzl
@@ -20,6 +20,7 @@ load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps")
 load("@com_grail_bazel_compdb//:deps.bzl", "bazel_compdb_deps")
 load("@io_bazel_rules_closure//closure:repositories.bzl", "rules_closure_dependencies", "rules_closure_toolchains")
+load("@python39//:defs.bzl", python_interpreter_target = "interpreter")
 load("@rules_hdl//:init.bzl", rules_hdl_init = "init")
 load("@rules_hdl//dependency_support:dependency_support.bzl", rules_hdl_dependency_support = "dependency_support")
 load("@rules_hdl//toolchains/cpython:cpython_toolchain.bzl", rules_hdl_register_cpython_repository = "register_cpython_repository")
@@ -32,7 +33,7 @@ def initialize_external_repositories():
     bazel_skylib_workspace()
     protobuf_deps()
     rules_hdl_register_cpython_repository()
-    rules_hdl_init()
+    rules_hdl_init(python_interpreter_target = python_interpreter_target)
     rules_hdl_dependency_support()
     rules_closure_dependencies()
     rules_closure_toolchains()
@@ -42,7 +43,7 @@ def initialize_external_repositories():
     pip_parse(
         name = "xls_pip_deps",
         requirements_lock = "@com_google_xls//dependency_support:pip_requirements_lock.txt",
-        python_interpreter = "python3",
+        python_interpreter_target = python_interpreter_target,
         timeout = 600000,
     )
     initialize_boost()

diff --git a/dependency_support/load_external.bzl b/dependency_support/load_external.bzl
@@ -201,6 +201,10 @@ def load_external_repositories():
         # Instead we patch it.
         #repo_mapping = {"@com_googlesource_code_re2": "@com_github_google_re2"},
         patches = ["@com_google_xls//dependency_support/com_github_grpc_grpc:grpc_patch.diff"],
+        repo_mapping = {
+            "@local_config_python": "@python39",
+            "@system_python": "@python39",
+        },
     )
 
     # Used by xlscc.

diff --git a/dependency_support/pip_requirements_lock.txt b/dependency_support/pip_requirements_lock.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    bazel run //dependency_support:pip_requirements.update
@@ -18,6 +18,10 @@ flask==2.3.2 \
     --hash=sha256:77fd4e1249d8c9923de34907236b747ced06e5467ecac1a7bb7115ae0e9670b0 \
     --hash=sha256:8c2f9abd47a9e8df7f0c3f091ce9497d011dc3b31effcf4c85a6e2b50f4114ef
     # via -r dependency_support/pip_requirements.in
+importlib-metadata==6.8.0 \
+    --hash=sha256:3ebb78df84a805d7698245025b975d9d67053cd94c79245ba4b3eb694abe68bb \
+    --hash=sha256:dbace7892d8c0c4ac1ad096662232f831d4e64f4c4545bd53016a3e9d4654743
+    # via flask
 itsdangerous==2.1.2 \
     --hash=sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44 \
     --hash=sha256:5dbbc68b317e5e42f327f9021763545dc3fc3bfe22e6deb96aaf1fc38874156a
@@ -207,3 +211,7 @@ werkzeug==2.3.3 \
     # via
     #   -r dependency_support/pip_requirements.in
     #   flask
+zipp==3.17.0 \
+    --hash=sha256:0e923e726174922dce09c53c59ad483ff7bbb8e572e00c7f7c46b88556409f31 \
+    --hash=sha256:84e64a1c28cf7e91ed2078bb8cc8c259cb19b76942096c8d7b84947690cabaf0
+    # via importlib-metadata
diff --git a/dependency_support/rules_hdl/workspace.bzl b/dependency_support/rules_hdl/workspace.bzl
@@ -41,4 +41,7 @@ def repo():
         urls = [
             "https://github.com/hdl/bazel_rules_hdl/archive/%s.tar.gz" % git_hash,
         ],
+        repo_mapping = {
+            "@rules_hdl_cpython": "@python39",
+        },
     )
diff --git a/docs_src/README.md b/docs_src/README.md
@@ -96,11 +96,6 @@ bazel 5.2.0
 ~/xls$ # This is important. Without this step, you may experience cryptic error messages:
 ~/xls$ sudo apt install python3-distutils python3-dev libtinfo5 python-is-python3
 
-~/xls$ # We're also going to update the Python/pip dependency lock file for your Python
-~/xls$ # version, since it's locked to one version by default. Without this, you may
-~/xls$ # also experience cryptic error messages.
-~/xls$ bazel run -c opt -- //xls/dependency_support:pip_requirements.update
-
 ~/xls$ # Now build/test in optimized build mode.
 ~/xls$ # If you don't plan on using the C++ frontend, which is not needed to get started,
 ~/xls$ # use this command line: