From 7476ef43eac6383be586448bef66ae6c11f32ac6 Mon Sep 17 00:00:00 2001 From: Nicolas Lopez Date: Wed, 2 Oct 2019 14:15:14 -0400 Subject: [PATCH] more platform fixes for rbe --- .bazelci/presubmit.yml | 16 ++++---- WORKSPACE | 7 +++- python/image.bzl | 4 +- python3/image.bzl | 4 +- toolchains/BUILD | 84 +++++++++++++++++++++++++++++++++++++++++ toolchains/python/BUILD | 52 +------------------------ 6 files changed, 103 insertions(+), 64 deletions(-) create mode 100644 toolchains/BUILD diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml index b999d9084..b4291c974 100644 --- a/.bazelci/presubmit.yml +++ b/.bazelci/presubmit.yml @@ -127,10 +127,10 @@ platforms: # Disabled e2e tests that pull from localhost in nested workspace - "-//testing/new_pusher_tests/..." build_flags: - - "--extra_toolchains=@bazel_toolchains//configs/ubuntu16_04_clang/latest:toolchain_docker" - - "--extra_execution_platforms=@bazel_toolchains//configs/ubuntu16_04_clang/latest:platform_docker" - - "--host_platform=@bazel_toolchains//configs/ubuntu16_04_clang/latest:platform_docker" - - "--platforms=@bazel_toolchains//configs/ubuntu16_04_clang/latest:platform_docker" + - "--extra_toolchains=@buildkite_config//config:cc-toolchain,//toolchains:rbe_cc_toolchain" + - "--extra_execution_platforms=//toolchains:rbe_platform,//toolchains:rbe_container_platform" + - "--host_platform=//toolchains:rbe_platform" + - "--platforms=//toolchains:rbe_platform" - "--keep_going" # For tests/container:set_env_make_vars - "--define=ENV_KEY=my_key" @@ -160,10 +160,10 @@ platforms: - "-//tests/contrib/automatic_container_release:configs_test" - "-//tests/contrib/automatic_container_release:configs_test_deps_spec_only" test_flags: - - "--extra_toolchains=@buildkite_config//cc:toolchain" - - "--extra_execution_platforms=//toolchains/python:rbe_platform" - - "--host_platform=//toolchains/python:rbe_platform" - - "--platforms=//toolchains/python:rbe_platform" + - "--extra_toolchains=@buildkite_config//config:cc-toolchain,//toolchains:rbe_cc_toolchain" + - "--extra_execution_platforms=//toolchains:rbe_platform,//toolchains:rbe_container_platform" + - "--host_platform=//toolchains:rbe_platform" + - "--platforms=//toolchains:rbe_platform" - "--keep_going" # For tests/container:set_env_make_vars_test - "--define=ENV_KEY=my_key" diff --git a/WORKSPACE b/WORKSPACE index 537515331..d682033c4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -403,7 +403,7 @@ dockerfile_image( # Register the default py_toolchain / platform for containerized execution register_toolchains("//toolchains/python:container_py_toolchain") -register_execution_platforms("//toolchains/python:local_container_platform") +register_execution_platforms("//toolchains:local_container_platform") http_archive( name = "bazel_toolchains", @@ -418,6 +418,11 @@ load("@bazel_toolchains//rules:rbe_repo.bzl", "rbe_autoconfig") rbe_autoconfig( name = "buildkite_config", + base_container_digest = "sha256: 4bfd33aa9ce73e28718385b8c01608a79bc6546906f01cf9329311cace1766a1", + digest = "sha256:c20046852a2d7910c55d76e0ec9c182b37532a9f0360d22dd5c9a1451b7c3a15", + registry = "marketplace.gcr.io", + repository = "google/bazel", + use_legacy_platform_definition = False, ) # gazelle:repo bazel_gazelle diff --git a/python/image.bzl b/python/image.bzl index 7563af431..5cf1ca390 100644 --- a/python/image.bzl +++ b/python/image.bzl @@ -43,7 +43,7 @@ def repositories(): # Register the default py_toolchain / platform for containerized execution native.register_toolchains("@io_bazel_rules_docker//toolchains/python:container_py_toolchain") - native.register_execution_platforms("@io_bazel_rules_docker//toolchains/python:local_container_platform") + native.register_execution_platforms("@io_bazel_rules_docker//toolchains:local_container_platform") excludes = native.existing_rules().keys() if "py_image_base" not in excludes: @@ -96,7 +96,7 @@ def py_image(name, base = None, deps = [], layers = [], **kwargs): name = binary_name, python_version = "PY2", deps = deps + layers, - exec_compatible_with = ["@io_bazel_rules_docker//toolchains/python:run_in_container"], + exec_compatible_with = ["@io_bazel_rules_docker//toolchains:run_in_container"], **kwargs ) diff --git a/python3/image.bzl b/python3/image.bzl index c8e5f4a6a..c3561020c 100644 --- a/python3/image.bzl +++ b/python3/image.bzl @@ -42,7 +42,7 @@ def repositories(): # Register the default py_toolchain / platform for containerized execution native.register_toolchains("@io_bazel_rules_docker//toolchains/python:container_py_toolchain") - native.register_execution_platforms("@io_bazel_rules_docker//toolchains/python:local_container_platform") + native.register_execution_platforms("@io_bazel_rules_docker//toolchains:local_container_platform") excludes = native.existing_rules().keys() if "py3_image_base" not in excludes: @@ -90,7 +90,7 @@ def py3_image(name, base = None, deps = [], layers = [], **kwargs): name = binary_name, python_version = "PY3", deps = deps + layers, - exec_compatible_with = ["@io_bazel_rules_docker//toolchains/python:run_in_container"], + exec_compatible_with = ["@io_bazel_rules_docker//toolchains:run_in_container"], **kwargs ) diff --git a/toolchains/BUILD b/toolchains/BUILD new file mode 100644 index 000000000..24b85ae6b --- /dev/null +++ b/toolchains/BUILD @@ -0,0 +1,84 @@ +# Copyright 2017 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@bazel_toolchains//rules/experimental/rbe:exec_properties.bzl", "merge_dicts") +load("@exec_properties//:constants.bzl", "DOCKER_SIBLINGS_CONTAINERS", "NETWORK_ON") + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # Apache 2.0 + +toolchain( + name = "rbe_cc_toolchain", + exec_compatible_with = [ + "@io_bazel_rules_docker//toolchains:run_in_container", + "@bazel_tools//platforms:x86_64", + "@bazel_tools//platforms:linux", + "@bazel_tools//tools/cpp:clang", + ], + target_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + toolchain = "@buildkite_config//cc:cc-compiler-k8", + toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", +) + +# Default host platform that has constraint_value set for the +# "@io_bazel_rules_docker//toolchains:run_in_container" constraint +platform( + name = "local_container_platform", + constraint_values = [ + "@io_bazel_rules_docker//toolchains:run_in_container", + ], + cpu_constraints = [ + "@platforms//cpu:x86_32", + "@platforms//cpu:x86_64", + "@platforms//cpu:ppc", + "@platforms//cpu:arm", + "@platforms//cpu:aarch64", + "@platforms//cpu:s390x", + ], + host_platform = True, + os_constraints = [ + "@platforms//os:osx", + "@platforms//os:freebsd", + "@platforms//os:linux", + "@platforms//os:windows", + ], +) + +platform( + name = "rbe_container_platform", + constraint_values = [ + "@io_bazel_rules_docker//toolchains:run_in_container", + ], + exec_properties = merge_dicts(DOCKER_SIBLINGS_CONTAINERS, NETWORK_ON), + parents = ["@buildkite_config//config:platform"], +) + +platform( + name = "rbe_platform", + exec_properties = merge_dicts(DOCKER_SIBLINGS_CONTAINERS, NETWORK_ON), + parents = ["@buildkite_config//config:platform"], +) + +# Constraints used to describe docker compatible toolchains. + +constraint_setting(name = "docker") + +constraint_value( + name = "run_in_container", + constraint_setting = ":docker", +) diff --git a/toolchains/python/BUILD b/toolchains/python/BUILD index 7b70c179e..bbd085318 100644 --- a/toolchains/python/BUILD +++ b/toolchains/python/BUILD @@ -14,8 +14,6 @@ load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") -load("@exec_properties//:constants.bzl", "DOCKER_SIBLINGS_CONTAINERS", "NETWORK_ON") - package(default_visibility = ["//visibility:public"]) licenses(["notice"]) # Apache 2.0 @@ -49,56 +47,8 @@ py_runtime_pair( toolchain( name = "container_py_toolchain", exec_compatible_with = [ - "@io_bazel_rules_docker//toolchains/python:run_in_container", + "@io_bazel_rules_docker//toolchains:run_in_container", ], toolchain = ":default_container_py_runtime_pair", toolchain_type = "@bazel_tools//tools/python:toolchain_type", ) - -# Default host platform that has constraint_value set for the -# "@io_bazel_rules_docker//toolchains/python:run_in_container" constraint -platform( - name = "local_container_platform", - constraint_values = [ - "@io_bazel_rules_docker//toolchains/python:run_in_container", - ], - cpu_constraints = [ - "@platforms//cpu:x86_32", - "@platforms//cpu:x86_64", - "@platforms//cpu:ppc", - "@platforms//cpu:arm", - "@platforms//cpu:aarch64", - "@platforms//cpu:s390x", - ], - host_platform = True, - os_constraints = [ - "@platforms//os:osx", - "@platforms//os:freebsd", - "@platforms//os:linux", - "@platforms//os:windows", - ], -) - -platform( - name = "rbe_container_platform", - constraint_values = [ - "@io_bazel_rules_docker//toolchains/python:run_in_container", - ], - parents = ["@buildkite_config//config:platform"], - exec_properties = DOCKER_SIBLINGS_CONTAINERS, -) - -platform( - name = "rbe_platform", - parents = ["@buildkite_config//config:platform"], - exec_properties = DOCKER_SIBLINGS_CONTAINERS, -) - -# Constraints used to describe docker compatible toolchains. - -constraint_setting(name = "docker") - -constraint_value( - name = "run_in_container", - constraint_setting = ":docker", -)