Refactors RBE support (flutter#49416)

This PR refactors the RBE build and adds support for mac-hosted builds
to use RBE. In particular, configurations files are downloaded from
flutter-internal CIPD instead of being included in the repo. The
download from CIPD is triggered by the presence of the `use_rbe` flag as
a custom var in the `.gclient` file.

This PR also turns on RBE for Linux and macOS builds that do not produce
artifacts.

flutter/buildroot#807

Author: zanderso

.gitignore

@@ -130,4 +130,9 @@ app.*.symbols
 
 # Prebuilt binaries.
 /prebuilts/
+
+# GN build support for protobufs vended by Fuchsia.
 /build/secondary/third_party/protobuf
+
+# RBE support configurations and scripts vended from CIPD
+/build/rbe

DEPS

@@ -54,7 +54,7 @@ vars = {
   # https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client
   'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',
 
-  'reclient_version': 'git_revision:81e819b39d4743462857cc55430d898b9fcca1af',
+  'reclient_version': 'git_revision:f3883c2237b0eb9cc9524cb571b5ab8378f257e4',
 
   'gcloud_version': 'version:2@444.0.0.chromium.3',
 
@@ -127,6 +127,10 @@ vars = {
   # specified by GOMA_DIR, or installed in the default goma install location.
   'use_cipd_goma': False,
 
+  # When this is true, the Flutter Engine's configuration files and scripts for
+  # RBE will be downloaded from CIPD. This option is only usable by Googlers.
+  'use_rbe': False,
+
   # This is not downloaded be default because it increases the
   # `gclient sync` time by between 1 and 3 minutes. This option is enabled
   # in flutter/ci/builders/mac_impeller_cmake_example.json, and is likely to
@@ -267,7 +271,7 @@ allowed_hosts = [
 ]
 
 deps = {
-  'src': 'https://github.com/flutter/buildroot.git' + '@' + '8c274b21f1ad4f2aec0a5e0ae8f4264393045b4b',
+  'src': 'https://github.com/flutter/buildroot.git' + '@' + 'a43582b52d361bc3da156a8e6eab6dd947ca339d',
 
   'src/flutter/third_party/rapidjson':
    Var('flutter_git') + '/third_party/rapidjson' + '@' + 'ef3564c5c8824989393b87df25355baf35ff544b',
@@ -844,7 +848,7 @@ deps = {
         'version': Var('clang_version'),
       }
     ],
-    'condition': 'host_os == "linux" and host_cpu == "x64"',
+    'condition': 'host_os == "linux" or host_os == "mac"',
     'dep_type': 'cipd',
   },
 
@@ -904,15 +908,48 @@ deps = {
     'dep_type': 'cipd',
   },
 
-  # reclient.
+  # RBE binaries and configs.
   'src/buildtools/linux-x64/reclient': {
     'packages': [
       {
         'package': 'infra/rbe/client/${{platform}}',
         'version': Var('reclient_version'),
       }
     ],
-    'condition': 'host_os == "linux" and host_cpu == "x64"',
+    'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/mac-arm64/reclient': {
+    'packages': [
+      {
+        'package': 'infra/rbe/client/${{platform}}',
+        'version': Var('reclient_version'),
+      }
+    ],
+    'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/mac-x64/reclient': {
+    'packages': [
+      {
+        'package': 'infra/rbe/client/${{platform}}',
+        'version': Var('reclient_version'),
+      }
+    ],
+    'condition': 'use_rbe and host_os == "mac" and host_cpu == "x64"',
+    'dep_type': 'cipd',
+  },
+
+  'src/flutter/build/rbe': {
+    'packages': [
+      {
+        'package': 'flutter_internal/rbe/reclient_cfgs',
+        'version': 'U42C0v8jI-_YREjd8rbDEt0evvqvLWJ_NTkaiJ_Clt8C',
+      }
+    ],
+    'condition': 'use_rbe',
     'dep_type': 'cipd',
   },
 
@@ -924,7 +961,18 @@ deps = {
         'version': Var('gcloud_version'),
       }
     ],
-    'condition': 'host_os == "linux" and host_cpu == "x64"',
+    'condition': 'use_rbe and host_os == "linux" and host_cpu == "x64"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/mac-arm64/gcloud': {
+    'packages': [
+      {
+        'package': 'infra/3pp/tools/gcloud/${{platform}}',
+        'version': Var('gcloud_version'),
+      }
+    ],
+    'condition': 'use_rbe and host_os == "mac" and host_cpu == "arm64"',
     'dep_type': 'cipd',
   },
 

ci/builders/linux_android_emulator.json

@@ -7,6 +7,9 @@
                 "kvm=1",
                 "cores=8"
             ],
+            "gclient_variables": {
+                "use_rbe": true
+            },
             "gn": [
                 "--android",
                 "--android-cpu=x64",
@@ -69,6 +72,9 @@
                 "kvm=1",
                 "cores=8"
             ],
+            "gclient_variables": {
+                "use_rbe": true
+            },
             "gn": [
                 "--android",
                 "--android-cpu=x86",

ci/builders/linux_unopt.json

@@ -6,6 +6,9 @@
                 "os=Linux",
                 "cores=32"
             ],
+            "gclient_variables": {
+                "use_rbe": true
+            },
             "gn": [
                 "--runtime-mode",
                 "debug",
@@ -84,6 +87,9 @@
                 "os=Linux"
             ],
             "dependencies": [],
+            "gclient_variables": {
+                "use_rbe": true
+            },
             "gn": [
                 "--android",
                 "--embedder-for-target",
@@ -112,6 +118,9 @@
                     "version": "last_updated:2023-02-03T15:32:01-0800"
                 }
             ],
+            "gclient_variables": {
+                "use_rbe": true
+            },
             "gn": [
                 "--android",
                 "--unoptimized",

ci/builders/mac_unopt.json

@@ -16,15 +16,19 @@
                 "mac_model=Macmini8,1"
             ],
             "gclient_variables": {
-                "download_android_deps": false
+                "download_android_deps": false,
+                "use_rbe": true
             },
             "gn": [
                 "--runtime-mode",
                 "debug",
                 "--unoptimized",
                 "--no-lto",
                 "--prebuilt-dart-sdk",
-                "--enable-impeller-3d"
+                "--enable-impeller-3d",
+                "--rbe",
+                "--no-goma",
+                "--xcode-symlinks"
             ],
             "name": "host_debug_unopt",
             "ninja": {
@@ -75,14 +79,18 @@
                 "cpu=x86"
             ],
             "gclient_variables": {
-                "download_android_deps": false
+                "download_android_deps": false,
+                "use_rbe": true
             },
             "gn": [
                 "--ios",
                 "--runtime-mode",
                 "debug",
                 "--simulator",
-                "--no-lto"
+                "--no-lto",
+                "--rbe",
+                "--no-goma",
+                "--xcode-symlinks"
             ],
             "name": "ios_debug_sim",
             "ninja": {
@@ -132,7 +140,8 @@
                 "cpu=arm64"
             ],
             "gclient_variables": {
-                "download_android_deps": false
+                "download_android_deps": false,
+                "use_rbe": true
             },
             "gn": [
                "--runtime-mode",
@@ -142,7 +151,10 @@
                "--prebuilt-dart-sdk",
                "--force-mac-arm64",
                "--mac-cpu",
-               "arm64"
+               "arm64",
+               "--rbe",
+               "--no-goma",
+               "--xcode-symlinks"
             ],
             "name": "host_debug_unopt_arm64",
             "ninja": {
@@ -181,7 +193,8 @@
                 "cpu=arm64"
             ],
             "gclient_variables": {
-                "download_android_deps": false
+                "download_android_deps": false,
+                "use_rbe": true
             },
             "gn": [
                 "--ios",
@@ -191,7 +204,10 @@
                 "--no-lto",
                 "--force-mac-arm64",
                 "--simulator-cpu",
-                "arm64"
+                "arm64",
+                "--rbe",
+                "--no-goma",
+                "--xcode-symlinks"
             ],
             "name": "ios_debug_sim_arm64",
             "ninja": {
@@ -251,7 +267,8 @@
                 "cpu=arm64"
             ],
             "gclient_variables": {
-                "download_android_deps": false
+                "download_android_deps": false,
+                "use_rbe": true
             },
             "gn": [
                 "--ios",
@@ -262,7 +279,10 @@
                 "--force-mac-arm64",
                 "--simulator-cpu",
                 "arm64",
-                "--darwin-extension-safe"
+                "--darwin-extension-safe",
+                "--rbe",
+                "--no-goma",
+                "--xcode-symlinks"
             ],
             "name": "ios_debug_sim_arm64_extension_safe",
             "ninja": {

ci/builders/standalone/linux_benchmarks.json

@@ -4,7 +4,8 @@
         "os=Linux"
     ],
     "gclient_variables": {
-        "download_android_deps": false
+        "download_android_deps": false,
+        "use_rbe": true
     },
     "gn": [
         "--runtime-mode",

tools/gn

@@ -205,8 +205,6 @@ def buildtools_dir():
   host_cpu = get_host_cpu()
   if host_os == 'win':
     host_os = 'windows'
-  if host_os == 'mac' and host_cpu == 'arm64':
-    host_cpu = 'x64'
   return '%s-%s' % (host_os, host_cpu)
 
 
@@ -217,9 +215,10 @@ def setup_rbe(args):
   if not args.rbe:
     return rbe_gn_args
 
-  if get_host_os() not in ['linux']:
+  if get_host_os() not in ['linux', 'mac']:
     print(
-        'The --rbe flag has no effect. RBE is currently only supported on Linux.'
+        'The --rbe flag has no effect. RBE is currently only supported on '
+        'macOS and Linux.'
     )
     return rbe_gn_args
 
@@ -239,7 +238,9 @@ def setup_rbe(args):
     )
     bootstrap_path = os.path.join(cipd_reclient_dir, 'bootstrap')
     reproxy_path = os.path.join(cipd_reclient_dir, 'reproxy')
-    rbe_cfg_path = os.path.join(SRC_ROOT, 'build', 'rbe.cfg')
+    rbe_cfg_path = os.path.join(
+        SRC_ROOT, 'flutter', 'build', 'rbe', 'reclient.cfg'
+    )
     bootstrap_cmd = [
         bootstrap_path,
         '--re_proxy=' + reproxy_path,
@@ -260,8 +261,20 @@ def setup_rbe(args):
     rbe_gn_args['rbe_dial_timeout'] = args.rbe_dial_timeout
   if args.rbe_platform:
     rbe_gn_args['rbe_platform'] = args.rbe_platform
-  if args.rbe_dir:
-    rbe_gn_args['rbe_dir'] = args.rbe_dir
+
+  rbe_gn_args['rbe_dir'] = os.path.join(
+      SRC_ROOT, 'buildtools', buildtools_dir(), 'reclient'
+  )
+
+  rbe_gn_args['rbe_cfg'] = os.path.join(
+      SRC_ROOT, 'flutter', 'build', 'rbe',
+      'rewrapper-' + buildtools_dir() + '.cfg'
+  )
+
+  if sys.platform == 'darwin':
+    if (not running_on_luci or args.xcode_symlinks or
+        os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'):
+      rbe_gn_args['create_xcode_symlinks'] = True
 
   return rbe_gn_args
 
@@ -285,8 +298,16 @@ def setup_goma(args):
   # care of starting and stopping the compiler proxy.
   running_on_luci = os.environ.get('LUCI_CONTEXT') is not None
 
+  # The GOMA client has no arm64 binary, so run the x64 binary through
+  # Rosetta.
+  buildtools_platform = buildtools_dir()
+  if buildtools_platform == 'mac-arm64':
+    buildtools_platform = 'mac-x64'
+
   # Prefer the goma fetched by gclient if it exists.
-  cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma')
+  cipd_goma_dir = os.path.join(
+      SRC_ROOT, 'buildtools', buildtools_platform, 'goma'
+  )
 
   # Next, if GOMA_DIR is set, use that install.
   goma_dir = os.environ.get('GOMA_DIR')