[reclient] Add option for remote links.

The configuration file is _not_ production ready, as the docker
container image is likely larger than needed - so using remote
links is not yet production ready. But the functionality technicaly
works, and remote links are required for some other applications in the
future, such as not downloading intermediates.

Note that it adding nacl link support shouldn't be much extra work from
now (I expect a new config file is all we need), but it's not work
that seems worth doing ATM.

I also took the chance to trim down hardcoded inputs a bit more, except
for the introduced links where we do need some of those assets (eg ldd).

Change-Id: I8e1d36ce2f7099ab2ffad3bc611dc02111f975e5
Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/2910712
Auto-Submit: Rubens Farias <rubensf@google.com>
Reviewed-by: Dirk Pranke <dpranke@google.com>
Commit-Queue: Dirk Pranke <dpranke@google.com>
Cr-Commit-Position: refs/heads/master@{#885618}

build/toolchain/gcc_toolchain.gni

@@ -206,16 +206,26 @@ template("gcc_toolchain") {
     } else {
       compiler_prefix = "${toolchain_cc_wrapper} "
     }
-    if (use_goma_thin_lto && toolchain_uses_goma && use_thin_lto) {
+
+    if (defined(toolchain_args.use_rbe_links)) {
+      toolchain_uses_rbe_links = toolchain_args.use_rbe_links
+    } else {
+      toolchain_uses_rbe_links = use_rbe_links
+    }
+
+    if (toolchain_uses_rbe_links) {
+      link_prefix = "${rbe_bin_dir}/rewrapper -cfg=../../buildtools/reclient_cfgs/rewrapper_linux_link.cfg -exec_root=${rbe_exec_root} "
+      not_needed([ "goma_path" ])
+    } else if (use_goma_thin_lto && toolchain_uses_goma && use_thin_lto) {
       # goma_ld.py uses autoninja in an attempt to set a reasonable
       # number of jobs, but this results in too low a value on
       # Chrome OS builders. So we pass in an explicit value.
-      goma_ld =
+      link_prefix =
           "$python_path " +
           rebase_path("//tools/clang/scripts/goma_ld.py", root_build_dir) +
           " --gomacc ${goma_path} --jobs 200 -- "
     } else {
-      goma_ld = ""
+      link_prefix = ""
       not_needed([ "goma_path" ])
     }
 
@@ -268,7 +278,7 @@ template("gcc_toolchain") {
     cxx = compiler_prefix + invoker.cxx
     asm = asm_prefix + invoker.cc
     ar = invoker.ar
-    ld = "$goma_ld${invoker.ld}"
+    ld = link_prefix + invoker.ld
     if (defined(invoker.readelf)) {
       readelf = invoker.readelf
     } else {
@@ -408,7 +418,10 @@ template("gcc_toolchain") {
       soname = "{{target_output_name}}{{output_extension}}"  # e.g. "libfoo.so".
       sofile = "{{output_dir}}/$soname"  # Possibly including toolchain dir.
       rspfile = sofile + ".rsp"
-      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      if (!use_rbe_links) {
+        pool = "//build/toolchain:link_pool($default_toolchain)"
+      }
 
       if (defined(invoker.strip)) {
         unstripped_sofile = "{{root_out_dir}}/lib.unstripped/$soname"
@@ -504,7 +517,10 @@ template("gcc_toolchain") {
       soname = "{{target_output_name}}{{output_extension}}"  # e.g. "libfoo.so".
       sofile = "{{output_dir}}/$soname"
       rspfile = sofile + ".rsp"
-      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      if (!use_rbe_links) {
+        pool = "//build/toolchain:link_pool($default_toolchain)"
+      }
 
       if (defined(invoker.strip)) {
         unstripped_sofile = "{{root_out_dir}}/lib.unstripped/$soname"
@@ -550,7 +566,10 @@ template("gcc_toolchain") {
       outfile = "{{output_dir}}/$exename"
       rspfile = "$outfile.rsp"
       unstripped_outfile = outfile
-      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      if (!use_rbe_links) {
+        pool = "//build/toolchain:link_pool($default_toolchain)"
+      }
 
       # Use this for {{output_extension}} expansions unless a target manually
       # overrides it (in which case {{output_extension}} will be what the target

build/toolchain/nacl_toolchain.gni

@@ -66,6 +66,9 @@ template("nacl_toolchain") {
           rbe_cc_cfg_file = "${rbe_cfg_dir}/rewrapper_linux_nacl.cfg"
         }
       }
+
+      # Nacl Links are not yet supported.
+      use_rbe_links = false
     }
   }
 }

build/toolchain/rbe.gni

@@ -16,6 +16,9 @@ declare_args() {
   # Set to true to enable remote compilation using RBE.
   use_rbe = false
 
+  # Set to true to enable remote compilation of links using RBE.
+  use_rbe_links = false
+
   # Set to the path of the RBE reclient configuration file.
   rbe_cc_cfg_file = ""
 }

buildtools/reclient_cfgs/rewrapper_linux.cfg

@@ -2,4 +2,4 @@ platform=container-image=docker://gcr.io/cloud-marketplace/google/rbe-ubuntu16-0
 server_address=unix:///tmp/reproxy.sock
 labels=type=compile,compiler=clang,lang=cpp
 exec_strategy=remote_local_fallback
-inputs=third_party/llvm-build/Release+Asserts/bin/clang,third_party/llvm-build/Release+Asserts/bin/clang++,third_party/llvm-build/Release+Asserts/bin/clang-cl,third_party/llvm-build/Release+Asserts/lib/libstdc++.so.6
+inputs=third_party/llvm-build/Release+Asserts/lib/libstdc++.so.6

buildtools/reclient_cfgs/rewrapper_linux_link.cfg

@@ -0,0 +1,5 @@
+platform=container-image=docker://gcr.io/foundry-x-experiments/chrome-test-builder@sha256:d3646f1c57b57076684cd9f152033acb6d16427231abcbe45c0a36f24148aca4
+server_address=unix:///tmp/reproxy.sock
+labels=type=link,tool=clang
+exec_strategy=remote
+inputs=third_party/llvm-build/Release+Asserts,build/linux/debian_sid_amd64-sysroot/usr/lib/gcc

buildtools/reclient_cfgs/rewrapper_linux_nacl.cfg

@@ -2,4 +2,4 @@ platform=container-image=docker://gcr.io/cloud-marketplace/google/rbe-ubuntu16-0
 server_address=unix:///tmp/reproxy.sock
 labels=type=compile,compiler=nacl,lang=cpp
 exec_strategy=remote_local_fallback
-inputs=third_party/llvm-build/Release+Asserts/bin/clang,third_party/llvm-build/Release+Asserts/bin/clang++,third_party/llvm-build/Release+Asserts/bin/clang-cl,third_party/llvm-build/Release+Asserts/lib/libstdc++.so.6,native_client/toolchain/linux_x86/pnacl_newlib
+inputs=third_party/llvm-build/Release+Asserts/lib/libstdc++.so.6,native_client/toolchain/linux_x86/pnacl_newlib