diff --git a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java
index c3456eb687968c..da81887c632906 100644
--- a/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java
+++ b/src/main/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloader.java
@@ -23,6 +23,7 @@
 import build.bazel.remote.execution.v2.RequestMetadata;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
 import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
 import com.google.devtools.build.lib.bazel.repository.downloader.Downloader;
 import com.google.devtools.build.lib.bazel.repository.downloader.HashOutputStream;
@@ -171,7 +172,7 @@ static FetchBlobRequest newFetchBlobRequest(
       requestBuilder.addQualifiers(
           Qualifier.newBuilder()
               .setName(QUALIFIER_AUTH_HEADERS)
-              .setValue(authHeadersJson(authHeaders))
+              .setValue(authHeadersJson(urls, authHeaders))
               .build());
     }
 
@@ -197,15 +198,24 @@ private OutputStream newOutputStream(
     return out;
   }
 
-  private static String authHeadersJson(Map<URI, Map<String, String>> authHeaders) {
+  private static String authHeadersJson(
+      List<URL> urls, Map<URI, Map<String, String>> authHeaders) {
+    ImmutableSet<String> hostSet =
+        urls.stream().map(URL::getHost).collect(ImmutableSet.toImmutableSet());
     Map<String, JsonObject> subObjects = new TreeMap<>();
     for (Map.Entry<URI, Map<String, String>> entry : authHeaders.entrySet()) {
+      URI uri = entry.getKey();
+      // Only add headers that are relevant to the hosts.
+      if (!hostSet.contains(uri.getHost())) {
+        continue;
+      }
+
       JsonObject subObject = new JsonObject();
       Map<String, String> orderedHeaders = new TreeMap<>(entry.getValue());
       for (Map.Entry<String, String> subEntry : orderedHeaders.entrySet()) {
         subObject.addProperty(subEntry.getKey(), subEntry.getValue());
       }
-      subObjects.put(entry.getKey().toString(), subObject);
+      subObjects.put(uri.toString(), subObject);
     }
 
     JsonObject authHeadersJson = new JsonObject();
diff --git a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java
index 699076407647c2..6995ca911cb1fd 100644
--- a/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java
+++ b/src/test/java/com/google/devtools/build/lib/remote/downloader/GrpcRemoteDownloaderTest.java
@@ -320,9 +320,6 @@ public void testFetchBlobRequest() throws Exception {
             + "\"http://example.com\":{"
             + "\"Another-Header\":\"another header content\","
             + "\"Some-Header\":\"some header content\""
-            + "},"
-            + "\"http://example.org\":{"
-            + "\"Org-Header\":\"org header content\""
             + "}"
             + "}";