[8.0] Add support for HTTP Proxies for the GCS repository (#82737)

docs/reference/snapshot-restore/repository-gcs.asciidoc

@@ -191,6 +191,16 @@ are marked as `Secure`.
     can be specified explicitly. For example, it can be used to switch between projects when the
     same credentials are usable for both the production and the development projects.
 
+`proxy.host`::
+    The host name of a proxy to connect to the Google Cloud Storage through.
+
+`proxy.port`::
+    The port of a proxy to connect to the Google Cloud Storage through.
+
+`proxy.type`::
+    The proxy type for the client. Supported values are `direct`, `http`, and `socks`.
+    The default value is `direct` (no proxy).
+
 [[repository-gcs-repository]]
 ==== Repository Settings
 

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettings.java

@@ -14,15 +14,22 @@
 import org.elasticsearch.common.settings.SecureSetting;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsException;
+import org.elasticsearch.core.Nullable;
 import org.elasticsearch.core.TimeValue;
 
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.UncheckedIOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.net.URI;
+import java.net.UnknownHostException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Locale;
 import java.util.Map;
 import java.util.function.Function;
 
@@ -90,6 +97,29 @@ public class GoogleCloudStorageClientSettings {
         key -> new Setting<>(key, "repository-gcs", Function.identity(), Setting.Property.NodeScope, Setting.Property.DeprecatedWarning)
     );
 
+    /** The type of the proxy to connect to the GCS through. Can be DIRECT (aka no proxy), HTTP or SOCKS */
+    public static final Setting.AffixSetting<Proxy.Type> PROXY_TYPE_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.type",
+        (key) -> new Setting<>(key, "DIRECT", s -> Proxy.Type.valueOf(s.toUpperCase(Locale.ROOT)), Setting.Property.NodeScope)
+    );
+
+    /** The host name of a proxy to connect to the GCS through. */
+    static final Setting.AffixSetting<String> PROXY_HOST_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.host",
+        (key) -> Setting.simpleString(key, Setting.Property.NodeScope),
+        () -> PROXY_TYPE_SETTING
+    );
+
+    /** The port of a proxy to connect to the GCS through. */
+    static final Setting.AffixSetting<Integer> PROXY_PORT_SETTING = Setting.affixKeySetting(
+        PREFIX,
+        "proxy.port",
+        (key) -> Setting.intSetting(key, 0, 0, 65535, Setting.Property.NodeScope),
+        () -> PROXY_HOST_SETTING
+    );
+
     /** The credentials used by the client to connect to the Storage endpoint. */
     private final ServiceAccountCredentials credential;
 
@@ -111,14 +141,20 @@ public class GoogleCloudStorageClientSettings {
     /** The token server URI. This leases access tokens in the oauth flow. */
     private final URI tokenUri;
 
+    @Nullable
+    private final Proxy proxy;
+
     GoogleCloudStorageClientSettings(
         final ServiceAccountCredentials credential,
         final String endpoint,
         final String projectId,
         final TimeValue connectTimeout,
         final TimeValue readTimeout,
         final String applicationName,
-        final URI tokenUri
+        final URI tokenUri,
+        final Proxy.Type proxyType,
+        final String proxyHost,
+        final Integer proxyPort
     ) {
         this.credential = credential;
         this.endpoint = endpoint;
@@ -127,6 +163,13 @@ public class GoogleCloudStorageClientSettings {
         this.readTimeout = readTimeout;
         this.applicationName = applicationName;
         this.tokenUri = tokenUri;
+        try {
+            proxy = proxyType.equals(Proxy.Type.DIRECT)
+                ? null
+                : new Proxy(proxyType, new InetSocketAddress(InetAddress.getByName(proxyHost), proxyPort));
+        } catch (UnknownHostException e) {
+            throw new SettingsException("GCS proxy host is unknown.", e);
+        }
     }
 
     public ServiceAccountCredentials getCredential() {
@@ -157,6 +200,11 @@ public URI getTokenUri() {
         return tokenUri;
     }
 
+    @Nullable
+    public Proxy getProxy() {
+        return proxy;
+    }
+
     public static Map<String, GoogleCloudStorageClientSettings> load(final Settings settings) {
         final Map<String, GoogleCloudStorageClientSettings> clients = new HashMap<>();
         for (final String clientName : settings.getGroups(PREFIX).keySet()) {
@@ -178,7 +226,10 @@ static GoogleCloudStorageClientSettings getClientSettings(final Settings setting
             getConfigValue(settings, clientName, CONNECT_TIMEOUT_SETTING),
             getConfigValue(settings, clientName, READ_TIMEOUT_SETTING),
             getConfigValue(settings, clientName, APPLICATION_NAME_SETTING),
-            getConfigValue(settings, clientName, TOKEN_URI_SETTING)
+            getConfigValue(settings, clientName, TOKEN_URI_SETTING),
+            getConfigValue(settings, clientName, PROXY_TYPE_SETTING),
+            getConfigValue(settings, clientName, PROXY_HOST_SETTING),
+            getConfigValue(settings, clientName, PROXY_PORT_SETTING)
         );
     }
 

modules/repository-gcs/src/main/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageService.java

@@ -34,6 +34,7 @@
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.net.HttpURLConnection;
+import java.net.Proxy;
 import java.net.URI;
 import java.net.URL;
 import java.security.KeyStore;
@@ -140,6 +141,11 @@ private Storage createClient(GoogleCloudStorageClientSettings gcsClientSettings,
                 SecurityUtils.loadKeyStore(certTrustStore, keyStoreStream, "notasecret");
             }
             builder.trustCertificates(certTrustStore);
+            Proxy proxy = gcsClientSettings.getProxy();
+            if (proxy != null) {
+                builder.setProxy(proxy);
+                notifyProxyIsSet(proxy);
+            }
             return builder.build();
         });
 
@@ -272,4 +278,7 @@ static Integer toTimeout(final TimeValue timeout) {
         }
         return Math.toIntExact(timeout.getMillis());
     }
+
+    // used for unit testing
+    void notifyProxyIsSet(Proxy proxy) {}
 }

modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageClientSettingsTests.java

@@ -17,6 +17,9 @@
 import org.elasticsearch.core.Tuple;
 import org.elasticsearch.test.ESTestCase;
 
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Proxy;
 import java.net.URI;
 import java.nio.charset.StandardCharsets;
 import java.security.KeyPair;
@@ -34,6 +37,9 @@
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.CREDENTIALS_FILE_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.ENDPOINT_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROJECT_ID_SETTING;
+import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_HOST_SETTING;
+import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_PORT_SETTING;
+import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.READ_TIMEOUT_SETTING;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.getClientSettings;
 import static org.elasticsearch.repositories.gcs.GoogleCloudStorageClientSettings.loadCredential;
@@ -94,11 +100,35 @@ public void testProjectIdDefaultsToCredentials() throws Exception {
             CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
             READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
             APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY),
-            new URI("")
+            new URI(""),
+            PROXY_TYPE_SETTING.getDefault(Settings.EMPTY),
+            PROXY_HOST_SETTING.getDefault(Settings.EMPTY),
+            PROXY_PORT_SETTING.getDefault(Settings.EMPTY)
         );
         assertEquals(credential.getProjectId(), googleCloudStorageClientSettings.getProjectId());
     }
 
+    public void testLoadsProxySettings() throws Exception {
+        final String clientName = randomAlphaOfLength(5);
+        final ServiceAccountCredentials credential = randomCredential(clientName).v1();
+        final GoogleCloudStorageClientSettings googleCloudStorageClientSettings = new GoogleCloudStorageClientSettings(
+            credential,
+            ENDPOINT_SETTING.getDefault(Settings.EMPTY),
+            PROJECT_ID_SETTING.getDefault(Settings.EMPTY),
+            CONNECT_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
+            READ_TIMEOUT_SETTING.getDefault(Settings.EMPTY),
+            APPLICATION_NAME_SETTING.getDefault(Settings.EMPTY),
+            new URI(""),
+            Proxy.Type.HTTP,
+            "192.168.15.1",
+            8080
+        );
+        assertEquals(
+            new Proxy(Proxy.Type.HTTP, new InetSocketAddress(InetAddress.getByName("192.168.15.1"), 8080)),
+            googleCloudStorageClientSettings.getProxy()
+        );
+    }
+
     /** Generates a given number of GoogleCloudStorageClientSettings along with the Settings to build them from **/
     private Tuple<Map<String, GoogleCloudStorageClientSettings>, Settings> randomClients(
         final int nbClients,
@@ -192,7 +222,10 @@ private static GoogleCloudStorageClientSettings randomClient(
             connectTimeout,
             readTimeout,
             applicationName,
-            new URI("")
+            new URI(""),
+            PROXY_TYPE_SETTING.getDefault(Settings.EMPTY),
+            PROXY_HOST_SETTING.getDefault(Settings.EMPTY),
+            PROXY_PORT_SETTING.getDefault(Settings.EMPTY)
         );
     }
 

modules/repository-gcs/src/test/java/org/elasticsearch/repositories/gcs/GoogleCloudStorageServiceTests.java

@@ -12,6 +12,7 @@
 import com.google.cloud.http.HttpTransportOptions;
 import com.google.cloud.storage.Storage;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.settings.MockSecureSettings;
 import org.elasticsearch.common.settings.Setting;
@@ -21,6 +22,7 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.hamcrest.Matchers;
 
+import java.net.Proxy;
 import java.security.KeyPair;
 import java.security.KeyPairGenerator;
 import java.util.Base64;
@@ -58,8 +60,17 @@ public void testClientInitializer() throws Exception {
             )
             .put(GoogleCloudStorageClientSettings.ENDPOINT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), endpoint)
             .put(GoogleCloudStorageClientSettings.PROJECT_ID_SETTING.getConcreteSettingForNamespace(clientName).getKey(), projectIdName)
+            .put(GoogleCloudStorageClientSettings.PROXY_TYPE_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "HTTP")
+            .put(GoogleCloudStorageClientSettings.PROXY_HOST_SETTING.getConcreteSettingForNamespace(clientName).getKey(), "192.168.52.15")
+            .put(GoogleCloudStorageClientSettings.PROXY_PORT_SETTING.getConcreteSettingForNamespace(clientName).getKey(), 8080)
             .build();
-        final GoogleCloudStorageService service = new GoogleCloudStorageService();
+        SetOnce<Proxy> proxy = new SetOnce<>();
+        final GoogleCloudStorageService service = new GoogleCloudStorageService() {
+            @Override
+            void notifyProxyIsSet(Proxy p) {
+                proxy.set(p);
+            }
+        };
         service.refreshAndClearCache(GoogleCloudStorageClientSettings.load(settings));
         GoogleCloudStorageOperationsStats statsCollector = new GoogleCloudStorageOperationsStats("bucket");
         final IllegalArgumentException e = expectThrows(
@@ -84,6 +95,7 @@ public void testClientInitializer() throws Exception {
             Matchers.is((int) readTimeValue.millis())
         );
         assertThat(storage.getOptions().getCredentials(), Matchers.nullValue(Credentials.class));
+        assertThat(proxy.get().toString(), equalTo("HTTP @ /192.168.52.15:8080"));
     }
 
     public void testReinitClientSettings() throws Exception {