Merge remote-tracking branch 'civitaspo/client_configure' into client-configurable

Author: frsyuki

README.md

@@ -53,6 +53,60 @@
 
     - **session_token**: session token (string, required)
 
+- **client_config**: configure S3 client config (optional)
+
+  - **protocol**: (enum, `HTTP` or `HTTPS`, optional)
+
+  - **max_connections**: (int, optional)
+
+  - **user_agent** (string, optional)
+
+  - **local_address**: name of a hostname (string, optional)
+
+  - **proxy_host**: name of a hostname (string, optional)
+
+  - **proxy_port**: (int, optional)
+
+  - **proxy_username**: (string, optional)
+
+  - **proxy_password**: (string, optional)
+
+  - **proxy_domain**: (string, optional)
+
+  - **proxy_workstation**: (string, optional)
+
+  - **max_error_retry**: (int, optional)
+
+  - **socket_timeout**: (int, optional)
+
+  - **connection_timeout**: (int, optional)
+
+  - **request_timeout**: (int, optional)
+
+  - **use_reaper**: (boolean, optional)
+
+  - **use_gzip**: (boolean, optional)
+
+  - **signer_override**: (string, optional)
+
+  - **preemptive_basic_proxy_auth**: (boolean, optional)
+
+  - **connection_ttl**: (long, optional)
+
+  - **connection_max_idle_millis**: (long, optional)
+
+  - **use_tcp_keep_alive**: (boolean, optional)
+
+  - **response_metadata_cache_size**: (int, optional)
+
+  - **use_expect_continue**: (boolean, optional)
+
+  - **secure_random**: (optional)
+
+    - **algorithm**: (string, required)
+
+    - **provider**: (string, optional)
+
 * **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
 
 * **total_file_count_limit**: maximum number of files to read (integer, optional)

embulk-input-s3/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -1,19 +1,15 @@
 package org.embulk.input.s3;
 
 import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Iterator;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
 
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.slf4j.Logger;
-import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
@@ -23,7 +19,6 @@
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.AmazonServiceException;
-import com.amazonaws.Protocol;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
@@ -65,7 +60,9 @@ public interface PluginTask
         @ConfigDefault("null")
         public Optional<String> getAccessKeyId();
 
-        // TODO timeout, ssl, etc
+        @Config("client_config")
+        @ConfigDefault("{}")
+        public ClientConfigurationConfigurable.Task getClientConfigurationConfigurableTask();
 
         public FileList getFiles();
         public void setFiles(FileList files);
@@ -129,14 +126,8 @@ protected AWSCredentialsProvider getCredentialsProvider(PluginTask task)
 
     protected ClientConfiguration getClientConfiguration(PluginTask task)
     {
-        ClientConfiguration clientConfig = new ClientConfiguration();
-
-        //clientConfig.setProtocol(Protocol.HTTP);
-        clientConfig.setMaxConnections(50); // SDK default: 50
-        clientConfig.setMaxErrorRetry(3); // SDK default: 3
-        clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
-
-        return clientConfig;
+        ClientConfigurationConfigurable.Task configurableTask = task.getClientConfigurationConfigurableTask();
+        return ClientConfigurationConfigurable.getClientConfiguration(configurableTask);
     }
 
     private FileList listFiles(PluginTask task)