Proxy envrionment variable support (#333)

- Add support for setting up proxy from environment variable and integration test for proxy

Author: TingDaoK

.builder/action/aws-c-http-test.sh

@@ -0,0 +1,9 @@
+PROXY_SETUP=/tmp/setup_proxy_test_env.sh
+if [ -f "$PROXY_SETUP" ]; then
+    source $PROXY_SETUP
+    export AWS_PROXY_NO_VERIFY_PEER=on
+    echo "setting proxy integration test envrionment"
+fi
+
+cd ./build/aws-c-http/
+ctest --output-on-failure

CMakeLists.txt

@@ -5,7 +5,7 @@ if (POLICY CMP0069)
     cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags
 endif()
 
-option(ENABLE_PROXY_INTEGRATION_TESTS "Whether to run the proxy integration tests that rely on a proxy server installed and running locally" OFF)
+option(ENABLE_PROXY_INTEGRATION_TESTS "Whether to run the proxy integration tests that rely on pre-configured proxy" OFF)
 
 if (DEFINED CMAKE_PREFIX_PATH)
     file(TO_CMAKE_PATH "${CMAKE_PREFIX_PATH}" CMAKE_PREFIX_PATH)

builder.json

@@ -14,7 +14,7 @@
         { "name": "aws-c-auth" }
     ],
     "test_steps": [
-        "test",
+        ["bash", "{project_dir}/.builder/action/aws-c-http-test.sh"],
         ["{python}", "{project_dir}/integration-testing/http_client_test.py", "{install_dir}/bin/elasticurl{exe}"]
     ]
 }

codebuild/linux-integration-tests.yml

@@ -0,0 +1,25 @@
+version: 0.2
+#this build spec assumes the manylinux1 image for pypi
+#additional packages we installed: cmake 3.5, libcrypto 1.1.0j, gcc 4.8.4
+phases:
+  install:
+    commands:
+      - add-apt-repository ppa:ubuntu-toolchain-r/test
+      - apt-get update -y
+      - apt-get install gcc-7 cmake ninja-build python3 -y
+  pre_build:
+    commands:
+      - export CC=gcc-7
+      - export BUILDER_VERSION=v0.8.27
+      - export BUILDER_SOURCE=releases
+      - export BUILDER_HOST=https://d19elf31gohf1l.cloudfront.net
+  build:
+    commands:
+      - echo Build started on `date`
+      - aws s3 cp s3://aws-crt-test-stuff/setup_proxy_test_env.sh /tmp/setup_proxy_test_env.sh
+      - chmod a+xr /tmp/setup_proxy_test_env.sh
+      - python3 -c "from urllib.request import urlretrieve; urlretrieve('$BUILDER_HOST/$BUILDER_SOURCE/$BUILDER_VERSION/builder.pyz', 'builder.pyz')"
+      - python3 builder.pyz build -p aws-c-http --cmake-extra=-DENABLE_PROXY_INTEGRATION_TESTS=ON
+  post_build:
+    commands:
+      - echo Build completed on `date`

include/aws/http/connection.h

@@ -7,6 +7,7 @@
  */
 
 #include <aws/http/http.h>
+#include <aws/http/proxy.h>
 
 struct aws_client_bootstrap;
 struct aws_socket_options;
@@ -270,6 +271,13 @@ struct aws_http_client_connection_options {
      */
     const struct aws_http_proxy_options *proxy_options;
 
+    /*
+     * Optional.
+     * Configuration for using proxy from environment variable.
+     * Only works when proxy_options is not set.
+     */
+    const struct proxy_env_var_settings *proxy_ev_settings;
+
     /**
      * Optional
      * Configuration options related to connection health monitoring

include/aws/http/connection_manager.h

@@ -7,6 +7,7 @@
  */
 
 #include <aws/http/http.h>
+#include <aws/http/proxy.h>
 
 #include <aws/common/byte_buf.h>
 
@@ -37,10 +38,18 @@ struct aws_http_connection_manager_options {
     size_t initial_window_size;
     const struct aws_socket_options *socket_options;
     const struct aws_tls_connection_options *tls_connection_options;
-    const struct aws_http_proxy_options *proxy_options;
     const struct aws_http_connection_monitoring_options *monitoring_options;
     struct aws_byte_cursor host;
     uint16_t port;
+    /* Proxy configuration for http connection */
+    const struct aws_http_proxy_options *proxy_options;
+
+    /*
+     * Optional.
+     * Configuration for using proxy from environment variable.
+     * Only works when proxy_options is not set.
+     */
+    const struct proxy_env_var_settings *proxy_ev_settings;
 
     /*
      * Maximum number of connections this manager is allowed to contain

include/aws/http/proxy.h

@@ -31,6 +31,21 @@ enum aws_http_proxy_authentication_type {
     AWS_HPAT_BASIC,
 };
 
+enum aws_http_proxy_env_var_type {
+    /**
+     * Default.
+     * Disable reading from environment variable for proxy.
+     */
+    AWS_HPEV_DISABLE = 0,
+    /**
+     * Enable get proxy URL from environment variable, when the manual proxy options of connection manager is not set.
+     * env HTTPS_PROXY/https_proxy will be checked when the main connection use tls.
+     * env HTTP_PROXY/http_proxy will be checked when the main connection NOT use tls.
+     * The lower case version has precedence.
+     */
+    AWS_HPEV_ENABLE,
+};
+
 /**
  * Supported proxy connection types
  */
@@ -56,6 +71,27 @@ enum aws_http_proxy_connection_type {
     AWS_HPCT_HTTP_TUNNEL,
 };
 
+/*
+ * Configuration for using proxy from environment variable.
+ * Zero out as default settings.
+ */
+struct proxy_env_var_settings {
+    enum aws_http_proxy_env_var_type env_var_type;
+    /*
+     * Optional.
+     * If not set:
+     * If tls options are provided (for the main connection) use tunnel proxy type
+     * If tls options are not provided (for the main connection) use forward proxy type
+     */
+    enum aws_http_proxy_connection_type connection_type;
+    /*
+     * Optional.
+     * If not set, a default tls option will be created. when https used for Local to proxy connection.
+     * Must be distinct from the the tls_connection_options from aws_http_connection_manager_options
+     */
+    const struct aws_tls_connection_options *tls_options;
+};
+
 struct aws_http_proxy_strategy;
 
 /**
@@ -457,6 +493,19 @@ struct aws_http_proxy_config *aws_http_proxy_config_new_tunneling_from_proxy_opt
     struct aws_allocator *allocator,
     const struct aws_http_proxy_options *options);
 
+/**
+ * Create a persistent proxy configuration from non-persistent proxy options.
+ * Legacy connection type of proxy options will be rejected.
+ *
+ * @param allocator memory allocator to use
+ * @param options http proxy options to source proxy configuration from
+ * @return
+ */
+AWS_HTTP_API
+struct aws_http_proxy_config *aws_http_proxy_config_new_from_proxy_options(
+    struct aws_allocator *allocator,
+    const struct aws_http_proxy_options *options);
+
 /**
  * Clones an existing proxy configuration.  A refactor could remove this (do a "move" between the old and new user
  * data in the one spot it's used) but that should wait until we have better test cases for the logic where this

source/connection.c

@@ -1147,7 +1147,12 @@ int aws_http_client_connect(const struct aws_http_client_connection_options *opt
     if (options->proxy_options != NULL) {
         return aws_http_client_connect_via_proxy(options);
     } else {
-        return aws_http_client_connect_internal(options, NULL);
+        if (!options->proxy_ev_settings || options->proxy_ev_settings->env_var_type != AWS_HPEV_ENABLE) {
+            return aws_http_client_connect_internal(options, NULL);
+        } else {
+            /* Proxy through envrionment variable is enabled */
+            return aws_http_client_connect_via_proxy(options);
+        }
     }
 }
 

source/connection_manager.c

@@ -16,6 +16,7 @@
 #include <aws/io/logging.h>
 #include <aws/io/socket.h>
 #include <aws/io/tls_channel_handler.h>
+#include <aws/io/uri.h>
 
 #include <aws/common/clock.h>
 #include <aws/common/hash_table.h>
@@ -209,6 +210,8 @@ struct aws_http_connection_manager {
     struct aws_http_proxy_config *proxy_config;
     struct aws_http_connection_monitoring_options monitoring_options;
     struct aws_string *host;
+    struct proxy_env_var_settings proxy_ev_settings;
+    struct aws_tls_connection_options *proxy_ev_tls_options;
     uint16_t port;
 
     /*
@@ -632,7 +635,10 @@ static void s_aws_http_connection_manager_finish_destroy(struct aws_http_connect
         aws_tls_connection_options_clean_up(manager->tls_connection_options);
         aws_mem_release(manager->allocator, manager->tls_connection_options);
     }
-
+    if (manager->proxy_ev_tls_options) {
+        aws_tls_connection_options_clean_up(manager->proxy_ev_tls_options);
+        aws_mem_release(manager->allocator, manager->proxy_ev_tls_options);
+    }
     if (manager->proxy_config) {
         aws_http_proxy_config_destroy(manager->proxy_config);
     }
@@ -806,7 +812,6 @@ struct aws_http_connection_manager *aws_http_connection_manager_new(
             goto on_error;
         }
     }
-
     if (options->proxy_options) {
         manager->proxy_config = aws_http_proxy_config_new_from_manager_options(allocator, options);
         if (manager->proxy_config == NULL) {
@@ -830,7 +835,16 @@ struct aws_http_connection_manager *aws_http_connection_manager_new(
     manager->shutdown_complete_user_data = options->shutdown_complete_user_data;
     manager->enable_read_back_pressure = options->enable_read_back_pressure;
     manager->max_connection_idle_in_milliseconds = options->max_connection_idle_in_milliseconds;
-
+    if (options->proxy_ev_settings) {
+        manager->proxy_ev_settings = *options->proxy_ev_settings;
+    }
+    if (manager->proxy_ev_settings.tls_options) {
+        manager->proxy_ev_tls_options = aws_mem_calloc(allocator, 1, sizeof(struct aws_tls_connection_options));
+        if (aws_tls_connection_options_copy(manager->proxy_ev_tls_options, manager->proxy_ev_settings.tls_options)) {
+            goto on_error;
+        }
+        manager->proxy_ev_settings.tls_options = manager->proxy_ev_tls_options;
+    }
     s_schedule_connection_culling(manager);
 
     AWS_LOGF_INFO(AWS_LS_HTTP_CONNECTION_MANAGER, "id=%p: Successfully created", (void *)manager);
@@ -907,6 +921,7 @@ static int s_aws_http_connection_manager_new_connection(struct aws_http_connecti
     options.on_setup = s_aws_http_connection_manager_on_connection_setup;
     options.on_shutdown = s_aws_http_connection_manager_on_connection_shutdown;
     options.manual_window_management = manager->enable_read_back_pressure;
+    options.proxy_ev_settings = &manager->proxy_ev_settings;
 
     if (aws_http_connection_monitoring_options_is_valid(&manager->monitoring_options)) {
         options.monitoring_options = &manager->monitoring_options;