Http Proxy raw channel establishment API (#319)

* Reworks http proxy layer to use pass-through handlers rather than stripping or re-using the proxy handler
* Adds an API to create a raw channel through an http proxy.

Author: bretambrose

include/aws/http/http.h

@@ -49,6 +49,7 @@ enum aws_http_errors {
     AWS_ERROR_HTTP_PROXY_STRATEGY_NTLM_CHALLENGE_TOKEN_MISSING,
     AWS_ERROR_HTTP_PROXY_STRATEGY_TOKEN_RETRIEVAL_FAILURE,
     AWS_ERROR_HTTP_PROXY_CONNECT_FAILED_RETRYABLE,
+    AWS_ERROR_HTTP_PROTOCOL_SWITCH_FAILURE,
 
     AWS_ERROR_HTTP_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_HTTP_PACKAGE_ID)
 };

include/aws/http/private/connection_impl.h

@@ -169,6 +169,32 @@ struct aws_crt_statistics_http1_channel *aws_h1_connection_get_statistics(struct
 AWS_HTTP_API
 uint32_t aws_http_connection_get_next_stream_id(struct aws_http_connection *connection);
 
+/**
+ * Layers an http channel handler/connection onto a channel.  Moved from internal to private so that the proxy
+ * logic could apply a new http connection/handler after tunneling proxy negotiation (into http) is finished.
+ * This is a synchronous operation.
+ *
+ * @param alloc memory allocator to use
+ * @param channel channel to apply the http handler/connection to
+ * @param is_server should the handler behave like an http server
+ * @param is_using_tls is tls is being used (do an alpn check of the to-the-left channel handler)
+ * @param manual_window_management is manual window management enabled
+ * @param initial_window_size what should the initial window size be
+ * @param http1_options http1 options
+ * @param http2_options http2 options
+ * @return a new http connection or NULL on failure
+ */
+AWS_HTTP_API
+struct aws_http_connection *aws_http_connection_new_channel_handler(
+    struct aws_allocator *alloc,
+    struct aws_channel *channel,
+    bool is_server,
+    bool is_using_tls,
+    bool manual_window_management,
+    size_t initial_window_size,
+    const struct aws_http1_connection_options *http1_options,
+    const struct aws_http2_connection_options *http2_options);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_HTTP_CONNECTION_IMPL_H */

include/aws/http/private/proxy_impl.h

@@ -11,6 +11,7 @@
 #include <aws/http/connection.h>
 #include <aws/http/proxy.h>
 #include <aws/http/status_code.h>
+#include <aws/io/channel_bootstrap.h>
 #include <aws/io/socket.h>
 
 struct aws_http_connection_manager_options;
@@ -74,7 +75,19 @@ struct aws_http_proxy_user_data {
     enum aws_proxy_bootstrap_state state;
     int error_code;
     enum aws_http_status_code connect_status_code;
-    struct aws_http_connection *connection;
+
+    /*
+     * The initial http connection object between the client and the proxy.
+     */
+    struct aws_http_connection *proxy_connection;
+
+    /*
+     * The http connection object that gets surfaced to callers if http is the final protocol of proxy
+     * negotiation.
+     *
+     * In the case of a forwarding proxy, proxy_connection and final_connection are the same.
+     */
+    struct aws_http_connection *final_connection;
     struct aws_http_message *connect_request;
     struct aws_http_stream *connect_stream;
     struct aws_http_proxy_negotiator *proxy_negotiator;
@@ -84,15 +97,27 @@ struct aws_http_proxy_user_data {
      */
     struct aws_string *original_host;
     uint16_t original_port;
-    aws_http_on_client_connection_setup_fn *original_on_setup;
-    aws_http_on_client_connection_shutdown_fn *original_on_shutdown;
     void *original_user_data;
+    struct aws_tls_connection_options *original_tls_options;
+    struct aws_client_bootstrap *original_bootstrap;
+    struct aws_socket_options original_socket_options;
+    bool original_manual_window_management;
+    size_t original_initial_window_size;
+    struct aws_http1_connection_options original_http1_options;
 
-    struct aws_tls_connection_options *tls_options;
-    struct aws_client_bootstrap *bootstrap;
-    struct aws_socket_options socket_options;
-    bool manual_window_management;
-    size_t initial_window_size;
+    /*
+     * setup/shutdown callbacks.  We enforce via fatal assert that either the http callbacks are supplied or
+     * the channel callbacks are supplied but never both.
+     *
+     * When using a proxy to ultimately establish an http connection, use the http callbacks.
+     * When using a proxy to establish any other protocol connection, use the raw channel callbacks.
+     *
+     * In the future, we might consider a further refactor which only use raw channel callbacks.
+     */
+    aws_http_on_client_connection_setup_fn *original_http_on_setup;
+    aws_http_on_client_connection_shutdown_fn *original_http_on_shutdown;
+    aws_client_bootstrap_on_channel_event_fn *original_channel_on_setup;
+    aws_client_bootstrap_on_channel_event_fn *original_channel_on_shutdown;
 
     struct aws_http_proxy_config *proxy_config;
 };
@@ -106,7 +131,9 @@ AWS_EXTERN_C_BEGIN
 AWS_HTTP_API
 struct aws_http_proxy_user_data *aws_http_proxy_user_data_new(
     struct aws_allocator *allocator,
-    const struct aws_http_client_connection_options *options);
+    const struct aws_http_client_connection_options *options,
+    aws_client_bootstrap_on_channel_event_fn *on_channel_setup,
+    aws_client_bootstrap_on_channel_event_fn *on_channel_shutdown);
 
 AWS_HTTP_API
 void aws_http_proxy_user_data_destroy(struct aws_http_proxy_user_data *user_data);

include/aws/http/proxy.h

@@ -488,6 +488,19 @@ void aws_http_proxy_options_init_from_config(
     struct aws_http_proxy_options *options,
     const struct aws_http_proxy_config *config);
 
+/**
+ * Establish an arbitrary protocol connection through an http proxy via tunneling CONNECT.  Alpn is
+ * not required for this connection process to succeed, but we encourage its use if available.
+ *
+ * @param channel_options configuration options for the socket level connection
+ * @param proxy_options configuration options for the proxy connection
+ *
+ * @return AWS_OP_SUCCESS if the asynchronous channel kickoff succeeded, AWS_OP_ERR otherwise
+ */
+AWS_HTTP_API int aws_http_proxy_new_socket_channel(
+    struct aws_socket_channel_bootstrap_options *channel_options,
+    struct aws_http_proxy_options *proxy_options);
+
 AWS_EXTERN_C_END
 
-#endif /* AWS_PROXY_H */
+#endif /* AWS_PROXY_STRATEGY_H */

source/connection.c

@@ -70,7 +70,7 @@ static void s_server_unlock_synced_data(struct aws_http_server *server) {
 }
 
 /* Determine the http-version, create appropriate type of connection, and insert it into the channel. */
-static struct aws_http_connection *s_connection_new(
+struct aws_http_connection *aws_http_connection_new_channel_handler(
     struct aws_allocator *alloc,
     struct aws_channel *channel,
     bool is_server,
@@ -365,7 +365,7 @@ void aws_http_connection_release(struct aws_http_connection *connection) {
         /* When the channel's refcount reaches 0, it destroys its slots/handlers, which will destroy the connection */
         aws_channel_release_hold(connection->channel_slot->channel);
     } else {
-        AWS_ASSERT(prev_refcount != 0);
+        AWS_FATAL_ASSERT(prev_refcount != 0);
         AWS_LOGF_TRACE(
             AWS_LS_HTTP_CONNECTION,
             "id=%p: Connection refcount released, %zu remaining.",
@@ -402,7 +402,7 @@ static void s_server_bootstrap_on_accept_channel_setup(
     /* TODO: expose http1/2 options to server API */
     struct aws_http1_connection_options http1_options = AWS_HTTP1_CONNECTION_OPTIONS_INIT;
     struct aws_http2_connection_options http2_options = AWS_HTTP2_CONNECTION_OPTIONS_INIT;
-    connection = s_connection_new(
+    connection = aws_http_connection_new_channel_handler(
         server->alloc,
         channel,
         true,
@@ -733,7 +733,7 @@ static void s_client_bootstrap_on_channel_setup(
 
     AWS_LOGF_TRACE(AWS_LS_HTTP_CONNECTION, "static: Socket connected, creating client connection object.");
 
-    http_bootstrap->connection = s_connection_new(
+    http_bootstrap->connection = aws_http_connection_new_channel_handler(
         http_bootstrap->alloc,
         channel,
         false,

source/h1_connection.c

@@ -461,10 +461,77 @@ static void s_cross_thread_work_task(struct aws_channel_task *channel_task, void
     }
 }
 
+static bool s_aws_http_stream_was_successful_connect(struct aws_h1_stream *stream) {
+    struct aws_http_stream *base = &stream->base;
+    if (base->request_method != AWS_HTTP_METHOD_CONNECT) {
+        return false;
+    }
+
+    if (base->client_data == NULL) {
+        return false;
+    }
+
+    if (base->client_data->response_status != AWS_HTTP_STATUS_CODE_200_OK) {
+        return false;
+    }
+
+    return true;
+}
+
+/**
+ * Validate and perform a protocol switch on a connection.  Protocol switching essentially turns the connection's
+ * handler into a dummy pass-through.  It is valid to switch protocols to the same protocol resulting in a channel
+ * that has a "dead" http handler in the middle of the channel (which negotiated the CONNECT through the proxy) and
+ * a "live" handler on the end which takes the actual http requests.  By doing this, we get the exact same
+ * behavior whether we're transitioning to http or any other protocol: once the CONNECT succeeds
+ * the first http handler is put in pass-through mode and a new protocol (which could be http) is tacked onto the end.
+ */
+static int s_aws_http1_switch_protocols(struct aws_h1_connection *connection) {
+    AWS_FATAL_ASSERT(aws_channel_thread_is_callers_thread(connection->base.channel_slot->channel));
+
+    /* Switching protocols while there are multiple streams is too complex to deal with.
+     * Ensure stream_list has exactly this 1 stream in it. */
+    if (aws_linked_list_begin(&connection->thread_data.stream_list) !=
+        aws_linked_list_rbegin(&connection->thread_data.stream_list)) {
+        AWS_LOGF_ERROR(
+            AWS_LS_HTTP_CONNECTION,
+            "id=%p: Cannot switch protocols while further streams are pending, closing connection.",
+            (void *)&connection->base);
+
+        return aws_raise_error(AWS_ERROR_INVALID_STATE);
+    }
+
+    AWS_LOGF_TRACE(
+        AWS_LS_HTTP_CONNECTION,
+        "id=%p: Connection has switched protocols, another channel handler must be installed to"
+        " deal with further data.",
+        (void *)&connection->base);
+
+    connection->thread_data.has_switched_protocols = true;
+    { /* BEGIN CRITICAL SECTION */
+        aws_h1_connection_lock_synced_data(connection);
+        connection->synced_data.new_stream_error_code = AWS_ERROR_HTTP_SWITCHED_PROTOCOLS;
+        aws_h1_connection_unlock_synced_data(connection);
+    } /* END CRITICAL SECTION */
+
+    return AWS_OP_SUCCESS;
+}
+
 static void s_stream_complete(struct aws_h1_stream *stream, int error_code) {
     struct aws_h1_connection *connection =
         AWS_CONTAINER_OF(stream->base.owning_connection, struct aws_h1_connection, base);
 
+    /*
+     * If this is the end of a successful CONNECT request, mark ourselves as pass-through since the proxy layer
+     * will be tacking on a new http handler (and possibly a tls handler in-between).
+     */
+    if (error_code == AWS_ERROR_SUCCESS && s_aws_http_stream_was_successful_connect(stream)) {
+        if (s_aws_http1_switch_protocols(connection)) {
+            error_code = AWS_ERROR_HTTP_PROTOCOL_SWITCH_FAILURE;
+            s_shutdown_due_to_error(connection, error_code);
+        }
+    }
+
     /* Remove stream from list. */
     aws_linked_list_remove(&stream->node);
 
@@ -1034,31 +1101,9 @@ static int s_mark_head_done(struct aws_h1_stream *incoming_stream) {
         /* Only clients can receive informational headers.
          * Check whether we're switching protocols */
         if (incoming_stream->base.client_data->response_status == AWS_HTTP_STATUS_CODE_101_SWITCHING_PROTOCOLS) {
-
-            /* Switching protocols while there are multiple streams is too complex to deal with.
-             * Ensure stream_list has exactly this 1 stream in it. */
-            if (aws_linked_list_begin(&connection->thread_data.stream_list) !=
-                aws_linked_list_rbegin(&connection->thread_data.stream_list)) {
-                AWS_LOGF_ERROR(
-                    AWS_LS_HTTP_CONNECTION,
-                    "id=%p: Cannot switch protocols while further streams are pending, closing connection.",
-                    (void *)&connection->base);
-
-                return aws_raise_error(AWS_ERROR_INVALID_STATE);
+            if (s_aws_http1_switch_protocols(connection)) {
+                return AWS_OP_ERR;
             }
-
-            AWS_LOGF_TRACE(
-                AWS_LS_HTTP_CONNECTION,
-                "id=%p: Connection has switched protocols, another channel handler must be installed to"
-                " deal with further data.",
-                (void *)&connection->base);
-
-            connection->thread_data.has_switched_protocols = true;
-            { /* BEGIN CRITICAL SECTION */
-                aws_h1_connection_lock_synced_data(connection);
-                connection->synced_data.new_stream_error_code = AWS_ERROR_HTTP_SWITCHED_PROTOCOLS;
-                aws_h1_connection_unlock_synced_data(connection);
-            } /* END CRITICAL SECTION */
         }
     }
 

source/http.c

@@ -124,6 +124,9 @@ static struct aws_error_info s_errors[] = {
     AWS_DEFINE_ERROR_INFO_HTTP(
         AWS_ERROR_HTTP_PROXY_CONNECT_FAILED_RETRYABLE,
         "Proxy connection attempt failed but the negotiation could be continued on a new connection"),
+    AWS_DEFINE_ERROR_INFO_HTTP(
+        AWS_ERROR_HTTP_PROTOCOL_SWITCH_FAILURE,
+        "Internal state failure prevent connection from switching protocols"),
 };
 /* clang-format on */