Stat (#152)

* Base statistics gathering and monitoring
* Control rate of HTTP/1 writes (#180)
* HTTP/1 only writes 1 message at a time.
This avoids bloating memory with unsent messages.
This also helps improve the accuracy of our upload stats.
Still needs tweaking to deal with websocket upgrade edge-cases.
* Install websocket handler the moment 101 response comes in, do not wait for stream to complete.
Remove hacks where 101 response completed a stream, no 1xx response should complete a stream.
* Simplify confusing list logic.
Remove the server-only `waiting_stream_list`.
Do a simple scan over available streams to find the next one to work on. This is O(N) when pipelining, but I hear that no one does HTTP/1 pipelining, so in that case O(N=1). We can always re-optimize if we have data that pipelining is super popular and this is a hotspot.
* Update
* Windows warning

Co-authored-by: Michael Graeb <graebm@amazon.com>

Author: bretambrose

include/aws/http/connection.h

@@ -56,6 +56,24 @@ typedef void(
 typedef void(
     aws_http_on_client_connection_shutdown_fn)(struct aws_http_connection *connection, int error_code, void *user_data);
 
+/**
+ * Configuration options for connection monitoring
+ */
+struct aws_http_connection_monitoring_options {
+
+    /**
+     * minimum required throughput of the connection.  Throughput is only measured against the interval of time where
+     * there is actual io to perform.  Read and write throughput are measured and checked independently of one another.
+     */
+    uint64_t minimum_throughput_bytes_per_second;
+
+    /*
+     * amount of time, in seconds, throughput is allowed to drop below the minimum before the connection is shut down
+     * as unhealthy.
+     */
+    uint32_t allowable_throughput_failure_interval_seconds;
+};
+
 /**
  * Supported proxy authentication modes
  */
@@ -158,6 +176,12 @@ struct aws_http_client_connection_options {
      */
     const struct aws_http_proxy_options *proxy_options;
 
+    /**
+     * Optional
+     * Configuration options related to connection health monitoring
+     */
+    const struct aws_http_connection_monitoring_options *monitoring_options;
+
     /**
      * Optional.
      * A default size is set by AWS_HTTP_CLIENT_CONNECTION_OPTIONS_INIT.

include/aws/http/connection_manager.h

@@ -48,6 +48,7 @@ struct aws_http_connection_manager_options {
     const struct aws_socket_options *socket_options;
     const struct aws_tls_connection_options *tls_connection_options;
     const struct aws_http_proxy_options *proxy_options;
+    const struct aws_http_connection_monitoring_options *monitoring_options;
     struct aws_byte_cursor host;
     uint16_t port;
 

include/aws/http/http.h

@@ -16,12 +16,14 @@
  * permissions and limitations under the License.
  */
 
+#include <aws/common/logging.h>
 #include <aws/http/exports.h>
-
 #include <aws/io/io.h>
 
+#define AWS_C_HTTP_PACKAGE_ID 2
+
 enum aws_http_errors {
-    AWS_ERROR_HTTP_UNKNOWN = 0x0800,
+    AWS_ERROR_HTTP_UNKNOWN = AWS_ERROR_ENUM_BEGIN_RANGE(AWS_C_HTTP_PACKAGE_ID),
     AWS_ERROR_HTTP_HEADER_NOT_FOUND,
     AWS_ERROR_HTTP_INVALID_HEADER_FIELD,
     AWS_ERROR_HTTP_INVALID_HEADER_NAME,
@@ -46,17 +48,18 @@ enum aws_http_errors {
     AWS_ERROR_HTTP_SERVER_CLOSED,
     AWS_ERROR_HTTP_PROXY_TLS_CONNECT_FAILED,
     AWS_ERROR_HTTP_CONNECTION_MANAGER_SHUTTING_DOWN,
+    AWS_ERROR_HTTP_CHANNEL_THROUGHPUT_FAILURE,
     AWS_ERROR_HTTP_PROTOCOL_ERROR,
     AWS_ERROR_HTTP_STREAM_CLOSED,
     AWS_ERROR_HTTP_STREAM_IDS_EXHAUSTED,
     AWS_ERROR_HTTP_INVALID_FRAME_SIZE,
     AWS_ERROR_HTTP_COMPRESSION,
 
-    AWS_ERROR_HTTP_END_RANGE = 0x0C00,
+    AWS_ERROR_HTTP_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_HTTP_PACKAGE_ID)
 };
 
 enum aws_http_log_subject {
-    AWS_LS_HTTP_GENERAL = 0x800,
+    AWS_LS_HTTP_GENERAL = AWS_LOG_SUBJECT_BEGIN_RANGE(AWS_C_HTTP_PACKAGE_ID),
     AWS_LS_HTTP_CONNECTION,
     AWS_LS_HTTP_ENCODER,
     AWS_LS_HTTP_DECODER,

include/aws/http/private/connection_impl.h

@@ -30,28 +30,10 @@ struct aws_http_make_request_options;
 struct aws_http_request_handler_options;
 struct aws_http_stream;
 
-typedef int(aws_client_bootstrap_new_socket_channel_fn)(
-    struct aws_client_bootstrap *bootstrap,
-    const char *host_name,
-    uint16_t port,
-    const struct aws_socket_options *options,
-    aws_client_bootstrap_on_channel_setup_fn *setup_callback,
-    aws_client_bootstrap_on_channel_shutdown_fn *shutdown_callback,
-    void *user_data);
-
-typedef int(aws_client_bootstrap_new_tls_socket_channel_fn)(
-    struct aws_client_bootstrap *bootstrap,
-    const char *host_name,
-    uint16_t port,
-    const struct aws_socket_options *options,
-    const struct aws_tls_connection_options *connection_options,
-    aws_client_bootstrap_on_channel_setup_fn *setup_callback,
-    aws_client_bootstrap_on_channel_shutdown_fn *shutdown_callback,
-    void *user_data);
+typedef int aws_client_bootstrap_new_socket_channel_fn(struct aws_socket_channel_bootstrap_options *options);
 
 struct aws_http_connection_system_vtable {
     aws_client_bootstrap_new_socket_channel_fn *new_socket_channel;
-    aws_client_bootstrap_new_tls_socket_channel_fn *new_tls_socket_channel;
 };
 
 struct aws_http_connection_vtable {
@@ -90,6 +72,9 @@ struct aws_http_connection {
      * aws_http_streams will also acquire holds on their connection for the duration of their lifetime */
     struct aws_atomic_var refcount;
 
+    /* Starts at either 1 or 2, increments by two with each new stream */
+    struct aws_atomic_var next_stream_id;
+
     union {
         struct aws_http_connection_client_data {
             uint8_t delete_me; /* exists to prevent "empty struct" errors */
@@ -113,6 +98,7 @@ struct aws_http_client_bootstrap {
     struct aws_allocator *alloc;
     bool is_using_tls;
     size_t initial_window_size;
+    struct aws_http_connection_monitoring_options monitoring_options;
     void *user_data;
     aws_http_on_client_connection_setup_fn *on_setup;
     aws_http_on_client_connection_shutdown_fn *on_shutdown;
@@ -137,6 +123,20 @@ int aws_http_client_connect_internal(
 AWS_HTTP_API
 void aws_http_connection_acquire(struct aws_http_connection *connection);
 
+/**
+ * Allow tests to fake stats data
+ */
+AWS_HTTP_API
+struct aws_crt_statistics_http1_channel *aws_h1_connection_get_statistics(struct aws_http_connection *connection);
+
+/**
+ * Gets the next available stream id within the connection.  Valid for creating both h1 and h2 streams.
+ *
+ * Returns 0 if there was an error.
+ */
+AWS_HTTP_API
+uint32_t aws_http_connection_get_next_stream_id(struct aws_http_connection *connection);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_HTTP_CONNECTION_IMPL_H */

include/aws/http/private/connection_monitor.h

@@ -0,0 +1,56 @@
+#ifndef AWS_HTTP_HTTP_MONITOR_H
+#define AWS_HTTP_HTTP_MONITOR_H
+
+/*
+ * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <aws/http/connection.h>
+#include <aws/http/http.h>
+
+struct aws_allocator;
+struct aws_crt_statistics_handler;
+
+/*
+ * Needed by tests
+ */
+struct aws_statistics_handler_http_connection_monitor_impl {
+    struct aws_http_connection_monitoring_options options;
+
+    uint64_t throughput_failure_time_ms;
+    uint32_t last_incoming_stream_id;
+    uint32_t last_outgoing_stream_id;
+    uint64_t last_measured_throughput;
+};
+
+AWS_EXTERN_C_BEGIN
+
+/**
+ * Creates a new http connection monitor that regularly checks the connection's throughput and shuts the connection
+ * down if the a minimum threshold is not met for a configurable number of seconds.
+ */
+AWS_HTTP_API
+struct aws_crt_statistics_handler *aws_crt_statistics_handler_new_http_connection_monitor(
+    struct aws_allocator *allocator,
+    struct aws_http_connection_monitoring_options *options);
+
+/**
+ * Validates monitoring options to ensure they are sensible
+ */
+AWS_HTTP_API
+bool aws_http_connection_monitoring_options_is_valid(const struct aws_http_connection_monitoring_options *options);
+
+AWS_EXTERN_C_END
+
+#endif /* AWS_HTTP_HTTP_MONITOR_H */

include/aws/http/private/h2_connection.h

@@ -51,9 +51,6 @@ struct aws_h2_connection {
         /* New `aws_h2_stream *` that haven't moved to `thread_data` yet */
         struct aws_linked_list pending_stream_list;
 
-        /* Refers to the next stream id to vend */
-        uint32_t next_stream_id;
-
         /* If non-zero, reason to immediately reject new streams. (ex: closing) */
         int new_stream_error_code;
 

include/aws/http/private/h2_stream.h

@@ -27,7 +27,7 @@
     AWS_LOGF_##level(                                                                                                  \
         AWS_LS_HTTP_STREAM,                                                                                            \
         "id=%" PRIu32 " connection=%p state=%s: " text,                                                                \
-        (stream)->id,                                                                                                  \
+        (stream)->base.id,                                                                                             \
         (void *)(stream)->base.owning_connection,                                                                      \
         aws_h2_stream_state_to_str((stream)->thread_data.state),                                                       \
         __VA_ARGS__)
@@ -48,8 +48,6 @@ enum aws_h2_stream_state {
 struct aws_h2_stream {
     struct aws_http_stream base;
 
-    uint32_t id;
-
     struct aws_linked_list_node node;
 
     /* Only the event-loop thread may touch this data */

include/aws/http/private/request_response_impl.h

@@ -36,6 +36,8 @@ struct aws_http_stream {
     struct aws_allocator *alloc;
     struct aws_http_connection *owning_connection;
 
+    uint32_t id;
+
     bool manual_window_management;
 
     void *user_data;

include/aws/http/request_response.h

@@ -647,6 +647,12 @@ int aws_http_stream_send_response(struct aws_http_stream *stream, struct aws_htt
 AWS_HTTP_API
 void aws_http_stream_update_window(struct aws_http_stream *stream, size_t increment_size);
 
+/**
+ * Gets the Http/2 id associated with a stream.  Even h1 streams have an id (using the same allocation procedure
+ * as http/2) for easier tracking purposes.
+ */
+uint32_t aws_http_stream_get_id(struct aws_http_stream *stream);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_HTTP_REQUEST_RESPONSE_H */

include/aws/http/statistics.h

@@ -0,0 +1,63 @@
+#ifndef AWS_HTTP_STATISTICS_H
+#define AWS_HTTP_STATISTICS_H
+
+/*
+ * Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#include <aws/http/http.h>
+
+#include <aws/common/statistics.h>
+
+enum aws_crt_http_statistics_category {
+    AWSCRT_STAT_CAT_HTTP1_CHANNEL = AWS_CRT_STATISTICS_CATEGORY_BEGIN_RANGE(AWS_C_HTTP_PACKAGE_ID),
+};
+
+/**
+ * A statistics struct for http handlers.  Tracks the actual amount of time that incoming and outgoing requests are
+ * waiting for their IO to complete.
+ */
+struct aws_crt_statistics_http1_channel {
+    aws_crt_statistics_category_t category;
+
+    uint64_t pending_outgoing_stream_ms;
+    uint64_t pending_incoming_stream_ms;
+
+    uint32_t current_outgoing_stream_id;
+    uint32_t current_incoming_stream_id;
+};
+
+AWS_EXTERN_C_BEGIN
+
+/**
+ * Initializes a http channel handler statistics struct
+ */
+AWS_HTTP_API
+int aws_crt_statistics_http1_channel_init(struct aws_crt_statistics_http1_channel *stats);
+
+/**
+ * Cleans up a http channel handler statistics struct
+ */
+AWS_HTTP_API
+void aws_crt_statistics_http1_channel_cleanup(struct aws_crt_statistics_http1_channel *stats);
+
+/**
+ * Resets a http channel handler statistics struct's statistics
+ */
+AWS_HTTP_API
+void aws_crt_statistics_http1_channel_reset(struct aws_crt_statistics_http1_channel *stats);
+
+AWS_EXTERN_C_END
+
+#endif /* AWS_HTTP_STATISTICS_H */