telemetry API

include/aws/s3/private/s3_meta_request_impl.h

@@ -132,6 +132,8 @@ struct aws_s3_meta_request {
     aws_s3_meta_request_shutdown_fn *shutdown_callback;
     aws_s3_meta_request_progress_fn *progress_callback;
 
+    aws_s3_meta_request_telemetry_fn *telemetry_callback;
+
     /* Customer specified callbacks to be called by our specialized callback to calculate the response checksum. */
     aws_s3_meta_request_headers_callback_fn *headers_user_callback_after_checksum;
     aws_s3_meta_request_receive_body_callback_fn *body_user_callback_after_checksum;

include/aws/s3/private/s3_request.h

@@ -9,6 +9,7 @@
 #include <aws/common/byte_buf.h>
 #include <aws/common/linked_list.h>
 #include <aws/common/ref_count.h>
+#include <aws/common/thread.h>
 #include <aws/s3/s3.h>
 
 #include <aws/s3/private/s3_checksums.h>
@@ -23,6 +24,73 @@ enum aws_s3_request_flags {
     AWS_S3_REQUEST_FLAG_ALWAYS_SEND = 0x00000004,
 };
 
+/**
+ * Information sent in the telemetry_callback after each aws_s3_request finished/retried from meta request.
+ */
+struct aws_s3_request_metrics {
+    struct aws_allocator *allocator;
+
+    struct {
+        /* The time stamp when the request started by S3 client, which is prepared time by the client. Timestamps
+         * are from `aws_high_res_clock_get_ticks`. This will always be available. */
+        int64_t start_timestamp_ns;
+        /* The time stamp when the request finished by S3 client succeed or failed or to be retried. Timestamps
+         * are from `aws_high_res_clock_get_ticks`. This will always be available. */
+        int64_t end_timestamp_ns;
+        /* The time duration for the request from start to finish. end_timestamp_ns - start_timestamp_ns. This will
+         * always be available. */
+        int64_t total_duration_ns;
+
+        /* The time stamp when the request started to be encoded. -1 means data not available. Timestamp
+         * are from `aws_high_res_clock_get_ticks` */
+        int64_t send_start_timestamp_ns;
+        /* The time stamp when the request finished to be encoded. -1 means data not available.
+         * Timestamp are from `aws_high_res_clock_get_ticks` */
+        int64_t send_end_timestamp_ns;
+        /* The time duration for the request from start encoding to finish encoding (send_end_timestamp_ns -
+         * send_start_timestamp_ns). When send_end_timestamp_ns is -1, means data not available. */
+        int64_t sending_duration_ns;
+
+        /* The time stamp when the response started to be received from the network channel. -1 means data not
+         * available. Timestamp are from `aws_high_res_clock_get_ticks` */
+        int64_t receive_start_timestamp_ns;
+        /* The time stamp when the response finished to be received from the network channel. -1 means data not
+         * available. Timestamp are from `aws_high_res_clock_get_ticks` */
+        int64_t receive_end_timestamp_ns;
+        /* The time duration for the request from start receiving to finish receiving (receive_end_timestamp_ns -
+         * receive_start_timestamp_ns). When receive_end_timestamp_ns is 0, means data not available. */
+        int64_t receiving_duration_ns;
+    } time_metrics;
+
+    struct {
+        /* Response status code for the request */
+        int response_status;
+        /* HTTP Headers of the response received. */
+        struct aws_http_headers *response_headers;
+        /* Path and query of the request. */
+        struct aws_string *request_path_query;
+        /* The host address of the request. */
+        struct aws_string *host_address;
+        /* The the request ID header value. */
+        struct aws_string *request_id;
+    } req_resp_info_metrics;
+
+    struct {
+        /* The IP address of the request connected to */
+        struct aws_string *ip_address;
+        /* The pointer to the connection that request was made from */
+        void *connection_id;
+        /* The aws_thread_id_t to the thread that request ran on */
+        aws_thread_id_t thread_id;
+        /* The stream-id, which is the idex when the stream was activated. */
+        uint32_t stream_id;
+        /* CRT error code when the aws_s3_request finishes. */
+        int error_code;
+    } crt_info_metrics;
+
+    struct aws_ref_count ref_count;
+};
+
 /* Represents a single request made to S3. */
 struct aws_s3_request {
 
@@ -103,6 +171,8 @@ struct aws_s3_request {
         /* Returned response status of this request. */
         int response_status;
 
+        /* The metrics for the request telemetry */
+        struct aws_s3_request_metrics *metrics;
     } send_data;
 
     /* When true, response headers from the request will be stored in the request's response_headers variable. */
@@ -147,6 +217,11 @@ void aws_s3_request_acquire(struct aws_s3_request *request);
 AWS_S3_API
 void aws_s3_request_release(struct aws_s3_request *request);
 
+AWS_S3_API
+struct aws_s3_request_metrics *aws_s3_request_metrics_new(
+    struct aws_allocator *allocator,
+    struct aws_http_message *message);
+
 AWS_EXTERN_C_END
 
 #endif

include/aws/s3/private/s3_util.h

@@ -145,6 +145,8 @@ extern const struct aws_byte_cursor g_range_header_name;
 
 extern const struct aws_byte_cursor g_if_match_header_name;
 
+extern const struct aws_byte_cursor g_request_id_header_name;
+
 AWS_S3_API
 extern const struct aws_byte_cursor g_content_range_header_name;
 

include/aws/s3/s3.h

@@ -35,6 +35,7 @@ enum aws_s3_errors {
     AWS_ERROR_S3_RESUME_FAILED,
     AWS_ERROR_S3_OBJECT_MODIFIED,
     AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR,
+    AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE,
     AWS_ERROR_S3_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_S3_PACKAGE_ID)
 };
 

include/aws/s3/s3_client.h

@@ -26,6 +26,8 @@ struct aws_s3_meta_request_resume_token;
 struct aws_uri;
 struct aws_string;
 
+struct aws_s3_request_metrics;
+
 /**
  * A Meta Request represents a group of generated requests that are being done on behalf of the
  * original request. For example, one large GetObject request can be transformed into a series
@@ -138,6 +140,17 @@ typedef void(aws_s3_meta_request_progress_fn)(
     const struct aws_s3_meta_request_progress *progress,
     void *user_data);
 
+/**
+ * Invoked to report the telemetry of the meta request once a single request finishes. Invoked from the thread of the
+ * connection that request made from.
+ * Note: *metrics is only valid for the duration of the callback. If you need to keep it around, use
+ * `aws_s3_request_metrics_acquire`
+ */
+typedef void(aws_s3_meta_request_telemetry_fn)(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request_metrics *metrics,
+    void *user_data);
+
 typedef void(aws_s3_meta_request_shutdown_fn)(void *user_data);
 
 typedef void(aws_s3_client_shutdown_complete_callback_fn)(void *user_data);
@@ -412,6 +425,18 @@ struct aws_s3_meta_request_options {
      */
     aws_s3_meta_request_progress_fn *progress_callback;
 
+    /**
+     * Optional.
+     * To get telemetry metrics when a single request finishes.
+     * If set the request will keep track of the metrics from `aws_s3_request_metrics`, and fire the callback when the
+     * request finishes receiving response.
+     * See `aws_s3_meta_request_telemetry_fn`
+     *
+     * Notes:
+     * - The callback will be invoked multiple times from different threads.
+     */
+    aws_s3_meta_request_telemetry_fn *telemetry_callback;
+
     /**
      * Optional.
      * Endpoint override for request. Can be used to override scheme and port of
@@ -661,12 +686,170 @@ struct aws_s3_meta_request *aws_s3_meta_request_acquire(struct aws_s3_meta_reque
 AWS_S3_API
 struct aws_s3_meta_request *aws_s3_meta_request_release(struct aws_s3_meta_request *meta_request);
 
+/**
+ * Initialize the configuration for a default S3 signing.
+ */
 AWS_S3_API
 void aws_s3_init_default_signing_config(
     struct aws_signing_config_aws *signing_config,
     const struct aws_byte_cursor region,
     struct aws_credentials_provider *credentials_provider);
 
+/**
+ * Add a reference, keeping this object alive.
+ * The reference must be released when you are done with it, or it's memory will never be cleaned up.
+ * Always returns the same pointer that was passed in.
+ */
+AWS_S3_API
+struct aws_s3_request_metrics *aws_s3_request_metrics_acquire(struct aws_s3_request_metrics *metrics);
+
+/**
+ * Release a reference.
+ * When the reference count drops to 0, this object will be cleaned up.
+ * It's OK to pass in NULL (nothing happens).
+ * Always returns NULL.
+ */
+AWS_S3_API
+struct aws_s3_request_metrics *aws_s3_request_metrics_release(struct aws_s3_request_metrics *metrics);
+
+/*************************************  Getters for s3 request metrics ************************************************/
+/**
+ * Get the request ID from aws_s3_request_metrics.
+ * If unavailable, AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised.
+ * If available, out_request_id will be set to a string. Be warned this string's lifetime is tied to the metrics
+ * object.
+ **/
+AWS_S3_API
+int aws_s3_request_metrics_get_request_id(
+    const struct aws_s3_request_metrics *metrics,
+    const struct aws_string **out_request_id);
+
+/* Get the start time from aws_s3_request_metrics, which is when S3 client prepare the request to be sent. Always
+ * available. Timestamp are from `aws_high_res_clock_get_ticks`  */
+AWS_S3_API
+void aws_s3_request_metrics_get_start_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_start_time);
+
+/* Get the end time from aws_s3_request_metrics. Always available */
+AWS_S3_API
+void aws_s3_request_metrics_get_end_timestamp_ns(const struct aws_s3_request_metrics *metrics, uint64_t *out_end_time);
+
+/* Get the total duration time from aws_s3_request_metrics. Always available */
+AWS_S3_API
+void aws_s3_request_metrics_get_total_duration_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_total_duration);
+
+/* Get the time stamp when the request started to be encoded. Timestamps are from `aws_high_res_clock_get_ticks`
+ * AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if the request ended before it gets sent. */
+AWS_S3_API
+int aws_s3_request_metrics_get_send_start_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_send_start_time);
+
+/* Get the time stamp when the request finished to be encoded. Timestamps are from `aws_high_res_clock_get_ticks`
+ * AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data not available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_send_end_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_send_end_time);
+
+/* The time duration for the request from start encoding to finish encoding (send_end_timestamp_ns -
+ * send_start_timestamp_ns).
+ * AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data not available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_sending_duration_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_sending_duration);
+
+/* Get the time stamp when the response started to be received from the network channel. Timestamps are from
+ * `aws_high_res_clock_get_ticks` AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data not available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_receive_start_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_receive_start_time);
+
+/* Get the time stamp when the response finished to be received from the network channel. Timestamps are from
+ * `aws_high_res_clock_get_ticks` AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data not available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_receive_end_timestamp_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_receive_end_time);
+
+/* The time duration for the request from start receiving to finish receiving (receive_end_timestamp_ns -
+ * receive_start_timestamp_ns).
+ * AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data not available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_receiving_duration_ns(
+    const struct aws_s3_request_metrics *metrics,
+    uint64_t *out_receiving_duration);
+
+/* Get the response status code for the request. AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data not
+ * available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_response_status_code(
+    const struct aws_s3_request_metrics *metrics,
+    int *out_response_status);
+
+/* Get the HTTP Headers of the response received for the request. AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised
+ * if data not available. */
+AWS_S3_API
+int aws_s3_request_metrics_get_response_headers(
+    const struct aws_s3_request_metrics *metrics,
+    struct aws_http_headers **out_response_headers);
+
+/**
+ * Get the path and query of the request.
+ * If unavailable, AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised.
+ * If available, out_request_path_query will be set to a string. Be warned this string's lifetime is tied to the metrics
+ * object.
+ */
+AWS_S3_API
+void aws_s3_request_metrics_get_request_path_query(
+    const struct aws_s3_request_metrics *metrics,
+    const struct aws_string **out_request_path_query);
+
+/**
+ * Get the host_address of the request.
+ * If unavailable, AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised.
+ * If available, out_host_address will be set to a string. Be warned this string's lifetime is tied to the metrics
+ * object.
+ */
+AWS_S3_API
+void aws_s3_request_metrics_get_host_address(
+    const struct aws_s3_request_metrics *metrics,
+    const struct aws_string **out_host_address);
+
+/**
+ * Get the IP address of the request connected to.
+ * If unavailable, AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised.
+ * If available, out_ip_address will be set to a string. Be warned this string's lifetime is tied to the metrics object.
+ */
+AWS_S3_API
+int aws_s3_request_metrics_get_ip_address(
+    const struct aws_s3_request_metrics *metrics,
+    const struct aws_string **out_ip_address);
+
+/* Get the id of connection that request was made from. AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if data
+ * not available */
+AWS_S3_API
+int aws_s3_request_metrics_get_connection_id(const struct aws_s3_request_metrics *metrics, size_t *out_connection_id);
+
+/* Get the thread ID of the thread that request was made from. AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be raised if
+ * data not available */
+AWS_S3_API
+int aws_s3_request_metrics_get_thread_id(const struct aws_s3_request_metrics *metrics, aws_thread_id_t *out_thread_id);
+
+/* Get the stream-id, which is the idex when the stream was activated. AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE will be
+ * raised if data not available */
+AWS_S3_API
+int aws_s3_request_metrics_get_request_stream_id(const struct aws_s3_request_metrics *metrics, uint32_t *out_stream_id);
+
+/* Get the AWS CRT error code from request metrics. */
+AWS_S3_API
+int aws_s3_request_metrics_get_error_code(const struct aws_s3_request_metrics *out_metrics);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_S3_CLIENT_H */

source/s3.c

@@ -35,7 +35,8 @@ static struct aws_error_info s_errors[] = {
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_RESUMED_PART_CHECKSUM_MISMATCH, "Checksum does not match previously uploaded part"),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_RESUME_FAILED, "Resuming request failed"),
     AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_OBJECT_MODIFIED, "The object modifed during download."),
-    AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR, "Async error received from S3 and not recoverable from retry.")
+    AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR, "Async error received from S3 and not recoverable from retry."),
+    AWS_DEFINE_ERROR_INFO_S3(AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE, "The metric data is not available, the requests ends before the metric happens."),
 };
 /* clang-format on */
 

source/s3_client.c

@@ -1596,9 +1596,9 @@ static void s_s3_client_create_connection_for_request_default(
     struct aws_http_headers *message_headers = aws_http_message_get_headers(meta_request->initial_request_message);
     AWS_ASSERT(message_headers);
 
-    int get_header_result = aws_http_headers_get(message_headers, g_host_header_name, &host_header_value);
-    AWS_ASSERT(get_header_result == AWS_OP_SUCCESS);
-    (void)get_header_result;
+    int result = aws_http_headers_get(message_headers, g_host_header_name, &host_header_value);
+    AWS_ASSERT(result == AWS_OP_SUCCESS);
+    (void)result;
 
     if (aws_retry_strategy_acquire_retry_token(
             client->retry_strategy, &host_header_value, s_s3_client_acquired_retry_token, connection, 0)) {
@@ -1750,6 +1750,9 @@ void aws_s3_client_notify_connection_finished(
 
     struct aws_s3_endpoint *endpoint = meta_request->endpoint;
     AWS_PRECONDITION(endpoint);
+    if (request->send_data.metrics) {
+        request->send_data.metrics->crt_info_metrics.error_code = error_code;
+    }
 
     /* If we're trying to setup a retry... */
     if (finish_code == AWS_S3_CONNECTION_FINISH_CODE_RETRY) {