aws: Refactor libcurl metadata fetcher to accept RequestMessage as ar…

…gument (envoyproxy#20954)

Signed-off-by: Sunil Narasimhamurthy <sunnrs@amazon.com>

source/extensions/common/aws/BUILD

@@ -29,7 +29,7 @@ envoy_cc_library(
         "//source/common/common:logger_lib",
         "//source/common/common:utility_lib",
         "//source/common/crypto:utility_lib",
-        "//source/common/http:headers_lib",
+        "//source/common/http:message_lib",
         "//source/common/singleton:const_singleton",
     ],
 )
@@ -47,6 +47,7 @@ envoy_cc_library(
     external_deps = ["abseil_time"],
     deps = [
         ":credentials_provider_interface",
+        ":utility_lib",
         "//envoy/api:api_interface",
         "//source/common/common:logger_lib",
         "//source/common/common:thread_lib",
@@ -61,6 +62,7 @@ envoy_cc_library(
     hdrs = ["utility.h"],
     external_deps = ["curl"],
     deps = [
+        "//envoy/http:message_interface",
         "//source/common/common:empty_string",
         "//source/common/common:matchers_lib",
         "//source/common/common:utility_lib",

source/extensions/common/aws/credentials_provider.h

@@ -21,9 +21,10 @@ namespace Aws {
  */
 class Credentials {
 public:
-  Credentials(absl::string_view access_key_id = absl::string_view(),
-              absl::string_view secret_access_key = absl::string_view(),
-              absl::string_view session_token = absl::string_view()) {
+  explicit Credentials(absl::string_view access_key_id = absl::string_view(),
+                       absl::string_view secret_access_key = absl::string_view(),
+                       absl::string_view session_token = absl::string_view()) {
+    // TODO(suniltheta): Move credential expiration date in here
     if (!access_key_id.empty()) {
       access_key_id_ = std::string(access_key_id);
       if (!secret_access_key.empty()) {

source/extensions/common/aws/credentials_provider_impl.cc

@@ -3,8 +3,10 @@
 #include "envoy/common/exception.h"
 
 #include "source/common/common/lock_guard.h"
+#include "source/common/http/message_impl.h"
 #include "source/common/http/utility.h"
 #include "source/common/json/json_loader.h"
+#include "source/extensions/common/aws/utility.h"
 
 namespace Envoy {
 namespace Extensions {
@@ -71,20 +73,29 @@ void InstanceProfileCredentialsProvider::refresh() {
   ENVOY_LOG(debug, "Getting AWS credentials from the instance metadata");
 
   // First discover the Role of this instance
-  const auto instance_role_string =
-      metadata_fetcher_(EC2_METADATA_HOST, SECURITY_CREDENTIALS_PATH, "");
+  Http::RequestMessageImpl message;
+  message.headers().setMethod(Http::Headers::get().MethodValues.Get);
+  message.headers().setHost(EC2_METADATA_HOST);
+  message.headers().setPath(SECURITY_CREDENTIALS_PATH);
+  const auto instance_role_string = metadata_fetcher_(message);
   if (!instance_role_string) {
     ENVOY_LOG(error, "Could not retrieve credentials listing from the instance metadata");
     return;
   }
+  fetchCredentialFromInstanceRole(instance_role_string.value());
+}
 
-  const auto instance_role_list =
-      StringUtil::splitToken(StringUtil::trim(instance_role_string.value()), "\n");
+void InstanceProfileCredentialsProvider::fetchCredentialFromInstanceRole(
+    const std::string& instance_role) {
+  if (instance_role.empty()) {
+    return;
+  }
+  const auto instance_role_list = StringUtil::splitToken(StringUtil::trim(instance_role), "\n");
   if (instance_role_list.empty()) {
     ENVOY_LOG(error, "No AWS credentials were found in the instance metadata");
     return;
   }
-  ENVOY_LOG(debug, "AWS credentials list:\n{}", instance_role_string.value());
+  ENVOY_LOG(debug, "AWS credentials list:\n{}", instance_role);
 
   // Only one Role can be associated with an instance:
   // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
@@ -94,15 +105,27 @@ void InstanceProfileCredentialsProvider::refresh() {
   ENVOY_LOG(debug, "AWS credentials path: {}", credential_path);
 
   // Then fetch and parse the credentials
-  const auto credential_document = metadata_fetcher_(EC2_METADATA_HOST, credential_path, "");
+  Http::RequestMessageImpl message;
+  message.headers().setMethod(Http::Headers::get().MethodValues.Get);
+  message.headers().setHost(EC2_METADATA_HOST);
+  message.headers().setPath(credential_path);
+
+  const auto credential_document = metadata_fetcher_(message);
   if (!credential_document) {
     ENVOY_LOG(error, "Could not load AWS credentials document from the instance metadata");
     return;
   }
+  extractCredentials(credential_document.value());
+}
 
+void InstanceProfileCredentialsProvider::extractCredentials(
+    const std::string& credential_document_value) {
+  if (credential_document_value.empty()) {
+    return;
+  }
   Json::ObjectSharedPtr document_json;
   try {
-    document_json = Json::Factory::loadFromString(credential_document.value());
+    document_json = Json::Factory::loadFromString(credential_document_value);
   } catch (EnvoyException& e) {
     ENVOY_LOG(error, "Could not parse AWS credentials document: {}", e.what());
     return;
@@ -133,17 +156,27 @@ void TaskRoleCredentialsProvider::refresh() {
   absl::string_view host;
   absl::string_view path;
   Http::Utility::extractHostPathFromUri(credential_uri_, host, path);
-  const auto credential_document =
-      metadata_fetcher_(std::string(host.data(), host.size()),
-                        std::string(path.data(), path.size()), authorization_token_);
+
+  Http::RequestMessageImpl message;
+  message.headers().setMethod(Http::Headers::get().MethodValues.Get);
+  message.headers().setHost(host);
+  message.headers().setPath(path);
+  message.headers().setCopy(Http::CustomHeaders::get().Authorization, authorization_token_);
+  const auto credential_document = metadata_fetcher_(message);
   if (!credential_document) {
     ENVOY_LOG(error, "Could not load AWS credentials document from the task role");
     return;
   }
+  extractCredentials(credential_document.value());
+}
 
+void TaskRoleCredentialsProvider::extractCredentials(const std::string& credential_document_value) {
+  if (credential_document_value.empty()) {
+    return;
+  }
   Json::ObjectSharedPtr document_json;
   try {
-    document_json = Json::Factory::loadFromString(credential_document.value());
+    document_json = Json::Factory::loadFromString(credential_document_value);
   } catch (EnvoyException& e) {
     ENVOY_LOG(error, "Could not parse AWS credentials document from the task role: {}", e.what());
     return;

source/extensions/common/aws/credentials_provider_impl.h

@@ -4,6 +4,7 @@
 
 #include "envoy/api/api.h"
 #include "envoy/event/timer.h"
+#include "envoy/http/message.h"
 
 #include "source/common/common/logger.h"
 #include "source/common/common/thread.h"
@@ -31,8 +32,7 @@ class EnvironmentCredentialsProvider : public CredentialsProvider,
 class MetadataCredentialsProviderBase : public CredentialsProvider,
                                         public Logger::Loggable<Logger::Id::aws> {
 public:
-  using MetadataFetcher = std::function<absl::optional<std::string>(
-      const std::string& host, const std::string& path, const std::string& auth_token)>;
+  using MetadataFetcher = std::function<absl::optional<std::string>(Http::RequestMessage&)>;
 
   MetadataCredentialsProviderBase(Api::Api& api, const MetadataFetcher& metadata_fetcher)
       : api_(api), metadata_fetcher_(metadata_fetcher) {}
@@ -68,6 +68,8 @@ class InstanceProfileCredentialsProvider : public MetadataCredentialsProviderBas
 private:
   bool needsRefresh() override;
   void refresh() override;
+  void extractCredentials(const std::string& credential_document_value);
+  void fetchCredentialFromInstanceRole(const std::string& instance_role);
 };
 
 /**
@@ -90,6 +92,7 @@ class TaskRoleCredentialsProvider : public MetadataCredentialsProviderBase {
 
   bool needsRefresh() override;
   void refresh() override;
+  void extractCredentials(const std::string& credential_document_value);
 };
 
 /**

source/extensions/common/aws/signer_impl.h

@@ -48,7 +48,7 @@ using AwsSigV4HeaderExclusionVector = std::vector<envoy::type::matcher::v3::Stri
  * Implementation of the Signature V4 signing process.
  * See https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
  */
-class SignerImpl : public Signer, public Logger::Loggable<Logger::Id::http> {
+class SignerImpl : public Signer, public Logger::Loggable<Logger::Id::aws> {
 public:
   SignerImpl(absl::string_view service_name, absl::string_view region,
              const CredentialsProviderSharedPtr& credentials_provider, TimeSource& time_source,

source/extensions/common/aws/utility.cc

@@ -223,9 +223,7 @@ static size_t curlCallback(char* ptr, size_t, size_t nmemb, void* data) {
   return nmemb;
 }
 
-absl::optional<std::string> Utility::metadataFetcher(const std::string& host,
-                                                     const std::string& path,
-                                                     const std::string& auth_token) {
+absl::optional<std::string> Utility::fetchMetadata(Http::RequestMessage& message) {
   static const size_t MAX_RETRIES = 4;
   static const std::chrono::milliseconds RETRY_DELAY{1000};
   static const std::chrono::seconds TIMEOUT{5};
@@ -235,7 +233,10 @@ absl::optional<std::string> Utility::metadataFetcher(const std::string& host,
     return absl::nullopt;
   };
 
-  const std::string url = fmt::format("http://{}/{}", host, path);
+  const auto host = message.headers().getHostValue();
+  const auto path = message.headers().getPathValue();
+
+  const std::string url = fmt::format("http://{}{}", host, path);
   curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
   curl_easy_setopt(curl, CURLOPT_TIMEOUT, TIMEOUT.count());
   curl_easy_setopt(curl, CURLOPT_FAILONERROR, 1L);
@@ -246,9 +247,17 @@ absl::optional<std::string> Utility::metadataFetcher(const std::string& host,
   curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curlCallback);
 
   struct curl_slist* headers = nullptr;
-  if (!auth_token.empty()) {
-    const std::string auth = fmt::format("Authorization: {}", auth_token);
-    headers = curl_slist_append(headers, auth.c_str());
+  message.headers().iterate([&headers](const Http::HeaderEntry& entry) -> Http::HeaderMap::Iterate {
+    // Skip pseudo-headers
+    if (!entry.key().getStringView().empty() && entry.key().getStringView()[0] == ':') {
+      return Http::HeaderMap::Iterate::Continue;
+    }
+    const std::string header =
+        fmt::format("{}: {}", entry.key().getStringView(), entry.value().getStringView());
+    headers = curl_slist_append(headers, header.c_str());
+    return Http::HeaderMap::Iterate::Continue;
+  });
+  if (headers != nullptr) {
     curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
   }
 

source/extensions/common/aws/utility.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include "envoy/http/message.h"
+
 #include "source/common/common/matchers.h"
 #include "source/common/http/headers.h"
 
@@ -81,18 +83,15 @@ class Utility {
   /**
    * Fetch AWS instance or task metadata.
    *
-   * @param host host or ip address of the metadata endpoint.
-   * @param path path of the metadata document.
-   * @auth_token authentication token to pass in the request, empty string indicates no auth.
+   * @param message An HTTP request.
    * @return Metadata document or nullopt in case if unable to fetch it.
    *
    * @note In case of an error, function will log ENVOY_LOG_MISC(debug) message.
    *
    * @note This is not main loop safe method as it is blocking. It is intended to be used from the
    * gRPC auth plugins that are able to schedule blocking plugins on a different thread.
    */
-  static absl::optional<std::string>
-  metadataFetcher(const std::string& host, const std::string& path, const std::string& auth_token);
+  static absl::optional<std::string> fetchMetadata(Http::RequestMessage& message);
 };
 
 } // namespace Aws

source/extensions/filters/http/aws_lambda/config.cc

@@ -37,15 +37,16 @@ Http::FilterFactoryCb AwsLambdaFilterFactory::createFilterFactoryFromProtoTyped(
     const envoy::extensions::filters::http::aws_lambda::v3::Config& proto_config,
     const std::string& stat_prefix, Server::Configuration::FactoryContext& context) {
 
-  auto credentials_provider =
-      std::make_shared<Extensions::Common::Aws::DefaultCredentialsProviderChain>(
-          context.api(), Extensions::Common::Aws::Utility::metadataFetcher);
-
   const auto arn = parseArn(proto_config.arn());
   if (!arn) {
     throw EnvoyException(fmt::format("aws_lambda_filter: Invalid ARN: {}", proto_config.arn()));
   }
   const std::string region = arn->region();
+
+  auto credentials_provider =
+      std::make_shared<Extensions::Common::Aws::DefaultCredentialsProviderChain>(
+          context.api(), Extensions::Common::Aws::Utility::fetchMetadata);
+
   auto signer = std::make_shared<Extensions::Common::Aws::SignerImpl>(
       service_name, region, std::move(credentials_provider),
       context.mainThreadDispatcher().timeSource(),

source/extensions/filters/http/aws_request_signing/config.cc

@@ -20,7 +20,7 @@ Http::FilterFactoryCb AwsRequestSigningFilterFactory::createFilterFactoryFromPro
 
   auto credentials_provider =
       std::make_shared<Extensions::Common::Aws::DefaultCredentialsProviderChain>(
-          context.api(), Extensions::Common::Aws::Utility::metadataFetcher);
+          context.api(), Extensions::Common::Aws::Utility::fetchMetadata);
   const auto matcher_config = Extensions::Common::Aws::AwsSigV4HeaderExclusionVector(
       config.match_excluded_headers().begin(), config.match_excluded_headers().end());
   auto signer = std::make_unique<Extensions::Common::Aws::SignerImpl>(

source/extensions/grpc_credentials/aws_iam/config.cc

@@ -45,7 +45,7 @@ std::shared_ptr<grpc::ChannelCredentials> AwsIamGrpcCredentialsFactory::getChann
             const envoy::config::grpc_credential::v3::AwsIamConfig&>(
             *config_message, ProtobufMessage::getNullValidationVisitor());
         auto credentials_provider = std::make_shared<Common::Aws::DefaultCredentialsProviderChain>(
-            api, Common::Aws::Utility::metadataFetcher);
+            api, Common::Aws::Utility::fetchMetadata);
         auto signer = std::make_unique<Common::Aws::SignerImpl>(
             config.service_name(), getRegion(config), credentials_provider, api.timeSource(),
             // TODO: extend API to allow specifying header exclusion. ref: