Implement Api Key authorisation on requests

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -116,6 +116,7 @@ public abstract class ConnectionConfiguration<T> : IConnectionConfigurationValue
 		private readonly ElasticsearchUrlFormatter _urlFormatter;
 
 		private BasicAuthenticationCredentials _basicAuthCredentials;
+		private ApiKeyAuthenticationCredentials _apiKeyAuthCredentials;
 		private X509CertificateCollection _clientCertificates;
 		private Action<IApiCallDetails> _completedRequestHandler = DefaultCompletedRequestHandler;
 		private int _connectionLimit;
@@ -169,6 +170,7 @@ protected ConnectionConfiguration(IConnectionPool connectionPool, IConnection co
 
 		protected IElasticsearchSerializer UseThisRequestResponseSerializer { get; set; }
 		BasicAuthenticationCredentials IConnectionConfigurationValues.BasicAuthenticationCredentials => _basicAuthCredentials;
+		ApiKeyAuthenticationCredentials IConnectionConfigurationValues.ApiKeyAuthenticationCredentials => _apiKeyAuthCredentials;
 		SemaphoreSlim IConnectionConfigurationValues.BootstrapLock => _semaphore;
 		X509CertificateCollection IConnectionConfigurationValues.ClientCertificates => _clientCertificates;
 		IConnection IConnectionConfigurationValues.Connection => _connection;
@@ -434,6 +436,18 @@ public T BasicAuthentication(string username, string password) =>
 		public T BasicAuthentication(string username, SecureString password) =>
 			Assign(new BasicAuthenticationCredentials(username, password), (a, v) => a._basicAuthCredentials = v);
 
+		/// <summary>
+		/// Api Key to send with all requests to Elasticsearch
+		/// </summary>
+		public T ApiKeyAuthentication(string id, SecureString apiKey) =>
+			Assign(new ApiKeyAuthenticationCredentials(id, apiKey), (a, v) => a._apiKeyAuthCredentials = v);
+
+		/// <summary>
+		/// Api Key to send with all requests to Elasticsearch
+		/// </summary>
+		public T ApiKeyAuthentication(string id, string apiKey) =>
+			Assign(new ApiKeyAuthenticationCredentials(id, apiKey), (a, v) => a._apiKeyAuthCredentials = v);
+
 		/// <summary>
 		/// Allows for requests to be pipelined. http://en.wikipedia.org/wiki/HTTP_pipelining
 		/// <para>NOTE: HTTP pipelining must also be enabled in Elasticsearch for this to work properly.</para>
@@ -523,6 +537,7 @@ protected virtual void DisposeManagedResources()
 			_semaphore?.Dispose();
 			_proxyPassword?.Dispose();
 			_basicAuthCredentials?.Dispose();
+			_apiKeyAuthCredentials?.Dispose();
 		}
 
 		protected virtual bool HttpStatusCodeClassifier(HttpMethod method, int statusCode) =>

src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs

@@ -13,8 +13,19 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// <summary>
 		/// Basic access authorization credentials to specify with all requests.
 		/// </summary>
+		/// <remarks>
+		/// Cannot be used in conjuction with <see cref="ApiKeyAuthenticationCredentials"/>
+		/// </remarks>
 		BasicAuthenticationCredentials BasicAuthenticationCredentials { get; }
 
+		/// <summary>
+		/// Api Key authorization credentials to specify with all requests.
+		/// </summary>
+		/// <remarks>
+		/// Cannot be used in conjuction with <see cref="BasicAuthenticationCredentials"/>
+		/// </remarks>
+		ApiKeyAuthenticationCredentials ApiKeyAuthenticationCredentials { get; }
+
 		/// <summary> Provides a semaphoreslim to transport implementations that need to limit access to a resource</summary>
 		SemaphoreSlim BootstrapLock { get; }
 

src/Elasticsearch.Net/Configuration/RequestConfiguration.cs

@@ -22,8 +22,20 @@ public interface IRequestConfiguration
 		/// Basic access authorization credentials to specify with this request.
 		/// Overrides any credentials that are set at the global IConnectionSettings level.
 		/// </summary>
+		/// <remarks>
+		///	Cannot be used in conjunction with <see cref="ApiKeyAuthenticationCredentials"/>
+		/// </remarks>
 		BasicAuthenticationCredentials BasicAuthenticationCredentials { get; set; }
 
+		/// <summary>
+		/// An API-key authorization credentials to specify with this request.
+		/// Overrides any credentials that are set at the global IConnectionSettings level.
+		/// </summary>
+		/// <remarks>
+		///	Cannot be used in conjunction with <see cref="BasicAuthenticationCredentials"/>
+		/// </remarks>
+		ApiKeyAuthenticationCredentials ApiKeyAuthenticationCredentials { get; set; }
+
 		/// <summary>
 		/// Use the following client certificates to authenticate this single request
 		/// </summary>
@@ -102,6 +114,8 @@ public class RequestConfiguration : IRequestConfiguration
 		public IReadOnlyCollection<int> AllowedStatusCodes { get; set; }
 		public BasicAuthenticationCredentials BasicAuthenticationCredentials { get; set; }
 
+		public ApiKeyAuthenticationCredentials ApiKeyAuthenticationCredentials { get; set; }
+
 		public X509CertificateCollection ClientCertificates { get; set; }
 		public string ContentType { get; set; }
 		public bool? DisableDirectStreaming { get; set; }
@@ -138,6 +152,7 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 			Self.DisableDirectStreaming = config?.DisableDirectStreaming;
 			Self.AllowedStatusCodes = config?.AllowedStatusCodes;
 			Self.BasicAuthenticationCredentials = config?.BasicAuthenticationCredentials;
+			Self.ApiKeyAuthenticationCredentials = config?.ApiKeyAuthenticationCredentials;
 			Self.EnableHttpPipelining = config?.EnableHttpPipelining ?? true;
 			Self.RunAs = config?.RunAs;
 			Self.ClientCertificates = config?.ClientCertificates;
@@ -148,6 +163,7 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 		string IRequestConfiguration.Accept { get; set; }
 		IReadOnlyCollection<int> IRequestConfiguration.AllowedStatusCodes { get; set; }
 		BasicAuthenticationCredentials IRequestConfiguration.BasicAuthenticationCredentials { get; set; }
+		ApiKeyAuthenticationCredentials IRequestConfiguration.ApiKeyAuthenticationCredentials { get; set; }
 		X509CertificateCollection IRequestConfiguration.ClientCertificates { get; set; }
 		string IRequestConfiguration.ContentType { get; set; }
 		bool? IRequestConfiguration.DisableDirectStreaming { get; set; }
@@ -274,6 +290,30 @@ public RequestConfigurationDescriptor BasicAuthentication(string userName, Secur
 			return this;
 		}
 
+		public RequestConfigurationDescriptor ApiKeyAuthentication(string id, string apiKey)
+		{
+			Self.ApiKeyAuthenticationCredentials = new ApiKeyAuthenticationCredentials(id, apiKey);
+			return this;
+		}
+
+		public RequestConfigurationDescriptor ApiKeyAuthentication(string id, SecureString apiKey)
+		{
+			Self.ApiKeyAuthenticationCredentials = new ApiKeyAuthenticationCredentials(id, apiKey);
+			return this;
+		}
+
+		public RequestConfigurationDescriptor ApiKeyAuthentication(string base64EncodedApiKey)
+		{
+			Self.ApiKeyAuthenticationCredentials = new ApiKeyAuthenticationCredentials(base64EncodedApiKey);
+			return this;
+		}
+
+		public RequestConfigurationDescriptor ApiKeyAuthentication(SecureString base64EncodedApiKey)
+		{
+			Self.ApiKeyAuthenticationCredentials = new ApiKeyAuthenticationCredentials(base64EncodedApiKey);
+			return this;
+		}
+
 		public RequestConfigurationDescriptor EnableHttpPipelining(bool enable = true)
 		{
 			Self.EnableHttpPipelining = enable;

src/Elasticsearch.Net/Configuration/Security/ApiKeyAuthenticationCredentials.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Security;
+using System.Text;
+
+namespace Elasticsearch.Net
+{
+	/// <summary>
+	/// Credentials for Api Key Authentication
+	/// </summary>
+	public class ApiKeyAuthenticationCredentials : IDisposable
+	{
+		public ApiKeyAuthenticationCredentials()
+		{
+		}
+
+		public ApiKeyAuthenticationCredentials(string id, SecureString apiKey)
+		{
+			Base64EncodedApiKey = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{id}:{apiKey.CreateString()}")).CreateSecureString();
+		}
+
+		public ApiKeyAuthenticationCredentials(string id, string apiKey)
+		{
+			Base64EncodedApiKey = Convert.ToBase64String(Encoding.UTF8.GetBytes($"{id}:{apiKey}")).CreateSecureString();
+		}
+
+		public ApiKeyAuthenticationCredentials(string base64EncodedApiKey)
+		{
+			Base64EncodedApiKey = base64EncodedApiKey.CreateSecureString();
+		}
+
+		public ApiKeyAuthenticationCredentials(SecureString base64EncodedApiKey)
+		{
+			Base64EncodedApiKey = base64EncodedApiKey;
+		}
+
+		/// <summary>
+		/// The Base64 encoded api key with which to authenticate
+		/// Take the form, id:api_key, which is then base 64 encoded
+		/// </summary>
+		public SecureString Base64EncodedApiKey { get; }
+
+		public void Dispose() => Base64EncodedApiKey?.Dispose();
+	}
+}

src/Elasticsearch.Net/Connection/HttpConnection.cs

@@ -216,10 +216,59 @@ protected virtual HttpMessageHandler CreateHttpClientHandler(RequestData request
 		protected virtual HttpRequestMessage CreateHttpRequestMessage(RequestData requestData)
 		{
 			var request = CreateRequestMessage(requestData);
-			SetBasicAuthenticationIfNeeded(request, requestData);
+			SetAuthenticationIfNeeded(request, requestData);
 			return request;
 		}
 
+		protected virtual void SetAuthenticationIfNeeded(HttpRequestMessage requestMessage, RequestData requestData)
+		{
+			// Api Key authentication takes precedence
+			var apiKeySet = SetApiKeyAuthenticationIfNeeded(requestMessage, requestData);
+
+			if (!apiKeySet)
+				SetBasicAuthenticationIfNeeded(requestMessage, requestData);
+		}
+
+		// TODO - make private in 8.0 and only expose SetAuthenticationIfNeeded
+		protected virtual bool SetApiKeyAuthenticationIfNeeded(HttpRequestMessage requestMessage, RequestData requestData)
+		{
+			// ApiKey auth credentials take the following precedence (highest -> lowest):
+			// 1 - Specified on the request (highest precedence)
+			// 2 - Specified at the global IConnectionSettings level
+
+			string apiKey = null;
+			if (requestData.ApiKeyAuthenticationCredentials != null)
+				apiKey = requestData.ApiKeyAuthenticationCredentials.Base64EncodedApiKey.CreateString();
+
+			if (string.IsNullOrWhiteSpace(apiKey))
+				return false;
+
+			requestMessage.Headers.Authorization = new AuthenticationHeaderValue("ApiKey", apiKey);
+			return true;
+
+		}
+
+		// TODO - make private in 8.0 and only expose SetAuthenticationIfNeeded
+		protected virtual void SetBasicAuthenticationIfNeeded(HttpRequestMessage requestMessage, RequestData requestData)
+		{
+			// Basic auth credentials take the following precedence (highest -> lowest):
+			// 1 - Specified on the request (highest precedence)
+			// 2 - Specified at the global IConnectionSettings level
+			// 3 - Specified with the URI (lowest precedence)
+
+			string userInfo = null;
+			if (!requestData.Uri.UserInfo.IsNullOrEmpty())
+				userInfo = Uri.UnescapeDataString(requestData.Uri.UserInfo);
+			else if (requestData.BasicAuthorizationCredentials != null)
+				userInfo =
+					$"{requestData.BasicAuthorizationCredentials.Username}:{requestData.BasicAuthorizationCredentials.Password.CreateString()}";
+			if (!userInfo.IsNullOrEmpty())
+			{
+				var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(userInfo));
+				requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
+			}
+		}
+
 		protected virtual HttpRequestMessage CreateRequestMessage(RequestData requestData)
 		{
 			var method = ConvertHttpMethod(requestData.Method);
@@ -256,21 +305,6 @@ protected virtual HttpRequestMessage CreateRequestMessage(RequestData requestDat
 		private static void SetAsyncContent(HttpRequestMessage message, RequestData requestData, CancellationToken token) =>
 			message.Content = new RequestDataContent(requestData, token);
 
-		protected virtual void SetBasicAuthenticationIfNeeded(HttpRequestMessage requestMessage, RequestData requestData)
-		{
-			string userInfo = null;
-			if (!requestData.Uri.UserInfo.IsNullOrEmpty())
-				userInfo = Uri.UnescapeDataString(requestData.Uri.UserInfo);
-			else if (requestData.BasicAuthorizationCredentials != null)
-				userInfo =
-					$"{requestData.BasicAuthorizationCredentials.Username}:{requestData.BasicAuthorizationCredentials.Password.CreateString()}";
-			if (!userInfo.IsNullOrEmpty())
-			{
-				var credentials = Convert.ToBase64String(Encoding.UTF8.GetBytes(userInfo));
-				requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Basic", credentials);
-			}
-		}
-
 		private static System.Net.Http.HttpMethod ConvertHttpMethod(HttpMethod httpMethod)
 		{
 			switch (httpMethod)

src/Elasticsearch.Net/Connection/HttpWebRequestConnection.cs

@@ -142,7 +142,7 @@ CancellationToken cancellationToken
 		protected virtual HttpWebRequest CreateHttpWebRequest(RequestData requestData)
 		{
 			var request = CreateWebRequest(requestData);
-			SetBasicAuthenticationIfNeeded(request, requestData);
+			SetAuthenticationIfNeeded(requestData, request);
 			SetProxyIfNeeded(request, requestData);
 			SetServerCertificateValidationCallBackIfNeeded(request, requestData);
 			SetClientCertificates(request, requestData);
@@ -239,6 +239,16 @@ protected virtual void SetProxyIfNeeded(HttpWebRequest request, RequestData requ
 				request.Proxy = null;
 		}
 
+		protected virtual void SetAuthenticationIfNeeded(RequestData requestData, HttpWebRequest request)
+		{
+			// Api Key authentication takes precedence
+			var apiKeySet = SetApiKeyAuthenticationIfNeeded(request, requestData);
+
+			if (!apiKeySet)
+				SetBasicAuthenticationIfNeeded(request, requestData);
+		}
+
+		// TODO - make private in 8.0 and only expose SetAuthenticationIfNeeded
 		protected virtual void SetBasicAuthenticationIfNeeded(HttpWebRequest request, RequestData requestData)
 		{
 			// Basic auth credentials take the following precedence (highest -> lowest):
@@ -253,11 +263,30 @@ protected virtual void SetBasicAuthenticationIfNeeded(HttpWebRequest request, Re
 				userInfo =
 					$"{requestData.BasicAuthorizationCredentials.Username}:{requestData.BasicAuthorizationCredentials.Password.CreateString()}";
 
+			if (string.IsNullOrWhiteSpace(userInfo))
+				return;
 
-			if (!string.IsNullOrWhiteSpace(userInfo))
-				request.Headers["Authorization"] = "Basic " + Convert.ToBase64String(Encoding.UTF8.GetBytes(userInfo));
+			request.Headers["Authorization"] = $"Basic {Convert.ToBase64String(Encoding.UTF8.GetBytes(userInfo))}";
 		}
 
+		// TODO - make private in 8.0 and only expose SetAuthenticationIfNeeded
+		protected virtual bool SetApiKeyAuthenticationIfNeeded(HttpWebRequest request, RequestData requestData)
+		{
+			// ApiKey auth credentials take the following precedence (highest -> lowest):
+			// 1 - Specified on the request (highest precedence)
+			// 2 - Specified at the global IConnectionSettings level
+
+			string apiKey = null;
+			if (requestData.ApiKeyAuthenticationCredentials != null)
+				apiKey = requestData.ApiKeyAuthenticationCredentials.Base64EncodedApiKey.CreateString();
+
+			if (string.IsNullOrWhiteSpace(apiKey))
+				return false;
+
+			request.Headers["Authorization"] = $"ApiKey {apiKey}";
+			return true;
+
+		}
 
 		/// <summary>
 		/// Registers an APM async task cancellation on the threadpool

src/Elasticsearch.Net/Transport/Pipeline/RequestData.cs

@@ -76,6 +76,7 @@ IMemoryStreamFactory memoryStreamFactory
 			ProxyPassword = global.ProxyPassword;
 			DisableAutomaticProxyDetection = global.DisableAutomaticProxyDetection;
 			BasicAuthorizationCredentials = local?.BasicAuthenticationCredentials ?? global.BasicAuthenticationCredentials;
+			ApiKeyAuthenticationCredentials = local?.ApiKeyAuthenticationCredentials ?? global.ApiKeyAuthenticationCredentials;
 			AllowedStatusCodes = local?.AllowedStatusCodes ?? EmptyReadOnly<int>.Collection;
 			ClientCertificates = local?.ClientCertificates ?? global.ClientCertificates;
 			UserAgent = global.UserAgent;
@@ -86,6 +87,8 @@ IMemoryStreamFactory memoryStreamFactory
 		public string Accept { get; }
 		public IReadOnlyCollection<int> AllowedStatusCodes { get; }
 
+		public ApiKeyAuthenticationCredentials ApiKeyAuthenticationCredentials { get; }
+
 		public BasicAuthenticationCredentials BasicAuthorizationCredentials { get; }
 
 		public X509CertificateCollection ClientCertificates { get; }

src/Elasticsearch.Net/Transport/Pipeline/RequestPipeline.cs

@@ -22,7 +22,7 @@ public class RequestPipeline : IRequestPipeline
 		private readonly IDateTimeProvider _dateTimeProvider;
 		private readonly IMemoryStreamFactory _memoryStreamFactory;
 		private readonly IConnectionConfigurationValues _settings;
-		
+
 		private static DiagnosticSource DiagnosticSource { get; } = new DiagnosticListener(DiagnosticSources.RequestPipeline.SourceName);
 
 		public RequestPipeline(
@@ -392,7 +392,7 @@ public void Ping(Node node)
 		public async Task PingAsync(Node node, CancellationToken cancellationToken)
 		{
 			if (PingDisabled(node)) return;
-			
+
 			var pingData = CreatePingRequestData(node);
 			using (var audit = Audit(PingSuccess, node))
 			using (var d = DiagnosticSource.Diagnose<RequestData, IApiCallDetails>(DiagnosticSources.RequestPipeline.Ping, pingData))
@@ -432,7 +432,7 @@ public void Sniff()
 						audit.Path = requestData.PathAndQuery;
 						var response = _connection.Request<SniffResponse>(requestData);
 						d.EndState = response;
-						
+
 						ThrowBadAuthPipelineExceptionWhenNeeded(response);
 						//sniff should not silently accept bad but valid http responses
 						if (!response.Success)
@@ -469,7 +469,7 @@ public async Task SniffAsync(CancellationToken cancellationToken)
 						audit.Path = requestData.PathAndQuery;
 						var response = await _connection.RequestAsync<SniffResponse>(requestData, cancellationToken).ConfigureAwait(false);
 						d.EndState = response;
-						
+
 						ThrowBadAuthPipelineExceptionWhenNeeded(response);
 						//sniff should not silently accept bad but valid http responses
 						if (!response.Success)
@@ -554,6 +554,7 @@ private RequestData CreatePingRequestData(Node node)
 				PingTimeout = PingTimeout,
 				RequestTimeout = PingTimeout,
 				BasicAuthenticationCredentials = _settings.BasicAuthenticationCredentials,
+				ApiKeyAuthenticationCredentials = _settings.ApiKeyAuthenticationCredentials,
 				EnableHttpPipelining = RequestConfiguration?.EnableHttpPipelining ?? _settings.HttpPipeliningEnabled,
 				ForceNode = RequestConfiguration?.ForceNode
 			};

src/Tests/Tests.Core/Extensions/EphemeralClusterExtensions.cs

@@ -30,7 +30,10 @@ public static IElasticClient GetOrAddClient<TConfig>(
 				var settings = modifySettings(cluster.CreateConnectionSettings());
 
 				var current = (IConnectionConfigurationValues)settings;
-				var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null && current.ClientCertificates == null;
+				var notAlreadyAuthenticated = current.BasicAuthenticationCredentials == null
+					&& current.ApiKeyAuthenticationCredentials == null
+					&& current.ClientCertificates == null;
+
 				var noCertValidation = current.ServerCertificateValidationCallback == null;
 
 				if (cluster.ClusterConfiguration.EnableSecurity && notAlreadyAuthenticated)