Add support for the SQL APIs (#3414)

* Add QuerySql API methods and tests
* Add ClearSqlCursor API methods and tests
* Add TranslateSql API methods and tests

Author: Mpdreamz

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -80,9 +80,7 @@ public static void Generate(string downloadBranch, params string[] folders)
 			"xpack.rollup.rollup_search.json",
 			"xpack.rollup.start_job.json",
 			"xpack.rollup.stop_job.json",
-			"xpack.sql.clear_cursor.json",
-			"xpack.sql.query.json",
-			"xpack.sql.translate.json",
+
 			"xpack.ssl.certificates.json",
 
 			// 6.4 new  API's

src/CodeGeneration/ApiGenerator/Program.cs

@@ -41,7 +41,8 @@ static void Main(string[] args)
 			if (redownloadCoreSpecification)
 				RestSpecDownloader.Download(downloadBranch);
 
-			ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info", "MachineLearning", "Migration");
+			ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info",
+				"MachineLearning", "Migration", "Sql");
 		}
 	}
 }

src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs

@@ -2563,6 +2563,23 @@ public partial class PutUserRequestParameters : RequestParameters<PutUserRequest
 		///</summary>
 		public Refresh? Refresh { get => Q<Refresh?>("refresh"); set => Q("refresh", value); }
 	}
+	///<summary>Request options for XpackSqlClearCursor<pre>Clear SQL cursor</pre></summary>
+	public partial class ClearSqlCursorRequestParameters : RequestParameters<ClearSqlCursorRequestParameters> 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+	}
+	///<summary>Request options for XpackSqlQuery<pre>Execute SQL</pre></summary>
+	public partial class QuerySqlRequestParameters : RequestParameters<QuerySqlRequestParameters> 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+		///<summary>a short version of the Accept header, e.g. json, yaml</summary>
+		public string Format { get => Q<string>("format"); set => Q("format", value); }
+	}
+	///<summary>Request options for XpackSqlTranslate<pre>Translate SQL into Elasticsearch queries</pre></summary>
+	public partial class TranslateSqlRequestParameters : RequestParameters<TranslateSqlRequestParameters> 
+	{
+		public override HttpMethod DefaultHttpMethod => HttpMethod.POST;
+	}
 	///<summary>Request options for XpackWatcherAckWatch<pre>http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-ack-watch.html</pre></summary>
 	public partial class AcknowledgeWatchRequestParameters : RequestParameters<AcknowledgeWatchRequestParameters> 
 	{

src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs

@@ -3830,6 +3830,52 @@ public TResponse XpackSecurityPutUserPost<TResponse>(string username, PostData b
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
 		public Task<TResponse> XpackSecurityPutUserPostAsync<TResponse>(string username, PostData body, PutUserRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
 			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(POST, Url($"_xpack/security/user/{username.NotNull("username")}"), ctx, body, _params(requestParameters));
+		///<summary>POST on /_xpack/sql/close <para>Clear SQL cursor</para></summary>
+		///<param name="body">Specify the cursor value in the `cursor` element to clean the cursor.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public TResponse XpackSqlClearCursor<TResponse>(PostData body, ClearSqlCursorRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest<TResponse>(POST, Url($"_xpack/sql/close"), body, _params(requestParameters));
+		///<summary>POST on /_xpack/sql/close <para>Clear SQL cursor</para></summary>
+		///<param name="body">Specify the cursor value in the `cursor` element to clean the cursor.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public Task<TResponse> XpackSqlClearCursorAsync<TResponse>(PostData body, ClearSqlCursorRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(POST, Url($"_xpack/sql/close"), ctx, body, _params(requestParameters));
+		///<summary>POST on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="body">Use the `query` element to start a query. Use the `cursor` element to continue a query.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public TResponse XpackSqlQuery<TResponse>(PostData body, QuerySqlRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest<TResponse>(POST, Url($"_xpack/sql"), body, _params(requestParameters));
+		///<summary>POST on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="body">Use the `query` element to start a query. Use the `cursor` element to continue a query.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public Task<TResponse> XpackSqlQueryAsync<TResponse>(PostData body, QuerySqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(POST, Url($"_xpack/sql"), ctx, body, _params(requestParameters));
+		///<summary>GET on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public TResponse XpackSqlQueryGet<TResponse>(QuerySqlRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest<TResponse>(GET, Url($"_xpack/sql"), null, _params(requestParameters));
+		///<summary>GET on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public Task<TResponse> XpackSqlQueryGetAsync<TResponse>(QuerySqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(GET, Url($"_xpack/sql"), ctx, null, _params(requestParameters));
+		///<summary>POST on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="body">Specify the query in the `query` element.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public TResponse XpackSqlTranslate<TResponse>(PostData body, TranslateSqlRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest<TResponse>(POST, Url($"_xpack/sql/translate"), body, _params(requestParameters));
+		///<summary>POST on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="body">Specify the query in the `query` element.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public Task<TResponse> XpackSqlTranslateAsync<TResponse>(PostData body, TranslateSqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(POST, Url($"_xpack/sql/translate"), ctx, body, _params(requestParameters));
+		///<summary>GET on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public TResponse XpackSqlTranslateGet<TResponse>(TranslateSqlRequestParameters requestParameters = null)
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequest<TResponse>(GET, Url($"_xpack/sql/translate"), null, _params(requestParameters));
+		///<summary>GET on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		public Task<TResponse> XpackSqlTranslateGetAsync<TResponse>(TranslateSqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken))
+			where TResponse : class, IElasticsearchResponse, new() => this.DoRequestAsync<TResponse>(GET, Url($"_xpack/sql/translate"), ctx, null, _params(requestParameters));
 		///<summary>PUT on /_xpack/watcher/watch/{watch_id}/_ack <para>http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-ack-watch.html</para></summary>
 		///<param name="watch_id">Watch ID</param>
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>

src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs

@@ -3104,6 +3104,42 @@ public partial interface IElasticLowLevelClient
 		///<param name="body">The user to add</param>
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
 		Task<TResponse> XpackSecurityPutUserPostAsync<TResponse>(string username, PostData body, PutUserRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_xpack/sql/close <para>Clear SQL cursor</para></summary>
+		///<param name="body">Specify the cursor value in the `cursor` element to clean the cursor.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		TResponse XpackSqlClearCursor<TResponse>(PostData body, ClearSqlCursorRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_xpack/sql/close <para>Clear SQL cursor</para></summary>
+		///<param name="body">Specify the cursor value in the `cursor` element to clean the cursor.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		Task<TResponse> XpackSqlClearCursorAsync<TResponse>(PostData body, ClearSqlCursorRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="body">Use the `query` element to start a query. Use the `cursor` element to continue a query.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		TResponse XpackSqlQuery<TResponse>(PostData body, QuerySqlRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="body">Use the `query` element to start a query. Use the `cursor` element to continue a query.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		Task<TResponse> XpackSqlQueryAsync<TResponse>(PostData body, QuerySqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>GET on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		TResponse XpackSqlQueryGet<TResponse>(QuerySqlRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>GET on /_xpack/sql <para>Execute SQL</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		Task<TResponse> XpackSqlQueryGetAsync<TResponse>(QuerySqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="body">Specify the query in the `query` element.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		TResponse XpackSqlTranslate<TResponse>(PostData body, TranslateSqlRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>POST on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="body">Specify the query in the `query` element.</param>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		Task<TResponse> XpackSqlTranslateAsync<TResponse>(PostData body, TranslateSqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>GET on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		TResponse XpackSqlTranslateGet<TResponse>(TranslateSqlRequestParameters requestParameters = null) where TResponse : class, IElasticsearchResponse, new();
+		///<summary>GET on /_xpack/sql/translate <para>Translate SQL into Elasticsearch queries</para></summary>
+		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>
+		Task<TResponse> XpackSqlTranslateGetAsync<TResponse>(TranslateSqlRequestParameters requestParameters = null, CancellationToken ctx = default(CancellationToken)) where TResponse : class, IElasticsearchResponse, new();
 		///<summary>PUT on /_xpack/watcher/watch/{watch_id}/_ack <para>http://www.elastic.co/guide/en/elasticsearch/reference/current/watcher-api-ack-watch.html</para></summary>
 		///<param name="watch_id">Watch ID</param>
 		///<param name="requestParameters">A func that allows you to describe the querystring parameters &amp; request specific connection settings.</param>

src/Nest/CommonAbstractions/Infer/Fields/FieldsJsonConverter.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using Elasticsearch.Net;
 using Newtonsoft.Json;
 
 namespace Nest
@@ -16,22 +17,42 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
 		{
 			var fields = value as Fields;
 			writer.WriteStartArray();
-			var infer = serializer.GetConnectionSettings().Inferrer;
-			foreach (var f in fields?.ListOfFields ?? Enumerable.Empty<Field>())
+			if (fields != null)
 			{
-				writer.WriteValue(infer.Field(f));
+				var infer = serializer.GetConnectionSettings().Inferrer;
+				foreach (var f in fields.ListOfFields)
+				{
+					writer.WriteValue(infer.Field(f));
+				}
 			}
 			writer.WriteEndArray();
 		}
+
 		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
 		{
 			if (reader.TokenType != JsonToken.StartArray) return null;
 			var fields = new Fields();
 			while (reader.TokenType != JsonToken.EndArray)
 			{
-				var field = reader.ReadAsString();
-				if (reader.TokenType == JsonToken.String)
-					fields.And(field);
+				// as per https://github.com/elastic/elasticsearch/pull/29639 this can now be an array of objects
+				reader.Read();
+				switch (reader.TokenType)
+				{
+					case JsonToken.String:
+						fields.And((string)reader.Value);
+						break;
+					case JsonToken.StartObject:
+						/// TODO 6.4 this is temporary until we add proper support for doc_values format
+						reader.Read(); // "field";
+						var field = reader.ReadAsString();
+						fields.And(field);
+						while (reader.TokenType != JsonToken.EndObject)
+						{
+							reader.Read();
+						}
+						reader.Read(); // "}";
+						break;
+				}
 			}
 			return fields;
 		}

src/Nest/CommonAbstractions/LazyDocument/LazyDocumentJsonConverter.cs

@@ -27,4 +27,5 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 
 		public override bool CanConvert(Type objectType) => true;
 	}
+
 }

src/Nest/XPack/Sql/ClearSqlCursor/ClearSqlCursorRequest.cs

@@ -0,0 +1,32 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[MapsApi("xpack.sql.clear_cursor.json")]
+	[ContractJsonConverter(typeof(ReadAsTypeJsonConverter<ClearSqlCursorRequest>))]
+	public partial interface IClearSqlCursorRequest
+	{
+		/// <summary>
+		/// <para>You’ve reached the last page when there is no cursor returned in the results. Like Elasticsearch’s scroll,
+		/// SQL may keep state in Elasticsearch to support the cursor.
+		/// Unlike scroll, receiving the last page is enough to guarantee that the Elasticsearch state is cleared.
+		/// </para>
+		/// </summary>
+		[JsonProperty("cursor")]
+		string Cursor { get; set; }
+	}
+
+	public partial class ClearSqlCursorRequest
+	{
+		/// <inheritdoc cref="IQuerySqlRequest.Cursor"/>>
+		public string Cursor { get; set; }
+	}
+
+	public partial class ClearSqlCursorDescriptor
+	{
+		string IClearSqlCursorRequest.Cursor { get; set; }
+
+		/// <inheritdoc cref="IQuerySqlRequest.Cursor"/>>
+		public ClearSqlCursorDescriptor Cursor(string cursor) => Assign(a => a.Cursor = cursor);
+	}
+}

src/Nest/XPack/Sql/ClearSqlCursor/ClearSqlCursorResponse.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public interface IClearSqlCursorResponse : IResponse
+	{
+		[JsonProperty("succeeded")]
+		bool Succeeded { get; }
+	}
+
+	public class ClearSqlCursorResponse : ResponseBase, IClearSqlCursorResponse
+	{
+		public bool Succeeded { get; internal set; }
+	}
+}