Add support for the SQL APIs (#3414)

* Add QuerySql API methods and tests
* Add ClearSqlCursor API methods and tests
* Add TranslateSql API methods and tests

(cherry picked from commit a59a00a)

Author: Mpdreamz

src/CodeGeneration/ApiGenerator/ApiGenerator.cs

@@ -79,9 +79,7 @@ public static void Generate(string downloadBranch, params string[] folders)
 			"xpack.rollup.rollup_search.json",
 			"xpack.rollup.start_job.json",
 			"xpack.rollup.stop_job.json",
-			"xpack.sql.clear_cursor.json",
-			"xpack.sql.query.json",
-			"xpack.sql.translate.json",
+
 			"xpack.ssl.certificates.json",
 
 			// 6.4 new  API's

src/CodeGeneration/ApiGenerator/Program.cs

@@ -39,10 +39,8 @@ private static void Main(string[] args)
 			if (redownloadCoreSpecification)
 				RestSpecDownloader.Download(downloadBranch);
 
-			ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info", "MachineLearning", "Migration");
-
-			//ApiGenerator.Generate(); //generates everything under ApiSpecification
+			ApiGenerator.Generate(downloadBranch, "Core", "Graph", "License", "Security", "Watcher", "Info",
+				"MachineLearning", "Migration", "Sql");
 		}
-
 	}
 }

src/Elasticsearch.Net/Domain/RequestParameters/RequestParameters.Generated.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using Elasticsearch.Net;
 using Newtonsoft.Json;
 
 namespace Nest
@@ -16,22 +17,42 @@ public override void WriteJson(JsonWriter writer, object value, JsonSerializer s
 		{
 			var fields = value as Fields;
 			writer.WriteStartArray();
-			var infer = serializer.GetConnectionSettings().Inferrer;
-			foreach (var f in fields?.ListOfFields ?? Enumerable.Empty<Field>())
+			if (fields != null)
 			{
-				writer.WriteValue(infer.Field(f));
+				var infer = serializer.GetConnectionSettings().Inferrer;
+				foreach (var f in fields.ListOfFields)
+				{
+					writer.WriteValue(infer.Field(f));
+				}
 			}
 			writer.WriteEndArray();
 		}
+
 		public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
 		{
 			if (reader.TokenType != JsonToken.StartArray) return null;
 			var fields = new Fields();
 			while (reader.TokenType != JsonToken.EndArray)
 			{
-				var field = reader.ReadAsString();
-				if (reader.TokenType == JsonToken.String)
-					fields.And(field);
+				// as per https://github.com/elastic/elasticsearch/pull/29639 this can now be an array of objects
+				reader.Read();
+				switch (reader.TokenType)
+				{
+					case JsonToken.String:
+						fields.And((string)reader.Value);
+						break;
+					case JsonToken.StartObject:
+						/// TODO 6.4 this is temporary until we add proper support for doc_values format
+						reader.Read(); // "field";
+						var field = reader.ReadAsString();
+						fields.And(field);
+						while (reader.TokenType != JsonToken.EndObject)
+						{
+							reader.Read();
+						}
+						reader.Read(); // "}";
+						break;
+				}
 			}
 			return fields;
 		}

src/Elasticsearch.Net/ElasticLowLevelClient.Generated.cs

@@ -27,4 +27,5 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 
 		public override bool CanConvert(Type objectType) => true;
 	}
+
 }

src/Elasticsearch.Net/IElasticLowLevelClient.Generated.cs

@@ -0,0 +1,32 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[MapsApi("xpack.sql.clear_cursor.json")]
+	[ContractJsonConverter(typeof(ReadAsTypeJsonConverter<ClearSqlCursorRequest>))]
+	public partial interface IClearSqlCursorRequest
+	{
+		/// <summary>
+		/// <para>You’ve reached the last page when there is no cursor returned in the results. Like Elasticsearch’s scroll,
+		/// SQL may keep state in Elasticsearch to support the cursor.
+		/// Unlike scroll, receiving the last page is enough to guarantee that the Elasticsearch state is cleared.
+		/// </para>
+		/// </summary>
+		[JsonProperty("cursor")]
+		string Cursor { get; set; }
+	}
+
+	public partial class ClearSqlCursorRequest
+	{
+		/// <inheritdoc cref="IQuerySqlRequest.Cursor"/>>
+		public string Cursor { get; set; }
+	}
+
+	public partial class ClearSqlCursorDescriptor
+	{
+		string IClearSqlCursorRequest.Cursor { get; set; }
+
+		/// <inheritdoc cref="IQuerySqlRequest.Cursor"/>>
+		public ClearSqlCursorDescriptor Cursor(string cursor) => Assign(a => a.Cursor = cursor);
+	}
+}

src/Nest/CommonAbstractions/Infer/Fields/FieldsJsonConverter.cs

@@ -0,0 +1,17 @@
+using System;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public interface IClearSqlCursorResponse : IResponse
+	{
+		[JsonProperty("succeeded")]
+		bool Succeeded { get; }
+	}
+
+	public class ClearSqlCursorResponse : ResponseBase, IClearSqlCursorResponse
+	{
+		public bool Succeeded { get; internal set; }
+	}
+}

src/Nest/CommonAbstractions/LazyDocument/LazyDocumentJsonConverter.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Threading.Tasks;
+using Elasticsearch.Net;
+using System.Threading;
+
+namespace Nest
+{
+	public partial interface IElasticClient
+	{
+		/// <summary>
+		/// Clear a cursor returned by <see cref="IQuerySqlResponse.Cursor"/>. Not that this is only necessary if you wish to bail out early
+		/// </summary>
+		IClearSqlCursorResponse ClearSqlCursor(Func<ClearSqlCursorDescriptor, IClearSqlCursorRequest> selector = null);
+
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		IClearSqlCursorResponse ClearSqlCursor(IClearSqlCursorRequest request);
+
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		Task<IClearSqlCursorResponse> ClearSqlCursorAsync(Func<ClearSqlCursorDescriptor, IClearSqlCursorRequest> selector = null, CancellationToken cancellationToken = default(CancellationToken));
+
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		Task<IClearSqlCursorResponse> ClearSqlCursorAsync(IClearSqlCursorRequest request, CancellationToken cancellationToken = default(CancellationToken));
+	}
+
+	public partial class ElasticClient
+	{
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		public IClearSqlCursorResponse ClearSqlCursor(Func<ClearSqlCursorDescriptor, IClearSqlCursorRequest> selector = null) =>
+			this.ClearSqlCursor(selector.InvokeOrDefault(new ClearSqlCursorDescriptor()));
+
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		public IClearSqlCursorResponse ClearSqlCursor(IClearSqlCursorRequest request) =>
+			this.Dispatcher.Dispatch<IClearSqlCursorRequest, ClearSqlCursorRequestParameters, ClearSqlCursorResponse>(
+				request,
+				(p, d) =>this.LowLevelDispatch.XpackSqlClearCursorDispatch<ClearSqlCursorResponse>(p, d)
+			);
+
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		public Task<IClearSqlCursorResponse> ClearSqlCursorAsync(Func<ClearSqlCursorDescriptor, IClearSqlCursorRequest> selector = null, CancellationToken cancellationToken = default(CancellationToken)) =>
+			this.ClearSqlCursorAsync(selector.InvokeOrDefault(new ClearSqlCursorDescriptor()), cancellationToken);
+
+		/// <inheritdoc cref="ClearSqlCursor(System.Func{Nest.ClearSqlCursorDescriptor,Nest.IClearSqlCursorRequest})"/>
+		public Task<IClearSqlCursorResponse> ClearSqlCursorAsync(IClearSqlCursorRequest request, CancellationToken cancellationToken = default(CancellationToken)) =>
+			this.Dispatcher.DispatchAsync<IClearSqlCursorRequest, ClearSqlCursorRequestParameters, ClearSqlCursorResponse, IClearSqlCursorResponse>(
+				request,
+				cancellationToken,
+				(p, d, c) => this.LowLevelDispatch.XpackSqlClearCursorDispatchAsync<ClearSqlCursorResponse>(p, d, c)
+			);
+	}
+}