Collection deserialisation needs to check for null tokens

src/Nest/Aggregations/Bucket/SignificantTerms/IncludeExclude.cs

@@ -39,7 +39,10 @@ public IncludeExclude Deserialize(ref JsonReader reader, IJsonFormatterResolver
 			var token = reader.GetCurrentJsonToken();
 
 			if (token == JsonToken.Null)
+			{
+				reader.ReadNext();
 				return null;
+			}
 
 			IncludeExclude termsInclude;
 

src/Nest/Cat/CatHelpResponseBuilder.cs

@@ -14,7 +14,8 @@ public override object DeserializeResponse(IElasticsearchSerializer builtInSeria
 		{
 			var catResponse = new CatResponse<CatHelpRecord>();
 
-			if (!response.Success) return catResponse;
+			if (!response.Success)
+				return catResponse;
 
 			using (stream)
 			using (var ms = response.ConnectionConfiguration.MemoryStreamFactory.Create())
@@ -27,20 +28,13 @@ public override object DeserializeResponse(IElasticsearchSerializer builtInSeria
 			return catResponse;
 		}
 
-		private static void Parse(CatResponse<CatHelpRecord> catResponse, string body) =>
-			catResponse.Records = body.Split('\n')
-				.Skip(1)
-				.Select(f => new CatHelpRecord { Endpoint = f.Trim() })
-				.ToList();
-
 		public override async Task<object> DeserializeResponseAsync(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream,
-			CancellationToken ctx = default
-		)
+			CancellationToken ctx = default)
 		{
-
 			var catResponse = new CatResponse<CatHelpRecord>();
 
-			if (!response.Success) return catResponse;
+			if (!response.Success)
+				return catResponse;
 
 			using (stream)
 			using (var ms = response.ConnectionConfiguration.MemoryStreamFactory.Create())
@@ -52,5 +46,11 @@ public override async Task<object> DeserializeResponseAsync(IElasticsearchSerial
 
 			return catResponse;
 		}
+
+		private static void Parse(CatResponse<CatHelpRecord> catResponse, string body) =>
+			catResponse.Records = body.Split('\n')
+				.Skip(1)
+				.Select(f => new CatHelpRecord { Endpoint = f.Trim() })
+				.ToList();
 	}
 }

src/Nest/Cat/CatResponseBuilder.cs

@@ -4,38 +4,34 @@
 using System.Threading.Tasks;
 using Elasticsearch.Net;
 
-namespace Nest 
+namespace Nest
 {
 	internal class CatResponseBuilder<TCatRecord> : CustomResponseBuilderBase where TCatRecord : ICatRecord
 	{
 		public static CatResponseBuilder<TCatRecord> Instance { get; } = new CatResponseBuilder<TCatRecord>();
-		
+
 		public override object DeserializeResponse(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream)
 		{
 			var catResponse = new CatResponse<TCatRecord>();
 
-			if (!response.Success) return catResponse;
+			if (!response.Success)
+				return catResponse;
 
 			var records = builtInSerializer.Deserialize<IReadOnlyCollection<TCatRecord>>(stream);
 			catResponse.Records = records;
-
 			return catResponse;
-
 		}
 
-
 		public override async Task<object> DeserializeResponseAsync(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream, CancellationToken ctx = default)
 		{
 			var catResponse = new CatResponse<TCatRecord>();
 
-			if (!response.Success) return catResponse;
+			if (!response.Success)
+				return catResponse;
 
 			var records = await builtInSerializer.DeserializeAsync<IReadOnlyCollection<TCatRecord>>(stream, ctx);
 			catResponse.Records = records;
-
 			return catResponse;
-
 		}
-
 	}
-}
+}

src/Nest/Cluster/NodesHotThreads/NodeHotThreadsResponseBuilder.cs

@@ -7,12 +7,12 @@
 using System.Threading.Tasks;
 using Elasticsearch.Net;
 
-namespace Nest 
+namespace Nest
 {
 	internal class NodeHotThreadsResponseBuilder : CustomResponseBuilderBase
 	{
 		public static NodeHotThreadsResponseBuilder Instance { get; } = new NodeHotThreadsResponseBuilder();
-		
+
 		//::: {Dragonfly}{lvtIV72sRIWBGik7ulbuaw}{127.0.0.1}{127.0.0.1:9300}
 		private static readonly Regex NodeRegex = new Regex(@"^\s\{(?<name>.+?)\}\{(?<id>.+?)\}(?<hosts>.+)\n");
 
@@ -49,8 +49,11 @@ where matches.Success
 			return new NodesHotThreadsResponse(info.ToList());
 		}
 
-		public override object DeserializeResponse(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream) 
+		public override object DeserializeResponse(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream)
 		{
+			if (!response.Success)
+				return new NodesHotThreadsResponse();
+
 			using (stream)
 			using (var sr = new StreamReader(stream, Encoding.UTF8))
 			{
@@ -60,19 +63,21 @@ public override object DeserializeResponse(IElasticsearchSerializer builtInSeria
 		}
 
 		public override async Task<object> DeserializeResponseAsync(
-			IElasticsearchSerializer builtInSerializer, 
-			IApiCallDetails apiCallDetails, 
+			IElasticsearchSerializer builtInSerializer,
+			IApiCallDetails response,
 			Stream stream,
 			CancellationToken ctx = default
 		)
 		{
+			if (!response.Success)
+				return new NodesHotThreadsResponse();
+
 			using (stream)
 			using (var sr = new StreamReader(stream, Encoding.UTF8))
 			{
 				var plainTextResponse = await sr.ReadToEndAsync();
 				return Parse(plainTextResponse);
 			}
-
 		}
 	}
-}
+}

src/Nest/CommonAbstractions/LazyDocument/LazyDocumentFormatter.cs

@@ -111,8 +111,10 @@ public void Serialize(ref JsonWriter writer, LazyDocument value, IJsonFormatterR
 
 		public LazyDocument Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver)
 		{
-			if (reader.ReadIsNull())
+			if (reader.GetCurrentJsonToken() == JsonToken.Null)
+			{
 				return null;
+			}
 
 			var arraySegment = reader.ReadNextBlockSegment();
 

src/Nest/CommonAbstractions/SerializationBehavior/Attributes/MachineLearningDateTimeAttribute.cs

@@ -28,7 +28,11 @@ public DateTimeOffset Deserialize(ref JsonReader reader, IJsonFormatterResolver
 				var formatter = formatterResolver.GetFormatter<DateTimeOffset>();
 				return formatter.Deserialize(ref reader, formatterResolver);
 			}
-			if (token == JsonToken.Null) return default;
+			if (token == JsonToken.Null)
+			{
+				reader.ReadNext();
+				return default;
+			}
 
 			if (token == JsonToken.Number)
 			{
@@ -55,7 +59,11 @@ public DateTime Deserialize(ref JsonReader reader, IJsonFormatterResolver format
 				var formatter = formatterResolver.GetFormatter<DateTime>();
 				return formatter.Deserialize(ref reader, formatterResolver);
 			}
-			if (token == JsonToken.Null) return default;
+			if (token == JsonToken.Null)
+			{
+				reader.ReadNext();
+				return default;
+			}
 
 			if (token == JsonToken.Number)
 			{

src/Nest/CommonAbstractions/SerializationBehavior/JsonFormatters/DateTimeOffsetEpochMillisecondsFormatter.cs

@@ -26,8 +26,10 @@ internal class NullableDateTimeOffsetEpochMillisecondsFormatter : IJsonFormatter
 					return formatter.Deserialize(ref reader, formatterResolver);
 				}
 				case JsonToken.Null:
+				{
 					reader.ReadNext();
 					return null;
+				}
 				case JsonToken.Number:
 				{
 					var millisecondsSinceEpoch = reader.ReadDouble();

src/Nest/Document/Multiple/MultiGet/Request/MultiGetResponseBuilder.cs

@@ -13,8 +13,9 @@ internal class MultiGetResponseBuilder : CustomResponseBuilderBase
 
 		public override object DeserializeResponse(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream)
 		{
-			var statefulSerializer = builtInSerializer.CreateStateful(Formatter);
-			return statefulSerializer.Deserialize<MultiGetResponse>(stream);
+			return response.Success
+				? builtInSerializer.CreateStateful(Formatter).Deserialize<MultiGetResponse>(stream)
+				: new MultiGetResponse();
 		}
 
 		public override async Task<object> DeserializeResponseAsync(
@@ -24,8 +25,9 @@ public override async Task<object> DeserializeResponseAsync(
 			CancellationToken ctx = default
 		)
 		{
-			var statefulSerializer = builtInSerializer.CreateStateful(Formatter);
-			return await statefulSerializer.DeserializeAsync<MultiGetResponse>(stream, ctx);
+			return response.Success
+				? await builtInSerializer.CreateStateful(Formatter).DeserializeAsync<MultiGetResponse>(stream, ctx)
+				: new MultiGetResponse();
 		}
 	}
 }

src/Nest/Document/Single/Source/SourceRequestResponseBuilder.cs

@@ -11,10 +11,16 @@ public class SourceRequestResponseBuilder<TDocument> : CustomResponseBuilderBase
 
 		public override object DeserializeResponse(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream)
 		{
-			var source = builtInSerializer.Deserialize<TDocument>(stream);
-			return new SourceResponse<TDocument> { Body = source, };
+			return response.Success
+				? new SourceResponse<TDocument> { Body = builtInSerializer.Deserialize<TDocument>(stream) }
+				: new SourceResponse<TDocument>();
 		}
 
-		public override Task<object> DeserializeResponseAsync(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream, CancellationToken ctx = default) => throw new System.NotImplementedException();
+		public override async Task<object> DeserializeResponseAsync(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream, CancellationToken ctx = default)
+		{
+			return response.Success
+				? new SourceResponse<TDocument> { Body = await builtInSerializer.DeserializeAsync<TDocument>(stream) }
+				: new SourceResponse<TDocument>();
+		}
 	}
 }

src/Nest/ElasticClient.MachineLearning.cs

@@ -737,25 +737,29 @@ internal MachineLearningNamespace(ElasticClient client): base(client)
 		/// <para></para>
 		/// <a href = "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html">http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html</a>
 		/// </summary>
-		public PreviewDatafeedResponse<TResult> PreviewDatafeed<TResult>(Id datafeedId, Func<PreviewDatafeedDescriptor, IPreviewDatafeedRequest> selector = null) => PreviewDatafeed<TResult>(selector.InvokeOrDefault(new PreviewDatafeedDescriptor(datafeedId: datafeedId)));
+		public PreviewDatafeedResponse<TDocument> PreviewDatafeed<TDocument>(Id datafeedId, Func<PreviewDatafeedDescriptor, IPreviewDatafeedRequest> selector = null)
+			where TDocument : class => PreviewDatafeed<TDocument>(selector.InvokeOrDefault(new PreviewDatafeedDescriptor(datafeedId: datafeedId)));
 		/// <summary>
 		/// <c>GET</c> request to the <c>ml.preview_datafeed</c> API, read more about this API online:
 		/// <para></para>
 		/// <a href = "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html">http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html</a>
 		/// </summary>
-		public Task<PreviewDatafeedResponse<TResult>> PreviewDatafeedAsync<TResult>(Id datafeedId, Func<PreviewDatafeedDescriptor, IPreviewDatafeedRequest> selector = null, CancellationToken ct = default) => PreviewDatafeedAsync<TResult>(selector.InvokeOrDefault(new PreviewDatafeedDescriptor(datafeedId: datafeedId)), ct);
+		public Task<PreviewDatafeedResponse<TDocument>> PreviewDatafeedAsync<TDocument>(Id datafeedId, Func<PreviewDatafeedDescriptor, IPreviewDatafeedRequest> selector = null, CancellationToken ct = default)
+			where TDocument : class => PreviewDatafeedAsync<TDocument>(selector.InvokeOrDefault(new PreviewDatafeedDescriptor(datafeedId: datafeedId)), ct);
 		/// <summary>
 		/// <c>GET</c> request to the <c>ml.preview_datafeed</c> API, read more about this API online:
 		/// <para></para>
 		/// <a href = "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html">http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html</a>
 		/// </summary>
-		public PreviewDatafeedResponse<TResult> PreviewDatafeed<TResult>(IPreviewDatafeedRequest request) => DoRequest<IPreviewDatafeedRequest, PreviewDatafeedResponse<TResult>>(request, request.RequestParameters);
+		public PreviewDatafeedResponse<TDocument> PreviewDatafeed<TDocument>(IPreviewDatafeedRequest request)
+			where TDocument : class => DoRequest<IPreviewDatafeedRequest, PreviewDatafeedResponse<TDocument>>(request, ResponseBuilder(request.RequestParameters, PreviewDatafeedResponseBuilder<TDocument>.Instance));
 		/// <summary>
 		/// <c>GET</c> request to the <c>ml.preview_datafeed</c> API, read more about this API online:
 		/// <para></para>
 		/// <a href = "http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html">http://www.elastic.co/guide/en/elasticsearch/reference/current/ml-preview-datafeed.html</a>
 		/// </summary>
-		public Task<PreviewDatafeedResponse<TResult>> PreviewDatafeedAsync<TResult>(IPreviewDatafeedRequest request, CancellationToken ct = default) => DoRequestAsync<IPreviewDatafeedRequest, PreviewDatafeedResponse<TResult>>(request, request.RequestParameters, ct);
+		public Task<PreviewDatafeedResponse<TDocument>> PreviewDatafeedAsync<TDocument>(IPreviewDatafeedRequest request, CancellationToken ct = default)
+			where TDocument : class => DoRequestAsync<IPreviewDatafeedRequest, PreviewDatafeedResponse<TDocument>>(request, ResponseBuilder(request.RequestParameters, PreviewDatafeedResponseBuilder<TDocument>.Instance), ct);
 		/// <summary>
 		/// <c>PUT</c> request to the <c>ml.put_calendar</c> API, read more about this API online:
 		/// <para></para>

src/Nest/ElasticClient.cs

@@ -2,6 +2,7 @@
 using System.Threading;
 using System.Threading.Tasks;
 using Elasticsearch.Net;
+using Elasticsearch.Net.Specification.MachineLearningApi;
 
 namespace Nest
 {
@@ -54,6 +55,12 @@ protected Task<CatResponse<TCatRecord>> DoCatAsync<TRequest, TParams, TCatRecord
 				request.RequestParameters.CustomResponseBuilder = CatResponseBuilder<TCatRecord>.Instance;
 			return DoRequestAsync<TRequest, CatResponse<TCatRecord>>(request, request.RequestParameters, ct, r => ElasticClient.ForceJson(r));
 		}
+
+		internal IRequestParameters ResponseBuilder(PreviewDatafeedRequestParameters parameters, CustomResponseBuilderBase builder)
+		{
+			parameters.CustomResponseBuilder = builder;
+			return parameters;
+		}
 	}
 	/// <summary>
 	/// ElasticClient is NEST's strongly typed client which exposes fully mapped Elasticsearch endpoints

src/Nest/Indices/AliasManagement/Alias/Actions/IAliasAction.cs

@@ -23,7 +23,10 @@ public IAliasAction Deserialize(ref JsonReader reader, IJsonFormatterResolver fo
 		{
 			var token = reader.GetCurrentJsonToken();
 			if (token == JsonToken.Null)
+			{
+				reader.ReadNext();
 				return null;
+			}
 
 			var segment = reader.ReadNextBlockSegment();
 			var segmentReader = new JsonReader(segment.Array, segment.Offset);

src/Nest/QueryDsl/Span/Gap/SpanGapQuery.cs

@@ -63,7 +63,10 @@ public void Serialize(ref JsonWriter writer, ISpanGapQuery value, IJsonFormatter
 		public ISpanGapQuery Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver)
 		{
 			if (reader.GetCurrentJsonToken() == JsonToken.Null)
+			{
+				reader.ReadNext();
 				return null;
+			}
 
 			var count = 0;
 			var query = new SpanGapQuery();

src/Nest/QueryDsl/TermLevel/Fuzzy/FuzzyQueryFormatter.cs

@@ -50,8 +50,11 @@ public override void SerializeInternal(ref JsonWriter writer, IFuzzyQuery value,
 
 		public override IFuzzyQuery Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver)
 		{
-			if (reader.ReadIsNull())
+			if (reader.GetCurrentJsonToken() == JsonToken.Null)
+			{
+				reader.ReadNext();
 				return null;
+			}
 
 			var count = 0;
 			IFuzzyQuery query = null;

src/Nest/Search/MultiSearch/MultiSearchResponseBuilder.cs

@@ -13,8 +13,9 @@ internal class MultiSearchResponseBuilder : CustomResponseBuilderBase
 
 		public override object DeserializeResponse(IElasticsearchSerializer builtInSerializer, IApiCallDetails response, Stream stream)
 		{
-			var statefulSerializer = builtInSerializer.CreateStateful(Formatter);
-			return statefulSerializer.Deserialize<MultiSearchResponse>(stream);
+			return response.Success
+				? builtInSerializer.CreateStateful(Formatter).Deserialize<MultiSearchResponse>(stream)
+				: new MultiSearchResponse();
 		}
 
 		public override async Task<object> DeserializeResponseAsync(
@@ -24,8 +25,9 @@ public override async Task<object> DeserializeResponseAsync(
 			CancellationToken ctx = default
 		)
 		{
-			var statefulSerializer = builtInSerializer.CreateStateful(Formatter);
-			return await statefulSerializer.DeserializeAsync<MultiSearchResponse>(stream, ctx);
+			return response.Success
+				? await builtInSerializer.CreateStateful(Formatter).DeserializeAsync<MultiSearchResponse>(stream, ctx)
+				: new MultiSearchResponse();
 		}
 	}
 }

src/Nest/Search/Search/SourceFiltering/SourceFilterJsonConverter.cs

@@ -17,7 +17,10 @@ public ISourceFilter Deserialize(ref JsonReader reader, IJsonFormatterResolver f
 		{
 			var token = reader.GetCurrentJsonToken();
 			if (token == JsonToken.Null)
+			{
+				reader.ReadNext();
 				return null;
+			}
 
 			var filter = new SourceFilter();
 			switch (token)

src/Nest/Search/Suggesters/ContextSuggester/SuggestContextFormatter.cs

@@ -15,8 +15,11 @@ internal class SuggestContextFormatter : IJsonFormatter<ISuggestContext>
 
 		public ISuggestContext Deserialize(ref JsonReader reader, IJsonFormatterResolver formatterResolver)
 		{
-			if (reader.ReadIsNull())
+			if (reader.GetCurrentJsonToken() == JsonToken.Null)
+			{
+				reader.ReadNext();
 				return null;
+			}
 
 			var segment = reader.ReadNextBlockSegment();
 			var segmentReader = new JsonReader(segment.Array, segment.Offset);