Return distinct items from GetMany and SourceMany (#4353) (#4400)

This commit fixes a bug where a GetMany or SourceMany API call
with a repeated id would return a cartesian product of id and documents.

- enumerate only distinct ids when retrieving hits
or source from GetMany and SourceMany, so that the same id input
to either will return only a single document per target index.

- fix a bug in the MultiGetRequestFormatter whereby
the document index is removed when a request index is specified,
without checking whether the document index matches the request index.

Fixes #4342

(cherry-picked from commit 8cbc1fe)

Author: russcam

docs/aggregations/bucket/terms/terms-aggregation-usage.asciidoc

@@ -206,15 +206,15 @@ new TermsAggregation("states")
 [source,csharp]
 ----
 response.ShouldBeValid();
-var states = response.Aggregations.Terms("states");
+var states = response.Aggregations.Terms<StateOfBeing>("states");
 states.Should().NotBeNull();
 states.DocCountErrorUpperBound.Should().HaveValue();
 states.SumOtherDocCount.Should().HaveValue();
 states.Buckets.Should().NotBeNull();
 states.Buckets.Count.Should().BeGreaterThan(0);
 foreach (var item in states.Buckets)
 {
-    item.Key.Should().NotBeNullOrEmpty();
+    item.Key.Should().BeOfType<StateOfBeing>();
     item.DocCount.Should().BeGreaterOrEqualTo(1);
 }
 states.Meta.Should().NotBeNull().And.HaveCount(1);

docs/search/request/suggest-usage.asciidoc

@@ -26,6 +26,9 @@ See the Elasticsearch documentation on {ref_current}/search-suggesters.html[Sugg
 ----
 s => s
 .Query(q => ProjectFilter)
+.DocValueFields(d => d
+    .Field(f => f.State)
+)
 .Suggest(ss => ss
     .Term("my-term-suggest", t => t
         .MaxEdits(1)
@@ -89,6 +92,7 @@ s => s
 new SearchRequest<Project>
 {
     Query = ProjectFilter,
+    DocValueFields = Fields<Project>(f => f.State),
     Suggest = new SuggestContainer
     {
         {
@@ -185,6 +189,9 @@ new SearchRequest<Project>
       }
     }
   },
+  "docvalue_fields": [
+    "state"
+  ],
   "suggest": {
     "my-completion-suggest": {
       "completion": {
@@ -273,6 +280,8 @@ option.Source.Should().NotBeNull();
 option.Source.Name.Should().NotBeNullOrWhiteSpace();
 option.Source.ShouldAdhereToSourceSerializerWhenSet();
 option.Score.Should().BeGreaterThan(0);
+option.Fields.Should().NotBeNull().And.NotBeEmpty();
+option.Fields.Should().ContainKey("state");
 option.Contexts.Should().NotBeNull().And.NotBeEmpty();
 option.Contexts.Should().ContainKey("color");
 var colorContexts = option.Contexts["color"];

docs/search/writing-queries.asciidoc

@@ -16,7 +16,7 @@ please modify the original csharp file found at the link and submit the PR with
 === Writing queries
 
 Once you have data indexed within Elasticsearch, you're going to want to be able to search it. Elasticsearch
-offers a powerful query DSL to define queries to execute agains Elasticsearch. This DSL is based on JSON
+offers a powerful query DSL to define queries to execute against Elasticsearch. This DSL is based on JSON
 and is exposed in NEST in the form of both a Fluent API and an Object Initializer syntax
 
 ==== Match All query

src/Nest/Document/Multiple/MultiGet/ElasticClient-GetMany.cs

@@ -21,8 +21,8 @@ public static class GetManyExtensions
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">IEnumerable of ids as string for the documents to fetch</param>
-		/// <param name="index">Optionally override the default inferred index name for T</param>
-		/// <param name="type">Optionally overiide the default inferred typename for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client, IEnumerable<string> ids, IndexName index = null,
 			TypeName type = null
 		)
@@ -47,8 +47,8 @@ public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">IEnumerable of ids as ints for the documents to fetch</param>
-		/// <param name="index">Optionally override the default inferred index name for T</param>
-		/// <param name="type">Optionally overiide the default inferred typename for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client, IEnumerable<long> ids, IndexName index = null,
 			TypeName type = null
 		)
@@ -64,8 +64,8 @@ public static IEnumerable<IMultiGetHit<T>> GetMany<T>(this IElasticClient client
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">IEnumerable of ids as string for the documents to fetch</param>
-		/// <param name="index">Optionally override the default inferred index name for T</param>
-		/// <param name="type">Optionally overiide the default inferred typename for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static async Task<IEnumerable<IMultiGetHit<T>>> GetManyAsync<T>(
 			this IElasticClient client, IEnumerable<string> ids, IndexName index = null, TypeName type = null,
 			CancellationToken cancellationToken = default(CancellationToken)
@@ -93,8 +93,8 @@ public static async Task<IEnumerable<IMultiGetHit<T>>> GetManyAsync<T>(
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">IEnumerable of ids as ints for the documents to fetch</param>
-		/// <param name="index">Optionally override the default inferred index name for T</param>
-		/// <param name="type">Optionally overiide the default inferred typename for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static Task<IEnumerable<IMultiGetHit<T>>> GetManyAsync<T>(
 			this IElasticClient client, IEnumerable<long> ids, IndexName index = null, TypeName type = null,
 			CancellationToken cancellationToken = default(CancellationToken)

src/Nest/Document/Multiple/MultiGet/ElasticClient-SourceMany.cs

@@ -24,17 +24,16 @@ public static class SourceManyExtensions
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">A list of ids as string</param>
-		/// <param name="index">Optionally override the default inferred indexname for T</param>
-		/// <param name="type">Optionally override the default inferred indexname for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerable<string> ids, string index = null, string type = null)
 			where T : class
 		{
 			var result = client.MultiGet(s => s
+				.Index(index)
+				.Type(type)
 				.RequestConfiguration(r => r.ThrowExceptions())
-				.GetMany<T>(ids, (gs, i) => gs
-					.Index(index)
-					.Type(type)
-				)
+				.GetMany<T>(ids)
 			);
 			return result.SourceMany<T>(ids);
 		}
@@ -52,8 +51,8 @@ public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerab
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">A list of ids as int</param>
-		/// <param name="index">Optionally override the default inferred indexname for T</param>
-		/// <param name="type">Optionally override the default inferred indexname for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerable<long> ids, string index = null, string type = null)
 			where T : class => client.SourceMany<T>(ids.Select(i => i.ToString(CultureInfo.InvariantCulture)), index, type);
 
@@ -70,20 +69,19 @@ public static IEnumerable<T> SourceMany<T>(this IElasticClient client, IEnumerab
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">A list of ids as string</param>
-		/// <param name="index">Optionally override the default inferred indexname for T</param>
-		/// <param name="type">Optionally override the default inferred indexname for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static async Task<IEnumerable<T>> SourceManyAsync<T>(
 			this IElasticClient client, IEnumerable<string> ids, string index = null, string type = null,
 			CancellationToken cancellationToken = default(CancellationToken)
 		)
 			where T : class
 		{
 			var response = await client.MultiGetAsync(s => s
+					.Index(index)
+					.Type(type)
 					.RequestConfiguration(r => r.ThrowExceptions())
-					.GetMany<T>(ids, (gs, i) => gs
-						.Index(index)
-						.Type(type)
-					), cancellationToken)
+					.GetMany<T>(ids), cancellationToken)
 				.ConfigureAwait(false);
 			return response.SourceMany<T>(ids);
 		}
@@ -101,8 +99,8 @@ public static async Task<IEnumerable<T>> SourceManyAsync<T>(
 		/// <typeparam name="T">The type used to infer the default index and typename</typeparam>
 		/// <param name="client"></param>
 		/// <param name="ids">A list of ids as int</param>
-		/// <param name="index">Optionally override the default inferred indexname for T</param>
-		/// <param name="type">Optionally override the default inferred indexname for T</param>
+		/// <param name="index">Set the request level index name</param>
+		/// <param name="type">Set the request level type name</param>
 		public static Task<IEnumerable<T>> SourceManyAsync<T>(
 			this IElasticClient client, IEnumerable<long> ids, string index = null, string type = null,
 			CancellationToken cancellationToken = default(CancellationToken)

src/Nest/Document/Multiple/MultiGet/Request/MultiGetRequestJsonConverter.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
+using Elasticsearch.Net;
 using Newtonsoft.Json;
 
 namespace Nest
@@ -13,20 +15,50 @@ internal class MultiGetRequestJsonConverter : JsonConverter
 
 		public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
 		{
-			var request = value as IMultiGetRequest;
+			var request = (IMultiGetRequest)value;
 			writer.WriteStartObject();
 			if (!(request?.Documents.HasAny()).GetValueOrDefault(false))
 			{
 				writer.WriteEndObject();
 				return;
 			}
-			var docs = request.Documents.Select(d =>
-				{
-					if (request.Index != null) d.Index = null;
-					if (request.Type != null) d.Type = null;
-					return d;
-				})
-				.ToList();
+
+			List<IMultiGetOperation> docs;
+			var requestHasIndex = request.Index != null;
+			var requestHasType = request.Type != null;
+
+			if (requestHasIndex || requestHasType)
+			{
+				var settings = serializer.GetConnectionSettings();
+				var resolvedIndex = requestHasIndex
+					? request.Index.GetString(settings)
+					: null;
+				var resolvedType = requestHasType
+					? ((IUrlParameter)request.Type).GetString(settings)
+					: null;
+
+				docs = request.Documents.Select(d =>
+					{
+						if (requestHasIndex && d.Index != null)
+						{
+							var docIndex = d.Index.GetString(settings);
+							if (string.Equals(resolvedIndex, docIndex))
+								d.Index = null;
+						}
+
+						if (requestHasType && d.Type != null)
+						{
+							var docType = ((IUrlParameter)d.Type).GetString(settings);
+							if (string.Equals(resolvedType, docType))
+								d.Type = null;
+						}
+
+						return d;
+					})
+					.ToList();
+			}
+			else
+				docs = request.Documents.ToList();
 
 			var flatten = docs.All(p => p.CanBeFlattened);
 

src/Nest/Document/Multiple/MultiGet/Response/MultiGetResponse.cs

@@ -52,12 +52,28 @@ public FieldValues GetFieldValues<T>(string id) where T : class
 			return multiHit?.Fields ?? FieldValues.Empty;
 		}
 
+		/// <summary>
+		/// Retrieves the hits for each distinct id.
+		/// </summary>
+		/// <param name="ids">The ids to retrieve source for</param>
+		/// <typeparam name="T">The document type for the hits to return</typeparam>
+		/// <returns>An IEnumerable{T} of hits</returns>
 		public IEnumerable<IMultiGetHit<T>> GetMany<T>(IEnumerable<string> ids) where T : class
 		{
-			var docs = Hits.OfType<IMultiGetHit<T>>();
-			return from d in docs
-				join id in ids on d.Id equals id
-				select d;
+			HashSet<string> seenIndices = null;
+			foreach (var id in ids.Distinct())
+			{
+				if (seenIndices == null)
+					seenIndices = new HashSet<string>();
+				else
+					seenIndices.Clear();
+
+				foreach (var doc in Hits.OfType<IMultiGetHit<T>>())
+				{
+					if (string.Equals(doc.Id, id) && seenIndices.Add(doc.Index))
+						yield return doc;
+				}
+			}
 		}
 
 		public IEnumerable<IMultiGetHit<T>> GetMany<T>(IEnumerable<long> ids) where T : class =>
@@ -71,14 +87,16 @@ public T Source<T>(string id) where T : class
 
 		public T Source<T>(long id) where T : class => Source<T>(id.ToString(CultureInfo.InvariantCulture));
 
-		public IEnumerable<T> SourceMany<T>(IEnumerable<string> ids) where T : class
-		{
-			var docs = Hits.OfType<IMultiGetHit<T>>();
-			return from d in docs
-				join id in ids on d.Id equals id
-				where d.Found
-				select d.Source;
-		}
+		/// <summary>
+		/// Retrieves the source, if available, for each distinct id.
+		/// </summary>
+		/// <param name="ids">The ids to retrieve source for</param>
+		/// <typeparam name="T">The document type for the hits to return</typeparam>
+		/// <returns>An IEnumerable{T} of sources</returns>
+		public IEnumerable<T> SourceMany<T>(IEnumerable<string> ids) where T : class =>
+			from hit in GetMany<T>(ids)
+			where hit.Found
+			select hit.Source;
 
 		public IEnumerable<T> SourceMany<T>(IEnumerable<long> ids) where T : class =>
 			SourceMany<T>(ids.Select(i => i.ToString(CultureInfo.InvariantCulture)));