Add Multiplexer Token Filter (#3425)

(cherry picked from commit ef88c30)

Author: Mpdreamz

src/Nest/Analysis/TokenFilters/MultiplexerTokenFilter.cs

@@ -0,0 +1,50 @@
+using System.Collections;
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// A token filter of type multiplexer will emit multiple tokens at the same position, each version of the token having
+	/// been run through a different filter. Identical output tokens at the same position will be removed.
+	/// </summary>
+	public interface IMultiplexerTokenFilter : ITokenFilter
+	{
+		[JsonProperty("filters")]
+		IEnumerable<string> Filters { get; set; }
+
+		[JsonProperty("preserve_original")]
+		bool? PreserveOriginal { get; set; }
+	}
+	public class MultiplexerTokenFilter : TokenFilterBase, IMultiplexerTokenFilter
+	{
+		internal const string TokenType = "multiplexer";
+		public MultiplexerTokenFilter() : base(TokenType) { }
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.Filters"/>
+		public IEnumerable<string> Filters { get; set; }
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.PreserveOriginal"/>
+		public bool? PreserveOriginal { get; set; }
+
+	}
+	///<inheritdoc cref="IMultiplexerTokenFilter"/>
+	public class MultiplexerTokenFilterDescriptor
+		: TokenFilterDescriptorBase<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter>, IMultiplexerTokenFilter
+	{
+		protected override string Type => MultiplexerTokenFilter.TokenType;
+
+		IEnumerable<string> IMultiplexerTokenFilter.Filters { get; set; }
+		bool? IMultiplexerTokenFilter.PreserveOriginal { get; set; }
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.Filters"/>
+		public MultiplexerTokenFilterDescriptor Filters(IEnumerable<string> filters) => Assign(a => a.Filters = filters);
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.Filters"/>
+		public MultiplexerTokenFilterDescriptor Filters(params string[] filters) => Assign(a => a.Filters = filters);
+
+		/// <inheritdoc cref="IMultiplexerTokenFilter.PreserveOriginal"/>
+		public MultiplexerTokenFilterDescriptor PreserveOriginal(bool? preserve = true) => Assign(a => a.PreserveOriginal = preserve);
+	}
+
+}

src/Nest/Analysis/TokenFilters/TokenFilters.cs

@@ -317,5 +317,12 @@ public TokenFiltersDescriptor IcuNormalization(string name, Func<IcuNormalizatio
 		/// </summary>
 		public TokenFiltersDescriptor IcuTransform(string name, Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
 			Assign(name, selector.Invoke(new IcuTransformTokenFilterDescriptor()));
+
+		/// <summary>
+		///  A token filter of type multiplexer will emit multiple tokens at the same position, each version of the token
+		/// having been run through a different filter. Identical output tokens at the same position will be removed.
+		/// </summary>
+		public TokenFiltersDescriptor Multiplexer(string name, Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
+			Assign(name, selector.Invoke(new MultiplexerTokenFilterDescriptor()));
 	}
 }

src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs

@@ -323,6 +323,10 @@ public AnalyzeTokenFiltersDescriptor IcuNormalization(Func<IcuNormalizationToken
 		/// </summary>
 		public AnalyzeTokenFiltersDescriptor IcuTransform(Func<IcuTransformTokenFilterDescriptor, IIcuTransformTokenFilter> selector) =>
 			AssignIfNotNull(selector.Invoke(new IcuTransformTokenFilterDescriptor()));
+
+		///<inheritdoc cref="IMultiplexerTokenFilter"/>
+		public AnalyzeTokenFiltersDescriptor Multiplexer(Func<MultiplexerTokenFilterDescriptor, IMultiplexerTokenFilter> selector) =>
+			AssignIfNotNull(selector.Invoke(new MultiplexerTokenFilterDescriptor()));
 	}
 
 }

src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs

@@ -1,4 +1,5 @@
-using System;
+using System;
+using Elastic.Xunit.XunitPlumbing;
 using Nest;
 using Tests.Framework;
 
@@ -878,5 +879,28 @@ public class PhoneticTests : TokenFilterAssertionBase<PhoneticTests>
 			};
 
 		}
+
+		[SkipVersion("<6.4.0", "Introduced in 6.4.0")]
+		public class MultiplexerTests : TokenFilterAssertionBase<PhoneticTests>
+		{
+			public override string Name => "multiplexer";
+			public override object Json => new
+			{
+				filters = new[]{"lowercase", "lowercase, porter_stem"},
+				preserve_original = true
+			};
+
+			public override ITokenFilter Initializer => new MultiplexerTokenFilter
+			{
+				Filters = new[] {"lowercase", "lowercase, porter_stem"},
+				PreserveOriginal = true
+			};
+
+			public override FuncTokenFilters Fluent => (n, tf) => tf
+				.Multiplexer(n, t => t
+					.Filters("lowercase", "lowercase, porter_stem")
+					.PreserveOriginal()
+				);
+		}
 	}
 }