Add discard_compound_token to kuromoji tokenizer

Relates: elastic/elasticsearch#57421

Author: russcam

src/Nest/Analysis/Plugins/Kuromoji/KuromojiTokenizer.cs

@@ -21,6 +21,14 @@ public interface IKuromojiTokenizer : ITokenizer
 		[JsonFormatter(typeof(NullableStringBooleanFormatter))]
 		bool? DiscardPunctuation { get; set; }
 
+		/// <summary>
+		/// Whether original compound tokens should be discarded from the output with
+		/// <see cref="KuromojiTokenizationMode.Search"/> <see cref="Mode"/>. Defaults to `false`.
+		/// </summary>
+		[DataMember(Name ="discard_compound_token")]
+		[JsonFormatter(typeof(NullableStringBooleanFormatter))]
+		bool? DiscardCompoundToken { get; set; }
+
 		/// <summary>
 		/// The tokenization mode determines how the tokenizer handles compound and unknown words.
 		/// </summary>
@@ -64,6 +72,9 @@ public class KuromojiTokenizer : TokenizerBase, IKuromojiTokenizer
 		/// <inheritdoc />
 		public bool? DiscardPunctuation { get; set; }
 
+		/// <inheritdoc />
+		public bool? DiscardCompoundToken { get; set; }
+
 		/// <inheritdoc />
 		public KuromojiTokenizationMode? Mode { get; set; }
 
@@ -86,32 +97,35 @@ public class KuromojiTokenizerDescriptor
 	{
 		protected override string Type => "kuromoji_tokenizer";
 		bool? IKuromojiTokenizer.DiscardPunctuation { get; set; }
-
+		bool? IKuromojiTokenizer.DiscardCompoundToken { get; set; }
 		KuromojiTokenizationMode? IKuromojiTokenizer.Mode { get; set; }
 		int? IKuromojiTokenizer.NBestCost { get; set; }
 		string IKuromojiTokenizer.NBestExamples { get; set; }
 		string IKuromojiTokenizer.UserDictionary { get; set; }
 		IEnumerable<string> IKuromojiTokenizer.UserDictionaryRules { get; set; }
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.Mode" />
 		public KuromojiTokenizerDescriptor Mode(KuromojiTokenizationMode? mode) => Assign(mode, (a, v) => a.Mode = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.DiscardPunctuation" />
 		public KuromojiTokenizerDescriptor DiscardPunctuation(bool? discard = true) => Assign(discard, (a, v) => a.DiscardPunctuation = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.DiscardCompoundToken" />
+		public KuromojiTokenizerDescriptor DiscardCompoundToken(bool? discard = true) => Assign(discard, (a, v) => a.DiscardCompoundToken = v);
+
+		/// <inheritdoc cref="IKuromojiTokenizer.UserDictionary" />
 		public KuromojiTokenizerDescriptor UserDictionary(string userDictionary) => Assign(userDictionary, (a, v) => a.UserDictionary = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.NBestExamples" />
 		public KuromojiTokenizerDescriptor NBestExamples(string examples) => Assign(examples, (a, v) => a.NBestExamples = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.NBestCost" />
 		public KuromojiTokenizerDescriptor NBestCost(int? cost) => Assign(cost, (a, v) => a.NBestCost = v);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.UserDictionaryRules" />
 		public KuromojiTokenizerDescriptor UserDictionaryRules(IEnumerable<string> rules) => Assign(rules, (a, v) => a.UserDictionaryRules = rules);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IKuromojiTokenizer.UserDictionaryRules" />
 		public KuromojiTokenizerDescriptor UserDictionaryRules(params string[] rules) => Assign(rules, (a, v) => a.UserDictionaryRules = rules);
 	}
 }

tests/Tests/Analysis/Tokenizers/TokenizerTests.cs

@@ -214,6 +214,34 @@ public class KuromojiTests : TokenizerAssertionBase<KuromojiTests>
 			public override string Name => "kuro";
 		}
 
+		[SkipVersion("<7.9.0", "discard_compound_token introduced in 7.9.0")]
+		public class KuromojiDiscardCompoundTokenTests : TokenizerAssertionBase<KuromojiDiscardCompoundTokenTests>
+		{
+			private const string Example = "/箱根山-箱根/成田空港-成田/";
+			private const string Inline = "東京スカイツリー,東京 スカイツリー,トウキョウ スカイツリー,カスタム名詞";
+
+			public override FuncTokenizer Fluent => (n, t) => t
+				.Kuromoji(n, e => e
+					.Mode(KuromojiTokenizationMode.Search)
+					.DiscardCompoundToken()
+				);
+
+			public override ITokenizer Initializer => new KuromojiTokenizer
+			{
+				Mode = KuromojiTokenizationMode.Search,
+				DiscardCompoundToken = true,
+			};
+
+			public override object Json => new
+			{
+				discard_compound_token = true,
+				mode = "search",
+				type = "kuromoji_tokenizer",
+			};
+
+			public override string Name => "kuro_discard_compound_token";
+		}
+
 		public class UaxTests : TokenizerAssertionBase<UaxTests>
 		{
 			public override FuncTokenizer Fluent => (n, t) => t.UaxEmailUrl(n, e => e