Add prebuilt ICU Analyzer

Relates: #3615, elastic/elasticsearch#34958

Author: russcam

src/Nest/Analysis/Analyzers/AnalyzerJsonConverter.cs

@@ -31,6 +31,7 @@ public override object ReadJson(JsonReader reader, Type objectType, object exist
 				case "fingerprint": return o.ToObject<FingerprintAnalyzer>(ElasticContractResolver.Empty);
 				case "kuromoji": return o.ToObject<KuromojiAnalyzer>(ElasticContractResolver.Empty);
 				case "nori": return o.ToObject<NoriAnalyzer>(ElasticContractResolver.Empty);
+				case "icu_analyzer": return o.ToObject<IcuAnalyzer>(ElasticContractResolver.Empty);
 				default:
 					if (o.Property("tokenizer") != null)
 						return o.ToObject<CustomAnalyzer>(ElasticContractResolver.Empty);

src/Nest/Analysis/Analyzers/Analyzers.cs

@@ -108,5 +108,9 @@ public AnalyzersDescriptor Kuromoji(string name, Func<KuromojiAnalyzerDescriptor
 		/// <inheritdoc cref="INoriAnalyzer" />
 		public AnalyzersDescriptor Nori(string name, Func<NoriAnalyzerDescriptor, INoriAnalyzer> selector) =>
 			Assign(name, selector?.Invoke(new NoriAnalyzerDescriptor()));
+
+		/// <inheritdoc cref="IIcuAnalyzer" />
+		public AnalyzersDescriptor Icu(string name, Func<IcuAnalyzerDescriptor, IIcuAnalyzer> selector) =>
+			Assign(name, selector?.Invoke(new IcuAnalyzerDescriptor()));
 	}
 }

src/Nest/Analysis/Plugins/Icu/Collation/IcuCollationAlternate.cs

@@ -8,6 +8,9 @@ namespace Nest
 	/// Sets the alternate handling for strength quaternary to be either shifted or non-ignorable.
 	/// Which boils down to ignoring punctuation and whitespace.
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuCollationAlternate
 	{

src/Nest/Analysis/Plugins/Icu/Collation/IcuCollationCaseFirst.cs

@@ -5,9 +5,12 @@
 namespace Nest
 {
 	/// <summary>
-	/// Sets the alternate handling for strength quaternary to be either shifted or non-ignorable.
-	/// Which boils down to ignoring punctuation and whitespace.
+	/// Controls which case is sorted first when case is not ignored for
+	/// strength tertiary. The default depends on the collation.
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuCollationCaseFirst
 	{

src/Nest/Analysis/Plugins/Icu/Collation/IcuCollationDecomposition.cs

@@ -13,6 +13,9 @@ namespace Nest
 	/// great many of the world’s languages do not require text normalization, most locales
 	/// set no as the default decomposition mode.
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuCollationDecomposition
 	{

src/Nest/Analysis/Plugins/Icu/Collation/IcuCollationStrength.cs

@@ -9,6 +9,9 @@ namespace Nest
 	/// difference considered significant during comparison.
 	/// See also: http://icu-project.org/apiref/icu4j/com/ibm/icu/text/Collator.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuCollationStrength
 	{

src/Nest/Analysis/Plugins/Icu/IcuAnalyzer.cs

@@ -0,0 +1,54 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	/// <summary>
+	/// An ICU analyzer that performs basic normalization, tokenization and character folding,
+	/// using the <see cref="IIcuNormalizationCharFilter" /> char filter,
+	/// <see cref="IIcuTokenizer" /> and <see cref="IcuNormalizationTokenFilter" /> token filter
+	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed and Elasticsearch 6.6.0+
+	/// </remarks>
+	public interface IIcuAnalyzer : IAnalyzer
+	{
+		/// <summary>
+		/// Normalization method. Default is <see cref="IcuNormalizationType.CompatibilityCaseFold" />
+		/// </summary>
+		[JsonProperty("method")]
+		IcuNormalizationType? Method { get; set; }
+
+		/// <summary>
+		/// Normalization mode. Default is <see cref="IcuNormalizationMode.Compose" />
+		/// </summary>
+		[JsonProperty("mode")]
+		IcuNormalizationMode? Mode { get; set; }
+	}
+
+	/// <inheritdoc cref="IIcuAnalyzer" />
+	public class IcuAnalyzer : AnalyzerBase, IIcuAnalyzer
+	{
+		public IcuAnalyzer() : base("icu_analyzer") { }
+
+		/// <inheritdoc />
+		public IcuNormalizationType? Method { get; set; }
+
+		/// <inheritdoc />
+		public IcuNormalizationMode? Mode { get; set; }
+	}
+
+	/// <inheritdoc cref="IIcuAnalyzer" />
+	public class IcuAnalyzerDescriptor : AnalyzerDescriptorBase<IcuAnalyzerDescriptor, IIcuAnalyzer>, IIcuAnalyzer
+	{
+		protected override string Type => "icu_analyzer";
+
+		IcuNormalizationType? IIcuAnalyzer.Method { get; set; }
+		IcuNormalizationMode? IIcuAnalyzer.Mode { get; set; }
+
+		/// <inheritdoc cref="IIcuAnalyzer.Method"/>
+		public IcuAnalyzerDescriptor Method(IcuNormalizationType? method) => Assign(a => a.Method = method);
+
+		/// <inheritdoc cref="IIcuAnalyzer.Mode"/>
+		public IcuAnalyzerDescriptor Mode(IcuNormalizationMode? mode) => Assign(a => a.Mode = mode);
+	}
+}

src/Nest/Analysis/Plugins/Icu/IcuCollationTokenFilter.cs

@@ -7,6 +7,9 @@ namespace Nest
 	/// defaults to using the DUCET collation, which is a best-effort attempt at language-neutral sorting.
 	/// Part of the `analysis-icu` plugin: https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	public interface IIcuCollationTokenFilter : ITokenFilter
 	{
 		/// <summary>
@@ -75,7 +78,7 @@ public interface IIcuCollationTokenFilter : ITokenFilter
 		string Variant { get; set; }
 	}
 
-	/// <inheritdoc />
+	/// <inheritdoc cref="IIcuCollationTokenFilter" />
 	public class IcuCollationTokenFilter : TokenFilterBase, IIcuCollationTokenFilter
 	{
 		public IcuCollationTokenFilter() : base("icu_collation") { }
@@ -114,7 +117,7 @@ public IcuCollationTokenFilter() : base("icu_collation") { }
 		public string Variant { get; set; }
 	}
 
-	/// <inheritdoc />
+	/// <inheritdoc cref="IIcuCollationTokenFilter" />
 	public class IcuCollationTokenFilterDescriptor
 		: TokenFilterDescriptorBase<IcuCollationTokenFilterDescriptor, IIcuCollationTokenFilter>, IIcuCollationTokenFilter
 	{
@@ -132,38 +135,38 @@ public class IcuCollationTokenFilterDescriptor
 		string IIcuCollationTokenFilter.VariableTop { get; set; }
 		string IIcuCollationTokenFilter.Variant { get; set; }
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Language" />
 		public IcuCollationTokenFilterDescriptor Language(string language) => Assign(a => a.Language = language);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Country" />
 		public IcuCollationTokenFilterDescriptor Country(string country) => Assign(a => a.Country = country);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Variant" />
 		public IcuCollationTokenFilterDescriptor Variant(string variant) => Assign(a => a.Variant = variant);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Strength" />
 		public IcuCollationTokenFilterDescriptor Strength(IcuCollationStrength? strength) => Assign(a => a.Strength = strength);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Decomposition" />
 		public IcuCollationTokenFilterDescriptor Decomposition(IcuCollationDecomposition? decomposition) =>
 			Assign(a => a.Decomposition = decomposition);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Alternate" />
 		public IcuCollationTokenFilterDescriptor Alternate(IcuCollationAlternate? alternate) => Assign(a => a.Alternate = alternate);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.CaseFirst" />
 		public IcuCollationTokenFilterDescriptor CaseFirst(IcuCollationCaseFirst? caseFirst) => Assign(a => a.CaseFirst = caseFirst);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.CaseLevel" />
 		public IcuCollationTokenFilterDescriptor CaseLevel(bool? caseLevel = true) => Assign(a => a.CaseLevel = caseLevel);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.Numeric" />
 		public IcuCollationTokenFilterDescriptor Numeric(bool? numeric = true) => Assign(a => a.Numeric = numeric);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.HiraganaQuaternaryMode" />
 		public IcuCollationTokenFilterDescriptor HiraganaQuaternaryMode(bool? mode = true) => Assign(a => a.HiraganaQuaternaryMode = mode);
 
-		/// <inheritdoc />
+		/// <inheritdoc cref="IIcuCollationTokenFilter.VariableTop" />
 		public IcuCollationTokenFilterDescriptor VariableTop(string variableTop) => Assign(a => a.VariableTop = variableTop);
 	}
 }

src/Nest/Analysis/Plugins/Icu/IcuFoldingTokenFilter.cs

@@ -4,8 +4,10 @@ namespace Nest
 {
 	/// <summary>
 	/// Case folding of Unicode characters based on UTR#30, like the ASCII-folding token filter on steroids.
-	/// Part of the `analysis-icu` plugin: https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	public interface IIcuFoldingTokenFilter : ITokenFilter
 	{
 		/// <summary>

src/Nest/Analysis/Plugins/Icu/IcuNormalizationCharFilter.cs

@@ -4,8 +4,10 @@ namespace Nest
 {
 	/// <summary>
 	/// Normalizes as defined here: http://userguide.icu-project.org/transforms/normalization
-	/// Part of the `analysis-icu` plugin: https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	public interface IIcuNormalizationCharFilter : ICharFilter
 	{
 		/// <summary>

src/Nest/Analysis/Plugins/Icu/IcuNormalizationTokenFilter.cs

@@ -4,8 +4,10 @@ namespace Nest
 {
 	/// <summary>
 	/// Normalizes as defined here: http://userguide.icu-project.org/transforms/normalization
-	/// Part of the `analysis-icu` plugin: https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	public interface IIcuNormalizationTokenFilter : ITokenFilter
 	{
 		/// <summary>

src/Nest/Analysis/Plugins/Icu/IcuTokenizer.cs

@@ -7,8 +7,10 @@ namespace Nest
 	/// like the standard tokenizer, but adds better support for some Asian languages by using a dictionary-based approach
 	/// to identify words in Thai, Lao, Chinese, Japanese, and Korean, and using custom rules to break Myanmar and Khmer
 	/// text into syllables.
-	/// Part of the `analysis-icu` plugin: https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	public interface IIcuTokenizer : ITokenizer
 	{
 		/// <summary>

src/Nest/Analysis/Plugins/Icu/IcuTransformTokenFilter.cs

@@ -5,8 +5,10 @@ namespace Nest
 	/// <summary>
 	/// Transforms are used to process Unicode text in many different ways, such as case mapping,
 	/// normalization, transliteration and bidirectional text handling.
-	/// Part of the `analysis-icu` plugin: https://www.elastic.co/guide/en/elasticsearch/plugins/current/analysis-icu.html
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	public interface IIcuTransformTokenFilter : ITokenFilter
 	{
 		/// <summary>

src/Nest/Analysis/Plugins/Icu/Normalization/IcuNormalizationMode.cs

@@ -7,6 +7,9 @@ namespace Nest
 	/// <summary>
 	/// Normalization mode https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuNormalizationMode
 	{

src/Nest/Analysis/Plugins/Icu/Normalization/IcuNormalizationType.cs

@@ -7,6 +7,9 @@ namespace Nest
 	/// <summary>
 	/// Normalization forms https://en.wikipedia.org/wiki/Unicode_equivalence#Normal_forms
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuNormalizationType
 	{

src/Nest/Analysis/Plugins/Icu/Transform/IcuNormalizationType.cs

@@ -7,6 +7,9 @@ namespace Nest
 	/// <summary>
 	/// Forward (default) for LTR and reverse for RTL
 	/// </summary>
+	/// <remarks>
+	/// Requires analysis-icu plugin to be installed
+	/// </remarks>
 	[JsonConverter(typeof(StringEnumConverter))]
 	public enum IcuTransformDirection
 	{

src/Tests/Tests/Analysis/Analyzers/AnalyzerTests.cs

@@ -69,14 +69,14 @@ public class SimpleTests : AnalyzerAssertionBase<SimpleTests>
 			public override string Name => "mySimple";
 		}
 
-		public class LanguageTests : AnalyzerAssertionBase<SimpleTests>
+		public class LanguageTests : AnalyzerAssertionBase<LanguageTests>
 		{
 			public override FuncTokenizer Fluent => (n, an) => an
 				.Language("myLanguage", a => a.Language(Language.Dutch));
 
 			public override IAnalyzer Initializer => new LanguageAnalyzer { Language = Language.Dutch };
 
-			public override object Json => new { type = "dutch" };
+			public override object Json => new { type = "Dutch" };
 			public override string Name => "myLanguage";
 		}
 
@@ -216,5 +216,29 @@ public class NoriTests : AnalyzerAssertionBase<NoriTests>
 
 			public override string Name => "nori";
 		}
+
+		[SkipVersion("<6.6.0", "introduced in 6.6.0")]
+		public class IcuTests : AnalyzerAssertionBase<IcuTests>
+		{
+			public override FuncTokenizer Fluent => (n, t) => t.Icu(n, e => e
+				.Method(IcuNormalizationType.Canonical)
+				.Mode(IcuNormalizationMode.Decompose)
+			);
+
+			public override IAnalyzer Initializer => new IcuAnalyzer
+			{
+				Method = IcuNormalizationType.Canonical,
+				Mode = IcuNormalizationMode.Decompose
+			};
+
+			public override object Json => new
+			{
+				type = "icu_analyzer",
+				method = "nfc",
+				mode = "decompose"
+			};
+
+			public override string Name => "icu_analyzer";
+		}
 	}
 }