Skip to content

Add support for condition token filter #3523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Jan 25, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 66 additions & 0 deletions src/Nest/Analysis/TokenFilters/ConditionTokenFilter.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using Newtonsoft.Json;

namespace Nest
{
/// <summary>
/// The conditional token filter takes a predicate script and a list of subfilters, and
/// only applies the subfilters to the current token if it matches the predicate.
/// </summary>
public interface IConditionTokenFilter : ITokenFilter
{
/// <summary>
/// a predicate script that determines whether or not the filters will be applied
/// to the current token. Note that only inline scripts are supported
/// </summary>
[JsonProperty("script")]
IScript Script { get; set; }

/// <summary>
///a chain of token filters to apply to the current token if the predicate
/// matches. These can be any token filters defined elsewhere in the index mappings.
/// </summary>
[JsonProperty("filter")]
IEnumerable<string> Filters { get; set; }
}

/// <inheritdoc cref="IConditionTokenFilter" />
public class ConditionTokenFilter : TokenFilterBase, IConditionTokenFilter
{
public ConditionTokenFilter() : base("condition") { }

/// <inheritdoc cref="IConditionTokenFilter.Script" />
public IScript Script { get; set; }

/// <inheritdoc cref="IConditionTokenFilter.Filters" />
public IEnumerable<string> Filters { get; set; }
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can this be a single filter or enumerable of filters?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, this does not allow inline token filter declarations only references.

I've only seen it being documented as an array of string.

}

/// <inheritdoc cref="IConditionTokenFilter" />
public class ConditionTokenFilterDescriptor
: TokenFilterDescriptorBase<ConditionTokenFilterDescriptor, IConditionTokenFilter>, IConditionTokenFilter
{
protected override string Type => "condition";

IScript IConditionTokenFilter.Script { get; set; }
IEnumerable<string> IConditionTokenFilter.Filters { get; set; }

/// <inheritdoc cref="IConditionTokenFilter.Script" />
public ConditionTokenFilterDescriptor Script(Func<ScriptDescriptor, IScript> scriptSelector) =>
Assign(a => a.Script = scriptSelector?.Invoke(new ScriptDescriptor()));

/// <inheritdoc cref="IConditionTokenFilter.Script" />
public ConditionTokenFilterDescriptor Script(string predicate) =>
Assign(a => a.Script = new InlineScript(predicate));

/// <inheritdoc cref="IConditionTokenFilter.Filters" />
public ConditionTokenFilterDescriptor Filters(params string[] filters) =>
Assign(a => a.Filters = filters);

/// <inheritdoc cref="IConditionTokenFilter.Filters" />
public ConditionTokenFilterDescriptor Filters(IEnumerable<string> filters) =>
Assign(a => a.Filters = filters);

}
}
4 changes: 4 additions & 0 deletions src/Nest/Analysis/TokenFilters/TokenFilters.cs
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,10 @@ public TokenFiltersDescriptor Standard(string name, Func<StandardTokenFilterDesc
public TokenFiltersDescriptor Stemmer(string name, Func<StemmerTokenFilterDescriptor, IStemmerTokenFilter> selector) =>
Assign(name, selector?.Invoke(new StemmerTokenFilterDescriptor()));

/// <inheritdoc cref="IConditionTokenFilter" />
public TokenFiltersDescriptor Condition(string name, Func<ConditionTokenFilterDescriptor, IConditionTokenFilter> selector) =>
Assign(name, selector?.Invoke(new ConditionTokenFilterDescriptor()));

/// <summary>
/// Overrides stemming algorithms, by applying a custom mapping, then protecting these terms from being modified by stemmers. Must be placed
/// before any stemming filters.
Expand Down
4 changes: 4 additions & 0 deletions src/Nest/Indices/Analyze/AnalyzeTokenFilters.cs
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,10 @@ public AnalyzeTokenFiltersDescriptor Standard(Func<StandardTokenFilterDescriptor
AssignIfNotNull(selector.InvokeOrDefault(new StandardTokenFilterDescriptor()));
#pragma warning restore 618

/// <inheritdoc cref="IConditionTokenFilter" />
public AnalyzeTokenFiltersDescriptor Condition(Func<ConditionTokenFilterDescriptor, IConditionTokenFilter> selector) =>
AssignIfNotNull(selector?.Invoke(new ConditionTokenFilterDescriptor()));

/// <summary>
/// A filter that stems words (similar to snowball, but with more options).
/// </summary>
Expand Down
34 changes: 31 additions & 3 deletions src/Tests/Tests/Analysis/TokenFilters/TokenFilterTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -896,8 +896,35 @@ public class NoriPartOfSpeechTests : TokenFilterAssertionBase<NoriPartOfSpeechTe
public override string Name => "nori_pos";
}

[SkipVersion("<6.5.0", "Introduced in 6.5.0")]
public class ConditionTests : TokenFilterAssertionBase<ConditionTests>
{
private readonly string _predicate = "token.getTerm().length() < 5";

public override FuncTokenFilters Fluent => (n, tf) => tf
.Condition(n, t => t
.Filters("lowercase", "lowercase, porter_stem")
.Script(_predicate)
);

public override ITokenFilter Initializer => new ConditionTokenFilter
{
Filters = new[] { "lowercase", "lowercase, porter_stem" },
Script = new InlineScript(_predicate)
};

public override object Json => new
{
type = "condition",
filter = new[] { "lowercase", "lowercase, porter_stem" },
script = new { source = _predicate }
};

public override string Name => "condition";
}

[SkipVersion("<6.4.0", "Introduced in 6.4.0")]
public class MultiplexerTests : TokenFilterAssertionBase<PhoneticTests>
public class MultiplexerTests : TokenFilterAssertionBase<MultiplexerTests>
{
public override FuncTokenFilters Fluent => (n, tf) => tf
.Multiplexer(n, t => t
Expand All @@ -913,6 +940,7 @@ public class MultiplexerTests : TokenFilterAssertionBase<PhoneticTests>

public override object Json => new
{
type = "multiplexer",
filters = new[] { "lowercase", "lowercase, porter_stem" },
preserve_original = true
};
Expand All @@ -921,11 +949,11 @@ public class MultiplexerTests : TokenFilterAssertionBase<PhoneticTests>
}

[SkipVersion("<6.4.0", "Introduced in 6.4.0")]
public class RemoveDuplicatesTests : TokenFilterAssertionBase<PhoneticTests>
public class RemoveDuplicatesTests : TokenFilterAssertionBase<RemoveDuplicatesTests>
{
public override FuncTokenFilters Fluent => (n, tf) => tf.RemoveDuplicates(n);
public override ITokenFilter Initializer => new RemoveDuplicatesTokenFilter { };
public override object Json => new { };
public override object Json => new { type = "remove_duplicates" };
public override string Name => "dupes";
}
}
Expand Down