-
Notifications
You must be signed in to change notification settings - Fork 1.2k
Implement ml.estimate_model_memory.json #4530
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
Show all changes
4 commits
Select commit
Hold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
125 changes: 125 additions & 0 deletions
125
src/Nest/XPack/MachineLearning/EstimateModelMemory/EstimateModelMemoryRequest.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,125 @@ | ||
using System; | ||
using System.Collections.Generic; | ||
using System.Linq.Expressions; | ||
using System.Runtime.Serialization; | ||
using Elasticsearch.Net.Utf8Json; | ||
|
||
namespace Nest | ||
{ | ||
[MapsApi("ml.estimate_model_memory.json")] | ||
[ReadAs(typeof(EstimateModelMemoryRequest))] | ||
public partial interface IEstimateModelMemoryRequest | ||
{ | ||
/// <summary> | ||
/// For a list of the properties that you can specify in the analysis_config component of the body | ||
/// of this API, see analysis_config. | ||
/// </summary> | ||
[DataMember(Name ="analysis_config")] | ||
IAnalysisConfig AnalysisConfig { get; set; } | ||
|
||
/// <summary> | ||
/// Estimates of the cardinality that will be observed for fields over the whole time period that | ||
/// the job analyzes data. To produce a good answer, values must be provided for fields referenced | ||
/// in the by_field_name, over_field_name and partition_field_name of any detectors. It does not matter | ||
/// if values are provided for other fields. If no detectors have a by_field_name, over_field_name or | ||
/// partition_field_name then overall_cardinality can be omitted from the request. | ||
/// </summary> | ||
[DataMember(Name = "overall_cardinality")] | ||
IOverallCardinality OverallCardinality { get; set; } | ||
|
||
/// <summary> | ||
/// Estimates of the highest cardinality in a single bucket that will be observed for influencer | ||
/// fields over the time period that the job analyzes data. To produce a good answer, values must | ||
/// be provided for all influencer fields. It does not matter if values are provided for fields | ||
/// that are not listed as influencers. If there are no influencers then max_bucket_cardinality | ||
/// can be omitted from the request. | ||
/// </summary> | ||
[DataMember(Name = "max_bucket_cardinality")] | ||
IMaxBucketCardinality MaxBucketCardinality { get; set; } | ||
} | ||
|
||
public partial class EstimateModelMemoryRequest | ||
{ | ||
/// <inheritdoc /> | ||
public IAnalysisConfig AnalysisConfig { get; set; } | ||
|
||
/// <inheritdoc /> | ||
public IOverallCardinality OverallCardinality { get; set; } | ||
|
||
/// <inheritdoc /> | ||
public IMaxBucketCardinality MaxBucketCardinality { get; set; } | ||
} | ||
|
||
public partial class EstimateModelMemoryDescriptor<TDocument> where TDocument : class | ||
{ | ||
IAnalysisConfig IEstimateModelMemoryRequest.AnalysisConfig { get; set; } | ||
IOverallCardinality IEstimateModelMemoryRequest.OverallCardinality { get; set; } | ||
IMaxBucketCardinality IEstimateModelMemoryRequest.MaxBucketCardinality { get; set; } | ||
|
||
/// <inheritdoc /> | ||
public EstimateModelMemoryDescriptor<TDocument> AnalysisConfig(Func<AnalysisConfigDescriptor<TDocument>, IAnalysisConfig> selector) => | ||
Assign(selector, (a, v) => a.AnalysisConfig = v?.Invoke(new AnalysisConfigDescriptor<TDocument>())); | ||
|
||
/// <inheritdoc /> | ||
public EstimateModelMemoryDescriptor<TDocument> OverallCardinality(Func<OverallCardinalityDescriptor<TDocument>, IPromise<IOverallCardinality>> analyzerSelector) => | ||
Assign(analyzerSelector, (a, v) => a.OverallCardinality = v?.Invoke(new OverallCardinalityDescriptor<TDocument>())?.Value); | ||
|
||
/// <inheritdoc /> | ||
public EstimateModelMemoryDescriptor<TDocument> MaxBucketCardinality(Func<MaxBucketCardinalityDescriptor<TDocument>, IPromise<IMaxBucketCardinality>> analyzerSelector) => | ||
Assign(analyzerSelector, (a, v) => a.MaxBucketCardinality = v?.Invoke(new MaxBucketCardinalityDescriptor<TDocument>())?.Value); | ||
} | ||
|
||
[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<OverallCardinality, IOverallCardinality, Field, long>))] | ||
public interface IOverallCardinality : IIsADictionary<Field, long> { } | ||
|
||
public class OverallCardinality : IsADictionaryBase<Field, long>, IOverallCardinality | ||
{ | ||
public OverallCardinality() { } | ||
|
||
public OverallCardinality(IDictionary<Field, long> container) : base(container) { } | ||
|
||
public void Add(Field field, long cardinality) => BackingDictionary.Add(field, cardinality); | ||
} | ||
|
||
public class OverallCardinality<T> : OverallCardinality where T : class | ||
{ | ||
public void Add<TValue>(Expression<Func<T, TValue>> field, long cardinality) => BackingDictionary.Add(field, cardinality); | ||
} | ||
|
||
public class OverallCardinalityDescriptor<T> : IsADictionaryDescriptorBase<OverallCardinalityDescriptor<T>, IOverallCardinality, Field, long> where T : class | ||
{ | ||
public OverallCardinalityDescriptor() : base(new OverallCardinality()) { } | ||
|
||
public OverallCardinalityDescriptor<T> Field(Field field, long cardinality) => Assign(field, cardinality); | ||
|
||
public OverallCardinalityDescriptor<T> Field<TValue>(Expression<Func<T, TValue>> field, long cardinality) => Assign(field, cardinality); | ||
} | ||
|
||
[JsonFormatter(typeof(VerbatimDictionaryKeysFormatter<MaxBucketCardinality, IMaxBucketCardinality, Field, long>))] | ||
public interface IMaxBucketCardinality : IIsADictionary<Field, long> { } | ||
|
||
public class MaxBucketCardinality : IsADictionaryBase<Field, long>, IMaxBucketCardinality | ||
{ | ||
public MaxBucketCardinality() { } | ||
|
||
public MaxBucketCardinality(IDictionary<Field, long> container) : base(container) { } | ||
|
||
public MaxBucketCardinality(Dictionary<Field, long> container) : base(container) { } | ||
|
||
public void Add(Field field, long cardinality) => BackingDictionary.Add(field, cardinality); | ||
} | ||
|
||
public class MaxBucketCardinality<T> : MaxBucketCardinality where T : class | ||
{ | ||
public void Add<TValue>(Expression<Func<T, TValue>> field, long cardinality) => BackingDictionary.Add(field, cardinality); | ||
} | ||
|
||
public class MaxBucketCardinalityDescriptor<T> : IsADictionaryDescriptorBase<MaxBucketCardinalityDescriptor<T>, IMaxBucketCardinality, Field, long> where T : class | ||
{ | ||
public MaxBucketCardinalityDescriptor() : base(new MaxBucketCardinality()) { } | ||
|
||
public MaxBucketCardinalityDescriptor<T> Field(Field field, long cardinality) => Assign(field, cardinality); | ||
|
||
public MaxBucketCardinalityDescriptor<T> Field<TValue>(Expression<Func<T, TValue>> field, long cardinality) => Assign(field, cardinality); | ||
} | ||
} |
10 changes: 10 additions & 0 deletions
10
src/Nest/XPack/MachineLearning/EstimateModelMemory/EstimateModelMemoryResponse.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,10 @@ | ||
using System.Runtime.Serialization; | ||
|
||
namespace Nest | ||
{ | ||
public class EstimateModelMemoryResponse : ResponseBase | ||
{ | ||
[DataMember(Name ="model_memory_estimate")] | ||
public string ModelMemoryEstimate { get; internal set; } | ||
} | ||
} |
102 changes: 102 additions & 0 deletions
102
tests/Tests/XPack/MachineLearning/EstimateModelMemory/EstimateModelMemoryApiTests.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,102 @@ | ||
using System; | ||
using Elastic.Xunit.XunitPlumbing; | ||
using Elasticsearch.Net; | ||
using Nest; | ||
using Tests.Core.Extensions; | ||
using Tests.Domain; | ||
using Tests.Framework.EndpointTests.TestState; | ||
using static Nest.Infer; | ||
|
||
namespace Tests.XPack.MachineLearning.EstimateModelMemory | ||
{ | ||
[SkipVersion("<7.7.0", "Introduced in 7.7.0")] | ||
public class EstimateModelMemoryApiTests : MachineLearningIntegrationTestBase<EstimateModelMemoryResponse, IEstimateModelMemoryRequest, EstimateModelMemoryDescriptor<Metric>, EstimateModelMemoryRequest> | ||
{ | ||
public EstimateModelMemoryApiTests(MachineLearningCluster cluster, EndpointUsage usage) : base(cluster, usage) { } | ||
|
||
protected override bool ExpectIsValid => true; | ||
protected override int ExpectStatusCode => 200; | ||
protected override Func<EstimateModelMemoryDescriptor<Metric>, IEstimateModelMemoryRequest> Fluent => f => f | ||
.AnalysisConfig(a => a | ||
.BucketSpan("30m") | ||
.Latency("0s") | ||
.Detectors(d => d.Sum(c => c.FieldName(r => r.Total))) | ||
) | ||
.OverallCardinality(m => | ||
m.Field(f => f.Response, 50) | ||
.Field(f => f.Accept, 10) | ||
) | ||
.MaxBucketCardinality(m => | ||
m.Field(f => f.Response, 500) | ||
.Field(f => f.Accept, 100) | ||
); | ||
|
||
protected override HttpMethod HttpMethod => HttpMethod.POST; | ||
|
||
protected override object ExpectJson => new | ||
{ | ||
analysis_config = new | ||
{ | ||
bucket_span = "30m", | ||
detectors = new[] | ||
{ | ||
new | ||
{ | ||
function = "sum", | ||
field_name = "total" | ||
} | ||
}, | ||
latency = "0s", | ||
}, | ||
overall_cardinality = new | ||
{ | ||
response = 50, | ||
accept = 10, | ||
}, | ||
max_bucket_cardinality = new | ||
{ | ||
response = 500, | ||
accept = 100, | ||
} | ||
}; | ||
|
||
protected override EstimateModelMemoryRequest Initializer => new EstimateModelMemoryRequest | ||
{ | ||
AnalysisConfig = new AnalysisConfig | ||
{ | ||
BucketSpan = "30m", | ||
Latency = "0s", | ||
Detectors = new[] | ||
{ | ||
new SumDetector | ||
{ | ||
FieldName = Field<Metric>(f => f.Total) | ||
} | ||
} | ||
}, | ||
OverallCardinality = new OverallCardinality | ||
{ | ||
{ Field<Metric>(f => f.Response), 50 }, | ||
{ Field<Metric>(f => f.Accept), 10 } | ||
}, | ||
MaxBucketCardinality = new MaxBucketCardinality | ||
{ | ||
{ Field<Metric>(f => f.Response), 500 }, | ||
{ Field<Metric>(f => f.Accept), 100 } | ||
} | ||
}; | ||
|
||
protected override string UrlPath => $"/_ml/anomaly_detectors/_estimate_model_memory"; | ||
|
||
protected override LazyResponses ClientUsage() => Calls( | ||
(client, f) => client.MachineLearning.EstimateModelMemory(f), | ||
(client, f) => client.MachineLearning.EstimateModelMemoryAsync(f), | ||
(client, r) => client.MachineLearning.EstimateModelMemory(r), | ||
(client, r) => client.MachineLearning.EstimateModelMemoryAsync(r) | ||
); | ||
|
||
protected override EstimateModelMemoryDescriptor<Metric> NewDescriptor() => new EstimateModelMemoryDescriptor<Metric>(); | ||
|
||
protected override void ExpectResponse(EstimateModelMemoryResponse response) => response.ShouldBeValid(); | ||
} | ||
codebrain marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} |
17 changes: 17 additions & 0 deletions
17
tests/Tests/XPack/MachineLearning/EstimateModelMemory/EstimateModelMemoryUrlTests.cs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
using System.Threading.Tasks; | ||
using Elastic.Xunit.XunitPlumbing; | ||
using Nest; | ||
using Tests.Framework.EndpointTests; | ||
using static Tests.Framework.EndpointTests.UrlTester; | ||
|
||
namespace Tests.XPack.MachineLearning.EstimateModelMemory | ||
{ | ||
public class EstimateModelMemoryUrlTests : UrlTestsBase | ||
{ | ||
[U] public override async Task Urls() => await POST("/_ml/anomaly_detectors/_estimate_model_memory") | ||
.Fluent(c => c.MachineLearning.EstimateModelMemory(new EstimateModelMemoryRequest())) | ||
.Request(c => c.MachineLearning.EstimateModelMemory(new EstimateModelMemoryRequest())) | ||
.FluentAsync(c => c.MachineLearning.EstimateModelMemoryAsync(new EstimateModelMemoryRequest())) | ||
.RequestAsync(c => c.MachineLearning.EstimateModelMemoryAsync(new EstimateModelMemoryRequest())); | ||
} | ||
} |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.