Skip to content

Commit d516278

Browse files
MpdreamzMpdreamz
committed
SeedNode() on DefaultSeeder parallelizes where possible (#3389)
(cherry picked from commit b8e412e)
1 parent 9201256 commit d516278

File tree

3 files changed

+130
-129
lines changed

3 files changed

+130
-129
lines changed

‎src/Tests/Tests.Core/ManagedElasticsearch/NodeSeeders/DefaultSeeder.cs

Lines changed: 127 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,7 @@
1+
using System;
12
using System.Collections.Generic;
3+
using System.Runtime.CompilerServices;
4+
using System.Threading.Tasks;
25
using Nest;
36
using Tests.Configuration;
47
using Tests.Core.Client;
@@ -36,22 +39,46 @@ public DefaultSeeder(IElasticClient client) : this(client, null) { }
3639
public void SeedNode()
3740
{
3841
if (!TestClient.Configuration.ForceReseed && this.AlreadySeeded()) return;
39-
// Ensure a clean slate by deleting everything regardless of whether they may already exist
40-
this.ClusterSettings();
41-
this.DeleteIndicesAndTemplates();
42-
// and now recreate everything
43-
this.CreateIndicesAndSeedIndexData();
42+
43+
var t = Task.Run(async () => await this.SeedNodeAsync());
44+
45+
t.Wait(TimeSpan.FromSeconds(40));
46+
}
47+
public void SeedNodeNoData()
48+
{
49+
if (!TestClient.Configuration.ForceReseed && this.AlreadySeeded()) return;
50+
51+
var t = Task.Run(async () => await this.SeedNodeNoDataAsync());
52+
53+
t.Wait(TimeSpan.FromSeconds(40));
4454
}
4555

4656
// Sometimes we run against an manually started elasticsearch when
4757
// writing tests to cut down on cluster startup times.
4858
// If raw_fields exists assume this cluster is already seeded.
4959
private bool AlreadySeeded() => this.Client.IndexTemplateExists(TestsIndexTemplateName).Exists;
5060

51-
public void ClusterSettings()
61+
private async Task SeedNodeAsync()
62+
{
63+
// Ensure a clean slate by deleting everything regardless of whether they may already exist
64+
await this.DeleteIndicesAndTemplatesAsync();
65+
await this.ClusterSettingsAsync();
66+
// and now recreate everything
67+
await this.CreateIndicesAndSeedIndexDataAsync();
68+
}
69+
private async Task SeedNodeNoDataAsync()
70+
{
71+
// Ensure a clean slate by deleting everything regardless of whether they may already exist
72+
await this.DeleteIndicesAndTemplatesAsync();
73+
await this.ClusterSettingsAsync();
74+
// and now recreate everything
75+
await this.CreateIndicesAsync();
76+
}
77+
78+
public async Task ClusterSettingsAsync()
5279
{
5380
if (TestConfiguration.Instance.InRange("<6.1.0")) return;
54-
var putSettingsResponse = this.Client.ClusterPutSettings(s=>s
81+
var putSettingsResponse = await this.Client.ClusterPutSettingsAsync(s=>s
5582
.Transient(t=>t
5683
.Add("cluster.routing.use_adaptive_replica_selection", true)
5784
)
@@ -60,109 +87,111 @@ public void ClusterSettings()
6087
putSettingsResponse.ShouldBeValid();
6188
}
6289

63-
public void DeleteIndicesAndTemplates()
90+
public async Task DeleteIndicesAndTemplatesAsync()
6491
{
65-
if (this.Client.IndexTemplateExists(TestsIndexTemplateName).Exists)
66-
this.Client.DeleteIndexTemplate(TestsIndexTemplateName);
67-
if (this.Client.IndexExists(Infer.Indices<Project>()).Exists)
68-
this.Client.DeleteIndex(typeof(Project));
69-
if (this.Client.IndexExists(Infer.Indices<Developer>()).Exists)
70-
this.Client.DeleteIndex(typeof(Developer));
71-
if (this.Client.IndexExists(Infer.Indices<ProjectPercolation>()).Exists)
72-
this.Client.DeleteIndex(typeof(ProjectPercolation));
73-
}
74-
75-
public void CreateIndices()
76-
{
77-
this.CreateIndexTemplate();
78-
this.CreateProjectIndex();
79-
this.CreateDeveloperIndex();
80-
this.CreatePercolatorIndex();
81-
}
82-
83-
private void SeedIndexData()
84-
{
85-
this.Client.IndexMany(Project.Projects);
86-
this.Client.IndexMany(Developer.Developers);
87-
this.Client.IndexDocument(new ProjectPercolation
92+
var tasks = new Task[]
8893
{
89-
Id = "1",
90-
Query = new MatchAllQuery()
91-
});
92-
this.Client.Bulk(b => b
93-
.IndexMany(
94-
CommitActivity.CommitActivities,
95-
(d, c) => d.Document(c).Routing(c.ProjectName)
96-
)
97-
);
98-
this.Client.Refresh(Nest.Indices.Index(typeof(Project), typeof(Developer), typeof(ProjectPercolation)));
94+
this.Client.DeleteIndexTemplateAsync(TestsIndexTemplateName),
95+
this.Client.DeleteIndexAsync(typeof(Project)),
96+
this.Client.DeleteIndexAsync(typeof(Developer)),
97+
this.Client.DeleteIndexAsync(typeof(ProjectPercolation))
98+
};
99+
await Task.WhenAll(tasks);
99100
}
100101

101-
private void CreateIndicesAndSeedIndexData()
102+
private async Task CreateIndicesAndSeedIndexDataAsync()
102103
{
103-
this.CreateIndices();
104-
this.SeedIndexData();
104+
await this.CreateIndicesAsync();
105+
await this.SeedIndexDataAsync();
105106
}
106107

107-
private void CreateIndexTemplate()
108+
public async Task CreateIndicesAsync()
108109
{
109-
var putTemplateResult = this.Client.PutIndexTemplate(new PutIndexTemplateRequest(TestsIndexTemplateName)
110+
var indexTemplateResponse = await this.CreateIndexTemplateAsync();
111+
indexTemplateResponse.ShouldBeValid();
112+
113+
var tasks = new []
110114
{
111-
IndexPatterns = new[] {"*"},
112-
Settings = this.IndexSettings
115+
this.CreateProjectIndexAsync(),
116+
this.CreateDeveloperIndexAsync(),
117+
this.CreatePercolatorIndexAsync(),
118+
};
119+
await Task.WhenAll(tasks).ContinueWith(t =>
120+
{
121+
foreach(var r in t.Result)
122+
r.ShouldBeValid();
113123
});
114-
putTemplateResult.ShouldBeValid();
115124
}
116125

117-
private void CreateDeveloperIndex()
126+
private async Task SeedIndexDataAsync()
118127
{
119-
var createDeveloperIndex = this.Client.CreateIndex(Infer.Index<Developer>(), c => c
120-
.Mappings(map => map
121-
.Map<Developer>(m => m
122-
.AutoMap()
123-
.Properties(DeveloperProperties)
128+
var tasks = new Task[]
129+
{
130+
this.Client.IndexManyAsync(Project.Projects),
131+
this.Client.IndexManyAsync(Developer.Developers),
132+
this.Client.IndexDocumentAsync(new ProjectPercolation
133+
{
134+
Id = "1",
135+
Query = new MatchAllQuery()
136+
}),
137+
this.Client.BulkAsync(b => b
138+
.IndexMany(
139+
CommitActivity.CommitActivities,
140+
(d, c) => d.Document(c).Routing(c.ProjectName)
124141
)
125142
)
126-
);
127-
createDeveloperIndex.ShouldBeValid();
143+
};
144+
await Task.WhenAll(tasks);
145+
await this.Client.RefreshAsync(Indices.Index(typeof(Project), typeof(Developer), typeof(ProjectPercolation)));
128146
}
129147

130-
private void CreateProjectIndex()
148+
private Task<IPutIndexTemplateResponse> CreateIndexTemplateAsync() => this.Client.PutIndexTemplateAsync(new PutIndexTemplateRequest(TestsIndexTemplateName)
131149
{
132-
var createProjectIndex = this.Client.CreateIndex(typeof(Project), c => c
133-
.Settings(settings => settings
134-
.Analysis(ProjectAnalysisSettings)
150+
IndexPatterns = new[] {"*"},
151+
Settings = this.IndexSettings
152+
});
153+
154+
private Task<ICreateIndexResponse> CreateDeveloperIndexAsync() => this.Client.CreateIndexAsync(Infer.Index<Developer>(), c => c
155+
.Mappings(map => map
156+
.Map<Developer>(m => m
157+
.AutoMap()
158+
.Properties(DeveloperProperties)
135159
)
136-
.Aliases(aliases => aliases
137-
.Alias(ProjectsAliasName)
138-
.Alias(ProjectsAliasFilter, a => a
139-
.Filter<Project>(f => f.Term(p => p.Join, Infer.Relation<Project>()))
140-
)
141-
.Alias(CommitsAliasFilter, a => a
142-
.Filter<CommitActivity>(f => f.Term(p => p.Join, Infer.Relation<CommitActivity>()))
143-
)
160+
)
161+
);
162+
163+
private Task<ICreateIndexResponse> CreateProjectIndexAsync() => this.Client.CreateIndexAsync(typeof(Project), c => c
164+
.Settings(settings => settings
165+
.Analysis(ProjectAnalysisSettings)
166+
)
167+
.Aliases(aliases => aliases
168+
.Alias(ProjectsAliasName)
169+
.Alias(ProjectsAliasFilter, a => a
170+
.Filter<Project>(f => f.Term(p => p.Join, Infer.Relation<Project>()))
171+
)
172+
.Alias(CommitsAliasFilter, a => a
173+
.Filter<CommitActivity>(f => f.Term(p => p.Join, Infer.Relation<CommitActivity>()))
144174
)
145-
.Mappings(map => map
146-
.Map<Project>(m => m
147-
.RoutingField(r=>r.Required())
148-
.AutoMap()
149-
.Properties(ProjectProperties)
150-
.Properties<CommitActivity>(props => props
151-
.Object<Developer>(o => o
152-
.AutoMap()
153-
.Name(p => p.Committer)
154-
.Properties(DeveloperProperties)
155-
)
156-
.Text(t => t
157-
.Name(p => p.ProjectName)
158-
.Index(false)
159-
)
175+
)
176+
.Mappings(map => map
177+
.Map<Project>(m => m
178+
.RoutingField(r=>r.Required())
179+
.AutoMap()
180+
.Properties(ProjectProperties)
181+
.Properties<CommitActivity>(props => props
182+
.Object<Developer>(o => o
183+
.AutoMap()
184+
.Name(p => p.Committer)
185+
.Properties(DeveloperProperties)
186+
)
187+
.Text(t => t
188+
.Name(p => p.ProjectName)
189+
.Index(false)
160190
)
161191
)
162192
)
163-
);
164-
createProjectIndex.ShouldBeValid();
165-
}
193+
)
194+
);
166195

167196
public static IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis)
168197
{
@@ -190,23 +219,18 @@ public static IAnalysis ProjectAnalysisSettings(AnalysisDescriptor analysis)
190219
}
191220

192221

193-
private void CreatePercolatorIndex()
194-
{
195-
var createPercolatedIndex = this.Client.CreateIndex(typeof(ProjectPercolation), c => c
196-
.Settings(s => s
197-
.AutoExpandReplicas("0-all")
198-
.Analysis(DefaultSeeder.ProjectAnalysisSettings)
199-
)
200-
.Mappings(map => map
201-
.Map<ProjectPercolation>(m => m
202-
.AutoMap()
203-
.Properties(PercolatedQueryProperties)
204-
)
222+
private Task<ICreateIndexResponse> CreatePercolatorIndexAsync() => this.Client.CreateIndexAsync(typeof(ProjectPercolation), c => c
223+
.Settings(s => s
224+
.AutoExpandReplicas("0-all")
225+
.Analysis(DefaultSeeder.ProjectAnalysisSettings)
226+
)
227+
.Mappings(map => map
228+
.Map<ProjectPercolation>(m => m
229+
.AutoMap()
230+
.Properties(PercolatedQueryProperties)
205231
)
206-
);
207-
208-
createPercolatedIndex.ShouldBeValid();
209-
}
232+
)
233+
);
210234

211235
public static PropertiesDescriptor<TProject> ProjectProperties<TProject>(PropertiesDescriptor<TProject> props)
212236
where TProject : Project => props

‎src/Tests/Tests.Profiling/Framework/ProfilingCluster.cs

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,6 @@ namespace Tests.Profiling.Framework
55
{
66
public class ProfilingCluster : ClientTestClusterBase
77
{
8-
protected override void SeedCluster()
9-
{
10-
var seeder = new DefaultSeeder(this.Client);
11-
seeder.DeleteIndicesAndTemplates();
12-
seeder.CreateIndices();
13-
}
8+
protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNodeNoData();
149
}
1510
}

‎src/Tests/Tests/Document/Multiple/Reindex/ReindexApiTests.cs

Lines changed: 2 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,25 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Linq;
4-
using System.Reactive.Linq;
54
using System.Threading;
6-
using Elastic.Xunit.Sdk;
75
using Elastic.Xunit.XunitPlumbing;
86
using FluentAssertions;
97
using Nest;
108
using Tests.Core.Extensions;
119
using Tests.Core.ManagedElasticsearch.Clusters;
1210
using Tests.Core.ManagedElasticsearch.NodeSeeders;
1311
using Tests.Domain;
14-
using Tests.Framework;
15-
using Tests.Framework.Integration;
16-
using Tests.Framework.ManagedElasticsearch;
17-
using Tests.Framework.ManagedElasticsearch.Clusters;
18-
using Tests.Framework.ManagedElasticsearch.NodeSeeders;
19-
using Xunit;
2012

2113
namespace Tests.Document.Multiple.Reindex
2214
{
2315
public class ReindexCluster : ClientTestClusterBase
2416
{
25-
protected override void SeedCluster()
26-
{
27-
var seeder = new DefaultSeeder(this.Client);
28-
seeder.DeleteIndicesAndTemplates();
29-
seeder.CreateIndices();
30-
}
17+
protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNodeNoData();
3118
}
3219

3320
public class ManualReindexCluster : ClientTestClusterBase
3421
{
35-
protected override void SeedCluster()
36-
{
37-
var seeder = new DefaultSeeder(this.Client);
38-
seeder.DeleteIndicesAndTemplates();
39-
seeder.CreateIndices();
40-
}
22+
protected override void SeedCluster() => new DefaultSeeder(this.Client).SeedNodeNoData();
4123
}
4224

4325
public class ReindexApiTests : IClusterFixture<ManualReindexCluster>

0 commit comments

Comments
 (0)