Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixing concurrent search tests with one slice #11071

Merged
merged 2 commits into from
Nov 5, 2023
Merged

Fixing concurrent search tests with one slice #11071

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3ce73bf
Fixing concurrent search tests with one slice (#11071)
ticheng-aws Sep 28, 2023
f09a93a
Remove changes for non-flaky tests (#11071)
ticheng-aws Nov 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/search/nested/SimpleNestedIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,7 @@ public void testMultiNested() throws Exception {
refresh();
// check the numDocs
assertDocumentCount("test", 7);
indexRandomForConcurrentSearch("test");

// do some multi nested queries
SearchResponse searchResponse = client().prepareSearch("test")
Expand Down Expand Up @@ -485,6 +486,7 @@ public void testExplain() throws Exception {
)
.setRefreshPolicy(IMMEDIATE)
.get();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch("test")
.setQuery(nestedQuery("nested1", termQuery("nested1.n_field1", "n_value1"), ScoreMode.Total))
Expand Down Expand Up @@ -968,6 +970,10 @@ public void testNestedSortWithMultiLevelFiltering() throws Exception {

// https://github.com/elastic/elasticsearch/issues/31554
public void testLeakingSortValues() throws Exception {
assumeFalse(
"Concurrent search case muted pending fix: https://github.com/opensearch-project/OpenSearch/issues/11065",
internalCluster().clusterService().getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
);
assertAcked(
prepareCreate("test").setSettings(Settings.builder().put("number_of_shards", 1))
.setMapping(
Expand Down Expand Up @@ -1035,6 +1041,7 @@ public void testLeakingSortValues() throws Exception {
.get();

refresh();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch()
.setQuery(termQuery("_id", 2))
Expand Down Expand Up @@ -1627,6 +1634,7 @@ public void testCheckFixedBitSetCache() throws Exception {
client().prepareIndex("test").setId("1").setSource("field", "value").get();
refresh();
ensureSearchable("test");
indexRandomForConcurrentSearch("test");

// No nested mapping yet, there shouldn't be anything in the fixed bit set cache
ClusterStatsResponse clusterStatsResponse = client().admin().cluster().prepareClusterStats().get();
Expand Down
1 change: 1 addition & 0 deletions server/src/internalClusterTest/java/org/opensearch/search/pit/PitMultiNodeIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ public void clearIndex() {
public void testPit() throws Exception {
CreatePitRequest request = new CreatePitRequest(TimeValue.timeValueDays(1), true);
request.setIndices(new String[] { "index" });
indexRandomForConcurrentSearch("index");
ActionFuture<CreatePitResponse> execute = client().execute(CreatePitAction.INSTANCE, request);
CreatePitResponse pitResponse = execute.get();
SearchResponse searchResponse = client().prepareSearch("index")
Expand Down
10 changes: 6 additions & 4 deletions server/src/internalClusterTest/java/org/opensearch/search/preference/SearchPreferenceIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,6 @@
import org.opensearch.test.OpenSearchIntegTestCase;
import org.opensearch.test.ParameterizedOpenSearchIntegTestCase;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -99,7 +98,7 @@ public Settings nodeSettings(int nodeOrdinal) {
}

// see #2896
public void testStopOneNodePreferenceWithRedState() throws IOException {
public void testStopOneNodePreferenceWithRedState() throws Exception {
assertAcked(
prepareCreate("test").setSettings(
Settings.builder().put("index.number_of_shards", cluster().numDataNodes() + 2).put("index.number_of_replicas", 0)
Expand All @@ -110,6 +109,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException {
client().prepareIndex("test").setId("" + i).setSource("field1", "value1").get();
}
refresh();
indexRandomForConcurrentSearch("test");
internalCluster().stopRandomDataNode();
client().admin().cluster().prepareHealth().setWaitForStatus(ClusterHealthStatus.RED).get();
String[] preferences = new String[] {
Expand Down Expand Up @@ -138,7 +138,7 @@ public void testStopOneNodePreferenceWithRedState() throws IOException {
assertThat("_only_local", searchResponse.getFailedShards(), greaterThanOrEqualTo(0));
}

public void testNoPreferenceRandom() {
public void testNoPreferenceRandom() throws Exception {
assertAcked(
prepareCreate("test").setSettings(
// this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
Expand All @@ -149,6 +149,7 @@ public void testNoPreferenceRandom() {

client().prepareIndex("test").setSource("field1", "value1").get();
refresh();
indexRandomForConcurrentSearch("test");

final Client client = internalCluster().smartClient();
SearchResponse searchResponse = client.prepareSearch("test").setQuery(matchAllQuery()).get();
Expand Down Expand Up @@ -201,7 +202,7 @@ public void testThatSpecifyingNonExistingNodesReturnsUsefulError() {
}
}

public void testNodesOnlyRandom() {
public void testNodesOnlyRandom() throws Exception {
assertAcked(
prepareCreate("test").setSettings(
// this test needs at least a replica to make sure two consecutive searches go to two different copies of the same data
Expand All @@ -211,6 +212,7 @@ public void testNodesOnlyRandom() {
ensureGreen();
client().prepareIndex("test").setSource("field1", "value1").get();
refresh();
indexRandomForConcurrentSearch("test");

final Client client = internalCluster().smartClient();
// multiple wildchar to cover multi-param usecase
Expand Down
6 changes: 6 additions & 0 deletions server/src/internalClusterTest/java/org/opensearch/search/query/QueryStringIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ public void testDocWithAllTypes() throws Exception {
String docBody = copyToStringFromClasspath("/org/opensearch/search/query/all-example-document.json");
reqs.add(client().prepareIndex("test").setId("1").setSource(docBody, MediaTypeRegistry.JSON));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
assertHits(resp.getHits(), "1");
Expand Down Expand Up @@ -225,6 +226,7 @@ public void testKeywordWithWhitespace() throws Exception {
reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar"));
reqs.add(client().prepareIndex("test").setId("3").setSource("f1", "foo bar"));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(queryStringQuery("foo")).get();
assertHits(resp.getHits(), "3");
Expand All @@ -245,6 +247,7 @@ public void testRegexCaseInsensitivity() throws Exception {
indexRequests.add(client().prepareIndex("messages").setId("1").setSource("message", "message: this is a TLS handshake"));
indexRequests.add(client().prepareIndex("messages").setId("2").setSource("message", "message: this is a tcp handshake"));
indexRandom(true, false, indexRequests);
indexRandomForConcurrentSearch("messages");

SearchResponse response = client().prepareSearch("messages").setQuery(queryStringQuery("/TLS/").defaultField("message")).get();
assertNoFailures(response);
Expand Down Expand Up @@ -282,6 +285,7 @@ public void testAllFields() throws Exception {
List<IndexRequestBuilder> reqs = new ArrayList<>();
reqs.add(client().prepareIndex("test_1").setId("1").setSource("f1", "foo", "f2", "eggplant"));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test_1");

SearchResponse resp = client().prepareSearch("test_1")
.setQuery(queryStringQuery("foo eggplant").defaultOperator(Operator.AND))
Expand Down Expand Up @@ -374,6 +378,7 @@ public void testLimitOnExpandedFields() throws Exception {

client().prepareIndex("testindex").setId("1").setSource("field_A0", "foo bar baz").get();
refresh();
indexRandomForConcurrentSearch("testindex");

// single field shouldn't trigger the limit
doAssertOneHitForQueryString("field_A0:foo");
Expand Down Expand Up @@ -465,6 +470,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
List<IndexRequestBuilder> indexRequests = new ArrayList<>();
indexRequests.add(client().prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one"));
indexRandom(true, false, indexRequests);
indexRandomForConcurrentSearch("test");

// The wildcard field matches aliases for both a text and geo_point field.
// By default, the geo_point field should be ignored when building the query.
Expand Down
12 changes: 8 additions & 4 deletions server/src/internalClusterTest/java/org/opensearch/search/query/ScriptScoreQueryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,13 +109,14 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
// 1) only matched docs retrieved
// 2) score is calculated based on a script with params
// 3) min score applied
public void testScriptScore() {
public void testScriptScore() throws Exception {
assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
int docCount = 10;
for (int i = 1; i <= docCount; i++) {
client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get();
}
refresh();
indexRandomForConcurrentSearch("test-index");

Map<String, Object> params = new HashMap<>();
params.put("param1", 0.1);
Expand All @@ -135,13 +136,14 @@ public void testScriptScore() {
assertOrderedSearchHits(resp, "10", "8", "6");
}

public void testScriptScoreBoolQuery() {
public void testScriptScoreBoolQuery() throws Exception {
assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
int docCount = 10;
for (int i = 1; i <= docCount; i++) {
client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + i, "field2", i).get();
}
refresh();
indexRandomForConcurrentSearch("test-index");

Map<String, Object> params = new HashMap<>();
params.put("param1", 0.1);
Expand All @@ -155,7 +157,7 @@ public void testScriptScoreBoolQuery() {
}

// test that when the internal query is rewritten script_score works well
public void testRewrittenQuery() {
public void testRewrittenQuery() throws Exception {
assertAcked(
prepareCreate("test-index2").setSettings(Settings.builder().put("index.number_of_shards", 1))
.setMapping("field1", "type=date", "field2", "type=double")
Expand All @@ -164,6 +166,7 @@ public void testRewrittenQuery() {
client().prepareIndex("test-index2").setId("2").setSource("field1", "2019-10-01", "field2", 2).get();
client().prepareIndex("test-index2").setId("3").setSource("field1", "2019-11-01", "field2", 3).get();
refresh();
indexRandomForConcurrentSearch("test-index2");

RangeQueryBuilder rangeQB = new RangeQueryBuilder("field1").from("2019-01-01"); // the query should be rewritten to from:null
Map<String, Object> params = new HashMap<>();
Expand All @@ -174,14 +177,15 @@ public void testRewrittenQuery() {
assertOrderedSearchHits(resp, "3", "2", "1");
}

public void testDisallowExpensiveQueries() {
public void testDisallowExpensiveQueries() throws Exception {
try {
assertAcked(prepareCreate("test-index").setMapping("field1", "type=text", "field2", "type=double"));
int docCount = 10;
for (int i = 1; i <= docCount; i++) {
client().prepareIndex("test-index").setId("" + i).setSource("field1", "text" + (i % 2), "field2", i).get();
}
refresh();
indexRandomForConcurrentSearch("test-index");

Map<String, Object> params = new HashMap<>();
params.put("param1", 0.1);
Expand Down
17 changes: 16 additions & 1 deletion server/src/internalClusterTest/java/org/opensearch/search/query/SimpleQueryStringIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,7 @@ public void testSimpleQueryString() throws ExecutionException, InterruptedExcept
client().prepareIndex("test").setId("5").setSource("body", "quux baz spaghetti"),
client().prepareIndex("test").setId("6").setSource("otherbody", "spaghetti")
);
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar")).get();
assertHitCount(searchResponse, 3L);
Expand Down Expand Up @@ -199,6 +200,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
client().prepareIndex("test").setId("3").setSource("body", "foo bar"),
client().prepareIndex("test").setId("4").setSource("body", "foo baz bar")
);
indexRandomForConcurrentSearch("test");

logger.info("--> query 1");
SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar").minimumShouldMatch("2")).get();
Expand Down Expand Up @@ -235,6 +237,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
client().prepareIndex("test").setId("7").setSource("body2", "foo bar", "other", "foo"),
client().prepareIndex("test").setId("8").setSource("body2", "foo baz bar", "other", "foo")
);
indexRandomForConcurrentSearch("test");

logger.info("--> query 5");
searchResponse = client().prepareSearch()
Expand All @@ -256,7 +259,7 @@ public void testSimpleQueryStringMinimumShouldMatch() throws Exception {
assertSearchHits(searchResponse, "6", "7", "8");
}

public void testNestedFieldSimpleQueryString() throws IOException {
public void testNestedFieldSimpleQueryString() throws Exception {
assertAcked(
prepareCreate("test").setMapping(
jsonBuilder().startObject()
Expand All @@ -275,6 +278,7 @@ public void testNestedFieldSimpleQueryString() throws IOException {
);
client().prepareIndex("test").setId("1").setSource("body", "foo bar baz").get();
refresh();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("foo bar baz").field("body")).get();
assertHitCount(searchResponse, 1L);
Expand Down Expand Up @@ -359,6 +363,8 @@ public void testSimpleQueryStringLenient() throws ExecutionException, Interrupte
client().prepareIndex("test2").setId("10").setSource("field", 5)
);
refresh();
indexRandomForConcurrentSearch("test1");
indexRandomForConcurrentSearch("test2");

SearchResponse searchResponse = client().prepareSearch()
.setAllowPartialSearchResults(true)
Expand Down Expand Up @@ -419,6 +425,7 @@ public void testSimpleQueryStringUsesFieldAnalyzer() throws Exception {
client().prepareIndex("test").setId("2").setSource("foo", 234, "bar", "bcd").get();

refresh();
indexRandomForConcurrentSearch("test");

SearchResponse searchResponse = client().prepareSearch().setQuery(simpleQueryStringQuery("123").field("foo").field("bar")).get();
assertHitCount(searchResponse, 1L);
Expand Down Expand Up @@ -469,6 +476,7 @@ public void testBasicAllQuery() throws Exception {
reqs.add(client().prepareIndex("test").setId("2").setSource("f2", "Bar"));
reqs.add(client().prepareIndex("test").setId("3").setSource("f3", "foo bar baz"));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
assertHitCount(resp, 2L);
Expand All @@ -492,6 +500,7 @@ public void testWithDate() throws Exception {
reqs.add(client().prepareIndex("test").setId("1").setSource("f1", "foo", "f_date", "2015/09/02"));
reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01"));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
assertHits(resp.getHits(), "1", "2");
Expand Down Expand Up @@ -523,6 +532,7 @@ public void testWithLotsOfTypes() throws Exception {
client().prepareIndex("test").setId("2").setSource("f1", "bar", "f_date", "2015/09/01", "f_float", "1.8", "f_ip", "127.0.0.2")
);
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo bar")).get();
assertHits(resp.getHits(), "1", "2");
Expand Down Expand Up @@ -550,6 +560,7 @@ public void testDocWithAllTypes() throws Exception {
String docBody = copyToStringFromClasspath("/org/opensearch/search/query/all-example-document.json");
reqs.add(client().prepareIndex("test").setId("1").setSource(docBody, MediaTypeRegistry.JSON));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
assertHits(resp.getHits(), "1");
Expand Down Expand Up @@ -596,6 +607,7 @@ public void testKeywordWithWhitespace() throws Exception {
reqs.add(client().prepareIndex("test").setId("2").setSource("f1", "bar"));
reqs.add(client().prepareIndex("test").setId("3").setSource("f1", "foo bar"));
indexRandom(true, false, reqs);
indexRandomForConcurrentSearch("test");

SearchResponse resp = client().prepareSearch("test").setQuery(simpleQueryStringQuery("foo")).get();
assertHits(resp.getHits(), "3");
Expand Down Expand Up @@ -663,6 +675,7 @@ public void testFieldAlias() throws Exception {
indexRequests.add(client().prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two"));
indexRequests.add(client().prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three"));
indexRandom(true, false, indexRequests);
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_alias")).get();

Expand All @@ -681,6 +694,7 @@ public void testFieldAliasWithWildcardField() throws Exception {
indexRequests.add(client().prepareIndex("test").setId("2").setSource("f3", "value", "f2", "two"));
indexRequests.add(client().prepareIndex("test").setId("3").setSource("f3", "another value", "f2", "three"));
indexRandom(true, false, indexRequests);
indexRandomForConcurrentSearch("test");

SearchResponse response = client().prepareSearch("test").setQuery(simpleQueryStringQuery("value").field("f3_*")).get();

Expand All @@ -697,6 +711,7 @@ public void testFieldAliasOnDisallowedFieldType() throws Exception {
List<IndexRequestBuilder> indexRequests = new ArrayList<>();
indexRequests.add(client().prepareIndex("test").setId("1").setSource("f3", "text", "f2", "one"));
indexRandom(true, false, indexRequests);
indexRandomForConcurrentSearch("test");

// The wildcard field matches aliases for both a text and boolean field.
// By default, the boolean field should be ignored when building the query.
Expand Down
Loading