Skip to content

Another TPCH fix: Wait for all shards #5106

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Swiddis merged 3 commits into from
Feb 3, 2026
+22 −17
Merged

Another TPCH fix: Wait for all shards #5106

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c73e8eb
Another TCPH fix: Wait for all shards
Swiddis Feb 3, 2026
2357a4f
Set replicas to 0 for integ test mappings by default
Swiddis Feb 3, 2026
410a638
Add more ctx to a comment
Swiddis Feb 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 0 additions & 13 deletions integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,19 +210,6 @@ protected synchronized void loadIndex(Index index, RestClient client) throws IOE
createIndexByRestClient(client, indexName, mapping);
loadDataByRestClient(client, indexName, dataSet);
}
// loadIndex() could directly return when isIndexExist()=true,
// e.g. the index is created in the cluster but data hasn't been flushed.
// We block loadIndex() until data loaded to resolve
// https://github.com/opensearch-project/sql/issues/4261
int countDown = 3; // 1500ms timeout
while (countDown != 0 && getDocCount(client, indexName) == 0) {
try {
Thread.sleep(500);
countDown--;
} catch (InterruptedException e) {
throw new IOException(e);
}
}
}

protected synchronized void loadIndex(Index index) throws IOException {
Expand Down
26 changes: 22 additions & 4 deletions integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,12 +46,29 @@ public class TestUtils {
*/
public static void createIndexByRestClient(RestClient client, String indexName, String mapping) {
Request request = new Request("PUT", "/" + indexName);
if (!isNullOrEmpty(mapping)) {
request.setJsonEntity(mapping);
}
JSONObject jsonObject = isNullOrEmpty(mapping) ? new JSONObject() : new JSONObject(mapping);
setZeroReplicas(jsonObject);
request.setJsonEntity(jsonObject.toString());
performRequest(client, request);
}

/**
* Sets number_of_replicas to 0 in the index settings. This makes multi-node behavior consistent
* (<a href="https://github.com/opensearch-project/sql/issues/4261">4261</a>) and prevents tests
* from hanging on single-node clusters when using wait_for_active_shards=all.
*
* @param jsonObject the index creation JSON object to modify
*/
private static void setZeroReplicas(JSONObject jsonObject) {
JSONObject settings =
jsonObject.has("settings") ? jsonObject.getJSONObject("settings") : new JSONObject();
JSONObject indexSettings =
settings.has("index") ? settings.getJSONObject("index") : new JSONObject();
indexSettings.put("number_of_replicas", 0);
settings.put("index", indexSettings);
jsonObject.put("settings", settings);
}

/**
* https://github.com/elastic/elasticsearch/pull/49959<br>
* Deprecate creation of dot-prefixed index names except for hidden and system indices. Create
Expand Down Expand Up @@ -99,7 +116,8 @@ public static boolean isIndexExist(RestClient client, String indexName) {
public static void loadDataByRestClient(
RestClient client, String indexName, String dataSetFilePath) throws IOException {
Path path = Paths.get(getResourceFilePath(dataSetFilePath));
Request request = new Request("POST", "/" + indexName + "/_bulk?refresh=true");
Request request =
new Request("POST", "/" + indexName + "/_bulk?refresh=wait_for&wait_for_active_shards=all");
request.setJsonEntity(new String(Files.readAllBytes(path)));
performRequest(client, request);
}
Expand Down
Loading