Skip to content

Commit aea9b41

Browse files
quangdutranquangdutran
authored
Add setting to exclude certain indices from insight query (#308)
Signed-off-by: Du Tran <quangdutran809@gmail.com>
1 parent 16be187 commit aea9b41

File tree

9 files changed

+268
-8
lines changed

9 files changed

+268
-8
lines changed

‎src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,7 @@ public List<Setting<?>> getSettings() {
153153
QueryInsightsSettings.TOP_N_EXPORTER_DELETE_AFTER,
154154
QueryInsightsSettings.TOP_N_EXPORTER_TYPE,
155155
QueryInsightsSettings.TOP_N_EXPORTER_TEMPLATE_PRIORITY,
156+
QueryInsightsSettings.TOP_N_QUERIES_EXCLUDED_INDICES,
156157
QueryCategorizationSettings.SEARCH_QUERY_FIELD_TYPE_CACHE_SIZE_KEY
157158
);
158159
}

‎src/main/java/org/opensearch/plugin/insights/core/listener/QueryInsightsListener.java‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010

1111
import static org.opensearch.plugin.insights.rules.model.SearchQueryRecord.DEFAULT_TOP_N_QUERY_MAP;
1212
import static org.opensearch.plugin.insights.settings.QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING;
13+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_EXCLUDED_INDICES;
1314
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_NAME;
1415
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_TYPE;
1516
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.TOP_N_QUERIES_GROUP_BY;
@@ -22,7 +23,11 @@
2223
import java.util.List;
2324
import java.util.Locale;
2425
import java.util.Map;
26+
import java.util.Optional;
27+
import java.util.Set;
2528
import java.util.concurrent.TimeUnit;
29+
import java.util.regex.Pattern;
30+
import java.util.stream.Collectors;
2631
import org.apache.logging.log4j.LogManager;
2732
import org.apache.logging.log4j.Logger;
2833
import org.opensearch.action.search.SearchPhaseContext;
@@ -32,6 +37,7 @@
3237
import org.opensearch.action.search.SearchTask;
3338
import org.opensearch.cluster.service.ClusterService;
3439
import org.opensearch.common.inject.Inject;
40+
import org.opensearch.core.index.Index;
3541
import org.opensearch.core.tasks.resourcetracker.TaskResourceInfo;
3642
import org.opensearch.plugin.insights.core.metrics.OperationalMetric;
3743
import org.opensearch.plugin.insights.core.metrics.OperationalMetricsCounter;
@@ -42,7 +48,9 @@
4248
import org.opensearch.plugin.insights.rules.model.MetricType;
4349
import org.opensearch.plugin.insights.rules.model.SearchQueryRecord;
4450
import org.opensearch.plugin.insights.settings.QueryInsightsSettings;
51+
import org.opensearch.search.builder.SearchSourceBuilder;
4552
import org.opensearch.tasks.Task;
53+
import reactor.util.annotation.NonNull;
4654

4755
/**
4856
* The listener for query insights services.
@@ -58,6 +66,7 @@ public final class QueryInsightsListener extends SearchRequestOperationsListener
5866
private boolean groupingFieldNameEnabled;
5967
private boolean groupingFieldTypeEnabled;
6068
private final QueryShapeGenerator queryShapeGenerator;
69+
private Set<Pattern> excludedIndicesPattern;
6170

6271
/**
6372
* Constructor for QueryInsightsListener
@@ -134,6 +143,15 @@ public QueryInsightsListener(
134143
this.queryInsightsService.validateMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N));
135144
this.queryInsightsService.setMaximumGroups(clusterService.getClusterSettings().get(TOP_N_QUERIES_MAX_GROUPS_EXCLUDING_N));
136145

146+
clusterService.getClusterSettings()
147+
.addSettingsUpdateConsumer(
148+
QueryInsightsSettings.TOP_N_QUERIES_EXCLUDED_INDICES,
149+
this::setExcludedIndices,
150+
this::validateExcludedIndices
151+
);
152+
validateExcludedIndices(clusterService.getClusterSettings().get(TOP_N_QUERIES_EXCLUDED_INDICES));
153+
setExcludedIndices(clusterService.getClusterSettings().get(TOP_N_QUERIES_EXCLUDED_INDICES));
154+
137155
// Internal settings for grouping attributes
138156
clusterService.getClusterSettings().addSettingsUpdateConsumer(TOP_N_QUERIES_GROUPING_FIELD_NAME, this::setGroupingFieldNameEnabled);
139157
setGroupingFieldNameEnabled(clusterService.getClusterSettings().get(TOP_N_QUERIES_GROUPING_FIELD_NAME));
@@ -147,6 +165,13 @@ public QueryInsightsListener(
147165
setSearchQueryMetricsEnabled(clusterService.getClusterSettings().get(SEARCH_QUERY_METRICS_ENABLED_SETTING));
148166
}
149167

168+
private void setExcludedIndices(List<String> excludedIndices) {
169+
this.excludedIndicesPattern = excludedIndices.stream()
170+
.map(index -> index.contains("*") ? index.replace("*", ".*") : index)
171+
.map(Pattern::compile)
172+
.collect(Collectors.toSet());
173+
}
174+
150175
/**
151176
* Enable or disable top queries insights collection for {@link MetricType}.
152177
* This function will enable or disable the corresponding listeners
@@ -221,9 +246,35 @@ public void onRequestFailure(final SearchPhaseContext context, final SearchReque
221246
constructSearchQueryRecord(context, searchRequestContext);
222247
}
223248

224-
private void constructSearchQueryRecord(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
249+
private boolean skipSearchRequest(final SearchRequestContext searchRequestContext) {
225250
// Skip profile queries
226-
if (searchRequestContext.getRequest().source().profile()) {
251+
if (Optional.ofNullable(searchRequestContext)
252+
.map(SearchRequestContext::getRequest)
253+
.map(SearchRequest::source)
254+
.map(SearchSourceBuilder::profile)
255+
.orElse(false)) {
256+
return true;
257+
}
258+
259+
if (excludedIndicesPattern.isEmpty()) {
260+
return false;
261+
}
262+
263+
return Optional.ofNullable(searchRequestContext)
264+
.map(SearchRequestContext::getSuccessfulSearchShardIndices)
265+
.map(indices -> indices.stream().map(Index::getName).anyMatch(this::matchedExcludedIndices))
266+
.orElse(false);
267+
}
268+
269+
private boolean matchedExcludedIndices(String indexName) {
270+
if (indexName == null || excludedIndicesPattern == null) {
271+
return false;
272+
}
273+
return excludedIndicesPattern.stream().anyMatch(pattern -> pattern.matcher(indexName).matches());
274+
}
275+
276+
private void constructSearchQueryRecord(final SearchPhaseContext context, final SearchRequestContext searchRequestContext) {
277+
if (skipSearchRequest(searchRequestContext)) {
227278
return;
228279
}
229280

@@ -307,4 +358,22 @@ private void constructSearchQueryRecord(final SearchPhaseContext context, final
307358
}
308359
}
309360

361+
/**
362+
* Validate the index name for excluded indices
363+
* @param excludedIndices list of index to validate
364+
*/
365+
public void validateExcludedIndices(@NonNull List<String> excludedIndices) {
366+
for (String index : excludedIndices) {
367+
if (index == null) {
368+
throw new IllegalArgumentException("Excluded index name cannot be null.");
369+
}
370+
if (index.isBlank()) {
371+
throw new IllegalArgumentException("Excluded index name cannot be blank.");
372+
}
373+
if (index.chars().anyMatch(Character::isUpperCase)) {
374+
throw new IllegalArgumentException("Index name must be lowercase.");
375+
}
376+
}
377+
}
378+
310379
}

‎src/main/java/org/opensearch/plugin/insights/core/service/QueryInsightsService.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,6 @@ public QueryInsightsService(
197197
this.setExporterDeleteAfterAndDelete(clusterService.getClusterSettings().get(TOP_N_EXPORTER_DELETE_AFTER));
198198
this.setExporterAndReaderType(SinkType.parse(clusterService.getClusterSettings().get(TOP_N_EXPORTER_TYPE)));
199199
this.setTemplatePriority(clusterService.getClusterSettings().get(TOP_N_EXPORTER_TEMPLATE_PRIORITY));
200-
201200
this.searchQueryCategorizer = SearchQueryCategorizer.getInstance(metricsRegistry);
202201
this.enableSearchQueryMetricsFeature(false);
203202
this.groupingType = DEFAULT_GROUPING_TYPE;

‎src/main/java/org/opensearch/plugin/insights/settings/QueryInsightsSettings.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,12 @@
99
package org.opensearch.plugin.insights.settings;
1010

1111
import java.util.Arrays;
12+
import java.util.Collections;
1213
import java.util.HashSet;
14+
import java.util.List;
1315
import java.util.Set;
1416
import java.util.concurrent.TimeUnit;
17+
import java.util.function.Function;
1518
import org.opensearch.common.settings.Setting;
1619
import org.opensearch.common.unit.TimeValue;
1720
import org.opensearch.plugin.insights.core.exporter.SinkType;
@@ -281,6 +284,17 @@ public class QueryInsightsSettings {
281284
Setting.Property.Dynamic
282285
);
283286

287+
/**
288+
* Settings for the list of indices that excluded from top queries.
289+
*/
290+
public static final Setting<List<String>> TOP_N_QUERIES_EXCLUDED_INDICES = Setting.listSetting(
291+
TOP_N_QUERIES_SETTING_PREFIX + ".excluded_indices",
292+
Collections.emptyList(),
293+
Function.identity(),
294+
Setting.Property.NodeScope,
295+
Setting.Property.Dynamic
296+
);
297+
284298
/**
285299
* Index pattern glob for matching top query indices
286300
*/

‎src/test/java/org/opensearch/plugin/insights/QueryInsightsPluginTests.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ public void testGetSettings() {
8585
QueryInsightsSettings.TOP_N_EXPORTER_DELETE_AFTER,
8686
QueryInsightsSettings.TOP_N_EXPORTER_TYPE,
8787
QueryInsightsSettings.TOP_N_EXPORTER_TEMPLATE_PRIORITY,
88+
QueryInsightsSettings.TOP_N_QUERIES_EXCLUDED_INDICES,
8889
QueryCategorizationSettings.SEARCH_QUERY_FIELD_TYPE_CACHE_SIZE_KEY
8990
),
9091
queryInsightsPlugin.getSettings()

‎src/test/java/org/opensearch/plugin/insights/QueryInsightsRestTestCase.java‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@
5959

6060
public abstract class QueryInsightsRestTestCase extends OpenSearchRestTestCase {
6161
protected static final String QUERY_INSIGHTS_INDICES_PREFIX = "top_queries";
62+
private static final String DEFAULT_KEYWORD = "timestamp";
6263
private static final Logger logger = Logger.getLogger(QueryInsightsRestTestCase.class.getName());
6364
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).withZone(ZoneOffset.UTC);
6465

@@ -254,6 +255,15 @@ protected void doSearch(int times) throws IOException {
254255
}
255256
}
256257

258+
protected void doSearch(int times, String indices) throws IOException {
259+
for (int i = 0; i < times; i++) {
260+
Request request = new Request("GET", "/" + indices + "/_search?size=20&pretty");
261+
request.setJsonEntity(searchBody());
262+
Response response = client().performRequest(request);
263+
Assert.assertEquals(200, response.getStatusLine().getStatusCode());
264+
}
265+
}
266+
257267
protected void doSearch(String queryType, int times) throws IOException {
258268
for (int i = 0; i < times; i++) {
259269
// Do Search
@@ -303,9 +313,9 @@ private String searchBody(String queryType) {
303313
}
304314
}
305315

306-
protected int countTopQueries(String json) {
316+
protected int countTopQueries(String json, String keyword) {
307317
// Basic pattern to match JSON array elements in `top_queries`
308-
Pattern pattern = Pattern.compile("\\{\\s*\"timestamp\"");
318+
Pattern pattern = Pattern.compile("\\{\\s*\"" + keyword + "\"");
309319
Matcher matcher = pattern.matcher(json);
310320

311321
int count = 0;
@@ -332,7 +342,7 @@ protected void waitForEmptyTopQueriesResponse() throws IOException, InterruptedE
332342

333343
String responseBody = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8);
334344

335-
if (countTopQueries(responseBody) == 0) {
345+
if (countTopQueries(responseBody, DEFAULT_KEYWORD) == 0) {
336346
isEmpty = true;
337347
} else {
338348
Thread.sleep(1000); // Sleep before retrying
@@ -345,14 +355,18 @@ protected void waitForEmptyTopQueriesResponse() throws IOException, InterruptedE
345355
}
346356

347357
protected void assertTopQueriesCount(int expectedTopQueriesCount, String type) throws IOException, InterruptedException {
358+
assertTopQueriesCount(expectedTopQueriesCount, type, DEFAULT_KEYWORD);
359+
}
360+
361+
protected void assertTopQueriesCount(int expectedTopQueriesCount, String type, String index) throws IOException, InterruptedException {
348362
// Ensure records are drained to the top queries service
349363
Thread.sleep(QueryInsightsSettings.QUERY_RECORD_QUEUE_DRAIN_INTERVAL.millis());
350364

351365
// run five times to make sure the records are drained to the top queries services
352366
for (int i = 0; i < 5; i++) {
353367
String responseBody = getTopQueries(type);
354368

355-
int topNArraySize = countTopQueries(responseBody);
369+
int topNArraySize = countTopQueries(responseBody, index);
356370

357371
if (topNArraySize < expectedTopQueriesCount) {
358372
// Ensure records are drained to the top queries service

‎src/test/java/org/opensearch/plugin/insights/QueryInsightsTestUtils.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@ public static void registerAllQueryInsightsSettings(ClusterSettings clusterSetti
358358
clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_GROUPING_FIELD_TYPE);
359359
clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_EXPORTER_DELETE_AFTER);
360360
clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_EXPORTER_TEMPLATE_PRIORITY);
361+
clusterSettings.registerSetting(QueryInsightsSettings.TOP_N_QUERIES_EXCLUDED_INDICES);
361362
clusterSettings.registerSetting(QueryCategorizationSettings.SEARCH_QUERY_METRICS_ENABLED_SETTING);
362363
}
363364
}

0 commit comments

Comments
 (0)