-
Notifications
You must be signed in to change notification settings - Fork 1.8k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Categorize search queries by type and log query shape (#10724)
* Search Query Categorizor initial skeleton using QueryBuilderVisitor Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Integrate metrics framework, add counters and log query shape Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Update changelog Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Add level attribute to QueryBuilderVisitor and as a tag in Counters Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Log query shape as debug log Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Integrate metrics framework, refactor code and update tests Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Fix build Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Add javadocs Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Minor fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Spotless check changes Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Address comments, add agg and sort counters, add feature flag, refactoring Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Build fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * spotless check Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Fix tests Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Dynamic feature flag with callback Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Minor fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Add initialization in callback Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Address comments Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Add exception handling Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Refactoring and renaming Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Minor fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Fix changelog and minor refactoring Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Address review comments Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Add unit tests Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Address review comments and add complex query unit test Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Add sort order as a tag to sort counter Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Address review comments Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Address final comments Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Build fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Fix build tests failure Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Minor fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Minor fix Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Empty commit Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> * Remove extra newline Signed-off-by: Michael Froh <froh@amazon.com> * Empty commit Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> --------- Signed-off-by: Siddhant Deshmukh <deshsid@amazon.com> Signed-off-by: Michael Froh <froh@amazon.com> Co-authored-by: Michael Froh <froh@amazon.com>
- Loading branch information
Showing
10 changed files
with
656 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
81 changes: 81 additions & 0 deletions
81
server/src/main/java/org/opensearch/action/search/SearchQueryCategorizer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,81 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.search; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.opensearch.index.query.QueryBuilder; | ||
| import org.opensearch.index.query.QueryBuilderVisitor; | ||
| import org.opensearch.index.query.QueryShapeVisitor; | ||
| import org.opensearch.search.aggregations.AggregatorFactories; | ||
| import org.opensearch.search.builder.SearchSourceBuilder; | ||
| import org.opensearch.search.sort.SortBuilder; | ||
| import org.opensearch.telemetry.metrics.MetricsRegistry; | ||
| import org.opensearch.telemetry.metrics.tags.Tags; | ||
|
|
||
| import java.util.List; | ||
| import java.util.ListIterator; | ||
|
|
||
| /** | ||
| * Class to categorize the search queries based on the type and increment the relevant counters. | ||
| * Class also logs the query shape. | ||
| */ | ||
| final class SearchQueryCategorizer { | ||
|
|
||
| private static final Logger log = LogManager.getLogger(SearchQueryCategorizer.class); | ||
|
|
||
| final SearchQueryCounters searchQueryCounters; | ||
|
|
||
| public SearchQueryCategorizer(MetricsRegistry metricsRegistry) { | ||
| searchQueryCounters = new SearchQueryCounters(metricsRegistry); | ||
| } | ||
|
|
||
| public void categorize(SearchSourceBuilder source) { | ||
| QueryBuilder topLevelQueryBuilder = source.query(); | ||
|
|
||
| logQueryShape(topLevelQueryBuilder); | ||
| incrementQueryTypeCounters(topLevelQueryBuilder); | ||
| incrementQueryAggregationCounters(source.aggregations()); | ||
| incrementQuerySortCounters(source.sorts()); | ||
| } | ||
|
|
||
| private void incrementQuerySortCounters(List<SortBuilder<?>> sorts) { | ||
| if (sorts != null && sorts.size() > 0) { | ||
| for (ListIterator<SortBuilder<?>> it = sorts.listIterator(); it.hasNext();) { | ||
| SortBuilder sortBuilder = it.next(); | ||
| String sortOrder = sortBuilder.order().toString(); | ||
| searchQueryCounters.sortCounter.add(1, Tags.create().addTag("sort_order", sortOrder)); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private void incrementQueryAggregationCounters(AggregatorFactories.Builder aggregations) { | ||
| if (aggregations != null) { | ||
| searchQueryCounters.aggCounter.add(1); | ||
| } | ||
| } | ||
|
|
||
| private void incrementQueryTypeCounters(QueryBuilder topLevelQueryBuilder) { | ||
| if (topLevelQueryBuilder == null) { | ||
| return; | ||
| } | ||
| QueryBuilderVisitor searchQueryVisitor = new SearchQueryCategorizingVisitor(searchQueryCounters); | ||
| topLevelQueryBuilder.visit(searchQueryVisitor); | ||
| } | ||
|
|
||
| private void logQueryShape(QueryBuilder topLevelQueryBuilder) { | ||
| if (topLevelQueryBuilder == null) { | ||
| return; | ||
| } | ||
| QueryShapeVisitor shapeVisitor = new QueryShapeVisitor(); | ||
| topLevelQueryBuilder.visit(shapeVisitor); | ||
| log.debug("Query shape : {}", shapeVisitor.prettyPrintTree(" ")); | ||
| } | ||
|
|
||
| } |
73 changes: 73 additions & 0 deletions
73
server/src/main/java/org/opensearch/action/search/SearchQueryCategorizingVisitor.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.search; | ||
|
|
||
| import org.apache.lucene.search.BooleanClause; | ||
| import org.opensearch.index.query.BoolQueryBuilder; | ||
| import org.opensearch.index.query.MatchPhraseQueryBuilder; | ||
| import org.opensearch.index.query.MatchQueryBuilder; | ||
| import org.opensearch.index.query.MultiMatchQueryBuilder; | ||
| import org.opensearch.index.query.QueryBuilder; | ||
| import org.opensearch.index.query.QueryBuilderVisitor; | ||
| import org.opensearch.index.query.QueryStringQueryBuilder; | ||
| import org.opensearch.index.query.RangeQueryBuilder; | ||
| import org.opensearch.index.query.RegexpQueryBuilder; | ||
| import org.opensearch.index.query.TermQueryBuilder; | ||
| import org.opensearch.index.query.WildcardQueryBuilder; | ||
| import org.opensearch.index.query.functionscore.FunctionScoreQueryBuilder; | ||
| import org.opensearch.telemetry.metrics.tags.Tags; | ||
|
|
||
| /** | ||
| * Class to visit the querybuilder tree and also track the level information. | ||
| * Increments the counters related to Search Query type. | ||
| */ | ||
| final class SearchQueryCategorizingVisitor implements QueryBuilderVisitor { | ||
| private static final String LEVEL_TAG = "level"; | ||
| private final int level; | ||
| private final SearchQueryCounters searchQueryCounters; | ||
|
|
||
| public SearchQueryCategorizingVisitor(SearchQueryCounters searchQueryCounters) { | ||
| this(searchQueryCounters, 0); | ||
| } | ||
|
|
||
| private SearchQueryCategorizingVisitor(SearchQueryCounters counters, int level) { | ||
| this.searchQueryCounters = counters; | ||
| this.level = level; | ||
| } | ||
|
|
||
| public void accept(QueryBuilder qb) { | ||
| if (qb instanceof BoolQueryBuilder) { | ||
| searchQueryCounters.boolCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof FunctionScoreQueryBuilder) { | ||
| searchQueryCounters.functionScoreCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof MatchQueryBuilder) { | ||
| searchQueryCounters.matchCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof MatchPhraseQueryBuilder) { | ||
| searchQueryCounters.matchPhrasePrefixCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof MultiMatchQueryBuilder) { | ||
| searchQueryCounters.multiMatchCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof QueryStringQueryBuilder) { | ||
| searchQueryCounters.queryStringQueryCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof RangeQueryBuilder) { | ||
| searchQueryCounters.rangeCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof RegexpQueryBuilder) { | ||
| searchQueryCounters.regexCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof TermQueryBuilder) { | ||
| searchQueryCounters.termCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else if (qb instanceof WildcardQueryBuilder) { | ||
| searchQueryCounters.wildcardCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } else { | ||
| searchQueryCounters.otherQueryCounter.add(1, Tags.create().addTag(LEVEL_TAG, level)); | ||
| } | ||
| } | ||
|
|
||
| public QueryBuilderVisitor getChildVisitor(BooleanClause.Occur occur) { | ||
| return new SearchQueryCategorizingVisitor(searchQueryCounters, level + 1); | ||
| } | ||
| } |
117 changes: 117 additions & 0 deletions
117
server/src/main/java/org/opensearch/action/search/SearchQueryCounters.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.search; | ||
|
|
||
| import org.opensearch.telemetry.metrics.Counter; | ||
| import org.opensearch.telemetry.metrics.MetricsRegistry; | ||
|
|
||
| /** | ||
| * Class contains all the Counters related to search query types. | ||
| */ | ||
| final class SearchQueryCounters { | ||
| private static final String UNIT = "1"; | ||
| private final MetricsRegistry metricsRegistry; | ||
|
|
||
| // Counters related to Query types | ||
| public final Counter aggCounter; | ||
| public final Counter boolCounter; | ||
| public final Counter functionScoreCounter; | ||
| public final Counter matchCounter; | ||
| public final Counter matchPhrasePrefixCounter; | ||
| public final Counter multiMatchCounter; | ||
| public final Counter otherQueryCounter; | ||
| public final Counter queryStringQueryCounter; | ||
| public final Counter rangeCounter; | ||
| public final Counter regexCounter; | ||
|
|
||
| public final Counter sortCounter; | ||
| public final Counter skippedCounter; | ||
| public final Counter termCounter; | ||
| public final Counter totalCounter; | ||
| public final Counter wildcardCounter; | ||
|
|
||
| public SearchQueryCounters(MetricsRegistry metricsRegistry) { | ||
| this.metricsRegistry = metricsRegistry; | ||
| this.aggCounter = metricsRegistry.createCounter( | ||
| "search.query.type.agg.count", | ||
| "Counter for the number of top level agg search queries", | ||
| UNIT | ||
| ); | ||
| this.boolCounter = metricsRegistry.createCounter( | ||
| "search.query.type.bool.count", | ||
| "Counter for the number of top level and nested bool search queries", | ||
| UNIT | ||
| ); | ||
| this.functionScoreCounter = metricsRegistry.createCounter( | ||
| "search.query.type.functionscore.count", | ||
| "Counter for the number of top level and nested function score search queries", | ||
| UNIT | ||
| ); | ||
| this.matchCounter = metricsRegistry.createCounter( | ||
| "search.query.type.match.count", | ||
| "Counter for the number of top level and nested match search queries", | ||
| UNIT | ||
| ); | ||
| this.matchPhrasePrefixCounter = metricsRegistry.createCounter( | ||
| "search.query.type.matchphrase.count", | ||
| "Counter for the number of top level and nested match phrase prefix search queries", | ||
| UNIT | ||
| ); | ||
| this.multiMatchCounter = metricsRegistry.createCounter( | ||
| "search.query.type.multimatch.count", | ||
| "Counter for the number of top level and nested multi match search queries", | ||
| UNIT | ||
| ); | ||
| this.otherQueryCounter = metricsRegistry.createCounter( | ||
| "search.query.type.other.count", | ||
| "Counter for the number of top level and nested search queries that do not match any other categories", | ||
| UNIT | ||
| ); | ||
| this.queryStringQueryCounter = metricsRegistry.createCounter( | ||
| "search.query.type.querystringquery.count", | ||
| "Counter for the number of top level and nested queryStringQuery search queries", | ||
| UNIT | ||
| ); | ||
| this.rangeCounter = metricsRegistry.createCounter( | ||
| "search.query.type.range.count", | ||
| "Counter for the number of top level and nested range search queries", | ||
| UNIT | ||
| ); | ||
| this.regexCounter = metricsRegistry.createCounter( | ||
| "search.query.type.regex.count", | ||
| "Counter for the number of top level and nested regex search queries", | ||
| UNIT | ||
| ); | ||
| this.skippedCounter = metricsRegistry.createCounter( | ||
| "search.query.type.skipped.count", | ||
| "Counter for the number queries skipped due to error", | ||
| UNIT | ||
| ); | ||
| this.sortCounter = metricsRegistry.createCounter( | ||
| "search.query.type.sort.count", | ||
| "Counter for the number of top level sort search queries", | ||
| UNIT | ||
| ); | ||
| this.termCounter = metricsRegistry.createCounter( | ||
| "search.query.type.term.count", | ||
| "Counter for the number of top level and nested term search queries", | ||
| UNIT | ||
| ); | ||
| this.totalCounter = metricsRegistry.createCounter( | ||
| "search.query.type.total.count", | ||
| "Counter for the number of top level and nested search queries", | ||
| UNIT | ||
| ); | ||
| this.wildcardCounter = metricsRegistry.createCounter( | ||
| "search.query.type.wildcard.count", | ||
| "Counter for the number of top level and nested wildcard search queries", | ||
| UNIT | ||
| ); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.