[Manual Backport 2.x] [ignoreFilterIfFieldNotInIndex] Parse query string filter to determine fields (#6126)

package.json

@@ -202,6 +202,8 @@
     "json-stringify-safe": "5.0.1",
     "lodash": "^4.17.21",
     "lru-cache": "^4.1.5",
+    "lucene": "^2.1.1",
+    "mathjs": "^11.8.2",
     "minimatch": "^3.0.4",
     "moment": "^2.24.0",
     "moment-timezone": "^0.5.27",
@@ -310,6 +312,7 @@
     "@types/listr": "^0.14.0",
     "@types/lodash": "^4.14.170",
     "@types/lru-cache": "^5.1.0",
+    "@types/lucene": "^2.1.7",
     "@types/markdown-it": "^0.0.7",
     "@types/minimatch": "^2.0.29",
     "@types/mocha-next": "npm:@types/mocha@^10.0.1",

src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.test.ts

@@ -28,7 +28,7 @@
  * under the License.
  */
 
-import { Filter } from '../filters';
+import { Filter, QueryStringFilter } from '../filters';
 import { filterMatchesIndex } from './filter_matches_index';
 import { IIndexPattern } from '../../index_patterns';
 
@@ -80,4 +80,34 @@ describe('filterMatchesIndex', () => {
 
     expect(filterMatchesIndex(filter, indexPattern)).toBe(true);
   });
+
+  it('should return false if a query string filter cannot be parsed', () => {
+    const filter = {
+      meta: { key: 'query', type: 'query_string' },
+      query: { query_string: { query: 'foo"bar' } },
+    } as QueryStringFilter;
+    const indexPattern = { id: 'bar', fields: [{ name: 'foo' }] } as IIndexPattern;
+
+    expect(filterMatchesIndex(filter, indexPattern)).toBe(false);
+  });
+
+  it('should return true if a query string filter references fields in an index', () => {
+    const filter = {
+      meta: { key: 'query', type: 'query_string' },
+      query: { query_string: { query: 'foo: bar AND baz: firzle' } },
+    } as QueryStringFilter;
+    const indexPattern = { id: 'bar', fields: [{ name: 'foo' }, { name: 'baz' }] } as IIndexPattern;
+
+    expect(filterMatchesIndex(filter, indexPattern)).toBe(true);
+  });
+
+  it('should return false if a query string filter references fields not in an index', () => {
+    const filter = {
+      meta: { key: 'query', type: 'query_string' },
+      query: { query_string: { query: 'foo: bar AND baz: firzle' } },
+    } as QueryStringFilter;
+    const indexPattern = { id: 'bar', fields: [{ name: 'foo' }] } as IIndexPattern;
+
+    expect(filterMatchesIndex(filter, indexPattern)).toBe(false);
+  });
 });

src/plugins/data/common/opensearch_query/opensearch_query/filter_matches_index.ts

@@ -28,14 +28,58 @@
  * under the License.
  */
 
+import { uniq } from 'lodash';
+import { parse, AST } from 'lucene';
 import { IIndexPattern, IFieldType } from '../../index_patterns';
-import { Filter } from '../filters';
+import { Filter, QueryStringFilter } from '../filters';
+
+const implicitLuceneField = '<implicit>';
+
+function getLuceneFields(ast: AST): string[] {
+  const fields: string[] = [];
+
+  // Parse left side of AST (if it exists)
+  if ('left' in ast && ast.left) {
+    if ('field' in ast.left) {
+      if (ast.left.field && ast.left.field !== implicitLuceneField) {
+        fields.push(ast.left.field);
+      }
+    } else {
+      fields.push(...getLuceneFields(ast.left));
+    }
+  }
+
+  // Parse right side of AST (if it exists)
+  if ('right' in ast && ast.right) {
+    if ('field' in ast.right) {
+      if (ast.right.field && ast.right.field !== implicitLuceneField) {
+        fields.push(ast.right.field);
+      }
+    } else {
+      fields.push(...getLuceneFields(ast.right));
+    }
+  }
+  return fields;
+}
 
 export function filterMatchesIndex(filter: Filter, indexPattern?: IIndexPattern | null) {
   if (!filter.meta?.key || !indexPattern) {
     return true;
   }
 
+  if (filter.meta?.type === 'query_string') {
+    const qsFilter = filter as QueryStringFilter;
+    try {
+      const ast = parse(qsFilter.query.query_string.query);
+      const filterFields = uniq(getLuceneFields(ast));
+      return filterFields.every((filterField) =>
+        indexPattern.fields.some((field: IFieldType) => field.name === filterField)
+      );
+    } catch {
+      return false;
+    }
+  }
+
   if (filter.meta?.type === 'custom') {
     return filter.meta.index === indexPattern.id;
   }

src/plugins/data/common/opensearch_query/opensearch_query/index.ts

@@ -33,3 +33,4 @@ export { buildQueryFromFilters } from './from_filters';
 export { luceneStringToDsl } from './lucene_string_to_dsl';
 export { decorateQuery } from './decorate_query';
 export { getOpenSearchQueryConfig } from './get_opensearch_query_config';
+export { filterMatchesIndex } from './filter_matches_index';

src/plugins/data/public/ui/filter_bar/filter_item.tsx

@@ -44,6 +44,7 @@ import {
   toggleFilterPinned,
   toggleFilterDisabled,
   getIndexPatternFromFilter,
+  filterMatchesIndex,
 } from '../../../common';
 import { getIndexPatterns } from '../../services';
 
@@ -268,11 +269,7 @@ export function FilterItem(props: Props) {
     const ip = getIndexPatternFromFilter(filter, indexPatterns);
     if (ip) return true;
 
-    const allFields = indexPatterns.map((indexPattern) => {
-      return indexPattern.fields.map((field) => field.name);
-    });
-    const flatFields = allFields.reduce((acc: string[], it: string[]) => [...acc, ...it], []);
-    return flatFields.includes(filter.meta?.key || '');
+    return indexPatterns.some((indexPattern) => filterMatchesIndex(filter, indexPattern));
   }
 
   function getValueLabel(): LabelOptions {

yarn.lock

@@ -3403,6 +3403,11 @@
   resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.1.tgz#c48c2e27b65d2a153b19bfc1a317e30872e01eef"
   integrity sha512-ssE3Vlrys7sdIzs5LOxCzTVMsU7i9oa/IaW92wF32JFb3CVczqOkru2xspuKczHEbG3nvmPY7IFqVmGGHdNbYw==
 
+"@types/lucene@^2.1.7":
+  version "2.1.7"
+  resolved "https://registry.yarnpkg.com/@types/lucene/-/lucene-2.1.7.tgz#fbdea914c5b7d91fd164664ccc6019ed210e729b"
+  integrity sha512-i3J0OV0RoJSskOJUa76Hgz09deabWwfJajsUxc1M05HryjPpPEKqtRklKe0+O0XVhdrFIiFO1/SInXpDCacfNA==
+
 "@types/markdown-it@^0.0.7":
   version "0.0.7"
   resolved "https://registry.yarnpkg.com/@types/markdown-it/-/markdown-it-0.0.7.tgz#75070485a3d8ad11e7deb8287f4430be15bf4d39"
@@ -12248,6 +12253,11 @@ lru-queue@^0.1.0:
   dependencies:
     es5-ext "~0.10.2"
 
+lucene@^2.1.1:
+  version "2.1.1"
+  resolved "https://registry.yarnpkg.com/lucene/-/lucene-2.1.1.tgz#e710cc123b214eaf72a4c5f1da06943c0af44d86"
+  integrity sha512-l0qCX+pgXEZh/7sYQNG+vzhOIFRPjlJJkQ/irk9n7Ak3d+1MrU6F7IV31KILwFkUn153oLK8a2AIt48DzLdVPg==
+
 lz-string@^1.4.4:
   version "1.4.4"
   resolved "https://registry.yarnpkg.com/lz-string/-/lz-string-1.4.4.tgz#c0d8eaf36059f705796e1e344811cf4c498d3a26"