[backend] manage long text filter like description and add search ope…

…rator #4940 (#5633)
OpenCTI-Platform · Jun 3, 2024 · d8fc6fb · d8fc6fb
1 parent e6802a8
commit d8fc6fb
Show file tree

Hide file tree

Showing 9 changed files with 343 additions and 10 deletions.
diff --git a/opencti-platform/opencti-front/lang/front/en.json b/opencti-platform/opencti-front/lang/front/en.json
@@ -2661,4 +2661,4 @@
   "Missing token": "Missing token",
   "Generative AI is a beta feature as we are currently fine-tuning our models. Consider checking important information.": "Generative AI is a beta feature as we are currently fine-tuning our models. Consider checking important information.",
   "Accept": "Accept"
-}
+}
diff --git a/opencti-platform/opencti-front/src/components/filters/FilterChipPopover.tsx b/opencti-platform/opencti-front/src/components/filters/FilterChipPopover.tsx
@@ -9,7 +9,16 @@ import { SelectChangeEvent } from '@mui/material/Select';
 import SearchScopeElement from '@components/common/lists/SearchScopeElement';
 import Chip from '@mui/material/Chip';
 import { OptionValue } from '@components/common/lists/FilterAutocomplete';
-import { dateFilters, Filter, getAvailableOperatorForFilter, getSelectedOptions, integerFilters, isStixObjectTypes, textFilters } from '../../utils/filters/filtersUtils';
+import {
+  dateFilters,
+  Filter,
+  getAvailableOperatorForFilter,
+  getSelectedOptions,
+  integerFilters,
+  isStixObjectTypes,
+  longTextFilters,
+  textFilters,
+} from '../../utils/filters/filtersUtils';
 import { useFormatter } from '../i18n';
 import ItemIcon from '../ItemIcon';
 import { getOptionsFromEntities, getUseSearch } from '../../utils/filters/SearchEntitiesUtil';
@@ -48,6 +57,7 @@ const OperatorKeyValues: {
   not_starts_with: 'Not starts with',
   ends_with: 'Ends with',
   not_ends_with: 'Not ends with',
+  search: 'Search',
 };
 
 interface BasicNumberInputProps {
@@ -204,6 +214,7 @@ export const FilterChipPopover: FunctionComponent<FilterChipMenuProps> = ({
       dateFilters.includes(fKey)
       || integerFilters.includes(fKey)
       || textFilters.includes(fKey)
+      || longTextFilters.includes(fKey)
     );
   };
 
@@ -316,7 +327,7 @@ export const FilterChipPopover: FunctionComponent<FilterChipMenuProps> = ({
         />
       );
     }
-    if (textFilters.includes(fKey)) {
+    if (textFilters.includes(fKey) || longTextFilters.includes(fKey)) {
       return (
         <BasicTextInput
           filter={filter}

diff --git a/opencti-platform/opencti-front/src/private/components/analyses/Reports.tsx b/opencti-platform/opencti-front/src/private/components/analyses/Reports.tsx
@@ -152,6 +152,7 @@ const Reports: FunctionComponent = () => {
             'created_at',
             'objects',
             'name',
+            'description',
           ]}
         >
           {queryRef && (

diff --git a/opencti-platform/opencti-front/src/schema/relay.schema.graphql b/opencti-platform/opencti-front/src/schema/relay.schema.graphql
@@ -102,6 +102,7 @@ enum FilterOperator {
   script
   nil
   not_nil
+  search
 }
 
 input FilterGroup {

diff --git a/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx b/opencti-platform/opencti-front/src/utils/filters/filtersUtils.tsx
@@ -54,11 +54,14 @@ export const integerFilters = [
 
 export const textFilters = [
   'name',
-  'description',
   'value',
   'pattern',
 ];
 
+export const longTextFilters = [
+  'description',
+];
+
 // filters that can have 'eq' or 'not_eq' operator
 export const EqFilters = [
   'objectLabel',
@@ -641,6 +644,9 @@ export const getDefaultOperatorFilter = (filterKey: string) => {
   if (textFilters.includes(filterKey)) {
     return 'starts_with';
   }
+  if (longTextFilters.includes(filterKey)) {
+    return 'search';
+  }
   return 'eq';
 };
 
@@ -689,6 +695,10 @@ export const getAvailableOperatorForFilterKey = (filterKey: string): string[] =>
     return ['eq', 'not_eq', 'nil', 'not_nil', 'contains', 'not_contains',
       'starts_with', 'not_starts_with', 'ends_with', 'not_ends_with'];
   }
+  if (longTextFilters.includes(filterKey)) {
+    return ['search'];
+  }
+
   return ['eq', 'not_eq', 'nil', 'not_nil'];
 };
 

diff --git a/opencti-platform/opencti-graphql/config/schema/opencti.graphql b/opencti-platform/opencti-graphql/config/schema/opencti.graphql
@@ -103,6 +103,7 @@ enum FilterOperator {
   script
   nil
   not_nil
+  search
 }
 
 input FilterGroup {

diff --git a/opencti-platform/opencti-graphql/src/database/engine.js b/opencti-platform/opencti-graphql/src/database/engine.js
@@ -1417,23 +1417,27 @@ const BASE_SEARCH_ATTRIBUTES = [
   'result_name',
 ];
 
-export const elGenerateFullTextSearchShould = (search, args = {}) => {
+function processSearch(search, args) {
   const { useWildcardPrefix = ES_DEFAULT_WILDCARD_PREFIX } = args;
   let decodedSearch;
   try {
-    decodedSearch = decodeURIComponent(search).trim();
+    decodedSearch = decodeURIComponent(search)
+      .trim();
   } catch (e) {
     decodedSearch = search.trim();
   }
   let remainingSearch = decodedSearch;
   const exactSearch = (decodedSearch.match(/"[^"]+"/g) || []) //
-    .filter((e) => isNotEmptyField(e.replace(/"/g, '').trim()));
+    .filter((e) => isNotEmptyField(e.replace(/"/g, '')
+      .trim()));
   for (let index = 0; index < exactSearch.length; index += 1) {
     remainingSearch = remainingSearch.replace(exactSearch[index], '');
   }
   const querySearch = [];
 
-  const partialSearch = remainingSearch.replace(/"/g, '').trim().split(' ');
+  const partialSearch = remainingSearch.replace(/"/g, '')
+    .trim()
+    .split(' ');
 
   for (let searchIndex = 0; searchIndex < partialSearch.length; searchIndex += 1) {
     const partialElement = partialSearch[searchIndex];
@@ -1445,6 +1449,14 @@ export const elGenerateFullTextSearchShould = (search, args = {}) => {
       }
     }
   }
+  return {
+    exactSearch,
+    querySearch
+  };
+}
+
+export const elGenerateFullTextSearchShould = (search, args = {}) => {
+  const { exactSearch, querySearch } = processSearch(search, args);
   // Return the elastic search engine expected bool should terms
   // Build the search for all exact match (between double quotes)
   const shouldSearch = [];
@@ -1522,6 +1534,47 @@ export const elGenerateFullTextSearchShould = (search, args = {}) => {
   return shouldSearch;
 };
 
+export const elGenerateFieldTextSearchShould = (search, arrayKeys, args = {}) => {
+  const { exactSearch, querySearch } = processSearch(search, args);
+  const cleanExactSearch = R.uniq(exactSearch.map((e) => e.replace(/"|http?:/g, '')));
+  const shouldSearch = [];
+  shouldSearch.push(
+    ...cleanExactSearch.map((ex) => [
+      {
+        multi_match: {
+          type: 'phrase',
+          query: ex,
+          lenient: true,
+          fields: arrayKeys,
+        },
+      }
+    ]).flat()
+  );
+  // Build the search for all other fields
+  const searchPhrase = R.uniq(querySearch).join(' ');
+  if (searchPhrase) {
+    shouldSearch.push(...[
+      {
+        query_string: {
+          query: searchPhrase,
+          analyze_wildcard: true,
+          fields: arrayKeys,
+        },
+      },
+      {
+        multi_match: {
+          type: 'phrase',
+          query: searchPhrase,
+          lenient: true,
+          fields: arrayKeys,
+        },
+      }
+    ]);
+  }
+
+  return shouldSearch;
+};
+
 const BASE_FIELDS = ['_index', 'internal_id', 'standard_id', 'sort', 'base_type', 'entity_type',
   'connections', 'first_seen', 'last_seen', 'start_time', 'stop_time'];
 
@@ -1583,7 +1636,16 @@ const buildLocalMustFilter = async (validFilter) => {
     if (arrayKeys.length > 1) {
       throw UnsupportedError('Filter must have only one field', { keys: arrayKeys });
     } else {
-      valuesFiltering.push({
+      const schemaKey = schemaAttributesDefinition.getAttributeByName(R.head(arrayKeys));
+      valuesFiltering.push(schemaKey?.type === 'string' && schemaKey?.format === 'text' ? {
+        bool: {
+          must_not: {
+            wildcard: {
+              [R.head(arrayKeys)]: '*'
+            }
+          },
+        }
+      } : {
         bool: {
           must_not: {
             exists: {
@@ -1597,7 +1659,21 @@ const buildLocalMustFilter = async (validFilter) => {
     if (arrayKeys.length > 1) {
       throw UnsupportedError('Filter must have only one field', { keys: arrayKeys });
     } else {
-      valuesFiltering.push({ exists: { field: R.head(arrayKeys) } });
+      const schemaKey = schemaAttributesDefinition.getAttributeByName(R.head(arrayKeys));
+      valuesFiltering.push(schemaKey?.type === 'string' && schemaKey?.format === 'text' ? {
+        bool: {
+          must:
+            {
+              wildcard: {
+                [R.head(arrayKeys)]: '*'
+              }
+            },
+        }
+      } : {
+        exists: {
+          field: R.head(arrayKeys)
+        }
+      });
     }
   }
   // 03. Handle values according to the operator
@@ -1666,6 +1742,15 @@ const buildLocalMustFilter = async (validFilter) => {
             script: values[i].toString()
           },
         });
+      } else if (operator === 'search') {
+        const shouldSearch = elGenerateFieldTextSearchShould(values[i].toString(), arrayKeys);
+        const bool = {
+          bool: {
+            should: shouldSearch,
+            minimum_should_match: 1,
+          },
+        };
+        valuesFiltering.push(bool);
       } else {
         if (arrayKeys.length > 1) {
           throw UnsupportedError('Filter must have only one field', { keys: arrayKeys });

diff --git a/opencti-platform/opencti-graphql/src/generated/graphql.ts b/opencti-platform/opencti-graphql/src/generated/graphql.ts
@@ -7191,6 +7191,7 @@ export enum FilterOperator {
   NotNil = 'not_nil',
   NotStartsWith = 'not_starts_with',
   Script = 'script',
+  Search = 'search',
   StartsWith = 'starts_with',
   Wildcard = 'wildcard'
 }