[7.x] [SECURITY SOLUTION] [Detections] Increase lookback when gap is detected (#68339) (#70371)

* add POC logic to modify the 'from' param in the search

* fixes formatting for appending gap diff to from

* computes new max signals based on how many intervals of rule runs were missed when gap in consecutive rule runs is detected

* adds logging, fixes bug where we could end up with negative values for diff, adds calculatedFrom to the search after query

* remove console.log and for some reason two eslint disables were added so i removed one of them

* rename variables, add test based on log message - need to figure out a better way to test this

* remove unused import

* fully re-worked the algorithm for searching discrete time periods, still need search_after because a user could submit a rule with a custom maxSignals so that would still serve a purpose. This needs heavy refactoring though, and tests.

* updated loop to include maxSignals per time interval tuple, this way we guarantee maxSignals per full rule interval. Needs some refactoring though.

* move logic into utils function, utils function still needs refactoring

* adds unit tests and cleans up new util function for determining time intervals for searching to occur

* more code cleanup

* remove more logging statements

* fix type errors

* updates unit tests and fixes bug where search result would return 0 hits but we were accessing property on non-existent hit item

* fix rebase conflict

* fixes a bug where a negative gap could exist if a rule ran before the lookback time, also fixes a bug where the search and bulk loop would return false when successful.

* gap is a duration, not a number.

* remove logging variable

* remove logging function from test

* fix type import from rebase with master

* updates missed test when rebased with master, removes unused import

* modify log statements to include meta information for logged rule events, adds tests

* remove unnecessary ts-ignores

* indentation on stringify

* adds a test to ensure we are parsing the elapsed time correctly

Author: dhurley14

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.test.ts

@@ -6,14 +6,20 @@
 
 import uuid from 'uuid';
 import { filterEventsAgainstList } from './filter_events_with_list';
+import { buildRuleMessageFactory } from './rule_messages';
 import { mockLogger, repeatedSearchResultsWithSortId } from './__mocks__/es_results';
 
 import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
 import { getListItemResponseMock } from '../../../../../lists/common/schemas/response/list_item_schema.mock';
 import { listMock } from '../../../../../lists/server/mocks';
 
 const someGuids = Array.from({ length: 13 }).map((x) => uuid.v4());
-
+const buildRuleMessage = buildRuleMessageFactory({
+  id: 'fake id',
+  ruleId: 'fake rule id',
+  index: 'fakeindex',
+  name: 'fake name',
+});
 describe('filterEventsAgainstList', () => {
   let listClient = listMock.getListClient();
   beforeEach(() => {
@@ -33,6 +39,7 @@ describe('filterEventsAgainstList', () => {
         '3.3.3.3',
         '7.7.7.7',
       ]),
+      buildRuleMessage,
     });
     expect(res.hits.hits.length).toEqual(4);
   });
@@ -57,6 +64,7 @@ describe('filterEventsAgainstList', () => {
         listClient,
         exceptionsList: [exceptionItem],
         eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)),
+        buildRuleMessage,
       });
       expect(res.hits.hits.length).toEqual(4);
     });
@@ -91,6 +99,7 @@ describe('filterEventsAgainstList', () => {
           '3.3.3.3',
           '7.7.7.7',
         ]),
+        buildRuleMessage,
       });
       expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip');
       expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual(
@@ -118,6 +127,7 @@ describe('filterEventsAgainstList', () => {
         listClient,
         exceptionsList: [exceptionItem],
         eventSearchResult: repeatedSearchResultsWithSortId(4, 4, someGuids.slice(0, 3)),
+        buildRuleMessage,
       });
       expect(res.hits.hits.length).toEqual(0);
     });
@@ -152,6 +162,7 @@ describe('filterEventsAgainstList', () => {
           '3.3.3.3',
           '7.7.7.7',
         ]),
+        buildRuleMessage,
       });
       expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].type).toEqual('ip');
       expect((listClient.getListItemByValues as jest.Mock).mock.calls[0][0].listId).toEqual(

x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_events_with_list.ts

@@ -8,6 +8,7 @@ import { Logger } from 'src/core/server';
 
 import { ListClient } from '../../../../../lists/server';
 import { SignalSearchResponse, SearchTypes } from './types';
+import { BuildRuleMessage } from './rule_messages';
 import {
   entriesList,
   EntryList,
@@ -19,16 +20,20 @@ interface FilterEventsAgainstList {
   exceptionsList: ExceptionListItemSchema[];
   logger: Logger;
   eventSearchResult: SignalSearchResponse;
+  buildRuleMessage: BuildRuleMessage;
 }
 
 export const filterEventsAgainstList = async ({
   listClient,
   exceptionsList,
   logger,
   eventSearchResult,
+  buildRuleMessage,
 }: FilterEventsAgainstList): Promise<SignalSearchResponse> => {
   try {
+    logger.debug(buildRuleMessage(`exceptionsList: ${JSON.stringify(exceptionsList, null, 2)}`));
     if (exceptionsList == null || exceptionsList.length === 0) {
+      logger.debug(buildRuleMessage('about to return original search result'));
       return eventSearchResult;
     }
 
@@ -86,7 +91,7 @@ export const filterEventsAgainstList = async ({
               return false;
             });
             const diff = eventSearchResult.hits.hits.length - filteredEvents.length;
-            logger.debug(`Lists filtered out ${diff} events`);
+            logger.debug(buildRuleMessage(`Lists filtered out ${diff} events`));
             return filteredEvents;
           });