[7.x] [Security Solution] Converge detection engine on single schema representation (#96186)

x-pack/plugins/alerting/server/saved_objects/migrations.test.ts

@@ -699,6 +699,282 @@ describe('7.11.2', () => {
   });
 });
 
+describe('7.13.0', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+    encryptedSavedObjectsSetup.createMigration.mockImplementation(
+      (shouldMigrateWhenPredicate, migration) => migration
+    );
+  });
+  test('security solution alerts get migrated and remove null values', () => {
+    const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
+    const alert = getMockData({
+      alertTypeId: 'siem.signals',
+      params: {
+        author: ['Elastic'],
+        buildingBlockType: null,
+        description:
+          "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.",
+        ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3',
+        index: ['packetbeat-*'],
+        falsePositives: [
+          "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.",
+        ],
+        from: 'now-6m',
+        immutable: true,
+        query:
+          'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us',
+        language: 'lucene',
+        license: 'Elastic License',
+        outputIndex: '.siem-signals-rylandherrick_2-default',
+        savedId: null,
+        timelineId: null,
+        timelineTitle: null,
+        meta: null,
+        filters: null,
+        maxSignals: 100,
+        riskScore: 73,
+        riskScoreMapping: [],
+        ruleNameOverride: null,
+        severity: 'high',
+        severityMapping: null,
+        threat: null,
+        threatFilters: null,
+        timestampOverride: null,
+        to: 'now',
+        type: 'query',
+        references: [
+          'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html',
+        ],
+        note:
+          'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.',
+        version: 1,
+        exceptionsList: null,
+        threshold: {
+          field: null,
+          value: 5,
+        },
+      },
+    });
+
+    expect(migration713(alert, migrationContext)).toEqual({
+      ...alert,
+      attributes: {
+        ...alert.attributes,
+        params: {
+          author: ['Elastic'],
+          description:
+            "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.",
+          ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3',
+          index: ['packetbeat-*'],
+          falsePositives: [
+            "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.",
+          ],
+          from: 'now-6m',
+          immutable: true,
+          query:
+            'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us',
+          language: 'lucene',
+          license: 'Elastic License',
+          outputIndex: '.siem-signals-rylandherrick_2-default',
+          maxSignals: 100,
+          riskScore: 73,
+          riskScoreMapping: [],
+          severity: 'high',
+          severityMapping: [],
+          threat: [],
+          to: 'now',
+          type: 'query',
+          references: [
+            'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html',
+          ],
+          note:
+            'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.',
+          version: 1,
+          exceptionsList: [],
+          threshold: {
+            field: [],
+            value: 5,
+            cardinality: [],
+          },
+        },
+      },
+    });
+  });
+
+  test('non-null values in security solution alerts are not modified', () => {
+    const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
+    const alert = getMockData({
+      alertTypeId: 'siem.signals',
+      params: {
+        author: ['Elastic'],
+        buildingBlockType: 'default',
+        description:
+          "This rule detects a known command and control pattern in network events. The FIN7 threat group is known to use this command and control technique, while maintaining persistence in their target's network.",
+        ruleId: '4a4e23cf-78a2-449c-bac3-701924c269d3',
+        index: ['packetbeat-*'],
+        falsePositives: [
+          "This rule could identify benign domains that are formatted similarly to FIN7's command and control algorithm. Alerts should be investigated by an analyst to assess the validity of the individual observations.",
+        ],
+        from: 'now-6m',
+        immutable: true,
+        query:
+          'event.category:(network OR network_traffic) AND type:(tls OR http) AND network.transport:tcp AND destination.domain:/[a-zA-Z]{4,5}.(pw|us|club|info|site|top)/ AND NOT destination.domain:zoom.us',
+        language: 'lucene',
+        license: 'Elastic License',
+        outputIndex: '.siem-signals-rylandherrick_2-default',
+        savedId: 'saved-id',
+        timelineId: 'timeline-id',
+        timelineTitle: 'timeline-title',
+        meta: {
+          field: 'value',
+        },
+        filters: ['filters'],
+        maxSignals: 100,
+        riskScore: 73,
+        riskScoreMapping: ['risk-score-mapping'],
+        ruleNameOverride: 'field.name',
+        severity: 'high',
+        severityMapping: ['severity-mapping'],
+        threat: [
+          {
+            framework: 'MITRE ATT&CK',
+            tactic: {
+              id: 'TA0011',
+              name: 'Command and Control',
+              reference: 'https://attack.mitre.org/tactics/TA0011/',
+            },
+            technique: [
+              {
+                id: 'T1483',
+                name: 'Domain Generation Algorithms',
+                reference: 'https://attack.mitre.org/techniques/T1483/',
+              },
+            ],
+          },
+        ],
+        threatFilters: ['threat-filter'],
+        timestampOverride: 'event.ingested',
+        to: 'now',
+        type: 'query',
+        references: [
+          'https://www.fireeye.com/blog/threat-research/2018/08/fin7-pursuing-an-enigmatic-and-evasive-global-criminal-operation.html',
+        ],
+        note:
+          'In the event this rule identifies benign domains in your environment, the `destination.domain` field in the rule can be modified to include those domains. Example: `...AND NOT destination.domain:(zoom.us OR benign.domain1 OR benign.domain2)`.',
+        version: 1,
+        exceptionsList: ['exceptions-list'],
+      },
+    });
+
+    expect(migration713(alert, migrationContext)).toEqual(alert);
+  });
+
+  test('security solution threshold alert with string in threshold.field is migrated to array', () => {
+    const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
+    const alert = getMockData({
+      alertTypeId: 'siem.signals',
+      params: {
+        threshold: {
+          field: 'host.id',
+          value: 5,
+        },
+      },
+    });
+
+    expect(migration713(alert, migrationContext)).toEqual({
+      ...alert,
+      attributes: {
+        ...alert.attributes,
+        params: {
+          threshold: {
+            field: ['host.id'],
+            value: 5,
+            cardinality: [],
+          },
+          exceptionsList: [],
+          riskScoreMapping: [],
+          severityMapping: [],
+          threat: [],
+        },
+      },
+    });
+  });
+
+  test('security solution threshold alert with empty string in threshold.field is migrated to empty array', () => {
+    const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
+    const alert = getMockData({
+      alertTypeId: 'siem.signals',
+      params: {
+        threshold: {
+          field: '',
+          value: 5,
+        },
+      },
+    });
+
+    expect(migration713(alert, migrationContext)).toEqual({
+      ...alert,
+      attributes: {
+        ...alert.attributes,
+        params: {
+          threshold: {
+            field: [],
+            value: 5,
+            cardinality: [],
+          },
+          exceptionsList: [],
+          riskScoreMapping: [],
+          severityMapping: [],
+          threat: [],
+        },
+      },
+    });
+  });
+
+  test('security solution threshold alert with array in threshold.field and cardinality is left alone', () => {
+    const migration713 = getMigrations(encryptedSavedObjectsSetup)['7.13.0'];
+    const alert = getMockData({
+      alertTypeId: 'siem.signals',
+      params: {
+        threshold: {
+          field: ['host.id'],
+          value: 5,
+          cardinality: [
+            {
+              field: 'source.ip',
+              value: 10,
+            },
+          ],
+        },
+      },
+    });
+
+    expect(migration713(alert, migrationContext)).toEqual({
+      ...alert,
+      attributes: {
+        ...alert.attributes,
+        params: {
+          threshold: {
+            field: ['host.id'],
+            value: 5,
+            cardinality: [
+              {
+                field: 'source.ip',
+                value: 10,
+              },
+            ],
+          },
+          exceptionsList: [],
+          riskScoreMapping: [],
+          severityMapping: [],
+          threat: [],
+        },
+      },
+    });
+  });
+});
+
 function getUpdatedAt(): string {
   const updatedAt = new Date();
   updatedAt.setHours(updatedAt.getHours() + 2);

x-pack/plugins/alerting/server/saved_objects/migrations.ts

@@ -11,6 +11,7 @@ import {
   SavedObjectMigrationFn,
   SavedObjectMigrationContext,
   SavedObjectAttributes,
+  SavedObjectAttribute,
 } from '../../../../../src/core/server';
 import { RawAlert, RawAlertAction } from '../types';
 import { EncryptedSavedObjectsPluginSetup } from '../../../encrypted_saved_objects/server';
@@ -30,6 +31,9 @@ export const isAnyActionSupportIncidents = (doc: SavedObjectUnsanitizedDoc<RawAl
     SUPPORT_INCIDENTS_ACTION_TYPES.includes(action.actionTypeId)
   );
 
+export const isSecuritySolutionRule = (doc: SavedObjectUnsanitizedDoc<RawAlert>): boolean =>
+  doc.attributes.alertTypeId === 'siem.signals';
+
 export function getMigrations(
   encryptedSavedObjects: EncryptedSavedObjectsPluginSetup
 ): SavedObjectMigrationMap {
@@ -59,10 +63,16 @@ export function getMigrations(
     pipeMigrations(restructureConnectorsThatSupportIncident)
   );
 
+  const migrationSecurityRules713 = encryptedSavedObjects.createMigration<RawAlert, RawAlert>(
+    (doc): doc is SavedObjectUnsanitizedDoc<RawAlert> => isSecuritySolutionRule(doc),
+    pipeMigrations(removeNullsFromSecurityRules)
+  );
+
   return {
     '7.10.0': executeMigrationWithErrorHandling(migrationWhenRBACWasIntroduced, '7.10.0'),
     '7.11.0': executeMigrationWithErrorHandling(migrationAlertUpdatedAtAndNotifyWhen, '7.11.0'),
     '7.11.2': executeMigrationWithErrorHandling(migrationActions7112, '7.11.2'),
+    '7.13.0': executeMigrationWithErrorHandling(migrationSecurityRules713, '7.13.0'),
   };
 }
 
@@ -333,6 +343,68 @@ function restructureConnectorsThatSupportIncident(
   };
 }
 
+function convertNullToUndefined(attribute: SavedObjectAttribute) {
+  return attribute != null ? attribute : undefined;
+}
+
+function removeNullsFromSecurityRules(
+  doc: SavedObjectUnsanitizedDoc<RawAlert>
+): SavedObjectUnsanitizedDoc<RawAlert> {
+  const {
+    attributes: { params },
+  } = doc;
+  return {
+    ...doc,
+    attributes: {
+      ...doc.attributes,
+      params: {
+        ...params,
+        buildingBlockType: convertNullToUndefined(params.buildingBlockType),
+        note: convertNullToUndefined(params.note),
+        index: convertNullToUndefined(params.index),
+        language: convertNullToUndefined(params.language),
+        license: convertNullToUndefined(params.license),
+        outputIndex: convertNullToUndefined(params.outputIndex),
+        savedId: convertNullToUndefined(params.savedId),
+        timelineId: convertNullToUndefined(params.timelineId),
+        timelineTitle: convertNullToUndefined(params.timelineTitle),
+        meta: convertNullToUndefined(params.meta),
+        query: convertNullToUndefined(params.query),
+        filters: convertNullToUndefined(params.filters),
+        riskScoreMapping: params.riskScoreMapping != null ? params.riskScoreMapping : [],
+        ruleNameOverride: convertNullToUndefined(params.ruleNameOverride),
+        severityMapping: params.severityMapping != null ? params.severityMapping : [],
+        threat: params.threat != null ? params.threat : [],
+        threshold:
+          params.threshold != null &&
+          typeof params.threshold === 'object' &&
+          !Array.isArray(params.threshold)
+            ? {
+                field: Array.isArray(params.threshold.field)
+                  ? params.threshold.field
+                  : params.threshold.field === '' || params.threshold.field == null
+                  ? []
+                  : [params.threshold.field],
+                value: params.threshold.value,
+                cardinality:
+                  params.threshold.cardinality != null ? params.threshold.cardinality : [],
+              }
+            : undefined,
+        timestampOverride: convertNullToUndefined(params.timestampOverride),
+        exceptionsList:
+          params.exceptionsList != null
+            ? params.exceptionsList
+            : params.exceptions_list != null
+            ? params.exceptions_list
+            : params.lists != null
+            ? params.lists
+            : [],
+        threatFilters: convertNullToUndefined(params.threatFilters),
+      },
+    },
+  };
+}
+
 function pipeMigrations(...migrations: AlertMigration[]): AlertMigration {
   return (doc: SavedObjectUnsanitizedDoc<RawAlert>) =>
     migrations.reduce((migratedDoc, nextMigration) => nextMigration(migratedDoc), doc);