[Security Solution][Detections] Adoption telemetry (#71102)

* style: sort plugin interface

* WIP: UsageCollector for Security Adoption

This uses ML and raw ES calls to query our ML Jobs and Rules, and parse
them into a format to be consumed by telemetry.

Still to come:
* initialization
* tests

* Initialize usage collectors during plugin setup

* Rename usage key

The service seems to convert colons to underscores, so let's just use an
underscure.

* Collector is ready if we have a kibana index

* Refactor collector to generate options in a function

This allows us to test our adherence to the collector API, focusing
particularly on the fetch function.

* Refactor usage collector in anticipation of endpoint data

We're going to have our usage data under one key corresponding to the
app, so this nests the existing data under a 'detections' key while
allowing another fetching function to be plugged into the
main collector under a separate key.

* Update our collector to satisfy telemetry tooling

* inlines collector options
* inlines schema object
* makes DetectionsUsage an interface instead of a type alias

* Extracts telemetry mappings via scripts/telemetry_extract

* Refactor detections usage logic to perform one loop instead of two

We were previously performing two loops over each set of data: one to
format it down to just the data we need, and another to convert that
into usage data. We now perform both steps within a single loop.

* Refactor detections telemetry to be nested

* Extract new nested detections telemetry mappings

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: rylnd

x-pack/plugins/security_solution/server/plugin.ts

@@ -16,6 +16,7 @@ import {
   PluginInitializerContext,
   SavedObjectsClient,
 } from '../../../../src/core/server';
+import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server';
 import { PluginSetupContract as AlertingSetup } from '../../alerts/server';
 import { SecurityPluginSetup as SecuritySetup } from '../../security/server';
 import { PluginSetupContract as FeaturesSetup } from '../../features/server';
@@ -46,17 +47,19 @@ import { ArtifactClient, ManifestManager } from './endpoint/services';
 import { EndpointAppContextService } from './endpoint/endpoint_app_context_services';
 import { EndpointAppContext } from './endpoint/types';
 import { registerDownloadExceptionListRoute } from './endpoint/routes/artifacts';
+import { initUsageCollectors } from './usage';
 
 export interface SetupPlugins {
   alerts: AlertingSetup;
   encryptedSavedObjects?: EncryptedSavedObjectsSetup;
   features: FeaturesSetup;
   licensing: LicensingPluginSetup;
+  lists?: ListPluginSetup;
+  ml?: MlSetup;
   security?: SecuritySetup;
   spaces?: SpacesSetup;
   taskManager?: TaskManagerSetupContract;
-  ml?: MlSetup;
-  lists?: ListPluginSetup;
+  usageCollection?: UsageCollectionSetup;
 }
 
 export interface StartPlugins {
@@ -106,9 +109,15 @@ export class Plugin implements IPlugin<PluginSetup, PluginStart, SetupPlugins, S
     this.logger.debug('plugin setup');
 
     const config = await this.config$.pipe(first()).toPromise();
+    const globalConfig = await this.context.config.legacy.globalConfig$.pipe(first()).toPromise();
 
     initSavedObjects(core.savedObjects);
     initUiSettings(core.uiSettings);
+    initUsageCollectors({
+      kibanaIndex: globalConfig.kibana.index,
+      ml: plugins.ml,
+      usageCollection: plugins.usageCollection,
+    });
 
     const endpointContext: EndpointAppContext = {
       logFactory: this.context.logger,

x-pack/plugins/security_solution/server/usage/collector.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from '../../../../../src/core/server';
+import { CollectorDependencies } from './types';
+import { DetectionsUsage, fetchDetectionsUsage } from './detections';
+
+export type RegisterCollector = (deps: CollectorDependencies) => void;
+export interface UsageData {
+  detections: DetectionsUsage;
+}
+
+export const registerCollector: RegisterCollector = ({ kibanaIndex, ml, usageCollection }) => {
+  if (!usageCollection) {
+    return;
+  }
+
+  const collector = usageCollection.makeUsageCollector<UsageData>({
+    type: 'security_solution',
+    schema: {
+      detections: {
+        detection_rules: {
+          custom: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+          elastic: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+        },
+        ml_jobs: {
+          custom: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+          elastic: {
+            enabled: { type: 'long' },
+            disabled: { type: 'long' },
+          },
+        },
+      },
+    },
+    isReady: () => kibanaIndex.length > 0,
+    fetch: async (callCluster: LegacyAPICaller): Promise<UsageData> => ({
+      detections: await fetchDetectionsUsage(kibanaIndex, callCluster, ml),
+    }),
+  });
+
+  usageCollection.registerCollector(collector);
+};

x-pack/plugins/security_solution/server/usage/detections.mocks.ts

@@ -0,0 +1,162 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { INTERNAL_IMMUTABLE_KEY } from '../../common/constants';
+
+export const getMockJobSummaryResponse = () => [
+  {
+    id: 'linux_anomalous_network_activity_ecs',
+    description:
+      'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)',
+    groups: ['auditbeat', 'process', 'siem'],
+    processed_record_count: 141889,
+    memory_status: 'ok',
+    jobState: 'opened',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-linux_anomalous_network_activity_ecs',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'started',
+    latestTimestampMs: 1594085401911,
+    earliestTimestampMs: 1593054845656,
+    latestResultsTimestampMs: 1594085401911,
+    isSingleMetricViewerJob: true,
+    nodeName: 'node',
+  },
+  {
+    id: 'linux_anomalous_network_port_activity_ecs',
+    description:
+      'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)',
+    groups: ['auditbeat', 'process', 'siem'],
+    processed_record_count: 0,
+    memory_status: 'ok',
+    jobState: 'closed',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-linux_anomalous_network_port_activity_ecs',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'stopped',
+    isSingleMetricViewerJob: true,
+  },
+  {
+    id: 'other_job',
+    description: 'a job that is custom',
+    groups: ['auditbeat', 'process'],
+    processed_record_count: 0,
+    memory_status: 'ok',
+    jobState: 'closed',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-other',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'stopped',
+    isSingleMetricViewerJob: true,
+  },
+  {
+    id: 'another_job',
+    description: 'another job that is custom',
+    groups: ['auditbeat', 'process'],
+    processed_record_count: 0,
+    memory_status: 'ok',
+    jobState: 'opened',
+    hasDatafeed: true,
+    datafeedId: 'datafeed-another',
+    datafeedIndices: ['auditbeat-*'],
+    datafeedState: 'started',
+    isSingleMetricViewerJob: true,
+  },
+];
+
+export const getMockListModulesResponse = () => [
+  {
+    id: 'siem_auditbeat',
+    title: 'SIEM Auditbeat',
+    description:
+      'Detect suspicious network activity and unusual processes in Auditbeat data (beta).',
+    type: 'Auditbeat data',
+    logoFile: 'logo.json',
+    defaultIndexPattern: 'auditbeat-*',
+    query: {
+      bool: {
+        filter: [
+          {
+            term: {
+              'agent.type': 'auditbeat',
+            },
+          },
+        ],
+      },
+    },
+    jobs: [
+      {
+        id: 'linux_anomalous_network_activity_ecs',
+        config: {
+          job_type: 'anomaly_detector',
+          description:
+            'SIEM Auditbeat: Looks for unusual processes using the network which could indicate command-and-control, lateral movement, persistence, or data exfiltration activity (beta)',
+          groups: ['siem', 'auditbeat', 'process'],
+          analysis_config: {
+            bucket_span: '15m',
+            detectors: [
+              {
+                detector_description: 'rare by "process.name"',
+                function: 'rare',
+                by_field_name: 'process.name',
+              },
+            ],
+            influencers: ['host.name', 'process.name', 'user.name', 'destination.ip'],
+          },
+          allow_lazy_open: true,
+          analysis_limits: {
+            model_memory_limit: '64mb',
+          },
+          data_description: {
+            time_field: '@timestamp',
+          },
+        },
+      },
+      {
+        id: 'linux_anomalous_network_port_activity_ecs',
+        config: {
+          job_type: 'anomaly_detector',
+          description:
+            'SIEM Auditbeat: Looks for unusual destination port activity that could indicate command-and-control, persistence mechanism, or data exfiltration activity (beta)',
+          groups: ['siem', 'auditbeat', 'network'],
+          analysis_config: {
+            bucket_span: '15m',
+            detectors: [
+              {
+                detector_description: 'rare by "destination.port"',
+                function: 'rare',
+                by_field_name: 'destination.port',
+              },
+            ],
+            influencers: ['host.name', 'process.name', 'user.name', 'destination.ip'],
+          },
+          allow_lazy_open: true,
+          analysis_limits: {
+            model_memory_limit: '32mb',
+          },
+          data_description: {
+            time_field: '@timestamp',
+          },
+        },
+      },
+    ],
+    datafeeds: [],
+    kibana: {},
+  },
+];
+
+export const getMockRulesResponse = () => ({
+  hits: {
+    hits: [
+      { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:false`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: true, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:false`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+      { _source: { alert: { enabled: false, tags: [`${INTERNAL_IMMUTABLE_KEY}:true`] } } },
+    ],
+  },
+});

x-pack/plugins/security_solution/server/usage/detections.test.ts

@@ -0,0 +1,107 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { LegacyAPICaller } from '../../../../../src/core/server';
+import { elasticsearchServiceMock } from '../../../../../src/core/server/mocks';
+import { jobServiceProvider } from '../../../ml/server/models/job_service';
+import { DataRecognizer } from '../../../ml/server/models/data_recognizer';
+import { mlServicesMock } from '../lib/machine_learning/mocks';
+import {
+  getMockJobSummaryResponse,
+  getMockListModulesResponse,
+  getMockRulesResponse,
+} from './detections.mocks';
+import { fetchDetectionsUsage } from './detections';
+
+jest.mock('../../../ml/server/models/job_service');
+jest.mock('../../../ml/server/models/data_recognizer');
+
+describe('Detections Usage', () => {
+  describe('fetchDetectionsUsage()', () => {
+    let callClusterMock: jest.Mocked<LegacyAPICaller>;
+    let mlMock: ReturnType<typeof mlServicesMock.create>;
+
+    beforeEach(() => {
+      callClusterMock = elasticsearchServiceMock.createLegacyClusterClient().callAsInternalUser;
+      mlMock = mlServicesMock.create();
+    });
+
+    it('returns zeroed counts if both calls are empty', async () => {
+      const result = await fetchDetectionsUsage('', callClusterMock, mlMock);
+
+      expect(result).toEqual({
+        detection_rules: {
+          custom: {
+            enabled: 0,
+            disabled: 0,
+          },
+          elastic: {
+            enabled: 0,
+            disabled: 0,
+          },
+        },
+        ml_jobs: {
+          custom: {
+            enabled: 0,
+            disabled: 0,
+          },
+          elastic: {
+            enabled: 0,
+            disabled: 0,
+          },
+        },
+      });
+    });
+
+    it('tallies rules data given rules results', async () => {
+      (callClusterMock as jest.Mock).mockResolvedValue(getMockRulesResponse());
+      const result = await fetchDetectionsUsage('', callClusterMock, mlMock);
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          detection_rules: {
+            custom: {
+              enabled: 1,
+              disabled: 1,
+            },
+            elastic: {
+              enabled: 2,
+              disabled: 3,
+            },
+          },
+        })
+      );
+    });
+
+    it('tallies jobs data given jobs results', async () => {
+      const mockJobSummary = jest.fn().mockResolvedValue(getMockJobSummaryResponse());
+      const mockListModules = jest.fn().mockResolvedValue(getMockListModulesResponse());
+      (jobServiceProvider as jest.Mock).mockImplementation(() => ({
+        jobsSummary: mockJobSummary,
+      }));
+      (DataRecognizer as jest.Mock).mockImplementation(() => ({
+        listModules: mockListModules,
+      }));
+
+      const result = await fetchDetectionsUsage('', callClusterMock, mlMock);
+
+      expect(result).toEqual(
+        expect.objectContaining({
+          ml_jobs: {
+            custom: {
+              enabled: 1,
+              disabled: 1,
+            },
+            elastic: {
+              enabled: 1,
+              disabled: 1,
+            },
+          },
+        })
+      );
+    });
+  });
+});