[Security_Solution] Split up indices (#69589) (#69718)

jonathan-buttner · elasticmachine · web-flow · commit c3a553694409 · 2020-06-23T15:20:24.000-04:00
* Fixing resolver alert generation

* Splitting indices up

* Removing tests that could randomly fail because of the generation code

* Adding support for multiple indices

* Updating archives with the new index names

* Removing alerts data stream

* Switching to process instead of fake

Co-authored-by: Elastic Machine &lt;elasticmachine@users.noreply.github.com&gt;

Co-authored-by: Elastic Machine &lt;elasticmachine@users.noreply.github.com&gt;
diff --git a/x-pack/plugins/security_solution/common/endpoint/constants.ts b/x-pack/plugins/security_solution/common/endpoint/constants.ts
@@ -4,7 +4,8 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-export const eventsIndexPattern = 'events-endpoint-*';
+export const eventsIndexPattern = 'logs-endpoint.events.*';
+export const alertsIndexPattern = 'logs-endpoint.alerts-*';
 export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
 export const policyIndexPattern = 'metrics-endpoint.policy-*';
 export const telemetryIndexPattern = 'metrics-endpoint.telemetry-*';
diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts
@@ -16,14 +16,15 @@ export async function indexHostsAndAlerts(
   metadataIndex: string,
   policyIndex: string,
   eventIndex: string,
+  alertIndex: string,
   alertsPerHost: number,
   options: TreeOptions = {}
 ) {
   const random = seedrandom(seed);
   for (let i = 0; i < numHosts; i++) {
     const generator = new EndpointDocGenerator(random);
     await indexHostDocs(numDocs, client, metadataIndex, policyIndex, generator);
-    await indexAlerts(client, eventIndex, generator, alertsPerHost, options);
+    await indexAlerts(client, eventIndex, alertIndex, generator, alertsPerHost, options);
   }
   await client.indices.refresh({
     index: eventIndex,
@@ -65,7 +66,8 @@ async function indexHostDocs(
 
 async function indexAlerts(
   client: Client,
-  index: string,
+  eventIndex: string,
+  alertIndex: string,
   generator: EndpointDocGenerator,
   numAlerts: number,
   options: TreeOptions = {}
@@ -82,9 +84,14 @@ async function indexAlerts(
     }
     const body = resolverDocs.reduce(
       // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      (array: Array<Record<string, any>>, doc) => (
-        array.push({ create: { _index: index } }, doc), array
-      ),
+      (array: Array<Record<string, any>>, doc) => {
+        let index = eventIndex;
+        if (doc.event.kind === 'alert') {
+          index = alertIndex;
+        }
+        array.push({ create: { _index: index } }, doc);
+        return array;
+      },
       []
     );
     await client.bulk({ body, refresh: 'true' });
diff --git a/x-pack/plugins/security_solution/public/endpoint_alerts/store/middleware.ts b/x-pack/plugins/security_solution/public/endpoint_alerts/store/middleware.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { eventsIndexPattern } from '../../../common/endpoint/constants';
+import { alertsIndexPattern } from '../../../common/endpoint/constants';
 import { IIndexPattern } from '../../../../../../src/plugins/data/public';
 import {
   AlertResultList,
@@ -49,10 +49,10 @@ export const alertMiddlewareFactory: ImmutableMiddlewareFactory<AlertListState>
   async function fetchIndexPatterns(): Promise<IIndexPattern[]> {
     const { indexPatterns } = depsStart.data;
     const fields = await indexPatterns.getFieldsForWildcard({
-      pattern: eventsIndexPattern,
+      pattern: alertsIndexPattern,
     });
     const indexPattern: IIndexPattern = {
-      title: eventsIndexPattern,
+      title: alertsIndexPattern,
       fields,
     };
 
diff --git a/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts b/x-pack/plugins/security_solution/scripts/endpoint/resolver_generator.ts
@@ -95,19 +95,25 @@ async function main() {
     eventIndex: {
       alias: 'ei',
       describe: 'index to store events in',
-      default: 'events-endpoint-1',
+      default: 'logs-endpoint.events.process-default',
+      type: 'string',
+    },
+    alertIndex: {
+      alias: 'ai',
+      describe: 'index to store alerts in',
+      default: 'logs-endpoint.alerts-default',
       type: 'string',
     },
     metadataIndex: {
       alias: 'mi',
       describe: 'index to store host metadata in',
-      default: 'metrics-endpoint.metadata-default-1',
+      default: 'metrics-endpoint.metadata-default',
       type: 'string',
     },
     policyIndex: {
       alias: 'pi',
       describe: 'index to store host policy in',
-      default: 'metrics-endpoint.policy-default-1',
+      default: 'metrics-endpoint.policy-default',
       type: 'string',
     },
     ancestors: {
@@ -192,7 +198,10 @@ async function main() {
 
   const client = new Client(clientOptions);
   if (argv.delete) {
-    await deleteIndices([argv.eventIndex, argv.metadataIndex, argv.policyIndex], client);
+    await deleteIndices(
+      [argv.eventIndex, argv.metadataIndex, argv.policyIndex, argv.alertIndex],
+      client
+    );
   }
 
   let seed = argv.seed;
@@ -209,6 +218,7 @@ async function main() {
     argv.metadataIndex,
     argv.policyIndex,
     argv.eventIndex,
+    argv.alertIndex,
     argv.alertsPerHost,
     {
       ancestors: argv.ancestors,
diff --git a/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/details/index.ts b/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/details/index.ts
@@ -5,7 +5,7 @@
  */
 import { GetResponse } from 'elasticsearch';
 import { KibanaRequest, RequestHandler } from 'kibana/server';
-import { eventsIndexPattern } from '../../../../../common/endpoint/constants';
+import { alertsIndexPattern } from '../../../../../common/endpoint/constants';
 import { AlertEvent } from '../../../../../common/endpoint/types';
 import { EndpointAppContext } from '../../../types';
 import { AlertDetailsRequestParams } from '../../../../../common/endpoint_alerts/types';
@@ -34,7 +34,7 @@ export const alertDetailsHandlerWrapper = function (
         ctx,
         req.params,
         response,
-        eventsIndexPattern
+        alertsIndexPattern
       );
 
       const currentHostInfo = await getHostData(
diff --git a/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/list/index.ts b/x-pack/plugins/security_solution/server/endpoint/alerts/handlers/list/index.ts
@@ -4,7 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import { RequestHandler } from 'kibana/server';
-import { eventsIndexPattern } from '../../../../../common/endpoint/constants';
+import { alertsIndexPattern } from '../../../../../common/endpoint/constants';
 import { EndpointAppContext } from '../../../types';
 import { searchESForAlerts } from '../lib';
 import { getRequestData, mapToAlertResultList } from './lib';
@@ -23,7 +23,7 @@ export const alertListHandlerWrapper = function (
       const response = await searchESForAlerts(
         ctx.core.elasticsearch.legacy.client,
         reqData,
-        eventsIndexPattern
+        alertsIndexPattern
       );
       const mappedBody = await mapToAlertResultList(ctx, endpointAppContext, reqData, response);
       return res.ok({ body: mappedBody });
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/alerts.ts
@@ -7,7 +7,7 @@
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler, Logger } from 'kibana/server';
 import { validateAlerts } from '../../../../common/endpoint/schema/resolver';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { alertsIndexPattern, eventsIndexPattern } from '../../../../common/endpoint/constants';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
 
@@ -23,7 +23,7 @@ export function handleAlerts(
     try {
       const client = context.core.elasticsearch.legacy.client;
 
-      const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+      const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
       return res.ok({
         body: await fetcher.alerts(alerts, afterAlert),
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/ancestry.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/ancestry.ts
@@ -6,7 +6,7 @@
 
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateAncestry } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
@@ -23,7 +23,7 @@ export function handleAncestry(
     try {
       const client = context.core.elasticsearch.legacy.client;
 
-      const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+      const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
       const ancestorInfo = await fetcher.ancestors(ancestors);
 
       return res.ok({
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/children.ts
@@ -6,7 +6,7 @@
 
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateChildren } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
@@ -22,7 +22,7 @@ export function handleChildren(
     } = req;
     try {
       const client = context.core.elasticsearch.legacy.client;
-      const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+      const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
       return res.ok({
         body: await fetcher.children(children, generations, afterChild),
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/events.ts
@@ -6,7 +6,7 @@
 
 import { TypeOf } from '@kbn/config-schema';
 import { RequestHandler, Logger } from 'kibana/server';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateEvents } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { EndpointAppContext } from '../../types';
@@ -23,7 +23,7 @@ export function handleEvents(
     try {
       const client = context.core.elasticsearch.legacy.client;
 
-      const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+      const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
       return res.ok({
         body: await fetcher.events(events, afterEvent),
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/alerts.ts
@@ -15,7 +15,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com
 export class AlertsQuery extends ResolverQuery<PaginatedResults> {
   constructor(
     private readonly pagination: PaginationBuilder,
-    indexPattern: string,
+    indexPattern: string | string[],
     endpointID?: string
   ) {
     super(indexPattern, endpointID);
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/base.ts
@@ -25,13 +25,16 @@ export abstract class ResolverQuery<T> implements MSearchQuery {
    *  we need `endpointID` for legacy data is because we don't have a cross endpoint unique identifier for process
    *  events. Instead we use `unique_pid/ppid` and `endpointID` to uniquely identify a process event.
    */
-  constructor(private readonly indexPattern: string, private readonly endpointID?: string) {}
+  constructor(
+    private readonly indexPattern: string | string[],
+    private readonly endpointID?: string
+  ) {}
 
   private static createIdsArray(ids: string | string[]): string[] {
     return Array.isArray(ids) ? ids : [ids];
   }
 
-  private buildQuery(ids: string | string[]): { query: JsonObject; index: string } {
+  private buildQuery(ids: string | string[]): { query: JsonObject; index: string | string[] } {
     const idsArray = ResolverQuery.createIdsArray(ids);
     if (this.endpointID) {
       return { query: this.legacyQuery(this.endpointID, idsArray), index: legacyEventIndexPattern };
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/children.ts
@@ -15,7 +15,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com
 export class ChildrenQuery extends ResolverQuery<PaginatedResults> {
   constructor(
     private readonly pagination: PaginationBuilder,
-    indexPattern: string,
+    indexPattern: string | string[],
     endpointID?: string
   ) {
     super(indexPattern, endpointID);
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/queries/events.ts
@@ -15,7 +15,7 @@ import { JsonObject } from '../../../../../../../../src/plugins/kibana_utils/com
 export class EventsQuery extends ResolverQuery<PaginatedResults> {
   constructor(
     private readonly pagination: PaginationBuilder,
-    indexPattern: string,
+    indexPattern: string | string[],
     endpointID?: string
   ) {
     super(indexPattern, endpointID);
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/tree.ts
@@ -6,7 +6,7 @@
 
 import { RequestHandler, Logger } from 'kibana/server';
 import { TypeOf } from '@kbn/config-schema';
-import { eventsIndexPattern } from '../../../../common/endpoint/constants';
+import { eventsIndexPattern, alertsIndexPattern } from '../../../../common/endpoint/constants';
 import { validateTree } from '../../../../common/endpoint/schema/resolver';
 import { Fetcher } from './utils/fetch';
 import { Tree } from './utils/tree';
@@ -34,7 +34,7 @@ export function handleTree(
     try {
       const client = context.core.elasticsearch.legacy.client;
 
-      const fetcher = new Fetcher(client, id, eventsIndexPattern, endpointID);
+      const fetcher = new Fetcher(client, id, eventsIndexPattern, alertsIndexPattern, endpointID);
 
       const [childrenNodes, ancestry, relatedEvents, relatedAlerts] = await Promise.all([
         fetcher.children(children, generations, afterChild),
diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts b/x-pack/plugins/security_solution/server/endpoint/routes/resolver/utils/fetch.ts
@@ -39,9 +39,13 @@ export class Fetcher {
      */
     private readonly id: string,
     /**
-     * Index pattern for searching ES
+     * Index pattern for searching ES for events
      */
-    private readonly indexPattern: string,
+    private readonly eventsIndexPattern: string,
+    /**
+     * Index pattern for searching ES for alerts
+     */
+    private readonly alertsIndexPattern: string,
     /**
      * This is used for searching legacy events
      */
@@ -109,7 +113,7 @@ export class Fetcher {
   public async alerts(limit: number, after?: string): Promise<ResolverRelatedAlerts> {
     const query = new AlertsQuery(
       PaginationBuilder.createBuilder(limit, after),
-      this.indexPattern,
+      this.alertsIndexPattern,
       this.endpointID
     );
 
@@ -140,7 +144,7 @@ export class Fetcher {
   }
 
   private async getNode(entityID: string): Promise<LifecycleNode | undefined> {
-    const query = new LifecycleQuery(this.indexPattern, this.endpointID);
+    const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID);
     const results = await query.search(this.client, entityID);
     if (results.length === 0) {
       return;
@@ -172,7 +176,7 @@ export class Fetcher {
       return;
     }
 
-    const query = new LifecycleQuery(this.indexPattern, this.endpointID);
+    const query = new LifecycleQuery(this.eventsIndexPattern, this.endpointID);
     const results = await query.search(this.client, ancestors);
 
     if (results.length === 0) {
@@ -210,7 +214,7 @@ export class Fetcher {
   private async doEvents(limit: number, after?: string) {
     const query = new EventsQuery(
       PaginationBuilder.createBuilder(limit, after),
-      this.indexPattern,
+      this.eventsIndexPattern,
       this.endpointID
     );
 
@@ -243,10 +247,10 @@ export class Fetcher {
 
     const childrenQuery = new ChildrenQuery(
       PaginationBuilder.createBuilder(limit, after),
-      this.indexPattern,
+      this.eventsIndexPattern,
       this.endpointID
     );
-    const lifecycleQuery = new LifecycleQuery(this.indexPattern, this.endpointID);
+    const lifecycleQuery = new LifecycleQuery(this.eventsIndexPattern, this.endpointID);
 
     const { totals, results } = await childrenQuery.search(this.client, ids);
     if (results.length === 0) {
@@ -262,7 +266,10 @@ export class Fetcher {
   }
 
   private async doStats(tree: Tree) {
-    const statsQuery = new StatsQuery(this.indexPattern, this.endpointID);
+    const statsQuery = new StatsQuery(
+      [this.eventsIndexPattern, this.alertsIndexPattern],
+      this.endpointID
+    );
     const ids = tree.ids();
     const res = await statsQuery.search(this.client, ids);
     const alerts = res.alerts;
diff --git a/x-pack/test/api_integration/apis/endpoint/alerts/index.ts b/x-pack/test/api_integration/apis/endpoint/alerts/index.ts
diff --git a/x-pack/test/api_integration/apis/endpoint/data_stream_helper.ts b/x-pack/test/api_integration/apis/endpoint/data_stream_helper.ts
diff --git a/x-pack/test/api_integration/apis/endpoint/resolver.ts b/x-pack/test/api_integration/apis/endpoint/resolver.ts
diff --git a/x-pack/test/api_integration/services/resolver.ts b/x-pack/test/api_integration/services/resolver.ts
diff --git a/x-pack/test/functional/es_archives/endpoint/alerts/api_feature/data.json.gz b/x-pack/test/functional/es_archives/endpoint/alerts/api_feature/data.json.gz
diff --git a/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/data.json.gz b/x-pack/test/functional/es_archives/endpoint/alerts/host_api_feature/data.json.gz
diff --git a/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/metadata/api_feature/data.json
diff --git a/x-pack/test/functional/es_archives/endpoint/policy/data.json.gz b/x-pack/test/functional/es_archives/endpoint/policy/data.json.gz
