[Ingest] check and create default indices during setup (#64809) (#65400)

* check and create default indices during setup

* fix typos

* add kibana issue

* add date mapping

* add removeDocValues to each field to stop apps from creating a new index pattern

* update snapshot

* add property to test data
# Conflicts:
#	x-pack/plugins/ingest_manager/common/constants/epm.ts

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: neptunian

x-pack/plugins/ingest_manager/common/constants/epm.ts

@@ -7,3 +7,4 @@
 export const PACKAGES_SAVED_OBJECT_TYPE = 'epm-packages';
 export const INDEX_PATTERN_SAVED_OBJECT_TYPE = 'index-pattern';
 export const DEFAULT_REGISTRY_URL = 'https://epr-experimental.elastic.co';
+export const INDEX_PATTERN_PLACEHOLDER_SUFFIX = '-index_pattern_placeholder';

x-pack/plugins/ingest_manager/server/constants/index.ts

@@ -9,6 +9,7 @@ export {
   AGENT_TYPE_TEMPORARY,
   AGENT_POLLING_THRESHOLD_MS,
   AGENT_POLLING_INTERVAL,
+  INDEX_PATTERN_PLACEHOLDER_SUFFIX,
   // Routes
   PLUGIN_ID,
   EPM_API_ROUTES,

x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/__snapshots__/install.test.ts.snap

@@ -5,11 +5,14 @@
  */
 
 import { SavedObjectsClientContract } from 'src/core/server';
-import { INDEX_PATTERN_SAVED_OBJECT_TYPE } from '../../../../constants';
+import {
+  INDEX_PATTERN_SAVED_OBJECT_TYPE,
+  INDEX_PATTERN_PLACEHOLDER_SUFFIX,
+} from '../../../../constants';
 import * as Registry from '../../registry';
 import { loadFieldsFromYaml, Fields, Field } from '../../fields/field';
 import { getPackageKeysByStatus } from '../../packages/get';
-import { InstallationStatus, RegistryPackage } from '../../../../types';
+import { InstallationStatus, RegistryPackage, CallESAsCurrentUser } from '../../../../types';
 
 interface FieldFormatMap {
   [key: string]: FieldFormatMapItem;
@@ -63,6 +66,7 @@ export interface IndexPatternField {
   enabled?: boolean;
   script?: string;
   lang?: string;
+  readFromDocValues: boolean;
 }
 export enum IndexPatternType {
   logs = 'logs',
@@ -234,6 +238,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa
     searchable: field.searchable ?? true,
     aggregatable: field.aggregatable ?? true,
     doc_values: field.doc_values ?? true,
+    readFromDocValues: field.doc_values ?? true,
   };
 
   // if type exists, check if it exists in the map
@@ -251,6 +256,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa
     newField.aggregatable = false;
     newField.analyzed = false;
     newField.doc_values = field.doc_values ?? false;
+    newField.readFromDocValues = field.doc_values ?? false;
     newField.indexed = false;
     newField.searchable = false;
   }
@@ -262,6 +268,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa
       newField.aggregatable = false;
       newField.analyzed = false;
       newField.doc_values = false;
+      newField.readFromDocValues = false;
       newField.indexed = false;
       newField.searchable = false;
     }
@@ -276,6 +283,7 @@ export const transformField = (field: Field, i: number, fields: Fields): IndexPa
     newField.script = field.script;
     newField.lang = 'painless';
     newField.doc_values = false;
+    newField.readFromDocValues = false;
   }
 
   return newField;
@@ -357,3 +365,42 @@ const getFieldFormatParams = (field: Field): FieldFormatParams => {
   if (field.open_link_in_current_tab) params.openLinkInCurrentTab = field.open_link_in_current_tab;
   return params;
 };
+
+export const ensureDefaultIndices = async (callCluster: CallESAsCurrentUser) =>
+  // create placeholder indices to supress errors in the kibana Dashboards app
+  // that no matching indices exist https://github.com/elastic/kibana/issues/62343
+  Promise.all(
+    Object.keys(IndexPatternType).map(async indexPattern => {
+      const defaultIndexPatternName = indexPattern + INDEX_PATTERN_PLACEHOLDER_SUFFIX;
+      const indexExists = await doesIndexExist(defaultIndexPatternName, callCluster);
+      if (!indexExists) {
+        try {
+          await callCluster('transport.request', {
+            method: 'PUT',
+            path: `/${defaultIndexPatternName}`,
+            body: {
+              mappings: {
+                properties: {
+                  '@timestamp': { type: 'date' },
+                },
+              },
+            },
+          });
+        } catch (putErr) {
+          throw new Error(`${defaultIndexPatternName} could not be created`);
+        }
+      }
+    })
+  );
+
+export const doesIndexExist = async (indexName: string, callCluster: CallESAsCurrentUser) => {
+  try {
+    await callCluster('transport.request', {
+      method: 'HEAD',
+      path: indexName,
+    });
+    return true;
+  } catch (err) {
+    return false;
+  }
+};

x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/install.ts

@@ -15,6 +15,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 0,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -26,6 +27,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 0,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -37,6 +39,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 0,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -48,6 +51,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 2,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -59,6 +63,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 0,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -70,6 +75,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 0,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -81,6 +87,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 0,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: true,
   },
@@ -92,6 +99,7 @@ export const dupeFields: IndexPatternField[] = [
     count: 1,
     indexed: true,
     doc_values: true,
+    readFromDocValues: true,
     scripted: false,
     analyzed: false,
   },

x-pack/plugins/ingest_manager/server/services/epm/kibana/index_pattern/tests/test_data.ts

@@ -11,6 +11,7 @@ import { CallESAsCurrentUser } from '../types';
 import { agentConfigService } from './agent_config';
 import { outputService } from './output';
 import { ensureInstalledDefaultPackages } from './epm/packages/install';
+import { ensureDefaultIndices } from './epm/kibana/index_pattern/install';
 import {
   packageToConfigDatasource,
   Datasource,
@@ -38,6 +39,7 @@ export async function setupIngestManager(
     ensureInstalledDefaultPackages(soClient, callCluster),
     outputService.ensureDefaultOutput(soClient),
     agentConfigService.ensureDefaultAgentConfig(soClient),
+    ensureDefaultIndices(callCluster),
     settingsService.getSettings(soClient).catch((e: any) => {
       if (e.isBoom && e.output.statusCode === 404) {
         const http = appContextService.getHttpSetup();