Merge branch '7.x' into backport/7.x/pr-64809

Author: elasticmachine

.eslintignore

@@ -25,7 +25,7 @@ target
 /src/plugins/vis_type_timelion/public/_generated_/**
 /src/plugins/vis_type_timelion/public/webpackShims/jquery.flot.*
 /x-pack/legacy/plugins/**/__tests__/fixtures/**
-/x-pack/legacy/plugins/apm/e2e/cypress/**/snapshots.js
+/x-pack/plugins/apm/e2e/cypress/**/snapshots.js
 /x-pack/legacy/plugins/canvas/canvas_plugin
 /x-pack/legacy/plugins/canvas/canvas_plugin_src/lib/flot-charts
 /x-pack/legacy/plugins/canvas/shareable_runtime/build

.eslintrc.js

@@ -310,7 +310,7 @@ module.exports = {
     {
       files: [
         'x-pack/test/functional/apps/**/*.js',
-        'x-pack/legacy/plugins/apm/**/*.js',
+        'x-pack/plugins/apm/**/*.js',
         'test/*/config.ts',
         'test/*/config_open.ts',
         'test/*/{tests,test_suites,apis,apps}/**/*',
@@ -397,7 +397,7 @@ module.exports = {
         'x-pack/**/*.test.js',
         'x-pack/test_utils/**/*',
         'x-pack/gulpfile.js',
-        'x-pack/legacy/plugins/apm/public/utils/testHelpers.js',
+        'x-pack/plugins/apm/public/utils/testHelpers.js',
       ],
       rules: {
         'import/no-extraneous-dependencies': [
@@ -523,15 +523,15 @@ module.exports = {
      * APM overrides
      */
     {
-      files: ['x-pack/legacy/plugins/apm/**/*.js'],
+      files: ['x-pack/plugins/apm/**/*.js'],
       rules: {
         'no-unused-vars': ['error', { ignoreRestSiblings: true }],
         'no-console': ['warn', { allow: ['error'] }],
       },
     },
     {
       plugins: ['react-hooks'],
-      files: ['x-pack/legacy/plugins/apm/**/*.{ts,tsx}'],
+      files: ['x-pack/plugins/apm/**/*.{ts,tsx}'],
       rules: {
         'react-hooks/rules-of-hooks': 'error', // Checks rules of Hooks
         'react-hooks/exhaustive-deps': ['error', { additionalHooks: '^useFetcher$' }],

.gitignore

@@ -44,6 +44,6 @@ package-lock.json
 *.sublime-*
 npm-debug.log*
 .tern-project
-x-pack/legacy/plugins/apm/tsconfig.json
+x-pack/plugins/apm/tsconfig.json
 apm.tsconfig.json
-/x-pack/legacy/plugins/apm/e2e/snapshots.js
+/x-pack/plugins/apm/e2e/snapshots.js

x-pack/plugins/ingest_manager/common/services/decode_cloud_id.test.ts

@@ -0,0 +1,100 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { decodeCloudId } from './decode_cloud_id';
+
+describe('Ingest Manager - decodeCloudId', () => {
+  it('parses various CloudID formats', () => {
+    const tests = [
+      {
+        cloudID:
+          'staging:dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443',
+        expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443',
+      },
+      {
+        cloudID:
+          'dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443',
+        expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443',
+      },
+      {
+        cloudID:
+          ':dXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        expectedEsURL: 'https://cec6f261a74bf24ce33bb8811b84294f.us-east-1.aws.found.io:443',
+        expectedKibanaURL: 'https://c6c2ca6d042249af0cc7d7a9e9625743.us-east-1.aws.found.io:443',
+      },
+      {
+        cloudID:
+          'gcp-cluster:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZmNlJDBjZDVjZDU2OGVlYmU1M2M4OWViN2NhZTViYWM4YjM3',
+        expectedEsURL: 'https://8a0283af041f195f7729bc04c66a0fce.us-central1.gcp.cloud.es.io:443',
+        expectedKibanaURL:
+          'https://0cd5cd568eebe53c89eb7cae5bac8b37.us-central1.gcp.cloud.es.io:443',
+      },
+      {
+        cloudID:
+          'custom-port:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvOjkyNDMkYWMzMWViYjkwMjQxNzczMTU3MDQzYzM0ZmQyNmZkNDYkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA=',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9243',
+      },
+      {
+        cloudID:
+          'different-es-kb-port:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2OjkyNDMkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244',
+      },
+      {
+        cloudID:
+          'only-kb-set:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2JGE0YzA2MjMwZTQ4YzhmY2U3YmU4OGEwNzRhM2JiM2UwOjkyNDQ=',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:443',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244',
+      },
+      {
+        cloudID:
+          'host-and-kb-set:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvOjkyNDMkYWMzMWViYjkwMjQxNzczMTU3MDQzYzM0ZmQyNmZkNDYkYTRjMDYyMzBlNDhjOGZjZTdiZTg4YTA3NGEzYmIzZTA6OTI0NA==',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:9243',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:9244',
+      },
+      {
+        cloudID:
+          'extra-items:dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJGFjMzFlYmI5MDI0MTc3MzE1NzA0M2MzNGZkMjZmZDQ2JGE0YzA2MjMwZTQ4YzhmY2U3YmU4OGEwNzRhM2JiM2UwJGFub3RoZXJpZCRhbmRhbm90aGVy',
+        expectedEsURL: 'https://ac31ebb90241773157043c34fd26fd46.us-central1.gcp.cloud.es.io:443',
+        expectedKibanaURL:
+          'https://a4c06230e48c8fce7be88a074a3bb3e0.us-central1.gcp.cloud.es.io:443',
+      },
+    ];
+
+    for (const test of tests) {
+      const decoded = decodeCloudId(test.cloudID);
+      expect(decoded).toBeTruthy();
+      expect(decoded?.elasticsearchUrl === test.expectedEsURL).toBe(true);
+      expect(decoded?.kibanaUrl === test.expectedKibanaURL).toBe(true);
+    }
+  });
+
+  it('returns undefined for invalid formats', () => {
+    const tests = [
+      {
+        cloudID:
+          'staging:garbagedXMtZWFzdC0xLmF3cy5mb3VuZC5pbyRjZWM2ZjI2MWE3NGJmMjRjZTMzYmI4ODExYjg0Mjk0ZiRjNmMyY2E2ZDA0MjI0OWFmMGNjN2Q3YTllOTYyNTc0Mw==',
+        errorMsg: 'base64 decoding failed',
+      },
+      {
+        cloudID: 'dXMtY2VudHJhbDEuZ2NwLmNsb3VkLmVzLmlvJDhhMDI4M2FmMDQxZjE5NWY3NzI5YmMwNGM2NmEwZg==',
+        errorMsg: 'Expected at least 3 parts',
+      },
+    ];
+
+    for (const test of tests) {
+      const decoded = decodeCloudId(test.cloudID);
+      expect(decoded).toBe(undefined);
+      // decodeCloudId currently only logs; not throws errors
+    }
+  });
+});

x-pack/plugins/ingest_manager/common/services/decode_cloud_id.ts

@@ -0,0 +1,65 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// decodeCloudId decodes the c.id into c.esURL and c.kibURL
+export function decodeCloudId(
+  cid: string
+):
+  | {
+      host: string;
+      defaultPort: string;
+      elasticsearchUrl: string;
+      kibanaUrl: string;
+    }
+  | undefined {
+  // 1. Ignore anything before `:`.
+  const id = cid.split(':').pop();
+  if (!id) {
+    // throw new Error(`Unable to decode ${id}`);
+    // eslint-disable-next-line no-console
+    console.debug(`Unable to decode ${id}`);
+    return;
+  }
+
+  // 2. base64 decode
+  let decoded: string | undefined;
+  try {
+    decoded = Buffer.from(id, 'base64').toString('utf8');
+  } catch {
+    // throw new Error(`base64 decoding failed on ${id}`);
+    // eslint-disable-next-line no-console
+    console.debug(`base64 decoding failed on ${id}`);
+    return;
+  }
+
+  // 3. separate based on `$`
+  const words = decoded.split('$');
+  if (words.length < 3) {
+    // throw new Error(`Expected at least 3 parts in ${decoded}`);
+    // eslint-disable-next-line no-console
+    console.debug(`Expected at least 3 parts in ${decoded}`);
+    return;
+  }
+  // 4. extract port from the ES and Kibana host
+  const [host, defaultPort] = extractPortFromName(words[0]);
+  const [esId, esPort] = extractPortFromName(words[1], defaultPort);
+  const [kbId, kbPort] = extractPortFromName(words[2], defaultPort);
+  // 5. form the URLs
+  const esUrl = `https://${esId}.${host}:${esPort}`;
+  const kbUrl = `https://${kbId}.${host}:${kbPort}`;
+  return {
+    host,
+    defaultPort,
+    elasticsearchUrl: esUrl,
+    kibanaUrl: kbUrl,
+  };
+}
+// extractPortFromName takes a string in the form `id:port` and returns the
+// Id and the port. If there's no `:`, the default port is returned
+function extractPortFromName(word: string, defaultPort = '443') {
+  const [host, port = defaultPort] = word.split(':');
+  return [host, port];
+}

x-pack/plugins/ingest_manager/common/services/index.ts

@@ -9,3 +9,4 @@ export * from './routes';
 export { packageToConfigDatasourceInputs, packageToConfigDatasource } from './package_to_config';
 export { storedDatasourceToAgentDatasource } from './datasource_to_agent_datasource';
 export { AgentStatusKueryHelper };
+export { decodeCloudId } from './decode_cloud_id';

x-pack/plugins/ingest_manager/common/types/index.ts

@@ -15,7 +15,6 @@ export interface IngestManagerConfigType {
   fleet: {
     enabled: boolean;
     tlsCheckDisabled: boolean;
-    defaultOutputHost: string;
     kibana: {
       host?: string;
       ca_sha256?: string;

x-pack/plugins/ingest_manager/server/index.ts

@@ -32,7 +32,7 @@ export const config = {
         ca_sha256: schema.maybe(schema.string()),
       }),
       elasticsearch: schema.object({
-        host: schema.string({ defaultValue: 'http://localhost:9200' }),
+        host: schema.maybe(schema.string()),
         ca_sha256: schema.maybe(schema.string()),
       }),
     }),

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -165,6 +165,7 @@ export class IngestManagerPlugin
     const config = await this.config$.pipe(first()).toPromise();
 
     // Register routes
+    registerSetupRoutes(router, config);
     registerAgentConfigRoutes(router);
     registerDatasourceRoutes(router);
     registerOutputRoutes(router);
@@ -177,7 +178,6 @@ export class IngestManagerPlugin
     }
 
     if (config.fleet.enabled) {
-      registerSetupRoutes(router);
       registerAgentRoutes(router);
       registerEnrollmentApiKeyRoutes(router);
       registerInstallScriptRoutes({

x-pack/plugins/ingest_manager/server/routes/setup/index.ts

@@ -6,13 +6,14 @@
 import { IRouter } from 'src/core/server';
 
 import { PLUGIN_ID, FLEET_SETUP_API_ROUTES, SETUP_API_ROUTE } from '../../constants';
+import { IngestManagerConfigType } from '../../../common';
 import {
   getFleetStatusHandler,
   createFleetSetupHandler,
   ingestManagerSetupHandler,
 } from './handlers';
 
-export const registerRoutes = (router: IRouter) => {
+export const registerRoutes = (router: IRouter, config: IngestManagerConfigType) => {
   // Ingest manager setup
   router.post(
     {
@@ -24,6 +25,11 @@ export const registerRoutes = (router: IRouter) => {
     },
     ingestManagerSetupHandler
   );
+
+  if (!config.fleet.enabled) {
+    return;
+  }
+
   // Get Fleet setup
   router.get(
     {