[Ingest Manager] Support registration of server side callbacks for Create Datasource API (#69428)

* Ingest: Expose `registerExternalCallback()` method out of Ingest server `start` lifecycle
* Ingest: Add support for External Callbacks on REST `createDatasourceHandler()`
* Ingest: expose DatasourceServices to Plugin start interface
* Endpoint: Added Endpoint Ingest handler for Create Datasources
  - Also moved the temporary logic from the middleware
    to the handler (still temporary)

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: paul-tavares

x-pack/plugins/ingest_manager/server/index.ts

@@ -11,6 +11,7 @@ export {
   IngestManagerSetupContract,
   IngestManagerSetupDeps,
   IngestManagerStartContract,
+  ExternalCallback,
 } from './plugin';
 
 export const config = {
@@ -42,6 +43,8 @@ export const config = {
 
 export type IngestManagerConfigType = TypeOf<typeof config.schema>;
 
+export { DatasourceServiceInterface } from './services/datasource';
+
 export const plugin = (initializerContext: PluginInitializerContext) => {
   return new IngestManagerPlugin(initializerContext);
 };

x-pack/plugins/ingest_manager/server/mocks.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { loggingSystemMock, savedObjectsServiceMock } from 'src/core/server/mocks';
+import { IngestManagerAppContext } from './plugin';
+import { encryptedSavedObjectsMock } from '../../encrypted_saved_objects/server/mocks';
+import { securityMock } from '../../security/server/mocks';
+import { DatasourceServiceInterface } from './services/datasource';
+
+export const createAppContextStartContractMock = (): IngestManagerAppContext => {
+  return {
+    encryptedSavedObjectsStart: encryptedSavedObjectsMock.createStart(),
+    savedObjects: savedObjectsServiceMock.createStartContract(),
+    security: securityMock.createSetup(),
+    logger: loggingSystemMock.create().get(),
+    isProductionMode: true,
+    kibanaVersion: '8.0.0',
+  };
+};
+
+export const createDatasourceServiceMock = () => {
+  return {
+    assignPackageStream: jest.fn(),
+    buildDatasourceFromPackage: jest.fn(),
+    bulkCreate: jest.fn(),
+    create: jest.fn(),
+    delete: jest.fn(),
+    get: jest.fn(),
+    getByIDs: jest.fn(),
+    list: jest.fn(),
+    update: jest.fn(),
+  } as jest.Mocked<DatasourceServiceInterface>;
+};

x-pack/plugins/ingest_manager/server/plugin.ts

@@ -45,13 +45,14 @@ import {
   registerSettingsRoutes,
   registerAppRoutes,
 } from './routes';
-import { IngestManagerConfigType } from '../common';
+import { IngestManagerConfigType, NewDatasource } from '../common';
 import {
   appContextService,
   licenseService,
   ESIndexPatternSavedObjectService,
   ESIndexPatternService,
   AgentService,
+  datasourceService,
 } from './services';
 import { getAgentStatusById } from './services/agents';
 import { CloudSetup } from '../../cloud/server';
@@ -92,12 +93,31 @@ const allSavedObjectTypes = [
   ENROLLMENT_API_KEYS_SAVED_OBJECT_TYPE,
 ];
 
+/**
+ * Callbacks supported by the Ingest plugin
+ */
+export type ExternalCallback = [
+  'datasourceCreate',
+  (newDatasource: NewDatasource) => Promise<NewDatasource>
+];
+
+export type ExternalCallbacksStorage = Map<ExternalCallback[0], Set<ExternalCallback[1]>>;
+
 /**
  * Describes public IngestManager plugin contract returned at the `startup` stage.
  */
 export interface IngestManagerStartContract {
   esIndexPatternService: ESIndexPatternService;
   agentService: AgentService;
+  /**
+   * Services for Ingest's Datasources
+   */
+  datasourceService: typeof datasourceService;
+  /**
+   * Register callbacks for inclusion in ingest API processing
+   * @param args
+   */
+  registerExternalCallback: (...args: ExternalCallback) => void;
 }
 
 export class IngestManagerPlugin
@@ -237,6 +257,10 @@ export class IngestManagerPlugin
       agentService: {
         getAgentStatusById,
       },
+      datasourceService,
+      registerExternalCallback: (...args: ExternalCallback) => {
+        return appContextService.addExternalCallback(...args);
+      },
     };
   }