[ML] job saved objects initialization (#82639) (#83115)

* [ML] job saved objects initialization

* fixing job count logic

* adding missing files

* attempting to fix build crash

* fixing kibana.json

* changes based on review

* removing accidentally added export

* adding intialization promise

* use finally so errors dont stop initialization

* function rename

* removing duplicate header

* adding job initialization count to log message

* adding error to log message

* moving initialization file

* moving intialization file back again to fix git stash issues

* removing .kibana index search

* creating internal saved object client

* code clean up

* removing commented code

* adding check for spaces enabled

* adding ids to saved objects

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

x-pack/plugins/ml/common/types/capabilities.ts

@@ -6,6 +6,7 @@
 
 import { KibanaRequest } from 'kibana/server';
 import { PLUGIN_ID } from '../constants/app';
+import { ML_SAVED_OBJECT_TYPE } from './saved_objects';
 
 export const apmUserMlCapabilities = {
   canGetJobs: false,
@@ -78,7 +79,13 @@ export function getPluginPrivileges() {
   const adminMlCapabilitiesKeys = Object.keys(adminMlCapabilities);
   const allMlCapabilitiesKeys = [...adminMlCapabilitiesKeys, ...userMlCapabilitiesKeys];
   // TODO: include ML in base privileges for the `8.0` release: https://github.com/elastic/kibana/issues/71422
-  const savedObjects = ['index-pattern', 'dashboard', 'search', 'visualization', 'ml-job'];
+  const savedObjects = [
+    'index-pattern',
+    'dashboard',
+    'search',
+    'visualization',
+    ML_SAVED_OBJECT_TYPE,
+  ];
   const privilege = {
     app: [PLUGIN_ID, 'kibana'],
     excludeFromBasePrivileges: true,

x-pack/plugins/ml/common/types/saved_objects.ts

@@ -5,3 +5,4 @@
  */
 
 export type JobType = 'anomaly-detector' | 'data-frame-analytics';
+export const ML_SAVED_OBJECT_TYPE = 'ml-job';

x-pack/plugins/ml/public/application/services/ml_api_service/saved_objects.ts

@@ -9,6 +9,7 @@
 import { HttpService } from '../http_service';
 
 import { basePath } from './index';
+import { JobType } from '../../../../common/types/saved_objects';
 
 export const savedObjectsApiProvider = (httpService: HttpService) => ({
   jobsSpaces() {
@@ -17,4 +18,20 @@ export const savedObjectsApiProvider = (httpService: HttpService) => ({
       method: 'GET',
     });
   },
+  assignJobToSpace(jobType: JobType, jobIds: string[], spaces: string[]) {
+    const body = JSON.stringify({ jobType, jobIds, spaces });
+    return httpService.http<any>({
+      path: `${basePath()}/saved_objects/assign_job_to_space`,
+      method: 'POST',
+      body,
+    });
+  },
+  removeJobFromSpace(jobType: JobType, jobIds: string[], spaces: string[]) {
+    const body = JSON.stringify({ jobType, jobIds, spaces });
+    return httpService.http<any>({
+      path: `${basePath()}/saved_objects/remove_job_from_space`,
+      method: 'POST',
+      body,
+    });
+  },
 });

x-pack/plugins/ml/server/lib/capabilities/check_capabilities.ts

@@ -6,7 +6,7 @@
 
 import { KibanaRequest } from 'kibana/server';
 import type { MlClient } from '../../lib/ml_client';
-import { mlLog } from '../../client/log';
+import { mlLog } from '../../lib/log';
 import {
   MlCapabilities,
   adminMlCapabilities,

x-pack/plugins/ml/server/lib/capabilities/upgrade.ts

@@ -5,7 +5,7 @@
  */
 
 import type { MlClient } from '../../lib/ml_client';
-import { mlLog } from '../../client/log';
+import { mlLog } from '../../lib/log';
 
 export function upgradeCheckProvider(mlClient: MlClient) {
   async function isUpgradeInProgress(): Promise<boolean> {

x-pack/plugins/ml/server/lib/check_annotations/index.ts

@@ -5,7 +5,7 @@
  */
 
 import { IScopedClusterClient } from 'kibana/server';
-import { mlLog } from '../../client/log';
+import { mlLog } from '../../lib/log';
 
 import {
   ML_ANNOTATIONS_INDEX_ALIAS_READ,

x-pack/plugins/ml/server/lib/route_guard.ts

@@ -12,6 +12,7 @@ import {
   RequestHandler,
   SavedObjectsClientContract,
 } from 'kibana/server';
+import { SpacesPluginSetup } from '../../../spaces/server';
 
 import { jobSavedObjectServiceFactory, JobSavedObjectService } from '../saved_objects';
 import { MlLicense } from '../../common/license';
@@ -28,14 +29,27 @@ type Handler = (handlerParams: {
 }) => ReturnType<RequestHandler>;
 
 type GetMlSavedObjectClient = (request: KibanaRequest) => SavedObjectsClientContract | null;
+type GetInternalSavedObjectClient = () => SavedObjectsClientContract | null;
 
 export class RouteGuard {
   private _mlLicense: MlLicense;
   private _getMlSavedObjectClient: GetMlSavedObjectClient;
+  private _getInternalSavedObjectClient: GetInternalSavedObjectClient;
+  private _spacesPlugin: SpacesPluginSetup | undefined;
+  private _isMlReady: () => Promise<void>;
 
-  constructor(mlLicense: MlLicense, getSavedObject: GetMlSavedObjectClient) {
+  constructor(
+    mlLicense: MlLicense,
+    getSavedObject: GetMlSavedObjectClient,
+    getInternalSavedObject: GetInternalSavedObjectClient,
+    spacesPlugin: SpacesPluginSetup | undefined,
+    isMlReady: () => Promise<void>
+  ) {
     this._mlLicense = mlLicense;
     this._getMlSavedObjectClient = getSavedObject;
+    this._getInternalSavedObjectClient = getInternalSavedObject;
+    this._spacesPlugin = spacesPlugin;
+    this._isMlReady = isMlReady;
   }
 
   public fullLicenseAPIGuard(handler: Handler) {
@@ -56,13 +70,19 @@ export class RouteGuard {
       }
 
       const mlSavedObjectClient = this._getMlSavedObjectClient(request);
-      if (mlSavedObjectClient === null) {
+      const internalSavedObjectsClient = this._getInternalSavedObjectClient();
+      if (mlSavedObjectClient === null || internalSavedObjectsClient === null) {
         return response.badRequest({
           body: { message: 'saved object client has not been initialized' },
         });
       }
 
-      const jobSavedObjectService = jobSavedObjectServiceFactory(mlSavedObjectClient);
+      const jobSavedObjectService = jobSavedObjectServiceFactory(
+        mlSavedObjectClient,
+        internalSavedObjectsClient,
+        this._spacesPlugin !== undefined,
+        this._isMlReady
+      );
       const client = context.core.elasticsearch.client;
 
       return handler({

x-pack/plugins/ml/server/lib/spaces_utils.ts

@@ -6,36 +6,31 @@
 
 import { Legacy } from 'kibana';
 import { KibanaRequest } from 'kibana/server';
-import { Space, SpacesPluginSetup } from '../../../spaces/server';
+import { SpacesPluginSetup } from '../../../spaces/server';
 
 export type RequestFacade = KibanaRequest | Legacy.Request;
 
-interface GetActiveSpaceResponse {
-  valid: boolean;
-  space?: Space;
-}
-
-export function spacesUtilsProvider(spacesPlugin: SpacesPluginSetup, request: RequestFacade) {
-  async function activeSpace(): Promise<GetActiveSpaceResponse> {
-    try {
-      return {
-        valid: true,
-        space: await spacesPlugin.spacesService.getActiveSpace(request),
-      };
-    } catch (e) {
-      return {
-        valid: false,
-      };
+export function spacesUtilsProvider(
+  spacesPlugin: SpacesPluginSetup | undefined,
+  request: RequestFacade
+) {
+  async function isMlEnabledInSpace(): Promise<boolean> {
+    if (spacesPlugin === undefined) {
+      // if spaces is disabled force isMlEnabledInSpace to be true
+      return true;
     }
+    const space = await spacesPlugin.spacesService.getActiveSpace(request);
+    return space.disabledFeatures.includes('ml') === false;
   }
 
-  async function isMlEnabledInSpace(): Promise<boolean> {
-    const { valid, space } = await activeSpace();
-    if (valid === true && space !== undefined) {
-      return space.disabledFeatures.includes('ml') === false;
+  async function getAllSpaces(): Promise<string[] | null> {
+    if (spacesPlugin === undefined) {
+      return null;
     }
-    return true;
+    const client = await spacesPlugin.spacesService.scopedClient(request);
+    const spaces = await client.getAll();
+    return spaces.map((s) => s.id);
   }
 
-  return { isMlEnabledInSpace };
+  return { isMlEnabledInSpace, getAllSpaces };
 }

x-pack/plugins/ml/server/models/bucket_span_estimator/bucket_span_estimator.js

@@ -6,7 +6,7 @@
 
 import { cloneDeep, each, remove, sortBy, get } from 'lodash';
 
-import { mlLog } from '../../client/log';
+import { mlLog } from '../../lib/log';
 
 import { INTERVALS } from './intervals';
 import { singleSeriesCheckerFactory } from './single_series_checker';

x-pack/plugins/ml/server/models/bucket_span_estimator/single_series_checker.js

@@ -10,7 +10,7 @@
  * Bucket spans: 5m, 10m, 30m, 1h, 3h
  */
 
-import { mlLog } from '../../client/log';
+import { mlLog } from '../../lib/log';
 import { INTERVALS, LONG_INTERVALS } from './intervals';
 
 export function singleSeriesCheckerFactory({ asCurrentUser }) {

x-pack/plugins/ml/server/models/data_recognizer/data_recognizer.ts

@@ -37,7 +37,7 @@ import {
   prefixDatafeedId,
   splitIndexPatternNames,
 } from '../../../common/util/job_utils';
-import { mlLog } from '../../client/log';
+import { mlLog } from '../../lib/log';
 import { calculateModelMemoryLimitProvider } from '../calculate_model_memory_limit';
 import { fieldsServiceProvider } from '../fields_service';
 import { jobServiceProvider } from '../job_service';

x-pack/plugins/ml/server/plugin.ts

@@ -15,15 +15,15 @@ import {
   CapabilitiesStart,
   IClusterClient,
   SavedObjectsServiceStart,
-  SavedObjectsClientContract,
 } from 'kibana/server';
 import { DEFAULT_APP_CATEGORIES } from '../../../../src/core/server';
+import { SpacesPluginSetup } from '../../spaces/server';
 import { PluginsSetup, RouteInitialization } from './types';
 import { PLUGIN_ID } from '../common/constants/app';
 import { MlCapabilities } from '../common/types/capabilities';
 
 import { initMlTelemetry } from './lib/telemetry';
-import { initMlServerLog } from './client/log';
+import { initMlServerLog } from './lib/log';
 import { initSampleDataSets } from './lib/sample_data_sets';
 
 import { annotationRoutes } from './routes/annotations';
@@ -50,7 +50,11 @@ import { getPluginPrivileges } from '../common/types/capabilities';
 import { setupCapabilitiesSwitcher } from './lib/capabilities';
 import { registerKibanaSettings } from './lib/register_settings';
 import { trainedModelsRoutes } from './routes/trained_models';
-import { setupSavedObjects } from './saved_objects';
+import {
+  setupSavedObjects,
+  jobSavedObjectsInitializationFactory,
+  savedObjectClientsFactory,
+} from './saved_objects';
 import { RouteGuard } from './lib/route_guard';
 
 export type MlPluginSetup = SharedServices;
@@ -63,14 +67,19 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
   private capabilities: CapabilitiesStart | null = null;
   private clusterClient: IClusterClient | null = null;
   private savedObjectsStart: SavedObjectsServiceStart | null = null;
+  private spacesPlugin: SpacesPluginSetup | undefined;
+  private isMlReady: Promise<void>;
+  private setMlReady: () => void = () => {};
 
   constructor(ctx: PluginInitializerContext) {
     this.log = ctx.logger.get();
     this.version = ctx.env.packageInfo.branch;
     this.mlLicense = new MlLicense();
+    this.isMlReady = new Promise((resolve) => (this.setMlReady = resolve));
   }
 
   public setup(coreSetup: CoreSetup, plugins: PluginsSetup): MlPluginSetup {
+    this.spacesPlugin = plugins.spaces;
     const { admin, user, apmUser } = getPluginPrivileges();
 
     plugins.features.registerKibanaFeature({
@@ -120,18 +129,19 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
     setupCapabilitiesSwitcher(coreSetup, plugins.licensing.license$, this.log);
     setupSavedObjects(coreSetup.savedObjects);
 
-    const getMlSavedObjectsClient = (request: KibanaRequest): SavedObjectsClientContract | null => {
-      if (this.savedObjectsStart === null) {
-        return null;
-      }
-      return this.savedObjectsStart.getScopedClient(request, {
-        includedHiddenTypes: ['ml-job'],
-      });
-    };
+    const { getInternalSavedObjectsClient, getMlSavedObjectsClient } = savedObjectClientsFactory(
+      () => this.savedObjectsStart
+    );
 
     const routeInit: RouteInitialization = {
       router: coreSetup.http.createRouter(),
-      routeGuard: new RouteGuard(this.mlLicense, getMlSavedObjectsClient),
+      routeGuard: new RouteGuard(
+        this.mlLicense,
+        getMlSavedObjectsClient,
+        getInternalSavedObjectsClient,
+        plugins.spaces,
+        () => this.isMlReady
+      ),
       mlLicense: this.mlLicense,
     };
 
@@ -176,7 +186,9 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
         plugins.spaces,
         plugins.cloud,
         resolveMlCapabilities,
-        () => this.clusterClient
+        () => this.clusterClient,
+        () => getInternalSavedObjectsClient(),
+        () => this.isMlReady
       ),
     };
   }
@@ -185,6 +197,16 @@ export class MlServerPlugin implements Plugin<MlPluginSetup, MlPluginStart, Plug
     this.capabilities = coreStart.capabilities;
     this.clusterClient = coreStart.elasticsearch.client;
     this.savedObjectsStart = coreStart.savedObjects;
+
+    // check whether the job saved objects exist
+    // and create them if needed.
+    const { initializeJobs } = jobSavedObjectsInitializationFactory(
+      coreStart,
+      this.spacesPlugin !== undefined
+    );
+    initializeJobs().finally(() => {
+      this.setMlReady();
+    });
   }
 
   public stop() {

x-pack/plugins/ml/server/routes/saved_objects.ts

@@ -6,7 +6,7 @@
 
 import { wrapError } from '../client/error_wrapper';
 import { RouteInitialization } from '../types';
-import { checksFactory } from '../saved_objects';
+import { checksFactory, repairFactory } from '../saved_objects';
 import { jobsAndSpaces, repairJobObjects } from './schemas/saved_objects';
 
 /**
@@ -67,7 +67,7 @@ export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization)
     routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => {
       try {
         const { simulate } = request.query;
-        const { repairJobs } = checksFactory(client, jobSavedObjectService);
+        const { repairJobs } = repairFactory(client, jobSavedObjectService);
         const savedObjects = await repairJobs(simulate);
 
         return response.ok({
@@ -100,7 +100,7 @@ export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization)
     routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => {
       try {
         const { simulate } = request.query;
-        const { initSavedObjects } = checksFactory(client, jobSavedObjectService);
+        const { initSavedObjects } = repairFactory(client, jobSavedObjectService);
         const savedObjects = await initSavedObjects(simulate);
 
         return response.ok({

x-pack/plugins/ml/server/routes/system.ts

@@ -9,7 +9,7 @@ import { schema } from '@kbn/config-schema';
 import { Request } from '@hapi/hapi';
 import { IScopedClusterClient } from 'kibana/server';
 import { wrapError } from '../client/error_wrapper';
-import { mlLog } from '../client/log';
+import { mlLog } from '../lib/log';
 import { capabilitiesProvider } from '../lib/capabilities';
 import { spacesUtilsProvider } from '../lib/spaces_utils';
 import { RouteInitialization, SystemRouteDeps } from '../types';
@@ -117,11 +117,7 @@ export function systemRoutes(
     },
     routeGuard.basicLicenseAPIGuard(async ({ mlClient, request, response }) => {
       try {
-        // if spaces is disabled force isMlEnabledInSpace to be true
-        const { isMlEnabledInSpace } =
-          spaces !== undefined
-            ? spacesUtilsProvider(spaces, (request as unknown) as Request)
-            : { isMlEnabledInSpace: async () => true };
+        const { isMlEnabledInSpace } = spacesUtilsProvider(spaces, (request as unknown) as Request);
 
         const mlCapabilities = await resolveMlCapabilities(request);
         if (mlCapabilities === null) {