From 3701886895babdd7480d19759202069d45ec8474 Mon Sep 17 00:00:00 2001 From: James Gowdy Date: Tue, 10 Nov 2020 12:05:41 +0000 Subject: [PATCH] removing .kibana index search --- x-pack/plugins/ml/server/lib/spaces_utils.ts | 41 +++++------ x-pack/plugins/ml/server/plugin.ts | 2 +- .../plugins/ml/server/routes/saved_objects.ts | 23 ++++-- x-pack/plugins/ml/server/routes/system.ts | 6 +- .../plugins/ml/server/saved_objects/checks.ts | 73 ++++++------------- .../ml/server/saved_objects/initialization.ts | 54 +++++++------- .../plugins/ml/server/saved_objects/repair.ts | 9 ++- .../ml/server/saved_objects/service.ts | 2 +- .../shared_services/providers/system.ts | 5 +- 9 files changed, 92 insertions(+), 123 deletions(-) diff --git a/x-pack/plugins/ml/server/lib/spaces_utils.ts b/x-pack/plugins/ml/server/lib/spaces_utils.ts index c3f5b013e36e5b..e1606061b0ff19 100644 --- a/x-pack/plugins/ml/server/lib/spaces_utils.ts +++ b/x-pack/plugins/ml/server/lib/spaces_utils.ts @@ -6,36 +6,31 @@ import { Legacy } from 'kibana'; import { KibanaRequest } from 'kibana/server'; -import { Space, SpacesPluginSetup } from '../../../spaces/server'; +import { SpacesPluginSetup } from '../../../spaces/server'; export type RequestFacade = KibanaRequest | Legacy.Request; -interface GetActiveSpaceResponse { - valid: boolean; - space?: Space; -} - -export function spacesUtilsProvider(spacesPlugin: SpacesPluginSetup, request: RequestFacade) { - async function activeSpace(): Promise { - try { - return { - valid: true, - space: await spacesPlugin.spacesService.getActiveSpace(request), - }; - } catch (e) { - return { - valid: false, - }; +export function spacesUtilsProvider( + spacesPlugin: SpacesPluginSetup | undefined, + request: RequestFacade +) { + async function isMlEnabledInSpace(): Promise { + if (spacesPlugin === undefined) { + // if spaces is disabled force isMlEnabledInSpace to be true + return true; } + const space = await spacesPlugin.spacesService.getActiveSpace(request); + return space.disabledFeatures.includes('ml') === false; } - async function isMlEnabledInSpace(): Promise { - const { valid, space } = await activeSpace(); - if (valid === true && space !== undefined) { - return space.disabledFeatures.includes('ml') === false; + async function getAllSpaces(): Promise { + if (spacesPlugin === undefined) { + return null; } - return true; + const client = await spacesPlugin.spacesService.scopedClient(request); + const spaces = await client.getAll(); + return spaces.map((s) => s.id); } - return { isMlEnabledInSpace }; + return { isMlEnabledInSpace, getAllSpaces }; } diff --git a/x-pack/plugins/ml/server/plugin.ts b/x-pack/plugins/ml/server/plugin.ts index 182a4b388ac44b..e949c926692cba 100644 --- a/x-pack/plugins/ml/server/plugin.ts +++ b/x-pack/plugins/ml/server/plugin.ts @@ -162,7 +162,7 @@ export class MlServerPlugin implements Plugin { + routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => { try { - const { checkStatus } = checksFactory(client, jobSavedObjectService); + const { checkStatus } = checksFactory(client, jobSavedObjectService, request, spacesPlugin); const status = await checkStatus(); return response.ok({ @@ -67,7 +71,7 @@ export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization) routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => { try { const { simulate } = request.query; - const { repairJobs } = repairFactory(client, jobSavedObjectService); + const { repairJobs } = repairFactory(client, jobSavedObjectService, request, spacesPlugin); const savedObjects = await repairJobs(simulate); return response.ok({ @@ -100,7 +104,12 @@ export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization) routeGuard.fullLicenseAPIGuard(async ({ client, request, response, jobSavedObjectService }) => { try { const { simulate } = request.query; - const { initSavedObjects } = repairFactory(client, jobSavedObjectService); + const { initSavedObjects } = repairFactory( + client, + jobSavedObjectService, + request, + spacesPlugin + ); const savedObjects = await initSavedObjects(simulate); return response.ok({ @@ -196,9 +205,9 @@ export function savedObjectsRoutes({ router, routeGuard }: RouteInitialization) tags: ['access:ml:canGetJobs'], }, }, - routeGuard.fullLicenseAPIGuard(async ({ response, jobSavedObjectService, client }) => { + routeGuard.fullLicenseAPIGuard(async ({ request, response, jobSavedObjectService, client }) => { try { - const { checkStatus } = checksFactory(client, jobSavedObjectService); + const { checkStatus } = checksFactory(client, jobSavedObjectService, request, spacesPlugin); const allStatuses = Object.values((await checkStatus()).savedObjects).flat(); const body = allStatuses diff --git a/x-pack/plugins/ml/server/routes/system.ts b/x-pack/plugins/ml/server/routes/system.ts index 96524fe77fc6f5..412108e4fee13e 100644 --- a/x-pack/plugins/ml/server/routes/system.ts +++ b/x-pack/plugins/ml/server/routes/system.ts @@ -117,11 +117,7 @@ export function systemRoutes( }, routeGuard.basicLicenseAPIGuard(async ({ mlClient, request, response }) => { try { - // if spaces is disabled force isMlEnabledInSpace to be true - const { isMlEnabledInSpace } = - spaces !== undefined - ? spacesUtilsProvider(spaces, (request as unknown) as Request) - : { isMlEnabledInSpace: async () => true }; + const { isMlEnabledInSpace } = spacesUtilsProvider(spaces, (request as unknown) as Request); const mlCapabilities = await resolveMlCapabilities(request); if (mlCapabilities === null) { diff --git a/x-pack/plugins/ml/server/saved_objects/checks.ts b/x-pack/plugins/ml/server/saved_objects/checks.ts index a910f2fa18a77d..c11419ae25a346 100644 --- a/x-pack/plugins/ml/server/saved_objects/checks.ts +++ b/x-pack/plugins/ml/server/saved_objects/checks.ts @@ -4,14 +4,15 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IScopedClusterClient } from 'kibana/server'; -import { SearchResponse } from 'elasticsearch'; +import { IScopedClusterClient, KibanaRequest } from 'kibana/server'; +import type { SpacesPluginSetup } from '../../../spaces/server'; import type { JobSavedObjectService } from './service'; -import { JobType, ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects'; +import { JobType } from '../../common/types/saved_objects'; import { Job } from '../../common/types/anomaly_detection_jobs'; import { Datafeed } from '../../common/types/anomaly_detection_jobs'; import { DataFrameAnalyticsConfig } from '../../common/types/data_frame_analytics'; +import { spacesUtilsProvider } from '../lib/spaces_utils'; interface JobSavedObjectStatus { jobId: string; @@ -32,14 +33,6 @@ interface JobStatus { }; } -interface SavedObjectJob { - [ML_SAVED_OBJECT_TYPE]: { - job_id: string; - type: JobType; - }; - namespaces: string[]; -} - interface StatusResponse { savedObjects: { [type in JobType]: JobSavedObjectStatus[]; @@ -51,10 +44,21 @@ interface StatusResponse { export function checksFactory( client: IScopedClusterClient, - jobSavedObjectService: JobSavedObjectService + jobSavedObjectService: JobSavedObjectService, + request: KibanaRequest, + spacesPlugin?: SpacesPluginSetup ) { + const { getAllSpaces } = spacesUtilsProvider(spacesPlugin, request); async function checkStatus(): Promise { - const jobObjects = await jobSavedObjectService.getAllJobObjects(undefined, false); + const allSpaces = await getAllSpaces(); + const allJobObjects = await jobSavedObjectService.getAllJobObjects(undefined, false); + const jobObjects = + allSpaces === null + ? allJobObjects + : allJobObjects.filter((j) => + j.namespaces?.some((s) => s === '*' || allSpaces.includes(s)) + ); + // load all non-space jobs and datafeeds const { body: adJobs } = await client.asInternalUser.ml.getJobs<{ jobs: Job[] }>(); const { body: datafeeds } = await client.asInternalUser.ml.getDatafeeds<{ @@ -93,16 +97,15 @@ export function checksFactory( } ); - const nonSpaceSavedObjects = await _loadAllJobSavedObjects(); const nonSpaceADObjectIds = new Set( - nonSpaceSavedObjects - .filter(({ type }) => type === 'anomaly-detector') - .map(({ jobId }) => jobId) + allJobObjects + .filter(({ attributes }) => attributes.type === 'anomaly-detector') + .map(({ attributes }) => attributes.job_id) ); const nonSpaceDFAObjectIds = new Set( - nonSpaceSavedObjects - .filter(({ type }) => type === 'data-frame-analytics') - .map(({ jobId }) => jobId) + allJobObjects + .filter(({ attributes }) => attributes.type === 'data-frame-analytics') + .map(({ attributes }) => attributes.job_id) ); const adObjectIds = new Set( @@ -161,35 +164,5 @@ export function checksFactory( }; } - async function _loadAllJobSavedObjects() { - const { body } = await client.asInternalUser.search>({ - index: '.kibana*', - size: 1000, - _source: ['ml-job.job_id', 'ml-job.type', 'namespaces'], - body: { - query: { - bool: { - filter: [ - { - term: { - type: ML_SAVED_OBJECT_TYPE, - }, - }, - ], - }, - }, - }, - }); - - return body.hits.hits.map(({ _source }) => { - const { job_id: jobId, type } = _source[ML_SAVED_OBJECT_TYPE]; - return { - jobId, - type, - spaces: _source.namespaces, - }; - }); - } - return { checkStatus }; } diff --git a/x-pack/plugins/ml/server/saved_objects/initialization.ts b/x-pack/plugins/ml/server/saved_objects/initialization.ts index 11f1497b906fae..149adc29a8974f 100644 --- a/x-pack/plugins/ml/server/saved_objects/initialization.ts +++ b/x-pack/plugins/ml/server/saved_objects/initialization.ts @@ -4,10 +4,10 @@ * you may not use this file except in compliance with the Elastic License. */ -import { IScopedClusterClient, CoreStart } from 'kibana/server'; +import { IScopedClusterClient, CoreStart, SavedObjectsClient, KibanaRequest } from 'kibana/server'; import { getInternalSavedObjectsClient } from './util'; import { repairFactory } from './repair'; -import { jobSavedObjectServiceFactory } from './service'; +import { jobSavedObjectServiceFactory, JobObject } from './service'; import { mlLog } from '../lib/log'; import { ML_SAVED_OBJECT_TYPE } from '../../common/types/saved_objects'; @@ -28,17 +28,24 @@ export function jobSavedObjectsInitializationFactory(core: CoreStart) { */ async function initializeJobs() { try { - if ((await _needsInitializing()) === false) { - // ml job saved objects have already been initialized - return; - } - mlLog.info('Initializing job saved objects'); const savedObjectsClient = getInternalSavedObjectsClient(core); const jobSavedObjectService = jobSavedObjectServiceFactory( savedObjectsClient, () => Promise.resolve() // pretend isMlReady, to allow us to initialize the saved objects ); - const { initSavedObjects } = repairFactory(client, jobSavedObjectService); + + if ((await _needsInitializing(savedObjectsClient)) === false) { + // ml job saved objects have already been initialized + return; + } + + mlLog.info('Initializing job saved objects'); + const { initSavedObjects } = repairFactory( + client, + jobSavedObjectService, + {} as KibanaRequest, // request won't be used as this spaces plugin isn't being used. + undefined // spaces plugin isn't needed, as we'll be repairing all jobs + ); const { jobs } = await initSavedObjects(); mlLog.info(`${jobs.length} job saved objects initialized for * space`); } catch (error) { @@ -46,8 +53,8 @@ export function jobSavedObjectsInitializationFactory(core: CoreStart) { } } - async function _needsInitializing() { - if (await _jobSavedObjectsExist()) { + async function _needsInitializing(savedObjectsClient: SavedObjectsClient) { + if (await _jobSavedObjectsExist(savedObjectsClient)) { // at least one ml saved object exists // this has been initialized before return false; @@ -63,26 +70,15 @@ export function jobSavedObjectsInitializationFactory(core: CoreStart) { return false; } - async function _jobSavedObjectsExist(size: number = 1) { - const { body } = await client.asInternalUser.search({ - index: '.kibana*', - size, - body: { - query: { - bool: { - filter: [ - { - term: { - type: ML_SAVED_OBJECT_TYPE, - }, - }, - ], - }, - }, - }, - }); + async function _jobSavedObjectsExist(savedObjectsClient: SavedObjectsClient) { + const options = { + type: ML_SAVED_OBJECT_TYPE, + perPage: 0, + namespaces: ['*'], + }; - return body.hits.total.value > 0; + const { total } = await savedObjectsClient.find(options); + return total > 0; } async function _jobsExist() { diff --git a/x-pack/plugins/ml/server/saved_objects/repair.ts b/x-pack/plugins/ml/server/saved_objects/repair.ts index 9271032f83aec1..a7ee7bf19d8bdf 100644 --- a/x-pack/plugins/ml/server/saved_objects/repair.ts +++ b/x-pack/plugins/ml/server/saved_objects/repair.ts @@ -5,7 +5,8 @@ */ import Boom from '@hapi/boom'; -import { IScopedClusterClient } from 'kibana/server'; +import { IScopedClusterClient, KibanaRequest } from 'kibana/server'; +import type { SpacesPluginSetup } from '../../../spaces/server'; import type { JobObject, JobSavedObjectService } from './service'; import { JobType } from '../../common/types/saved_objects'; import { checksFactory } from './checks'; @@ -14,9 +15,11 @@ import { Datafeed } from '../../common/types/anomaly_detection_jobs'; export function repairFactory( client: IScopedClusterClient, - jobSavedObjectService: JobSavedObjectService + jobSavedObjectService: JobSavedObjectService, + request: KibanaRequest, + spacesPlugin?: SpacesPluginSetup ) { - const { checkStatus } = checksFactory(client, jobSavedObjectService); + const { checkStatus } = checksFactory(client, jobSavedObjectService, request, spacesPlugin); async function repairJobs(simulate: boolean = false) { type Result = Record; diff --git a/x-pack/plugins/ml/server/saved_objects/service.ts b/x-pack/plugins/ml/server/saved_objects/service.ts index 1ae1b302307be5..52a0aa78347b58 100644 --- a/x-pack/plugins/ml/server/saved_objects/service.ts +++ b/x-pack/plugins/ml/server/saved_objects/service.ts @@ -87,7 +87,7 @@ export function jobSavedObjectServiceFactory( throw new MLJobNotFound('job not found'); } - await savedObjectsClient.delete(ML_SAVED_OBJECT_TYPE, job.id); + await savedObjectsClient.delete(ML_SAVED_OBJECT_TYPE, job.id, { force: true }); } async function createAnomalyDetectionJob(jobId: string, datafeedId?: string) { diff --git a/x-pack/plugins/ml/server/shared_services/providers/system.ts b/x-pack/plugins/ml/server/shared_services/providers/system.ts index d704a3d0e3727a..c7c50eb74595e1 100644 --- a/x-pack/plugins/ml/server/shared_services/providers/system.ts +++ b/x-pack/plugins/ml/server/shared_services/providers/system.ts @@ -44,10 +44,7 @@ export function getMlSystemProvider( return await getGuards(request, savedObjectsClient) .isMinimumLicense() .ok(async ({ mlClient }) => { - const { isMlEnabledInSpace } = - spaces !== undefined - ? spacesUtilsProvider(spaces, request) - : { isMlEnabledInSpace: async () => true }; + const { isMlEnabledInSpace } = spacesUtilsProvider(spaces, request); const mlCapabilities = await resolveMlCapabilities(request); if (mlCapabilities === null) {