[ML] Adding space aware jobs (#77916)

* [ML] Adding space aware jobs

* adding mlClient

* switching to type includes

* adding additional job checks

* fixing conflict

* adding dfa checks

* refactoring jobs in spaces checks

* filtering calendars

* adding initial job object status and repair endpoints

* enabling repair endpoint

* fixing listed jobs in status

* adding datafeed repair

* updating shared services

* adding results job id check

* fixing conflicts

* don't remove SO on delete

* fixing non-ml plugins

* filtering job audit messages

* fixing types

* fixing tests

* adding job ids wildcard support

* removing empty migration test

* fixing tests and disabling spaces test user

* adding saved objects all permission

* fixing calendars

* updating job 404

* updating job wildcard search

* renaming services

* fixing conflicts

* fixing log tests

* disabling apm test

* skipping more apm tests

* optimzing repair

* fixing types

* updating apm test archive to include ML saved objects

* enabling disabled test

* removing comment

* adding space assigning endpoints

* adding saved object default permissions

* removing commented code

* loading all jobs for all spaces for status check

* adding spaces list endpoint

* adding job spaces to management page

* adding trained model fltering

* fixing trained model id check and job wildcard check

* fixing types

* fixing bug when adding new job to calendar

* changes based on review

* updating schema

* changes based on review

* fixing types

* rolling back http service injection

* fixing http service injection

* adding errrors to repair endpoint response

* updating api doc

* improving types

* disabling id check on ad get endpoints

* fixing tests

* fixing group requests

* adding comments

* using filter in saved object search

* fixing fake request issue

* removing console log

* making job saved object hidden

* removing acccidentally included file

* renaming saved object client

* updating apidoc

* unhiding ml saved objects

* moving route guard

* improving error when SOC is null

* fixing types after merge with master

* fixing tests

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Author: 3 people

x-pack/plugins/apm/server/lib/alerts/register_transaction_duration_anomaly_alert_type.ts

@@ -73,8 +73,14 @@ export function registerTransactionDurationAnomalyAlertType({
       }
       const alertParams = params;
       const request = {} as KibanaRequest;
-      const { mlAnomalySearch } = ml.mlSystemProvider(request);
-      const anomalyDetectors = ml.anomalyDetectorsProvider(request);
+      const { mlAnomalySearch } = ml.mlSystemProvider(
+        request,
+        services.savedObjectsClient
+      );
+      const anomalyDetectors = ml.anomalyDetectorsProvider(
+        request,
+        services.savedObjectsClient
+      );
 
       const mlJobs = await getMLJobs(anomalyDetectors, alertParams.environment);
 
@@ -94,6 +100,7 @@ export function registerTransactionDurationAnomalyAlertType({
         return {};
       }
 
+      const jobIds = mlJobs.map((job) => job.job_id);
       const anomalySearchParams = {
         terminateAfter: 1,
         body: {
@@ -102,7 +109,7 @@ export function registerTransactionDurationAnomalyAlertType({
             bool: {
               filter: [
                 { term: { result_type: 'record' } },
-                { terms: { job_id: mlJobs.map((job) => job.job_id) } },
+                { terms: { job_id: jobIds } },
                 {
                   range: {
                     timestamp: {
@@ -163,7 +170,8 @@ export function registerTransactionDurationAnomalyAlertType({
       };
 
       const response = ((await mlAnomalySearch(
-        anomalySearchParams
+        anomalySearchParams,
+        jobIds
       )) as unknown) as {
         hits: { total: { value: number } };
         aggregations?: {

x-pack/plugins/apm/server/lib/helpers/setup_request.ts

@@ -123,8 +123,8 @@ function getMlSetup(
   request: KibanaRequest
 ) {
   return {
-    mlSystem: ml.mlSystemProvider(request),
-    anomalyDetectors: ml.anomalyDetectorsProvider(request),
+    mlSystem: ml.mlSystemProvider(request, savedObjectsClient),
+    anomalyDetectors: ml.anomalyDetectorsProvider(request, savedObjectsClient),
     modules: ml.modulesProvider(request, savedObjectsClient),
   };
 }

x-pack/plugins/apm/server/lib/service_map/get_service_anomalies.ts

@@ -104,7 +104,7 @@ export async function getServiceAnomalies({
     },
   };
 
-  const response = await ml.mlSystem.mlAnomalySearch(params);
+  const response = await ml.mlSystem.mlAnomalySearch(params, mlJobIds);
 
   return {
     mlJobIds,

x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/fetcher.ts

@@ -81,7 +81,7 @@ export async function anomalySeriesFetcher({
     const response: ESSearchResponse<
       unknown,
       typeof params
-    > = (await ml.mlSystem.mlAnomalySearch(params)) as any;
+    > = (await ml.mlSystem.mlAnomalySearch(params, [jobId])) as any;
 
     return response;
   } catch (err) {

x-pack/plugins/apm/server/lib/transactions/charts/get_anomaly_data/get_ml_bucket_size.ts

@@ -49,7 +49,7 @@ export async function getMlBucketSize({
   };
 
   try {
-    const resp = await ml.mlSystem.mlAnomalySearch<ESResponse>(params);
+    const resp = await ml.mlSystem.mlAnomalySearch<ESResponse>(params, [jobId]);
     return resp.hits.hits[0]?._source.bucket_span;
   } catch (err) {
     const isHttpError = 'statusCode' in err;

x-pack/plugins/infra/server/lib/infra_ml/common.ts

@@ -62,7 +62,8 @@ export async function getLogEntryDatasets(
           endTime,
           COMPOSITE_AGGREGATION_BATCH_SIZE,
           afterLatestBatchKey
-        )
+        ),
+        jobIds
       )
     );
 

x-pack/plugins/infra/server/lib/infra_ml/metrics_hosts_anomalies.ts

@@ -177,7 +177,8 @@ async function fetchMetricsHostsAnomalies(
 
   const results = decodeOrThrow(metricsHostsAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination)
+      createMetricsHostsAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination),
+      jobIds
     )
   );
 

x-pack/plugins/infra/server/lib/infra_ml/metrics_k8s_anomalies.ts

@@ -173,7 +173,8 @@ async function fetchMetricK8sAnomalies(
 
   const results = decodeOrThrow(metricsK8sAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createMetricsK8sAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination)
+      createMetricsK8sAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination),
+      jobIds
     )
   );
 

x-pack/plugins/infra/server/lib/log_analysis/common.ts

@@ -62,7 +62,8 @@ export async function getLogEntryDatasets(
           endTime,
           COMPOSITE_AGGREGATION_BATCH_SIZE,
           afterLatestBatchKey
-        )
+        ),
+        jobIds
       )
     );
 

x-pack/plugins/infra/server/lib/log_analysis/log_entry_anomalies.ts

@@ -223,7 +223,8 @@ async function fetchLogEntryAnomalies(
 
   const results = decodeOrThrow(logEntryAnomaliesResponseRT)(
     await mlSystem.mlAnomalySearch(
-      createLogEntryAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination, datasets)
+      createLogEntryAnomaliesQuery(jobIds, startTime, endTime, sort, expandedPagination, datasets),
+      jobIds
     )
   );