[Security Solution] add handling for endpoint package spec v2

endpoint package managed transform names are changing with the endpoint package migration to
package spec v2. this commit updates places where we rely on these transform names. backwards compatible.

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -29,14 +29,20 @@ export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
 /** index that the metadata transform writes to (destination) and that is used by endpoint APIs */
 export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current_*';
 
+// endpoint package V2 has an additional prefix in the transform names
+const PACKAGE_V2_PREFIX = 'logs-';
+
 /** The metadata Transform Name prefix with NO (package) version) */
 export const metadataTransformPrefix = 'endpoint.metadata_current-default';
+export const METADATA_CURRENT_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${metadataTransformPrefix}`;
 
 // metadata transforms pattern for matching all metadata transform ids
 export const METADATA_TRANSFORMS_PATTERN = 'endpoint.metadata_*';
+export const METADATA_TRANSFORMS_PATTERN_V2 = `${PACKAGE_V2_PREFIX}${METADATA_TRANSFORMS_PATTERN}`;
 
 // united metadata transform id
 export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default';
+export const METADATA_UNITED_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${METADATA_UNITED_TRANSFORM}`;
 
 // united metadata transform destination index
 export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default';

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -111,8 +111,8 @@ export const indexHostsAndAlerts = usageTracker.track(
 
     const shouldWaitForEndpointMetadataDocs = fleet;
     if (shouldWaitForEndpointMetadataDocs) {
-      await waitForMetadataTransformsReady(client);
-      await stopMetadataTransforms(client);
+      await waitForMetadataTransformsReady(client, epmEndpointPackage.version);
+      await stopMetadataTransforms(client, epmEndpointPackage.version);
     }
 
     for (let i = 0; i < numHosts; i++) {
@@ -147,7 +147,8 @@ export const indexHostsAndAlerts = usageTracker.track(
     if (shouldWaitForEndpointMetadataDocs) {
       await startMetadataTransforms(
         client,
-        response.agents.map((agent) => agent.agent?.id ?? '')
+        response.agents.map((agent) => agent.agent?.id ?? ''),
+        epmEndpointPackage.version
       );
     }
 

x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts

@@ -5,28 +5,33 @@
  * 2.0.
  */
 
+import semverLte from 'semver/functions/lte';
+
 import type { Client } from '@elastic/elasticsearch';
 import type { TransformGetTransformStatsTransformStats } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import { usageTracker } from '../data_loaders/usage_tracker';
 import {
   metadataCurrentIndexPattern,
   metadataTransformPrefix,
+  METADATA_CURRENT_TRANSFORM_V2,
   METADATA_TRANSFORMS_PATTERN,
+  METADATA_TRANSFORMS_PATTERN_V2,
   METADATA_UNITED_TRANSFORM,
+  METADATA_UNITED_TRANSFORM_V2,
 } from '../constants';
 
 export const waitForMetadataTransformsReady = usageTracker.track(
   'waitForMetadataTransformsReady',
-  async (esClient: Client): Promise<void> => {
-    await waitFor(() => areMetadataTransformsReady(esClient));
+  async (esClient: Client, version: string): Promise<void> => {
+    await waitFor(() => areMetadataTransformsReady(esClient, version));
   }
 );
 
 export const stopMetadataTransforms = usageTracker.track(
   'stopMetadataTransforms',
-  async (esClient: Client): Promise<void> => {
-    const transformIds = await getMetadataTransformIds(esClient);
+  async (esClient: Client, version: string): Promise<void> => {
+    const transformIds = await getMetadataTransformIds(esClient, version);
 
     await Promise.all(
       transformIds.map((transformId) =>
@@ -46,14 +51,18 @@ export const startMetadataTransforms = usageTracker.track(
   async (
     esClient: Client,
     // agentIds to wait for
-    agentIds: string[]
+    agentIds: string[],
+    version: string
   ): Promise<void> => {
-    const transformIds = await getMetadataTransformIds(esClient);
+    const transformIds = await getMetadataTransformIds(esClient, version);
+    const isV2 = isEndpointPackageV2(version);
+    const currentTransformPrefix = isV2 ? METADATA_CURRENT_TRANSFORM_V2 : metadataTransformPrefix;
     const currentTransformId = transformIds.find((transformId) =>
-      transformId.startsWith(metadataTransformPrefix)
+      transformId.startsWith(currentTransformPrefix)
     );
+    const unitedTransformPrefix = isV2 ? METADATA_UNITED_TRANSFORM_V2 : METADATA_UNITED_TRANSFORM;
     const unitedTransformId = transformIds.find((transformId) =>
-      transformId.startsWith(METADATA_UNITED_TRANSFORM)
+      transformId.startsWith(unitedTransformPrefix)
     );
     if (!currentTransformId || !unitedTransformId) {
       // eslint-disable-next-line no-console
@@ -88,22 +97,26 @@ export const startMetadataTransforms = usageTracker.track(
 );
 
 async function getMetadataTransformStats(
-  esClient: Client
+  esClient: Client,
+  version: string
 ): Promise<TransformGetTransformStatsTransformStats[]> {
+  const transformId = isEndpointPackageV2(version)
+    ? METADATA_TRANSFORMS_PATTERN_V2
+    : METADATA_TRANSFORMS_PATTERN;
   return (
     await esClient.transform.getTransformStats({
-      transform_id: METADATA_TRANSFORMS_PATTERN,
+      transform_id: transformId,
       allow_no_match: true,
     })
   ).transforms;
 }
 
-async function getMetadataTransformIds(esClient: Client): Promise<string[]> {
-  return (await getMetadataTransformStats(esClient)).map((transform) => transform.id);
+async function getMetadataTransformIds(esClient: Client, version: string): Promise<string[]> {
+  return (await getMetadataTransformStats(esClient, version)).map((transform) => transform.id);
 }
 
-async function areMetadataTransformsReady(esClient: Client): Promise<boolean> {
-  const transforms = await getMetadataTransformStats(esClient);
+async function areMetadataTransformsReady(esClient: Client, version: string): Promise<boolean> {
+  const transforms = await getMetadataTransformStats(esClient, version);
   return !transforms.some(
     // TODO TransformGetTransformStatsTransformStats type needs to be updated to include health
     (transform: TransformGetTransformStatsTransformStats & { health?: { status: string } }) =>
@@ -159,3 +172,8 @@ async function waitFor(
     }
   }
 }
+
+const MIN_ENDPOINT_PACKAGE_V2_VERSION = '8.12.0';
+export function isEndpointPackageV2(version: string) {
+  return semverLte(MIN_ENDPOINT_PACKAGE_V2_VERSION, version);
+}

x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts

@@ -70,6 +70,7 @@ describe('check metadata transforms task', () => {
           { type: ElasticsearchAssetType.transform } as EsAssetReference,
           { type: ElasticsearchAssetType.transform } as EsAssetReference,
         ],
+        version: '8.11.0',
       } as Installation);
   });
 

x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts

@@ -19,10 +19,14 @@ import type {
 import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server';
 import { ElasticsearchAssetType, FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
 import type { EndpointAppContext } from '../../types';
-import { METADATA_TRANSFORMS_PATTERN } from '../../../../common/endpoint/constants';
+import {
+  METADATA_TRANSFORMS_PATTERN,
+  METADATA_TRANSFORMS_PATTERN_V2,
+} from '../../../../common/endpoint/constants';
 import { WARNING_TRANSFORM_STATES } from '../../../../common/constants';
 import { wrapErrorIfNeeded } from '../../utils';
 import { stateSchemaByVersion, emptyState, type LatestTaskStateSchema } from './task_state';
+import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms';
 
 const SCOPE = ['securitySolution'];
 const INTERVAL = '2h';
@@ -108,11 +112,21 @@ export class CheckMetadataTransformsTask {
     const [{ elasticsearch }] = await core.getStartServices();
     const esClient = elasticsearch.client.asInternalUser;
 
+    const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
+    const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
+    if (!installation) {
+      this.logger.info('no endpoint installation found');
+      return { state: taskInstance.state };
+    }
+
+    const transformName = isEndpointPackageV2(installation.version)
+      ? METADATA_TRANSFORMS_PATTERN_V2
+      : METADATA_TRANSFORMS_PATTERN;
     let transformStatsResponse: TransportResult<TransformGetTransformStatsResponse>;
     try {
       transformStatsResponse = await esClient?.transform.getTransformStats(
         {
-          transform_id: METADATA_TRANSFORMS_PATTERN,
+          transform_id: transformName,
         },
         { meta: true }
       );
@@ -124,12 +138,6 @@ export class CheckMetadataTransformsTask {
       return { state: taskInstance.state };
     }
 
-    const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
-    const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
-    if (!installation) {
-      this.logger.info('no endpoint installation found');
-      return { state: taskInstance.state };
-    }
     const expectedTransforms = installation.installed_es.filter(
       (asset) => asset.type === ElasticsearchAssetType.transform
     );

x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts

@@ -7,6 +7,8 @@
 
 import type { TypeOf } from '@kbn/config-schema';
 import type { Logger, RequestHandler } from '@kbn/core/server';
+import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
+
 import type {
   MetadataListResponse,
   EndpointSortableField,
@@ -25,7 +27,9 @@ import {
   ENDPOINT_DEFAULT_SORT_DIRECTION,
   ENDPOINT_DEFAULT_SORT_FIELD,
   METADATA_TRANSFORMS_PATTERN,
+  METADATA_TRANSFORMS_PATTERN_V2,
 } from '../../../../common/endpoint/constants';
+import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms';
 
 export const getLogger = (endpointAppContext: EndpointAppContext): Logger => {
   return endpointAppContext.logFactory.get('metadata');
@@ -99,13 +103,21 @@ export const getMetadataRequestHandler = function (
 };
 
 export function getMetadataTransformStatsHandler(
+  endpointAppContext: EndpointAppContext,
   logger: Logger
 ): RequestHandler<unknown, unknown, unknown, SecuritySolutionRequestHandlerContext> {
   return async (context, _, response) => {
     const esClient = (await context.core).elasticsearch.client.asInternalUser;
+    const packageClient = endpointAppContext.service.getInternalFleetServices().packages;
+    const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
+    const transformName =
+      installation?.version && !isEndpointPackageV2(installation.version)
+        ? METADATA_TRANSFORMS_PATTERN
+        : METADATA_TRANSFORMS_PATTERN_V2;
+
     try {
       const transformStats = await esClient.transform.getTransformStats({
-        transform_id: METADATA_TRANSFORMS_PATTERN,
+        transform_id: transformName,
         allow_no_match: true,
       });
       return response.ok({

x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts

@@ -103,7 +103,7 @@ export function registerEndpointRoutes(
       withEndpointAuthz(
         { all: ['canReadSecuritySolution'] },
         logger,
-        getMetadataTransformStatsHandler(logger)
+        getMetadataTransformStatsHandler(endpointAppContext, logger)
       )
     );
 }

x-pack/test/security_solution_endpoint/services/endpoint.ts

@@ -12,8 +12,10 @@ import { Client } from '@elastic/elasticsearch';
 import {
   metadataCurrentIndexPattern,
   metadataTransformPrefix,
+  METADATA_CURRENT_TRANSFORM_V2,
   METADATA_UNITED_INDEX,
   METADATA_UNITED_TRANSFORM,
+  METADATA_UNITED_TRANSFORM_V2,
   HOST_METADATA_GET_ROUTE,
   METADATA_DATASTREAM,
 } from '@kbn/security-solution-plugin/common/endpoint/constants';
@@ -22,6 +24,8 @@ import {
   IndexedHostsAndAlertsResponse,
   indexHostsAndAlerts,
 } from '@kbn/security-solution-plugin/common/endpoint/index_data';
+import { getEndpointPackageInfo } from '@kbn/security-solution-plugin/common/endpoint/utils/package';
+import { isEndpointPackageV2 } from '@kbn/security-solution-plugin/common/endpoint/utils/transforms';
 import { installOrUpgradeEndpointFleetPackage } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/setup_fleet_for_endpoint';
 import { EndpointError } from '@kbn/security-solution-plugin/common/endpoint/errors';
 import { STARTED_TRANSFORM_STATES } from '@kbn/security-solution-plugin/common/constants';
@@ -116,11 +120,23 @@ export class EndpointTestResources extends FtrService {
       customIndexFn,
     } = options;
 
+    let currentTransformName = metadataTransformPrefix;
+    let unitedTransformName = METADATA_UNITED_TRANSFORM;
+    if (waitUntilTransformed && customIndexFn) {
+      const endpointPackage = await getEndpointPackageInfo(this.kbnClient);
+      const isV2 = isEndpointPackageV2(endpointPackage.version);
+
+      if (isV2) {
+        currentTransformName = METADATA_CURRENT_TRANSFORM_V2;
+        unitedTransformName = METADATA_UNITED_TRANSFORM_V2;
+      }
+    }
+
     if (waitUntilTransformed && customIndexFn) {
       // need this before indexing docs so that the united transform doesn't
       // create a checkpoint with a timestamp after the doc timestamps
-      await this.stopTransform(metadataTransformPrefix);
-      await this.stopTransform(METADATA_UNITED_TRANSFORM);
+      await this.stopTransform(currentTransformName);
+      await this.stopTransform(unitedTransformName);
     }
 
     // load data into the system
@@ -147,10 +163,10 @@ export class EndpointTestResources extends FtrService {
         );
 
     if (waitUntilTransformed && customIndexFn) {
-      await this.startTransform(metadataTransformPrefix);
+      await this.startTransform(currentTransformName);
       const metadataIds = Array.from(new Set(indexedData.hosts.map((host) => host.agent.id)));
       await this.waitForEndpoints(metadataIds, waitTimeout);
-      await this.startTransform(METADATA_UNITED_TRANSFORM);
+      await this.startTransform(unitedTransformName);
     }
 
     if (waitUntilTransformed) {
@@ -342,4 +358,9 @@ export class EndpointTestResources extends FtrService {
 
     return response;
   }
+
+  async isEndpointPackageV2(): Promise<boolean> {
+    const endpointPackage = await getEndpointPackageInfo(this.kbnClient);
+    return isEndpointPackageV2(endpointPackage.version);
+  }
 }