Revert "[Security Solution] add handling for endpoint package spec v2 (…

…elastic#169901)"

This reverts commit 82880fe.

x-pack/plugins/security_solution/common/endpoint/constants.ts

@@ -29,20 +29,14 @@ export const metadataIndexPattern = 'metrics-endpoint.metadata-*';
 /** index that the metadata transform writes to (destination) and that is used by endpoint APIs */
 export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current_*';
 
-// endpoint package V2 has an additional prefix in the transform names
-const PACKAGE_V2_PREFIX = 'logs-';
-
 /** The metadata Transform Name prefix with NO (package) version) */
 export const metadataTransformPrefix = 'endpoint.metadata_current-default';
-export const METADATA_CURRENT_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${metadataTransformPrefix}`;
 
 // metadata transforms pattern for matching all metadata transform ids
 export const METADATA_TRANSFORMS_PATTERN = 'endpoint.metadata_*';
-export const METADATA_TRANSFORMS_PATTERN_V2 = `${PACKAGE_V2_PREFIX}${METADATA_TRANSFORMS_PATTERN}`;
 
 // united metadata transform id
 export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default';
-export const METADATA_UNITED_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${METADATA_UNITED_TRANSFORM}`;
 
 // united metadata transform destination index
 export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default';

x-pack/plugins/security_solution/common/endpoint/index_data.ts

@@ -111,8 +111,8 @@ export const indexHostsAndAlerts = usageTracker.track(
 
     const shouldWaitForEndpointMetadataDocs = fleet;
     if (shouldWaitForEndpointMetadataDocs) {
-      await waitForMetadataTransformsReady(client, epmEndpointPackage.version);
-      await stopMetadataTransforms(client, epmEndpointPackage.version);
+      await waitForMetadataTransformsReady(client);
+      await stopMetadataTransforms(client);
     }
 
     for (let i = 0; i < numHosts; i++) {
@@ -147,8 +147,7 @@ export const indexHostsAndAlerts = usageTracker.track(
     if (shouldWaitForEndpointMetadataDocs) {
       await startMetadataTransforms(
         client,
-        response.agents.map((agent) => agent.agent?.id ?? ''),
-        epmEndpointPackage.version
+        response.agents.map((agent) => agent.agent?.id ?? '')
       );
     }
 

x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts

@@ -5,33 +5,28 @@
  * 2.0.
  */
 
-import semverLte from 'semver/functions/lte';
-
 import type { Client } from '@elastic/elasticsearch';
 import type { TransformGetTransformStatsTransformStats } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
 
 import { usageTracker } from '../data_loaders/usage_tracker';
 import {
   metadataCurrentIndexPattern,
   metadataTransformPrefix,
-  METADATA_CURRENT_TRANSFORM_V2,
   METADATA_TRANSFORMS_PATTERN,
-  METADATA_TRANSFORMS_PATTERN_V2,
   METADATA_UNITED_TRANSFORM,
-  METADATA_UNITED_TRANSFORM_V2,
 } from '../constants';
 
 export const waitForMetadataTransformsReady = usageTracker.track(
   'waitForMetadataTransformsReady',
-  async (esClient: Client, version: string): Promise<void> => {
-    await waitFor(() => areMetadataTransformsReady(esClient, version));
+  async (esClient: Client): Promise<void> => {
+    await waitFor(() => areMetadataTransformsReady(esClient));
   }
 );
 
 export const stopMetadataTransforms = usageTracker.track(
   'stopMetadataTransforms',
-  async (esClient: Client, version: string): Promise<void> => {
-    const transformIds = await getMetadataTransformIds(esClient, version);
+  async (esClient: Client): Promise<void> => {
+    const transformIds = await getMetadataTransformIds(esClient);
 
     await Promise.all(
       transformIds.map((transformId) =>
@@ -51,18 +46,14 @@ export const startMetadataTransforms = usageTracker.track(
   async (
     esClient: Client,
     // agentIds to wait for
-    agentIds: string[],
-    version: string
+    agentIds: string[]
   ): Promise<void> => {
-    const transformIds = await getMetadataTransformIds(esClient, version);
-    const isV2 = isEndpointPackageV2(version);
-    const currentTransformPrefix = isV2 ? METADATA_CURRENT_TRANSFORM_V2 : metadataTransformPrefix;
+    const transformIds = await getMetadataTransformIds(esClient);
     const currentTransformId = transformIds.find((transformId) =>
-      transformId.startsWith(currentTransformPrefix)
+      transformId.startsWith(metadataTransformPrefix)
     );
-    const unitedTransformPrefix = isV2 ? METADATA_UNITED_TRANSFORM_V2 : METADATA_UNITED_TRANSFORM;
     const unitedTransformId = transformIds.find((transformId) =>
-      transformId.startsWith(unitedTransformPrefix)
+      transformId.startsWith(METADATA_UNITED_TRANSFORM)
     );
     if (!currentTransformId || !unitedTransformId) {
       // eslint-disable-next-line no-console
@@ -97,26 +88,22 @@ export const startMetadataTransforms = usageTracker.track(
 );
 
 async function getMetadataTransformStats(
-  esClient: Client,
-  version: string
+  esClient: Client
 ): Promise<TransformGetTransformStatsTransformStats[]> {
-  const transformId = isEndpointPackageV2(version)
-    ? METADATA_TRANSFORMS_PATTERN_V2
-    : METADATA_TRANSFORMS_PATTERN;
   return (
     await esClient.transform.getTransformStats({
-      transform_id: transformId,
+      transform_id: METADATA_TRANSFORMS_PATTERN,
       allow_no_match: true,
     })
   ).transforms;
 }
 
-async function getMetadataTransformIds(esClient: Client, version: string): Promise<string[]> {
-  return (await getMetadataTransformStats(esClient, version)).map((transform) => transform.id);
+async function getMetadataTransformIds(esClient: Client): Promise<string[]> {
+  return (await getMetadataTransformStats(esClient)).map((transform) => transform.id);
 }
 
-async function areMetadataTransformsReady(esClient: Client, version: string): Promise<boolean> {
-  const transforms = await getMetadataTransformStats(esClient, version);
+async function areMetadataTransformsReady(esClient: Client): Promise<boolean> {
+  const transforms = await getMetadataTransformStats(esClient);
   return !transforms.some(
     // TODO TransformGetTransformStatsTransformStats type needs to be updated to include health
     (transform: TransformGetTransformStatsTransformStats & { health?: { status: string } }) =>
@@ -172,8 +159,3 @@ async function waitFor(
     }
   }
 }
-
-const MIN_ENDPOINT_PACKAGE_V2_VERSION = '8.12.0';
-export function isEndpointPackageV2(version: string) {
-  return semverLte(MIN_ENDPOINT_PACKAGE_V2_VERSION, version);
-}

x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts

@@ -70,7 +70,6 @@ describe('check metadata transforms task', () => {
           { type: ElasticsearchAssetType.transform } as EsAssetReference,
           { type: ElasticsearchAssetType.transform } as EsAssetReference,
         ],
-        version: '8.11.0',
       } as Installation);
   });
 

x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts

@@ -19,14 +19,10 @@ import type {
 import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server';
 import { ElasticsearchAssetType, FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
 import type { EndpointAppContext } from '../../types';
-import {
-  METADATA_TRANSFORMS_PATTERN,
-  METADATA_TRANSFORMS_PATTERN_V2,
-} from '../../../../common/endpoint/constants';
+import { METADATA_TRANSFORMS_PATTERN } from '../../../../common/endpoint/constants';
 import { WARNING_TRANSFORM_STATES } from '../../../../common/constants';
 import { wrapErrorIfNeeded } from '../../utils';
 import { stateSchemaByVersion, emptyState, type LatestTaskStateSchema } from './task_state';
-import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms';
 
 const SCOPE = ['securitySolution'];
 const INTERVAL = '2h';
@@ -112,21 +108,11 @@ export class CheckMetadataTransformsTask {
     const [{ elasticsearch }] = await core.getStartServices();
     const esClient = elasticsearch.client.asInternalUser;
 
-    const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
-    const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
-    if (!installation) {
-      this.logger.info('no endpoint installation found');
-      return { state: taskInstance.state };
-    }
-
-    const transformName = isEndpointPackageV2(installation.version)
-      ? METADATA_TRANSFORMS_PATTERN_V2
-      : METADATA_TRANSFORMS_PATTERN;
     let transformStatsResponse: TransportResult<TransformGetTransformStatsResponse>;
     try {
       transformStatsResponse = await esClient?.transform.getTransformStats(
         {
-          transform_id: transformName,
+          transform_id: METADATA_TRANSFORMS_PATTERN,
         },
         { meta: true }
       );
@@ -138,6 +124,12 @@ export class CheckMetadataTransformsTask {
       return { state: taskInstance.state };
     }
 
+    const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages;
+    const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
+    if (!installation) {
+      this.logger.info('no endpoint installation found');
+      return { state: taskInstance.state };
+    }
     const expectedTransforms = installation.installed_es.filter(
       (asset) => asset.type === ElasticsearchAssetType.transform
     );

x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts

@@ -7,8 +7,6 @@
 
 import type { TypeOf } from '@kbn/config-schema';
 import type { Logger, RequestHandler } from '@kbn/core/server';
-import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common';
-
 import type {
   MetadataListResponse,
   EndpointSortableField,
@@ -27,9 +25,7 @@ import {
   ENDPOINT_DEFAULT_SORT_DIRECTION,
   ENDPOINT_DEFAULT_SORT_FIELD,
   METADATA_TRANSFORMS_PATTERN,
-  METADATA_TRANSFORMS_PATTERN_V2,
 } from '../../../../common/endpoint/constants';
-import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms';
 
 export const getLogger = (endpointAppContext: EndpointAppContext): Logger => {
   return endpointAppContext.logFactory.get('metadata');
@@ -103,21 +99,13 @@ export const getMetadataRequestHandler = function (
 };
 
 export function getMetadataTransformStatsHandler(
-  endpointAppContext: EndpointAppContext,
   logger: Logger
 ): RequestHandler<unknown, unknown, unknown, SecuritySolutionRequestHandlerContext> {
   return async (context, _, response) => {
     const esClient = (await context.core).elasticsearch.client.asInternalUser;
-    const packageClient = endpointAppContext.service.getInternalFleetServices().packages;
-    const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE);
-    const transformName =
-      installation?.version && !isEndpointPackageV2(installation.version)
-        ? METADATA_TRANSFORMS_PATTERN
-        : METADATA_TRANSFORMS_PATTERN_V2;
-
     try {
       const transformStats = await esClient.transform.getTransformStats({
-        transform_id: transformName,
+        transform_id: METADATA_TRANSFORMS_PATTERN,
         allow_no_match: true,
       });
       return response.ok({

x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts

@@ -103,7 +103,7 @@ export function registerEndpointRoutes(
       withEndpointAuthz(
         { all: ['canReadSecuritySolution'] },
         logger,
-        getMetadataTransformStatsHandler(endpointAppContext, logger)
+        getMetadataTransformStatsHandler(logger)
       )
     );
 }

x-pack/test/security_solution_endpoint/services/endpoint.ts

@@ -12,10 +12,8 @@ import { Client } from '@elastic/elasticsearch';
 import {
   metadataCurrentIndexPattern,
   metadataTransformPrefix,
-  METADATA_CURRENT_TRANSFORM_V2,
   METADATA_UNITED_INDEX,
   METADATA_UNITED_TRANSFORM,
-  METADATA_UNITED_TRANSFORM_V2,
   HOST_METADATA_GET_ROUTE,
   METADATA_DATASTREAM,
 } from '@kbn/security-solution-plugin/common/endpoint/constants';
@@ -24,8 +22,6 @@ import {
   IndexedHostsAndAlertsResponse,
   indexHostsAndAlerts,
 } from '@kbn/security-solution-plugin/common/endpoint/index_data';
-import { getEndpointPackageInfo } from '@kbn/security-solution-plugin/common/endpoint/utils/package';
-import { isEndpointPackageV2 } from '@kbn/security-solution-plugin/common/endpoint/utils/transforms';
 import { installOrUpgradeEndpointFleetPackage } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/setup_fleet_for_endpoint';
 import { EndpointError } from '@kbn/security-solution-plugin/common/endpoint/errors';
 import { STARTED_TRANSFORM_STATES } from '@kbn/security-solution-plugin/common/constants';
@@ -120,23 +116,11 @@ export class EndpointTestResources extends FtrService {
       customIndexFn,
     } = options;
 
-    let currentTransformName = metadataTransformPrefix;
-    let unitedTransformName = METADATA_UNITED_TRANSFORM;
-    if (waitUntilTransformed && customIndexFn) {
-      const endpointPackage = await getEndpointPackageInfo(this.kbnClient);
-      const isV2 = isEndpointPackageV2(endpointPackage.version);
-
-      if (isV2) {
-        currentTransformName = METADATA_CURRENT_TRANSFORM_V2;
-        unitedTransformName = METADATA_UNITED_TRANSFORM_V2;
-      }
-    }
-
     if (waitUntilTransformed && customIndexFn) {
       // need this before indexing docs so that the united transform doesn't
       // create a checkpoint with a timestamp after the doc timestamps
-      await this.stopTransform(currentTransformName);
-      await this.stopTransform(unitedTransformName);
+      await this.stopTransform(metadataTransformPrefix);
+      await this.stopTransform(METADATA_UNITED_TRANSFORM);
     }
 
     // load data into the system
@@ -163,10 +147,10 @@ export class EndpointTestResources extends FtrService {
         );
 
     if (waitUntilTransformed && customIndexFn) {
-      await this.startTransform(currentTransformName);
+      await this.startTransform(metadataTransformPrefix);
       const metadataIds = Array.from(new Set(indexedData.hosts.map((host) => host.agent.id)));
       await this.waitForEndpoints(metadataIds, waitTimeout);
-      await this.startTransform(unitedTransformName);
+      await this.startTransform(METADATA_UNITED_TRANSFORM);
     }
 
     if (waitUntilTransformed) {
@@ -358,9 +342,4 @@ export class EndpointTestResources extends FtrService {
 
     return response;
   }
-
-  async isEndpointPackageV2(): Promise<boolean> {
-    const endpointPackage = await getEndpointPackageInfo(this.kbnClient);
-    return isEndpointPackageV2(endpointPackage.version);
-  }
 }