diff --git a/x-pack/plugins/security_solution/common/endpoint/constants.ts b/x-pack/plugins/security_solution/common/endpoint/constants.ts index e4effec5bf4d39..892cb717e1a2cf 100644 --- a/x-pack/plugins/security_solution/common/endpoint/constants.ts +++ b/x-pack/plugins/security_solution/common/endpoint/constants.ts @@ -29,14 +29,20 @@ export const metadataIndexPattern = 'metrics-endpoint.metadata-*'; /** index that the metadata transform writes to (destination) and that is used by endpoint APIs */ export const metadataCurrentIndexPattern = 'metrics-endpoint.metadata_current_*'; +// endpoint package V2 has an additional prefix in the transform names +const PACKAGE_V2_PREFIX = 'logs-'; + /** The metadata Transform Name prefix with NO (package) version) */ export const metadataTransformPrefix = 'endpoint.metadata_current-default'; +export const METADATA_CURRENT_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${metadataTransformPrefix}`; // metadata transforms pattern for matching all metadata transform ids export const METADATA_TRANSFORMS_PATTERN = 'endpoint.metadata_*'; +export const METADATA_TRANSFORMS_PATTERN_V2 = `${PACKAGE_V2_PREFIX}${METADATA_TRANSFORMS_PATTERN}`; // united metadata transform id export const METADATA_UNITED_TRANSFORM = 'endpoint.metadata_united-default'; +export const METADATA_UNITED_TRANSFORM_V2 = `${PACKAGE_V2_PREFIX}${METADATA_UNITED_TRANSFORM}`; // united metadata transform destination index export const METADATA_UNITED_INDEX = '.metrics-endpoint.metadata_united_default'; diff --git a/x-pack/plugins/security_solution/common/endpoint/index_data.ts b/x-pack/plugins/security_solution/common/endpoint/index_data.ts index 3bbb49cb30707e..5caf47b1ade330 100644 --- a/x-pack/plugins/security_solution/common/endpoint/index_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/index_data.ts @@ -111,8 +111,8 @@ export const indexHostsAndAlerts = usageTracker.track( const shouldWaitForEndpointMetadataDocs = fleet; if (shouldWaitForEndpointMetadataDocs) { - await waitForMetadataTransformsReady(client); - await stopMetadataTransforms(client); + await waitForMetadataTransformsReady(client, epmEndpointPackage.version); + await stopMetadataTransforms(client, epmEndpointPackage.version); } for (let i = 0; i < numHosts; i++) { @@ -147,7 +147,8 @@ export const indexHostsAndAlerts = usageTracker.track( if (shouldWaitForEndpointMetadataDocs) { await startMetadataTransforms( client, - response.agents.map((agent) => agent.agent?.id ?? '') + response.agents.map((agent) => agent.agent?.id ?? ''), + epmEndpointPackage.version ); } diff --git a/x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts b/x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts index b689a1d7c20e64..2f40fc0367bc61 100644 --- a/x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts +++ b/x-pack/plugins/security_solution/common/endpoint/utils/transforms.ts @@ -5,6 +5,8 @@ * 2.0. */ +import semverLte from 'semver/functions/lte'; + import type { Client } from '@elastic/elasticsearch'; import type { TransformGetTransformStatsTransformStats } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; @@ -12,21 +14,24 @@ import { usageTracker } from '../data_loaders/usage_tracker'; import { metadataCurrentIndexPattern, metadataTransformPrefix, + METADATA_CURRENT_TRANSFORM_V2, METADATA_TRANSFORMS_PATTERN, + METADATA_TRANSFORMS_PATTERN_V2, METADATA_UNITED_TRANSFORM, + METADATA_UNITED_TRANSFORM_V2, } from '../constants'; export const waitForMetadataTransformsReady = usageTracker.track( 'waitForMetadataTransformsReady', - async (esClient: Client): Promise => { - await waitFor(() => areMetadataTransformsReady(esClient)); + async (esClient: Client, version: string): Promise => { + await waitFor(() => areMetadataTransformsReady(esClient, version)); } ); export const stopMetadataTransforms = usageTracker.track( 'stopMetadataTransforms', - async (esClient: Client): Promise => { - const transformIds = await getMetadataTransformIds(esClient); + async (esClient: Client, version: string): Promise => { + const transformIds = await getMetadataTransformIds(esClient, version); await Promise.all( transformIds.map((transformId) => @@ -46,14 +51,18 @@ export const startMetadataTransforms = usageTracker.track( async ( esClient: Client, // agentIds to wait for - agentIds: string[] + agentIds: string[], + version: string ): Promise => { - const transformIds = await getMetadataTransformIds(esClient); + const transformIds = await getMetadataTransformIds(esClient, version); + const isV2 = isEndpointPackageV2(version); + const currentTransformPrefix = isV2 ? METADATA_CURRENT_TRANSFORM_V2 : metadataTransformPrefix; const currentTransformId = transformIds.find((transformId) => - transformId.startsWith(metadataTransformPrefix) + transformId.startsWith(currentTransformPrefix) ); + const unitedTransformPrefix = isV2 ? METADATA_UNITED_TRANSFORM_V2 : METADATA_UNITED_TRANSFORM; const unitedTransformId = transformIds.find((transformId) => - transformId.startsWith(METADATA_UNITED_TRANSFORM) + transformId.startsWith(unitedTransformPrefix) ); if (!currentTransformId || !unitedTransformId) { // eslint-disable-next-line no-console @@ -88,22 +97,26 @@ export const startMetadataTransforms = usageTracker.track( ); async function getMetadataTransformStats( - esClient: Client + esClient: Client, + version: string ): Promise { + const transformId = isEndpointPackageV2(version) + ? METADATA_TRANSFORMS_PATTERN_V2 + : METADATA_TRANSFORMS_PATTERN; return ( await esClient.transform.getTransformStats({ - transform_id: METADATA_TRANSFORMS_PATTERN, + transform_id: transformId, allow_no_match: true, }) ).transforms; } -async function getMetadataTransformIds(esClient: Client): Promise { - return (await getMetadataTransformStats(esClient)).map((transform) => transform.id); +async function getMetadataTransformIds(esClient: Client, version: string): Promise { + return (await getMetadataTransformStats(esClient, version)).map((transform) => transform.id); } -async function areMetadataTransformsReady(esClient: Client): Promise { - const transforms = await getMetadataTransformStats(esClient); +async function areMetadataTransformsReady(esClient: Client, version: string): Promise { + const transforms = await getMetadataTransformStats(esClient, version); return !transforms.some( // TODO TransformGetTransformStatsTransformStats type needs to be updated to include health (transform: TransformGetTransformStatsTransformStats & { health?: { status: string } }) => @@ -159,3 +172,8 @@ async function waitFor( } } } + +const MIN_ENDPOINT_PACKAGE_V2_VERSION = '8.12.0'; +export function isEndpointPackageV2(version: string) { + return semverLte(MIN_ENDPOINT_PACKAGE_V2_VERSION, version); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts index dabd2feed57e2e..c5b641853e4dbc 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.test.ts @@ -70,6 +70,7 @@ describe('check metadata transforms task', () => { { type: ElasticsearchAssetType.transform } as EsAssetReference, { type: ElasticsearchAssetType.transform } as EsAssetReference, ], + version: '8.11.0', } as Installation); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts index ca7a73a2721924..c36fcbec8b879e 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/metadata/check_metadata_transforms_task.ts @@ -19,10 +19,14 @@ import type { import { throwUnrecoverableError } from '@kbn/task-manager-plugin/server'; import { ElasticsearchAssetType, FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common'; import type { EndpointAppContext } from '../../types'; -import { METADATA_TRANSFORMS_PATTERN } from '../../../../common/endpoint/constants'; +import { + METADATA_TRANSFORMS_PATTERN, + METADATA_TRANSFORMS_PATTERN_V2, +} from '../../../../common/endpoint/constants'; import { WARNING_TRANSFORM_STATES } from '../../../../common/constants'; import { wrapErrorIfNeeded } from '../../utils'; import { stateSchemaByVersion, emptyState, type LatestTaskStateSchema } from './task_state'; +import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms'; const SCOPE = ['securitySolution']; const INTERVAL = '2h'; @@ -108,11 +112,21 @@ export class CheckMetadataTransformsTask { const [{ elasticsearch }] = await core.getStartServices(); const esClient = elasticsearch.client.asInternalUser; + const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages; + const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE); + if (!installation) { + this.logger.info('no endpoint installation found'); + return { state: taskInstance.state }; + } + + const transformName = isEndpointPackageV2(installation.version) + ? METADATA_TRANSFORMS_PATTERN_V2 + : METADATA_TRANSFORMS_PATTERN; let transformStatsResponse: TransportResult; try { transformStatsResponse = await esClient?.transform.getTransformStats( { - transform_id: METADATA_TRANSFORMS_PATTERN, + transform_id: transformName, }, { meta: true } ); @@ -124,12 +138,6 @@ export class CheckMetadataTransformsTask { return { state: taskInstance.state }; } - const packageClient = this.endpointAppContext.service.getInternalFleetServices().packages; - const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE); - if (!installation) { - this.logger.info('no endpoint installation found'); - return { state: taskInstance.state }; - } const expectedTransforms = installation.installed_es.filter( (asset) => asset.type === ElasticsearchAssetType.transform ); diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts index 9131304d529a3c..93593efef13444 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/handlers.ts @@ -7,6 +7,8 @@ import type { TypeOf } from '@kbn/config-schema'; import type { Logger, RequestHandler } from '@kbn/core/server'; +import { FLEET_ENDPOINT_PACKAGE } from '@kbn/fleet-plugin/common'; + import type { MetadataListResponse, EndpointSortableField, @@ -25,7 +27,9 @@ import { ENDPOINT_DEFAULT_SORT_DIRECTION, ENDPOINT_DEFAULT_SORT_FIELD, METADATA_TRANSFORMS_PATTERN, + METADATA_TRANSFORMS_PATTERN_V2, } from '../../../../common/endpoint/constants'; +import { isEndpointPackageV2 } from '../../../../common/endpoint/utils/transforms'; export const getLogger = (endpointAppContext: EndpointAppContext): Logger => { return endpointAppContext.logFactory.get('metadata'); @@ -99,13 +103,21 @@ export const getMetadataRequestHandler = function ( }; export function getMetadataTransformStatsHandler( + endpointAppContext: EndpointAppContext, logger: Logger ): RequestHandler { return async (context, _, response) => { const esClient = (await context.core).elasticsearch.client.asInternalUser; + const packageClient = endpointAppContext.service.getInternalFleetServices().packages; + const installation = await packageClient.getInstallation(FLEET_ENDPOINT_PACKAGE); + const transformName = + installation?.version && !isEndpointPackageV2(installation.version) + ? METADATA_TRANSFORMS_PATTERN + : METADATA_TRANSFORMS_PATTERN_V2; + try { const transformStats = await esClient.transform.getTransformStats({ - transform_id: METADATA_TRANSFORMS_PATTERN, + transform_id: transformName, allow_no_match: true, }); return response.ok({ diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts index 2f96e8ba82bc9c..75d0fb2135fc80 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts @@ -103,7 +103,7 @@ export function registerEndpointRoutes( withEndpointAuthz( { all: ['canReadSecuritySolution'] }, logger, - getMetadataTransformStatsHandler(logger) + getMetadataTransformStatsHandler(endpointAppContext, logger) ) ); } diff --git a/x-pack/test/security_solution_endpoint/services/endpoint.ts b/x-pack/test/security_solution_endpoint/services/endpoint.ts index 3ef21fd9a2b2af..1f66ab6ac8ed5e 100644 --- a/x-pack/test/security_solution_endpoint/services/endpoint.ts +++ b/x-pack/test/security_solution_endpoint/services/endpoint.ts @@ -12,8 +12,10 @@ import { Client } from '@elastic/elasticsearch'; import { metadataCurrentIndexPattern, metadataTransformPrefix, + METADATA_CURRENT_TRANSFORM_V2, METADATA_UNITED_INDEX, METADATA_UNITED_TRANSFORM, + METADATA_UNITED_TRANSFORM_V2, HOST_METADATA_GET_ROUTE, METADATA_DATASTREAM, } from '@kbn/security-solution-plugin/common/endpoint/constants'; @@ -22,6 +24,8 @@ import { IndexedHostsAndAlertsResponse, indexHostsAndAlerts, } from '@kbn/security-solution-plugin/common/endpoint/index_data'; +import { getEndpointPackageInfo } from '@kbn/security-solution-plugin/common/endpoint/utils/package'; +import { isEndpointPackageV2 } from '@kbn/security-solution-plugin/common/endpoint/utils/transforms'; import { installOrUpgradeEndpointFleetPackage } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/setup_fleet_for_endpoint'; import { EndpointError } from '@kbn/security-solution-plugin/common/endpoint/errors'; import { STARTED_TRANSFORM_STATES } from '@kbn/security-solution-plugin/common/constants'; @@ -116,11 +120,23 @@ export class EndpointTestResources extends FtrService { customIndexFn, } = options; + let currentTransformName = metadataTransformPrefix; + let unitedTransformName = METADATA_UNITED_TRANSFORM; + if (waitUntilTransformed && customIndexFn) { + const endpointPackage = await getEndpointPackageInfo(this.kbnClient); + const isV2 = isEndpointPackageV2(endpointPackage.version); + + if (isV2) { + currentTransformName = METADATA_CURRENT_TRANSFORM_V2; + unitedTransformName = METADATA_UNITED_TRANSFORM_V2; + } + } + if (waitUntilTransformed && customIndexFn) { // need this before indexing docs so that the united transform doesn't // create a checkpoint with a timestamp after the doc timestamps - await this.stopTransform(metadataTransformPrefix); - await this.stopTransform(METADATA_UNITED_TRANSFORM); + await this.stopTransform(currentTransformName); + await this.stopTransform(unitedTransformName); } // load data into the system @@ -147,10 +163,10 @@ export class EndpointTestResources extends FtrService { ); if (waitUntilTransformed && customIndexFn) { - await this.startTransform(metadataTransformPrefix); + await this.startTransform(currentTransformName); const metadataIds = Array.from(new Set(indexedData.hosts.map((host) => host.agent.id))); await this.waitForEndpoints(metadataIds, waitTimeout); - await this.startTransform(METADATA_UNITED_TRANSFORM); + await this.startTransform(unitedTransformName); } if (waitUntilTransformed) { @@ -342,4 +358,9 @@ export class EndpointTestResources extends FtrService { return response; } + + async isEndpointPackageV2(): Promise { + const endpointPackage = await getEndpointPackageInfo(this.kbnClient); + return isEndpointPackageV2(endpointPackage.version); + } } diff --git a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts index 332bf153524e0f..48109e13257caf 100644 --- a/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts +++ b/x-pack/test/security_solution_endpoint_api_int/apis/metadata.ts @@ -16,7 +16,9 @@ import { METADATA_TRANSFORMS_STATUS_ROUTE, METADATA_UNITED_INDEX, METADATA_UNITED_TRANSFORM, + METADATA_UNITED_TRANSFORM_V2, metadataTransformPrefix, + METADATA_CURRENT_TRANSFORM_V2, } from '@kbn/security-solution-plugin/common/endpoint/constants'; import { AGENTS_INDEX } from '@kbn/fleet-plugin/common'; import { indexFleetEndpointPolicy } from '@kbn/security-solution-plugin/common/endpoint/data_loaders/index_fleet_endpoint_policy'; @@ -44,8 +46,7 @@ export default function ({ getService }: FtrProviderContext) { const endpointTestResources = getService('endpointTestResources'); const log = getService('log'); - // FLAKY: https://github.com/elastic/kibana/issues/151854 - describe.skip('test metadata apis', () => { + describe('test metadata apis', () => { describe('list endpoints GET route', () => { const numberOfHostsInFixture = 2; let agent1Timestamp: number; @@ -400,7 +401,17 @@ export default function ({ getService }: FtrProviderContext) { }); describe('get metadata transforms', () => { - const testRegex = /endpoint\.metadata_(united|current)-default-*/; + const testRegex = /(endpoint|logs-endpoint)\.metadata_(united|current)-default-*/; + let currentTransformName = metadataTransformPrefix; + let unitedTransformName = METADATA_UNITED_TRANSFORM; + + before(async () => { + const isPackageV2 = await endpointTestResources.isEndpointPackageV2(); + if (isPackageV2) { + currentTransformName = METADATA_CURRENT_TRANSFORM_V2; + unitedTransformName = METADATA_UNITED_TRANSFORM_V2; + } + }); it('should respond forbidden if no fleet access', async () => { await getService('supertestWithoutAuth') @@ -411,8 +422,8 @@ export default function ({ getService }: FtrProviderContext) { }); it('correctly returns stopped transform stats', async () => { - await stopTransform(getService, `${metadataTransformPrefix}*`); - await stopTransform(getService, `${METADATA_UNITED_TRANSFORM}*`); + await stopTransform(getService, `${currentTransformName}*`); + await stopTransform(getService, `${unitedTransformName}*`); const { body } = await supertest .get(METADATA_TRANSFORMS_STATUS_ROUTE) @@ -428,17 +439,17 @@ export default function ({ getService }: FtrProviderContext) { expect(transforms.length).to.eql(2); const currentTransform = transforms.find((transform) => - transform.id.startsWith(metadataTransformPrefix) + transform.id.startsWith(currentTransformName) ); expect(currentTransform).to.be.ok(); const unitedTransform = transforms.find((transform) => - transform.id.startsWith(METADATA_UNITED_TRANSFORM) + transform.id.startsWith(unitedTransformName) ); expect(unitedTransform).to.be.ok(); - await startTransform(getService, metadataTransformPrefix); - await startTransform(getService, METADATA_UNITED_TRANSFORM); + await startTransform(getService, currentTransformName); + await startTransform(getService, unitedTransformName); }); it('correctly returns started transform stats', async () => { @@ -456,12 +467,12 @@ export default function ({ getService }: FtrProviderContext) { expect(transforms.length).to.eql(2); const currentTransform = transforms.find((transform) => - transform.id.startsWith(metadataTransformPrefix) + transform.id.startsWith(currentTransformName) ); expect(currentTransform).to.be.ok(); const unitedTransform = transforms.find((transform) => - transform.id.startsWith(METADATA_UNITED_TRANSFORM) + transform.id.startsWith(unitedTransformName) ); expect(unitedTransform).to.be.ok(); });