From 1a699657261c710c758f6e46474fdcaf2184af9f Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Thu, 9 Jul 2020 11:48:21 -0400 Subject: [PATCH] [Security Solution][Endpoint] User Manifest Cleanup + Artifact Compression (#70759) (#71246) * Stateless exception list translation with improved runtime checks * use flatMap and reduce to simplify logic * Update to new manifest format * Fix test fixture SO data type * Fix another test fixture data type * Fix sha256 reference in artifact_client * Refactor to remove usages of 'then' and tidy up a bit * Zlib compression * prefer byteLength to length * Make ingestManager optional for security-solution startup * Fix download functionality * Use eql for deep equality check * Fix base64 download bug * Add test for artifact download * Add more tests to ensure cached versions of artifacts are correct * Convert to new format * Deflate * missed some refs * partial fix to wrapper format * update fixtures and integration test * Fixing unit tests * small bug fixes * artifact and manifest versioning changes * Remove access tag from download endpoint * Adding decompression to integration test * Removing tag from route * add try/catch in ingest callback handler * Fixing * Removing last expect from unit test for tag * type fixes * Add compression type to manifest * Reverting ingestManager back to being required for now Co-authored-by: Alex Kahan Co-authored-by: Elastic Machine Co-authored-by: Alex Kahan Co-authored-by: Elastic Machine --- .../common/endpoint/generate_data.ts | 4 +- .../common/endpoint/schema/common.ts | 3 +- .../policy/store/policy_details/index.test.ts | 4 +- .../endpoint_app_context_services.test.ts | 16 +- .../endpoint/endpoint_app_context_services.ts | 22 +- .../server/endpoint/ingest_integration.ts | 34 +- .../endpoint/lib/artifacts/cache.test.ts | 27 +- .../server/endpoint/lib/artifacts/cache.ts | 6 +- .../server/endpoint/lib/artifacts/common.ts | 5 +- .../endpoint/lib/artifacts/lists.test.ts | 12 +- .../server/endpoint/lib/artifacts/lists.ts | 16 +- .../endpoint/lib/artifacts/manifest.test.ts | 63 +-- .../server/endpoint/lib/artifacts/manifest.ts | 3 +- .../lib/artifacts/manifest_entry.test.ts | 12 +- .../endpoint/lib/artifacts/manifest_entry.ts | 11 +- .../server/endpoint/lib/artifacts/task.ts | 32 +- .../artifacts/download_exception_list.test.ts | 27 +- .../artifacts/download_exception_list.ts | 12 +- .../server/endpoint/routes/metadata/index.ts | 83 ++-- .../endpoint/routes/metadata/metadata.test.ts | 33 +- .../endpoint/schemas/artifacts/common.ts | 13 +- .../response/download_artifact_schema.ts | 5 +- .../schemas/artifacts/saved_objects.ts | 4 +- .../artifacts/artifact_client.test.ts | 2 +- .../services/artifacts/artifact_client.ts | 2 +- .../artifacts/manifest_client.mock.ts | 4 +- .../artifacts/manifest_client.test.ts | 2 +- .../manifest_manager/manifest_manager.mock.ts | 5 +- .../manifest_manager/manifest_manager.test.ts | 50 ++- .../manifest_manager/manifest_manager.ts | 378 ++++++++++-------- .../security_solution/server/plugin.ts | 41 +- .../apis/endpoint/artifacts/index.ts | 17 +- .../endpoint/artifacts/api_feature/data.json | 20 +- 33 files changed, 564 insertions(+), 404 deletions(-) diff --git a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts index 6720f3523d5c73..339e5554ccb121 100644 --- a/x-pack/plugins/security_solution/common/endpoint/generate_data.ts +++ b/x-pack/plugins/security_solution/common/endpoint/generate_data.ts @@ -1036,8 +1036,8 @@ export class EndpointDocGenerator { config: { artifact_manifest: { value: { - manifest_version: 'v0', - schema_version: '1.0.0', + manifest_version: 'WzAsMF0=', + schema_version: 'v1', artifacts: {}, }, }, diff --git a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts index fdb2570314cd0d..014673ebe6398d 100644 --- a/x-pack/plugins/security_solution/common/endpoint/schema/common.ts +++ b/x-pack/plugins/security_solution/common/endpoint/schema/common.ts @@ -10,6 +10,7 @@ export const compressionAlgorithm = t.keyof({ none: null, zlib: null, }); +export type CompressionAlgorithm = t.TypeOf; export const encryptionAlgorithm = t.keyof({ none: null, @@ -20,7 +21,7 @@ export const identifier = t.string; export const manifestVersion = t.string; export const manifestSchemaVersion = t.keyof({ - '1.0.0': null, + v1: null, }); export type ManifestSchemaVersion = t.TypeOf; diff --git a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts index 0bd623b27f4fbc..102fd40c976729 100644 --- a/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts +++ b/x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts @@ -43,8 +43,8 @@ describe('policy details: ', () => { config: { artifact_manifest: { value: { - manifest_version: 'v0', - schema_version: '1.0.0', + manifest_version: 'WzAsMF0=', + schema_version: 'v1', artifacts: {}, }, }, diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts index 2daf259941cbfb..7642db23812e12 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts @@ -8,14 +8,14 @@ import { httpServerMock } from '../../../../../src/core/server/mocks'; import { EndpointAppContextService } from './endpoint_app_context_services'; describe('test endpoint app context services', () => { - it('should throw error on getAgentService if start is not called', async () => { - const endpointAppContextService = new EndpointAppContextService(); - expect(() => endpointAppContextService.getAgentService()).toThrow(Error); - }); - it('should return undefined on getManifestManager if start is not called', async () => { - const endpointAppContextService = new EndpointAppContextService(); - expect(endpointAppContextService.getManifestManager()).toEqual(undefined); - }); + // it('should return undefined on getAgentService if dependencies are not enabled', async () => { + // const endpointAppContextService = new EndpointAppContextService(); + // expect(endpointAppContextService.getAgentService()).toEqual(undefined); + // }); + // it('should return undefined on getManifestManager if dependencies are not enabled', async () => { + // const endpointAppContextService = new EndpointAppContextService(); + // expect(endpointAppContextService.getManifestManager()).toEqual(undefined); + // }); it('should throw error on getScopedSavedObjectsClient if start is not called', async () => { const endpointAppContextService = new EndpointAppContextService(); expect(() => diff --git a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts index 97a82049634c40..f51e8c6be10409 100644 --- a/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts +++ b/x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts @@ -4,20 +4,21 @@ * you may not use this file except in compliance with the Elastic License. */ import { - SavedObjectsServiceStart, KibanaRequest, + Logger, + SavedObjectsServiceStart, SavedObjectsClientContract, } from 'src/core/server'; import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server'; import { getPackageConfigCreateCallback } from './ingest_integration'; import { ManifestManager } from './services/artifacts'; -export type EndpointAppContextServiceStartContract = Pick< - IngestManagerStartContract, - 'agentService' +export type EndpointAppContextServiceStartContract = Partial< + Pick > & { - manifestManager?: ManifestManager | undefined; - registerIngestCallback: IngestManagerStartContract['registerExternalCallback']; + logger: Logger; + manifestManager?: ManifestManager; + registerIngestCallback?: IngestManagerStartContract['registerExternalCallback']; savedObjectsStart: SavedObjectsServiceStart; }; @@ -35,20 +36,17 @@ export class EndpointAppContextService { this.manifestManager = dependencies.manifestManager; this.savedObjectsStart = dependencies.savedObjectsStart; - if (this.manifestManager !== undefined) { + if (this.manifestManager && dependencies.registerIngestCallback) { dependencies.registerIngestCallback( 'packageConfigCreate', - getPackageConfigCreateCallback(this.manifestManager) + getPackageConfigCreateCallback(dependencies.logger, this.manifestManager) ); } } public stop() {} - public getAgentService(): AgentService { - if (!this.agentService) { - throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`); - } + public getAgentService(): AgentService | undefined { return this.agentService; } diff --git a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts index ace5aec77ed2c5..1acec1e7c53ac7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts +++ b/x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { Logger } from '../../../../../src/core/server'; import { NewPackageConfig } from '../../../ingest_manager/common/types/models'; import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config'; import { NewPolicyData } from '../../common/endpoint/types'; @@ -13,6 +14,7 @@ import { ManifestManager } from './services/artifacts'; * Callback to handle creation of PackageConfigs in Ingest Manager */ export const getPackageConfigCreateCallback = ( + logger: Logger, manifestManager: ManifestManager ): ((newPackageConfig: NewPackageConfig) => Promise) => { const handlePackageConfigCreate = async ( @@ -27,8 +29,19 @@ export const getPackageConfigCreateCallback = ( // follow the types/schema expected let updatedPackageConfig = newPackageConfig as NewPolicyData; - const wrappedManifest = await manifestManager.refresh({ initialize: true }); - if (wrappedManifest !== null) { + // get snapshot based on exception-list-agnostic SOs + // with diffs from last dispatched manifest, if it exists + const snapshot = await manifestManager.getSnapshot({ initialize: true }); + + if (snapshot === null) { + logger.warn('No manifest snapshot available.'); + return updatedPackageConfig; + } + + if (snapshot.diffs.length > 0) { + // create new artifacts + await manifestManager.syncArtifacts(snapshot, 'add'); + // Until we get the Default Policy Configuration in the Endpoint package, // we will add it here manually at creation time. // @ts-ignore @@ -42,7 +55,7 @@ export const getPackageConfigCreateCallback = ( streams: [], config: { artifact_manifest: { - value: wrappedManifest.manifest.toEndpointFormat(), + value: snapshot.manifest.toEndpointFormat(), }, policy: { value: policyConfigFactory(), @@ -57,9 +70,18 @@ export const getPackageConfigCreateCallback = ( try { return updatedPackageConfig; } finally { - // TODO: confirm creation of package config - // then commit. - await manifestManager.commit(wrappedManifest); + if (snapshot.diffs.length > 0) { + // TODO: let's revisit the way this callback happens... use promises? + // only commit when we know the package config was created + try { + await manifestManager.commit(snapshot.manifest); + + // clean up old artifacts + await manifestManager.syncArtifacts(snapshot, 'delete'); + } catch (err) { + logger.error(err); + } + } } }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts index 5a0fb913455529..00c764d0b912e7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts @@ -8,6 +8,7 @@ import { ExceptionsCache } from './cache'; describe('ExceptionsCache tests', () => { let cache: ExceptionsCache; + const body = Buffer.from('body'); beforeEach(() => { jest.clearAllMocks(); @@ -15,29 +16,33 @@ describe('ExceptionsCache tests', () => { }); test('it should cache', async () => { - cache.set('test', 'body'); + cache.set('test', body); const cacheResp = cache.get('test'); - expect(cacheResp).toEqual('body'); + expect(cacheResp).toEqual(body); }); test('it should handle cache miss', async () => { - cache.set('test', 'body'); + cache.set('test', body); const cacheResp = cache.get('not test'); expect(cacheResp).toEqual(undefined); }); test('it should handle cache eviction', async () => { - cache.set('1', 'a'); - cache.set('2', 'b'); - cache.set('3', 'c'); + const a = Buffer.from('a'); + const b = Buffer.from('b'); + const c = Buffer.from('c'); + const d = Buffer.from('d'); + cache.set('1', a); + cache.set('2', b); + cache.set('3', c); const cacheResp = cache.get('1'); - expect(cacheResp).toEqual('a'); + expect(cacheResp).toEqual(a); - cache.set('4', 'd'); + cache.set('4', d); const secondResp = cache.get('1'); expect(secondResp).toEqual(undefined); - expect(cache.get('2')).toEqual('b'); - expect(cache.get('3')).toEqual('c'); - expect(cache.get('4')).toEqual('d'); + expect(cache.get('2')).toEqual(b); + expect(cache.get('3')).toEqual(c); + expect(cache.get('4')).toEqual(d); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts index b7a4c2feb6bf84..b9d3bae4e6ef9a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts @@ -10,7 +10,7 @@ const DEFAULT_MAX_SIZE = 10; * FIFO cache implementation for artifact downloads. */ export class ExceptionsCache { - private cache: Map; + private cache: Map; private queue: string[]; private maxSize: number; @@ -20,7 +20,7 @@ export class ExceptionsCache { this.maxSize = maxSize || DEFAULT_MAX_SIZE; } - set(id: string, body: string) { + set(id: string, body: Buffer) { if (this.queue.length + 1 > this.maxSize) { const entry = this.queue.shift(); if (entry !== undefined) { @@ -31,7 +31,7 @@ export class ExceptionsCache { this.cache.set(id, body); } - get(id: string): string | undefined { + get(id: string): Buffer | undefined { return this.cache.get(id); } } diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts index cf381475220835..9ad4554b302036 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts @@ -8,10 +8,11 @@ export const ArtifactConstants = { GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist', SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2', SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'], - SCHEMA_VERSION: '1.0.0', + SCHEMA_VERSION: 'v1', }; export const ManifestConstants = { SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2', - SCHEMA_VERSION: '1.0.0', + SCHEMA_VERSION: 'v1', + INITIAL_VERSION: 'WzAsMF0=', }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts index 0a1cd556e6e91f..acde455f77cb4b 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts @@ -46,7 +46,7 @@ describe('buildEventTypeSignal', () => { const first = getFoundExceptionListItemSchemaMock(); mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -87,7 +87,7 @@ describe('buildEventTypeSignal', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -133,7 +133,7 @@ describe('buildEventTypeSignal', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -171,7 +171,7 @@ describe('buildEventTypeSignal', () => { first.data[0].entries = testEntries; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp).toEqual({ entries: [expectedEndpointExceptions], }); @@ -193,7 +193,7 @@ describe('buildEventTypeSignal', () => { .mockReturnValueOnce(first) .mockReturnValueOnce(second) .mockReturnValueOnce(third); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp.entries.length).toEqual(3); }); @@ -202,7 +202,7 @@ describe('buildEventTypeSignal', () => { exceptionsResponse.data = []; exceptionsResponse.total = 0; mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse); - const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0'); + const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1'); expect(resp.entries.length).toEqual(0); }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts index a13781519b5083..556405adff62f9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts @@ -5,6 +5,7 @@ */ import { createHash } from 'crypto'; +import { deflate } from 'zlib'; import { ExceptionListItemSchema } from '../../../../../lists/common/schemas'; import { validate } from '../../../../common/validate'; @@ -34,6 +35,7 @@ export async function buildArtifact( const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions)); const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex'); + // Keep compression info empty in case its a duplicate. Lazily compress before committing if needed. return { identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`, compressionAlgorithm: 'none', @@ -95,7 +97,7 @@ export function translateToEndpointExceptions( exc: FoundExceptionListItemSchema, schemaVersion: string ): TranslatedExceptionListItem[] { - if (schemaVersion === '1.0.0') { + if (schemaVersion === 'v1') { return exc.data.map((item) => { return translateItem(schemaVersion, item); }); @@ -180,3 +182,15 @@ function translateEntry( } } } + +export async function compressExceptionList(buffer: Buffer): Promise { + return new Promise((resolve, reject) => { + deflate(buffer, function (err, buf) { + if (err) { + reject(err); + } else { + resolve(buf); + } + }); + }); +} diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts index 3e5fdbf9484ca9..d3212eb3faf4dd 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts @@ -10,6 +10,7 @@ import { getInternalArtifactMock, getInternalArtifactMockWithDiffs, } from '../../schemas/artifacts/saved_objects.mock'; +import { ManifestConstants } from './common'; import { Manifest } from './manifest'; describe('manifest', () => { @@ -20,41 +21,45 @@ describe('manifest', () => { let manifest2: Manifest; beforeAll(async () => { - const artifactLinux = await getInternalArtifactMock('linux', '1.0.0'); - const artifactMacos = await getInternalArtifactMock('macos', '1.0.0'); - const artifactWindows = await getInternalArtifactMock('windows', '1.0.0'); + const artifactLinux = await getInternalArtifactMock('linux', 'v1'); + const artifactMacos = await getInternalArtifactMock('macos', 'v1'); + const artifactWindows = await getInternalArtifactMock('windows', 'v1'); artifacts.push(artifactLinux); artifacts.push(artifactMacos); artifacts.push(artifactWindows); - manifest1 = new Manifest(now, '1.0.0', 'v0'); + manifest1 = new Manifest(now, 'v1', ManifestConstants.INITIAL_VERSION); manifest1.addEntry(artifactLinux); manifest1.addEntry(artifactMacos); manifest1.addEntry(artifactWindows); manifest1.setVersion('abcd'); - const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', '1.0.0'); - manifest2 = new Manifest(new Date(), '1.0.0', 'v0'); + const newArtifactLinux = await getInternalArtifactMockWithDiffs('linux', 'v1'); + manifest2 = new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION); manifest2.addEntry(newArtifactLinux); manifest2.addEntry(artifactMacos); manifest2.addEntry(artifactWindows); }); test('Can create manifest with valid schema version', () => { - const manifest = new Manifest(new Date(), '1.0.0', 'v0'); + const manifest = new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION); expect(manifest).toBeInstanceOf(Manifest); }); test('Cannot create manifest with invalid schema version', () => { expect(() => { - new Manifest(new Date(), 'abcd' as ManifestSchemaVersion, 'v0'); + new Manifest( + new Date(), + 'abcd' as ManifestSchemaVersion, + ManifestConstants.INITIAL_VERSION + ); }).toThrow(); }); test('Manifest transforms correctly to expected endpoint format', async () => { expect(manifest1.toEndpointFormat()).toStrictEqual({ artifacts: { - 'endpoint-exceptionlist-linux-1.0.0': { + 'endpoint-exceptionlist-linux-v1': { compression_algorithm: 'none', encryption_algorithm: 'none', decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', @@ -62,9 +67,9 @@ describe('manifest', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, - 'endpoint-exceptionlist-macos-1.0.0': { + 'endpoint-exceptionlist-macos-v1': { compression_algorithm: 'none', encryption_algorithm: 'none', decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', @@ -72,9 +77,9 @@ describe('manifest', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, - 'endpoint-exceptionlist-windows-1.0.0': { + 'endpoint-exceptionlist-windows-v1': { compression_algorithm: 'none', encryption_algorithm: 'none', decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', @@ -82,11 +87,11 @@ describe('manifest', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }, }, manifest_version: 'abcd', - schema_version: '1.0.0', + schema_version: 'v1', }); }); @@ -94,9 +99,9 @@ describe('manifest', () => { expect(manifest1.toSavedObject()).toStrictEqual({ created: now.getTime(), ids: [ - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ], }); }); @@ -106,12 +111,12 @@ describe('manifest', () => { expect(diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', type: 'delete', }, { id: - 'endpoint-exceptionlist-linux-1.0.0-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', + 'endpoint-exceptionlist-linux-v1-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51', type: 'add', }, ]); @@ -119,7 +124,7 @@ describe('manifest', () => { test('Manifest returns data for given artifact', async () => { const artifact = artifacts[0]; - const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`); + const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.decodedSha256}`); expect(returned).toEqual(artifact); }); @@ -127,34 +132,34 @@ describe('manifest', () => { const entries = manifest1.getEntries(); const keys = Object.keys(entries); expect(keys).toEqual([ - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', ]); }); test('Manifest returns true if contains artifact', async () => { const found = manifest1.contains( - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); expect(found).toEqual(true); }); test('Manifest can be created from list of artifacts', async () => { - const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0'); + const manifest = Manifest.fromArtifacts(artifacts, 'v1', ManifestConstants.INITIAL_VERSION); expect( manifest.contains( - 'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-linux-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-macos-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); expect( manifest.contains( - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ) ).toEqual(true); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts index c343568226e229..c0124602ddb811 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.ts @@ -11,6 +11,7 @@ import { ManifestSchemaVersion, } from '../../../../common/endpoint/schema/common'; import { ManifestSchema, manifestSchema } from '../../../../common/endpoint/schema/manifest'; +import { ManifestConstants } from './common'; import { ManifestEntry } from './manifest_entry'; export interface ManifestDiff { @@ -104,7 +105,7 @@ export class Manifest { public toEndpointFormat(): ManifestSchema { const manifestObj: ManifestSchema = { - manifest_version: this.version ?? 'v0', + manifest_version: this.version ?? ManifestConstants.INITIAL_VERSION, schema_version: this.schemaVersion, artifacts: {}, }; diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts index a52114ad902586..7ea2a07210c556 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.test.ts @@ -14,7 +14,7 @@ describe('manifest_entry', () => { let manifestEntry: ManifestEntry; beforeAll(async () => { - artifact = await getInternalArtifactMock('windows', '1.0.0'); + artifact = await getInternalArtifactMock('windows', 'v1'); manifestEntry = new ManifestEntry(artifact); }); @@ -24,12 +24,12 @@ describe('manifest_entry', () => { test('Correct doc_id is returned', () => { expect(manifestEntry.getDocId()).toEqual( - 'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + 'endpoint-exceptionlist-windows-v1-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); test('Correct identifier is returned', () => { - expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-1.0.0'); + expect(manifestEntry.getIdentifier()).toEqual('endpoint-exceptionlist-windows-v1'); }); test('Correct sha256 is returned', () => { @@ -48,7 +48,7 @@ describe('manifest_entry', () => { test('Correct url is returned', () => { expect(manifestEntry.getUrl()).toEqual( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735' ); }); @@ -65,8 +65,10 @@ describe('manifest_entry', () => { decoded_size: 430, encoded_size: 430, relative_url: - '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', + '/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735', }); }); + + // TODO: add test for entry with compression }); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts index c23258c4c3ba47..b35e0c2b9ad6e5 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest_entry.ts @@ -5,6 +5,7 @@ */ import { InternalArtifactSchema } from '../../schemas/artifacts'; +import { CompressionAlgorithm } from '../../../../common/endpoint/schema/common'; import { ManifestEntrySchema } from '../../../../common/endpoint/schema/manifest'; export class ManifestEntry { @@ -15,13 +16,17 @@ export class ManifestEntry { } public getDocId(): string { - return `${this.getIdentifier()}-${this.getEncodedSha256()}`; + return `${this.getIdentifier()}-${this.getDecodedSha256()}`; } public getIdentifier(): string { return this.artifact.identifier; } + public getCompressionAlgorithm(): CompressionAlgorithm { + return this.artifact.compressionAlgorithm; + } + public getEncodedSha256(): string { return this.artifact.encodedSha256; } @@ -39,7 +44,7 @@ export class ManifestEntry { } public getUrl(): string { - return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getEncodedSha256()}`; + return `/api/endpoint/artifacts/download/${this.getIdentifier()}/${this.getDecodedSha256()}`; } public getArtifact(): InternalArtifactSchema { @@ -48,7 +53,7 @@ export class ManifestEntry { public getRecord(): ManifestEntrySchema { return { - compression_algorithm: 'none', + compression_algorithm: this.getCompressionAlgorithm(), encryption_algorithm: 'none', decoded_sha256: this.getDecodedSha256(), decoded_size: this.getDecodedSize(), diff --git a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts index 78b60e9e61f3ed..aa7f56e815d58f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts +++ b/x-pack/plugins/security_solution/server/endpoint/lib/artifacts/task.ts @@ -88,20 +88,22 @@ export class ManifestTask { return; } - manifestManager - .refresh() - .then((wrappedManifest) => { - if (wrappedManifest) { - return manifestManager.dispatch(wrappedManifest); - } - }) - .then((wrappedManifest) => { - if (wrappedManifest) { - return manifestManager.commit(wrappedManifest); - } - }) - .catch((err) => { - this.logger.error(err); - }); + try { + // get snapshot based on exception-list-agnostic SOs + // with diffs from last dispatched manifest + const snapshot = await manifestManager.getSnapshot(); + if (snapshot && snapshot.diffs.length > 0) { + // create new artifacts + await manifestManager.syncArtifacts(snapshot, 'add'); + // write to ingest-manager package config + await manifestManager.dispatch(snapshot.manifest); + // commit latest manifest state to user-artifact-manifest SO + await manifestManager.commit(snapshot.manifest); + // clean up old artifacts + await manifestManager.syncArtifacts(snapshot, 'delete'); + } + } catch (err) { + this.logger.error(err); + } }; } diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts index fbcd3bd130dfd5..8c6faee7f7a5d7 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.test.ts @@ -3,6 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { deflateSync, inflateSync } from 'zlib'; import { ILegacyClusterClient, IRouter, @@ -29,7 +30,7 @@ import { createMockEndpointAppContextServiceStartContract } from '../../mocks'; import { createMockConfig } from '../../../lib/detection_engine/routes/__mocks__'; import { WrappedTranslatedExceptionList } from '../../schemas/artifacts/lists'; -const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-1.0.0`; +const mockArtifactName = `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-windows-v1`; const expectedEndpointExceptions: WrappedTranslatedExceptionList = { entries: [ { @@ -93,7 +94,6 @@ describe('test alerts route', () => { let mockScopedClient: jest.Mocked; let mockSavedObjectClient: jest.Mocked; let mockResponse: jest.Mocked; - // @ts-ignore let routeConfig: RouteConfig; let routeHandler: RequestHandler; let endpointAppContextService: EndpointAppContextService; @@ -114,8 +114,9 @@ describe('test alerts route', () => { // The authentication with the Fleet Plugin needs a separate scoped SO Client ingestSavedObjectClient = savedObjectsClientMock.create(); ingestSavedObjectClient.find.mockReturnValue(Promise.resolve(mockIngestSOResponse)); - // @ts-ignore - startContract.savedObjectsStart.getScopedClient.mockReturnValue(ingestSavedObjectClient); + (startContract.savedObjectsStart.getScopedClient as jest.Mock).mockReturnValue( + ingestSavedObjectClient + ); endpointAppContextService.start(startContract); registerDownloadExceptionListRoute( @@ -146,11 +147,11 @@ describe('test alerts route', () => { references: [], attributes: { identifier: mockArtifactName, - schemaVersion: '1.0.0', + schemaVersion: 'v1', sha256: '123456', encoding: 'application/json', created: Date.now(), - body: Buffer.from(JSON.stringify(expectedEndpointExceptions)).toString('base64'), + body: deflateSync(JSON.stringify(expectedEndpointExceptions)).toString('base64'), size: 100, }, }; @@ -163,6 +164,8 @@ describe('test alerts route', () => { path.startsWith('/api/endpoint/artifacts/download') )!; + expect(routeConfig.options).toEqual(undefined); + await routeHandler( ({ core: { @@ -176,14 +179,16 @@ describe('test alerts route', () => { ); const expectedHeaders = { - 'content-encoding': 'application/json', - 'content-disposition': `attachment; filename=${mockArtifactName}.json`, + 'content-encoding': 'identity', + 'content-disposition': `attachment; filename=${mockArtifactName}.zz`, }; expect(mockResponse.ok).toBeCalled(); expect(mockResponse.ok.mock.calls[0][0]?.headers).toEqual(expectedHeaders); - const artifact = mockResponse.ok.mock.calls[0][0]?.body; - expect(artifact).toEqual(Buffer.from(mockArtifact.attributes.body, 'base64').toString()); + const artifact = inflateSync(mockResponse.ok.mock.calls[0][0]?.body as Buffer).toString(); + expect(artifact).toEqual( + inflateSync(Buffer.from(mockArtifact.attributes.body, 'base64')).toString() + ); }); it('should handle fetching a non-existent artifact', async () => { @@ -233,7 +238,7 @@ describe('test alerts route', () => { // Add to the download cache const mockArtifact = expectedEndpointExceptions; const cacheKey = `${mockArtifactName}-${mockSha}`; - cache.set(cacheKey, JSON.stringify(mockArtifact)); + cache.set(cacheKey, Buffer.from(JSON.stringify(mockArtifact))); // TODO: add compression here [routeConfig, routeHandler] = routerMock.get.mock.calls.find(([{ path }]) => path.startsWith('/api/endpoint/artifacts/download') diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts index 337393e768a8f2..1b364a04a4272a 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/artifacts/download_exception_list.ts @@ -43,9 +43,7 @@ export function registerDownloadExceptionListRoute( DownloadArtifactRequestParamsSchema >(downloadArtifactRequestParamsSchema), }, - options: { tags: [] }, }, - // @ts-ignore async (context, req, res) => { let scopedSOClient: SavedObjectsClientContract; const logger = endpointContext.logFactory.get('download_exception_list'); @@ -55,19 +53,19 @@ export function registerDownloadExceptionListRoute( scopedSOClient = endpointContext.service.getScopedSavedObjectsClient(req); await authenticateAgentWithAccessToken(scopedSOClient, req); } catch (err) { - if (err.output.statusCode === 401) { + if ((err.isBoom ? err.output.statusCode : err.statusCode) === 401) { return res.unauthorized(); } else { return res.notFound(); } } - const buildAndValidateResponse = (artName: string, body: string): IKibanaResponse => { + const buildAndValidateResponse = (artName: string, body: Buffer): IKibanaResponse => { const artifact: HttpResponseOptions = { body, headers: { - 'content-encoding': 'application/json', - 'content-disposition': `attachment; filename=${artName}.json`, + 'content-encoding': 'identity', + 'content-disposition': `attachment; filename=${artName}.zz`, }, }; @@ -90,7 +88,7 @@ export function registerDownloadExceptionListRoute( return scopedSOClient .get(ArtifactConstants.SAVED_OBJECT_TYPE, id) .then((artifact: SavedObject) => { - const body = Buffer.from(artifact.attributes.body, 'base64').toString(); + const body = Buffer.from(artifact.attributes.body, 'base64'); cache.set(id, body); return buildAndValidateResponse(artifact.attributes.identifier, body); }) diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts index 235e7152b83cf1..4b2eb3ea1ddb01 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/index.ts @@ -18,6 +18,7 @@ import { HostStatus, } from '../../../../common/endpoint/types'; import { EndpointAppContext } from '../../types'; +import { AgentService } from '../../../../../ingest_manager/server'; import { Agent, AgentStatus } from '../../../../../ingest_manager/common/types/models'; import { findAllUnenrolledAgentIds } from './support/unenroll'; @@ -26,8 +27,9 @@ interface HitSource { } interface MetadataRequestContext { + agentService: AgentService; + logger: Logger; requestHandlerContext: RequestHandlerContext; - endpointAppContext: EndpointAppContext; } const HOST_STATUS_MAPPING = new Map([ @@ -35,8 +37,12 @@ const HOST_STATUS_MAPPING = new Map([ ['offline', HostStatus.OFFLINE], ]); +const getLogger = (endpointAppContext: EndpointAppContext): Logger => { + return endpointAppContext.logFactory.get('metadata'); +}; + export function registerEndpointRoutes(router: IRouter, endpointAppContext: EndpointAppContext) { - const logger = endpointAppContext.logFactory.get('metadata'); + const logger = getLogger(endpointAppContext); router.post( { path: '/api/endpoint/metadata', @@ -66,12 +72,23 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp }) ), }, - options: { authRequired: true }, + options: { authRequired: true, tags: ['access:securitySolution'] }, }, async (context, req, res) => { try { + const agentService = endpointAppContext.service.getAgentService(); + if (agentService === undefined) { + throw new Error('agentService not available'); + } + + const metadataRequestContext: MetadataRequestContext = { + agentService, + logger, + requestHandlerContext: context, + }; + const unenrolledAgentIds = await findAllUnenrolledAgentIds( - endpointAppContext.service.getAgentService(), + agentService, context.core.savedObjects.client ); @@ -88,11 +105,9 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp 'search', queryParams )) as SearchResponse; + return res.ok({ - body: await mapToHostResultList(queryParams, response, { - endpointAppContext, - requestHandlerContext: context, - }), + body: await mapToHostResultList(queryParams, response, metadataRequestContext), }); } catch (err) { logger.warn(JSON.stringify(err, null, 2)); @@ -107,17 +122,22 @@ export function registerEndpointRoutes(router: IRouter, endpointAppContext: Endp validate: { params: schema.object({ id: schema.string() }), }, - options: { authRequired: true }, + options: { authRequired: true, tags: ['access:securitySolution'] }, }, async (context, req, res) => { + const agentService = endpointAppContext.service.getAgentService(); + if (agentService === undefined) { + return res.internalError({ body: 'agentService not available' }); + } + + const metadataRequestContext: MetadataRequestContext = { + agentService, + logger, + requestHandlerContext: context, + }; + try { - const doc = await getHostData( - { - endpointAppContext, - requestHandlerContext: context, - }, - req.params.id - ); + const doc = await getHostData(metadataRequestContext, req.params.id); if (doc) { return res.ok({ body: doc }); } @@ -164,17 +184,16 @@ async function findAgent( metadataRequestContext: MetadataRequestContext, hostMetadata: HostMetadata ): Promise { - const logger = metadataRequestContext.endpointAppContext.logFactory.get('metadata'); try { - return await metadataRequestContext.endpointAppContext.service - .getAgentService() - .getAgent( - metadataRequestContext.requestHandlerContext.core.savedObjects.client, - hostMetadata.elastic.agent.id - ); + return await metadataRequestContext.agentService.getAgent( + metadataRequestContext.requestHandlerContext.core.savedObjects.client, + hostMetadata.elastic.agent.id + ); } catch (e) { if (e.isBoom && e.output.statusCode === 404) { - logger.warn(`agent with id ${hostMetadata.elastic.agent.id} not found`); + metadataRequestContext.logger.warn( + `agent with id ${hostMetadata.elastic.agent.id} not found` + ); return undefined; } else { throw e; @@ -217,7 +236,7 @@ async function enrichHostMetadata( ): Promise { let hostStatus = HostStatus.ERROR; let elasticAgentId = hostMetadata?.elastic?.agent?.id; - const log = logger(metadataRequestContext.endpointAppContext); + const log = metadataRequestContext.logger; try { /** * Get agent status by elastic agent id if available or use the host id. @@ -228,12 +247,10 @@ async function enrichHostMetadata( log.warn(`Missing elastic agent id, using host id instead ${elasticAgentId}`); } - const status = await metadataRequestContext.endpointAppContext.service - .getAgentService() - .getAgentStatusById( - metadataRequestContext.requestHandlerContext.core.savedObjects.client, - elasticAgentId - ); + const status = await metadataRequestContext.agentService.getAgentStatusById( + metadataRequestContext.requestHandlerContext.core.savedObjects.client, + elasticAgentId + ); hostStatus = HOST_STATUS_MAPPING.get(status) || HostStatus.ERROR; } catch (e) { if (e.isBoom && e.output.statusCode === 404) { @@ -248,7 +265,3 @@ async function enrichHostMetadata( host_status: hostStatus, }; } - -const logger = (endpointAppContext: EndpointAppContext): Logger => { - return endpointAppContext.logFactory.get('metadata'); -}; diff --git a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts index 42cce382ec20ca..81027b42eb64fb 100644 --- a/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/routes/metadata/metadata.test.ts @@ -47,8 +47,9 @@ describe('test endpoint route', () => { let routeHandler: RequestHandler; // eslint-disable-next-line @typescript-eslint/no-explicit-any let routeConfig: RouteConfig; - let mockAgentService: ReturnType< - typeof createMockEndpointAppContextServiceStartContract + // tests assume that ingestManager is enabled, and thus agentService is available + let mockAgentService: Required< + ReturnType >['agentService']; let endpointAppContextService: EndpointAppContextService; const noUnenrolledAgent = { @@ -70,7 +71,7 @@ describe('test endpoint route', () => { endpointAppContextService = new EndpointAppContextService(); const startContract = createMockEndpointAppContextServiceStartContract(); endpointAppContextService.start(startContract); - mockAgentService = startContract.agentService; + mockAgentService = startContract.agentService!; registerEndpointRoutes(routerMock, { logFactory: loggingSystemMock.create(), @@ -97,7 +98,7 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] }); expect(mockResponse.ok).toBeCalled(); const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList; expect(endpointResultList.hosts.length).toEqual(1); @@ -139,7 +140,7 @@ describe('test endpoint route', () => { expect(mockScopedClient.callAsCurrentUser.mock.calls[0][1]?.body?.query).toEqual({ match_all: {}, }); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] }); expect(mockResponse.ok).toBeCalled(); const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList; expect(endpointResultList.hosts.length).toEqual(1); @@ -202,7 +203,7 @@ describe('test endpoint route', () => { ], }, }); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ authRequired: true, tags: ['access:securitySolution'] }); expect(mockResponse.ok).toBeCalled(); const endpointResultList = mockResponse.ok.mock.calls[0][0]?.body as HostResultList; expect(endpointResultList.hosts.length).toEqual(1); @@ -234,7 +235,10 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ + authRequired: true, + tags: ['access:securitySolution'], + }); expect(mockResponse.notFound).toBeCalled(); const message = mockResponse.notFound.mock.calls[0][0]?.body; expect(message).toEqual('Endpoint Not Found'); @@ -263,7 +267,10 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ + authRequired: true, + tags: ['access:securitySolution'], + }); expect(mockResponse.ok).toBeCalled(); const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo; expect(result).toHaveProperty('metadata.Endpoint'); @@ -298,7 +305,10 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ + authRequired: true, + tags: ['access:securitySolution'], + }); expect(mockResponse.ok).toBeCalled(); const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo; expect(result.host_status).toEqual(HostStatus.ERROR); @@ -328,7 +338,10 @@ describe('test endpoint route', () => { ); expect(mockScopedClient.callAsCurrentUser).toHaveBeenCalledTimes(1); - expect(routeConfig.options).toEqual({ authRequired: true }); + expect(routeConfig.options).toEqual({ + authRequired: true, + tags: ['access:securitySolution'], + }); expect(mockResponse.ok).toBeCalled(); const result = mockResponse.ok.mock.calls[0][0]?.body as HostInfo; expect(result.host_status).toEqual(HostStatus.ERROR); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts index 3c066e150288ae..d5a30951e9398d 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/common.ts @@ -6,14 +6,19 @@ import * as t from 'io-ts'; -export const body = t.string; +export const buffer = new t.Type( + 'buffer', + (input: unknown): input is Buffer => Buffer.isBuffer(input), + (input, context) => (Buffer.isBuffer(input) ? t.success(input) : t.failure(input, context)), + t.identity +); -export const created = t.number; // TODO: Make this into an ISO Date string check +export const created = t.number; export const encoding = t.keyof({ - 'application/json': null, + identity: null, }); export const schemaVersion = t.keyof({ - '1.0.0': null, + v1: null, }); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts index 537f7707889e44..3705062449c608 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/response/download_artifact_schema.ts @@ -5,9 +5,8 @@ */ import * as t from 'io-ts'; -import { encoding } from '../common'; +import { buffer, encoding } from '../common'; -const body = t.string; const headers = t.exact( t.type({ 'content-encoding': encoding, @@ -17,7 +16,7 @@ const headers = t.exact( export const downloadArtifactResponseSchema = t.exact( t.type({ - body, + body: buffer, headers, }) ); diff --git a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts index e4cd7f48a2901e..aa11f4409269aa 100644 --- a/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts +++ b/x-pack/plugins/security_solution/server/endpoint/schemas/artifacts/saved_objects.ts @@ -12,7 +12,9 @@ import { sha256, size, } from '../../../../common/endpoint/schema/common'; -import { body, created } from './common'; +import { created } from './common'; + +export const body = t.string; // base64 export const internalArtifactSchema = t.exact( t.type({ diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts index 08e29b5c6b82b4..3e3b12c04d65c8 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.test.ts @@ -27,7 +27,7 @@ describe('artifact_client', () => { test('can create artifact', async () => { const savedObjectsClient = savedObjectsClientMock.create(); const artifactClient = getArtifactClientMock(savedObjectsClient); - const artifact = await getInternalArtifactMock('linux', '1.0.0'); + const artifact = await getInternalArtifactMock('linux', 'v1'); await artifactClient.createArtifact(artifact); expect(savedObjectsClient.create).toHaveBeenCalledWith( ArtifactConstants.SAVED_OBJECT_TYPE, diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts index e899905602c8df..ca53a891c4d6be 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/artifact_client.ts @@ -16,7 +16,7 @@ export class ArtifactClient { } public getArtifactId(artifact: InternalArtifactSchema) { - return `${artifact.identifier}-${artifact.encodedSha256}`; + return `${artifact.identifier}-${artifact.decodedSha256}`; } public async getArtifact(id: string): Promise> { diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts index bfeacbcedf2cb9..d869ed9493abc9 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.mock.ts @@ -12,7 +12,7 @@ export const getManifestClientMock = ( savedObjectsClient?: SavedObjectsClientContract ): ManifestClient => { if (savedObjectsClient !== undefined) { - return new ManifestClient(savedObjectsClient, '1.0.0'); + return new ManifestClient(savedObjectsClient, 'v1'); } - return new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + return new ManifestClient(savedObjectsClientMock.create(), 'v1'); }; diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts index 5780c6279ee6ae..fe3f193bc8ff59 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_client.test.ts @@ -14,7 +14,7 @@ import { ManifestClient } from './manifest_client'; describe('manifest_client', () => { describe('ManifestClient sanity checks', () => { test('can create ManifestClient', () => { - const manifestClient = new ManifestClient(savedObjectsClientMock.create(), '1.0.0'); + const manifestClient = new ManifestClient(savedObjectsClientMock.create(), 'v1'); expect(manifestClient).toBeInstanceOf(ManifestClient); }); diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts index 483b3434d63f2c..dfbe2572076d00 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.mock.ts @@ -15,6 +15,7 @@ import { buildArtifact, getFullEndpointExceptionList, } from '../../../lib/artifacts'; +import { ManifestConstants } from '../../../lib/artifacts/common'; import { InternalArtifactSchema } from '../../../schemas/artifacts'; import { getArtifactClientMock } from '../artifact_client.mock'; import { getManifestClientMock } from '../manifest_client.mock'; @@ -69,13 +70,13 @@ async function mockBuildExceptionListArtifacts( export class ManifestManagerMock extends ManifestManager { // @ts-ignore private buildExceptionListArtifacts = async () => { - return mockBuildExceptionListArtifacts('linux', '1.0.0'); + return mockBuildExceptionListArtifacts('linux', 'v1'); }; // @ts-ignore private getLastDispatchedManifest = jest .fn() - .mockResolvedValue(new Manifest(new Date(), '1.0.0', 'v0')); + .mockResolvedValue(new Manifest(new Date(), 'v1', ManifestConstants.INITIAL_VERSION)); // @ts-ignore private getManifestClient = jest diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts index 1d6dffadde61a9..b1cbc41459f15f 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.test.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ +import { inflateSync } from 'zlib'; import { savedObjectsClientMock } from 'src/core/server/mocks'; import { ArtifactConstants, @@ -15,32 +16,33 @@ import { getPackageConfigServiceMock, getManifestManagerMock } from './manifest_ describe('manifest_manager', () => { describe('ManifestManager sanity checks', () => { - test('ManifestManager can refresh manifest', async () => { + test('ManifestManager can snapshot manifest', async () => { const manifestManager = getManifestManagerMock(); - const manifestWrapper = await manifestManager.refresh(); - expect(manifestWrapper!.diffs).toEqual([ + const snapshot = await manifestManager.getSnapshot(); + expect(snapshot!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + 'endpoint-exceptionlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); - expect(manifestWrapper!.manifest).toBeInstanceOf(Manifest); + expect(snapshot!.manifest).toBeInstanceOf(Manifest); }); test('ManifestManager populates cache properly', async () => { const cache = new ExceptionsCache(5); const manifestManager = getManifestManagerMock({ cache }); - const manifestWrapper = await manifestManager.refresh(); - expect(manifestWrapper!.diffs).toEqual([ + const snapshot = await manifestManager.getSnapshot(); + expect(snapshot!.diffs).toEqual([ { id: - 'endpoint-exceptionlist-linux-1.0.0-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', + 'endpoint-exceptionlist-linux-v1-1a8295e6ccb93022c6f5ceb8997b29f2912389b3b38f52a8f5a2ff7b0154b1bc', type: 'add', }, ]); - const diff = manifestWrapper!.diffs[0]; - const entry = JSON.parse(cache.get(diff!.id)!); + await manifestManager.syncArtifacts(snapshot!, 'add'); + const diff = snapshot!.diffs[0]; + const entry = JSON.parse(inflateSync(cache.get(diff!.id)! as Buffer).toString()); expect(entry).toEqual({ entries: [ { @@ -73,16 +75,16 @@ describe('manifest_manager', () => { test('ManifestManager can dispatch manifest', async () => { const packageConfigService = getPackageConfigServiceMock(); const manifestManager = getManifestManagerMock({ packageConfigService }); - const manifestWrapperRefresh = await manifestManager.refresh(); - const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); - expect(manifestWrapperRefresh).toEqual(manifestWrapperDispatch); - const entries = manifestWrapperDispatch!.manifest.getEntries(); + const snapshot = await manifestManager.getSnapshot(); + const dispatched = await manifestManager.dispatch(snapshot!.manifest); + expect(dispatched).toEqual(true); + const entries = snapshot!.manifest.getEntries(); const artifact = Object.values(entries)[0].getArtifact(); expect( packageConfigService.update.mock.calls[0][2].inputs[0].config.artifact_manifest.value ).toEqual({ - manifest_version: 'v0', - schema_version: '1.0.0', + manifest_version: ManifestConstants.INITIAL_VERSION, + schema_version: 'v1', artifacts: { [artifact.identifier]: { compression_algorithm: 'none', @@ -91,7 +93,7 @@ describe('manifest_manager', () => { encoded_sha256: artifact.encodedSha256, decoded_size: artifact.decodedSize, encoded_size: artifact.encodedSize, - relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.encodedSha256}`, + relative_url: `/api/endpoint/artifacts/download/${artifact.identifier}/${artifact.decodedSha256}`, }, }, }); @@ -103,15 +105,21 @@ describe('manifest_manager', () => { savedObjectsClient, }); - const manifestWrapperRefresh = await manifestManager.refresh(); - const manifestWrapperDispatch = await manifestManager.dispatch(manifestWrapperRefresh); + const snapshot = await manifestManager.getSnapshot(); + await manifestManager.syncArtifacts(snapshot!, 'add'); + const diff = { id: 'abcd', type: 'delete', }; - manifestWrapperDispatch!.diffs.push(diff); + snapshot!.diffs.push(diff); + + const dispatched = await manifestManager.dispatch(snapshot!.manifest); + expect(dispatched).toEqual(true); + + await manifestManager.commit(snapshot!.manifest); - await manifestManager.commit(manifestWrapperDispatch); + await manifestManager.syncArtifacts(snapshot!, 'delete'); // created new artifact expect(savedObjectsClient.create.mock.calls[0][0]).toEqual( diff --git a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts index f7bc711d4bd055..9726e28f541868 100644 --- a/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts +++ b/x-pack/plugins/security_solution/server/endpoint/services/artifacts/manifest_manager/manifest_manager.ts @@ -4,7 +4,8 @@ * you may not use this file except in compliance with the Elastic License. */ -import { Logger, SavedObjectsClientContract, SavedObject } from 'src/core/server'; +import { Logger, SavedObjectsClientContract } from 'src/core/server'; +import { createHash } from 'crypto'; import { PackageConfigServiceInterface } from '../../../../../../ingest_manager/server'; import { ExceptionListClient } from '../../../../../../lists/server'; import { ManifestSchemaVersion } from '../../../../../common/endpoint/schema/common'; @@ -17,9 +18,10 @@ import { ExceptionsCache, ManifestDiff, } from '../../../lib/artifacts'; -import { InternalArtifactSchema, InternalManifestSchema } from '../../../schemas/artifacts'; +import { InternalArtifactSchema } from '../../../schemas/artifacts'; import { ArtifactClient } from '../artifact_client'; import { ManifestClient } from '../manifest_client'; +import { compressExceptionList } from '../../../lib/artifacts/lists'; export interface ManifestManagerContext { savedObjectsClient: SavedObjectsClientContract; @@ -30,11 +32,11 @@ export interface ManifestManagerContext { cache: ExceptionsCache; } -export interface ManifestRefreshOpts { +export interface ManifestSnapshotOpts { initialize?: boolean; } -export interface WrappedManifest { +export interface ManifestSnapshot { manifest: Manifest; diffs: ManifestDiff[]; } @@ -56,215 +58,259 @@ export class ManifestManager { this.cache = context.cache; } - private getManifestClient(schemaVersion: string): ManifestClient { + /** + * Gets a ManifestClient for the provided schemaVersion. + * + * @param schemaVersion + */ + private getManifestClient(schemaVersion: string) { return new ManifestClient(this.savedObjectsClient, schemaVersion as ManifestSchemaVersion); } - private async buildExceptionListArtifacts( - schemaVersion: string - ): Promise { - const artifacts: InternalArtifactSchema[] = []; + /** + * Builds an array of artifacts (one per supported OS) based on the current + * state of exception-list-agnostic SO's. + * + * @param schemaVersion + */ + private async buildExceptionListArtifacts(schemaVersion: string) { + return ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS.reduce( + async (acc: Promise, os) => { + const exceptionList = await getFullEndpointExceptionList( + this.exceptionListClient, + os, + schemaVersion + ); + const artifacts = await acc; + const artifact = await buildArtifact(exceptionList, os, schemaVersion); + artifacts.push(artifact); + return Promise.resolve(artifacts); + }, + Promise.resolve([]) + ); + } - for (const os of ArtifactConstants.SUPPORTED_OPERATING_SYSTEMS) { - const exceptionList = await getFullEndpointExceptionList( - this.exceptionListClient, - os, - schemaVersion + /** + * Returns the last dispatched manifest based on the current state of the + * user-artifact-manifest SO. + * + * @param schemaVersion + */ + private async getLastDispatchedManifest(schemaVersion: string) { + try { + const manifestClient = this.getManifestClient(schemaVersion); + const manifestSo = await manifestClient.getManifest(); + + if (manifestSo.version === undefined) { + throw new Error('No version returned for manifest.'); + } + + const manifest = new Manifest( + new Date(manifestSo.attributes.created), + schemaVersion, + manifestSo.version ); - const artifact = await buildArtifact(exceptionList, os, schemaVersion); - artifacts.push(artifact); + for (const id of manifestSo.attributes.ids) { + const artifactSo = await this.artifactClient.getArtifact(id); + manifest.addEntry(artifactSo.attributes); + } + return manifest; + } catch (err) { + if (err.output.statusCode !== 404) { + throw err; + } + return null; } - - return artifacts; } - private async getLastDispatchedManifest(schemaVersion: string): Promise { - return this.getManifestClient(schemaVersion) - .getManifest() - .then(async (manifestSo: SavedObject) => { - if (manifestSo.version === undefined) { - throw new Error('No version returned for manifest.'); - } - const manifest = new Manifest( - new Date(manifestSo.attributes.created), - schemaVersion, - manifestSo.version - ); - - for (const id of manifestSo.attributes.ids) { - const artifactSo = await this.artifactClient.getArtifact(id); - manifest.addEntry(artifactSo.attributes); - } - - return manifest; - }) - .catch((err) => { - if (err.output.statusCode !== 404) { - throw err; - } + /** + * Snapshots a manifest based on current state of exception-list-agnostic SOs. + * + * @param opts TODO + */ + public async getSnapshot(opts?: ManifestSnapshotOpts) { + try { + let oldManifest: Manifest | null; + + // Get the last-dispatched manifest + oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + + if (oldManifest === null && opts !== undefined && opts.initialize) { + oldManifest = new Manifest( + new Date(), + ManifestConstants.SCHEMA_VERSION, + ManifestConstants.INITIAL_VERSION + ); // create empty manifest + } else if (oldManifest == null) { + this.logger.debug('Manifest does not exist yet. Waiting...'); return null; - }); - } + } - public async refresh(opts?: ManifestRefreshOpts): Promise { - let oldManifest: Manifest | null; + // Build new exception list artifacts + const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + + // Build new manifest + const newManifest = Manifest.fromArtifacts( + artifacts, + ManifestConstants.SCHEMA_VERSION, + oldManifest.getVersion() + ); - // Get the last-dispatched manifest - oldManifest = await this.getLastDispatchedManifest(ManifestConstants.SCHEMA_VERSION); + // Get diffs + const diffs = newManifest.diff(oldManifest); - if (oldManifest === null && opts !== undefined && opts.initialize) { - oldManifest = new Manifest(new Date(), ManifestConstants.SCHEMA_VERSION, 'v0'); // create empty manifest - } else if (oldManifest == null) { - this.logger.debug('Manifest does not exist yet. Waiting...'); + return { + manifest: newManifest, + diffs, + }; + } catch (err) { + this.logger.error(err); return null; } + } - // Build new exception list artifacts - const artifacts = await this.buildExceptionListArtifacts(ArtifactConstants.SCHEMA_VERSION); + /** + * Syncs artifacts based on provided snapshot. + * + * Creates artifacts that do not yet exist and cleans up old artifacts that have been + * superceded by this snapshot. + * + * Can be filtered to apply one or both operations. + * + * @param snapshot + * @param diffType + */ + public async syncArtifacts(snapshot: ManifestSnapshot, diffType?: 'add' | 'delete') { + const filteredDiffs = snapshot.diffs.reduce((diffs: ManifestDiff[], diff) => { + if (diff.type === diffType || diffType === undefined) { + diffs.push(diff); + } else if (!['add', 'delete'].includes(diff.type)) { + // TODO: replace with io-ts schema + throw new Error(`Unsupported diff type: ${diff.type}`); + } + return diffs; + }, []); - // Build new manifest - const newManifest = Manifest.fromArtifacts( - artifacts, - ManifestConstants.SCHEMA_VERSION, - oldManifest.getVersion() - ); + const adds = filteredDiffs.filter((diff) => { + return diff.type === 'add'; + }); - // Get diffs - const diffs = newManifest.diff(oldManifest); - - // Create new artifacts - for (const diff of diffs) { - if (diff.type === 'add') { - const artifact = newManifest.getArtifact(diff.id); - try { - await this.artifactClient.createArtifact(artifact); - - // Cache the body of the artifact - this.cache.set(diff.id, Buffer.from(artifact.body, 'base64').toString()); - } catch (err) { - if (err.status === 409) { - // This artifact already existed... - this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); - } else { - throw err; - } + const deletes = filteredDiffs.filter((diff) => { + return diff.type === 'delete'; + }); + + for (const diff of adds) { + const artifact = snapshot.manifest.getArtifact(diff.id); + const compressedArtifact = await compressExceptionList(Buffer.from(artifact.body, 'base64')); + artifact.body = compressedArtifact.toString('base64'); + artifact.encodedSize = compressedArtifact.byteLength; + artifact.compressionAlgorithm = 'zlib'; + artifact.encodedSha256 = createHash('sha256').update(compressedArtifact).digest('hex'); + + try { + await this.artifactClient.createArtifact(artifact); + } catch (err) { + if (err.status === 409) { + this.logger.debug(`Tried to create artifact ${diff.id}, but it already exists.`); + } else { + throw err; } } + // Cache the body of the artifact + this.cache.set(diff.id, Buffer.from(artifact.body, 'base64')); } - return { - manifest: newManifest, - diffs, - }; + for (const diff of deletes) { + await this.artifactClient.deleteArtifact(diff.id); + // TODO: should we delete the cache entry here? + this.logger.info(`Cleaned up artifact ${diff.id}`); + } } /** - * Dispatches the manifest by writing it to the endpoint packageConfig. + * Dispatches the manifest by writing it to the endpoint package config. * - * @return {WrappedManifest | null} WrappedManifest if all dispatched, else null */ - public async dispatch(wrappedManifest: WrappedManifest | null): Promise { - if (wrappedManifest === null) { - this.logger.debug('wrappedManifest was null, aborting dispatch'); - return null; - } - - function showDiffs(diffs: ManifestDiff[]) { - return diffs.map((diff) => { - const op = diff.type === 'add' ? '(+)' : '(-)'; - return `${op}${diff.id}`; + public async dispatch(manifest: Manifest) { + let paging = true; + let page = 1; + let success = true; + + while (paging) { + const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, { + page, + perPage: 20, + kuery: 'ingest-package-configs.package.name:endpoint', }); - } - if (wrappedManifest.diffs.length > 0) { - this.logger.info(`Dispatching new manifest with diffs: ${showDiffs(wrappedManifest.diffs)}`); - - let paging = true; - let page = 1; - let success = true; - - while (paging) { - const { items, total } = await this.packageConfigService.list(this.savedObjectsClient, { - page, - perPage: 20, - kuery: 'ingest-package-configs.package.name:endpoint', - }); - - for (const packageConfig of items) { - const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; - - if ( - newPackageConfig.inputs.length > 0 && - newPackageConfig.inputs[0].config !== undefined - ) { - const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { - value: {}, - }; - artifactManifest.value = wrappedManifest.manifest.toEndpointFormat(); - newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; - - await this.packageConfigService - .update(this.savedObjectsClient, id, newPackageConfig) - .then((response) => { - this.logger.debug(`Updated package config ${id}`); - }) - .catch((err) => { - success = false; - this.logger.debug(`Error updating package config ${id}`); - this.logger.error(err); - }); - } else { + for (const packageConfig of items) { + const { id, revision, updated_at, updated_by, ...newPackageConfig } = packageConfig; + if (newPackageConfig.inputs.length > 0 && newPackageConfig.inputs[0].config !== undefined) { + const artifactManifest = newPackageConfig.inputs[0].config.artifact_manifest ?? { + value: {}, + }; + artifactManifest.value = manifest.toEndpointFormat(); + newPackageConfig.inputs[0].config.artifact_manifest = artifactManifest; + + try { + await this.packageConfigService.update(this.savedObjectsClient, id, newPackageConfig); + this.logger.debug( + `Updated package config ${id} with manifest version ${manifest.getVersion()}` + ); + } catch (err) { success = false; - this.logger.debug(`Package config ${id} has no config.`); + this.logger.debug(`Error updating package config ${id}`); + this.logger.error(err); } + } else { + success = false; + this.logger.debug(`Package config ${id} has no config.`); } - - paging = page * items.length < total; - page++; } - return success ? wrappedManifest : null; - } else { - this.logger.debug('No manifest diffs [no-op]'); + paging = page * items.length < total; + page++; } - return null; + // TODO: revisit success logic + return success; } - public async commit(wrappedManifest: WrappedManifest | null) { - if (wrappedManifest === null) { - this.logger.debug('wrappedManifest was null, aborting commit'); - return; - } - - const manifestClient = this.getManifestClient(wrappedManifest.manifest.getSchemaVersion()); + /** + * Commits a manifest to indicate that it has been dispatched. + * + * @param manifest + */ + public async commit(manifest: Manifest) { + const manifestClient = this.getManifestClient(manifest.getSchemaVersion()); // Commit the new manifest - if (wrappedManifest.manifest.getVersion() === 'v0') { - await manifestClient.createManifest(wrappedManifest.manifest.toSavedObject()); + if (manifest.getVersion() === ManifestConstants.INITIAL_VERSION) { + await manifestClient.createManifest(manifest.toSavedObject()); } else { - const version = wrappedManifest.manifest.getVersion(); - if (version === 'v0') { + const version = manifest.getVersion(); + if (version === ManifestConstants.INITIAL_VERSION) { throw new Error('Updating existing manifest with baseline version. Bad state.'); } - await manifestClient.updateManifest(wrappedManifest.manifest.toSavedObject(), { + await manifestClient.updateManifest(manifest.toSavedObject(), { version, }); } - this.logger.info(`Commited manifest ${wrappedManifest.manifest.getVersion()}`); + this.logger.info(`Committed manifest ${manifest.getVersion()}`); + } - // Clean up old artifacts - for (const diff of wrappedManifest.diffs) { - try { - if (diff.type === 'delete') { - await this.artifactClient.deleteArtifact(diff.id); - this.logger.info(`Cleaned up artifact ${diff.id}`); - } - } catch (err) { - this.logger.error(err); - } - } + /** + * Confirms that a packageConfig exists with provided name. + */ + public async confirmPackageConfigExists(name: string) { + // TODO: what if there are multiple results? uh oh. + const { total } = await this.packageConfigService.list(this.savedObjectsClient, { + page: 1, + perPage: 20, + kuery: `ingest-package-configs.name:${name}`, + }); + return total > 0; } } diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 568aa0e85de937..86b8b9b5cf61df 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -24,7 +24,7 @@ import { ListPluginSetup } from '../../lists/server'; import { EncryptedSavedObjectsPluginSetup as EncryptedSavedObjectsSetup } from '../../encrypted_saved_objects/server'; import { SpacesPluginSetup as SpacesSetup } from '../../spaces/server'; import { LicensingPluginSetup } from '../../licensing/server'; -import { IngestManagerStartContract } from '../../ingest_manager/server'; +import { IngestManagerStartContract, ExternalCallback } from '../../ingest_manager/server'; import { TaskManagerSetupContract, TaskManagerStartContract } from '../../task_manager/server'; import { initServer } from './init_server'; import { compose } from './lib/compose/kibana'; @@ -54,14 +54,14 @@ export interface SetupPlugins { licensing: LicensingPluginSetup; security?: SecuritySetup; spaces?: SpacesSetup; - taskManager: TaskManagerSetupContract; + taskManager?: TaskManagerSetupContract; ml?: MlSetup; lists?: ListPluginSetup; } export interface StartPlugins { - ingestManager: IngestManagerStartContract; - taskManager: TaskManagerStartContract; + ingestManager?: IngestManagerStartContract; + taskManager?: TaskManagerStartContract; } // eslint-disable-next-line @typescript-eslint/no-empty-interface @@ -227,11 +227,15 @@ export class Plugin implements IPlugin { + return plugins.taskManager && plugins.lists; + }; + + if (exceptionListsSetupEnabled()) { this.lists = plugins.lists; this.manifestTask = new ManifestTask({ endpointAppContext: endpointContext, - taskManager: plugins.taskManager, + taskManager: plugins.taskManager!, }); } @@ -245,32 +249,41 @@ export class Plugin implements IPlugin void) | undefined; + + const exceptionListsStartEnabled = () => { + return this.lists && plugins.taskManager && plugins.ingestManager; + }; + + if (exceptionListsStartEnabled()) { + const exceptionListClient = this.lists!.getExceptionListClient(savedObjectsClient, 'kibana'); const artifactClient = new ArtifactClient(savedObjectsClient); + + registerIngestCallback = plugins.ingestManager!.registerExternalCallback; manifestManager = new ManifestManager({ savedObjectsClient, artifactClient, exceptionListClient, - packageConfigService: plugins.ingestManager.packageConfigService, + packageConfigService: plugins.ingestManager!.packageConfigService, logger: this.logger, cache: this.exceptionsCache, }); } this.endpointAppContextService.start({ - agentService: plugins.ingestManager.agentService, + agentService: plugins.ingestManager?.agentService, + logger: this.logger, manifestManager, - registerIngestCallback: plugins.ingestManager.registerExternalCallback, + registerIngestCallback, savedObjectsStart: core.savedObjects, }); - if (this.manifestTask) { + if (exceptionListsStartEnabled() && this.manifestTask) { this.manifestTask.start({ - taskManager: plugins.taskManager, + taskManager: plugins.taskManager!, }); } else { - this.logger.debug('Manifest task not available.'); + this.logger.debug('User artifacts task not available.'); } return {}; diff --git a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts index 1f1c6f27b636aa..ca59d396839ae2 100644 --- a/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts +++ b/x-pack/test/api_integration/apis/endpoint/artifacts/index.ts @@ -5,6 +5,7 @@ */ import expect from '@kbn/expect'; +import { inflateSync } from 'zlib'; import { FtrProviderContext } from '../../../ftr_provider_context'; import { getSupertestWithoutAuth, setupIngest } from '../../fleet/agents/services'; @@ -61,7 +62,7 @@ export default function (providerContext: FtrProviderContext) { it('should fail to find artifact with invalid hash', async () => { await supertestWithoutAuth - .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/abcd') + .get('/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-v1/abcd') .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() @@ -71,14 +72,14 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { - const artifactJson = JSON.parse(response.text); + const artifactJson = JSON.parse(inflateSync(response.body).toString()); expect(artifactJson).to.eql({ entries: [ { @@ -118,26 +119,26 @@ export default function (providerContext: FtrProviderContext) { it('should download an artifact with correct hash from cache', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { - JSON.parse(response.text); + JSON.parse(inflateSync(response.body).toString()); }) .then(async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-v1/d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey ${agentAccessAPIKey}`) .send() .expect(200) .expect((response) => { - const artifactJson = JSON.parse(response.text); + const artifactJson = JSON.parse(inflateSync(response.body).toString()); expect(artifactJson).to.eql({ entries: [ { @@ -178,7 +179,7 @@ export default function (providerContext: FtrProviderContext) { it('should fail on invalid api key', async () => { await supertestWithoutAuth .get( - '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' + '/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-v1/1825fb19fcc6dc391cae0bc4a2e96dd7f728a0c3ae9e1469251ada67f9e1b975' ) .set('kbn-xsrf', 'xxx') .set('authorization', `ApiKey iNvAlId`) diff --git a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json index 3433070c080096..bd1010240f86c3 100644 --- a/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json +++ b/x-pack/test/functional/es_archives/endpoint/artifacts/api_feature/data.json @@ -1,19 +1,19 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "id": "endpoint:user-artifact:v2:endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "index": ".kibana", "source": { "references": [ ], "endpoint:user-artifact:v2": { - "body": "eyJlbnRyaWVzIjpbeyJ0eXBlIjoic2ltcGxlIiwiZW50cmllcyI6W3siZmllbGQiOiJhY3RpbmdQcm9jZXNzLmZpbGUuc2lnbmVyIiwib3BlcmF0b3IiOiJpbmNsdWRlZCIsInR5cGUiOiJleGFjdF9jYXNlZCIsInZhbHVlIjoiRWxhc3RpYywgTi5WLiJ9LHsiZW50cmllcyI6W3siZmllbGQiOiJzaWduZXIiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJFdmlsIn0seyJmaWVsZCI6InRydXN0ZWQiLCJvcGVyYXRvciI6ImluY2x1ZGVkIiwidHlwZSI6ImV4YWN0X2Nhc2VkIiwidmFsdWUiOiJ0cnVlIn1dLCJmaWVsZCI6ImZpbGUuc2lnbmF0dXJlIiwidHlwZSI6Im5lc3RlZCJ9XX1dfQ==", + "body": "eJylkM8KwjAMxl9Fci59gN29iicvMqR02QjUbiSpKGPvbiw6ETwpuX1/fh9kBszKhALNcQa9TQgNCJ2nhOA+vJ4wdWaGqJSHPY8RRXxPCb3QkJEtP07IQUe2GOWYSoedqU8qXq16ikGqeAmpPNRtCqIU3WbnDx4WN38d/WvhQqmCXzDlIlojP9CsjLC0bqWtHwhaGN/1jHVkae3u+6N6Sg==", "created": 1593016187465, - "compressionAlgorithm": "none", + "compressionAlgorithm": "zlib", "encryptionAlgorithm": "none", - "identifier": "endpoint-exceptionlist-linux-1.0.0", - "encodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "encodedSize": 358, + "identifier": "endpoint-exceptionlist-linux-v1", + "encodedSha256": "5caaeabcb7864d47157fc7c28d5a7398b4f6bbaaa565d789c02ee809253b7613", + "encodedSize": 160, "decodedSha256": "d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", "decodedSize": 358 }, @@ -26,7 +26,7 @@ { "type": "doc", "value": { - "id": "endpoint:user-artifact-manifest:v2:endpoint-manifest-1.0.0", + "id": "endpoint:user-artifact-manifest:v2:endpoint-manifest-v1", "index": ".kibana", "source": { "references": [ @@ -34,9 +34,9 @@ "endpoint:user-artifact-manifest:v2": { "created": 1593183699663, "ids": [ - "endpoint-exceptionlist-linux-1.0.0-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", - "endpoint-exceptionlist-macos-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", - "endpoint-exceptionlist-windows-1.0.0-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" + "endpoint-exceptionlist-linux-v1-d2a9c760005b08d43394e59a8701ae75c80881934ccf15a006944452b80f7f9f", + "endpoint-exceptionlist-macos-v1-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658", + "endpoint-exceptionlist-windows-v1-d801aa1fb7ddcc330a5e3173372ea6af4a3d08ec58074478e85aa5603e926658" ] }, "type": "endpoint:user-artifact-manifest:v2",