[Security Solution][Endpoint] User Manifest Cleanup + Artifact Compre…

…ssion (#70759)

* Stateless exception list translation with improved runtime checks

* use flatMap and reduce to simplify logic

* Update to new manifest format

* Fix test fixture SO data type

* Fix another test fixture data type

* Fix sha256 reference in artifact_client

* Refactor to remove usages of 'then' and tidy up a bit

* Zlib compression

* prefer byteLength to length

* Make ingestManager optional for security-solution startup

* Fix download functionality

* Use eql for deep equality check

* Fix base64 download bug

* Add test for artifact download

* Add more tests to ensure cached versions of artifacts are correct

* Convert to new format

* Deflate

* missed some refs

* partial fix to wrapper format

* update fixtures and integration test

* Fixing unit tests

* small bug fixes

* artifact and manifest versioning changes

* Remove access tag from download endpoint

* Adding decompression to integration test

* Removing tag from route

* add try/catch in ingest callback handler

* Fixing

* Removing last expect from unit test for tag

* type fixes

* Add compression type to manifest

* Reverting ingestManager back to being required for now

Co-authored-by: Alex Kahan <alexander.kahan@elastic.co>
Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

x-pack/plugins/security_solution/common/endpoint/generate_data.ts

@@ -1036,8 +1036,8 @@ export class EndpointDocGenerator {
           config: {
             artifact_manifest: {
               value: {
-                manifest_version: 'v0',
-                schema_version: '1.0.0',
+                manifest_version: 'WzAsMF0=',
+                schema_version: 'v1',
                 artifacts: {},
               },
             },

x-pack/plugins/security_solution/common/endpoint/schema/common.ts

@@ -10,6 +10,7 @@ export const compressionAlgorithm = t.keyof({
   none: null,
   zlib: null,
 });
+export type CompressionAlgorithm = t.TypeOf<typeof compressionAlgorithm>;
 
 export const encryptionAlgorithm = t.keyof({
   none: null,
@@ -20,7 +21,7 @@ export const identifier = t.string;
 export const manifestVersion = t.string;
 
 export const manifestSchemaVersion = t.keyof({
-  '1.0.0': null,
+  v1: null,
 });
 export type ManifestSchemaVersion = t.TypeOf<typeof manifestSchemaVersion>;
 

x-pack/plugins/security_solution/public/management/pages/policy/store/policy_details/index.test.ts

@@ -43,8 +43,8 @@ describe('policy details: ', () => {
               config: {
                 artifact_manifest: {
                   value: {
-                    manifest_version: 'v0',
-                    schema_version: '1.0.0',
+                    manifest_version: 'WzAsMF0=',
+                    schema_version: 'v1',
                     artifacts: {},
                   },
                 },

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.test.ts

@@ -8,14 +8,14 @@ import { httpServerMock } from '../../../../../src/core/server/mocks';
 import { EndpointAppContextService } from './endpoint_app_context_services';
 
 describe('test endpoint app context services', () => {
-  it('should throw error on getAgentService if start is not called', async () => {
-    const endpointAppContextService = new EndpointAppContextService();
-    expect(() => endpointAppContextService.getAgentService()).toThrow(Error);
-  });
-  it('should return undefined on getManifestManager if start is not called', async () => {
-    const endpointAppContextService = new EndpointAppContextService();
-    expect(endpointAppContextService.getManifestManager()).toEqual(undefined);
-  });
+  // it('should return undefined on getAgentService if dependencies are not enabled', async () => {
+  //   const endpointAppContextService = new EndpointAppContextService();
+  //   expect(endpointAppContextService.getAgentService()).toEqual(undefined);
+  // });
+  // it('should return undefined on getManifestManager if dependencies are not enabled', async () => {
+  //   const endpointAppContextService = new EndpointAppContextService();
+  //   expect(endpointAppContextService.getManifestManager()).toEqual(undefined);
+  // });
   it('should throw error on getScopedSavedObjectsClient if start is not called', async () => {
     const endpointAppContextService = new EndpointAppContextService();
     expect(() =>

x-pack/plugins/security_solution/server/endpoint/endpoint_app_context_services.ts

@@ -4,20 +4,21 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 import {
-  SavedObjectsServiceStart,
   KibanaRequest,
+  Logger,
+  SavedObjectsServiceStart,
   SavedObjectsClientContract,
 } from 'src/core/server';
 import { AgentService, IngestManagerStartContract } from '../../../ingest_manager/server';
 import { getPackageConfigCreateCallback } from './ingest_integration';
 import { ManifestManager } from './services/artifacts';
 
-export type EndpointAppContextServiceStartContract = Pick<
-  IngestManagerStartContract,
-  'agentService'
+export type EndpointAppContextServiceStartContract = Partial<
+  Pick<IngestManagerStartContract, 'agentService'>
 > & {
-  manifestManager?: ManifestManager | undefined;
-  registerIngestCallback: IngestManagerStartContract['registerExternalCallback'];
+  logger: Logger;
+  manifestManager?: ManifestManager;
+  registerIngestCallback?: IngestManagerStartContract['registerExternalCallback'];
   savedObjectsStart: SavedObjectsServiceStart;
 };
 
@@ -35,20 +36,17 @@ export class EndpointAppContextService {
     this.manifestManager = dependencies.manifestManager;
     this.savedObjectsStart = dependencies.savedObjectsStart;
 
-    if (this.manifestManager !== undefined) {
+    if (this.manifestManager && dependencies.registerIngestCallback) {
       dependencies.registerIngestCallback(
         'packageConfigCreate',
-        getPackageConfigCreateCallback(this.manifestManager)
+        getPackageConfigCreateCallback(dependencies.logger, this.manifestManager)
       );
     }
   }
 
   public stop() {}
 
-  public getAgentService(): AgentService {
-    if (!this.agentService) {
-      throw new Error(`must call start on ${EndpointAppContextService.name} to call getter`);
-    }
+  public getAgentService(): AgentService | undefined {
     return this.agentService;
   }
 

x-pack/plugins/security_solution/server/endpoint/ingest_integration.ts

@@ -4,6 +4,7 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import { Logger } from '../../../../../src/core/server';
 import { NewPackageConfig } from '../../../ingest_manager/common/types/models';
 import { factory as policyConfigFactory } from '../../common/endpoint/models/policy_config';
 import { NewPolicyData } from '../../common/endpoint/types';
@@ -13,6 +14,7 @@ import { ManifestManager } from './services/artifacts';
  * Callback to handle creation of PackageConfigs in Ingest Manager
  */
 export const getPackageConfigCreateCallback = (
+  logger: Logger,
   manifestManager: ManifestManager
 ): ((newPackageConfig: NewPackageConfig) => Promise<NewPackageConfig>) => {
   const handlePackageConfigCreate = async (
@@ -27,8 +29,19 @@ export const getPackageConfigCreateCallback = (
     // follow the types/schema expected
     let updatedPackageConfig = newPackageConfig as NewPolicyData;
 
-    const wrappedManifest = await manifestManager.refresh({ initialize: true });
-    if (wrappedManifest !== null) {
+    // get snapshot based on exception-list-agnostic SOs
+    // with diffs from last dispatched manifest, if it exists
+    const snapshot = await manifestManager.getSnapshot({ initialize: true });
+
+    if (snapshot === null) {
+      logger.warn('No manifest snapshot available.');
+      return updatedPackageConfig;
+    }
+
+    if (snapshot.diffs.length > 0) {
+      // create new artifacts
+      await manifestManager.syncArtifacts(snapshot, 'add');
+
       // Until we get the Default Policy Configuration in the Endpoint package,
       // we will add it here manually at creation time.
       // @ts-ignore
@@ -42,7 +55,7 @@ export const getPackageConfigCreateCallback = (
               streams: [],
               config: {
                 artifact_manifest: {
-                  value: wrappedManifest.manifest.toEndpointFormat(),
+                  value: snapshot.manifest.toEndpointFormat(),
                 },
                 policy: {
                   value: policyConfigFactory(),
@@ -57,9 +70,18 @@ export const getPackageConfigCreateCallback = (
     try {
       return updatedPackageConfig;
     } finally {
-      // TODO: confirm creation of package config
-      // then commit.
-      await manifestManager.commit(wrappedManifest);
+      if (snapshot.diffs.length > 0) {
+        // TODO: let's revisit the way this callback happens... use promises?
+        // only commit when we know the package config was created
+        try {
+          await manifestManager.commit(snapshot.manifest);
+
+          // clean up old artifacts
+          await manifestManager.syncArtifacts(snapshot, 'delete');
+        } catch (err) {
+          logger.error(err);
+        }
+      }
     }
   };
 

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.test.ts

@@ -8,36 +8,41 @@ import { ExceptionsCache } from './cache';
 
 describe('ExceptionsCache tests', () => {
   let cache: ExceptionsCache;
+  const body = Buffer.from('body');
 
   beforeEach(() => {
     jest.clearAllMocks();
     cache = new ExceptionsCache(3);
   });
 
   test('it should cache', async () => {
-    cache.set('test', 'body');
+    cache.set('test', body);
     const cacheResp = cache.get('test');
-    expect(cacheResp).toEqual('body');
+    expect(cacheResp).toEqual(body);
   });
 
   test('it should handle cache miss', async () => {
-    cache.set('test', 'body');
+    cache.set('test', body);
     const cacheResp = cache.get('not test');
     expect(cacheResp).toEqual(undefined);
   });
 
   test('it should handle cache eviction', async () => {
-    cache.set('1', 'a');
-    cache.set('2', 'b');
-    cache.set('3', 'c');
+    const a = Buffer.from('a');
+    const b = Buffer.from('b');
+    const c = Buffer.from('c');
+    const d = Buffer.from('d');
+    cache.set('1', a);
+    cache.set('2', b);
+    cache.set('3', c);
     const cacheResp = cache.get('1');
-    expect(cacheResp).toEqual('a');
+    expect(cacheResp).toEqual(a);
 
-    cache.set('4', 'd');
+    cache.set('4', d);
     const secondResp = cache.get('1');
     expect(secondResp).toEqual(undefined);
-    expect(cache.get('2')).toEqual('b');
-    expect(cache.get('3')).toEqual('c');
-    expect(cache.get('4')).toEqual('d');
+    expect(cache.get('2')).toEqual(b);
+    expect(cache.get('3')).toEqual(c);
+    expect(cache.get('4')).toEqual(d);
   });
 });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/cache.ts

@@ -10,7 +10,7 @@ const DEFAULT_MAX_SIZE = 10;
  * FIFO cache implementation for artifact downloads.
  */
 export class ExceptionsCache {
-  private cache: Map<string, string>;
+  private cache: Map<string, Buffer>;
   private queue: string[];
   private maxSize: number;
 
@@ -20,7 +20,7 @@ export class ExceptionsCache {
     this.maxSize = maxSize || DEFAULT_MAX_SIZE;
   }
 
-  set(id: string, body: string) {
+  set(id: string, body: Buffer) {
     if (this.queue.length + 1 > this.maxSize) {
       const entry = this.queue.shift();
       if (entry !== undefined) {
@@ -31,7 +31,7 @@ export class ExceptionsCache {
     this.cache.set(id, body);
   }
 
-  get(id: string): string | undefined {
+  get(id: string): Buffer | undefined {
     return this.cache.get(id);
   }
 }

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts

@@ -8,10 +8,11 @@ export const ArtifactConstants = {
   GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
   SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2',
   SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
-  SCHEMA_VERSION: '1.0.0',
+  SCHEMA_VERSION: 'v1',
 };
 
 export const ManifestConstants = {
   SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2',
-  SCHEMA_VERSION: '1.0.0',
+  SCHEMA_VERSION: 'v1',
+  INITIAL_VERSION: 'WzAsMF0=',
 };

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts

@@ -46,7 +46,7 @@ describe('buildEventTypeSignal', () => {
 
     const first = getFoundExceptionListItemSchemaMock();
     mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
-    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
     expect(resp).toEqual({
       entries: [expectedEndpointExceptions],
     });
@@ -87,7 +87,7 @@ describe('buildEventTypeSignal', () => {
     first.data[0].entries = testEntries;
     mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
 
-    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
     expect(resp).toEqual({
       entries: [expectedEndpointExceptions],
     });
@@ -133,7 +133,7 @@ describe('buildEventTypeSignal', () => {
     first.data[0].entries = testEntries;
     mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
 
-    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
     expect(resp).toEqual({
       entries: [expectedEndpointExceptions],
     });
@@ -171,7 +171,7 @@ describe('buildEventTypeSignal', () => {
     first.data[0].entries = testEntries;
     mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
 
-    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
     expect(resp).toEqual({
       entries: [expectedEndpointExceptions],
     });
@@ -193,7 +193,7 @@ describe('buildEventTypeSignal', () => {
       .mockReturnValueOnce(first)
       .mockReturnValueOnce(second)
       .mockReturnValueOnce(third);
-    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
     expect(resp.entries.length).toEqual(3);
   });
 
@@ -202,7 +202,7 @@ describe('buildEventTypeSignal', () => {
     exceptionsResponse.data = [];
     exceptionsResponse.total = 0;
     mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse);
-    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
+    const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', 'v1');
     expect(resp.entries.length).toEqual(0);
   });
 });

x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts

@@ -5,6 +5,7 @@
  */
 
 import { createHash } from 'crypto';
+import { deflate } from 'zlib';
 import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
 import { validate } from '../../../../common/validate';
 
@@ -34,6 +35,7 @@ export async function buildArtifact(
   const exceptionsBuffer = Buffer.from(JSON.stringify(exceptions));
   const sha256 = createHash('sha256').update(exceptionsBuffer.toString()).digest('hex');
 
+  // Keep compression info empty in case its a duplicate. Lazily compress before committing if needed.
   return {
     identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
     compressionAlgorithm: 'none',
@@ -95,7 +97,7 @@ export function translateToEndpointExceptions(
   exc: FoundExceptionListItemSchema,
   schemaVersion: string
 ): TranslatedExceptionListItem[] {
-  if (schemaVersion === '1.0.0') {
+  if (schemaVersion === 'v1') {
     return exc.data.map((item) => {
       return translateItem(schemaVersion, item);
     });
@@ -180,3 +182,15 @@ function translateEntry(
     }
   }
 }
+
+export async function compressExceptionList(buffer: Buffer): Promise<Buffer> {
+  return new Promise((resolve, reject) => {
+    deflate(buffer, function (err, buf) {
+      if (err) {
+        reject(err);
+      } else {
+        resolve(buf);
+      }
+    });
+  });
+}