Skip to content

Commit f5b77e1

Browse files
madireyAlex Kahanelasticmachine
authored
[Security Solution][Endpoint] Fix base64 download bug and adopt new user artifact/manifest format (#70998)
* Fix base64 download bug * Add test for artifact download * Add more tests to ensure cached versions of artifacts are correct * Convert to new format * missed some refs * partial fix to wrapper format * update fixtures and integration test * Fixing unit tests Co-authored-by: Alex Kahan <alexander.kahan@elastic.co> Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
1 parent 8facae7 commit f5b77e1

File tree

19 files changed

+349
-145
lines changed

19 files changed

+349
-145
lines changed

‎x-pack/plugins/security_solution/common/endpoint/schema/manifest.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,10 @@ import {
1919
export const manifestEntrySchema = t.exact(
2020
t.type({
2121
relative_url: relativeUrl,
22-
precompress_sha256: sha256,
23-
precompress_size: size,
24-
postcompress_sha256: sha256,
25-
postcompress_size: size,
22+
decoded_sha256: sha256,
23+
decoded_size: size,
24+
encoded_sha256: sha256,
25+
encoded_size: size,
2626
compression_algorithm: compressionAlgorithm,
2727
encryption_algorithm: encryptionAlgorithm,
2828
})

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/common.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,12 @@
66

77
export const ArtifactConstants = {
88
GLOBAL_ALLOWLIST_NAME: 'endpoint-exceptionlist',
9-
SAVED_OBJECT_TYPE: 'endpoint:user-artifact',
9+
SAVED_OBJECT_TYPE: 'endpoint:user-artifact:v2',
1010
SUPPORTED_OPERATING_SYSTEMS: ['linux', 'macos', 'windows'],
1111
SCHEMA_VERSION: '1.0.0',
1212
};
1313

1414
export const ManifestConstants = {
15-
SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest',
15+
SAVED_OBJECT_TYPE: 'endpoint:user-artifact-manifest:v2',
1616
SCHEMA_VERSION: '1.0.0',
1717
};

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.test.ts‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@ describe('buildEventTypeSignal', () => {
2121

2222
test('it should convert the exception lists response to the proper endpoint format', async () => {
2323
const expectedEndpointExceptions = {
24-
exceptions_list: [
24+
type: 'simple',
25+
entries: [
2526
{
2627
entries: [
2728
{
@@ -46,7 +47,9 @@ describe('buildEventTypeSignal', () => {
4647
const first = getFoundExceptionListItemSchemaMock();
4748
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
4849
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
49-
expect(resp).toEqual(expectedEndpointExceptions);
50+
expect(resp).toEqual({
51+
entries: [expectedEndpointExceptions],
52+
});
5053
});
5154

5255
test('it should convert simple fields', async () => {
@@ -57,7 +60,8 @@ describe('buildEventTypeSignal', () => {
5760
];
5861

5962
const expectedEndpointExceptions = {
60-
exceptions_list: [
63+
type: 'simple',
64+
entries: [
6165
{
6266
field: 'server.domain',
6367
operator: 'included',
@@ -84,7 +88,9 @@ describe('buildEventTypeSignal', () => {
8488
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
8589

8690
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
87-
expect(resp).toEqual(expectedEndpointExceptions);
91+
expect(resp).toEqual({
92+
entries: [expectedEndpointExceptions],
93+
});
8894
});
8995

9096
test('it should convert fields case sensitive', async () => {
@@ -100,7 +106,8 @@ describe('buildEventTypeSignal', () => {
100106
];
101107

102108
const expectedEndpointExceptions = {
103-
exceptions_list: [
109+
type: 'simple',
110+
entries: [
104111
{
105112
field: 'server.domain',
106113
operator: 'included',
@@ -127,7 +134,9 @@ describe('buildEventTypeSignal', () => {
127134
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
128135

129136
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
130-
expect(resp).toEqual(expectedEndpointExceptions);
137+
expect(resp).toEqual({
138+
entries: [expectedEndpointExceptions],
139+
});
131140
});
132141

133142
test('it should ignore unsupported entries', async () => {
@@ -147,7 +156,8 @@ describe('buildEventTypeSignal', () => {
147156
];
148157

149158
const expectedEndpointExceptions = {
150-
exceptions_list: [
159+
type: 'simple',
160+
entries: [
151161
{
152162
field: 'server.domain',
153163
operator: 'included',
@@ -162,7 +172,9 @@ describe('buildEventTypeSignal', () => {
162172
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(first);
163173

164174
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
165-
expect(resp).toEqual(expectedEndpointExceptions);
175+
expect(resp).toEqual({
176+
entries: [expectedEndpointExceptions],
177+
});
166178
});
167179

168180
test('it should convert the exception lists response to the proper endpoint format while paging', async () => {
@@ -182,7 +194,7 @@ describe('buildEventTypeSignal', () => {
182194
.mockReturnValueOnce(second)
183195
.mockReturnValueOnce(third);
184196
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
185-
expect(resp.exceptions_list.length).toEqual(6);
197+
expect(resp.entries.length).toEqual(3);
186198
});
187199

188200
test('it should handle no exceptions', async () => {
@@ -191,6 +203,6 @@ describe('buildEventTypeSignal', () => {
191203
exceptionsResponse.total = 0;
192204
mockExceptionClient.findExceptionListItem = jest.fn().mockReturnValueOnce(exceptionsResponse);
193205
const resp = await getFullEndpointExceptionList(mockExceptionClient, 'linux', '1.0.0');
194-
expect(resp.exceptions_list.length).toEqual(0);
206+
expect(resp.entries.length).toEqual(0);
195207
});
196208
});

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/lists.ts‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
*/
66

77
import { createHash } from 'crypto';
8+
import { ExceptionListItemSchema } from '../../../../../lists/common/schemas';
89
import { validate } from '../../../../common/validate';
910

1011
import { Entry, EntryNested } from '../../../../../lists/common/schemas/types/entries';
@@ -14,13 +15,14 @@ import {
1415
InternalArtifactSchema,
1516
TranslatedEntry,
1617
WrappedTranslatedExceptionList,
17-
wrappedExceptionList,
18+
wrappedTranslatedExceptionList,
1819
TranslatedEntryNestedEntry,
1920
translatedEntryNestedEntry,
2021
translatedEntry as translatedEntryType,
2122
TranslatedEntryMatcher,
2223
translatedEntryMatchMatcher,
2324
translatedEntryMatchAnyMatcher,
25+
TranslatedExceptionListItem,
2426
} from '../../schemas';
2527
import { ArtifactConstants } from './common';
2628

@@ -36,10 +38,10 @@ export async function buildArtifact(
3638
identifier: `${ArtifactConstants.GLOBAL_ALLOWLIST_NAME}-${os}-${schemaVersion}`,
3739
compressionAlgorithm: 'none',
3840
encryptionAlgorithm: 'none',
39-
decompressedSha256: sha256,
40-
compressedSha256: sha256,
41-
decompressedSize: exceptionsBuffer.byteLength,
42-
compressedSize: exceptionsBuffer.byteLength,
41+
decodedSha256: sha256,
42+
encodedSha256: sha256,
43+
decodedSize: exceptionsBuffer.byteLength,
44+
encodedSize: exceptionsBuffer.byteLength,
4345
created: Date.now(),
4446
body: exceptionsBuffer.toString('base64'),
4547
};
@@ -50,7 +52,7 @@ export async function getFullEndpointExceptionList(
5052
os: string,
5153
schemaVersion: string
5254
): Promise<WrappedTranslatedExceptionList> {
53-
const exceptions: WrappedTranslatedExceptionList = { exceptions_list: [] };
55+
const exceptions: WrappedTranslatedExceptionList = { entries: [] };
5456
let numResponses = 0;
5557
let page = 1;
5658

@@ -68,7 +70,7 @@ export async function getFullEndpointExceptionList(
6870
if (response?.data !== undefined) {
6971
numResponses = response.data.length;
7072

71-
exceptions.exceptions_list = exceptions.exceptions_list.concat(
73+
exceptions.entries = exceptions.entries.concat(
7274
translateToEndpointExceptions(response, schemaVersion)
7375
);
7476

@@ -78,7 +80,7 @@ export async function getFullEndpointExceptionList(
7880
}
7981
} while (numResponses > 0);
8082

81-
const [validated, errors] = validate(exceptions, wrappedExceptionList);
83+
const [validated, errors] = validate(exceptions, wrappedTranslatedExceptionList);
8284
if (errors != null) {
8385
throw new Error(errors);
8486
}
@@ -92,19 +94,11 @@ export async function getFullEndpointExceptionList(
9294
export function translateToEndpointExceptions(
9395
exc: FoundExceptionListItemSchema,
9496
schemaVersion: string
95-
): TranslatedEntry[] {
97+
): TranslatedExceptionListItem[] {
9698
if (schemaVersion === '1.0.0') {
97-
return exc.data
98-
.flatMap((list) => {
99-
return list.entries;
100-
})
101-
.reduce((entries: TranslatedEntry[], entry) => {
102-
const translatedEntry = translateEntry(schemaVersion, entry);
103-
if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
104-
entries.push(translatedEntry);
105-
}
106-
return entries;
107-
}, []);
99+
return exc.data.map((item) => {
100+
return translateItem(schemaVersion, item);
101+
});
108102
} else {
109103
throw new Error('unsupported schemaVersion');
110104
}
@@ -124,6 +118,22 @@ function normalizeFieldName(field: string): string {
124118
return field.endsWith('.text') ? field.substring(0, field.length - 5) : field;
125119
}
126120

121+
function translateItem(
122+
schemaVersion: string,
123+
item: ExceptionListItemSchema
124+
): TranslatedExceptionListItem {
125+
return {
126+
type: item.type,
127+
entries: item.entries.reduce((translatedEntries: TranslatedEntry[], entry) => {
128+
const translatedEntry = translateEntry(schemaVersion, entry);
129+
if (translatedEntry !== undefined && translatedEntryType.is(translatedEntry)) {
130+
translatedEntries.push(translatedEntry);
131+
}
132+
return translatedEntries;
133+
}, []),
134+
};
135+
}
136+
127137
function translateEntry(
128138
schemaVersion: string,
129139
entry: Entry | EntryNested

‎x-pack/plugins/security_solution/server/endpoint/lib/artifacts/manifest.test.ts‎

Lines changed: 28 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -57,32 +57,32 @@ describe('manifest', () => {
5757
'endpoint-exceptionlist-linux-1.0.0': {
5858
compression_algorithm: 'none',
5959
encryption_algorithm: 'none',
60-
precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
61-
postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
62-
precompress_size: 268,
63-
postcompress_size: 268,
60+
decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
61+
encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
62+
decoded_size: 430,
63+
encoded_size: 430,
6464
relative_url:
65-
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
65+
'/api/endpoint/artifacts/download/endpoint-exceptionlist-linux-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
6666
},
6767
'endpoint-exceptionlist-macos-1.0.0': {
6868
compression_algorithm: 'none',
6969
encryption_algorithm: 'none',
70-
precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
71-
postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
72-
precompress_size: 268,
73-
postcompress_size: 268,
70+
decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
71+
encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
72+
decoded_size: 430,
73+
encoded_size: 430,
7474
relative_url:
75-
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
75+
'/api/endpoint/artifacts/download/endpoint-exceptionlist-macos-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
7676
},
7777
'endpoint-exceptionlist-windows-1.0.0': {
7878
compression_algorithm: 'none',
7979
encryption_algorithm: 'none',
80-
precompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
81-
postcompress_sha256: '70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
82-
precompress_size: 268,
83-
postcompress_size: 268,
80+
decoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
81+
encoded_sha256: '5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
82+
decoded_size: 430,
83+
encoded_size: 430,
8484
relative_url:
85-
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
85+
'/api/endpoint/artifacts/download/endpoint-exceptionlist-windows-1.0.0/5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
8686
},
8787
},
8888
manifest_version: 'abcd',
@@ -94,9 +94,9 @@ describe('manifest', () => {
9494
expect(manifest1.toSavedObject()).toStrictEqual({
9595
created: now.getTime(),
9696
ids: [
97-
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
98-
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
99-
'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
97+
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
98+
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
99+
'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
100100
],
101101
});
102102
});
@@ -106,36 +106,36 @@ describe('manifest', () => {
106106
expect(diffs).toEqual([
107107
{
108108
id:
109-
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
109+
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
110110
type: 'delete',
111111
},
112112
{
113113
id:
114-
'endpoint-exceptionlist-linux-1.0.0-69328f83418f4957470640ed6cc605be6abb5fe80e0e388fd74f9764ad7ed5d1',
114+
'endpoint-exceptionlist-linux-1.0.0-3d3546e94f70493021ee845be32c66e36ea7a720c64b4d608d8029fe949f7e51',
115115
type: 'add',
116116
},
117117
]);
118118
});
119119

120120
test('Manifest returns data for given artifact', async () => {
121121
const artifact = artifacts[0];
122-
const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.compressedSha256}`);
122+
const returned = manifest1.getArtifact(`${artifact.identifier}-${artifact.encodedSha256}`);
123123
expect(returned).toEqual(artifact);
124124
});
125125

126126
test('Manifest returns entries map', async () => {
127127
const entries = manifest1.getEntries();
128128
const keys = Object.keys(entries);
129129
expect(keys).toEqual([
130-
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
131-
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
132-
'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c',
130+
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
131+
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
132+
'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735',
133133
]);
134134
});
135135

136136
test('Manifest returns true if contains artifact', async () => {
137137
const found = manifest1.contains(
138-
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
138+
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
139139
);
140140
expect(found).toEqual(true);
141141
});
@@ -144,17 +144,17 @@ describe('manifest', () => {
144144
const manifest = Manifest.fromArtifacts(artifacts, '1.0.0', 'v0');
145145
expect(
146146
manifest.contains(
147-
'endpoint-exceptionlist-linux-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
147+
'endpoint-exceptionlist-linux-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
148148
)
149149
).toEqual(true);
150150
expect(
151151
manifest.contains(
152-
'endpoint-exceptionlist-macos-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
152+
'endpoint-exceptionlist-macos-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
153153
)
154154
).toEqual(true);
155155
expect(
156156
manifest.contains(
157-
'endpoint-exceptionlist-windows-1.0.0-70d2e0ee5db0073b242df9af32e64447b932b73c3e66de3a922c61a4077b1a9c'
157+
'endpoint-exceptionlist-windows-1.0.0-5f16e5e338c53e77cfa945c17c11b175c3967bf109aa87131de41fb93b149735'
158158
)
159159
).toEqual(true);
160160
});

0 commit comments

Comments
 (0)