[Security Solution][Exceptions] Add lowercase normalizer for case-insensitivity + deprecate _tags field (new OS field) (#77379)

* Finish adding .lower to exceptionable fields

* Add back migrations

* .lower -> .caseless

* Add separate field for os type

* updates

* Type updates

* Switch over to osTypes

* get rid of _tags

* Add tests for schema validation

* Remove remaining references to _tags

* Another round of test fixes

* DefaultArray tests

* More test fixes

* Fix remaining test failures

* types / tests

* more test updates

* lowercase os values

* Address feedback + fix test failure

* tests

* Fix integration test

* process.executable.path -> process.executable.caseless

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>

Author: madirey

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.test.ts

@@ -212,6 +212,37 @@ test('tests processing keyword field with multi fields with analyzed text field'
   expect(mappings).toEqual(keywordWithAnalyzedMultiFieldsMapping);
 });
 
+test('tests processing keyword field with multi fields with normalized keyword field', () => {
+  const keywordWithNormalizedMultiFieldsLiteralYml = `
+  - name: keywordWithNormalizedMultiField
+    type: keyword
+    multi_fields:
+      - name: normalized
+        type: keyword
+        normalizer: lowercase
+  `;
+
+  const keywordWithNormalizedMultiFieldsMapping = {
+    properties: {
+      keywordWithNormalizedMultiField: {
+        ignore_above: 1024,
+        type: 'keyword',
+        fields: {
+          normalized: {
+            type: 'keyword',
+            ignore_above: 1024,
+            normalizer: 'lowercase',
+          },
+        },
+      },
+    },
+  };
+  const fields: Field[] = safeLoad(keywordWithNormalizedMultiFieldsLiteralYml);
+  const processedFields = processFields(fields);
+  const mappings = generateMappings(processedFields);
+  expect(mappings).toEqual(keywordWithNormalizedMultiFieldsMapping);
+});
+
 test('tests processing object field with no other attributes', () => {
   const objectFieldLiteralYml = `
 - name: objectField

x-pack/plugins/ingest_manager/server/services/epm/elasticsearch/template/template.ts

@@ -189,6 +189,9 @@ function generateKeywordMapping(field: Field): IndexTemplateMapping {
   if (field.ignore_above) {
     mapping.ignore_above = field.ignore_above;
   }
+  if (field.normalizer) {
+    mapping.normalizer = field.normalizer;
+  }
   return mapping;
 }
 

x-pack/plugins/ingest_manager/server/services/epm/fields/field.ts

@@ -20,6 +20,7 @@ export interface Field {
   index?: boolean;
   required?: boolean;
   multi_fields?: Fields;
+  normalizer?: string;
   doc_values?: boolean;
   copy_to?: string;
   analyzer?: string;

x-pack/plugins/lists/README.md

@@ -113,12 +113,6 @@ You should see the new exception list created like so:
 
 ```sh
 {
-  "_tags": [
-    "endpoint",
-    "process",
-    "malware",
-    "os:linux"
-  ],
   "created_at": "2020-05-28T19:16:31.052Z",
   "created_by": "yo",
   "description": "This is a sample endpoint type exception",
@@ -141,12 +135,6 @@ And you can attach exception list items like so:
 
 ```ts
 {
-  "_tags": [
-    "endpoint",
-    "process",
-    "malware",
-    "os:linux"
-  ],
   "comments": [],
   "created_at": "2020-05-28T19:17:21.099Z",
   "created_by": "yo",
@@ -173,6 +161,7 @@ And you can attach exception list items like so:
   "list_id": "endpoint_list",
   "name": "Sample Endpoint Exception List",
   "namespace_type": "single",
+  "os_types": ["linux"],
   "tags": [
     "user added string for a tag",
     "malware"
@@ -222,19 +211,14 @@ or for finding exception lists:
 {
   "data": [
     {
-      "_tags": [
-        "endpoint",
-        "process",
-        "malware",
-        "os:linux"
-      ],
       "created_at": "2020-05-28T19:16:31.052Z",
       "created_by": "yo",
       "description": "This is a sample endpoint type exception",
       "id": "bcb94680-a117-11ea-ad9d-c71f4820e65b",
       "list_id": "endpoint_list",
       "name": "Sample Endpoint Exception List",
       "namespace_type": "single",
+      "os_types": ["linux"],
       "tags": [
         "user added string for a tag",
         "malware"

x-pack/plugins/lists/common/constants.mock.ts

@@ -5,6 +5,7 @@
  */
 import moment from 'moment';
 
+import { OsTypeArray } from './schemas/common';
 import { EntriesArray } from './schemas/types';
 import { EndpointEntriesArray } from './schemas/types/endpoint';
 export const DATE_NOW = '2020-04-20T15:25:31.830Z';
@@ -68,7 +69,7 @@ export const ENDPOINT_ENTRIES: EndpointEntriesArray = [
   { field: 'some.not.nested.field', operator: 'included', type: 'match', value: 'some value' },
 ];
 export const ITEM_TYPE = 'simple';
-export const _TAGS = [];
+export const OS_TYPES: OsTypeArray = ['windows'];
 export const TAGS = [];
 export const COMMENTS = [];
 export const FILTER = 'name:Nicolas Bourbaki';

x-pack/plugins/lists/common/schemas/common/schemas.test.ts

@@ -27,6 +27,8 @@ import {
   esDataTypeUnion,
   exceptionListType,
   operator,
+  osType,
+  osTypeArrayOrUndefined,
   type,
 } from './schemas';
 
@@ -379,4 +381,35 @@ describe('Common schemas', () => {
       expect(message.schema).toEqual({});
     });
   });
+
+  describe('osType', () => {
+    test('it will validate a correct osType', () => {
+      const payload = 'windows';
+      const decoded = osType.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual(payload);
+    });
+
+    test('it will fail to validate an incorrect osType', () => {
+      const payload = 'foo';
+      const decoded = osType.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([
+        'Invalid value "foo" supplied to ""linux" | "macos" | "windows""',
+      ]);
+      expect(message.schema).toEqual({});
+    });
+
+    test('it will default to an empty array when osTypeArrayOrUndefined is used', () => {
+      const payload = undefined;
+      const decoded = osTypeArrayOrUndefined.decode(payload);
+      const checked = exactCheck(payload, decoded);
+      const message = pipe(checked, foldLeftRight);
+      expect(getPaths(left(message.errors))).toEqual([]);
+      expect(message.schema).toEqual([]);
+    });
+  });
 });

x-pack/plugins/lists/common/schemas/common/schemas.ts

@@ -9,7 +9,7 @@
 import * as t from 'io-ts';
 
 import { DefaultNamespace } from '../types/default_namespace';
-import { DefaultStringArray, NonEmptyString } from '../../shared_imports';
+import { DefaultArray, DefaultStringArray, NonEmptyString } from '../../shared_imports';
 
 export const name = t.string;
 export type Name = t.TypeOf<typeof name>;
@@ -211,11 +211,6 @@ export type Tags = t.TypeOf<typeof tags>;
 export const tagsOrUndefined = t.union([tags, t.undefined]);
 export type TagsOrUndefined = t.TypeOf<typeof tagsOrUndefined>;
 
-export const _tags = DefaultStringArray;
-export type _Tags = t.TypeOf<typeof _tags>;
-export const _tagsOrUndefined = t.union([_tags, t.undefined]);
-export type _TagsOrUndefined = t.TypeOf<typeof _tagsOrUndefined>;
-
 export const exceptionListType = t.keyof({ detection: null, endpoint: null });
 export const exceptionListTypeOrUndefined = t.union([exceptionListType, t.undefined]);
 export type ExceptionListType = t.TypeOf<typeof exceptionListType>;
@@ -317,3 +312,16 @@ export type Immutable = t.TypeOf<typeof immutable>;
 
 export const immutableOrUndefined = t.union([immutable, t.undefined]);
 export type ImmutableOrUndefined = t.TypeOf<typeof immutableOrUndefined>;
+
+export const osType = t.keyof({
+  linux: null,
+  macos: null,
+  windows: null,
+});
+export type OsType = t.TypeOf<typeof osType>;
+
+export const osTypeArray = DefaultArray(osType);
+export type OsTypeArray = t.TypeOf<typeof osTypeArray>;
+
+export const osTypeArrayOrUndefined = t.union([osTypeArray, t.undefined]);
+export type OsTypeArrayOrUndefined = t.OutputOf<typeof osTypeArray>;

x-pack/plugins/lists/common/schemas/request/create_endpoint_list_item_schema.mock.ts

@@ -11,20 +11,20 @@ import {
   ITEM_TYPE,
   META,
   NAME,
+  OS_TYPES,
   TAGS,
-  _TAGS,
 } from '../../constants.mock';
 
 import { CreateEndpointListItemSchema } from './create_endpoint_list_item_schema';
 
 export const getCreateEndpointListItemSchemaMock = (): CreateEndpointListItemSchema => ({
-  _tags: _TAGS,
   comments: COMMENTS,
   description: DESCRIPTION,
   entries: ENDPOINT_ENTRIES,
   item_id: undefined,
   meta: META,
   name: NAME,
+  os_types: OS_TYPES,
   tags: TAGS,
   type: ITEM_TYPE,
 });

x-pack/plugins/lists/common/schemas/request/create_endpoint_list_item_schema.test.ts

@@ -174,19 +174,6 @@ describe('create_endpoint_list_item_schema', () => {
     expect(message.schema).toEqual(outputPayload);
   });
 
-  test('it should pass validation when supplied an undefined for "_tags" but return an array and generate a correct body not counting the auto generated uuid', () => {
-    const inputPayload = getCreateEndpointListItemSchemaMock();
-    const outputPayload = getCreateEndpointListItemSchemaMock();
-    delete inputPayload._tags;
-    outputPayload._tags = [];
-    const decoded = createEndpointListItemSchema.decode(inputPayload);
-    const checked = exactCheck(inputPayload, decoded);
-    const message = pipe(checked, foldLeftRight);
-    delete (message.schema as CreateEndpointListItemSchema).item_id;
-    expect(getPaths(left(message.errors))).toEqual([]);
-    expect(message.schema).toEqual(outputPayload);
-  });
-
   test('it should pass validation when supplied an undefined for "item_id" and auto generate a uuid', () => {
     const inputPayload = getCreateEndpointListItemSchemaMock();
     delete inputPayload.item_id;

x-pack/plugins/lists/common/schemas/request/create_endpoint_list_item_schema.ts

@@ -8,13 +8,13 @@ import * as t from 'io-ts';
 
 import {
   ItemId,
+  OsTypeArray,
   Tags,
-  _Tags,
-  _tags,
   description,
   exceptionListItemType,
   meta,
   name,
+  osTypeArrayOrUndefined,
   tags,
 } from '../common/schemas';
 import { RequiredKeepUndefined } from '../../types';
@@ -34,10 +34,10 @@ export const createEndpointListItemSchema = t.intersection([
   ),
   t.exact(
     t.partial({
-      _tags, // defaults to empty array if not set during decode
       comments: DefaultCreateCommentsArray, // defaults to empty array if not set during decode
       item_id: DefaultUuid, // defaults to GUID (uuid v4) if not set during decode
       meta, // defaults to undefined if not set during decode
+      os_types: osTypeArrayOrUndefined, // defaults to empty array if not set during decode
       tags, // defaults to empty array if not set during decode
     })
   ),
@@ -48,11 +48,11 @@ export type CreateEndpointListItemSchema = t.OutputOf<typeof createEndpointListI
 // This type is used after a decode since some things are defaults after a decode.
 export type CreateEndpointListItemSchemaDecoded = Omit<
   RequiredKeepUndefined<t.TypeOf<typeof createEndpointListItemSchema>>,
-  '_tags' | 'tags' | 'item_id' | 'entries' | 'comments'
+  'tags' | 'item_id' | 'entries' | 'comments' | 'os_types'
 > & {
-  _tags: _Tags;
   comments: CreateCommentsArray;
   tags: Tags;
   item_id: ItemId;
   entries: EntriesArray;
+  os_types: OsTypeArray;
 };