Review use of non-ecs-schema

[brokensound77]

The exceptions defined in the non-ecs-schema has grown significantly as of late, mostly to accommodate winlogbeat-specific fields.

Old Version

{
  "endgame-*": {
    "endgame": {
      "metadata": {
        "type": "keyword"
      },
      "event_subtype_full": "keyword"
    }
  },
  "winlogbeat-*": {
    "winlog": {
      "event_data": {
        "AccessList": "keyword",
        "AllowedToDelegateTo": "keyword",
        "AttributeLDAPDisplayName": "keyword",
        "AttributeValue": "keyword",
        "CallerProcessName": "keyword", 
        "CallTrace": "keyword",
        "ClientProcessId": "keyword",
        "GrantedAccess": "keyword",
        "NewTargetUserName": "keyword",
        "ObjectDN": "keyword",
        "OldTargetUserName": "keyword",
        "OriginalFileName": "keyword",
        "ParentProcessId": "keyword",
        "RelativeTargetName": "keyword",
        "ShareName": "keyword",
        "SubjectLogonId": "keyword",
        "TargetImage": "keyword",
        "TargetLogonId": "keyword",
        "TargetProcessGUID": "keyword",
        "TargetSid": "keyword"
      }
    },
    "winlog.logon.type": "keyword",
    "powershell.file.script_block_text": "text"
  },
  "filebeat-*": {
    "o365.audit.NewValue": "keyword",
    "o365audit.Parameters.ForwardTo": "keyword",
    "o365audit.Parameters.ForwardAsAttachmentTo": "keyword",
    "o365audit.Parameters.RedirectTo": "keyword"
  },
  "logs-endpoint.events.*": {
    "process.Ext.token.integrity_level_name": "keyword",
    "process.parent.Ext.real.pid": "long"
  },
  "logs-windows.*": {
    "powershell.file.script_block_text": "text"
  }
}

Jan 30th non-ecs-schema

{
  "endgame-*": {
    "endgame": {
      "metadata": {
        "type": "keyword"
      },
      "event_subtype_full": "keyword"
    }
  },
  "winlogbeat-*": {
    "winlog": {
      "event_data": {
        "AccessList": "keyword",
        "AccessMask": "keyword",
        "AccessMaskDescription": "keyword",
        "AllowedToDelegateTo": "keyword",
        "AttributeLDAPDisplayName": "keyword",
        "AttributeValue": "keyword",
        "CallerProcessName": "keyword",
        "CallTrace": "keyword",
        "ClientProcessId": "keyword",
        "GrantedAccess": "keyword",
        "NewTargetUserName": "keyword",
        "ObjectClass": "keyword",
        "ObjectDN": "keyword",
        "ObjectName": "keyword",
        "OldTargetUserName": "keyword",
        "OriginalFileName": "keyword",
        "ParentProcessId": "keyword",
        "ProcessName": "keyword",
        "Properties": "keyword",
        "RelativeTargetName": "keyword",
        "ShareName": "keyword",
        "SubjectLogonId": "keyword",
        "SubjectUserName": "keyword", 
        "SubjectUserSid": "keyword",
        "TargetUserName": "keyword",
        "TargetImage": "keyword",
        "TargetLogonId": "keyword",
        "TargetProcessGUID": "keyword",
        "TargetSid": "keyword",
      	"SchemaFriendlyName": "keyword",
        "Resource": "keyword",
        "PrivilegeList": "keyword",
        "AuthenticationPackageName" : "keyword",
        "TargetUserSid" : "keyword",
        "LogonProcessName": "keyword",
        "DnsHostName" : "keyword", 
        "ServiceFileName": "keyword", 
        "ImagePath": "keyword", 
        "TaskName": "keyword", 
        "Status": "keyword",
        "EnabledPrivilegeList": "keyword", 
        "OperationType": "keyword"
      }
    },
    "winlog.logon.type": "keyword", 
    "winlog.logon.id": "keyword",
    "powershell.file.script_block_text": "text"
  },
  "filebeat-*": {
    "o365.audit.NewValue": "keyword"
  },
  "logs-endpoint.events.*": {
    "process.Ext.token.integrity_level_name": "keyword",
    "process.parent.Ext.real.pid": "long", 
    "process.Ext.effective_parent.executable": "keyword", 
    "process.Ext.effective_parent.name": "keyword",
    "file.Ext.header_bytes": "keyword", 
    "file.Ext.entropy": "long",
    "file.size": "long",
    "file.Ext.original.name": "keyword",
    "dll.Ext.relative_file_creation_time": "double", 
    "dll.Ext.relative_file_name_modify_time": "double",
    "process.Ext.relative_file_name_modify_time": "double",
    "process.Ext.relative_file_creation_time": "double"
  },
  "logs-windows.*": {
    "powershell.file.script_block_text": "text"
  },
  "logs-kubernetes.*": {
    "kubernetes.audit.objectRef.resource": "keyword",
    "kubernetes.audit.objectRef.subresource": "keyword",
    "kubernetes.audit.verb": "keyword",
    "kubernetes.audit.user.username": "keyword",
    "kubernetes.audit.impersonatedUser.username": "keyword",
    "kubernetes.audit.annotations.authorization_k8s_io/decision": "keyword",
    "kubernetes.audit.annotations.authorization_k8s_io/reason": "keyword",
    "kubernetes.audit.user.groups": "text",
    "kubernetes.audit.requestObject.spec.containers.securityContext.privileged": "boolean",
    "kubernetes.audit.requestObject.spec.containers.securityContext.allowPrivilegeEscalation": "boolean",
    "kubernetes.audit.requestObject.spec.securityContext.runAsUser": "long",
    "kubernetes.audit.requestObject.spec.containers.securityContext.runAsUser": "long",
    "kubernetes.audit.requestObject.spec.hostPID": "boolean",
    "kubernetes.audit.requestObject.spec.hostNetwork": "boolean",
    "kubernetes.audit.requestObject.spec.hostIPC": "boolean",
    "kubernetes.audit.requestObject.spec.volumes.hostPath.path": "keyword",
    "kubernetes.audit.requestObject.spec.type": "keyword",
    "kubernetes.audit.requestObject.rules.resources": "keyword",
    "kubernetes.audit.requestObject.rules.verb": "keyword",
    "kubernetes.audit.objectRef.namespace": "keyword",
    "kubernetes.audit.objectRef.serviceAccountName": "keyword",
    "kubernetes.audit.requestObject.spec.serviceAccountName": "keyword",
    "kubernetes.audit.responseStatus.reason": "keyword",
    "kubernetes.audit.requestObject.spec.containers.securityContext.capabilities.add": "keyword", 
    "kubernetes.audit.requestObject.spec.containers.image": "text"
  },
  ".alerts-security.*": {
    "signal.rule.name": "keyword",
    "kibana.alert.rule.threat.tactic.id": "keyword"
  }
}

We need to review this as well as the rules using it for:

• filebeat fields: rules can define the module/dataset and not need to define them here
• integration fields: we do not parse integration specific schemas at the moment, so these may be able to defer to filebeat fields with dataset. "powershell.file.script_block_text": "text" may be definable in winlogbeat.
• since there are so many specific to winlog.event_data, we should look into auto parsing it based on the existence of the winlogbeat-* index pattern, similar to how modules and datasets are parsed for filebeat rules

[botelastic]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[Mikaayenson]

Update Jan 26 2023

A lot of this will go away after 2470 is merged. We can shrink this file once it is. In the interim, there are a few rules mentioned below that may have to be tuned due to potentially bad fields.

1. The field o365.audit.NewValue is most likely a bad field. It's in our non-ecs-schema.json file and used in this rule https://github.com/elastic/detection-rules/blob/main/rules/integrations/o365/persistence_microsoft_365_teams_custom_app_interaction_allowed.toml#L50. Based on the o365 integration docs, this field doesn't exist. Either way, this rule will get picked up by our integrations validation so the field can be removed from the non-ecs-schema.
2. Once [FR] Add Integration Schema Query Validation #2470 is merged, we can remove all integrations.
3. I think the winlog ones will get picked up by the windows integrations. However similar to o365, one of the fields, winlog.event_data.TargetProcessGUID is used in this rule https://github.com/elastic/detection-rules/blob/main/rules/windows/defense_evasion_suspicious_process_creation_calltrace.toml#L75, and the field is in our non-ecs-schema file, but it doesn't exist in winlog beats fields per the docs, nor the windows integration.

Internal discussion.

We need to find a way to:

• not maintain a custom non-ecs-schema (when possible)
  • maybe the integrations validation PR covers most of this
• remove the entries from the non-ecs-schema when no longer used
  • unit test?
• guarantee that a field exists for flattened fields. (to prevent typos slipping into rules)
  • process for RAD

[Mikaayenson]

1. The field o365.audit.NewValue is most likely a bad field. It's in our non-ecs-schema.json file and used in this rule https://github.com/elastic/detection-rules/blob/main/rules/integrations/o365/persistence_microsoft_365_teams_custom_app_interaction_allowed.toml#L50. Based on the o365 integration docs, this field doesn't exist. Either way, this rule will get picked up by our integrations validation so the field can be removed from the non-ecs-schema.

PR to resolve 1: elastic/integrations#5115

[Mikaayenson]

3. I think the winlog ones will get picked up by the windows integrations. However similar to o365, one of the fields, winlog.event_data.TargetProcessGUID is used in this rule https://github.com/elastic/detection-rules/blob/main/rules/windows/defense_evasion_suspicious_process_creation_calltrace.toml#L75, and the field is in our non-ecs-schema file, but it doesn't exist in winlog beats fields per the docs, nor the windows integration.

PR to resolve 3: elastic/integrations#5120