Do not merge-test

.script/tests/asimParsersTest/ASimFilteringTest.py

@@ -288,6 +288,14 @@ def main():
                 if parser_file['EquivalentBuiltInParser'] in read_exclusion_list_from_csv():
                     print(f"{YELLOW}The parser {parser_file_path} is listed in the exclusions file. Therefore, this workflow run will not fail because of it. To allow this parser to cause the workflow to fail, please remove its name from the exclusions list file located at: {exclusion_file_path}{RESET}")
                     sys.stdout.flush()
+                # If Failure count is due to EventResult and EventSchema is AuditEvent, then ignore the failure. 
+                # Audit Event is a special case where 'EventResult' validations could be partial like only 'Success' events.
+                elif len(result.failures) == 1:
+                    failure_message = result.failures[0][1]
+                    if "eventresult - validations for this parameter are partial" in failure_message:
+                        if parser_file['Normalization']['Schema'] == 'AuditEvent':
+                            print(f"{YELLOW} This single failure is due to partial result in 'EventResult' field in 'AuditEvent' schema. Audit Event is a special case where 'EventResult' validations could be partial like only 'Success' events. Ignoring this error. {RESET}")
+                            sys.stdout.flush()
                 else:
                     print(f"::error::Tests failed for {parser_file_path}")
                     sys.stdout.flush()  # Explicitly flush stdout
@@ -585,13 +593,19 @@ def get_substrings_list(self, rows, num_of_substrings, delimiter):
 
             value = row[COLUMN_INDEX_IN_ROW]
             post = get_postfix(value, rows, substrings_list, delimiter)
-            # Post will equal value if: value dont contain the delimiter, post is in the list, post is contained in an item in the list.
-            if post != value:
+
+            # Add post to the list if it's not already present
+            if post not in substrings_list:
                 substrings_list.append(post)
-            else:
+
+            # If the list has reached the required number of substrings, break the loop
+            if len(substrings_list) == num_of_substrings:
+                break
+
+            # If post is equal to value, also add pre to the list
+            if post == value:
                 pre = get_prefix(value, rows, substrings_list, delimiter)
-                # pre will equal value if: value dont contain the delimiter, pre is in the list, pre is contained in an item in the list.
-                if pre != value:
+                if pre not in substrings_list:
                     substrings_list.append(pre)
 
         return substrings_list

Parsers/ASimAuditEvent/ARM/ASimAuditEvent/ASimAuditEvent.json

@@ -35,7 +35,7 @@
             "displayName": "Audit event ASIM parser",
             "category": "ASIM",
             "FunctionAlias": "ASimAuditEvent",
-            "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimAuditEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser);\nlet BuiltInDisabled=toscalar('ExcludeASimAuditEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nunion isfuzzy=true\n  vimAuditEventEmpty, \n  ASimAuditEventMicrosoftExchangeAdmin365  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftExchangeAdmin365' in (DisabledParsers))),\n  ASimAuditEventMicrosoftWindowsEvents  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftWindowsEvents' in (DisabledParsers))),\n  ASimAuditEventMicrosoftSecurityEvents  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftSecurityEvents' in (DisabledParsers))),\n  ASimAuditEventMicrosoftEvent  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftEvents' in (DisabledParsers))),\n  ASimAuditEventAzureActivity  (BuiltInDisabled or ('ExcludeASimAuditEventAzureActivity' in (DisabledParsers))),\n  ASimAuditEventCiscoMeraki  (BuiltInDisabled or ('ExcludeASimAuditEventCiscoMeraki' in (DisabledParsers))),\n  ASimAuditEventCiscoMerakiSyslog  (BuiltInDisabled or ('ExcludeASimAuditEventCiscoMerakiSyslog' in (DisabledParsers))),\n  ASimAuditEventBarracudaWAF (BuiltInDisabled or ('ExcludeASimAuditEventBarracudaWAF' in (DisabledParsers))),\n  ASimAuditEventBarracudaCEF (BuiltInDisabled or ('ExcludeASimAuditEventBarracudaCEF' in (DisabledParsers))),\n  ASimAuditEventCiscoISE  (BuiltInDisabled or ('ExcludeASimAuditEventCiscoISE' in (DisabledParsers))),\n  ASimAuditEventVectraXDRAudit(BuiltInDisabled or ('ExcludeASimAuditEventVectraXDRAudit' in (DisabledParsers))),\n  ASimAuditEventSentinelOne  (BuiltInDisabled or ('ExcludeASimAuditEventSentinelOne' in (DisabledParsers))),\n  ASimAuditEventCrowdStrikeFalconHost(BuiltInDisabled or ('ExcludeASimAuditEventCrowdStrikeFalconHost' in (DisabledParsers))),\n  ASimAuditEventVMwareCarbonBlackCloud(BuiltInDisabled or ('ExcludeASimAuditEventVMwareCarbonBlackCloud' in (DisabledParsers)))\n",
+            "query": "let DisabledParsers=materialize(_GetWatchlist('ASimDisabledParsers') | where SearchKey in ('Any', 'ExcludeASimAuditEvent') | extend SourceSpecificParser=column_ifexists('SourceSpecificParser','') | distinct SourceSpecificParser);\nlet BuiltInDisabled=toscalar('ExcludeASimAuditEventBuiltIn' in (DisabledParsers) or 'Any' in (DisabledParsers)); \nunion isfuzzy=true\n  vimAuditEventEmpty, \n  ASimAuditEventMicrosoftExchangeAdmin365  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftExchangeAdmin365' in (DisabledParsers))),\n  ASimAuditEventMicrosoftWindowsEvents  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftWindowsEvents' in (DisabledParsers))),\n  ASimAuditEventMicrosoftSecurityEvents  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftSecurityEvents' in (DisabledParsers))),\n  ASimAuditEventMicrosoftEvent  (BuiltInDisabled or ('ExcludeASimAuditEventMicrosoftEvents' in (DisabledParsers))),\n  ASimAuditEventAzureActivity  (BuiltInDisabled or ('ExcludeASimAuditEventAzureActivity' in (DisabledParsers))),\n  ASimAuditEventCiscoMeraki  (BuiltInDisabled or ('ExcludeASimAuditEventCiscoMeraki' in (DisabledParsers))),\n  ASimAuditEventCiscoMerakiSyslog  (BuiltInDisabled or ('ExcludeASimAuditEventCiscoMerakiSyslog' in (DisabledParsers))),\n  ASimAuditEventBarracudaWAF (BuiltInDisabled or ('ExcludeASimAuditEventBarracudaWAF' in (DisabledParsers))),\n  ASimAuditEventBarracudaCEF (BuiltInDisabled or ('ExcludeASimAuditEventBarracudaCEF' in (DisabledParsers))),\n  ASimAuditEventCiscoISE  (BuiltInDisabled or ('ExcludeASimAuditEventCiscoISE' in (DisabledParsers))),\n  ASimAuditEventVectraXDRAudit(BuiltInDisabled or ('ExcludeASimAuditEventVectraXDRAudit' in (DisabledParsers))),\n  ASimAuditEventSentinelOne  (BuiltInDisabled or ('ExcludeASimAuditEventSentinelOne' in (DisabledParsers))),\n  ASimAuditEventCrowdStrikeFalconHost(BuiltInDisabled or ('ExcludeASimAuditEventCrowdStrikeFalconHost' in (DisabledParsers))),\n  ASimAuditEventVMwareCarbonBlackCloud(BuiltInDisabled or ('ExcludeASimAuditEventVMwareCarbonBlackCloud' in (DisabledParsers))),\n  ASimAuditEventInfobloxBloxOne(BuiltInDisabled or ('ExcludeASimAuditEventInfobloxBloxOne' in (DisabledParsers))),\n  ASimAuditEventIllumioSaaSCore(BuiltInDisabled or ('ExcludeASimAuditEventIllumioSaaSCore' in (DisabledParsers)))\n",
             "version": 1,
             "functionParameters": "pack:bool=False"
           }

Parsers/ASimAuditEvent/ARM/ASimAuditEventIllumioSaaSCore/README.md

@@ -0,0 +1,18 @@
+# Illumio Core ASIM AuditEvent Normalization Parser
+
+ARM template for ASIM AuditEvent schema parser for Illumio Core.
+
+This ASIM parser supports normalizing Illumio Core audit events logs ingested in 'Illumio_Auditable_Events_CL' table to the ASIM Audit Event schema.
+
+
+The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace.
+
+For more information, see:
+
+- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM)
+- [Deploy all of ASIM](https://aka.ms/DeployASIM)
+- [ASIM AuditEvent normalization schema reference](https://aka.ms/ASimAuditEventDoc)
+
+<br>
+
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimAuditEvent%2FARM%2FASimAuditEventIllumioSaaSCore%2FASimAuditEventIllumioSaaSCore.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimAuditEvent%2FARM%2FASimAuditEventIllumioSaaSCore%2FASimAuditEventIllumioSaaSCore.json)

Parsers/ASimAuditEvent/ARM/ASimAuditEventInfobloxBloxOne/ASimAuditEventInfobloxBloxOne.json

@@ -0,0 +1,46 @@
+{
+  "$schema": "https://schema.management.azure.com/schemas/2019-08-01/deploymentTemplate.json#",
+  "contentVersion": "1.0.0.0",
+  "parameters": {
+    "Workspace": {
+      "type": "string",
+      "metadata": {
+        "description": "The Microsoft Sentinel workspace into which the function will be deployed. Has to be in the selected Resource Group."
+      }
+    },
+    "WorkspaceRegion": {
+      "type": "string",
+      "defaultValue": "[resourceGroup().location]",
+      "metadata": {
+        "description": "The region of the selected workspace. The default value will use the Region selection above."
+      }
+    }
+  },
+  "resources": [
+    {
+      "type": "Microsoft.OperationalInsights/workspaces",
+      "apiVersion": "2017-03-15-preview",
+      "name": "[parameters('Workspace')]",
+      "location": "[parameters('WorkspaceRegion')]",
+      "resources": [
+        {
+          "type": "savedSearches",
+          "apiVersion": "2020-08-01",
+          "name": "ASimAuditEventInfobloxBloxOne",
+          "dependsOn": [
+            "[concat('Microsoft.OperationalInsights/workspaces/', parameters('Workspace'))]"
+          ],
+          "properties": {
+            "etag": "*",
+            "displayName": "AuditEvent ASIM parser for Infoblox BloxOne",
+            "category": "ASIM",
+            "FunctionAlias": "ASimAuditEventInfobloxBloxOne",
+            "query": "let EventSeverityLookup = datatable (LogSeverity:string, EventSeverity:string) [ \"0\", \"Low\", \"1\", \"Low\", \"2\", \"Low\", \"3\", \"Low\", \"4\", \"Medium\", \"5\", \"Medium\", \"6\", \"Medium\", \"7\", \"High\", \"8\", \"High\", \"9\", \"High\", \"10\", \"High\" ]; let OperationLookup = datatable (DeviceAction:string, Object:string, ObjectType:string) [ \"CreateSecurityPolicy\", \"Security Policy\", \"Policy Role\", \"UpdateSecurityPolicy\", \"Security Policy\", \"Policy\", \"Create\", \"Network Resource\", \"Service\", \"Update\", \"Network Resource\", \"Service\", \"Restore\", \"Infoblox Resource\", \"Service\", \"CreateOrGetDoHFQDN\", \"DOHFQDN\", \"Service\", \"CreateOrUpdateDfpService\", \"Dfp Service\", \"Service\", \"MoveToRecyclebin\", \"Recyclebin\", \"Other\", \"CreateCategoryFilter\", \"Category Filter\", \"Other\", \"GetLookalikeThreatCounts\", \"Lookalike Threat Counts\", \"Other\", \"GetLookalikeDomainCounts\", \"Lookalike Domain Counts\", \"Other\", \"CreateRoamingDeviceGroup\", \"Roaming Device Group\", \"Configuration Atom\", \"UpdatePartialRoamingDeviceGroup\", \"Partial Roaming Device Group\", \"Configuration Atom\" ]; let parser = (disabled:bool=false) { CommonSecurityLog | where not(disabled) and DeviceVendor == \"Infoblox\" and DeviceEventClassID has \"AUDIT\" | parse-kv AdditionalExtensions as (InfobloxHTTPReqBody:string, InfobloxHTTPRespBody:string) with (pair_delimiter=\";\", kv_delimiter=\"=\") | lookup EventSeverityLookup on LogSeverity | lookup OperationLookup on DeviceAction | invoke _ASIM_ResolveDvcFQDN('CollectorHostName') | project-rename EventResult = EventOutcome, Operation = DeviceAction, ActorUsername = SourceUserName, SrcIpAddr = SourceIP, EventOriginalSeverity = LogSeverity, EventMessage = Message, EventOriginalType = DeviceEventClassID, EventUid = _ItemId | extend Dvc = DvcHostname, EventEndTime = TimeGenerated, EventStartTime = TimeGenerated, EventType = case( Operation has_any (\"update\", \"upsert\"), \"Set\", Operation has \"create\", \"Create\", Operation has \"delete\", \"Delete\", \"Other\" ), Object = iff(isempty(Object), \"Infoblox Network Resource\", Object), ObjectType = iff(isempty(ObjectType), \"Service\", ObjectType), Src = SrcIpAddr, ActorUserType = _ASIM_GetUserType(ActorUsername, \"\"), AdditionalFields = bag_pack( \"InfobloxHTTPReqBody\", InfobloxHTTPReqBody, \"InfobloxHTTPRespBody\", InfobloxHTTPRespBody ), User = ActorUsername, IpAddr = SrcIpAddr, ActorUsernameType = _ASIM_GetUsernameType(ActorUsername) | extend EventCount = toint(1), EventProduct = \"BloxOne\", EventVendor = \"Infoblox\", EventSchema = \"AuditEvent\", EventSchemaVersion = \"0.1\" | project-away Source*, Destination*, Device*, AdditionalExtensions, CommunicationDirection, Protocol, SimplifiedDeviceAction, ExternalID, EndTime, FieldDevice*, Flex*, File*, Old*, MaliciousIP*, OriginalLogSeverity, Process*, ReceivedBytes, SentBytes, Remote*, Request*, StartTime, TenantId, ReportReferenceLink, ReceiptTime, Indicator*, _ResourceId, ThreatConfidence, ThreatDescription, ThreatSeverity, Computer, ApplicationProtocol, ExtID, Reason, Activity, Infoblox* }; parser(disabled=disabled)",
+            "version": 1,
+            "functionParameters": "disabled:bool=False"
+          }
+        }
+      ]
+    }
+  ]
+}

Parsers/ASimAuditEvent/ARM/ASimAuditEventInfobloxBloxOne/README.md

@@ -0,0 +1,18 @@
+# Infoblox BloxOne ASIM AuditEvent Normalization Parser
+
+ARM template for ASIM AuditEvent schema parser for Infoblox BloxOne.
+
+This ASIM parser supports normalizing AuditEvent logs from Infoblox BloxOne to the ASIM AuditEvent normalized schema. These events are captured through the Azure Monitor Agent (AMA) which are sent by the Data Connector Service of Infoblox BloxOne.
+
+
+The Advanced Security Information Model (ASIM) enables you to use and create source-agnostic content, simplifying your analysis of the data in your Microsoft Sentinel workspace.
+
+For more information, see:
+
+- [Normalization and the Advanced Security Information Model (ASIM)](https://aka.ms/AboutASIM)
+- [Deploy all of ASIM](https://aka.ms/DeployASIM)
+- [ASIM AuditEvent normalization schema reference](https://aka.ms/ASimAuditEventDoc)
+
+<br>
+
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimAuditEvent%2FARM%2FASimAuditEventInfobloxBloxOne%2FASimAuditEventInfobloxBloxOne.json) [![Deploy to Azure Gov](https://aka.ms/deploytoazuregovbutton)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2FAzure-Sentinel%2Fmaster%2FParsers%2FASimAuditEvent%2FARM%2FASimAuditEventInfobloxBloxOne%2FASimAuditEventInfobloxBloxOne.json)

Parsers/ASimAuditEvent/ARM/FullDeploymentAuditEvent.json

@@ -178,6 +178,46 @@
         }
       }
     },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2020-10-01",
+      "name": "linkedASimAuditEventIllumioSaaSCore",
+      "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+          "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimAuditEvent/ARM/ASimAuditEventIllumioSaaSCore/ASimAuditEventIllumioSaaSCore.json",
+          "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+          "Workspace": {
+            "value": "[parameters('Workspace')]"
+          },
+          "WorkspaceRegion": {
+            "value": "[parameters('WorkspaceRegion')]"
+          }
+        }
+      }
+    },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2020-10-01",
+      "name": "linkedASimAuditEventInfobloxBloxOne",
+      "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+          "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimAuditEvent/ARM/ASimAuditEventInfobloxBloxOne/ASimAuditEventInfobloxBloxOne.json",
+          "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+          "Workspace": {
+            "value": "[parameters('Workspace')]"
+          },
+          "WorkspaceRegion": {
+            "value": "[parameters('WorkspaceRegion')]"
+          }
+        }
+      }
+    },
     {
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2020-10-01",
@@ -498,6 +538,46 @@
         }
       }
     },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2020-10-01",
+      "name": "linkedvimAuditEventIllumioSaaSCore",
+      "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+          "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimAuditEvent/ARM/vimAuditEventIllumioSaaSCore/vimAuditEventIllumioSaaSCore.json",
+          "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+          "Workspace": {
+            "value": "[parameters('Workspace')]"
+          },
+          "WorkspaceRegion": {
+            "value": "[parameters('WorkspaceRegion')]"
+          }
+        }
+      }
+    },
+    {
+      "type": "Microsoft.Resources/deployments",
+      "apiVersion": "2020-10-01",
+      "name": "linkedvimAuditEventInfobloxBloxOne",
+      "properties": {
+        "mode": "Incremental",
+        "templateLink": {
+          "uri": "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/Parsers/ASimAuditEvent/ARM/vimAuditEventInfobloxBloxOne/vimAuditEventInfobloxBloxOne.json",
+          "contentVersion": "1.0.0.0"
+        },
+        "parameters": {
+          "Workspace": {
+            "value": "[parameters('Workspace')]"
+          },
+          "WorkspaceRegion": {
+            "value": "[parameters('WorkspaceRegion')]"
+          }
+        }
+      }
+    },
     {
       "type": "Microsoft.Resources/deployments",
       "apiVersion": "2020-10-01",