Merge branch 'certeu:main' into xdr_certificateauth

requirements.txt

@@ -1,5 +1,5 @@
 PyYAML==6.0.1
-pySigma==0.11.17
+pySigma==0.11.18
 ruamel.yaml==0.18.1
 azure-common==1.1.28
 azure-core==1.30.1

setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = detect-droid
-version = 0.2.5
+version = 0.2.7
 author = cert-eu/mlc
 author_email = services@cert.europa.eu
 description = Detection Rules Optimisation Integration Deployment
@@ -17,7 +17,7 @@ package_dir =
 python_requires = >=3.11.8
 install_requires =
     PyYAML==6.0.1
-    pySigma==0.11.17
+    pySigma==0.11.18
     ruamel.yaml==0.18.1
     azure-common==1.1.28
     azure-core==1.30.1

src/droid/__main__.py

@@ -403,6 +403,7 @@ def main(argv=None) -> None:
 
         if export_error:
             logger.error("Error in exporting the rules")
+            exit(1)
         else:
             logger.info("Successfully exported the rules")
 

src/droid/export.py

@@ -101,8 +101,11 @@ def export_rule_raw(parameters: dict, export_config: dict, logger_param: dict):
                 try:
                     platform.create_rule(rule_content, rule_converted, rule_file)
                 except:
-                    logger.error(f"Error in creating search for rule {rule_file}")
-                    error_i = True
+                    if rule_content.get("custom", {}).get("ignore_export_error", False):
+                        logger.warning(f"(Ignoring) Error in creating search for rule {rule_file}")
+                    else:
+                        logger.error(f"Error in creating search for rule {rule_file}")
+                        error_i = True
         if error_i:
             error = True
             return error
@@ -122,8 +125,12 @@ def export_rule_raw(parameters: dict, export_config: dict, logger_param: dict):
             try:
                 platform.create_rule(rule_content, rule_converted, rule_file)
             except Exception as e:
-                logger.error(f"Error in creating search for rule {rule_file} - error: {e}")
-                error = True
+                if rule_content.get("custom", {}).get("ignore_export_error", False):
+                    logger.warning(f"(Ignoring) Error in creating search for rule {rule_file} - error: {e}")
+                    error = False
+                else:
+                    logger.error(f"Error in creating search for rule {rule_file} - error: {e}")
+                    error = True
         if error:
             return error
     else:

src/droid/integrity.py

@@ -207,7 +207,11 @@ def integrity_rule_ms_xdr_mssp(rule_converted, rule_content, platform: Microsoft
             logger.error(f"Couldn't check the integrity for the rule {rule_file} on tenant {tenant_id} from {group} - error {e}")
             return error
 
-        error = integrity_rule_ms_xdr(rule_converted, rule_content, platform, rule_file, parameters, logger, error, saved_search=saved_search)
+        if not saved_search:
+            error = True
+            logger.error(f"Rule not found {rule_file} in {group} - {tenant_id}")
+        else:
+            error = integrity_rule_ms_xdr(rule_converted, rule_content, platform, rule_file, parameters, logger, error, saved_search=saved_search)
 
         if error:
             error_occured = True
@@ -394,17 +398,30 @@ def integrity_rule_raw(parameters: dict, export_config: dict, logger_param: dict
             rule_content = load_rule(rule_file)
             rule_converted = rule_content["detection"]
             error = integrity_rule(parameters, rule_converted, rule_content, platform, rule_file, error, logger_param)
+
             if error:
-                error_i = True
-        if error_i:
-            error = True
-            return error
+                custom_settings = rule_content.get("custom", {})
+                if custom_settings.get("ignore_export_error", False):
+                    logger.warning(f"(Ignoring) rule not found {rule_file}")
+                elif custom_settings.get("removed", False):
+                    logger.info(f"Rule not found and intended to be removed {rule_file}")
+                else:
+                    error_i = True
+
+        return error_i
+
 
     elif path.is_file():
         rule_file = path
         rule_content = load_rule(rule_file)
         rule_converted = rule_content["detection"]
         error = integrity_rule(parameters, rule_converted, rule_content, platform, rule_file, error, logger_param)
+        if error and rule_content.get("custom", {}).get("ignore_export_error", False):
+            logger.warning(f"(Ignoring) rule not found {rule_file}")
+            error = False
+        elif error and rule_content.get("custom", {}).get("removed", False):
+            logger.info(f"Rule not found and intended to be removed {rule_file}")
+            error = False
     else:
         print(f"The path {path} is neither a directory nor a file.")
 

src/droid/platforms/ms_xdr.py

@@ -107,7 +107,6 @@ def __init__(
     def get_export_list_mssp(self) -> list:
 
         if self._export_list_mssp:
-            self.logger.info("Integrity check for designated customers")
             return self._export_list_mssp
         else:
             self.logger.error("No export_list_mssp found")
@@ -128,7 +127,7 @@ def run_xdr_search(self, rule_converted, rule_file, tenant_id=None):
             results, status_code = self._post(
                 url="/security/runHuntingQuery", payload=payload, tenant_id=tenant_id
             )
-
+            time.sleep(2)
             if "error" in results:
                 self.logger.error(
                     f"Error while running the query {results['error']['message']}"
@@ -252,7 +251,7 @@ def acquire_token(self, tenant_id=None):
                 self.logger.error(
                     f'Failed to acquire token: {result["error_description"]}'
                 )
-                exit()
+                raise Exception(f"Token acquisition failed: {result.get('error', 'Unknown error')}")
 
     def process_query_period(self, query_period: str, rule_file: str):
         """Process the query period time
@@ -391,6 +390,7 @@ def create_rule(self, rule_content, rule_converted, rule_file):
 
         if self._export_mssp:
             if self._export_list_mssp:
+                error = False
                 self.logger.info("Exporting to designated customers")
                 for group, info in self._export_list_mssp.items():
 
@@ -417,7 +417,9 @@ def create_rule(self, rule_content, rule_converted, rule_file):
                                 "error": e,
                             },
                         )
-                        raise
+                        error = True
+                if error:
+                    raise
             else:
                 self.logger.error(
                     "Export list not found. Please provide the list of designated customers"
@@ -502,7 +504,7 @@ def push_detection_rule(
         rule_converted=None,
         tenant_id=None,
     ):
-        existing_rule = self.get_rule(rule_content["id"])
+        existing_rule = self.get_rule(rule_content["id"], tenant_id=tenant_id)
         if existing_rule:
             self.logger.info("Rule already exists")
             if not self.check_rule_changes(existing_rule, alert_rule):

src/droid/platforms/sentinel.py

@@ -15,6 +15,7 @@
 from azure.mgmt.securityinsight.models import EventGroupingSettings
 from azure.mgmt.securityinsight.models import IncidentConfiguration
 from azure.mgmt.securityinsight.models import GroupingConfiguration
+from azure.mgmt.securityinsight.models import EntityMapping, FieldMapping
 from azure.monitor.query import LogsQueryClient, LogsQueryStatus
 from datetime import datetime, timedelta, timezone
 from azure.core.exceptions import HttpResponseError, ResourceNotFoundError
@@ -420,6 +421,14 @@ def create_rule(self, rule_content, rule_converted, rule_file):
         else:
             enabled = True
 
+        # Handling the entities
+        entity_mappings = []
+        if rule_content.get('custom', {}).get('entity_mappings'):
+            for mapping in rule_content['custom']['entity_mappings']:
+                field_mappings = [FieldMapping(identifier=field['identifier'], column_name=field['column_name'])
+                                    for field in mapping['field_mappings']]
+                entity_mappings.append(EntityMapping(entity_type=mapping['entity_type'], field_mappings=field_mappings))
+
         # Handling the severity
         if rule_content['level'] == 'critical':
             severity = 'high'
@@ -471,6 +480,7 @@ def create_rule(self, rule_content, rule_converted, rule_file):
             event_grouping_settings=EventGroupingSettings(aggregation_kind="SingleAlert"),
             incident_configuration=IncidentConfiguration(create_incident=create_incident, grouping_configuration=grouping_config),
             tactics=self.mitre_tactics(rule_content),
+            entity_mappings=entity_mappings,
             techniques=self.mitre_techniques(rule_content)
         )
 
@@ -482,6 +492,7 @@ def create_rule(self, rule_content, rule_converted, rule_file):
 
         if self._export_mssp:
             if self._export_list_mssp:
+                error = False
                 self.logger.info("Exporting to designated customers")
                 for group, info in self._export_list_mssp.items():
 
@@ -507,7 +518,9 @@ def create_rule(self, rule_content, rule_converted, rule_file):
                         self.logger.info(f"Successfully exported the rule {rule_file} to {workspace_name}")
                     except Exception as e:
                         self.logger.error(f"Failed to export the rule {rule_file} to {workspace_name} - error: {e}")
-                        raise
+                        error = True
+                if error:
+                    raise
             else:
                 self.logger.error("Export list not found. Please provide the list of designated customers")
                 raise

src/droid/search.py

@@ -82,18 +82,17 @@ def search_rule_ms_xdr_mssp(rule_converted, platform: MicrosoftXDRPlatform, rule
 
             logger.info(f"Successfully searched the rule {rule_file}")
 
-            if result > 0: # If the rule has match
+            if result > 0:  # If the rule has a match
                 logger.warning(f"{result} Matches found for {rule_file}")
                 search_warning = True
-                return error, search_warning
             else:
                 logger.info(f"No hits for {rule_file}")
-                return error, search_warning
 
         except Exception as e:
             logger.error(f"Couldn't search for the rule {rule_file} for tenant {tenant_id} - error {e}")
             error = True
-            return error, search_warning
+
+    return error, search_warning
 
 def search_rule_ms_xdr(rule_converted, platform: MicrosoftXDRPlatform, rule_file, parameters, logger, error, search_warning):