[client] Limit stix_ids explosion by rewriting the standard_id in cli…

…ent python (#659)

pycti/connector/opencti_connector_helper.py

@@ -849,6 +849,13 @@ def __init__(self, config: Dict, playbook_compatible=False) -> None:
             False,
             True,
         )
+        self.keep_original_id = get_config_variable(
+            "CONNECTOR_KEEP_ORIGINAL_ID",
+            ["connector", "keep_original_id"],
+            config,
+            False,
+            False,
+        )
         self.bundle_send_to_directory = get_config_variable(
             "CONNECTOR_SEND_TO_DIRECTORY",
             ["connector", "send_to_directory"],
@@ -1571,6 +1578,7 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list:
         file_name = kwargs.get("file_name", None)
         bundle_send_to_queue = kwargs.get("send_to_queue", self.bundle_send_to_queue)
         cleanup_inconsistent_bundle = kwargs.get("cleanup_inconsistent_bundle", False)
+        keep_original_id = kwargs.get("keep_original_id", self.keep_original_id)
         bundle_send_to_directory = kwargs.get(
             "send_to_directory", self.bundle_send_to_directory
         )
@@ -1581,6 +1589,11 @@ def send_stix2_bundle(self, bundle: str, **kwargs) -> list:
             "send_to_directory_retention", self.bundle_send_to_directory_retention
         )
 
+        # Bundle ids must be rewritten
+        bundle = self.api.stix2.prepare_bundle_ids(
+            bundle=bundle, use_json=True, keep_original_id=keep_original_id
+        )
+
         # In case of enrichment ingestion, ensure the sharing if needed
         if self.enrichment_shared_organizations is not None:
             # Every element of the bundle must be enriched with the same organizations

pycti/entities/opencti_attack_pattern.py

@@ -222,15 +222,18 @@ def __init__(self, opencti):
 
     @staticmethod
     def generate_id(name, x_mitre_id=None):
-        name = name.lower().strip()
         if x_mitre_id is not None:
             data = {"x_mitre_id": x_mitre_id}
         else:
-            data = {"name": name}
+            data = {"name": name.lower().strip()}
         data = canonicalize(data, utf8=False)
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "attack-pattern--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return AttackPattern.generate_id(data.get("name"), data.get("x_mitre_id"))
+
     """
         List Attack-Pattern objects
 

pycti/entities/opencti_campaign.py

@@ -216,6 +216,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "campaign--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Campaign.generate_id(data["name"])
+
     """
         List Campaign objects
 

pycti/entities/opencti_case_incident.py

@@ -461,6 +461,10 @@ def generate_id(name, created):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "case-incident--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return CaseIncident.generate_id(data["name"], data["created"])
+
     """
         List Case Incident objects
         

pycti/entities/opencti_case_rfi.py

@@ -457,6 +457,10 @@ def generate_id(name, created):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "case-rfi--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return CaseRfi.generate_id(data["name"], data["created"])
+
     """
         List Case Rfi objects
         

pycti/entities/opencti_case_rft.py

@@ -457,6 +457,10 @@ def generate_id(name, created):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "case-rft--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return CaseRft.generate_id(data["name"], data["created"])
+
     """
         List Case Rft objects
         

pycti/entities/opencti_channel.py

@@ -212,6 +212,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "channel--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Channel.generate_id(data["name"])
+
     """
         List Channel objects
 

pycti/entities/opencti_course_of_action.py

@@ -196,15 +196,18 @@ def __init__(self, opencti):
 
     @staticmethod
     def generate_id(name, x_mitre_id=None):
-        name = name.lower().strip()
         if x_mitre_id is not None:
             data = {"x_mitre_id": x_mitre_id}
         else:
-            data = {"name": name}
+            data = {"name": name.lower().strip()}
         data = canonicalize(data, utf8=False)
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "course-of-action--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return CourseOfAction.generate_id(data.get("name"), data.get("x_mitre_id"))
+
     """
         List Course-Of-Action objects
 

pycti/entities/opencti_data_component.py

@@ -246,6 +246,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "data-component--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return DataComponent.generate_id(data["name"])
+
     """
         List Data-Component objects
 

pycti/entities/opencti_data_source.py

@@ -204,6 +204,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "data-source--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return DataSource.generate_id(data["name"])
+
     """
         List Data-Source objects
 

pycti/entities/opencti_event.py

@@ -216,6 +216,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "event--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Event.generate_id(data["name"])
+
     """
         List Event objects
 

pycti/entities/opencti_external_reference.py

@@ -68,6 +68,12 @@ def generate_id(url=None, source_name=None, external_id=None):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "external-reference--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return ExternalReference.generate_id(
+            data.get("url"), data.get("source_name"), data.get("external_id")
+        )
+
     """
         List External-Reference objects
 

pycti/entities/opencti_feedback.py

@@ -419,6 +419,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "feedback--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Feedback.generate_id(data["name"])
+
     """
         List Feedback objects
         

pycti/entities/opencti_grouping.py

@@ -406,6 +406,10 @@ def generate_id(name, context, created):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "grouping--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Grouping.generate_id(data["name"], data["context"])
+
     """
         List Grouping objects
 

pycti/entities/opencti_identity.py

@@ -226,12 +226,15 @@ def __init__(self, opencti):
 
     @staticmethod
     def generate_id(name, identity_class):
-        name = name.lower().strip()
-        data = {"name": name, "identity_class": identity_class}
+        data = {"name": name.lower().strip(), "identity_class": identity_class.lower()}
         data = canonicalize(data, utf8=False)
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "identity--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Identity.generate_id(data["name"], data["identity_class"])
+
     """
         List Identity objects
 

pycti/entities/opencti_incident.py

@@ -225,6 +225,10 @@ def generate_id(name, created):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "incident--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Incident.generate_id(data["name"], data["created"])
+
     """
         List Incident objects
 

pycti/entities/opencti_indicator.py

@@ -29,6 +29,10 @@ def generate_id(pattern):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "indicator--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Indicator.generate_id(data["pattern"])
+
     def list(self, **kwargs):
         """List Indicator objects
 

pycti/entities/opencti_infrastructure.py

@@ -239,6 +239,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "infrastructure--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Infrastructure.generate_id(data["name"])
+
     def list(self, **kwargs):
         """List Infrastructure objects
 

pycti/entities/opencti_intrusion_set.py

@@ -222,6 +222,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "intrusion-set--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return IntrusionSet.generate_id(data["name"])
+
     """
         List Intrusion-Set objects
 

pycti/entities/opencti_kill_chain_phase.py

@@ -28,6 +28,10 @@ def generate_id(phase_name, kill_chain_name):
             phase_name=phase_name, kill_chain_name=kill_chain_name
         )
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return KillChainPhase.generate_id(data["phase_name"], data["kill_chain_name"])
+
     """
         List Kill-Chain-Phase objects
 

pycti/entities/opencti_language.py

@@ -224,6 +224,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "language--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Language.generate_id(data["name"])
+
     """
         List Language objects
 

pycti/entities/opencti_location.py

@@ -210,15 +210,33 @@ def __init__(self, opencti):
 
     @staticmethod
     def generate_id(name, x_opencti_location_type, latitude=None, longitude=None):
-        name = name.lower().strip()
-        if x_opencti_location_type == "position":
-            data = {"name": name, "latitude": latitude, "longitude": longitude}
+        if x_opencti_location_type == "Position":
+            if latitude is not None and longitude is None:
+                data = {"latitude": latitude}
+            elif latitude is None and longitude is not None:
+                data = {"longitude": longitude}
+            elif latitude is not None and longitude is not None:
+                data = {"latitude": latitude, "longitude": longitude}
+            else:
+                data = {"name": name.lower().strip()}
         else:
-            data = {"name": name, "x_opencti_location_type": x_opencti_location_type}
+            data = {
+                "name": name.lower().strip(),
+                "x_opencti_location_type": x_opencti_location_type,
+            }
         data = canonicalize(data, utf8=False)
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "location--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Location.generate_id(
+            data.get("name"),
+            data.get("x_opencti_location_type"),
+            data.get("latitude"),
+            data.get("longitude"),
+        )
+
     """
         List Location objects
 

pycti/entities/opencti_malware.py

@@ -250,6 +250,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "malware--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Malware.generate_id(data["name"])
+
     """
         List Malware objects
 

pycti/entities/opencti_malware_analysis.py

@@ -219,13 +219,21 @@ def __init__(self, opencti):
         """
 
     @staticmethod
-    def generate_id(result_name):
+    def generate_id(result_name, product, submitted):
         result_name = result_name.lower().strip()
-        data = {"result_name": result_name}
+        data = {"result_name": result_name, "product": product}
+        if submitted is not None:
+            data = {**data, "submitted": submitted}
         data = canonicalize(data, utf8=False)
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "malware-analysis--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return MalwareAnalysis.generate_id(
+            data["result_name"], data["product"], data.get("submitted")
+        )
+
     """
         List Malware analysis objects
 

pycti/entities/opencti_marking_definition.py

@@ -31,6 +31,12 @@ def generate_id(definition, definition_type):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "marking-definition--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return MarkingDefinition.generate_id(
+            data["definition"], data["definition_type"]
+        )
+
     """
         List Marking-Definition objects
 

pycti/entities/opencti_narrative.py

@@ -202,6 +202,10 @@ def generate_id(name):
         id = str(uuid.uuid5(uuid.UUID("00abedb4-aa42-466c-9c01-fed23315a9b7"), data))
         return "narrative--" + id
 
+    @staticmethod
+    def generate_id_from_data(data):
+        return Narrative.generate_id(data["name"])
+
     """
         List Narrative objects