Event UUIDs

bbot/core/event/base.py

@@ -1,5 +1,6 @@
 import io
 import re
+import uuid
 import json
 import base64
 import logging
@@ -58,9 +59,10 @@ class BaseEvent:
 
     Attributes:
         type (str): Specifies the type of the event, e.g., `IP_ADDRESS`, `DNS_NAME`.
-        id (str): A unique identifier for the event.
+        id (str): An identifier for the event (event type + sha1 hash of data). NOT universally unique.
+        uuid (UUID): A universally unique identifier for the event.
         data (str or dict): The main data for the event, e.g., a URL or IP address.
-        data_graph (str): Representation of `self.data` for Neo4j graph nodes.
+        data_graph (str): Representation of `self.data` for graph nodes (e.g. Neo4j).
         data_human (str): Representation of `self.data` for human output.
         data_id (str): Representation of `self.data` used to calculate the event's ID (and ultimately its hash, which is used for deduplication)
         data_json (str): Representation of `self.data` to be used in JSON serialization.
@@ -75,6 +77,7 @@ class BaseEvent:
         resolved_hosts (list of str): List of hosts to which the event data resolves, applicable for URLs and DNS names.
         parent (BaseEvent): The parent event that led to the discovery of this event.
         parent_id (str): The `id` attribute of the parent event.
+        parent_uuid (str): The `uuid` attribute of the parent event.
         tags (set of str): Descriptive tags for the event, e.g., `mx-record`, `in-scope`.
         module (BaseModule): The module that discovered the event.
         module_sequence (str): The sequence of modules that participated in the discovery.
@@ -154,7 +157,7 @@ def __init__(
         Raises:
             ValidationError: If either `scan` or `parent` are not specified and `_dummy` is False.
         """
-
+        self.uuid = uuid.uuid4()
         self._id = None
         self._hash = None
         self._data = None
@@ -166,6 +169,7 @@ def __init__(
         self._parent = None
         self._priority = None
         self._parent_id = None
+        self._parent_uuid = None
         self._host_original = None
         self._scope_distance = None
         self._module_priority = None
@@ -393,7 +397,7 @@ def parent_chain(self):
         parent_chain = []
         if self.parent is not None and self.parent is not self:
             parent_chain = self.parent.parent_chain
-        return parent_chain + [self.id]
+        return parent_chain + [str(self.uuid)]
 
     @property
     def words(self):
@@ -567,6 +571,13 @@ def parent_id(self):
             return parent_id
         return self._parent_id
 
+    @property
+    def parent_uuid(self):
+        parent_uuid = getattr(self.get_parent(), "uuid", None)
+        if parent_uuid is not None:
+            return parent_uuid
+        return self._parent_uuid
+
     @property
     def validators(self):
         """
@@ -733,12 +744,12 @@ def json(self, mode="json", siem_friendly=False):
         Returns:
             dict: JSON-serializable dictionary representation of the event object.
         """
-        # type, ID, scope description
         j = dict()
-        for i in ("type", "id", "scope_description"):
+        # type, ID, scope description
+        for i in ("type", "id", "uuid", "scope_description"):
             v = getattr(self, i, "")
             if v:
-                j.update({i: v})
+                j.update({i: str(v)})
         # event data
         data_attr = getattr(self, f"data_{mode}", None)
         if data_attr is not None:
@@ -769,6 +780,9 @@ def json(self, mode="json", siem_friendly=False):
         parent_id = self.parent_id
         if parent_id:
             j["parent"] = parent_id
+        parent_uuid = self.parent_uuid
+        if parent_uuid:
+            j["parent_uuid"] = parent_uuid
         # tags
         if self.tags:
             j.update({"tags": list(self.tags)})
@@ -1691,6 +1705,9 @@ def event_from_json(j, siem_friendly=False):
             data = j["data"]
         kwargs["data"] = data
         event = make_event(**kwargs)
+        event_uuid = j.get("uuid", None)
+        if event_uuid is not None:
+            event.uuid = uuid.UUID(event_uuid)
 
         resolved_hosts = j.get("resolved_hosts", [])
         event._resolved_hosts = set(resolved_hosts)
@@ -1700,6 +1717,9 @@ def event_from_json(j, siem_friendly=False):
         parent_id = j.get("parent", None)
         if parent_id is not None:
             event._parent_id = parent_id
+        parent_uuid = j.get("parent_uuid", None)
+        if parent_uuid is not None:
+            event._parent_uuid = uuid.UUID(parent_uuid)
         return event
     except KeyError as e:
         raise ValidationError(f"Event missing required field: {e}")

bbot/core/helpers/names_generator.py

@@ -75,6 +75,7 @@
     "ethereal",
     "euphoric",
     "evil",
+    "expired",
     "exquisite",
     "extreme",
     "ferocious",
@@ -87,7 +88,6 @@
     "foreboding",
     "frenetic",
     "frolicking",
-    "frothy",
     "furry",
     "fuzzy",
     "gay",
@@ -149,6 +149,7 @@
     "muscular",
     "mushy",
     "mysterious",
+    "nascent",
     "naughty",
     "nefarious",
     "negligent",

bbot/core/helpers/regexes.py

@@ -152,3 +152,5 @@
 # for use in recursive_decode()
 encoded_regex = re.compile(r"%[0-9a-fA-F]{2}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\[ntrbv]")
 backslash_regex = re.compile(r"(?P<slashes>\\+)(?P<char>[ntrvb])")
+
+uuid_regex = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")

bbot/modules/internal/dnsresolve.py

@@ -90,6 +90,7 @@ async def handle_event(self, event, **kwargs):
         # if there weren't any DNS children and it's not an IP address, tag as unresolved
         if not main_host_event.raw_dns_records and not event_is_ip:
             main_host_event.add_tag("unresolved")
+            main_host_event.type = "DNS_NAME_UNRESOLVED"
 
         # main_host_event.add_tag(f"resolve-distance-{main_host_event.dns_resolve_distance}")
 
@@ -109,10 +110,6 @@ async def handle_event(self, event, **kwargs):
             if not self.minimal:
                 await self.emit_dns_children(main_host_event)
 
-            # If the event is unresolved, change its type to DNS_NAME_UNRESOLVED
-            if main_host_event.type == "DNS_NAME" and "unresolved" in main_host_event.tags:
-                main_host_event.type = "DNS_NAME_UNRESOLVED"
-
             # emit the main DNS_NAME or IP_ADDRESS
             if (
                 new_event

bbot/scanner/scanner.py

@@ -115,6 +115,8 @@ def __init__(
             dispatcher (Dispatcher, optional): Dispatcher object to use. Defaults to new Dispatcher.
             **kwargs (list[str], optional): Additional keyword arguments (passed through to `Preset`).
         """
+        self._root_event = None
+
         if scan_id is not None:
             self.id = str(id)
         else:
@@ -945,13 +947,15 @@ def root_event(self):
             }
             ```
         """
-        root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True)
-        root_event._id = self.id
-        root_event.scope_distance = 0
-        root_event.parent = root_event
-        root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET")
-        root_event.discovery_context = f"Scan {self.name} started at {root_event.timestamp}"
-        return root_event
+        if self._root_event is None:
+            root_event = self.make_event(data=self.json, event_type="SCAN", dummy=True)
+            root_event._id = self.id
+            root_event.scope_distance = 0
+            root_event.parent = root_event
+            root_event.module = self._make_dummy_module(name="TARGET", _type="TARGET")
+            root_event.discovery_context = f"Scan {self.name} started at {root_event.timestamp}"
+            self._root_event = root_event
+        return self._root_event
 
     @property
     def dns_strings(self):

bbot/test/test_step_1/test_events.py

@@ -4,6 +4,7 @@
 
 from ..bbot_fixtures import *
 from bbot.scanner import Scanner
+from bbot.core.helpers.regexes import uuid_regex
 
 
 @pytest.mark.asyncio
@@ -412,34 +413,73 @@ async def test_events(events, helpers):
         == "http://xn--12c1bik6bbd8ab6hd1b5jc6jta.com/ทดสอบ"
     )
 
+    # test event uuid
+    import uuid
+
+    parent_event1 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+    parent_event2 = scan.make_event("evilcorp.com", parent=scan.root_event, context="test context")
+
+    event1 = scan.make_event("evilcorp.com:80", parent=parent_event1, context="test context")
+    assert hasattr(event1, "uuid")
+    assert isinstance(event1.uuid, uuid.UUID)
+    event2 = scan.make_event("evilcorp.com:80", parent=parent_event2, context="test context")
+    assert hasattr(event2, "uuid")
+    assert isinstance(event2.uuid, uuid.UUID)
+    # ids should match because the event type + data is the same
+    assert event1.id == event2.id
+    # but uuids should be unique!
+    assert event1.uuid != event2.uuid
+    # parent ids should match
+    assert event1.parent_id == event2.parent_id == parent_event1.id == parent_event2.id
+    # uuids should not
+    assert event1.parent_uuid == parent_event1.uuid
+    assert event2.parent_uuid == parent_event2.uuid
+    assert event1.parent_uuid != event2.parent_uuid
+
     # test event serialization
     from bbot.core.event import event_from_json
 
     db_event = scan.make_event("evilcorp.com:80", parent=scan.root_event, context="test context")
+    assert db_event.parent == scan.root_event
+    assert db_event.parent is scan.root_event
     db_event._resolved_hosts = {"127.0.0.1"}
     db_event.scope_distance = 1
     assert db_event.discovery_context == "test context"
     assert db_event.discovery_path == ["test context"]
-    assert db_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert len(db_event.parent_chain) == 1
+    assert all([uuid_regex.match(u) for u in db_event.parent_chain])
+    assert db_event.parent_chain[0] == str(db_event.uuid)
+    assert db_event.parent.uuid == scan.root_event.uuid
+    assert db_event.parent_uuid == scan.root_event.uuid
     timestamp = db_event.timestamp.isoformat()
     json_event = db_event.json()
+    assert isinstance(json_event["uuid"], str)
+    assert json_event["uuid"] == str(db_event.uuid)
+    print(f"{json_event} / {db_event.uuid} / {db_event.parent_uuid} / {scan.root_event.uuid}")
+    assert json_event["parent_uuid"] == str(scan.root_event.uuid)
     assert json_event["scope_distance"] == 1
     assert json_event["data"] == "evilcorp.com:80"
     assert json_event["type"] == "OPEN_TCP_PORT"
     assert json_event["host"] == "evilcorp.com"
     assert json_event["timestamp"] == timestamp
     assert json_event["discovery_context"] == "test context"
     assert json_event["discovery_path"] == ["test context"]
-    assert json_event["parent_chain"] == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert json_event["parent_chain"] == db_event.parent_chain
+    assert json_event["parent_chain"][0] == str(db_event.uuid)
     reconstituted_event = event_from_json(json_event)
+    assert isinstance(reconstituted_event.uuid, uuid.UUID)
+    assert str(reconstituted_event.uuid) == json_event["uuid"]
+    assert str(reconstituted_event.parent_uuid) == json_event["parent_uuid"]
+    assert reconstituted_event.uuid == db_event.uuid
+    assert reconstituted_event.parent_uuid == scan.root_event.uuid
     assert reconstituted_event.scope_distance == 1
     assert reconstituted_event.timestamp.isoformat() == timestamp
     assert reconstituted_event.data == "evilcorp.com:80"
     assert reconstituted_event.type == "OPEN_TCP_PORT"
     assert reconstituted_event.host == "evilcorp.com"
     assert reconstituted_event.discovery_context == "test context"
     assert reconstituted_event.discovery_path == ["test context"]
-    assert reconstituted_event.parent_chain == ["OPEN_TCP_PORT:5098b5e3fc65b13bb4a5cee4201c2e160fa4ffac"]
+    assert reconstituted_event.parent_chain == db_event.parent_chain
     assert "127.0.0.1" in reconstituted_event.resolved_hosts
     hostless_event = scan.make_event("asdf", "ASDF", dummy=True)
     hostless_event_json = hostless_event.json()

bbot/test/test_step_2/module_tests/test_module_json.py

@@ -9,6 +9,9 @@ def check(self, module_test, events):
         dns_data = "blacklanternsecurity.com"
         context_data = f"Scan {module_test.scan.name} seeded with DNS_NAME: blacklanternsecurity.com"
 
+        scan_event = [e for e in events if e.type == "SCAN"][0]
+        dns_event = [e for e in events if e.type == "DNS_NAME"][0]
+
         # json events
         txt_file = module_test.scan.home / "output.json"
         lines = list(module_test.scan.helpers.read_file(txt_file))
@@ -22,24 +25,34 @@ def check(self, module_test, events):
         dns_json = dns_json[0]
         assert scan_json["data"]["name"] == module_test.scan.name
         assert scan_json["data"]["id"] == module_test.scan.id
+        assert scan_json["id"] == module_test.scan.id
+        assert scan_json["uuid"] == str(module_test.scan.root_event.uuid)
+        assert scan_json["parent_uuid"] == str(module_test.scan.root_event.uuid)
         assert scan_json["data"]["target"]["seeds"] == ["blacklanternsecurity.com"]
         assert scan_json["data"]["target"]["whitelist"] == ["blacklanternsecurity.com"]
         assert dns_json["data"] == dns_data
+        assert dns_json["id"] == str(dns_event.id)
+        assert dns_json["uuid"] == str(dns_event.uuid)
+        assert dns_json["parent_uuid"] == str(module_test.scan.root_event.uuid)
         assert dns_json["discovery_context"] == context_data
         assert dns_json["discovery_path"] == [context_data]
-        assert dns_json["parent_chain"] == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"]
+        assert dns_json["parent_chain"] == [dns_json["uuid"]]
 
         # event objects reconstructed from json
         scan_reconstructed = event_from_json(scan_json)
         dns_reconstructed = event_from_json(dns_json)
         assert scan_reconstructed.data["name"] == module_test.scan.name
         assert scan_reconstructed.data["id"] == module_test.scan.id
+        assert scan_reconstructed.uuid == scan_event.uuid
+        assert scan_reconstructed.parent_uuid == scan_event.uuid
         assert scan_reconstructed.data["target"]["seeds"] == ["blacklanternsecurity.com"]
         assert scan_reconstructed.data["target"]["whitelist"] == ["blacklanternsecurity.com"]
         assert dns_reconstructed.data == dns_data
+        assert dns_reconstructed.uuid == dns_event.uuid
+        assert dns_reconstructed.parent_uuid == module_test.scan.root_event.uuid
         assert dns_reconstructed.discovery_context == context_data
         assert dns_reconstructed.discovery_path == [context_data]
-        assert dns_reconstructed.parent_chain == ["DNS_NAME:1e57014aa7b0715bca68e4f597204fc4e1e851fc"]
+        assert dns_reconstructed.parent_chain == [dns_json["uuid"]]
 
 
 class TestJSONSIEMFriendly(ModuleTestBase):