dedup dns children by (rdtype, child) not parent host

bbot/modules/internal/dnsresolve.py

@@ -210,7 +210,7 @@ async def emit_dns_children(self, event):
                 if rdtype == "PTR":
                     child_event.add_tag("ptr")
 
-                child_hash = hash(f"{event.host}:{module}:{child_host}")
+                child_hash = hash(f"{module}:{child_host}")
                 # if we haven't emitted this one before
                 if child_hash not in self.children_emitted:
                     # and it's either in-scope or inside our dns search distance

bbot/test/test_step_1/test_manager_deduplication.py

@@ -128,7 +128,7 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in default_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]])
 
-    assert len(all_events) == 27
+    assert len(all_events) == 26
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "accept_dupes.test.notreal" and str(e.module) == "accept_dupes"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "default_module.test.notreal" and str(e.module) == "default_module"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "no_suppress_dupes.test.notreal" and str(e.module) == "no_suppress_dupes" and e.parent.data == "accept_dupes.test.notreal"])
@@ -140,7 +140,9 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "per_hostport_only.test.notreal" and str(e.module) == "per_hostport_only"])
     assert 1 == len([e for e in all_events if e.type == "DNS_NAME" and e.data == "test.notreal" and str(e.module) == "SEED" and "SCAN:" in e.parent.data["id"]])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "test.notreal"])
-    assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"])
+    # the second 127.0.0.3 (parent=default_module.test.notreal) is now deduped at the DNSResolve layer
+    # because (rdtype, child) collides with the test.notreal->127.0.0.3 edge already emitted
+    assert 0 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.3" and str(e.module) == "A" and e.parent.data == "default_module.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.5" and str(e.module) == "A" and e.parent.data == "no_suppress_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.6" and str(e.module) == "A" and e.parent.data == "accept_dupes.test.notreal"])
     assert 1 == len([e for e in all_events if e.type == "IP_ADDRESS" and e.data == "127.0.0.7" and str(e.module) == "A" and e.parent.data == "per_hostport_only.test.notreal"])

bbot/test/test_step_2/module_tests/test_module_dnscommonsrv.py

@@ -55,7 +55,7 @@ async def new_run_live(*command, check=False, text=True, **kwargs):
         )
 
     def check(self, module_test, events):
-        assert len(events) == 20
+        assert len(events) == 19
         assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "blacklanternsecurity.com"])
         assert 1 == len(
             [
@@ -75,7 +75,9 @@ def check(self, module_test, events):
                 and str(e.module) == "dnscommonsrv"
             ]
         ), "Failed to detect subdomain 2"
-        assert 2 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"]), (
+        # cross-parent (rdtype, child) dedup in DNSResolve.emit_dns_children collapses
+        # the two SRV->asdf edges into a single DNS_NAME event
+        assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "asdf.blacklanternsecurity.com"]), (
             "Failed to detect subdomain 3"
         )
         assert 1 == len([e for e in events if e.type == "DNS_NAME" and e.data == "api.blacklanternsecurity.com"]), (
@@ -109,6 +111,6 @@ def check(self, module_test, events):
             ]
         ), "Failed to emit RAW_DNS_RECORD for _ldap._tcp.gc._msdcs.blacklanternsecurity.com"
         assert 2 == len([e for e in events if e.type == "RAW_DNS_RECORD"])
-        assert 10 == len([e for e in events if e.type == "DNS_NAME"])
+        assert 9 == len([e for e in events if e.type == "DNS_NAME"])
         assert 5 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED"])
         assert 5 == len([e for e in events if e.type == "DNS_NAME_UNRESOLVED" and str(e.module) == "speculate"])

bbot/test/test_step_2/module_tests/test_module_dnsresolve.py

@@ -94,6 +94,57 @@ def check(self, module_test, events):
         )
 
 
+class TestDNSResolveSharedNameserverDedup(ModuleTestBase):
+    """
+    Multiple in-scope parents that share the same NS/SOA records should not cause the
+    shared nameserver hostname to be re-emitted once per parent. The dedup in
+    DNSResolve.emit_dns_children must collapse identical (rdtype, child) pairs across
+    parents, not key on parent host.
+    """
+
+    module_name = "dnsresolve"
+    targets = ["domain-a.test", "domain-b.test", "domain-c.test"]
+    config_overrides = {"dns": {"minimal": False}, "scope": {"report_distance": 1}}
+
+    async def setup_after_prep(self, module_test):
+        shared_ns = ["shared-ns1.cloudprovider.test.", "shared-ns2.cloudprovider.test."]
+        shared_soa = ["shared-ns1.cloudprovider.test. admin.cloudprovider.test. 1 7200 3600 1209600 3600"]
+        await module_test.mock_dns(
+            {
+                "domain-a.test": {"A": ["192.168.0.1"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-b.test": {"A": ["192.168.0.2"], "NS": shared_ns, "SOA": shared_soa},
+                "domain-c.test": {"A": ["192.168.0.3"], "NS": shared_ns, "SOA": shared_soa},
+                # resolve the shared nameservers so they keep the DNS_NAME type
+                "shared-ns1.cloudprovider.test": {"A": ["192.168.99.1"]},
+                "shared-ns2.cloudprovider.test": {"A": ["192.168.99.2"]},
+            }
+        )
+
+    def check(self, module_test, events):
+        from collections import Counter
+
+        ns_counts = Counter(
+            e.data for e in events if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "NS"
+        )
+        soa_counts = Counter(
+            e.data for e in events if e.type in ("DNS_NAME", "DNS_NAME_UNRESOLVED") and str(e.module) == "SOA"
+        )
+
+        # each shared nameserver hostname is referenced by all three in-scope parents
+        # but should still be emitted exactly once
+        assert ns_counts == {
+            "shared-ns1.cloudprovider.test": 1,
+            "shared-ns2.cloudprovider.test": 1,
+        }, (
+            f"Expected each shared NS hostname to be emitted exactly once across parents, "
+            f"got: {dict(ns_counts)}. emit_dns_children dedup is keyed on "
+            f"(parent_host, rdtype, child) instead of (rdtype, child)."
+        )
+        assert soa_counts == {"shared-ns1.cloudprovider.test": 1}, (
+            f"Expected the shared SOA hostname to be emitted exactly once across parents, got: {dict(soa_counts)}."
+        )
+
+
 class TestDNSResolveFilterPTRsDisabled(ModuleTestBase):
     """Test that PTR-derived hostnames ARE promoted to in-scope when filter_ptrs is disabled."""
 

bbot/test/test_step_2/template_tests/test_template_subdomain_enum.py

@@ -127,8 +127,10 @@ async def mock_query(query):
 
     def check(self, module_test, events):
         assert self.queries == ["walmart.cn"]
-        assert len(events) == 7
-        assert 2 == len(
+        assert len(events) == 6
+        # cross-parent (rdtype, child) dedup in DNSResolve.emit_dns_children collapses
+        # the three walmart.cn->127.0.0.1 edges into a single IP_ADDRESS event
+        assert 1 == len(
             [
                 e
                 for e in events