Dogstatsd magic tags

aggregator.py

@@ -494,7 +494,7 @@ def parse_event_packet(self, packet):
                 elif m[0] == u'#':
                     event['tags'] = sorted(m[1:].split(u','))
             return event
-        except IndexError, ValueError:
+        except (IndexError, ValueError):
             raise Exception(u'Unparseable event packet: %s' % packet)
 
     def submit_packets(self, packets):
@@ -511,7 +511,31 @@ def submit_packets(self, packets):
                 self.count += 1
                 parsed_packets = self.parse_metric_packet(packet)
                 for name, value, mtype, tags, sample_rate in parsed_packets:
-                    self.submit_metric(name, value, mtype, tags=tags, sample_rate=sample_rate)
+                    hostname, device_name, tags = self._extract_magic_tags(tags)
+                    self.submit_metric(name, value, mtype, tags=tags, hostname=hostname,
+                        device_name=device_name, sample_rate=sample_rate)
+
+    def _extract_magic_tags(self, tags):
+        """Magic tags (host, device) override metric hostname and device_name attributes"""
+        hostname = None
+        device_name = None
+        # This implementation avoid list operations for the common case
+        if tags:
+            tags_to_remove = []
+            for tag in tags:
+                if tag.startswith('host:'):
+                    hostname = tag[5:]
+                    tags_to_remove.append(tag)
+                elif tag.startswith('device:'):
+                    device_name = tag[7:]
+                    tags_to_remove.append(tag)
+            if tags_to_remove:
+                # tags is a tuple already sorted, we convert it into a list to pop elements
+                tags = list(tags)
+                for tag in tags_to_remove:
+                    tags.remove(tag)
+                tags = tuple(tags) or None
+        return hostname, device_name, tags
 
     def submit_metric(self, name, value, mtype, tags=None, hostname=None,
                                 device_name=None, timestamp=None, sample_rate=1):
@@ -591,7 +615,8 @@ def submit_metric(self, name, value, mtype, tags=None, hostname=None,
         # Note: if you change the way that context is created, please also change create_empty_metrics,
         #  which counts on this order
 
-        hostname = hostname or self.hostname
+        # Keep hostname with empty string to unset it
+        hostname = hostname if hostname is not None else self.hostname
 
         if tags is None:
             context = (name, tuple(), hostname, device_name)
@@ -712,8 +737,9 @@ def submit_metric(self, name, value, mtype, tags=None, hostname=None,
                                 device_name=None, timestamp=None, sample_rate=1):
         # Avoid calling extra functions to dedupe tags if there are none
 
-        hostname = hostname or self.hostname
-
+        # Keep hostname with empty string to unset it
+        hostname = hostname if hostname is not None else self.hostname
+
         if tags is None:
             context = (name, tuple(), hostname, device_name)
         else:
@@ -781,7 +807,7 @@ def flush(self):
         return metrics
 
 
-def api_formatter(metric, value, timestamp, tags, hostname, device_name=None,
+def api_formatter(metric, value, timestamp, tags, hostname=None, device_name=None,
         metric_type=None, interval=None):
     return {
         'metric': metric,

tests/test_dogstatsd.py

@@ -13,7 +13,7 @@ class TestUnitDogStatsd(unittest.TestCase):
     @staticmethod
     def sort_metrics(metrics):
         def sort_by(m):
-            return (m['metric'], ','.join(m['tags'] or []))
+            return (m['metric'], m['host'], m['device_name'], ','.join(m['tags'] or []))
         return sorted(metrics, key=sort_by)
 
     @staticmethod
@@ -94,8 +94,40 @@ def test_tags(self):
         nt.assert_equal(third['points'][0][1], 16)
         nt.assert_equal(third['host'], 'myhost')
 
+    def test_magic_tags(self):
+        stats = MetricsAggregator('myhost')
+        stats.submit_packets('my.gauge.a:1|c|#host:test-a')
+        stats.submit_packets('my.gauge.b:4|c|#tag1,tag2,host:test-b')
+        stats.submit_packets('my.gauge.b:8|c|#host:test-b,tag2,tag1')
+        stats.submit_packets('my.gauge.c:10|c|#tag3')
+        stats.submit_packets('my.gauge.c:16|c|#device:floppy,tag3')
+
+        metrics = self.sort_metrics(stats.flush())
+
+        nt.assert_equal(len(metrics), 4)
+        first, second, third, fourth = metrics
+
+        nt.assert_equal(first['metric'], 'my.gauge.a')
+        nt.assert_equal(first['tags'], None)
+        nt.assert_equal(first['points'][0][1], 1)
+        nt.assert_equal(first['host'], 'test-a')
+
+        nt.assert_equal(second['metric'], 'my.gauge.b')
+        nt.assert_equal(second['tags'], ('tag1', 'tag2'))
+        nt.assert_equal(second['points'][0][1], 12)
+        nt.assert_equal(second['host'], 'test-b')
+
+        nt.assert_equal(third['metric'], 'my.gauge.c')
+        nt.assert_equal(third['tags'], ('tag3', ))
+        nt.assert_equal(third['points'][0][1], 10)
+        nt.assert_equal(third['device_name'], None)
+
+        nt.assert_equal(fourth['metric'], 'my.gauge.c')
+        nt.assert_equal(fourth['tags'], ('tag3', ))
+        nt.assert_equal(fourth['points'][0][1], 16)
+        nt.assert_equal(fourth['device_name'], 'floppy')
+
     def test_tags_gh442(self):
-        import util
         import dogstatsd
         from aggregator import api_formatter