Merge pull request #168 from mozilla/bhr-stack

Bug 1675103 - Set `payload.hangs[].stack` to string in telemetry.bhr.4

Author: acmiyaguchi

bin/generate_commit

@@ -38,6 +38,10 @@ function _generate_schemas() {
         --mps-branch "$mps_branch_source" \
         --out-dir ./telemetry
 
+    mozilla-schema-generator generate-bhr-ping \
+        --mps-branch "$mps_branch_source" \
+        --out-dir ./telemetry
+
     mozilla-schema-generator generate-common-pings \
         --common-pings-config "$COMMON_PINGS_PATH" \
         --mps-branch "$mps_branch_source" \

mozilla_schema_generator/__main__.py

@@ -12,6 +12,7 @@
 import click
 import yaml
 
+from .bhr_ping import BhrPing
 from .common_ping import CommonPing
 from .config import Config
 from .glean_ping import GleanPing
@@ -88,6 +89,19 @@ def generate_main_ping(config, out_dir, split, pretty, mps_branch):
         config_data = yaml.safe_load(f)
 
     config = Config("main", config_data)
+    schemas = schema_generator.generate_schema(config, split=False)
+    # schemas introduces an extra layer to the actual schema
+    dump_schema(schemas, out_dir, pretty, version=4)
+
+
+@click.command()
+@common_options
+def generate_bhr_ping(out_dir, split, pretty, mps_branch):
+    schema_generator = BhrPing(mps_branch=mps_branch)
+    if out_dir:
+        out_dir = Path(out_dir)
+
+    config = Config("bhr", {})
     schemas = schema_generator.generate_schema(config, split=split)
     dump_schema(schemas, out_dir, pretty, version=4)
 
@@ -186,15 +200,7 @@ def generate_glean_pings(
     config = Config("glean", config_data)
 
     for repo in repos:
-        write_schema(
-            repo,
-            config,
-            out_dir,
-            split,
-            pretty,
-            generic_schema,
-            mps_branch,
-        )
+        write_schema(repo, config, out_dir, split, pretty, generic_schema, mps_branch)
 
 
 def write_schema(repo, config, out_dir, split, pretty, generic_schema, mps_branch):
@@ -242,6 +248,7 @@ def main(args=None):
 
 
 main.add_command(generate_main_ping)
+main.add_command(generate_bhr_ping)
 main.add_command(generate_glean_pings)
 main.add_command(generate_common_pings)
 

mozilla_schema_generator/bhr_ping.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from .common_ping import CommonPing
+from .utils import prepend_properties
+
+
+class BhrPing(CommonPing):
+    schema_url = (
+        "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas"
+        "/{branch}/schemas/telemetry/bhr/bhr.4.schema.json"
+    )
+
+    def __init__(self, **kwargs):
+        super().__init__(self.schema_url, **kwargs)
+
+    def _update_env(self, schema):
+        # hangs is an array of objects
+        stack = prepend_properties(("payload", "hangs")) + (
+            "items",
+            "properties",
+            "stack",
+        )
+        schema.set_schema_elem(
+            stack,
+            {
+                "type": "string",
+                "description": (
+                    "JSON representation of the stack field."
+                    " Injected by mozilla-schema-generator."
+                ),
+            },
+            # this may otherwise overwrite the "items" fields
+            propagate=False,
+        )
+
+        return super()._update_env(schema)

mozilla_schema_generator/common_ping.py

@@ -41,10 +41,7 @@ def get_schema(self):
     def _update_env(self, schema):
         integer = {"type": "integer"}
         string = {"type": "string"}
-        string_map = {
-            "type": "object",
-            "additionalProperties": string,
-        }
+        string_map = {"type": "object", "additionalProperties": string}
 
         def with_description(dtype: dict, comment: str) -> dict:
             """Add a description to the types defined above."""

mozilla_schema_generator/probes.py

@@ -68,19 +68,11 @@ class MainProbe(Probe):
 
     first_added_key = "first_added"
 
-    histogram_schema = {
-        "type": "string",
-    }
+    histogram_schema = {"type": "string"}
 
     parent_processes = {"main"}
 
-    child_processes = {
-        "content",
-        "gpu",
-        "extension",
-        "dynamic",
-        "socket",
-    }
+    child_processes = {"content", "gpu", "extension", "dynamic", "socket"}
 
     processes_map = {
         "all_childs": child_processes,

setup.py

@@ -33,16 +33,10 @@
         "console_scripts": [
             "mozilla-schema-generator=mozilla_schema_generator.__main__:main",
             "validate-bigquery=mozilla_schema_generator.validate_bigquery:validate",
-        ],
+        ]
     },
     include_package_data=True,
-    install_requires=[
-        "click",
-        "jsonschema",
-        "pyyaml",
-        "requests",
-        "gitpython",
-    ],
+    install_requires=["click", "jsonschema", "pyyaml", "requests", "gitpython"],
     license="MIT",
     zip_safe=False,
     keywords="mozilla-schema-generator",

tests/test_bhr.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozilla_schema_generator.bhr_ping import BhrPing
+from mozilla_schema_generator.config import Config
+
+
+def test_schema_contains_hangs_stacks():
+    schema = BhrPing().generate_schema(Config("bhr", {}))["bhr"][0].schema
+    hangs = schema["properties"]["payload"]["properties"]["hangs"]
+    assert hangs["items"]["properties"]["stack"]["type"] == "string"

tests/test_integration.py

@@ -102,7 +102,7 @@ def test_split_representation(self, schema, env, probes):  # noqa F811
                             "properties": {"test_probe": MainProbe.histogram_schema},
                         },
                     },
-                },
+                }
             ],
             "nested": [
                 {
@@ -231,10 +231,7 @@ def test_max_size(self, schema, env, probes):  # noqa F811
                         {
                             "second_level": False,
                             "details": {"keyed": False},
-                            "versions": {
-                                "first": "50",
-                                "last": "60",
-                            },
+                            "versions": {"first": "50", "last": "60"},
                         }
                     ]
                 },

tests/test_matcher.py

@@ -11,10 +11,7 @@
 class TestMatcher(object):
     def test_matches(self):
         match_obj = {
-            "details": {
-                "record_in_processes": {"contains": "main"},
-                "keyed": True,
-            },
+            "details": {"record_in_processes": {"contains": "main"}, "keyed": True},
             "table_group": "keyed_scalars",
             "type": "scalar",
         }

tests/test_probes.py

@@ -19,19 +19,15 @@ def glean_probe_defn():
                     "first": "2019-04-12 13:44:13",
                     "last": "2019-08-08 15:34:03",
                 },
-                "send_in_pings": [
-                    "metrics",
-                ],
+                "send_in_pings": ["metrics"],
             },
             {
                 "description": "Glean test description",
                 "dates": {
                     "first": "2019-08-08 15:34:14",
                     "last": "2019-08-08 15:45:14",
                 },
-                "send_in_pings": [
-                    "all-pings",
-                ],
+                "send_in_pings": ["all-pings"],
             },
         ],
         "name": "glean.error.invalid_value",
@@ -48,18 +44,14 @@ def glean_probe_defn_subset_pings():
                     "first": "2019-04-12 13:44:13",
                     "last": "2019-08-08 15:34:03",
                 },
-                "send_in_pings": [
-                    "metrics",
-                ],
+                "send_in_pings": ["metrics"],
             },
             {
                 "dates": {
                     "first": "2019-08-08 15:34:14",
                     "last": "2019-08-08 15:45:14",
                 },
-                "send_in_pings": [
-                    "baseline",
-                ],
+                "send_in_pings": ["baseline"],
             },
         ],
         "name": "glean.error.invalid_value",
@@ -138,17 +130,11 @@ def main_probe_defn():
                     "versions": {"first": "65", "last": "68"},
                 },
                 {
-                    "details": {
-                        "keyed": False,
-                        "kind": "string",
-                    },
+                    "details": {"keyed": False, "kind": "string"},
                     "versions": {"first": "61", "last": "64"},
                 },
                 {
-                    "details": {
-                        "keyed": False,
-                        "kind": "string",
-                    },
+                    "details": {"keyed": False, "kind": "string"},
                     "versions": {"first": "55", "last": "60"},
                 },
             ],
@@ -172,7 +158,7 @@ def main_probe_all_childs_defn():
                     },
                     "versions": {"first": "67", "last": "70"},
                 }
-            ],
+            ]
         },
         "name": "a11y.instantiators",
         "type": "scalar",
@@ -201,7 +187,7 @@ def main_probe_all_childs_and_main_defn():
                     },
                     "versions": {"first": "62", "last": "66"},
                 },
-            ],
+            ]
         },
         "name": "a11y.instantiators",
         "type": "scalar",
@@ -223,7 +209,7 @@ def main_probe_all_defn():
                     },
                     "versions": {"first": "67", "last": "70"},
                 }
-            ],
+            ]
         },
         "name": "a11y.instantiators",
         "type": "scalar",
@@ -245,7 +231,7 @@ def main_probe_max_description_length():
                     },
                     "versions": {"first": "67", "last": "70"},
                 }
-            ],
+            ]
         },
         "name": "a11y.instantiators",
         "type": "scalar",