Bug 1672448 Persist Glean retention_days setting to generated schema (#166)

This translates the `retention_days` setting from Glean repositories.yaml
to `mozPipelineMetadata.expiration_policy.delete_after_days` in all generated
JSON schemas for the application.

We will need a further step in the ops logic to act on this schema-level
metadata, setting the appropriate table-level retention policy in BQ.

Author: jklukas

bin/generate_commit

@@ -38,6 +38,10 @@ function _generate_schemas() {
         --mps-branch "$mps_branch_source" \
         --out-dir ./telemetry
 
+    mozilla-schema-generator generate-bhr-ping \
+        --mps-branch "$mps_branch_source" \
+        --out-dir ./telemetry
+
     mozilla-schema-generator generate-common-pings \
         --common-pings-config "$COMMON_PINGS_PATH" \
         --mps-branch "$mps_branch_source" \

mozilla_schema_generator/__main__.py

@@ -12,6 +12,7 @@
 import click
 import yaml
 
+from .bhr_ping import BhrPing
 from .common_ping import CommonPing
 from .config import Config
 from .glean_ping import GleanPing
@@ -88,6 +89,19 @@ def generate_main_ping(config, out_dir, split, pretty, mps_branch):
         config_data = yaml.safe_load(f)
 
     config = Config("main", config_data)
+    schemas = schema_generator.generate_schema(config, split=False)
+    # schemas introduces an extra layer to the actual schema
+    dump_schema(schemas, out_dir, pretty, version=4)
+
+
+@click.command()
+@common_options
+def generate_bhr_ping(out_dir, split, pretty, mps_branch):
+    schema_generator = BhrPing(mps_branch=mps_branch)
+    if out_dir:
+        out_dir = Path(out_dir)
+
+    config = Config("bhr", {})
     schemas = schema_generator.generate_schema(config, split=split)
     dump_schema(schemas, out_dir, pretty, version=4)
 
@@ -178,34 +192,23 @@ def generate_glean_pings(
     repos = GleanPing.get_repos()
 
     if repo is not None:
-        repos = [(r_name, r_id) for r_name, r_id in repos if r_id == repo]
+        repos = [r for r in repos if r["app_id"] == repo]
 
     with open(config, "r") as f:
         config_data = yaml.safe_load(f)
 
     config = Config("glean", config_data)
 
-    for repo_name, repo_id in repos:
-        write_schema(
-            repo_name,
-            repo_id,
-            config,
-            out_dir,
-            split,
-            pretty,
-            generic_schema,
-            mps_branch,
-        )
+    for repo in repos:
+        write_schema(repo, config, out_dir, split, pretty, generic_schema, mps_branch)
 
 
-def write_schema(
-    repo, repo_id, config, out_dir, split, pretty, generic_schema, mps_branch
-):
-    schema_generator = GleanPing(repo, repo_id, mps_branch=mps_branch)
+def write_schema(repo, config, out_dir, split, pretty, generic_schema, mps_branch):
+    schema_generator = GleanPing(repo, mps_branch=mps_branch)
     schemas = schema_generator.generate_schema(
         config, split=False, generic_schema=generic_schema
     )
-    dump_schema(schemas, out_dir and out_dir.joinpath(repo_id), pretty)
+    dump_schema(schemas, out_dir and out_dir.joinpath(repo["app_id"]), pretty)
 
 
 def dump_schema(schemas, out_dir, pretty, *, version=1):
@@ -245,6 +248,7 @@ def main(args=None):
 
 
 main.add_command(generate_main_ping)
+main.add_command(generate_bhr_ping)
 main.add_command(generate_glean_pings)
 main.add_command(generate_common_pings)
 

mozilla_schema_generator/bhr_ping.py

@@ -0,0 +1,40 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from .common_ping import CommonPing
+from .utils import prepend_properties
+
+
+class BhrPing(CommonPing):
+    schema_url = (
+        "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas"
+        "/{branch}/schemas/telemetry/bhr/bhr.4.schema.json"
+    )
+
+    def __init__(self, **kwargs):
+        super().__init__(self.schema_url, **kwargs)
+
+    def _update_env(self, schema):
+        # hangs is an array of objects
+        stack = prepend_properties(("payload", "hangs")) + (
+            "items",
+            "properties",
+            "stack",
+        )
+        schema.set_schema_elem(
+            stack,
+            {
+                "type": "string",
+                "description": (
+                    "JSON representation of the stack field."
+                    " Injected by mozilla-schema-generator."
+                ),
+            },
+            # this may otherwise overwrite the "items" fields
+            propagate=False,
+        )
+
+        return super()._update_env(schema)

mozilla_schema_generator/glean_ping.py

@@ -39,13 +39,14 @@ class GleanPing(GenericPing):
         "glean_client_info",
     }
 
-    def __init__(self, repo, app_id, **kwargs):  # TODO: Make env-url optional
+    def __init__(self, repo, **kwargs):  # TODO: Make env-url optional
         self.repo = repo
-        self.app_id = app_id
+        self.repo_name = repo["name"]
+        self.app_id = repo["app_id"]
         super().__init__(
             self.schema_url,
             self.schema_url,
-            self.probes_url_template.format(repo),
+            self.probes_url_template.format(self.repo_name),
             **kwargs,
         )
 
@@ -57,10 +58,10 @@ def get_dependencies(self):
         # map those back to the name of the repository in the repository file.
         try:
             dependencies = self._get_json(
-                self.dependencies_url_template.format(self.repo)
+                self.dependencies_url_template.format(self.repo_name)
             )
         except HTTPError:
-            logging.info(f"For {self.repo}, using default Glean dependencies")
+            logging.info(f"For {self.repo_name}, using default Glean dependencies")
             return self.default_dependencies
 
         dependency_library_names = list(dependencies.keys())
@@ -77,10 +78,10 @@ def get_dependencies(self):
                 dependencies.append(repos_by_dependency_name[name])
 
         if len(dependencies) == 0:
-            logging.info(f"For {self.repo}, using default Glean dependencies")
+            logging.info(f"For {self.repo_name}, using default Glean dependencies")
             return self.default_dependencies
 
-        logging.info(f"For {self.repo}, found Glean dependencies: {dependencies}")
+        logging.info(f"For {self.repo_name}, found Glean dependencies: {dependencies}")
         return dependencies
 
     def get_probes(self) -> List[GleanProbe]:
@@ -109,7 +110,7 @@ def get_probes(self) -> List[GleanProbe]:
                 "firefox-android-release",
             }
             if (
-                self.repo in issue_118_affected
+                self.repo_name in issue_118_affected
                 and probe.get_name() == "installation.timestamp"
             ):
                 logging.info(f"Writing column {probe.get_name()} for compatibility.")
@@ -134,7 +135,7 @@ def get_probes(self) -> List[GleanProbe]:
         return processed
 
     def get_pings(self) -> Set[str]:
-        url = self.ping_url_template.format(self.repo)
+        url = self.ping_url_template.format(self.repo_name)
         pings = GleanPing._get_json(url).keys()
 
         for dependency in self.get_dependencies():
@@ -156,14 +157,19 @@ def generate_schema(
             for matcher in matchers.values():
                 matcher.matcher["send_in_pings"]["contains"] = ping
             new_config = Config(ping, matchers=matchers)
+            retention_days = self.repo.get("retention_days", None)
 
-            defaults = {
-                "mozPipelineMetadata": {
-                    "bq_dataset_family": self.app_id.replace("-", "_"),
-                    "bq_table": ping.replace("-", "_") + "_v1",
-                    "bq_metadata_format": "structured",
-                }
+            pipeline_meta = {
+                "bq_dataset_family": self.app_id.replace("-", "_"),
+                "bq_table": ping.replace("-", "_") + "_v1",
+                "bq_metadata_format": "structured",
             }
+            if retention_days is not None:
+                expiration = pipeline_meta.get("expiration_policy", {})
+                expiration["delete_after_days"] = int(retention_days)
+                pipeline_meta["expiration_policy"] = expiration
+
+            defaults = {"mozPipelineMetadata": pipeline_meta}
 
             if generic_schema:  # Use the generic glean ping schema
                 schema = self.get_schema()
@@ -181,11 +187,7 @@ def generate_schema(
     @staticmethod
     def get_repos():
         """
-        Retrieve name and app_id for Glean repositories
+        Retrieve metadata for all non-library Glean repositories
         """
         repos = GleanPing._get_json(GleanPing.repos_url)
-        return [
-            (repo["name"], repo["app_id"])
-            for repo in repos
-            if "library_names" not in repo
-        ]
+        return [repo for repo in repos if "library_names" not in repo]

setup.py

@@ -20,7 +20,7 @@
 setup(
     name="mozilla-schema-generator",
     python_requires=">=3.6.0",
-    version="0.2.0",
+    version="0.3.0",
     description="Create full representations of schemas using the probe info service.",
     long_description=readme,
     long_description_content_type="text/markdown",

tests/test_bhr.py

@@ -0,0 +1,14 @@
+# -*- coding: utf-8 -*-
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from mozilla_schema_generator.bhr_ping import BhrPing
+from mozilla_schema_generator.config import Config
+
+
+def test_schema_contains_hangs_stacks():
+    schema = BhrPing().generate_schema(Config("bhr", {}))["bhr"][0].schema
+    hangs = schema["properties"]["payload"]["properties"]["hangs"]
+    assert hangs["items"]["properties"]["stack"]["type"] == "string"

tests/test_glean.py

@@ -19,7 +19,7 @@
 
 @pytest.fixture
 def glean():
-    return glean_ping.GleanPing("glean", "org-mozilla-glean")
+    return glean_ping.GleanPing({"name": "glean", "app_id": "org-mozilla-glean"})
 
 
 @pytest.fixture
@@ -61,7 +61,8 @@ def test_single_schema(self, glean, config):
 
     def test_get_repos(self):
         repos = glean_ping.GleanPing.get_repos()
-        assert ("fenix", "org-mozilla-fenix") in repos
+        names_ids = [(r["name"], r["app_id"]) for r in repos]
+        assert ("fenix", "org-mozilla-fenix") in names_ids
 
     def test_generic_schema(self, glean, config):
         schemas = glean.generate_schema(config, split=False, generic_schema=True)
@@ -79,7 +80,20 @@ def test_generic_schema(self, glean, config):
 
     def test_missing_data(self, config):
         # When there are no files, this should error
-
-        not_glean = NoProbeGleanPing("LeanGleanPingNoIding", "org-mozilla-lean")
+        repo = {"name": "LeanGleanPingNoIding", "app_id": "org-mozilla-lean"}
+        not_glean = NoProbeGleanPing(repo)
         with pytest.raises(requests.exceptions.HTTPError):
             not_glean.generate_schema(config, split=False)
+
+    def test_retention_days(self, config):
+        glean = glean_ping.GleanPing(
+            {"name": "glean", "app_id": "org-mozilla-glean", "retention_days": 90}
+        )
+        schemas = glean.generate_schema(config, split=False, generic_schema=True)
+
+        final_schemas = {k: schemas[k][0].schema for k in schemas}
+        for name, schema in final_schemas.items():
+            assert (
+                schema["mozPipelineMetadata"]["expiration_policy"]["delete_after_days"]
+                == 90
+            )