Merge pull request mozilla#647 from badboy/text-metric

Bug 1726327 - Implement the new text metric type

Author: badboy

CHANGELOG.md

@@ -11,6 +11,7 @@
   * Only up to 15 ping submissions every 60 seconds are now allowed.
 * [#658](https://github.com/mozilla/glean.js/pull/658): BUGFIX: Unblock ping uploading jobs after the maximum of upload failures are hit for a given uploading window.
 * [#661](https://github.com/mozilla/glean.js/pull/661): Include unminified version of library on Qt/QML builds.
+* [#647](https://github.com/mozilla/glean.js/pull/647): Implement the Text metric type.
 
 # v0.18.1 (2021-07-22)
 

benchmarks/size/webext/max.js

@@ -11,6 +11,7 @@ import EventMetricType from "@mozilla/glean/webext/private/metrics/event";
 import LabeledMetricType from "@mozilla/glean/webext/private/metrics/labeled";
 import QuantityMetricType from "@mozilla/glean/webext/private/metrics/quantity";
 import StringMetricType from "@mozilla/glean/webext/private/metrics/string";
+import TextMetricType from "@mozilla/glean/webext/private/metrics/text";
 import TimespanMetricType from "@mozilla/glean/webext/private/metrics/timespan";
 import UUIDMetricType from "@mozilla/glean/webext/private/metrics/uuid";
 import URLMetricType from "@mozilla/glean/webext/private/metrics/url";
@@ -28,6 +29,7 @@ console.log(
   JSON.stringify(LabeledMetricType),
   JSON.stringify(QuantityMetricType),
   JSON.stringify(StringMetricType),
+  JSON.stringify(TextMetricType),
   JSON.stringify(TimespanMetricType),
   JSON.stringify(UUIDMetricType),
   JSON.stringify(URLMetricType),

glean/src/cli.ts

@@ -19,7 +19,7 @@ const LOG_TAG = "CLI";
 const VIRTUAL_ENVIRONMENT_DIR = ".venv";
 
 // The version of glean_parser to install from PyPI.
-const GLEAN_PARSER_VERSION = "3.8.0";
+const GLEAN_PARSER_VERSION = "4.0.0";
 
 // This script runs a given Python module as a "main" module, like
 // `python -m module`. However, it first checks that the installed

glean/src/core/metrics/types/text.ts

@@ -0,0 +1,89 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type { CommonMetricData } from "../index.js";
+import { isString, truncateStringAtBoundaryWithError } from "../../utils.js";
+import { MetricType } from "../index.js";
+import { Context } from "../../context.js";
+import { Metric } from "../metric.js";
+
+// The maximum number of characters for text.
+export const TEXT_MAX_LENGTH = 200 * 1024;
+
+export class TextMetric extends Metric<string, string> {
+  constructor(v: unknown) {
+    super(v);
+  }
+
+  /**
+   * Validates that a given value is within bounds.
+   *
+   * @param v The value to validate.
+   * @returns Whether or not v is valid text data.
+   */
+  validate(v: unknown): v is string {
+    if (!isString(v)) {
+      return false;
+    }
+
+    if (v.length > TEXT_MAX_LENGTH) {
+      return false;
+    }
+
+    return true;
+  }
+
+  payload(): string {
+    return this._inner;
+  }
+}
+
+/**
+ * A text metric.
+ */
+class TextMetricType extends MetricType {
+  constructor(meta: CommonMetricData) {
+    super("text", meta);
+  }
+
+  /**
+   * Sets to a specified value.
+   *
+   * @param text the value to set.
+   */
+  set(text: string): void {
+    Context.dispatcher.launch(async () => {
+      if (!this.shouldRecord(Context.uploadEnabled)) {
+        return;
+      }
+
+      const truncatedValue = await truncateStringAtBoundaryWithError(this, text, TEXT_MAX_LENGTH);
+      const metric = new TextMetric(truncatedValue);
+      await Context.metricsDatabase.record(this, metric);
+    });
+  }
+
+  /**
+   * Test-only API.**
+   *
+   * Gets the currently stored value as a string.
+   *
+   * This doesn't clear the stored value.
+   *
+   * TODO: Only allow this function to be called on test mode (depends on Bug 1682771).
+   *
+   * @param ping the ping from which we want to retrieve this metrics value from.
+   *        Defaults to the first value in `sendInPings`.
+   * @returns The value found in storage or `undefined` if nothing was found.
+   */
+  async testGetValue(ping: string = this.sendInPings[0]): Promise<string | undefined> {
+    let metric: string | undefined;
+    await Context.dispatcher.testLaunch(async () => {
+      metric = await Context.metricsDatabase.getMetric<string>(ping, this);
+    });
+    return metric;
+  }
+}
+
+export default TextMetricType;

glean/src/core/metrics/utils.ts

@@ -12,6 +12,7 @@ import { CounterMetric } from "./types/counter.js";
 import { DatetimeMetric } from "./types/datetime.js";
 import { QuantityMetric } from "./types/quantity.js";
 import { StringMetric } from "./types/string.js";
+import { TextMetric } from "./types/text.js";
 import { TimespanMetric } from "./types/timespan.js";
 import { UrlMetric } from "./types/url.js";
 import { UUIDMetric } from "./types/uuid.js";
@@ -30,6 +31,7 @@ const METRIC_MAP: {
   "labeled_string": LabeledMetric,
   "quantity": QuantityMetric,
   "string": StringMetric,
+  "text": TextMetric,
   "timespan": TimespanMetric,
   "url": UrlMetric,
   "uuid": UUIDMetric,

glean/src/index/qt.ts

@@ -17,6 +17,7 @@ import EventMetricType from "../core/metrics/types/event.js";
 import LabeledMetricType from "../core/metrics/types/labeled.js";
 import QuantityMetricType from "../core/metrics/types/quantity.js";
 import StringMetricType from "../core/metrics/types/string.js";
+import TextMetricType from "../core/metrics/types/text.js";
 import TimespanMetricType from "../core/metrics/types/timespan.js";
 import UUIDMetricType from "../core/metrics/types/uuid.js";
 import URLMetricType from "../core/metrics/types/url.js";
@@ -124,6 +125,7 @@ export default {
     QuantityMetricType,
     StringMetricType,
     TimespanMetricType,
+    TextMetricType,
     UUIDMetricType,
     URLMetricType
   }

glean/tests/integration/schema/metrics.yaml

@@ -175,3 +175,18 @@ for_testing:
     expires: never
     send_in_pings:
       - testing
+  text:
+    type: text
+    description: |
+      Sample text metric.
+    bugs:
+      - https://bugzilla.mozilla.org/000000
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=000000#c3
+    notification_emails:
+      - me@mozilla.com
+    expires: never
+    send_in_pings:
+      - testing
+    data_sensitivity:
+      - highly_sensitive

glean/tests/integration/schema/schema.spec.ts

@@ -70,6 +70,7 @@ describe("schema", function() {
     metrics.labeledString["a_label"].set("ho");
     metrics.quantity.set(42);
     metrics.string.set("let's go");
+    metrics.text.set("let's gooooooooo");
     metrics.timespan.setRawNanos(10 * 10**6);
     metrics.uuid.generateAndSet();
     metrics.url.set("glean://test");

glean/tests/unit/core/metrics/text.spec.ts

@@ -0,0 +1,108 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import assert from "assert";
+import { Context } from "../../../../src/core/context";
+import { ErrorType } from "../../../../src/core/error/error_type";
+
+import Glean from "../../../../src/core/glean";
+import { Lifetime } from "../../../../src/core/metrics/lifetime";
+import TextMetricType, { TEXT_MAX_LENGTH } from "../../../../src/core/metrics/types/text";
+
+describe("TextMetric", function() {
+  const testAppId = `gleanjs.test.${this.title}`;
+
+  beforeEach(async function() {
+    await Glean.testResetGlean(testAppId);
+  });
+
+  it("attempting to get the value of a metric that hasn't been recorded doesn't error", async function() {
+    const metric = new TextMetricType({
+      category: "aCategory",
+      name: "aTextMetric",
+      sendInPings: ["aPing", "twoPing", "threePing"],
+      lifetime: Lifetime.Ping,
+      disabled: false
+    });
+
+    assert.strictEqual(await metric.testGetValue("aPing"), undefined);
+  });
+
+  it("attempting to set when glean upload is disabled is a no-op", async function() {
+    Glean.setUploadEnabled(false);
+
+    const metric = new TextMetricType({
+      category: "aCategory",
+      name: "aTextMetric",
+      sendInPings: ["aPing", "twoPing", "threePing"],
+      lifetime: Lifetime.Ping,
+      disabled: false
+    });
+
+    metric.set("some value");
+    assert.strictEqual(await metric.testGetValue("aPing"), undefined);
+  });
+
+  it("ping payload is correct", async function() {
+    const metric = new TextMetricType({
+      category: "aCategory",
+      name: "aTextMetric",
+      sendInPings: ["aPing"],
+      lifetime: Lifetime.Ping,
+      disabled: false
+    });
+
+    const validValues = [
+      "some value",
+      "<html><head><title>Website</title></head><body><h1>Text</h1></body>",
+      "some longer text\nwith newlines\nand also some quotes: \"once upon a time ...\"",
+    ];
+
+    for (const value of validValues) {
+      metric.set(value);
+      assert.strictEqual(await metric.testGetValue("aPing"), value);
+
+      const snapshot = await Context.metricsDatabase.getPingMetrics("aPing", true);
+      assert.deepStrictEqual(snapshot, {
+        "text": {
+          "aCategory.aTextMetric": value
+        }
+      });
+    }
+  });
+
+  it("set properly sets the value in all pings", async function() {
+    const metric = new TextMetricType({
+      category: "aCategory",
+      name: "aTextMetric",
+      sendInPings: ["aPing", "twoPing", "threePing"],
+      lifetime: Lifetime.Ping,
+      disabled: false
+    });
+
+    metric.set("some value");
+    assert.strictEqual(await metric.testGetValue("aPing"), "some value");
+    assert.strictEqual(await metric.testGetValue("twoPing"), "some value");
+    assert.strictEqual(await metric.testGetValue("threePing"), "some value");
+  });
+
+  it("truncates when text exceeds maximum length and records errors", async function () {
+    const metric = new TextMetricType({
+      category: "aCategory",
+      name: "aTextMetric",
+      sendInPings: ["aPing"],
+      lifetime: Lifetime.Ping,
+      disabled: false
+    });
+
+    const testText = `some value ${"a".repeat(TEXT_MAX_LENGTH)}`;
+    metric.set(testText);
+    const truncated = testText.substr(0, TEXT_MAX_LENGTH);
+
+    assert.strictEqual(await metric.testGetValue("aPing"), truncated);
+    assert.strictEqual(
+      await metric.testGetNumRecordedErrors(ErrorType.InvalidOverflow), 1
+    );
+  });
+});

samples/qt-qml-app/requirements.txt

@@ -1,3 +1,3 @@
 PySide2==5.15.2
 shiboken2==5.15.2
-glean_parser==3.8.0
+glean_parser==4.0.0