Implement PingType and ping maker

Author: brizental

src/constants.ts

@@ -0,0 +1,25 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const { version } = require("../package.json");
+
+export const GLEAN_SCHEMA_VERSION = 1;
+
+// The version for the current build of Glean.js
+export const GLEAN_VERSION = version;
+
+// The name of a "ping" that will include Glean ping_info metrics,
+// such as ping sequence numbers.
+//
+// Note that this is not really a ping,
+// but a neat way to gather metrics in the metrics database.
+export const PING_INFO_STORAGE = "glean_ping_info";
+
+// The name of a "ping" that will include the Glean client_info metrics,
+// such as ping sequence numbers.
+//
+// Note that this is not really a ping,
+// but a neat way to gather metrics in the metrics database.
+export const CLIENT_INFO_STORAGE = "glean_client_info";

src/glean.ts

@@ -17,6 +17,8 @@ class Glean {
   }
   // Whether or not to record metrics.
   private _uploadEnabled: boolean;
+  // The time the Glean object was initialized.
+  private _startTime: Date;
 
   private constructor() {
     if (!isUndefined(Glean._instance)) {
@@ -25,6 +27,7 @@ class Glean {
         Use Glean.instance instead to access the Glean singleton.`);
     }
 
+    this._startTime = new Date();
     this._db = {
       metrics: new MetricsDatabase(),
       pings: new PingsDatabase()
@@ -68,6 +71,15 @@ class Glean {
     Glean.instance._uploadEnabled = value;
   }
 
+  /**
+   * Gets the time this Glean instance was created.
+   *
+   * @returns The Date object representing the time this Glean instance was created.
+   */
+  static get startTime(): Date {
+    return Glean.instance._startTime;
+  }
+
   /**
    * **Test-only API**
    *

src/metrics/types/datetime.ts

@@ -40,6 +40,14 @@ export class DatetimeMetric extends Metric<DatetimeInternalRepresentation, strin
     super(v);
   }
 
+  static fromDate(v: Date, timeUnit: TimeUnit): DatetimeMetric {
+    return new DatetimeMetric({
+      timeUnit,
+      timezone: v.getTimezoneOffset(),
+      date: v.toISOString()
+    });
+  }
+
   /**
    * Gets the datetime data as a Date object.
    *
@@ -198,11 +206,7 @@ class DatetimeMetricType extends MetricType {
       break;
     }
 
-    const metric = new DatetimeMetric({
-      timeUnit: this.timeUnit,
-      timezone: value.getTimezoneOffset(),
-      date: value.toISOString(),
-    });
+    const metric = DatetimeMetric.fromDate(value, this.timeUnit);
     await Glean.metricsDatabase.record(this, metric);
   }
 

src/pings/index.ts

@@ -0,0 +1,65 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { v4 as UUIDv4 } from "uuid";
+
+import collectAndStorePing from "pings/maker";
+import Glean from "glean";
+
+/**
+ * Stores information about a ping.
+ *
+ * This is required so that given metric data queued on disk we can send
+ * pings with the correct settings, e.g. whether it has a client_id.
+ */
+class PingType {
+  /**
+   * Creates a new ping type for the given name,
+   * whether to include the client ID and whether to send this ping empty.
+   *
+   * @param name  The name of the ping.
+   * @param includeClientId Whether to include the client ID in the assembled ping when submitting.
+   * @param sendIfEmtpy Whether the ping should be sent empty or not.
+   * @param reasonCodes The valid reason codes for this ping.
+   */
+  constructor (
+    readonly name: string,
+    readonly includeClientId: boolean,
+    readonly sendIfEmtpy: boolean,
+    readonly reasonCodes: string[]
+  ) {}
+
+  /**
+   * Collects and submits a ping for eventual uploading.
+   *
+   * The ping content is assembled as soon as possible, but upload is not
+   * guaranteed to happen immediately, as that depends on the upload policies.
+   *
+   * If the ping currently contains no content, it will not be sent,
+   * unless it is configured to be sent if empty.
+   *
+   * @param reason The reason the ping was triggered. Included in the
+   *               `ping_info.reason` part of the payload.
+   *
+   * @returns Whether or not the ping was successfully submitted.
+   */
+  async submit(reason?: string): Promise<boolean> {
+    if (!Glean.uploadEnabled) {
+      console.info("Glean disabled: not submitting any pings.");
+      return false;
+    }
+
+    let correctedReason = reason;
+    if (reason && !this.reasonCodes.includes(reason)) {
+      console.error(`Invalid reason code ${reason} from ${this.name}. Ignoring.`);
+      correctedReason = undefined;
+    }
+
+    const identifier = UUIDv4();
+    await collectAndStorePing(identifier, this, correctedReason);
+    return true;
+  }
+}
+
+export default PingType;

src/pings/maker.ts

@@ -0,0 +1,219 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { GLEAN_SCHEMA_VERSION, GLEAN_VERSION, PING_INFO_STORAGE, CLIENT_INFO_STORAGE } from "../constants";
+import CounterMetricType, { CounterMetric } from "metrics/types/counter";
+import DatetimeMetricType, { DatetimeMetric } from "metrics/types/datetime";
+import { Lifetime } from "metrics";
+import TimeUnit from "metrics/time_unit";
+import { ClientInfo, PingInfo, PingHeaders, PingPayload } from "pings/database";
+import PingType from "pings";
+import Glean from "glean";
+
+/**
+ * Gets, and then increments, the sequence number for a given ping.
+ *
+ * @param ping The ping for which we want to get the sequence number.
+ *
+ * @returns The current number (before incrementing).
+ */
+export async function getSequenceNumber(ping: PingType): Promise<number> {
+  const seq = new CounterMetricType({
+    name: `${ping.name}#sequence`,
+    sendInPings: [PING_INFO_STORAGE],
+    lifetime: Lifetime.User,
+    disabled: false
+  });
+
+  const currentSeqData = await Glean.metricsDatabase.getMetric(PING_INFO_STORAGE, seq);
+  await seq.add(1);
+
+  if (currentSeqData) {
+    // Creating a new counter metric validates that the metric stored is actually a number.
+    // When we `add` we deal with getting rid of that number from storage,
+    // no need to worry about that here.
+    try {
+      const metric = new CounterMetric(currentSeqData);
+      return metric.payload();
+    } catch(e) {
+      console.warn(`Unexpected value found for sequence number in ping ${ping.name}. Ignoring.`);
+    }
+  }
+
+  return 0;
+}
+
+/**
+ * Gets the formatted start and end times for this ping
+ * and updates for the next ping.
+ *
+ * @param ping The ping for which we want to get the times.
+ *
+ * @returns An object containing start and times in their payload format.
+ */
+export async function getStartEndTimes(ping: PingType): Promise<{ startTime: string, endTime: string }> {
+  const start = new DatetimeMetricType({
+    name: `${ping.name}#start`,
+    sendInPings: [PING_INFO_STORAGE],
+    lifetime: Lifetime.User,
+    disabled: false
+  }, TimeUnit.Minute);
+
+  // "startTime" is the time the ping was generated the last time.
+  // If not available, we use the date the Glean object was initialized.
+  const startTimeData = await Glean.metricsDatabase.getMetric(PING_INFO_STORAGE, start);
+  let startTime: DatetimeMetric;
+  if (startTimeData) {
+    startTime = new DatetimeMetric(startTimeData);
+  } else {
+    startTime = DatetimeMetric.fromDate(Glean.startTime, TimeUnit.Minute);
+  }
+
+  // Update the start time with the current time.
+  const endTimeData = new Date();
+  await start.set(endTimeData);
+  const endTime = DatetimeMetric.fromDate(endTimeData, TimeUnit.Minute);
+
+  return {
+    startTime: startTime.payload(),
+    endTime: endTime.payload()
+  };
+}
+
+/**
+ * Builds the `ping_info` section of a ping.
+ *
+ * @param ping The ping to build the `ping_info` section for.
+ * @param reason The reason for submitting this ping.
+ *
+ * @returns The final `ping_info` section in its payload format.
+ */
+export async function buildPingInfoSection(ping: PingType, reason?: string): Promise<PingInfo> {
+  const seq = await getSequenceNumber(ping);
+  const { startTime, endTime } = await getStartEndTimes(ping);
+
+  const pingInfo: PingInfo = {
+    seq,
+    start_time: startTime,
+    end_time: endTime
+  };
+
+  if (reason) {
+    pingInfo.reason = reason;
+  }
+
+  return pingInfo;
+}
+
+/**
+ * Builds the `client_info` section of a ping.
+ *
+ * @param ping The ping to build the `client_info` section for.
+ *
+ * @returns The final `client_info` section in its payload format.
+ */
+export async function buildClientInfoSection(ping: PingType): Promise<ClientInfo> {
+  let clientInfo = await Glean.metricsDatabase.getPingMetrics(CLIENT_INFO_STORAGE, true);
+  if (!clientInfo) {
+    // TODO: Watch Bug 1685705 and change behaviour in here accordingly.
+    console.warn("Empty client info data. Will submit anyways.");
+    clientInfo = {};
+  }
+
+  let finalClientInfo: ClientInfo = {
+    "telemetry_sdk_build": GLEAN_VERSION
+  };
+  for (const metricType in clientInfo) {
+    finalClientInfo = { ...finalClientInfo, ...clientInfo[metricType] };
+  }
+
+  if (!ping.includeClientId) {
+    delete finalClientInfo["client_id"];
+  }
+
+  return finalClientInfo;
+}
+
+/**
+ * Gathers all the headers to be included to the final ping request.
+ *
+ * This guarantees that if headers are disabled after the ping collection,
+ * ping submission will still contain the desired headers.
+ *
+ * The current headers gathered here are:
+ * - [X-Debug-Id]
+ * - [X-Source-Tags]
+ *
+ * @returns An object with the headers to include
+ *          or `undefined` if there are no headers to include.
+ */
+export async function getPingHeaders(): Promise<PingHeaders | undefined> {
+  // TODO: Returning nothing for now until Bug 1685718 is resolved.
+  return;
+}
+
+/**
+ * Collects a snapshot for the given ping from storage and attach required meta information.
+ *
+ * @param ping The ping to collect for.
+ * @param reason An optional reason code to include in the ping.
+ *
+ * @returns A fully assembled JSON representation of the ping payload.
+ *          If there is no data stored for the ping, `undefined` is returned.
+ */
+export async function collectPing(ping: PingType, reason?: string): Promise<PingPayload | undefined> {
+  const metricsData = await Glean.metricsDatabase.getPingMetrics(ping.name, true);
+  if (!metricsData && !ping.sendIfEmtpy) {
+    console.info(`Storage for ${ping.name} empty. Bailing out.`);
+    return;
+  } else if (!metricsData) {
+    console.info(`Storage for ${ping.name} empty. Ping will still be sent.`);
+  }
+
+  const metrics = metricsData ? { metrics: metricsData } : {};
+  const pingInfo = await buildPingInfoSection(ping, reason);
+  const clientInfo = await buildClientInfoSection(ping);
+  return {
+    ...metrics,
+    ping_info: pingInfo,
+    client_info: clientInfo,
+  };
+}
+
+/**
+ * Build a pings submition path.
+ *
+ * @param identifier The pings UUID identifier.
+ * @param ping  The ping to build a path for.
+ *
+ * @returns The final submission path.
+ */
+function makePath(identifier: string, ping: PingType): string {
+  return `/submit/${Glean.applicationId}/${ping.name}/${GLEAN_SCHEMA_VERSION}/${identifier}`;
+}
+
+/**
+ * Collects and stores a ping on the pings database.
+ *
+ * @param identifier The pings UUID identifier.
+ * @param ping The ping to submit.
+ * @param reason An optional reason code to include in the ping.
+ *
+ * @returns A promise that is resolved once collection and storing is done.
+ */
+export async function collectAndStorePing(identifier: string, ping: PingType, reason?: string): Promise<void> {
+  const payload = await collectPing(ping, reason);
+  if (!payload) {
+    return;
+  }
+  const headers = await getPingHeaders();
+  return Glean.pingsDatabase.recordPing(
+    makePath(identifier, ping),
+    identifier,
+    payload,
+    headers
+  );
+}
+
+export default collectAndStorePing;