Merge pull request #415 from brizental/1712080-gzip

Bug 1712080 - Gzip ping paylod before upload

Author: Beatriz Rizental

CHANGELOG.md

@@ -5,6 +5,9 @@
 * [#411](https://github.com/mozilla/glean.js/pull/411): Tag all messages logged by Glean with the component they are coming from.
 * [#399](https://github.com/mozilla/glean.js/pull/399): Check if there are ping data before attempting to delete it.
   * This change lowers the amount of log messages related to attempting to delete inexistent data.
+* [#415](https://github.com/mozilla/glean.js/pull/415): Gzip ping paylod before upload
+  * This changes the signature of `Uploader.post` to accept `string | Uint8Array` for the `body` parameter, instead of only `string`.
+  * The gizpping functionality is disabled in Qt/QML environments. Follow [Bug 1716322](https://bugzilla.mozilla.org/show_bug.cgi?id=1716322) for updates on the status of this feature on that platform.
 
 # v0.15.0 (2021-06-03)
 

glean/package-lock.json

@@ -3,6 +3,7 @@
   "version": "0.15.0",
   "description": "An implementation of the Glean SDK, a modern cross-platform telemetry client, for JavaScript environments.",
   "type": "module",
+  "sideEffects": "false",
   "exports": {
     "./package.json": "./package.json",
     "./webext": "./dist/webext/index/webext.js",
@@ -51,7 +52,7 @@
     "build:webext:lib": "tsc -p ./tsconfig/webext/index.json",
     "build:webext:types": "tsc -p ./tsconfig/webext/types.json",
     "build:webext": "rm -rf dist/webext && run-s build:webext:lib build:webext:types",
-    "build:qt": "rm -rf dist/qt && webpack --config webpack.config.qt.js --mode production && ../bin/prepare-qml-module.sh",
+    "build:qt": "rm -rf dist/qt && webpack --config webpack.config.qt.js && ../bin/prepare-qml-module.sh",
     "build:docs": "rm -rf dist/docs && typedoc src/ --out dist/docs --tsconfig tsconfig/docs.json --theme minimal",
     "publish:docs": "NODE_DEBUG=gh-pages gh-pages --dotfiles --message \"[skip ci] Updates\" --dist dist/docs",
     "prepublishOnly": "cp ../README.md ./README.md && run-s build:cli build:webext",
@@ -106,6 +107,7 @@
     "webpack-cli": "^4.5.0"
   },
   "dependencies": {
+    "fflate": "^0.7.0",
     "jose": "^3.7.0",
     "uuid": "^8.3.2"
   },

glean/package.json

@@ -2,6 +2,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import { gzipSync } from "fflate";
+
 import type Platform from "../../platform/index.js";
 import type { Configuration } from "../config.js";
 import { GLEAN_VERSION } from "../constants.js";
@@ -124,25 +126,35 @@ class PingUploader implements PingsDatabaseObserver {
    */
   private async preparePingForUpload(ping: QueuedPing): Promise<{
     headers: Record<string, string>,
-    payload: string
+    payload: string | Uint8Array
   }> {
-    const stringifiedBody = JSON.stringify(ping.payload);
-
     let headers = ping.headers || {};
     headers = {
       ...ping.headers,
       "Content-Type": "application/json; charset=utf-8",
-      "Content-Length": stringifiedBody.length.toString(),
       "Date": (new Date()).toISOString(),
       "X-Client-Type": "Glean.js",
       "X-Client-Version": GLEAN_VERSION,
       "X-Telemetry-Agent": `Glean/${GLEAN_VERSION} (JS on ${await this.platformInfo.os()})`
     };
 
-    return {
-      headers,
-      payload: stringifiedBody
-    };
+    const stringifiedBody = JSON.stringify(ping.payload);
+    try {
+      const encoder = new TextEncoder();
+      const compressedBody = gzipSync(encoder.encode(stringifiedBody));
+      headers["Content-Encoding"] = "gzip";
+      headers["Content-Length"] = compressedBody.length.toString();
+      return {
+        headers,
+        payload: compressedBody
+      };
+    } catch {
+      headers["Content-Length"] = stringifiedBody.length.toString();
+      return {
+        headers,
+        payload: stringifiedBody
+      };
+    }
   }
 
   /**

glean/src/core/upload/index.ts

@@ -43,11 +43,13 @@ export interface Uploader {
    * Makes a POST request to a given url, with the given headers and body.
    *
    * @param url The URL to make the POST request
-   * @param body The stringified body of this post request
+   * @param body The body of this post request. The body may be a stringified JSON or, most likely,
+   *        a Uint8Array containing the gzipped version of said stringified JSON. We need to accept
+   *        both in case the compression fails.
    * @param headers Optional header to include in the request
    * @returns The status code of the response.
    */
-  post(url: string, body: string, headers?: Record<string, string>): Promise<UploadResult>;
+  post(url: string, body: string | Uint8Array, headers?: Record<string, string>): Promise<UploadResult>;
 }
 
 export default Uploader;

glean/src/core/upload/uploader.ts

@@ -0,0 +1,17 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file is a drop in replacement for `fflate` in QML.
+ * This library raises errors in QML. https://github.com/101arrowz/fflate/issues/71
+ * could be the the culprit, but it is unclear at this point.
+ */
+
+// eslint-disable-next-line jsdoc/require-jsdoc
+export function gzipSync() {
+  // We throw here because when the gzipping action throws the ping upload manager will
+  // catch and send the uncompressed ping, which is what we want on QML for the time being.
+  // We are trying to figure out how to actually add the gzipping step to QML on Bug 1716322.
+  throw new Error("Attempted to use `gzipSync` from QML, but that is not supported.");
+}

glean/src/platform/qt/fflate.stub.js

@@ -12,7 +12,7 @@ import type Platform from "../index.js";
 
 class MockUploader implements Uploader {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  post(_url: string, _body: string, _headers?: Record<string, string>): Promise<UploadResult> {
+  post(_url: string, _body: string | Uint8Array, _headers?: Record<string, string>): Promise<UploadResult> {
     const result: UploadResult = {
       result: UploadResultStatus.Success,
       status: 200

glean/src/platform/test/index.ts

@@ -10,7 +10,7 @@ import { DEFAULT_UPLOAD_TIMEOUT_MS, UploadResultStatus } from "../../core/upload
 const LOG_TAG = "platform.webext.Uploader";
 
 class BrowserUploader implements Uploader {
-  async post(url: string, body: string, headers: Record<string, string> = {}): Promise<UploadResult> {
+  async post(url: string, body: string | Uint8Array, headers: Record<string, string> = {}): Promise<UploadResult> {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), DEFAULT_UPLOAD_TIMEOUT_MS);
 

glean/src/platform/webext/uploader.ts

@@ -14,6 +14,7 @@ import { UploadResultStatus } from "../../../src/core/upload/uploader";
 // Generated files.
 import * as metrics from "./generated/forTesting";
 import * as pings from "./generated/pings";
+import { unzipPingPayload } from "../../utils";
 
 const GLEAN_SCHEMA_URL = "https://raw.githubusercontent.com/mozilla-services/mozilla-pipeline-schemas/main/schemas/glean/glean/glean.1.schema.json";
 
@@ -45,7 +46,7 @@ function fetchSchema(): Promise<JSONObject> {
  */
 class WaitableHttpClient implements Uploader {
   private waitingFor?: string;
-  private waitResolver?: (pingBody: string) => void;
+  private waitResolver?: (pingBody: JSONObject) => void;
 
   /**
    * Returns a promise that resolves once a ping is submitted or times out after a 2s wait.
@@ -56,12 +57,11 @@ class WaitableHttpClient implements Uploader {
   waitForPingSubmission(name: string): Promise<JSONObject> {
     this.waitingFor = name;
     return new Promise<JSONObject>((resolve, reject) => {
-      this.waitResolver = (pingBody: string) => {
+      this.waitResolver = (pingBody: JSONObject) => {
         this.waitingFor = undefined;
-        const parsedBody = JSON.parse(pingBody) as JSONObject;
         // Uncomment for debugging the ping payload.
-        // console.log(JSON.stringify(parsedBody, null, 2));
-        resolve(parsedBody);
+        // console.log(JSON.stringify(pingBody, null, 2));
+        resolve(pingBody);
       };
 
       setTimeout(() => reject(), 2000);
@@ -70,7 +70,7 @@ class WaitableHttpClient implements Uploader {
 
   post(url: string, body: string): Promise<UploadResult> {
     if (this.waitingFor && url.includes(this.waitingFor)) {
-      this.waitResolver?.(body);
+      this.waitResolver?.(unzipPingPayload(body));
     }
 
     return Promise.resolve({

glean/tests/integration/schema/schema.spec.ts

@@ -20,6 +20,7 @@ import { UploadResultStatus } from "../../../src/core/upload/uploader";
 import CounterMetricType from "../../../src/core/metrics/types/counter";
 import { Lifetime } from "../../../src/core/metrics/lifetime";
 import { Context } from "../../../src/core/context";
+import { unzipPingPayload } from "../../utils";
 
 const sandbox = sinon.createSandbox();
 
@@ -60,13 +61,13 @@ describe("PingEncryptionPlugin", function() {
 
     const postSpy = sandbox.spy(TestPlatform.uploader, "post").withArgs(
       sinon.match(makePath(Context.applicationId, pingId, ping)),
-      sinon.match.string
+      sinon.match.any
     );
 
     await collectAndStorePing(pingId, ping);
     assert.ok(postSpy.calledOnce);
 
-    const payload = JSON.parse(postSpy.args[0][1]) as JSONObject;
+    const payload = unzipPingPayload(postSpy.args[0][1]);
     assert.ok("payload" in payload);
     assert.ok(typeof payload.payload === "string");
   });
@@ -127,7 +128,7 @@ describe("PingEncryptionPlugin", function() {
 
     const { url, body } = await uploadPromise;
 
-    const parsedBody = JSON.parse(body) as JSONObject;
+    const parsedBody = unzipPingPayload(body);
     const { plaintext, protectedHeader } = await compactDecrypt(
       parsedBody?.payload as string,
       privateKey
@@ -152,3 +153,4 @@ describe("PingEncryptionPlugin", function() {
     assert.ok("typ" in protectedHeader);
   });
 });
+