Merge pull request #1834 from Dexterp37/send-beacon

Dexterp37 · web-flow · commit 7977cf3897a7 · 2023-11-30T18:06:28.000+01:00
Bug 1777182 - Add a `sendBeacon` uploader
diff --git a/glean/src/core/glean.ts b/glean/src/core/glean.ts
@@ -268,8 +268,7 @@ namespace Glean {
     Context.applicationId = sanitizeApplicationId(applicationId);
 
     // The configuration constructor will throw in case config has any incorrect prop.
-    const correctConfig = new Configuration(config);
-    Context.config = correctConfig;
+    Context.config = new Configuration(config);
 
     // Pre-set debug options for Glean from browser SessionStorage values.
     setDebugOptionsFromSessionStorage();
@@ -283,7 +282,7 @@ namespace Glean {
     Context.pingsDatabase = new PingsDatabase();
     Context.errorManager = new ErrorManager();
 
-    pingUploader = new PingUploadManager(correctConfig, Context.pingsDatabase);
+    pingUploader = new PingUploadManager(Context.config, Context.pingsDatabase);
 
     Context.initialized = true;
 
diff --git a/glean/src/core/upload/ping_body_overflow_error.ts b/glean/src/core/upload/ping_body_overflow_error.ts
@@ -0,0 +1,11 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Error to be thrown in case the final ping body is larger than MAX_PING_BODY_SIZE.
+export class PingBodyOverflowError extends Error {
+  constructor(message?: string) {
+    super(message);
+    this.name = "PingBodyOverflow";
+  }
+}
diff --git a/glean/src/core/upload/ping_request.ts b/glean/src/core/upload/ping_request.ts
@@ -0,0 +1,50 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { gzipSync, strToU8 } from "fflate";
+import { PingBodyOverflowError } from "./ping_body_overflow_error.js";
+
+/**
+ * Represents a payload to be transmitted by an uploading mechanism.
+ */
+export default class PingRequest<BodyType extends string | Uint8Array> {
+  constructor (
+    readonly identifier: string,
+    readonly headers: Record<string, string>,
+    readonly payload: BodyType,
+    readonly maxBodySize: number
+  ) {}
+
+  asCompressedPayload(): PingRequest<string | Uint8Array> {
+    // If this is already gzipped, do nothing.
+    if (this.headers["Content-Encoding"] == "gzip") {
+      return this;
+    }
+
+    // We prefer using `strToU8` instead of TextEncoder directly,
+    // because it will polyfill TextEncoder if it's not present in the environment.
+    // For example, IE doesn't provide TextEncoder.
+    const encodedBody = strToU8(this.payload as string);
+
+    let finalBody: string | Uint8Array;
+    const headers = this.headers;
+    try {
+      finalBody = gzipSync(encodedBody);
+      headers["Content-Encoding"] = "gzip";
+      headers["Content-Length"] = finalBody.length.toString();
+    } catch {
+      // Fall back to whatever we had.
+      return this;
+    }
+
+    if (finalBody.length > this.maxBodySize) {
+      throw new PingBodyOverflowError(
+        `Body for ping ${this.identifier} exceeds ${this.maxBodySize} bytes. Discarding.`
+      );
+    }
+
+    return new PingRequest<string | Uint8Array>(this.identifier, headers, finalBody, this.maxBodySize);
+  }
+}
+
diff --git a/glean/src/core/upload/uploader.ts b/glean/src/core/upload/uploader.ts
@@ -2,6 +2,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import type PingRequest from "./ping_request.js";
+
 // The timeout, in milliseconds, to use for all operations with the server.
 export const DEFAULT_UPLOAD_TIMEOUT_MS = 10_000;
 
@@ -51,7 +53,14 @@ export abstract class Uploader {
    * @param headers Optional headers to include in the request
    * @returns The status code of the response.
    */
-  abstract post(url: string, body: string | Uint8Array, headers?: Record<string, string>): Promise<UploadResult>;
+  abstract post(url: string, pingRequest: PingRequest<string | Uint8Array>): Promise<UploadResult>;
+
+  /**
+   * Whether or not this uploader supports submitting custom headers.
+   *
+   * @returns whether or not custom headers are supported.
+   */
+  abstract supportsCustomHeaders(): boolean;
 }
 
 export default Uploader;
diff --git a/glean/src/core/upload/worker.ts b/glean/src/core/upload/worker.ts
@@ -2,8 +2,6 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-import { gzipSync, strToU8 } from "fflate";
-
 import type { QueuedPing } from "./manager.js";
 import type Uploader from "./uploader.js";
 import type { UploadTask } from "./task.js";
@@ -14,17 +12,11 @@ import Policy from "./policy.js";
 import { UploadResult, UploadResultStatus } from "./uploader.js";
 import { UploadTaskTypes } from "./task.js";
 import { GLEAN_VERSION } from "../constants.js";
+import { PingBodyOverflowError } from "./ping_body_overflow_error.js";
+import PingRequest from "./ping_request.js";
 
 const PING_UPLOAD_WORKER_LOG_TAG = "core.Upload.PingUploadWorker";
 
-// Error to be thrown in case the final ping body is larger than MAX_PING_BODY_SIZE.
-class PingBodyOverflowError extends Error {
-  constructor(message?: string) {
-    super(message);
-    this.name = "PingBodyOverflow";
-  }
-}
-
 class PingUploadWorker {
   // Whether or not someone is blocking on the currentJob.
   isBlocking = false;
@@ -47,10 +39,7 @@ class PingUploadWorker {
    * @param ping The ping to include the headers in.
    * @returns The updated ping.
    */
-  private buildPingRequest(ping: QueuedPing): {
-    headers: Record<string, string>;
-    payload: string | Uint8Array;
-  } {
+  private buildPingRequest(ping: QueuedPing): PingRequest<string | Uint8Array> {
     let headers = ping.headers || {};
     headers = {
       ...ping.headers,
@@ -60,33 +49,15 @@ class PingUploadWorker {
     };
 
     const stringifiedBody = JSON.stringify(ping.payload);
-    // We prefer using `strToU8` instead of TextEncoder directly,
-    // because it will polyfill TextEncoder if it's not present in the environment.
-    // Environments that don't provide TextEncoder are IE and most importantly QML.
-    const encodedBody = strToU8(stringifiedBody);
-
-    let finalBody: string | Uint8Array;
-    let bodySizeInBytes: number;
-    try {
-      finalBody = gzipSync(encodedBody);
-      bodySizeInBytes = finalBody.length;
-      headers["Content-Encoding"] = "gzip";
-    } catch {
-      finalBody = stringifiedBody;
-      bodySizeInBytes = encodedBody.length;
-    }
 
-    if (bodySizeInBytes > this.policy.maxPingBodySize) {
+    if (stringifiedBody.length > this.policy.maxPingBodySize) {
       throw new PingBodyOverflowError(
         `Body for ping ${ping.identifier} exceeds ${this.policy.maxPingBodySize}bytes. Discarding.`
       );
     }
 
-    headers["Content-Length"] = bodySizeInBytes.toString();
-    return {
-      headers,
-      payload: finalBody
-    };
+    headers["Content-Length"] = stringifiedBody.length.toString();
+    return new PingRequest(ping.identifier, headers, stringifiedBody, this.policy.maxPingBodySize);
   }
 
   /**
@@ -99,12 +70,24 @@ class PingUploadWorker {
     try {
       const finalPing = this.buildPingRequest(ping);
 
+      let safeUploader = this.uploader;
+      if (!this.uploader.supportsCustomHeaders()) {
+        // Some options require us to submit custom headers. Unfortunately not all the
+        // uploaders support them (e.g. `sendBeacon`). In case headers are required, switch
+        // back to the default uploader that, for now, supports headers.
+        const needsHeaders = !(
+          (Context.config.sourceTags === undefined) && (Context.config.debugViewTag === undefined)
+        );
+        if (needsHeaders) {
+          safeUploader = Context.platform.uploader;
+        }
+      }
+
       // The POST call has to be asynchronous. Once the API call is triggered,
       // we rely on the browser's "keepalive" header.
-      return this.uploader.post(
+      return safeUploader.post(
         `${this.serverEndpoint}${ping.path}`,
-        finalPing.payload,
-        finalPing.headers
+        finalPing
       );
     } catch (e) {
       log(
diff --git a/glean/src/entry/web.ts b/glean/src/entry/web.ts
@@ -6,4 +6,5 @@ import platform from "../platform/browser/web/index.js";
 import { base } from "./base.js";
 
 export { default as Uploader, UploadResult, UploadResultStatus } from "../core/upload/uploader.js";
+export {default as BrowserSendBeaconUploader} from "../platform/browser/sendbeacon_uploader.js";
 export default base(platform);
diff --git a/glean/src/platform/browser/fetch_uploader.ts b/glean/src/platform/browser/fetch_uploader.ts
@@ -2,30 +2,34 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import type PingRequest from "../../core/upload/ping_request.js";
+
 import log, { LoggingLevel } from "../../core/log.js";
 import Uploader from "../../core/upload/uploader.js";
 import { DEFAULT_UPLOAD_TIMEOUT_MS, UploadResultStatus, UploadResult } from "../../core/upload/uploader.js";
 
-const LOG_TAG = "platform.browser.Uploader";
+const LOG_TAG = "platform.browser.FetchUploader";
 
-class BrowserUploader extends Uploader {
+class BrowserFetchUploader extends Uploader {
   timeoutMs: number = DEFAULT_UPLOAD_TIMEOUT_MS;
 
   async post(
     url: string,
-    body: string | Uint8Array,
-    headers: Record<string, string> = {},
+    pingRequest: PingRequest<string | Uint8Array>,
     keepalive = true
   ): Promise<UploadResult> {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
 
+    // We expect to have a gzipped payload.
+    const gzipRequest = pingRequest.asCompressedPayload();
+
     let response;
     try {
       response = await fetch(url.toString(), {
-        headers,
+        headers: gzipRequest.headers,
         method: "POST",
-        body: body,
+        body: gzipRequest.payload,
         keepalive,
         // Strips any cookies or authorization headers from the request.
         credentials: "omit",
@@ -42,7 +46,7 @@ class BrowserUploader extends Uploader {
           // Try again without `keepalive`, because that may be the issue.
           // This problem was observed in chromium versions below v81.
           // See: https://chromium.googlesource.com/chromium/src/+/26d70b36dd1c18244fb17b91d275332c8b73eab3
-          return this.post(url, body, headers, false);
+          return this.post(url, gzipRequest, false);
         }
 
         // From MDN: "A fetch() promise will reject with a TypeError
@@ -63,6 +67,10 @@ class BrowserUploader extends Uploader {
     clearTimeout(timeout);
     return new UploadResult(UploadResultStatus.Success, response.status);
   }
+
+  supportsCustomHeaders(): boolean {
+    return true;
+  }
 }
 
-export default new BrowserUploader();
+export default new BrowserFetchUploader();
diff --git a/glean/src/platform/browser/sendbeacon_uploader.ts b/glean/src/platform/browser/sendbeacon_uploader.ts
@@ -0,0 +1,42 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import type PingRequest from "../../core/upload/ping_request.js";
+
+import log, { LoggingLevel } from "../../core/log.js";
+import Uploader from "../../core/upload/uploader.js";
+import { DEFAULT_UPLOAD_TIMEOUT_MS, UploadResultStatus, UploadResult } from "../../core/upload/uploader.js";
+
+const LOG_TAG = "platform.browser.SendBeaconUploader";
+
+class BrowserSendBeaconUploader extends Uploader {
+  timeoutMs: number = DEFAULT_UPLOAD_TIMEOUT_MS;
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async post(
+    url: string,
+    pingRequest: PingRequest<string | Uint8Array>
+  ): Promise<UploadResult> {
+    // While the most appropriate type would be "application/json",
+    // using that would cause to send CORS preflight requests. We
+    // instead send the content as plain text and rely on the backend
+    // to do the appropriate validation/parsing.
+    const wasQueued = navigator.sendBeacon(url, pingRequest.payload);
+    if (wasQueued) {
+      // Unfortunately we don't know much more other than data was enqueued,
+      // it is the agent's responsibility to manage the submission. The only
+      // thing we can do is remove this from our internal queue.
+      return new UploadResult(UploadResultStatus.Success, 200);
+    }
+    log(LOG_TAG, "The `sendBeacon` call was not serviced by the browser. Deleting data.", LoggingLevel.Error);
+    // If the agent says there's a problem, there's not so much we can do.
+    return new UploadResult(UploadResultStatus.UnrecoverableFailure);
+  }
+
+  supportsCustomHeaders(): boolean {
+    return false;
+  }
+}
+
+export default new BrowserSendBeaconUploader();
diff --git a/glean/src/platform/browser/web/index.ts b/glean/src/platform/browser/web/index.ts
@@ -4,7 +4,7 @@
 
 import type Platform from "../../index.js";
 
-import uploader from "../uploader.js";
+import uploader from "../fetch_uploader.js";
 import info from "./platform_info.js";
 import Storage from "./storage.js";
 
diff --git a/glean/src/platform/test/index.ts b/glean/src/platform/test/index.ts
@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import type PingRequest from "../../core/upload/ping_request.js";
 import type Platform from "../index.js";
 import type PlatformInfo from "../../core/platform_info.js";
 
@@ -12,10 +13,14 @@ import { UploadResultStatus, UploadResult } from "../../core/upload/uploader.js"
 
 class MockUploader extends Uploader {
   // eslint-disable-next-line @typescript-eslint/no-unused-vars
-  post(_url: string, _body: string | Uint8Array, _headers?: Record<string, string>): Promise<UploadResult> {
+  post(_url: string, _pingRequest: PingRequest<string | Uint8Array>): Promise<UploadResult> {
     const result = new UploadResult(UploadResultStatus.Success, 200);
     return Promise.resolve(result);
   }
+
+  supportsCustomHeaders(): boolean {
+    return true;
+  }
 }
 
 const MockPlatformInfo: PlatformInfo = {
diff --git a/glean/tests/unit/core/upload/worker.spec.ts b/glean/tests/unit/core/upload/worker.spec.ts
@@ -130,7 +130,7 @@ describe("PingUploadWorker", function () {
     const url = postSpy.firstCall.args[0].split("/");
     const appId = url[url.length - 4];
     const documentId = url[url.length - 1];
-    const headers = postSpy.firstCall.args[2] || {};
+    const headers = postSpy.firstCall.args[1].headers || {};
 
     assert.strictEqual(documentId, MOCK_PING_IDENTIFIER);
     assert.strictEqual(appId, Context.applicationId);
@@ -139,7 +139,6 @@ describe("PingUploadWorker", function () {
     assert.ok("Content-Length" in headers);
     assert.ok("Content-Type" in headers);
     assert.ok("X-Telemetry-Agent" in headers);
-    assert.strictEqual(headers["Content-Encoding"], "gzip");
   });
 
   it("successful upload attempts to return the correct upload result", function () {
diff --git a/glean/tests/unit/platform/browser/fetch_uploader.spec.ts b/glean/tests/unit/platform/browser/fetch_uploader.spec.ts
diff --git a/glean/tests/unit/platform/browser/sendbeacon_uploader.spec.ts b/glean/tests/unit/platform/browser/sendbeacon_uploader.spec.ts
diff --git a/glean/tests/utils.ts b/glean/tests/utils.ts
