Merge branch 'release-v0.26.0' into release

brizental · brizental · commit 48395d21b961 · 2021-11-19T11:30:23.000+01:00
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -156,7 +156,7 @@ jobs:
             pip install -r requirements.txt
             glean_parser translate src/App/metrics.yaml src/App/pings.yaml \
               -f javascript -o src/App/generated \
-              --option platform=qt --option version="0.25"
+              --option platform=qt --option version="0.26"
       - run:
           name: Run Qt sample tests
           command: |
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,16 @@
 # Unreleased changes
 
-[Full changelog](https://github.com/mozilla/glean.js/compare/v0.25.0...main)
+[Full changelog](https://github.com/mozilla/glean.js/compare/v0.26.0...main)
+
+# v0.26.0 (2021-11-19)
+
+[Full changelog](https://github.com/mozilla/glean.js/compare/v0.25.0...v0.26.0)
+
+* [#965](https://github.com/mozilla/glean.js/pull/965): Attempt to infer the Python virtualenv folder from the environment before falling back to the default `.venv`.
+  * Users may provide a folder name through the `VIRTUAL_ENV` environment variable.
+  * If the user is inside an active virtualenv the `VIRTUAL_ENV` environment variable is already set by Python. See: https://docs.python.org/3/library/venv.html.
+* [#968](https://github.com/mozilla/glean.js/pull/968): Add runtime arguments type checking to `Glean.setUploadEnabled` API.
+* [#970](https://github.com/mozilla/glean.js/pull/970): BUGFIX: Guarantee uploading is immediatelly resumed if the uploader has been stopped due to any of the uploading limits being hit.
 
 # v0.25.0 (2021-11-15)
 
diff --git a/glean/package-lock.json b/glean/package-lock.json
diff --git a/glean/package.json b/glean/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@mozilla/glean",
-  "version": "0.25.0",
+  "version": "0.26.0",
   "description": "An implementation of the Glean SDK, a modern cross-platform telemetry client, for JavaScript environments.",
   "type": "module",
   "sideEffects": "false",
diff --git a/glean/src/cli.ts b/glean/src/cli.ts
@@ -14,9 +14,16 @@ import log, { LoggingLevel } from "./core/log.js";
 
 const LOG_TAG = "CLI";
 
-// The name of the directory which will contain the Python virtual environment
+// The name of the directory which contains / will contain the Python virtual environment
 // used to run the glean-parser.
-const VIRTUAL_ENVIRONMENT_DIR = ".venv";
+//
+// > When a virtual environment is active, the VIRTUAL_ENV environment variable
+// > is set to the path of the virtual environment. This can be used to check if
+// > one is running inside a virtual environment.
+//
+// See: https://docs.python.org/3/library/venv.html
+// (Also applies to envs created using virtualenv though)
+const VIRTUAL_ENVIRONMENT_DIR = process.env.VIRTUAL_ENV?.split("/").slice(-1)[0] || ".venv";
 
 // The version of glean_parser to install from PyPI.
 const GLEAN_PARSER_VERSION = "4.1.1";
diff --git a/glean/src/core/constants.ts b/glean/src/core/constants.ts
@@ -8,7 +8,7 @@ export const GLEAN_SCHEMA_VERSION = 1;
 //
 // PACKAGE_VERSION is defined as a global by webpack,
 // we need a default here for testing when the app is not build with webpack.
-export const GLEAN_VERSION = "0.25.0";
+export const GLEAN_VERSION = "0.26.0";
 
 // The name of a "ping" that will include Glean ping_info metrics,
 // such as ping sequence numbers.
diff --git a/glean/src/core/glean.ts b/glean/src/core/glean.ts
@@ -356,6 +356,15 @@ class Glean {
    * @param flag When true, enable metric collection.
    */
   static setUploadEnabled(flag: boolean): void {
+    if (!isBoolean(flag)) {
+      log(
+        LOG_TAG,
+        "Unable to change upload state, new value must be a boolean. Ignoring.",
+        LoggingLevel.Error
+      );
+      return;
+    }
+
     Context.dispatcher.launch(async () => {
       if (!Context.initialized) {
         log(
diff --git a/glean/src/core/upload/index.ts b/glean/src/core/upload/index.ts
@@ -93,6 +93,8 @@ class PingUploader implements PingsDatabaseObserver {
   private readonly platformInfo: PlatformInfo;
   // The server address we are sending pings to.
   private readonly serverEndpoint: string;
+  // Whether or not uploading is currently stopped due to limits having been hit.
+  private stopped = false;
 
   constructor(
     config: Configuration,
@@ -132,17 +134,21 @@ class PingUploader implements PingsDatabaseObserver {
     const { state: rateLimiterState, remainingTime } = this.rateLimiter.getState();
     if (rateLimiterState === RateLimiterState.Incrementing) {
       this.dispatcher.resume();
+      this.stopped = false;
     } else {
-      // Stop the dispatcher respecting the order of the dispatcher queue,
-      // to make sure the Stop command is enqueued _after_ previously enqueued requests.
-      this.dispatcher.stop(false);
+      if(!this.stopped) {
+        // Stop the dispatcher respecting the order of the dispatcher queue,
+        // to make sure the Stop command is enqueued _after_ previously enqueued requests.
+        this.dispatcher.stop(false);
+        this.stopped = true;
+      }
 
       if (rateLimiterState === RateLimiterState.Throttled) {
         log(
           LOG_TAG,
           [
-            "Attempted to upload a ping, but Glean is currently throttled.",
-            `Pending pings will be processed in ${(remainingTime || 0) / 1000}s.`
+            "Succesfully submitted a ping, but Glean is currently throttled.",
+            `Pending pings may be uploaded in ${(remainingTime || 0) / 1000}s.`
           ],
           LoggingLevel.Debug
         );
@@ -151,9 +157,9 @@ class PingUploader implements PingsDatabaseObserver {
         log(
           LOG_TAG,
           [
-            "Attempted to upload a ping, but Glean has reached maximum recoverable upload failures",
-            "for the current uploading window.",
-            `Will retry in ${(remainingTime || 0) / 1000}s.`
+            "Succesfully submitted a ping,",
+            "but Glean has reached maximum recoverable upload failures for the current uploading window.",
+            `May retry in ${(remainingTime || 0) / 1000}s.`
           ],
           LoggingLevel.Debug
         );
diff --git a/glean/src/metrics.yaml b/glean/src/metrics.yaml
@@ -2,9 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the metrics that are recorded by the Glean SDK.
-# APIs to use these pings are automatically generated at build time using
-# the `glean_parser` PyPI package.
+# This file defines the metrics that are recorded by the Glean JavaScript SDK.
 
 # Metrics in this file may make use of SDK reserved ping names. See
 # https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html
diff --git a/glean/src/pings.yaml b/glean/src/pings.yaml
@@ -2,9 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the pings that are recorded by the Glean SDK.
-# Their code APIs is automatically generated, at build time using,
-# the `glean_parser` PyPI package.
+# This file defines the pings that are recorded by the Glean JavaScript SDK.
 
 ---
 $schema: moz://mozilla.org/schemas/glean/pings/2-0-0
diff --git a/glean/tests/unit/core/glean.spec.ts b/glean/tests/unit/core/glean.spec.ts
@@ -605,4 +605,18 @@ describe("Glean", function() {
     await Glean.testResetGlean(testAppId, true);
     assert.strictEqual(Context.initialized, true);
   });
+
+  it("setUploadEnabled does nothing in case a non-boolean value is passed to it", async function() {
+    // Set the current upload value to false,
+    // strings are "truthy", this way we can be sure calling with the wrong type dod not work.
+    Glean.setUploadEnabled(false);
+    await Context.dispatcher.testBlockOnQueue();
+    assert.strictEqual(Context.uploadEnabled, false);
+
+    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    // @ts-ignore
+    Glean.setUploadEnabled("not a boolean");
+    await Context.dispatcher.testBlockOnQueue();
+    assert.strictEqual(Context.uploadEnabled, false);
+  });
 });
diff --git a/glean/tests/unit/core/upload/index.spec.ts b/glean/tests/unit/core/upload/index.spec.ts
@@ -25,7 +25,10 @@ const sandbox = sinon.createSandbox();
  * @param pingName the name of the ping to fill the database with, defaults to "ping".
  * @returns The array of identifiers of the pings added to the database.
  */
-async function fillUpPingsDatabase(numPings: number, pingName = "ping"): Promise<string[]> {
+async function fillUpPingsDatabase(
+  numPings: number,
+  pingName = "ping"
+): Promise<string[]> {
   const ping = new PingType({
     name: pingName,
     includeClientId: true,
@@ -246,4 +249,22 @@ describe("PingUploader", function() {
     assert.ok("X-Client-Version" in headers);
     assert.ok("X-Telemetry-Agent" in headers);
   });
+
+  it("dispatcher is only once stopped if upload limits are hit and is immediatelly restarted after", async function () {
+    const httpClient = new CounterUploader();
+    await Glean.testResetGlean(testAppId, true, { httpClient });
+
+    const stopSpy = sandbox.spy(Glean["pingUploader"]["dispatcher"], "stop");
+    await fillUpPingsDatabase((MAX_PINGS_PER_INTERVAL * 2) - 1);
+    await Glean["pingUploader"].testBlockOnPingsQueue();
+    assert.strictEqual(1, stopSpy.callCount);
+
+    // Reset the rate limiter so that we are not throttled anymore.
+    Glean["pingUploader"]["rateLimiter"]["reset"]();
+
+    // Add one more ping to the queue so that uploading is resumed.
+    await fillUpPingsDatabase(1);
+    await Glean["pingUploader"].testBlockOnPingsQueue();
+    assert.strictEqual(httpClient.count, MAX_PINGS_PER_INTERVAL * 2);
+  });
 });
diff --git a/glean/tests/unit/plugins/encryption.spec.ts b/glean/tests/unit/plugins/encryption.spec.ts
@@ -12,11 +12,8 @@ import type { JSONObject } from "../../../src/core/utils";
 import { WaitableUploader } from "../../utils";
 import PingEncryptionPlugin from "../../../src/plugins/encryption";
 import collectAndStorePing, { makePath } from "../../../src/core/pings/maker";
-import Uploader from "../../../src/core/upload/uploader";
-import { UploadResultStatus, UploadResult } from "../../../src/core/upload/uploader";
 import CounterMetricType from "../../../src/core/metrics/types/counter";
 import { Lifetime } from "../../../src/core/metrics/lifetime";
-import { unzipPingPayload } from "../../utils";
 
 const sandbox = sinon.createSandbox();
 
@@ -69,37 +66,6 @@ describe("PingEncryptionPlugin", function() {
   it("decrypting encrypted ping returns expected payload and protected headers", async function () {
     const { publicKey, privateKey } = await generateKeyPair("ECDH-ES");
 
-    // If we could use `Promise.defer()` (not cross-browser and deprecated), the following
-    // block of code would not be needed. We're basically replicating the deferred promise
-    // because we need to wait for the `post` method to be called in order to evaluate
-    // the arguments. Doing this with `sinon` alone would require us to use the `Glean.dispatcher`
-    // internals, which would defeat the point of having a custom uploader.
-    type UploadSignature = {url: string, body: string, headers?: Record<string, string>};
-    let uploadPromiseResolve: (value: UploadSignature) => void;
-    const uploadPromise = new Promise<UploadSignature>(r => uploadPromiseResolve = r);
-
-    class MockUploader extends Uploader {
-      post(url: string, body: string, headers?: Record<string, string>): Promise<UploadResult> {
-        uploadPromiseResolve({url, body, headers});
-        const result = new UploadResult(UploadResultStatus.Success, 200);
-        return Promise.resolve(result);
-      }
-    }
-
-    await Glean.testResetGlean(
-      testAppId,
-      true,
-      {
-        plugins: [
-          new PingEncryptionPlugin(await exportJWK(publicKey))
-        ],
-        debug: {
-          logPings: true
-        },
-        httpClient: new MockUploader(),
-      },
-    );
-
     const ping = new PingType({
       name: "encryptedping",
       includeClientId: true,
@@ -114,24 +80,34 @@ describe("PingEncryptionPlugin", function() {
       disabled: false
     });
 
+    const httpClient = new WaitableUploader();
+    const pingBody = httpClient.waitForPingSubmission("encryptedping");
+    await Glean.testResetGlean(
+      testAppId,
+      true,
+      {
+        plugins: [
+          new PingEncryptionPlugin(await exportJWK(publicKey))
+        ],
+        debug: {
+          logPings: true
+        },
+        httpClient,
+      },
+    );
+
     counter.add(37);
     ping.submit();
 
-    const { url, body } = await uploadPromise;
-
-    const parsedBody = unzipPingPayload(body);
     const { plaintext, protectedHeader } = await compactDecrypt(
-      parsedBody?.payload as string,
+      (await pingBody).payload as string,
       privateKey
     );
 
     const decoder = new TextDecoder();
     const decodedPayload = decoder.decode(plaintext);
     const decodedPayloadObject = JSON.parse(decodedPayload) as JSONObject;
 
-    // Check that this is the ping we were looking for.
-    assert.ok(url.includes("encryptedping"));
-
     // Do basic sanity checking on the ping payload.
     assert.notStrictEqual(decodedPayloadObject["client_info"], undefined);
     assert.notStrictEqual(decodedPayloadObject["ping_info"], undefined);
diff --git a/samples/browser/web/src/metrics.yaml b/samples/browser/web/src/metrics.yaml
@@ -2,14 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the metrics that are recorded by the Glean SDK.
-# APIs to use these pings are automatically generated at build time using
-# the `glean_parser` PyPI package.
-
-# Metrics in this file may make use of SDK reserved ping names. See
-# https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html
-# for additional information.
-
 ---
 $schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
 
diff --git a/samples/browser/web/src/pings.yaml b/samples/browser/web/src/pings.yaml
@@ -2,10 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the pings that are recorded by the Glean SDK.
-# Their code APIs is automatically generated, at build time using,
-# the `glean_parser` PyPI package.
-
 ---
 $schema: moz://mozilla.org/schemas/glean/pings/2-0-0
 
diff --git a/samples/node/metrics.yaml b/samples/node/metrics.yaml
@@ -2,14 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the metrics that are recorded by the Glean SDK.
-# APIs to use these pings are automatically generated at build time using
-# the `glean_parser` PyPI package.
-
-# Metrics in this file may make use of SDK reserved ping names. See
-# https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html
-# for additional information.
-
 ---
 $schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
 
diff --git a/samples/node/pings.yaml b/samples/node/pings.yaml
@@ -2,10 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the pings that are recorded by the Glean SDK.
-# Their code APIs is automatically generated, at build time using,
-# the `glean_parser` PyPI package.
-
 ---
 $schema: moz://mozilla.org/schemas/glean/pings/2-0-0
 
diff --git a/samples/qt/src/App/App.qml b/samples/qt/src/App/App.qml
@@ -5,8 +5,8 @@
 import QtQuick 2.0
 import QtQuick.Controls 2.0
 
-import org.mozilla.Glean 0.25
-import generated 0.25
+import org.mozilla.Glean 0.26
+import generated 0.26
 
 Rectangle {
   id: screen
diff --git a/samples/qt/src/App/metrics.yaml b/samples/qt/src/App/metrics.yaml
@@ -2,14 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the metrics that are recorded by the Glean SDK.
-# APIs to use these pings are automatically generated at build time using
-# the `glean_parser` PyPI package.
-
-# Metrics in this file may make use of SDK reserved ping names. See
-# https://mozilla.github.io/glean/book/dev/core/internal/reserved-ping-names.html
-# for additional information.
-
 ---
 $schema: moz://mozilla.org/schemas/glean/metrics/2-0-0
 
diff --git a/samples/qt/src/App/pings.yaml b/samples/qt/src/App/pings.yaml
@@ -2,10 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# This file defines the pings that are recorded by the Glean SDK.
-# Their code APIs is automatically generated, at build time using,
-# the `glean_parser` PyPI package.
-
 ---
 $schema: moz://mozilla.org/schemas/glean/pings/2-0-0
 
diff --git a/samples/qt/src/Tests/tst_maintests.qml b/samples/qt/src/Tests/tst_maintests.qml
@@ -1,8 +1,8 @@
 import QtQuick 2.0
 import QtTest 1.0
 
-import org.mozilla.Glean 0.25
-import generated 0.25
+import org.mozilla.Glean 0.26
+import generated 0.26
 
 import App 1.0
 

Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`	`1`	`{`
`2`	`2`	`"name": "@mozilla/glean",`
`3`		`- "version": "0.25.0",`
	`3`	`+ "version": "0.26.0",`
`4`	`4`	`"description": "An implementation of the Glean SDK, a modern cross-platform telemetry client, for JavaScript environments.",`
`5`	`5`	`"type": "module",`
`6`	`6`	`"sideEffects": "false",`