Bug 1679375 - Implement a Storage that uses the web ext storage API

.eslintrc

@@ -2,7 +2,8 @@
   "root": true,
   "env": {
     "node": true,
-    "browser": true
+    "browser": true,
+    "webextensions": true
   },
   "rules": {
     "indent": ["error", 2],

bin/build-size-check.sh

@@ -12,44 +12,49 @@ npm run build:qt
 qt_size=$(wc -c ./dist/glean.js | awk '{print $1}')
 qt_size_pretty=$(wc -c ./dist/glean.js | awk '{printf "%0.2f\n",$1/1024"."substr($2,1,2)}')
 
-npm run build:browser
-browser_size=$(wc -c ./dist/glean.js | awk '{print $1}')
-browser_size_pretty=$(wc -c ./dist/glean.js | awk '{printf "%0.2f\n",$1/1024"."substr($2,1,2)}')
+npm run build:webext
+webext_size=$(wc -c ./dist/glean.js | awk '{print $1}')
+webext_size_pretty=$(wc -c ./dist/glean.js | awk '{printf "%0.2f\n",$1/1024"."substr($2,1,2)}')
 
-git checkout -t origin/main
+git branch -f original-main origin/main
+# TODO: This is done in case there were changes to the package.json,
+# we should find a better way to deal with that though.
+# See: [Bug 1681484](https://bugzilla.mozilla.org/show_bug.cgi?id=1681484)
+git reset --hard HEAD
+git checkout original-main
 
 npm install
 
 npm run build:qt
 qt_size_main=$(wc -c ./dist/glean.js | awk '{print $1}')
 qt_size_main_pretty=$(wc -c ./dist/glean.js | awk '{printf "%0.2f\n",$1/1024"."substr($2,1,2)}')
 
-npm run build:browser
-browser_size_main=$(wc -c ./dist/glean.js | awk '{print $1}')
-browser_size_main_pretty=$(wc -c ./dist/glean.js | awk '{printf "%0.2f\n",$1/1024"."substr($2,1,2)}')
+npm run build:webext
+webext_size_main=$(wc -c ./dist/glean.js | awk '{print $1}')
+webext_size_main_pretty=$(wc -c ./dist/glean.js | awk '{printf "%0.2f\n",$1/1024"."substr($2,1,2)}')
 
 qt_diff=$(((($qt_size-$qt_size_main)%$qt_size_main)*100))
-browser_diff=$(((($browser_size-$browser_size_main)%$browser_size_main)*100))
+webext_diff=$(((($webext_size-$webext_size_main)%$webext_size_main)*100))
 
 [[ $qt_diff -ge 0 ]] && qt_emoji="📈" || qt_emoji="📉"
 [[ $qt_diff -ge 0 ]] && qt_result="Increase" || qt_result="Decrease"
 
-[[ $browser_diff -ge 0 ]] && browser_emoji="📈" || browser_emoji="📉"
-[[ $browser_diff -ge 0 ]] && browser_result="Increase" || browser_result="Decrease"
+[[ $webext_diff -ge 0 ]] && webext_emoji="📈" || webext_emoji="📉"
+[[ $webext_diff -ge 0 ]] && webext_result="Increase" || webext_result="Decrease"
 
 content="
   # Build size report
 
   Merging $CIRCLE_PULL_REQUEST into [main](https://github.com/brizental/glean.js) will:
 
-  * **$browser_result** the size of the browser build (\`npm run build:browser\`) by \`${browser_diff}%\`.
+  * **$webext_result** the size of the webext build (\`npm run build:webext\`) by \`${webext_diff}%\`.
   * **$qt_result** the size of the Qt build (\`npm run build:qt\`) by \`${qt_diff}%\`.
 
   ---
 
   | Build | Current size | New size | Size increase |
   |--:|:---:|:---:|:---:|
-  | browser | ${browser_size_main_pretty}K | ${browser_size_pretty}K | $browser_emoji ${browser_diff}% |
+  | webext | ${webext_size_main_pretty}K | ${webext_size_pretty}K | $webext_emoji ${webext_diff}% |
   | qt | ${qt_size_main_pretty}K | ${qt_size_pretty}K | $qt_emoji ${qt_diff}% |
 "
 

package.json

@@ -8,8 +8,8 @@
     "test:debug": "ts-mocha --paths -p ./tsconfig.json --inspect-brk",
     "lint": "eslint . --ext .ts,.js,.json",
     "fix": "eslint . --ext .ts,.js,.json --fix",
-    "build:browser": "webpack --config webpack.config.browser.js --mode production",
-    "dev:browser": "webpack --watch --config webpack.config.browser.js --mode development",
+    "build:webext": "webpack --config webpack.config.webext.js --mode production",
+    "dev:webext": "webpack --watch --config webpack.config.webext.js --mode development",
     "build:qt": "webpack --config webpack.config.qt.js --mode production",
     "dev:qt": "webpack --watch --config webpack.config.qt.js --mode development"
   },
@@ -42,6 +42,7 @@
     "ts-mocha": "^8.0.0",
     "ts-node": "^9.0.0",
     "typescript": "^4.0.5",
+    "web-ext-types": "^3.2.1",
     "webpack": "^5.4.0",
     "webpack-cli": "^4.2.0"
   }

samples/web-extension/README.md

@@ -10,10 +10,10 @@ Whenever this web extensions popup is opened it will trigger Glean.js events.
 
 ## How to run this sample
 
-1. Build Glean.js for the browser. On the root folder of this repository run:
+1. Build Glean.js for web extensions. On the root folder of this repository run:
 
 ```bash
-npm run build:browser
+npm run build:webext
 ```
 
 2. Build this sample. On this `web-extension` folder run:

src/index.ts

@@ -4,8 +4,13 @@
 
 // Importing this here just so the size increase will show on the PR comments,
 // once everything is implemented we remove it.
-// eslint-disable-next-line @typescript-eslint/no-unused-vars
 import StorageWeak from "storage/weak";
+import StoragePersistent from "storage/persistent";
+// If we leave the above imports unused they will not be added to the final webpack bundle.
+console.log(
+  StorageWeak,
+  StoragePersistent
+);
 
 export = {
   /**

src/storage/index.ts

@@ -2,8 +2,10 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+import { isString, isUndefined, isObject } from "utils";
+
 /**
- * The storage index is an ordered list of keys to navigate on the store
+ * The storage index in the ordered list of keys to navigate on the store
  * to reach a specific entry.
  *
  * # Example
@@ -31,6 +33,30 @@ export interface StorageObject {
   [key: string]: StorageValue;
 }
 
+/**
+ * Verifies if a given value is a valid StorageValue.
+ *
+ * @param v The value to verify
+ *
+ * @returns A special Typescript value (which compiles down to a boolean)
+ *          stating wether `v` is a valid StorageValue.
+ */
+export function isStorageValue(v: unknown): v is StorageValue {
+  if (isUndefined(v) || isString(v)) {
+    return true;
+  }
+
+  if (isObject(v)) {
+    if (Object.keys(v).length === 0) {
+      return true;
+    }
+    for (const key in v) {
+      return isStorageValue(v[key]);
+    }
+  }
+  return false;
+}
+
 export interface Store {
   /**
    * **Test-only API**
@@ -49,7 +75,8 @@ export interface Store {
    * @returns The value found for the given index on the storage.
    *          In case nothing has been recorded on the given index, returns `undefined`.
    *
-   * @throws In case the index is an empty array.
+   * @throws - In case the index is an empty array.
+   *         - In case a value that is not `string` or `object` is found.
    */
   get(index: StorageIndex): Promise<StorageValue>;
 

src/storage/persistent/webext.ts

@@ -0,0 +1,124 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+import { Store, StorageIndex, StorageValue, StorageObject, isStorageValue } from "storage";
+import { updateNestedObject, getValueFromNestedObject, deleteKeyFromNestedObject } from "storage/utils";
+import { isString, isUndefined } from "utils";
+
+type WebExtStoreQuery = { [x: string]: { [x: string]: null; } | null; };
+
+/**
+ * Persistent storage implementation based on the Promise-based
+ * [storage API](https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage)
+ * for web extensions.
+ *
+ * To make sure this implementation works on Chromium based browsers, the user must install the peer dependency
+ * [`mozilla/webextension-polyfill`](https://github.com/mozilla/webextension-polyfill).
+ */
+class WebExtStore implements Store {
+  private store;
+
+  // The main key under which all other entries will be recorded for this store instance.
+  private rootKey: string;
+
+  constructor(rootKey: string) {
+    if (typeof browser === "undefined") {
+      throw Error(
+        `The web extensions store should only be user in a browser extension context.
+        If running is a browser different from Firefox, make sure you have installed
+        the webextension-polyfill peer dependency. To do so, run \`npm i webextension-polyfill\`.`
+      );
+    }
+    this.store = browser.storage.local;
+    this.rootKey = rootKey;
+  }
+
+  async _testOnly_getWholeStore(): Promise<StorageObject> {
+    const result = await this.store.get({ [this.rootKey]: {} });
+    return  result[this.rootKey];
+  }
+
+  /**
+   * Build a query object to retrieve / update a given entry from the storage.
+   *
+   * @param index The index to the given entry on the storage.
+   *
+   * @return The query object.
+   */
+  private _buildQuery(index: StorageIndex): WebExtStoreQuery {
+    let partialQuery = null;
+    for (const key of index) {
+      partialQuery = { [key]: partialQuery };
+    }
+
+    return { [this.rootKey]: partialQuery };
+  }
+
+  /**
+   * Retrieves the full store and builds a query object on top of it.
+   *
+   * @param transformFn The transformation function to apply to the store.
+   *
+   * @returns The query object with the modified store.
+   */
+  private async _buildQueryFromStore(transformFn: (s: StorageObject) => StorageObject): Promise<StorageObject> {
+    const store = await this._testOnly_getWholeStore();
+    return { [this.rootKey]: transformFn(store) };
+  }
+
+  async get(index: StorageIndex): Promise<StorageValue> {
+    const query = this._buildQuery(index);
+    const response: browser.storage.StorageObject = await this.store.get(query);
+    if (!response) {
+      return;
+    }
+
+    if (isStorageValue(response)) {
+      if (!isUndefined(response) && !isString(response)) {
+        return getValueFromNestedObject(response, [ this.rootKey, ...index ]);
+      } else {
+        return response;
+      }
+    }
+
+    throw new Error(
+      `Unexpected value found in storage for index ${index}. Ignoring.
+      ${JSON.stringify(response, null, 2)}`
+    );
+  }
+
+  async update(
+    index: StorageIndex,
+    transformFn: (v: StorageValue) => Exclude<StorageValue, undefined>
+  ): Promise<void> {
+    if (index.length === 0) {
+      throw Error("The index must contain at least one property to update.");
+    }
+
+    // We need to get the full store object here, change it as requested and then re-save.
+    // This is necessary, because if we try to set a key to an inside object on the storage,
+    // it will erase any sibling keys that are not mentioned.
+    const query = await this._buildQueryFromStore(
+      store => updateNestedObject(store, index, transformFn)
+    );
+    return this.store.set(query);
+  }
+
+  async delete(index: StorageIndex): Promise<void> {
+    // The `remove API`[https://developer.mozilla.org/en-US/docs/Mozilla/Add-ons/WebExtensions/API/storage/StorageArea/remove]
+    // doesn't expose a way for us to delete nested keys.
+    // This means we need to get the whole store,
+    // make the necessary changes to it locally and then reset it.
+    try {
+      const query = await this._buildQueryFromStore(
+        store => deleteKeyFromNestedObject(store, index)
+      );
+      return this.store.set(query);
+    } catch(e) {
+      console.warn(e.message, "Ignoring");
+    }
+  }
+}
+
+export default WebExtStore;