Expose a "glean" command

.gitignore

@@ -121,3 +121,6 @@ web-ext-artifacts/
 
 # This is the name of the folder we will add the glean generated files in for our samples.
 generated/
+
+# This is the name of the Glean virtual environment
+.venv

CHANGELOG.md

@@ -11,6 +11,7 @@
 * [#101](https://github.com/mozilla/glean.js/pull/101): BUGFIX: Only validate Debug View Tag and Source Tags when they are present.
 * [#102](https://github.com/mozilla/glean.js/pull/102): BUGFIX: Include a Glean User-Agent header in all pings.
 * [#97](https://github.com/mozilla/glean.js/pull/97): Add support for labeled metric types (string, boolean and counter).
+* [#105](https://github.com/mozilla/glean.js/pull/105): Introduce and publish the `glean` command for using the `glean-parser` in a virtual environment.
 
 # v0.4.0 (2021-03-10)
 

glean/package.json

@@ -46,6 +46,9 @@
     "package.json",
     "dist/**/*"
   ],
+  "bin": {
+    "glean": "./dist/cli/cli.js"
+  },
   "scripts": {
     "test": "npm run test:core && npm run test:platform && npm run test:plugins",
     "test:core": "ts-mocha \"tests/core/**/*.spec.ts\" --recursive",
@@ -54,13 +57,14 @@
     "build:test-webext": "cd tests/platform/utils/webext/sample/ && npm install && npm run build:xpi",
     "lint": "eslint . --ext .ts,.js,.json --max-warnings=0",
     "fix": "eslint . --ext .ts,.js,.json --fix",
+    "build:cli": "tsc -p ./tsconfig/cli.json",
     "build:webext:lib:esm": "tsc -p ./tsconfig/webext/esm.json",
     "build:webext:lib:cjs": "tsc -p ./tsconfig/webext/cjs.json",
     "build:webext:lib:browser": "tsc -p ./tsconfig/webext/browser.json",
     "build:webext:types": "tsc -p ./tsconfig/webext/types.json",
     "build:webext": "rm -rf dist/webext && npm run build:webext:lib:esm && npm run build:webext:lib:cjs && npm run build:webext:lib:browser && npm run build:webext:types",
     "build:qt": "webpack --config webpack.config.qt.js --mode production",
-    "prepublishOnly": "cp ../README.md ./README.md && npm run build:webext",
+    "prepublishOnly": "cp ../README.md ./README.md && npm run build:cli && npm run build:webext",
     "postpublish": "rm ./README.md"
   },
   "repository": {

glean/src/cli.ts

@@ -0,0 +1,213 @@
+#!/usr/bin/env node
+
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+//require('ts-node').register();
+//require('./glean.ts');
+
+import * as exec from "child_process";
+import * as fs from "fs";
+import * as path from "path";
+import { argv, platform } from "process";
+import { promisify } from "util";
+
+// Define an async/await version of "exec".
+const execAsync = promisify(exec.exec);
+
+// The name of the directory which will contain the Python virtual environment
+// used to run the glean-parser.
+const VIRTUAL_ENVIRONMENT_DIR = ".venv";
+
+// The version of glean_parser to install from PyPI.
+const GLEAN_PARSER_VERSION = "2.5.0";
+
+// This script runs a given Python module as a "main" module, like
+// `python -m module`. However, it first checks that the installed
+// package is at the desired version, and if not, upgrades it using `pip`.
+//
+// ** IMPORTANT**
+// Keep this script in sync with the one in the Glean SDK (Gradle Plugin).
+//
+// Note: Groovy doesn't support embedded " in multi-line strings, so care
+// should be taken to use ' everywhere in this code snippet.
+const PYTHON_SCRIPT = `
+import importlib
+import subprocess
+import sys
+offline = sys.argv[1] == 'offline'
+module_name = sys.argv[2]
+expected_version = sys.argv[3]
+try:
+    module = importlib.import_module(module_name)
+except ImportError:
+    found_version = None
+else:
+    found_version = getattr(module, '__version__')
+if found_version != expected_version:
+    if not offline:
+        subprocess.check_call([
+            sys.executable,
+            '-m',
+            'pip',
+            'install',
+            '--upgrade',
+            f'{module_name}=={expected_version}'
+        ])
+    else:
+        print(f'Using Python environment at {sys.executable},')
+        print(f'expected glean_parser version {expected_version}, found {found_version}.')
+        sys.exit(1)
+try:
+    subprocess.check_call([
+        sys.executable,
+        '-m',
+        module_name
+    ] + sys.argv[4:])
+except:
+    # We don't need to show a traceback in this helper script.
+    # Only the output of the subprocess is interesting.
+    sys.exit(1)
+`;
+
+/**
+ * Gets the name of the Python binary, based on the host OS.
+ *
+ * @returns the name of the Python executable.
+ */
+function getSystemPythonBinName(): string {
+  return (platform === "win32") ? "python.exe" : "python3";
+}
+
+/**
+ * Gets the full path to the directory containing the python
+ * binaries in the virtual environment.
+ *
+ * Note that this directory changes depending on the host OS.
+ *
+ * @param venvRoot the root path of the virtual environment.
+ *
+ * @returns the full path to the directory containing the python
+ *          binaries in the virtual environment.
+ */
+function getPythonVenvBinariesPath(venvRoot: string): string {
+  if (platform === "win32") {
+    return path.join(venvRoot, "Scripts");
+  }
+
+  return path.join(venvRoot, "bin");
+}
+
+/**
+ * Checks if a Python virtual environment is available.
+ *
+ * @param venvPath the Python virtual environment directory.
+ *
+ * @returns `true` if the Python virtual environment exists and
+ *          is accessible, `false` otherwise.
+ */
+async function checkPythonVenvExists(venvPath: string): Promise<boolean> {
+  console.log(`Checking for a Glean virtual environment at ${venvPath}`);
+
+  const venvPython =
+    path.join(getPythonVenvBinariesPath(venvPath), getSystemPythonBinName());
+
+  const access = promisify(fs.access);
+
+  try {
+    await access(venvPath, fs.constants.F_OK);
+    await access(venvPython, fs.constants.F_OK);
+
+    return true;
+  } catch (e) {
+    return false;
+  }
+}
+
+/**
+ * Uses the system's Python interpreter to create a Python3 virtual environment.
+ *
+ * @param venvPath the directory in which to create the virtual environment.
+ *
+ * @returns `true` if the environment was correctly created, `false` otherwise.
+ */
+async function createPythonVenv(venvPath: string): Promise<boolean> {
+  console.log(`Creating a Glean virtual environment at ${venvPath}`);
+
+  const pipFilename = (platform === "win32") ? "pip3.exe" : "pip3";
+  const venvPip =
+    path.join(getPythonVenvBinariesPath(VIRTUAL_ENVIRONMENT_DIR), pipFilename);
+
+  const pipCmd = `${venvPip} install wheel`;
+  const venvCmd = `${getSystemPythonBinName()} -m venv ${VIRTUAL_ENVIRONMENT_DIR}`;
+
+  for (const cmd of [venvCmd, pipCmd]) {
+    try {
+      await execAsync(cmd);
+    } catch (e) {
+      console.error(e);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+/**
+ * Checks if a virtual environment for running the glean_parser exists,
+ * otherwise it creates it.
+ *
+ * @param projectRoot the project's root directory.
+ */
+async function setup(projectRoot: string) {
+  const venvRoot = path.join(projectRoot, VIRTUAL_ENVIRONMENT_DIR);
+
+  const venvExists = await checkPythonVenvExists(venvRoot);
+  if (venvExists) {
+    console.log(`Using Glean virtual environment at ${venvRoot}`);
+  } else if (!await createPythonVenv(venvRoot)){
+    console.error(`Failed to createa a Glean virtual environment at ${venvRoot}`);
+  }
+}
+
+/**
+ * Runs the glean_parser with the provided options.
+ *
+ * @param projectRoot the project's root directory.
+ * @param parserArgs the list of arguments passed to this command.
+ */
+async function runGlean(projectRoot: string, parserArgs: string[]) {
+  const venvRoot = path.join(projectRoot, VIRTUAL_ENVIRONMENT_DIR);
+  const pythonBin = path.join(getPythonVenvBinariesPath(venvRoot), getSystemPythonBinName());
+  const cmd = `${pythonBin} -c "${PYTHON_SCRIPT}" online glean_parser ${GLEAN_PARSER_VERSION} ${parserArgs.join(" ")}`;
+  try {
+    await execAsync(cmd);
+  } catch (e) {
+    console.error(e);
+  }
+}
+
+/**
+ * Runs the command.
+ *
+ * @param args the arguments passed to this process.
+ */
+async function run(args: string[]) {
+  if (args.length < 3) {
+    throw new Error("Not enough arguments. Please refer to https://mozilla.github.io/glean_parser/readme.html");
+  }
+
+  const projectRoot = process.cwd();
+  try {
+    await setup(projectRoot);
+  } catch (err) {
+    console.error("Failed to setup the Glean build environment", err);
+  }
+
+  await runGlean(projectRoot, args.slice(2));
+}
+
+run(argv).catch(e => {
+  console.error("There was an error running Glean", e);
+});

glean/tsconfig/cli.json

@@ -0,0 +1,9 @@
+{
+  "extends": "./base.json",
+  "include": [
+    "../src/cli.ts"
+  ],
+  "compilerOptions": {
+    "outDir": "../dist/cli"
+  }
+}

samples/web-extension/javascript/README.md

@@ -9,37 +9,30 @@ Whenever this web extensions popup is opened it will trigger Glean.js events.
 > of the directory where the manifest file is.
 
 ## How to run this sample
+Running the example requires Python 3.
 
-1. Generate metrics and pings files.
-
-```bash
-npm run glean_parser
-```
-
-> This command requires that you have [`glean_parser`](https://pypi.org/project/glean-parser/) available.
-> glean_parser is a Python package. To install it run `pip install glean_parser`.
-> Javascript support was added to glean_parser on version 2.1.0, make sure your version is up to date.
-
-2. Link the `@mozilla/glean` package. On the glean/ folder run:
+1. Link the `@mozilla/glean` package. On the glean/ folder run:
 
 ```bash
 npm link
 ```
 
-3. Link the `@mozilla/glean` package to this sample web extension. On this `web-extension` folder run:
+2. Link the `@mozilla/glean` package to this sample web extension. On this `web-extension` folder run:
 
 ```bash
+npm install
 npm link @mozilla/glean
 ```
 
-4. Build this sample. On this `web-extension` folder run:
+3. Build this sample. On this `web-extension` folder run:
 
 ```bash
-npm install
 npm run build
 ```
 
-5. Load the web extension on your browser of choice.
+> **Note** This operation will take some time on the first run, because it will create a virtual environment for running the glean-parser.
+
+4. Load the web extension on your browser of choice.
 
   - **Firefox**
     1. Go to [about:debugging#/runtime/this-firefox](about:debugging#/runtime/this-firefox);

samples/web-extension/javascript/package.json

@@ -4,9 +4,9 @@
   "description": "A sample web extension using Glean.js",
   "main": "dist/bundle.js",
   "scripts": {
-    "glean_parser": "glean_parser translate src/metrics.yaml src/pings.yaml -f javascript -o src/generated",
-    "build": "webpack --watch --config webpack.config.js --mode production",
-    "dev": "webpack --watch --config webpack.config.js --mode development"
+    "glean": "glean translate src/metrics.yaml src/pings.yaml -f javascript -o src/generated",
+    "build": "npm run glean && webpack --watch --config webpack.config.js --mode production",
+    "dev": "npm run glean && webpack --watch --config webpack.config.js --mode development"
   },
   "author": "The Glean Team <glean-team@mozilla.com>",
   "license": "MPL-2.0",

samples/web-extension/typescript/README.md

@@ -9,37 +9,30 @@ Whenever this web extensions popup is opened it will trigger Glean.js events.
 > of the directory where the manifest file is.
 
 ## How to run this sample
+Running the example requires Python 3.
 
-1. Generate metrics and pings files.
-
-```bash
-npm run glean_parser
-```
-
-> This command requires that you have [`glean_parser`](https://pypi.org/project/glean-parser/) available.
-> glean_parser is a Python package. To install it run `pip install glean_parser`.
-> Javascript support was added to glean_parser on version 2.1.0, make sure your version is up to date.
-
-2. Link the `@mozilla/glean` package. On the glean/ folder run:
+1. Link the `@mozilla/glean` package. On the glean/ folder run:
 
 ```bash
 npm link
 ```
 
-3. Link the `@mozilla/glean` package to this sample web extension. On this `web-extension` folder run:
+2. Link the `@mozilla/glean` package to this sample web extension. On this `web-extension` folder run:
 
 ```bash
+npm install
 npm link @mozilla/glean
 ```
 
-4. Build this sample. On this `web-extension` folder run:
+3. Build this sample. On this `web-extension` folder run:
 
 ```bash
-npm install
 npm run build
 ```
 
-5. Load the web extension on your browser of choice.
+> **Note** This operation will take some time on the first run, because it will create a virtual environment for running the glean-parser.
+
+4. Load the web extension on your browser of choice.
 
   - **Firefox**
     1. Go to [about:debugging#/runtime/this-firefox](about:debugging#/runtime/this-firefox);

samples/web-extension/typescript/package.json

@@ -4,9 +4,9 @@
   "description": "A sample web extension using Glean.js",
   "main": "dist/bundle.js",
   "scripts": {
-    "glean_parser": "glean_parser translate src/metrics.yaml src/pings.yaml -f typescript -o src/generated",
-    "build": "webpack --watch --config webpack.config.js --mode production",
-    "dev": "webpack --watch --config webpack.config.js --mode development"
+    "glean": "glean translate src/metrics.yaml src/pings.yaml -f typescript -o src/generated",
+    "build": "npm run glean && webpack --watch --config webpack.config.js --mode production",
+    "dev": "npm run glean && webpack --watch --config webpack.config.js --mode development"
   },
   "author": "The Glean Team <glean-team@mozilla.com>",
   "license": "MPL-2.0",