Add glean.js analytics to home page (mozilla#11402)

.circleci/config.yml

@@ -25,7 +25,7 @@ jobs:
 
   test_js:
     docker:
-      - image: "circleci/node:16-browsers"
+      - image: "circleci/python:3.9-node-browsers"
         auth:
           username: $DOCKERHUB_USERNAME
           password: $DOCKERHUB_PASSWORD

.gitignore

@@ -57,3 +57,4 @@ static_final
 static
 assets
 npm-debug.log
+media/js/libs/glean

Dockerfile

@@ -1,3 +1,24 @@
+########
+# Python dependencies builder
+#
+FROM python:3.9-slim-bullseye AS python-builder
+
+WORKDIR /app
+ENV LANG=C.UTF-8
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+ENV PATH="/venv/bin:$PATH"
+
+COPY docker/bin/apt-install /usr/local/bin/
+RUN apt-install gettext build-essential libxml2-dev libxslt1-dev libxslt1.1
+RUN python -m venv /venv
+
+COPY requirements/prod.txt ./requirements/
+
+# Install Python deps
+RUN pip install --require-hashes --no-cache-dir -r requirements/prod.txt
+
+
 ########
 # assets builder and dev server
 #
@@ -16,29 +37,16 @@ RUN npm ci
 COPY .eslintrc.js .eslintignore .stylelintrc .prettierrc.json .prettierignore webpack.config.js webpack.static.config.js ./
 COPY ./media ./media
 COPY ./tests/unit ./tests/unit
+COPY ./glean ./glean
 
-RUN npm run build
-
-
-########
-# Python dependencies builder
-#
-FROM python:3.9-slim-bullseye AS python-builder
-
-WORKDIR /app
-ENV LANG=C.UTF-8
-ENV PYTHONDONTWRITEBYTECODE=1
-ENV PYTHONUNBUFFERED=1
-ENV PATH="/venv/bin:$PATH"
-
+# Required for required glean_parser dependencies
 COPY docker/bin/apt-install /usr/local/bin/
-RUN apt-install gettext build-essential libxml2-dev libxslt1-dev libxslt1.1
-RUN python -m venv /venv
+RUN apt-install python3 python3-venv
+RUN python3 -m venv /.venv
+COPY --from=python-builder /venv /.venv
+ENV PATH="/.venv/bin:$PATH"
 
-COPY requirements/prod.txt ./requirements/
-
-# Install Python deps
-RUN pip install --require-hashes --no-cache-dir -r requirements/prod.txt
+RUN npm run build
 
 
 ########

Makefile

@@ -144,7 +144,7 @@ test_infra/fixtures/tls.json:
 build-ci: .docker-build-pull
 	${DC_CI} build --pull release
 #	tag intermediate images using cache
-	${DC_CI} build app assets builder app-base
+	${DC_CI} build app builder assets app-base
 	touch .docker-build-ci
 
 test-ci: .docker-build-ci

bedrock/base/templates/base-protocol.html

@@ -143,6 +143,8 @@
     {% endblock %}
 
     <!--[if !IE]><!-->
+    {% block glean %}{% endblock %}
+
     {% block js %}{% endblock %}
 
     {% if self.structured_data()|trim|length %}

bedrock/mozorg/templates/mozorg/home/base.html

@@ -33,3 +33,9 @@
 {% block structured_data %}
   {% include 'includes/structured-data/organizations/mozilla-corporation-organisation.json' %}
 {% endblock %}
+
+{% block glean %}
+  {% if switch('glean-analytics') %}
+    {{ js_bundle('glean') }}
+  {% endif %}
+{% endblock %}

docs/analytics.rst

@@ -159,4 +159,109 @@ click with the following structure:
     | Event Action: {{data-cta-type}} click
     | Event Label: {{data-cta-name}}
 
+Glean
+-----
+
+Currently in an evaluation phase, bedrock is now capable of running a parallel
+first-party analytics implementation alongside :abbr:`GTM (Google Tag Manager)`,
+using Mozilla's own `Glean`_ telemetry :abbr:`SDK (Software Development Kit)`.
+See the `Glean Book`_ for more developer reference documentation.
+
+Glean is currently behind a feature switch called ``SWITCH_GLEAN_ANALYTICS``.
+When the switch is enabled pages will load the Glean JavaScript bundle,
+which will do things like register page views and capture link clicks. Our
+implementation leverages the same HTML data attributes that we use for
+:abbr:`GTM (Google Tag Manager)` when tracking link clicks, so any attributes
+you add for :abbr:`GTM (Google Tag Manager)` should also be captured by Glean
+automatically.
+
+Debugging Pings
+~~~~~~~~~~~~~~~
+
+For all non-production environments, bedrock will automatically set a debug
+view tag for all pings. This means that when running on localhost, on a demo,
+or on a staging environment, ping data will not be sent to the production data
+pipeline. Instead, it will be sent to the `Glean debug dashboard`_ which can
+be used to test that pings are working correctly. All bedrock debug pings will
+register in the debug dashboard with the tag name ``moz-bedrock``.
+
+Logging Pings in the Console
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+When running bedrock locally, you can also set the following environment variable
+in your ``.env``` file to automatically log pings in the browser's web console.
+This can be especially useful when making updates to analytics code.
+
+.. code-block::
+
+    GLEAN_LOG_PINGS=True
+
+Defining Metrics and Pings
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All of the data we send to the Glean pipeline is defined in
+:abbr:`YAML (Yet Another Markup Language)` schema files in the ``./glean/``
+project root directory. The ``metrics.yaml`` file defines all the different
+metrics types we record, and the ``pings.yaml`` file defines the name of each
+ping event we use to send collections of individual metrics. These are all
+automatically documented in ``./glean/docs/``.
+
+.. Note::
+
+   Before running any Glean commands locally, always make sure you have first
+   activated your virtual environment by running ``pyenv activate bedrock``.
+
+When bedrock starts, we automatically run ``npm run glean`` which parses these
+schema files and then generates some JavaScript library code in
+``./media/js/libs/glean/``. This library code is not committed to the repository
+on purpose, in order to avoid people altering it and becoming out of sync with
+the schema. This library code is then imported into our Glean analytics code in
+``./media/js/glean/``, which is where we initiate page views and capture click
+events.
+
+Running ``npm run glean`` can also be performed independently of starting bedrock.
+It will also do things such as lint schema files and automatically generate the
+schema docs.
 
+.. Important::
+
+    All metrics and pings we record using Glean must first undergo a `data review`_
+    before being made active in production. Therefore anytime we make new additions
+    to these files, those changes should also undergo review.
+
+Using Glean pings in individual page bundles
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+All of our analytics code for Glean lives in a single bundle in the base template,
+which is intended to be shared across all web pages. There may be times where we
+want to send a ping from some JavaScript that exists only in a certain page
+specific bundle however. For instances like this, there is a global ``pageEventPing``
+helper available, which you can call from inside any custom event handler you write.
+
+For user initiated events, such as clicks:
+
+.. code-block:: javascript
+
+    if (typeof window.Mozilla.Glean !== 'undefined') {
+        window.Mozilla.Glean.pageEventPing({
+            label: 'Newsletters: mozilla-and-you',
+            type: 'Newsletter Signup Success'
+        });
+    }
+
+For non-interaction events that are not user initiated:
+
+.. code-block:: javascript
+
+    if (typeof window.Mozilla.Glean !== 'undefined') {
+        window.Mozilla.Glean.pageEventPing({
+            label: 'Auto Play',
+            type: 'Video'
+            nonInteraction: true
+        });
+    }
+
+.. _Glean: https://docs.telemetry.mozilla.org/concepts/glean/glean.html
+.. _Glean Book: https://mozilla.github.io/glean/book/index.html
+.. _Glean debug dashboard: https://debug-ping-preview.firebaseapp.com/
+.. _data revreview: https://wiki.mozilla.org/Data_Collection

docs/testing.rst

@@ -23,29 +23,34 @@ Installation
 ------------
 
 First follow the :ref:`installation instructions for bedrock<install>`, which
-will install the specific versions of Jasmine/Karma which are needed to run the
-unit tests, and guide you through installing pip and setting up a virtual
-environment for the functional tests. The additional requirements can then be
-installed by using the following commands:
+will install the dependencies required to run the various front-end test suites.
 
-.. code-block:: bash
 
-    $ source venv/bin/activate
+Running Jasmine tests using Karma
+---------------------------------
+
+To perform a single run of the Jasmine test suite using Firefox and Chrome,
+first make sure you have both browsers installed locally, and then activate
+your bedrock virtual env.
 
 .. code-block:: bash
 
-    $ pip install -r requirements/dev.txt
+    $ pyenv activate bedrock
 
-Running Jasmine tests using Karma
----------------------------------
-
-To perform a single run of the Jasmine test suite using Firefox, type the
-following command:
+You can then run the tests with the following command:
 
 .. code-block:: bash
 
     $ npm run test
 
+This will run all our front-end linters and formatting checks before running
+the Jasmine test suite. If you only want to run the tests themselves, you can
+run:
+
+.. code-block:: bash
+
+    $ npm run karma
+
 See the `Jasmine`_ documentation for tips on how to write JS behavioral or unit
 tests. We also use `Sinon`_ for creating test spies, stubs and mocks.
 

glean/docs/metrics.md

@@ -0,0 +1,105 @@
+<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. -->
+
+# Metrics
+
+This document enumerates the metrics collected by this project using the [Glean SDK](https://mozilla.github.io/glean/book/index.html).
+This project may depend on other projects which also collect metrics.
+This means you might have to go searching through the dependency tree to get a full picture of everything collected by this project.
+
+# Pings
+
+- [interaction](#interaction)
+- [non-interaction](#non-interaction)
+- [page-view](#page-view)
+
+## interaction
+
+A ping which is sent when a page element is
+interacted with.
+
+
+This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section).
+
+**Data reviews for this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1767442>
+
+**Bugs related to this ping:**
+
+- <https://github.com/mozilla/bedrock/issues/10746>
+
+All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections.
+
+In addition to those built-in metrics, the following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| element.clicked |[event](https://mozilla.github.io/glean/book/user/metrics/event.html) |An event containing metrics related to which element in the page was clicked.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)|<ul><li>label: Description of the page element that was clicked. Examples: 'Download Firefox', 'Get Mozilla VPN'. </li><li>position: The position of the element in the page. Examples: 'Primary', 'Secondary', 'Navigation', 'Footer'. </li><li>type: The type of element that was clicked. Examples: 'Button', 'Link'. </li></ul>|never |3 |
+| page.locale |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The locale of the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.page_event |[event](https://mozilla.github.io/glean/book/user/metrics/event.html) |An event containing metrics related to a page level completion or state that want to measure. This can be sent in either an interaction and non-interaction ping, depending upon the use-case. Examples: form completion, scroll events, banner impressions.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)|<ul><li>label: The label used to describe the event. Example: 'Newsletters: mozilla-and-you' </li><li>type: The type of event. Example: 'Newsletter Signup Success' </li></ul>|never |3 |
+| page.path |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The URL path of the page that was viewed, excluding locale.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.query_params |[labeled_string](https://mozilla.github.io/glean/book/user/metrics/labeled_strings.html) |Query parameters associated with the URL of the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)|<ul><li>utm_source</li><li>utm_campaign</li><li>utm_medium</li><li>utm_content</li><li>entrypoint_experiment</li><li>entrypoing_variation</li><li>experiment</li><li>variation</li><li>v</li><li>xv</li></ul>|never | |
+| page.referrer |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The referring URL that linked to the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.viewed |[datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) |The time a page was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+
+## non-interaction
+
+A ping which is sent when a non-user initiated event
+occurs. Examples: a specific banner impression is
+displayed, a video auto-plays on scroll.
+
+
+This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section).
+
+**Data reviews for this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1767442>
+
+**Bugs related to this ping:**
+
+- <https://github.com/mozilla/bedrock/issues/10746>
+
+All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections.
+
+In addition to those built-in metrics, the following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| page.locale |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The locale of the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.page_event |[event](https://mozilla.github.io/glean/book/user/metrics/event.html) |An event containing metrics related to a page level completion or state that want to measure. This can be sent in either an interaction and non-interaction ping, depending upon the use-case. Examples: form completion, scroll events, banner impressions.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)|<ul><li>label: The label used to describe the event. Example: 'Newsletters: mozilla-and-you' </li><li>type: The type of event. Example: 'Newsletter Signup Success' </li></ul>|never |3 |
+| page.path |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The URL path of the page that was viewed, excluding locale.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.query_params |[labeled_string](https://mozilla.github.io/glean/book/user/metrics/labeled_strings.html) |Query parameters associated with the URL of the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)|<ul><li>utm_source</li><li>utm_campaign</li><li>utm_medium</li><li>utm_content</li><li>entrypoint_experiment</li><li>entrypoing_variation</li><li>experiment</li><li>variation</li><li>v</li><li>xv</li></ul>|never | |
+| page.referrer |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The referring URL that linked to the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.viewed |[datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) |The time a page was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+
+## page-view
+
+A ping which is sent everytime a page is viewed.
+
+
+This ping includes the [client id](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section).
+
+**Data reviews for this ping:**
+
+- <https://bugzilla.mozilla.org/show_bug.cgi?id=1767442>
+
+**Bugs related to this ping:**
+
+- <https://github.com/mozilla/bedrock/issues/10746>
+
+All Glean pings contain built-in metrics in the [`ping_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-ping_info-section) and [`client_info`](https://mozilla.github.io/glean/book/user/pings/index.html#the-client_info-section) sections.
+
+In addition to those built-in metrics, the following metrics are added to the ping:
+
+| Name | Type | Description | Data reviews | Extras | Expiration | [Data Sensitivity](https://wiki.mozilla.org/Firefox/Data_Collection) |
+| --- | --- | --- | --- | --- | --- | --- |
+| page.locale |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The locale of the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.path |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The URL path of the page that was viewed, excluding locale.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.query_params |[labeled_string](https://mozilla.github.io/glean/book/user/metrics/labeled_strings.html) |Query parameters associated with the URL of the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)|<ul><li>utm_source</li><li>utm_campaign</li><li>utm_medium</li><li>utm_content</li><li>entrypoint_experiment</li><li>entrypoing_variation</li><li>experiment</li><li>variation</li><li>v</li><li>xv</li></ul>|never | |
+| page.referrer |[string](https://mozilla.github.io/glean/book/user/metrics/string.html) |The referring URL that linked to the page that was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+| page.viewed |[datetime](https://mozilla.github.io/glean/book/user/metrics/datetime.html) |The time a page was viewed.  |[Bug 1767442](https://bugzilla.mozilla.org/show_bug.cgi?id=1767442)||never |3 |
+
+Data categories are [defined here](https://wiki.mozilla.org/Firefox/Data_Collection).
+
+<!-- AUTOGENERATED BY glean_parser. DO NOT EDIT. -->
+