Added EUI Usage Analytics Script (#8296)

Co-authored-by: Tomasz Kajtoch <tomek@kajto.ch>

packages/eui-usage-analytics/README.md

@@ -0,0 +1,51 @@
+# EUI Usage Analytics
+
+This package contain a script used to collect EUI usage analytics from the Elastic product suite.
+
+Data is collected and shipped to an Elastic Cloud instance, from there the data can be analyzed any number of ways.
+
+![image of a dashboard](dashboard.png)
+
+## Notes
+
+There is not a lot of magic to this script, it does 2 key things:
+
+- It runs `react-scanner` to collect data on *all* React component usages in our products (Not just EUI)
+- It determines the "Code Owner" in which the component usage occured and includes it in the component usage record -- this is key to comparing usage across different teams.
+
+
+## Setup
+
+This script requires the following to run:
+
+1. The [kibana](https://github.com/elastic/kibana) directory must be cloned to the same directory as the eui repository.
+2. `CLOUD_ID_SECRET` and `AUTH_APIKEY_SECRET` of the Elatic Cloud instance for which you would like to ship the data.
+
+## Usage
+****
+This script must be run from this directory.
+
+```
+CLOUD_ID_SECRET=****** AUTH_APIKEY_SECRET=****** node index.js
+```
+
+## Schema
+
+This script will store data in an Elastic index named `eui_components`.
+
+It sends the following fields. Each record represents a component usage.
+
+- `@timestamp`
+- `scanDate` - Also a timestamp, but this is hardcoded so that an all entires from a single scan share the same date. This lets us group all records together from a single scan. ex: `2025-01-27T21:33:45.723Z`
+- `repository` - For now just `kibana`, but in the future this may also include `cloud` or other products we add to the scan.
+- `component` - The name of the component. ex: `EuiButton`
+- `codeOwners`- An array of codeowners that own the file in which this component usage occurred. ex: `[ '@elastic/kibana-management' ]`
+- `moduleName` - The module from which the component being used belongs. ex. `react`, `@elastic/eui`
+- `props` - A array of property name value pairs. ex: `[{ propName: 'className', propValue: 'test' }]`
+- `props_combined` - An array of property name value pairs contcatenated in a string. We use this since the prop names and values become disassociated when flattened in Elasticsearch. ex: `["className::test"]`
+- `fileName` - The file name in which a component usage occurs. ex: `/kibana/src/platform/plugins/shared/es_ui_shared/static/forms/components/fields/card_radio_group_field.tsx`
+- `sourceLocation` - A github link to the usage. ex: `'https://github.com/elastic/kibana/blob/main/src/platform/plugins/shared/es_ui_shared/static/forms/components/fields/card_radio_group_field.tsx#L51'`
+- `lineNumber` - ex. `51`
+- `lineColumn` - ex. `11`
+
+

packages/eui-usage-analytics/index.js

@@ -0,0 +1,39 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const { scan } = require('./scan');
+
+const { Client } = require('@elastic/elasticsearch');
+
+if (!process.env.CLOUD_ID_SECRET || !process.env.AUTH_APIKEY_SECRET) {
+  console.error(
+    'CLOUD_ID_SECRET and AUTH_APIKEY_SECRET environment variables must be set before running this script.'
+  );
+  process.exit(1);
+}
+
+const client = new Client({
+  cloud: {
+    id: process.env.CLOUD_ID_SECRET,
+  },
+  auth: {
+    apiKey: process.env.AUTH_APIKEY_SECRET,
+  },
+});
+
+const run = async () => {
+  const result = await scan();
+  const operations = result.flatMap((doc) => [
+    { index: { _index: 'eui_components' } },
+    doc,
+  ]);
+  const response = await client.bulk({ refresh: true, operations });
+  console.log(response);
+};
+
+run().catch((e) => console.error(e));

packages/eui-usage-analytics/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "@elastic/eui-usage-analytics",
+  "private": true,
+  "version": "0.1.0",
+  "description": "Scripts to collect analytics on EUI usage",
+  "dependencies": {
+    "@elastic/elasticsearch": "^8.14.0",
+    "codeowners": "^5.1.1",
+    "escodegen-wallaby": "^1.6.44",
+    "react-scanner": "^1.1.0"
+  }
+}

packages/eui-usage-analytics/scan.js

@@ -0,0 +1,137 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const scanner = require('react-scanner');
+const escodegen = require('escodegen-wallaby');
+const Codeowners = require('codeowners');
+
+const codeowners = new Codeowners('../../../kibana');
+const path = require('path');
+const cwd = path.resolve(__dirname);
+
+// NOTE: Do not add private repos to this list. If we plan to add private repos, we should do so via configuration rather than source.
+const repos = {
+  kibana: {
+    linkPrefix: 'https://github.com/elastic/kibana/blob/main/',
+    crawlFrom: [
+      /*
+       * Scanning the entirety of Kibana could lead to many false negatives and be.
+       * inefficient. These 3 crawl roots may not be 100% comprehensive, but they should cover
+       * most code usages 
+       */
+      '../../../kibana/src',
+      '../../../kibana/x-pack',
+      '../../../kibana/packages',
+    ],
+  },
+};
+
+const scannerConfig = {
+  rootDir: cwd,
+  exclude: ['node_modules', /^\.\w+/],
+  /**
+   * We extensions like .spec .stories. There could be other extension that are worth
+   * ignoring here.
+   */
+  globs: ['**/!(*.test|*.spec|*.stories).{jsx,tsx}'],
+  includeSubComponents: true,
+  /**
+   * count-components-and-props and other can be used to get helpful standalone summaries,
+   * but since we ship this to Elastic to summarize there, we just do a raw-report.
+   */
+  processors: ['raw-report'],
+  crawlFrom: './',
+  getPropValue: ({ node, propName, defaultGetPropValue }) => {
+    /**
+     * Certain complex types of prop values don't get seriealized, so you just
+     * see "(ArrowFunctionExpression)", etc. as the prop value. You can manually define
+     * serializers here. The serializer below lets us see values like
+     * `style::{ fontWeight: 'bold' }` instead of `JSXExpressionContainer` in data.
+     *
+     * This could be expanded further.
+     **/
+    if (propName === 'css' || propName === 'style') {
+      if (node.type === 'JSXExpressionContainer') {
+        try {
+          return escodegen.generate(node.expression);
+        } catch {
+          return defaultGetPropValue(node);
+        }
+      } else {
+        try {
+          return escodegen.generate(node);
+        } catch {
+          return defaultGetPropValue(node);
+        }
+      }
+    } else {
+      return defaultGetPropValue(node);
+    }
+  },
+};
+
+const scan = async () => {
+  let time = new Date();
+  let output = [];
+
+  await Promise.all(
+    Object.entries(repos).map(async ([repo, { crawlFrom, linkPrefix }]) => {
+      await Promise.all(
+        crawlFrom.map(async (kibanaCrawlDirs) => {
+          let newOutput = await scanner.run({
+            ...scannerConfig,
+            crawlFrom: kibanaCrawlDirs,
+          });
+
+          newOutput = Object.entries(newOutput).flatMap(
+            ([componentName, value]) => {
+              return value.instances?.map((instance) => {
+                let fileName;
+                let sourceLocation;
+                let owners = [];
+
+                let regex = /\/kibana\/(.*)$/;
+                if (instance.location?.file) {
+                  const result = regex.exec(instance.location.file);
+                  fileName = result[0];
+                  sourceLocation = `${linkPrefix}${result[1]}#L${instance.location.start.line}`;
+                  owners = codeowners.getOwner(result[1]);
+                }
+
+                return {
+                  '@timestamp': time,
+                  scanDate: time,
+                  component: componentName,
+                  codeOwners: owners,
+                  moduleName: instance.importInfo?.moduleName,
+                  props: Object.entries(instance.props).map(([k, v]) => ({
+                    propName: k,
+                    propValue: v,
+                  })),
+                  props_combined: Object.entries(instance.props).map(
+                    ([k, v]) => `${k}::${v}`
+                  ),
+                  fileName,
+                  sourceLocation,
+                  lineNumber: instance.location?.start?.line,
+                  lineColumn: instance.location?.start?.column,
+                  repository: repo,
+                };
+              });
+            }
+          );
+          output = output.concat(newOutput);
+        })
+      );
+    })
+  );
+
+  return output;
+};
+
+exports.scan = scan;

packages/eui-usage-analytics/test.js

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+const Codeowners = require('codeowners');
+const temp = new Codeowners("../../../kibana");
+
+const { scan } = require('./scan');
+
+const runScan = async () => {
+  const scanResult = await scan();
+  console.log(scanResult);
+};
+
+runScan();