[Obs AI Assistant] Use Mocha & Synthtrace for evaluation framework

packages/kbn-eslint-plugin-imports/README.mdx

@@ -39,6 +39,10 @@ This rule validates that every import request in the repsitory follows a standar
 
 This rule is not configurable, should never be skipped, and is auto-fixable.
 
+## `@kbn/imports/require_import`
+
+This rule validates that specific imports are present in a file. This allows you to e.g. require Mocha globals in test files automatically. Currently the only supported import type is a TypeScript reference type. This rule is auto-fixable.
+
 ## `@kbn/imports/exports_moved_packages`
 
 This rule assists package authors who are doing the good work of breaking up large packages. The goal is to define exports which used to be part of one package as having moved to another package. The configuration maintains this mapping and is designed to be extended in the future is additional needs arrise like targetting specific package types.

packages/kbn-eslint-plugin-imports/index.ts

@@ -12,6 +12,7 @@ import { UniformImportsRule } from './src/rules/uniform_imports';
 import { ExportsMovedPackagesRule } from './src/rules/exports_moved_packages';
 import { NoUnusedImportsRule } from './src/rules/no_unused_imports';
 import { NoBoundaryCrossingRule } from './src/rules/no_boundary_crossing';
+import { RequireImportRule } from './src/rules/require_import';
 
 /**
  * Custom ESLint rules, add `'@kbn/eslint-plugin-imports'` to your eslint config to use them
@@ -23,4 +24,5 @@ export const rules = {
   exports_moved_packages: ExportsMovedPackagesRule,
   no_unused_imports: NoUnusedImportsRule,
   no_boundary_crossing: NoBoundaryCrossingRule,
+  require_import: RequireImportRule,
 };

packages/kbn-eslint-plugin-imports/src/rules/require_import.test.ts

@@ -0,0 +1,89 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { RuleTester } from 'eslint';
+import { RequireImportRule } from './require_import';
+import dedent from 'dedent';
+
+const fmt = (str: TemplateStringsArray) => dedent(str) + '\n';
+
+const tsTester = [
+  '@typescript-eslint/parser',
+  new RuleTester({
+    parser: require.resolve('@typescript-eslint/parser'),
+    parserOptions: {
+      sourceType: 'module',
+      ecmaVersion: 2018,
+      ecmaFeatures: {
+        jsx: true,
+      },
+    },
+  }),
+] as const;
+
+const babelTester = [
+  '@babel/eslint-parser',
+  new RuleTester({
+    parser: require.resolve('@babel/eslint-parser'),
+    parserOptions: {
+      sourceType: 'module',
+      ecmaVersion: 2018,
+      requireConfigFile: false,
+      babelOptions: {
+        presets: ['@kbn/babel-preset/node_preset'],
+      },
+    },
+  }),
+] as const;
+
+for (const [name, tester] of [tsTester, babelTester]) {
+  describe(name, () => {
+    tester.run('@kbn/imports/require_import', RequireImportRule, {
+      valid: [
+        {
+          options: ['mocha'],
+          filename: 'foo.ts',
+          code: fmt`
+            import 'mocha';
+
+            /// <reference types="mocha"/>
+
+            describe(( ) => {
+              before(( ) => {
+              });
+            });
+            `,
+        },
+      ],
+      invalid: [
+        {
+          options: ['mocha'],
+          filename: 'foo.ts',
+          code: fmt`
+            describe(( ) => {
+              before(( ) => {
+              });
+            });
+            `,
+          output: fmt`/// <reference types="mocha"/>
+
+            describe(( ) => {
+              before(( ) => {
+              });
+            });`,
+          errors: [
+            {
+              line: 1,
+              message: `Required module 'mocha' is not imported as a type reference`,
+            },
+          ],
+        },
+      ],
+    });
+  });
+}

packages/kbn-eslint-plugin-imports/src/rules/require_import.ts

@@ -0,0 +1,105 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { Rule } from 'eslint';
+import { load } from 'cheerio';
+
+type StringModuleConfig = string;
+
+interface ObjectModuleConfig {
+  module: string;
+  as: ReferenceModuleAs;
+}
+
+type ModuleConfig = StringModuleConfig | ObjectModuleConfig;
+
+enum ReferenceModuleAs {
+  typeReference = 'typeReference',
+}
+
+export const RequireImportRule: Rule.RuleModule = {
+  meta: {
+    type: 'problem',
+    fixable: 'code',
+    docs: {
+      url: 'https://github.com/elastic/kibana/blob/main/packages/kbn-eslint-plugin-imports/README.mdx#kbnimportsrequire_import',
+    },
+    schema: {
+      type: 'array',
+      items: {
+        oneOf: [
+          {
+            type: 'string',
+          },
+          {
+            type: 'object',
+            additionalProperties: false,
+            additionalItems: false,
+            properties: {
+              module: {
+                type: 'string',
+              },
+              as: {
+                type: 'string',
+              },
+            },
+            required: ['module', 'type'],
+          },
+        ],
+      },
+    },
+  },
+
+  create(context) {
+    const requiredImports: ModuleConfig[] = context.options;
+
+    const mappedOptions: ObjectModuleConfig[] = requiredImports.map((config) => {
+      if (typeof config === 'string') {
+        return {
+          module: config,
+          as: ReferenceModuleAs.typeReference,
+        };
+      }
+      return config;
+    });
+
+    return {
+      'Program:exit': (node) => {
+        mappedOptions.forEach((option) => {
+          switch (option.as) {
+            case ReferenceModuleAs.typeReference:
+              const hasImport = node.comments?.some((comment) => {
+                const nodeText = comment.value.match(/\/\s*(<.*>)/)?.[1];
+                if (nodeText) {
+                  const parsedNode = load(nodeText, { xml: true })()._root?.children()[0];
+                  return (
+                    parsedNode &&
+                    parsedNode.name === 'reference' &&
+                    parsedNode.attribs.types === option.module
+                  );
+                }
+              });
+
+              if (!hasImport) {
+                context.report({
+                  node,
+                  message: `Required module '${option.module}' is not imported as a type reference`,
+                  fix(fixer) {
+                    return fixer.insertTextBefore(
+                      node.body[0],
+                      `/// <reference types="${option.module}"/>\n\n`
+                    );
+                  },
+                });
+              }
+          }
+        });
+      },
+    };
+  },
+};

x-pack/plugins/observability_ai_assistant/jest.config.js

@@ -8,11 +8,16 @@
 module.exports = {
   preset: '@kbn/test',
   rootDir: '../../..',
-  roots: ['<rootDir>/x-pack/plugins/observability_ai_assistant'],
+  roots: [
+    '<rootDir>/x-pack/plugins/observability_ai_assistant/public',
+    '<rootDir>/x-pack/plugins/observability_ai_assistant/common',
+    '<rootDir>/x-pack/plugins/observability_ai_assistant/server',
+  ],
   setupFiles: ['<rootDir>/x-pack/plugins/observability_ai_assistant/.storybook/jest_setup.js'],
   collectCoverage: true,
   collectCoverageFrom: [
     '<rootDir>/x-pack/plugins/observability_ai_assistant/{common,public,server}/**/*.{js,ts,tsx}',
   ],
+
   coverageReporters: ['html'],
 };

x-pack/plugins/observability_ai_assistant/scripts/evaluation/.eslintrc.json

@@ -0,0 +1,21 @@
+{
+  "overrides": [
+    {
+      "files": [
+        "**/*.spec.ts"
+      ],
+      "rules": {
+        "@kbn/imports/require_import": [
+          "error",
+          "@kbn/ambient-ftr-types"
+        ],
+        "@typescript-eslint/triple-slash-reference": [
+          "off"
+        ],
+        "spaced-comment": [
+          "off"
+        ]
+      }
+    }
+  ]
+}