feat(gen-ai): add eval tests COMPASS-10084 (#7641)

* add feature flag

* migrate prompts to compass

* co-pilot feedback

* clean up

* use edu api for gen ai

* clean up a bit

* fix url for test and ensure aggregations have content

* tests

* fix error handling

* clean up transport

* changes in field name

* use query parser

* fix check

* fix test

* clean up

* copilot feedback

* fix log id

* fix cors issue on e2e tests

* more tests

* add type

* wip

* fix alltext

* make expected output xml string

* add fixtures and run gen ai eval tests

* ts fixes

* reformat

* clean up scorer

* bootstrap

* remove extra files

* bootstrap

* fix prompts and data

* copilot review

* reduce num of docs

* reduce fixtures

* check fix

Author: mabaasit

package-lock.json

@@ -49,7 +49,8 @@
     "test-watch": "npm run test -- --watch",
     "test-ci": "npm run test-cov",
     "test-ci-electron": "npm run test-electron",
-    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write ."
+    "reformat": "npm run eslint . -- --fix && npm run prettier -- --write .",
+    "eval": "braintrust eval tests/evals"
   },
   "dependencies": {
     "@mongodb-js/atlas-service": "^0.74.1",
@@ -84,10 +85,13 @@
     "@types/mocha": "^9.0.0",
     "@types/react": "^17.0.5",
     "@types/sinon-chai": "^3.2.5",
+    "autoevals": "^0.0.130",
+    "braintrust": "^0.2.4",
     "chai": "^4.3.6",
     "depcheck": "^1.4.1",
     "electron-mocha": "^12.2.0",
     "mocha": "^10.2.0",
+    "mongodb-ns": "^3.1.0",
     "nyc": "^15.1.0",
     "p-queue": "^7.4.1",
     "sinon": "^9.2.3",

packages/compass-generative-ai/package.json

@@ -0,0 +1,42 @@
+import { streamText } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+import type {
+  ConversationEvalCaseInput,
+  ConversationTaskOutput,
+} from './types';
+
+export async function makeChatbotCall(
+  input: ConversationEvalCaseInput
+): Promise<ConversationTaskOutput> {
+  const openai = createOpenAI({
+    baseURL:
+      process.env.COMPASS_ASSISTANT_BASE_URL_OVERRIDE ??
+      'https://eval.knowledge-dev.mongodb.com/api/v1',
+    apiKey: '',
+    headers: {
+      'X-Request-Origin': 'compass-gen-ai-braintrust',
+      'User-Agent': 'mongodb-compass/x.x.x',
+    },
+  });
+  const result = streamText({
+    model: openai.responses('mongodb-slim-latest'),
+    temperature: undefined,
+    prompt: input.messages,
+    providerOptions: {
+      openai: {
+        instructions: input.instructions.content,
+        store: false,
+      },
+    },
+  });
+
+  const chunks: string[] = [];
+  for await (const chunk of result.toUIMessageStream()) {
+    if (chunk.type === 'text-delta') {
+      chunks.push(chunk.delta);
+    }
+  }
+  return {
+    messages: [{ content: chunks.join('') }],
+  };
+}