test(examples): e2e test for examples

.github/workflows/examples-tests.yml

@@ -0,0 +1,55 @@
+name: E2E Examples
+
+on:
+  push:
+    branches: ["main"]
+    paths-ignore:
+      - "**/*.md"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: Tests
+    timeout-minutes: 40
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        node-version: [18.x]
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Enable Corepack
+        run: corepack enable
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: "yarn"
+      - name: Install dependencies
+        run: yarn install --immutable
+      - name: Install ollama
+        run: curl -fsSL https://ollama.com/install.sh | sh
+      - name: Run ollama
+        run: |
+          ollama serve &
+          ollama pull llama3.1
+      - name: Call ollama API
+        run: |
+          curl -d '{"model": "llama3.1:latest", "stream": false, "prompt":"Whatever I say, asnwer with Yes"}' http://localhost:11434/api/generate
+      - name: Example Tests
+        env:
+          GENAI_API_KEY: ${{ secrets.GENAI_API_KEY }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          # TODO: enable WatsonX later
+          # WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }}
+          # WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
+          # WATSONX_SPACE_ID: ${{ secrets.WATSONX_SPACE_ID }}
+          # WATSONX_DEPLOYMENT_ID: ${{ secrets.WATSONX_DEPLOYMENT_ID }}
+        run: |
+          yarn test:examples

examples/vitest.examples.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig } from "vitest/config";
+import tsConfigPaths from "vite-tsconfig-paths";
+import packageJson from "../package.json";
+
+export default defineConfig({
+  test: {
+    globals: true,
+    passWithNoTests: true,
+    testTimeout: 120 * 1000,
+    printConsoleTrace: true,
+    setupFiles: ["./tests/setup.examples.ts"],
+    deps: {
+      interopDefault: false,
+    },
+    maxConcurrency: 10,
+  },
+  define: {
+    __LIBRARY_VERSION: JSON.stringify(packageJson.version),
+  },
+  plugins: [
+    tsConfigPaths({
+      projects: ["tsconfig.json"],
+    }),
+  ],
+});

package.json

@@ -130,6 +130,8 @@
     "test:e2e:watch": "vitest watch tests",
     "test:all": "vitest run",
     "test:watch": "vitest watch",
+    "test:examples": "vitest --config ./examples/vitest.examples.config.ts run tests/examples",
+    "test:examples:watch": "vitest --config ./examples/vitest.examples.config.ts run tests/examples",
     "prepare": "husky",
     "copyright": "./scripts/copyright.sh",
     "release": "release-it",
@@ -200,6 +202,7 @@
     "@types/eslint": "^9.6.1",
     "@types/eslint-config-prettier": "^6.11.3",
     "@types/eslint__js": "^8.42.3",
+    "@types/glob": "^8.1.0",
     "@types/js-yaml": "^4.0.9",
     "@types/mustache": "^4",
     "@types/needle": "^3.3.0",
@@ -234,7 +237,7 @@
     "temp-dir": "^3.0.0",
     "tsc-files": "^1.1.4",
     "tsup": "^8.3.0",
-    "tsx": "^4.19.0",
+    "tsx": "^4.19.1",
     "typescript": "^5.5.4",
     "typescript-eslint": "^8.2.0",
     "vite-tsconfig-paths": "^5.0.1",

tests/examples/examples.test.ts

@@ -0,0 +1,77 @@
+/**
+ * Copyright 2024 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { expect } from "vitest";
+import { exec } from "child_process";
+import { glob } from "glob";
+import { promisify } from "util";
+import { isTruthy } from "remeda";
+import { hasEnv } from "@/internals/env.js";
+
+const execAsync = promisify(exec);
+const include_pattern = process.env.INCLUDE_PATTERN || `./examples/**/*.ts`;
+const exclude_pattern = process.env.EXCLUDE_PATTERN || ``;
+const prefixLength = "examples/".length;
+
+const exclude: string[] = [
+  !hasEnv("WATSONX_API_KEY") && [
+    "llms/text.ts",
+    "llms/providers/watsonx_verbose.ts",
+    "llms/providers/watsonx.ts",
+  ],
+  !hasEnv("GROQ_API_KEY") && ["agents/sql.ts", "llms/providers/groq.ts"],
+  !hasEnv("OPENAI_API_KEY") && ["agents/bee_reusable.ts", "llms/providers/openai.ts"],
+  !hasEnv("IBM_VLLM_URL") && ["llms/providers/ibm-vllm.ts"],
+  !hasEnv("COHERE_API_KEY") && ["llms/providers/langchain.ts"],
+  ["llms/providers/bam.ts", "llms/providers/bam_verbose.ts"],
+]
+  .filter(isTruthy)
+  .flat(); // list of examples that are excluded
+
+describe("E2E Examples", async () => {
+  const exampleFiles = await glob(include_pattern, {
+    cwd: process.cwd(),
+    dot: false,
+    realpath: true,
+    ignore: [exclude_pattern],
+  });
+
+  for (const example of exampleFiles) {
+    if (exclude.includes(example.slice(prefixLength))) {
+      continue;
+    }
+    it.concurrent(`Run ${example}`, async () => {
+      await execAsync(`yarn start -- ${example} <<< "Hello world"`)
+        .then((stdout) => {
+          // eslint-disable-next-line no-console
+          console.log({
+            path: example,
+            result: stdout.stdout,
+            error: stdout.stderr,
+          });
+          expect(stdout.stderr).toBeFalsy();
+        })
+        .catch((error) => {
+          // eslint-disable-next-line no-console
+          console.log({
+            path: example,
+            errorCode: error.code,
+          });
+          expect(error.code).toBe(0);
+        });
+    });
+  }
+});

tests/setup.examples.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright 2024 IBM Corp.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import dotenv from "dotenv";
+import { FrameworkError } from "@/errors.js";
+dotenv.config();
+dotenv.config({
+  path: ".env.test",
+  override: true,
+});
+dotenv.config({
+  path: ".env.test.local",
+  override: true,
+});
+
+expect.addSnapshotSerializer({
+  serialize(val: FrameworkError): string {
+    return val.explain();
+  },
+  test(val): boolean {
+    return val && val instanceof FrameworkError;
+  },
+});

yarn.lock

@@ -2062,6 +2062,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/glob@npm:^8.1.0":
+  version: 8.1.0
+  resolution: "@types/glob@npm:8.1.0"
+  dependencies:
+    "@types/minimatch": "npm:^5.1.2"
+    "@types/node": "npm:*"
+  checksum: 10c0/ded07aa0d7a1caf3c47b85e262be82989ccd7933b4a14712b79c82fd45a239249811d9fc3a135b3e9457afa163e74a297033d7245b0dc63cd3d032f3906b053f
+  languageName: node
+  linkType: hard
+
 "@types/js-yaml@npm:^4.0.9":
   version: 4.0.9
   resolution: "@types/js-yaml@npm:4.0.9"
@@ -2110,6 +2120,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/minimatch@npm:^5.1.2":
+  version: 5.1.2
+  resolution: "@types/minimatch@npm:5.1.2"
+  checksum: 10c0/83cf1c11748891b714e129de0585af4c55dd4c2cafb1f1d5233d79246e5e1e19d1b5ad9e8db449667b3ffa2b6c80125c429dbee1054e9efb45758dbc4e118562
+  languageName: node
+  linkType: hard
+
 "@types/ms@npm:*":
   version: 0.7.34
   resolution: "@types/ms@npm:0.7.34"
@@ -2810,6 +2827,7 @@ __metadata:
     "@types/eslint": "npm:^9.6.1"
     "@types/eslint-config-prettier": "npm:^6.11.3"
     "@types/eslint__js": "npm:^8.42.3"
+    "@types/glob": "npm:^8.1.0"
     "@types/js-yaml": "npm:^4.0.9"
     "@types/mustache": "npm:^4"
     "@types/needle": "npm:^3.3.0"
@@ -2866,7 +2884,7 @@ __metadata:
     temp-dir: "npm:^3.0.0"
     tsc-files: "npm:^1.1.4"
     tsup: "npm:^8.3.0"
-    tsx: "npm:^4.19.0"
+    tsx: "npm:^4.19.1"
     turndown: "npm:^7.2.0"
     typescript: "npm:^5.5.4"
     typescript-eslint: "npm:^8.2.0"
@@ -11078,9 +11096,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"tsx@npm:^4.19.0":
-  version: 4.19.0
-  resolution: "tsx@npm:4.19.0"
+"tsx@npm:^4.19.1":
+  version: 4.19.1
+  resolution: "tsx@npm:4.19.1"
   dependencies:
     esbuild: "npm:~0.23.0"
     fsevents: "npm:~2.3.3"
@@ -11090,7 +11108,7 @@ __metadata:
       optional: true
   bin:
     tsx: dist/cli.mjs
-  checksum: 10c0/d14463a78067c6db84c677b79b14861de6d7f6fb0ffa5727cc500c4552459e936395a3854ad0112af0fd7b263bcedd62ce3929b036188eb10cd9902a607ffe34
+  checksum: 10c0/cbea9baf57e7406fa0ecc2c03b9bb2501ee740dc28c938f949180a646a28e5d65e7cccbfba340508923bfd45e90320ef9eef7f815cae4515b6ef2ee429edc7ee
   languageName: node
   linkType: hard