diff --git a/.github/workflows/examples-tests.yml b/.github/workflows/examples-tests.yml new file mode 100644 index 0000000..c01b8a8 --- /dev/null +++ b/.github/workflows/examples-tests.yml @@ -0,0 +1,55 @@ +name: E2E Examples + +on: + push: + branches: ["main"] + paths-ignore: + - "**/*.md" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Tests + timeout-minutes: 40 + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v4 + - name: Enable Corepack + run: corepack enable + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable + - name: Install ollama + run: curl -fsSL https://ollama.com/install.sh | sh + - name: Run ollama + run: | + ollama serve & + ollama pull llama3.1 + - name: Call ollama API + run: | + curl -d '{"model": "llama3.1:latest", "stream": false, "prompt":"Whatever I say, asnwer with Yes"}' http://localhost:11434/api/generate + - name: Example Tests + env: + GENAI_API_KEY: ${{ secrets.GENAI_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # TODO: enable WatsonX later + # WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} + # WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} + # WATSONX_SPACE_ID: ${{ secrets.WATSONX_SPACE_ID }} + # WATSONX_DEPLOYMENT_ID: ${{ secrets.WATSONX_DEPLOYMENT_ID }} + run: | + yarn test:examples diff --git a/examples/vitest.examples.config.ts b/examples/vitest.examples.config.ts new file mode 100644 index 0000000..6b2766d --- /dev/null +++ b/examples/vitest.examples.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from "vitest/config"; +import tsConfigPaths from "vite-tsconfig-paths"; +import packageJson from "../package.json"; + +export default defineConfig({ + test: { + globals: true, + passWithNoTests: true, + testTimeout: 120 * 1000, + printConsoleTrace: true, + setupFiles: ["./tests/setup.examples.ts"], + deps: { + interopDefault: false, + }, + }, + define: { + __LIBRARY_VERSION: JSON.stringify(packageJson.version), + }, + plugins: [ + tsConfigPaths({ + projects: ["tsconfig.json"], + }), + ], +}); diff --git a/package.json b/package.json index 2dd3d6f..6e4fed5 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,8 @@ "test:e2e:watch": "vitest watch tests", "test:all": "vitest run", "test:watch": "vitest watch", + "test:examples": "vitest --max-concurrency=10 --config ./examples/vitest.examples.config.ts run tests/examples", + "test:examples:watch": "vitest --max-concurrency=10 --config ./examples/vitest.examples.config.ts run tests/examples", "prepare": "husky", "copyright": "./scripts/copyright.sh", "release": "release-it", @@ -148,6 +150,7 @@ "@types/eslint": "^9.6.1", "@types/eslint-config-prettier": "^6.11.3", "@types/eslint__js": "^8.42.3", + "@types/glob": "^8.1.0", "@types/js-yaml": "^4.0.9", "@types/mustache": "^4", "@types/needle": "^3.3.0", @@ -181,7 +184,7 @@ "temp-dir": "^3.0.0", "tsc-files": "^1.1.4", "tsup": "^8.3.0", - "tsx": "^4.19.0", + "tsx": "^4.19.1", "typescript": "^5.5.4", "typescript-eslint": "^8.2.0", "vite-tsconfig-paths": "^5.0.1", diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts new file mode 100644 index 0000000..2012c63 --- /dev/null +++ b/tests/examples/examples.test.ts @@ -0,0 +1,46 @@ +/** + * Copyright 2024 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { exec } from "child_process"; +import { glob } from "glob"; +import { promisify } from "util"; + +const execAsync = promisify(exec); +const pattern = process.env.EXAMPLE_NAME || `./examples/**/*.ts`; + +const exclude: string[] = []; // list of examples that are excluded + +describe("E2E Examples", async () => { + const exampleFiles = await glob(pattern, { + cwd: process.cwd(), + dot: false, + realpath: true, + }); + + for (const example of exampleFiles) { + if (exclude.includes(example)) { + continue; + } + it.concurrent(`Run ${example}`, async () => { + const { stdout } = await execAsync(`yarn start -- ${example}`); + // eslint-disable-next-line no-console + console.log({ + path: example, + stdout, + }); + }); + } +}); diff --git a/tests/setup.examples.ts b/tests/setup.examples.ts new file mode 100644 index 0000000..b840865 --- /dev/null +++ b/tests/setup.examples.ts @@ -0,0 +1,36 @@ +/** + * Copyright 2024 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import dotenv from "dotenv"; +import { FrameworkError } from "@/errors.js"; +dotenv.config(); +dotenv.config({ + path: ".env.test", + override: true, +}); +dotenv.config({ + path: ".env.test.local", + override: true, +}); + +expect.addSnapshotSerializer({ + serialize(val: FrameworkError): string { + return val.explain(); + }, + test(val): boolean { + return val && val instanceof FrameworkError; + }, +}); diff --git a/yarn.lock b/yarn.lock index ddb46d2..b99cefc 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2046,6 +2046,16 @@ __metadata: languageName: node linkType: hard +"@types/glob@npm:^8.1.0": + version: 8.1.0 + resolution: "@types/glob@npm:8.1.0" + dependencies: + "@types/minimatch": "npm:^5.1.2" + "@types/node": "npm:*" + checksum: 10c0/ded07aa0d7a1caf3c47b85e262be82989ccd7933b4a14712b79c82fd45a239249811d9fc3a135b3e9457afa163e74a297033d7245b0dc63cd3d032f3906b053f + languageName: node + linkType: hard + "@types/js-yaml@npm:^4.0.9": version: 4.0.9 resolution: "@types/js-yaml@npm:4.0.9" @@ -2085,6 +2095,13 @@ __metadata: languageName: node linkType: hard +"@types/minimatch@npm:^5.1.2": + version: 5.1.2 + resolution: "@types/minimatch@npm:5.1.2" + checksum: 10c0/83cf1c11748891b714e129de0585af4c55dd4c2cafb1f1d5233d79246e5e1e19d1b5ad9e8db449667b3ffa2b6c80125c429dbee1054e9efb45758dbc4e118562 + languageName: node + linkType: hard + "@types/ms@npm:*": version: 0.7.34 resolution: "@types/ms@npm:0.7.34" @@ -2762,6 +2779,7 @@ __metadata: "@types/eslint": "npm:^9.6.1" "@types/eslint-config-prettier": "npm:^6.11.3" "@types/eslint__js": "npm:^8.42.3" + "@types/glob": "npm:^8.1.0" "@types/js-yaml": "npm:^4.0.9" "@types/mustache": "npm:^4" "@types/needle": "npm:^3.3.0" @@ -2817,7 +2835,7 @@ __metadata: temp-dir: "npm:^3.0.0" tsc-files: "npm:^1.1.4" tsup: "npm:^8.3.0" - tsx: "npm:^4.19.0" + tsx: "npm:^4.19.1" turndown: "npm:^7.2.0" typescript: "npm:^5.5.4" typescript-eslint: "npm:^8.2.0" @@ -10078,9 +10096,9 @@ __metadata: languageName: node linkType: hard -"tsx@npm:^4.19.0": - version: 4.19.0 - resolution: "tsx@npm:4.19.0" +"tsx@npm:^4.19.1": + version: 4.19.1 + resolution: "tsx@npm:4.19.1" dependencies: esbuild: "npm:~0.23.0" fsevents: "npm:~2.3.3" @@ -10090,7 +10108,7 @@ __metadata: optional: true bin: tsx: dist/cli.mjs - checksum: 10c0/d14463a78067c6db84c677b79b14861de6d7f6fb0ffa5727cc500c4552459e936395a3854ad0112af0fd7b263bcedd62ce3929b036188eb10cd9902a607ffe34 + checksum: 10c0/cbea9baf57e7406fa0ecc2c03b9bb2501ee740dc28c938f949180a646a28e5d65e7cccbfba340508923bfd45e90320ef9eef7f815cae4515b6ef2ee429edc7ee languageName: node linkType: hard