From 960e4073983458f8aec2cb1deefcb93a2a7598ed Mon Sep 17 00:00:00 2001 From: Akihiko Kuroda Date: Thu, 10 Oct 2024 10:45:38 -0400 Subject: [PATCH] test(examples): e2e test for examples Signed-off-by: Akihiko Kuroda --- .github/workflows/examples-tests.yml | 55 ++++++++++++++++++++++++++++ CONTRIBUTING.md | 1 - package.json | 6 ++- tests/examples/examples.test.ts | 41 +++++++++++++++++++++ tests/setup.examples.ts | 36 ++++++++++++++++++ vitest.examples.config.ts | 24 ++++++++++++ yarn.lock | 38 ++++++++++++++++--- 7 files changed, 194 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/examples-tests.yml create mode 100644 tests/examples/examples.test.ts create mode 100644 tests/setup.examples.ts create mode 100644 vitest.examples.config.ts diff --git a/.github/workflows/examples-tests.yml b/.github/workflows/examples-tests.yml new file mode 100644 index 0000000..c01b8a8 --- /dev/null +++ b/.github/workflows/examples-tests.yml @@ -0,0 +1,55 @@ +name: E2E Examples + +on: + push: + branches: ["main"] + paths-ignore: + - "**/*.md" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Tests + timeout-minutes: 40 + runs-on: ubuntu-latest + + strategy: + fail-fast: false + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v4 + - name: Enable Corepack + run: corepack enable + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node-version }} + cache: "yarn" + - name: Install dependencies + run: yarn install --immutable + - name: Install ollama + run: curl -fsSL https://ollama.com/install.sh | sh + - name: Run ollama + run: | + ollama serve & + ollama pull llama3.1 + - name: Call ollama API + run: | + curl -d '{"model": "llama3.1:latest", "stream": false, "prompt":"Whatever I say, asnwer with Yes"}' http://localhost:11434/api/generate + - name: Example Tests + env: + GENAI_API_KEY: ${{ secrets.GENAI_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + # TODO: enable WatsonX later + # WATSONX_API_KEY: ${{ secrets.WATSONX_API_KEY }} + # WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }} + # WATSONX_SPACE_ID: ${{ secrets.WATSONX_SPACE_ID }} + # WATSONX_DEPLOYMENT_ID: ${{ secrets.WATSONX_DEPLOYMENT_ID }} + run: | + yarn test:examples diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9245fc2..aeddeff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,7 +11,6 @@ If you are new to Bee contributing, we recommend you do the following before div - Read [Bee Overview](./docs/overview.md) to understand core concepts. - Read [Code of Conduct](./CODE_OF_CONDUCT.md). - ## Style and lint Bee uses the following tools to meet code quality standards and ensure a unified code style across the codebase. diff --git a/package.json b/package.json index 2dd3d6f..9cbae67 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,8 @@ "test:e2e:watch": "vitest watch tests", "test:all": "vitest run", "test:watch": "vitest watch", + "test:examples": "vitest --config ./vitest.examples.config.ts run tests/examples", + "test:examples:watch": "vitest --config ./vitest.examples.config.ts run tests/examples", "prepare": "husky", "copyright": "./scripts/copyright.sh", "release": "release-it", @@ -148,12 +150,14 @@ "@types/eslint": "^9.6.1", "@types/eslint-config-prettier": "^6.11.3", "@types/eslint__js": "^8.42.3", + "@types/glob": "^8.1.0", "@types/js-yaml": "^4.0.9", "@types/mustache": "^4", "@types/needle": "^3.3.0", "@types/node": "^20.16.1", "@types/object-hash": "^3.0.6", "@types/turndown": "^5.0.5", + "async-mutex": "^0.5.0", "dotenv": "^16.4.5", "embedme": "^1.22.1", "eslint": "^9.9.0", @@ -181,7 +185,7 @@ "temp-dir": "^3.0.0", "tsc-files": "^1.1.4", "tsup": "^8.3.0", - "tsx": "^4.19.0", + "tsx": "^4.19.1", "typescript": "^5.5.4", "typescript-eslint": "^8.2.0", "vite-tsconfig-paths": "^5.0.1", diff --git a/tests/examples/examples.test.ts b/tests/examples/examples.test.ts new file mode 100644 index 0000000..8ffd28f --- /dev/null +++ b/tests/examples/examples.test.ts @@ -0,0 +1,41 @@ +/** + * Copyright 2024 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { execSync } from "child_process"; +import { glob } from "glob"; +import { Semaphore } from "async-mutex"; + +const pattern = process.env.EXAMPLE_NAME || `./examples/**/*.ts`; + +describe("E2E Examples", async () => { + const semaphore = new Semaphore(5); + const exampleFiles = await glob(pattern, { + cwd: process.cwd(), + dot: false, + realpath: true, + }); + + for (const example of exampleFiles) { + it.concurrent(`Run ${example}`, async () => { + await semaphore.acquire(); + try { + execSync(`yarn start -- ${example}`).toString(); + } finally { + semaphore.release(); + } + }); + } +}); diff --git a/tests/setup.examples.ts b/tests/setup.examples.ts new file mode 100644 index 0000000..b840865 --- /dev/null +++ b/tests/setup.examples.ts @@ -0,0 +1,36 @@ +/** + * Copyright 2024 IBM Corp. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import dotenv from "dotenv"; +import { FrameworkError } from "@/errors.js"; +dotenv.config(); +dotenv.config({ + path: ".env.test", + override: true, +}); +dotenv.config({ + path: ".env.test.local", + override: true, +}); + +expect.addSnapshotSerializer({ + serialize(val: FrameworkError): string { + return val.explain(); + }, + test(val): boolean { + return val && val instanceof FrameworkError; + }, +}); diff --git a/vitest.examples.config.ts b/vitest.examples.config.ts new file mode 100644 index 0000000..a00aba0 --- /dev/null +++ b/vitest.examples.config.ts @@ -0,0 +1,24 @@ +import { defineConfig } from "vitest/config"; +import tsConfigPaths from "vite-tsconfig-paths"; +import packageJson from "./package.json"; + +export default defineConfig({ + test: { + globals: true, + passWithNoTests: true, + testTimeout: 120 * 1000, + printConsoleTrace: true, + setupFiles: ["./tests/setup.examples.ts"], + deps: { + interopDefault: false, + }, + }, + define: { + __LIBRARY_VERSION: JSON.stringify(packageJson.version), + }, + plugins: [ + tsConfigPaths({ + projects: ["tsconfig.json"], + }), + ], +}); diff --git a/yarn.lock b/yarn.lock index ddb46d2..3f34837 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2046,6 +2046,16 @@ __metadata: languageName: node linkType: hard +"@types/glob@npm:^8.1.0": + version: 8.1.0 + resolution: "@types/glob@npm:8.1.0" + dependencies: + "@types/minimatch": "npm:^5.1.2" + "@types/node": "npm:*" + checksum: 10c0/ded07aa0d7a1caf3c47b85e262be82989ccd7933b4a14712b79c82fd45a239249811d9fc3a135b3e9457afa163e74a297033d7245b0dc63cd3d032f3906b053f + languageName: node + linkType: hard + "@types/js-yaml@npm:^4.0.9": version: 4.0.9 resolution: "@types/js-yaml@npm:4.0.9" @@ -2085,6 +2095,13 @@ __metadata: languageName: node linkType: hard +"@types/minimatch@npm:^5.1.2": + version: 5.1.2 + resolution: "@types/minimatch@npm:5.1.2" + checksum: 10c0/83cf1c11748891b714e129de0585af4c55dd4c2cafb1f1d5233d79246e5e1e19d1b5ad9e8db449667b3ffa2b6c80125c429dbee1054e9efb45758dbc4e118562 + languageName: node + linkType: hard + "@types/ms@npm:*": version: 0.7.34 resolution: "@types/ms@npm:0.7.34" @@ -2682,6 +2699,15 @@ __metadata: languageName: node linkType: hard +"async-mutex@npm:^0.5.0": + version: 0.5.0 + resolution: "async-mutex@npm:0.5.0" + dependencies: + tslib: "npm:^2.4.0" + checksum: 10c0/9096e6ad6b674c894d8ddd5aa4c512b09bb05931b8746ebd634952b05685608b2b0820ed5c406e6569919ff5fe237ab3c491e6f2887d6da6b6ba906db3ee9c32 + languageName: node + linkType: hard + "async-retry@npm:1.3.3": version: 1.3.3 resolution: "async-retry@npm:1.3.3" @@ -2762,6 +2788,7 @@ __metadata: "@types/eslint": "npm:^9.6.1" "@types/eslint-config-prettier": "npm:^6.11.3" "@types/eslint__js": "npm:^8.42.3" + "@types/glob": "npm:^8.1.0" "@types/js-yaml": "npm:^4.0.9" "@types/mustache": "npm:^4" "@types/needle": "npm:^3.3.0" @@ -2770,6 +2797,7 @@ __metadata: "@types/turndown": "npm:^5.0.5" ajv: "npm:^8.17.1" ajv-formats: "npm:^3.0.1" + async-mutex: "npm:^0.5.0" bee-proto: "npm:0.0.2" dirty-json: "npm:0.9.2" dotenv: "npm:^16.4.5" @@ -2817,7 +2845,7 @@ __metadata: temp-dir: "npm:^3.0.0" tsc-files: "npm:^1.1.4" tsup: "npm:^8.3.0" - tsx: "npm:^4.19.0" + tsx: "npm:^4.19.1" turndown: "npm:^7.2.0" typescript: "npm:^5.5.4" typescript-eslint: "npm:^8.2.0" @@ -10078,9 +10106,9 @@ __metadata: languageName: node linkType: hard -"tsx@npm:^4.19.0": - version: 4.19.0 - resolution: "tsx@npm:4.19.0" +"tsx@npm:^4.19.1": + version: 4.19.1 + resolution: "tsx@npm:4.19.1" dependencies: esbuild: "npm:~0.23.0" fsevents: "npm:~2.3.3" @@ -10090,7 +10118,7 @@ __metadata: optional: true bin: tsx: dist/cli.mjs - checksum: 10c0/d14463a78067c6db84c677b79b14861de6d7f6fb0ffa5727cc500c4552459e936395a3854ad0112af0fd7b263bcedd62ce3929b036188eb10cd9902a607ffe34 + checksum: 10c0/cbea9baf57e7406fa0ecc2c03b9bb2501ee740dc28c938f949180a646a28e5d65e7cccbfba340508923bfd45e90320ef9eef7f815cae4515b6ef2ee429edc7ee languageName: node linkType: hard