Copilot Chat: Scenario Tests

.github/workflows/copilot-chat-tests.yml

@@ -0,0 +1,86 @@
+name: Copilot Chat Tests
+on:
+  workflow_dispatch:
+  push:
+    branches: [ "main", "feature*" ]
+    paths: 
+      - 'samples/apps/copilot-chat-app/**'
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    defaults:
+      run:
+        working-directory: samples/apps/copilot-chat-app/webapp
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v3
+
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16
+        cache-dependency-path: samples/apps/copilot-chat-app/webapp/yarn.lock
+        cache: 'yarn'
+
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v3
+      with:
+        dotnet-version: 6.0.x
+
+    - name: Install dependencies
+      run: yarn install
+
+    - name: Install Playwright Browsers
+      run: yarn playwright install --with-deps
+
+    - name: Update AIService configuration
+      working-directory: samples/apps/copilot-chat-app/webapi
+      env:
+        AzureOpenAI__ApiKey: ${{ secrets.AZUREOPENAI__APIKEY }}
+        AzureOpenAI__Endpoint: ${{ secrets.AZUREOPENAI__ENDPOINT }}
+      run: |
+        dotnet dev-certs https
+        dotnet user-secrets set "AIService:Key" "$AzureOpenAI__ApiKey"
+        dotnet user-secrets set "AIService:Endpoint" "$AzureOpenAI__Endpoint"
+
+    - name: Start service in background
+      working-directory: samples/apps/copilot-chat-app/webapi
+      run: |
+        dotnet run > service-log.txt 2>&1 &
+        for attempt in {0..20}; do
+          jobs
+          echo 'Waiting for service to start...';
+          if curl -k https://localhost:40443/healthz; then
+            echo;
+            echo 'Service started';
+            break;
+          fi;
+
+          sleep 5;
+        done
+
+    - name: Run Playwright tests
+      env:
+        REACT_APP_BACKEND_URI: https://localhost:40443/
+        REACT_APP_AAD_CLIENT_ID: ${{ secrets.COPILOT_CHAT_REACT_APP_AAD_CLIENT_ID }}
+        REACT_APP_AAD_AUTHORITY: https://login.microsoftonline.com/common
+        REACT_APP_TEST_USER_ACCOUNT: ${{ secrets.COPILOT_CHAT_TEST_USER_ACCOUNT }}
+        REACT_APP_TEST_USER_PASSWORD: ${{ secrets.COPILOT_CHAT_TEST_USER_PASSWORD }}
+      run: yarn playwright test
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: playwright-report
+        path: samples/apps/copilot-chat-app/webapp/playwright-report/
+        retention-days: 30
+
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: service-log
+        path: samples/apps/copilot-chat-app/webapi/service-log.txt
+        retention-days: 30

.gitignore

@@ -469,4 +469,7 @@ java/**/target
 java/.mvn/wrapper/maven-wrapper.jar
 
 # Java settings
-conf.properties
+conf.properties
+
+# Playwright
+playwright-report/

samples/apps/copilot-chat-app/webapp/.env.example

@@ -12,4 +12,8 @@ REACT_APP_SK_API_KEY=
 # Replace with your locally-trusted cert file
 # SSL_CRT_FILE=local-cert.crt
 # Replace with your locally-trusted cert key
-# SSL_KEY_FILE=local-cert.key
+# SSL_KEY_FILE=local-cert.key
+
+# For CI and testing purposes only
+REACT_APP_TEST_USER_ACCOUNT=
+REACT_APP_TEST_USER_PASSWORD=

samples/apps/copilot-chat-app/webapp/package.json

@@ -22,6 +22,7 @@
     "@fluentui/react-components": "^9.13.0",
     "@fluentui/react-icons": "^2.0.193",
     "@fluentui/react-northstar": "^0.66.4",
+    "@playwright/test": "^1.34.3",
     "@reduxjs/toolkit": "^1.9.1",
     "debug": "^4.3.4",
     "microsoft-cognitiveservices-speech-sdk": "^1.27.0",

samples/apps/copilot-chat-app/webapp/playwright.config.ts

@@ -0,0 +1,59 @@
+import { defineConfig, devices } from '@playwright/test';
+import dotenv from 'dotenv';
+import fs from 'fs';
+import path from 'path';
+import { fileURLToPath } from 'url';
+
+// Read the .env file if it exists. This is usually for testing locally.
+// Tests on CI should should get the environment variables from GitHub actions.
+const filename = fileURLToPath(import.meta.url);
+const dirname = path.dirname(filename);
+const envPath = path.resolve(dirname, '.env');
+if (fs.existsSync(envPath)) {
+  dotenv.config({ path: envPath });
+}
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+
+    /* Ignore certificate errors. */
+    ignoreHTTPSErrors: true,
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+    // Add more browsers here.
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: {
+    command: 'yarn start',
+    url: 'http://localhost:3000',
+    reuseExistingServer: !process.env.CI,
+    timeout: 120000,
+  },
+});

samples/apps/copilot-chat-app/webapp/src/components/chat/ChatHistoryItem.tsx

@@ -151,7 +151,11 @@ export const ChatHistoryItem: React.FC<ChatHistoryItemProps> = ({ message, getRe
 
     return (
         <>
-            <div className={isMe ? mergeClasses(classes.root, classes.alignEnd) : classes.root}>
+            <div
+                className={isMe ? mergeClasses(classes.root, classes.alignEnd) : classes.root}
+                data-testid={`chat-history-item-${messageIndex}`}
+                data-username={fullName}
+            >
                 {!isMe && <Persona className={classes.persona} avatar={avatar} presence={{ status: 'available' }} />}
                 <div className={isMe ? mergeClasses(classes.item, classes.me) : classes.item}>
                     <div className={classes.header}>

samples/apps/copilot-chat-app/webapp/src/components/views/BackendProbe.tsx

@@ -10,18 +10,22 @@ interface IData {
 
 const BackendProbe: FC<IData> = ({ uri, onBackendFound }) => {
     useEffect(() => {
-        const requestUrl = new URL('healthz', uri);
-        const fetchAsync = async () => {
-            try {
-                var result = await fetch(requestUrl);
-
-                if (result.ok) {
-                    onBackendFound();
-                }
-            } catch {}
-        };
-
-        fetchAsync();
+        const timer = setInterval(() => {
+            const requestUrl = new URL('healthz', uri);
+            const fetchAsync = async () => {
+                try {
+                    var result = await fetch(requestUrl);
+
+                    if (result.ok) {
+                        onBackendFound();
+                    }
+                } catch { }
+            };
+
+            fetchAsync();
+        }, 3000);
+
+        return () => clearInterval(timer);
     });
 
     return (

samples/apps/copilot-chat-app/webapp/tests/chat.test.ts

@@ -0,0 +1,51 @@
+import { expect, test } from '@playwright/test';
+
+/*
+This test is a simple end-to-end test that verifies the app can be
+loaded, and signed in. The user can send a message to the bot, and expect
+a response.
+*/
+test('get response from bot', async ({ page }) => {
+    // Make sure the server is running.
+    await page.goto('https://localhost:40443/healthz');
+    expect(page.getByText('Healthy')).toBeDefined();
+
+    await page.goto('/');
+    // Expect the page to contain a "Login" button.
+    await page.getByRole('button').click();
+    // Clicking the login button should redirect to the login page.
+    await expect(page).toHaveURL(new RegExp('^' + process.env.REACT_APP_AAD_AUTHORITY));
+    // Login with the test user.
+    await page.getByPlaceholder('Email, phone, or Skype').click();
+    await page.getByPlaceholder('Email, phone, or Skype').fill(process.env.REACT_APP_TEST_USER_ACCOUNT as string);
+    await page.getByRole('button', { name: 'Next' }).click();
+    await page.getByPlaceholder('Password').click();
+    await page.getByPlaceholder('Password').fill(process.env.REACT_APP_TEST_USER_PASSWORD as string);
+    await page.getByRole('button', { name: 'Sign in' }).click();
+
+    // Select No if asked to stay signed in.
+    const isAskingStaySignedIn = await page.$$("text='Stay signed in?'");
+    if (isAskingStaySignedIn) {
+        await page.getByRole('button', { name: 'No' }).click();
+    }
+
+    // After login, the page should redirect back to the app.
+    await expect(page).toHaveTitle('Copilot Chat');
+
+    // Send a message to the bot and wait for the response.
+    const responsePromise = page.waitForResponse('**/chat');
+    await page.locator('#chat-input').click();
+    await page.locator('#chat-input').fill('Hi!');
+    await page.locator('#chat-input').press('Enter');
+    await responsePromise;
+
+    // Expect the chat history to contain 3 messages.
+    // The first message is the welcome message from the bot.
+    // The second message is the user's message.
+    // The third message is the bot's response.
+    const chatHistoryItems = page.getByTestId(new RegExp('chat-history-item-*'));
+    expect((await chatHistoryItems.all()).length).toBe(3);
+
+    // Expect the third message to be the bot's response.
+    await expect(chatHistoryItems.nth(2)).toHaveAttribute('data-username', 'Copilot');
+});

samples/apps/copilot-chat-app/webapp/yarn.lock

@@ -2508,6 +2508,16 @@
     "@nodelib/fs.scandir" "2.1.5"
     fastq "^1.6.0"
 
+"@playwright/test@^1.34.3":
+  version "1.34.3"
+  resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.34.3.tgz#d9f1ac3f1a09633b5ca5351c50c308bf802bde53"
+  integrity sha512-zPLef6w9P6T/iT6XDYG3mvGOqOyb6eHaV9XtkunYs0+OzxBtrPAAaHotc0X+PJ00WPPnLfFBTl7mf45Mn8DBmw==
+  dependencies:
+    "@types/node" "*"
+    playwright-core "1.34.3"
+  optionalDependencies:
+    fsevents "2.3.2"
+
 "@pmmmwh/react-refresh-webpack-plugin@^0.5.3":
   version "0.5.10"
   resolved "https://registry.yarnpkg.com/@pmmmwh/react-refresh-webpack-plugin/-/react-refresh-webpack-plugin-0.5.10.tgz#2eba163b8e7dbabb4ce3609ab5e32ab63dda3ef8"
@@ -6147,7 +6157,7 @@ fs.realpath@^1.0.0:
   resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
   integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==
 
-fsevents@^2.3.2, fsevents@~2.3.2:
+fsevents@2.3.2, fsevents@^2.3.2, fsevents@~2.3.2:
   version "2.3.2"
   resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
   integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==
@@ -8667,6 +8677,11 @@ pkg-up@^3.1.0:
   dependencies:
     find-up "^3.0.0"
 
+playwright-core@1.34.3:
+  version "1.34.3"
+  resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.34.3.tgz#bc906ea1b26bb66116ce329436ee59ba2e78fe9f"
+  integrity sha512-2pWd6G7OHKemc5x1r1rp8aQcpvDh7goMBZlJv6Co5vCNLVcQJdhxRL09SGaY6HcyHH9aT4tiynZabMofVasBYw==
+
 please-upgrade-node@^3.2.0:
   version "3.2.0"
   resolved "https://registry.yarnpkg.com/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz#aeddd3f994c933e4ad98b99d9a556efa0e2fe942"