Setup build pipeline to run online tests

.github/workflows/ci.yml

@@ -1,9 +1,26 @@
 # .github/workflows/ci.yml
 name: CI
-on: [push, pull_request]
+
+on:
+  push:
+    branches: [ "main" ] 
+  pull_request_target:
+    branches: [ "main" ]
+  workflow_dispatch:     # manual run
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true  
+
+permissions:
+  pull-requests: read
+  contents: read
+  id-token: write  
 
 jobs:
   ci:
+    environment:
+      name: build-pipeline
     strategy:
       fail-fast: false
       matrix:
@@ -19,6 +36,29 @@ jobs:
     name: ${{ matrix.os }} ${{ matrix.task }} (py ${{ matrix.python-version }})
 
     steps:
+
+      # The following two steps (permissions checks) ensure that only users with write access can run this workflow on a PR (except the merge queue bot)
+      # PRs from forks we check the permissions of the user that triggered the workflow (github.triggering_actor)
+      # This means that if a user without write access opens a PR from a fork, they cannot run this workflow
+      # Users with write access can still run this workflow on a PR from a fork
+      # For PRs from the same repo, we allow the workflow to run as normal
+      - name: Get User Permission
+        if: ${{ github.event_name == 'pull_request_target' || github.triggering_actor != 'github-merge-queue[bot]' }}
+        id: checkAccess
+        uses: actions-cool/check-user-permission@v2
+        with:
+          require: write
+          username: ${{ github.triggering_actor }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check User Permission
+        if: ${{ (github.event_name == 'pull_request_target' || github.triggering_actor != 'github-merge-queue[bot]') && steps.checkAccess.outputs.require-result == 'false' }}
+        run: |
+          echo "${{ github.triggering_actor }} does not have permissions on this repo."
+          echo "Current permission level is ${{ steps.checkAccess.outputs.user-permission }}"
+          echo "Job originally triggered by ${{ github.actor }}"
+          exit 1  
+
       - uses: actions/checkout@v4
 
       - name: Set up Python
@@ -46,8 +86,25 @@ jobs:
         run: |
             uv run black -tpy312 -tpy313 -tpy314 typeagent test tools gmail demo --check
 
+      - name: Login to Azure
+        if: matrix.task == 'test'
+        uses: azure/login@v2.2.0
+        with:
+          client-id: ${{ secrets.AZURE_CLIENTID }}
+          tenant-id: ${{ secrets.AZURE_TENANTID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTIONID }}
+
+      - name: Get Keys
+        if: matrix.task == 'test'
+        run: |
+            uv run python tools/get_keys.py --vault build-pipeline-kv
+
       - name: Run Test
         shell: bash
         if: matrix.task == 'test'
         run: |
             uv run pytest
+
+      - name: Clean up Keys
+        run: |
+          node -e "try{require('fs').unlinkSync('./.env');}catch(e){}"

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+    "recommendations": [
+        "ms-python.python",
+        "ms-python.debugpy",
+        "ms-python.vscode-pylance",
+        "ms-python.vscode-python-envs"
+    ]
+}

.vscode/launch.json

@@ -0,0 +1,33 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "env": {
+                "PATH": "${env:PATH}"
+            }            
+        },
+        {
+            "name": "Python Debugger: Debug Tests",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "pytest",
+            "args": [
+                "${file}",
+                "-v"
+            ],
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "PATH": "${env:PATH}"
+            }
+        }
+    ]
+}

pyproject.toml

@@ -44,6 +44,8 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
+  "azure-mgmt-authorization>=4.0.0",
+  "azure-mgmt-keyvault>=12.1.1",  
   "coverage[toml]>=7.9.1",
   "google-api-python-client>=2.184.0",
   "google-auth-httplib2>=0.2.0",

test/test_mcp_server.py

@@ -91,6 +91,17 @@ async def test_mcp_server_query_conversation_slow(
     from mcp import ClientSession
     from mcp.client.stdio import stdio_client
 
+    # Pass through environment variables needed for authentication
+    # otherwise this test will fail in the CI on Windows only
+    if not (server_params.env) is None:
+        server_params.env.update(
+            {
+                k: v
+                for k, v in os.environ.items()
+                if k.startswith(("AZURE_", "OPENAI_")) or k in ("CREDENTIALS_JSON",)
+            }
+        )
+
     # Create client session and connect to server
     async with stdio_client(server_params) as (read, write):
         async with ClientSession(
@@ -123,7 +134,13 @@ async def test_mcp_server_query_conversation_slow(
             # Parse response (it should be JSON with success, answer, time_used)
             import json
 
-            response_data = json.loads(response_text)
+            try:
+                response_data = json.loads(response_text)
+            except json.JSONDecodeError as e:
+                pytest.fail(
+                    f"Response is not valid JSON: {e}\nResponse text: {response_text}"
+                )
+
             assert "success" in response_data
             assert "answer" in response_data
             assert "time_used" in response_data

test/test_online.py

@@ -0,0 +1,44 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+import pytest
+
+from fixtures import really_needs_auth  # type: ignore
+
+from typeagent.aitools.utils import create_async_openai_client
+
+
+@pytest.mark.asyncio
+async def test_why_is_sky_blue(really_needs_auth: None):
+    """Test that chat agent responds correctly to 'why is the sky blue?'"""
+
+    # Create an async OpenAI client
+    try:
+        client = create_async_openai_client()
+    except RuntimeError as e:
+        if "Neither OPENAI_API_KEY nor AZURE_OPENAI_API_KEY was provided." in str(e):
+            pytest.skip("API keys not configured")
+        raise
+
+    # Send the user request
+    response = await client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {
+                "role": "user",
+                "content": "why is the sky blue?",
+            }
+        ],
+        temperature=0,
+    )
+
+    # Get the response message
+    msg = response.choices[0].message.content
+    assert msg is not None, "Chat agent didn't respond"
+
+    print(f"Chat agent response: {msg}")
+
+    # Check that the response contains the expected keyword
+    assert (
+        "scatter" in msg.lower()
+    ), "Chat agent didn't respond with the expected message about scattering."

tools/get_keys.config.json

@@ -0,0 +1,40 @@
+{
+  "defaultDotEnvPath": "../.env",
+  "env": {
+    "shared": [
+      "AZURE_OPENAI_ENDPOINT",
+      "AZURE_OPENAI_API_KEY_GPT_v",
+      "AZURE_OPENAI_ENDPOINT_GPT_4_O",
+      "AZURE_OPENAI_ENDPOINT_GPT_4_O_MINI",
+      "AZURE_OPENAI_RESPONSE_FORMAT",
+      "AZURE_OPENAI_MAX_CONCURRENCY",
+      "AZURE_OPENAI_MAX_TIMEOUT",
+      "AZURE_OPENAI_MAX_RETRYATTEMPTS",
+      "AZURE_OPENAI_API_KEY",
+      "AZURE_OPENAI_API_KEY_GPT_35_TURBO",
+      "AZURE_OPENAI_ENDPOINT_GPT_v",
+      "AZURE_OPENAI_API_KEY_GPT_4_O",
+      "AZURE_OPENAI_API_KEY_GPT_4_O_MINI",
+      "AZURE_OPENAI_API_KEY_EMBEDDING",
+      "AZURE_OPENAI_ENDPOINT_EMBEDDING",
+      "AZURE_OPENAI_API_KEY_GPT_5",
+      "AZURE_OPENAI_ENDPOINT_GPT_5",
+      "AZURE_OPENAI_API_KEY_GPT_5_MINI",
+      "AZURE_OPENAI_ENDPOINT_GPT_5_MINI",
+      "AZURE_OPENAI_API_KEY_GPT_5_NANO",
+      "AZURE_OPENAI_ENDPOINT_GPT_5_NANO",
+      "AZURE_OPENAI_API_KEY_GPT_5_CHAT",
+      "AZURE_OPENAI_ENDPOINT_GPT_5_CHAT",
+      "OPENAI_ENDPOINT_LOCAL",
+      "OPENAI_API_KEY_LOCAL",
+      "OPENAI_MODEL_LOCAL",
+      "OPENAI_ORGANIZATION_LOCAL",
+      "OPENAI_RESPONSE_FORMAT_LOCAL"
+    ],
+    "private": [],
+    "delete": []
+  },
+  "vault": {
+    "shared": "aisystems"
+  }
+}