Setup integration test job and create initial OpenAI agent instrument integration tests

.github/workflows/ci.yml

@@ -85,6 +85,16 @@ jobs:
     - name: Run tests
       run: npm test
 
+    - name: Run integration tests
+      env:
+        AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+        AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
+        AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }}
+        AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
+        ENABLE_OBSERVABILITY: ${{ vars.ENABLE_OBSERVABILITY }}
+      run: |
+        npm run test:integration
+
     - name: Pack packages
       run: npm run pack
 

jest.integration.config.cjs

@@ -0,0 +1,27 @@
+// ------------------------------------------------------------------------------
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// ------------------------------------------------------------------------------
+
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  rootDir: '.',
+  testMatch: ['**/tests/observability/integration/**/*.test.ts'],
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json'],
+  collectCoverageFrom: [
+    'tests/observability/integration/**/*.ts',
+    '!tests/observability/integration/**/*.d.ts',
+    '!tests/observability/integration/**/*.test.ts',
+  ],
+  coverageDirectory: 'coverage/integration',
+  setupFilesAfterEnv: ['<rootDir>/tests/observability/integration/setup.ts'],
+  testTimeout: 60000,
+  globals: {
+    'ts-jest': {
+      tsconfig: {
+        module: 'commonjs',
+        esModuleInterop: true,
+      },
+    },
+  },
+};

package.json

@@ -8,8 +8,10 @@
     "build": "pnpm -r build",
     "build:watch": "npm run build:watch --workspaces --if-present",
     "clean": "npm run clean --workspaces --if-present && rimraf node_modules",
-    "test": "npm run test --workspaces --if-present",
+    "test": "npm run test --workspaces --if-present -- --testPathIgnorePatterns=/integration/",
     "test:watch": "npm run test:watch --workspaces --if-present",
+    "test:integration": "jest --config jest.integration.config.cjs",
+    "test:integration:watch": "jest --config jest.integration.config.cjs --watch",
     "lint": "npm run lint --workspaces --if-present",
     "lint:fix": "npm run lint:fix --workspaces --if-present",
     "ci": "npm run ci --workspaces --if-present",

packages/agents-a365-observability-extensions-openai/jest.config.json

@@ -1,7 +1,7 @@
 {
   "preset": "ts-jest",
   "testEnvironment": "node",
-  "roots": ["<rootDir>/src", "<rootDir>/../../tests/agents-a365-observability-extensions-openai"],
+  "roots": ["<rootDir>/src", "<rootDir>/../../tests/observability/extension/openai"],
   "testMatch": [
     "**/__tests__/**/*.ts",
     "**/?(*.)+(spec|test).ts"

packages/agents-a365-observability-extensions-openai/src/Constants.ts

@@ -34,3 +34,4 @@ export const GEN_AI_GRAPH_NODE_PARENT_ID = 'graph_node_parent_id';
 
 export const GEN_AI_REQUEST_CONTENT_KEY = 'gen_ai.request.content';
 export const GEN_AI_RESPONSE_CONTENT_KEY = 'gen_ai.response.content';
+export const GEN_AI_EXECUTION_PAYLOAD_KEY = 'gen_ai.execution.payload';

packages/agents-a365-observability-extensions-openai/src/OpenAIAgentsTraceProcessor.ts

@@ -43,6 +43,8 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
     ['mcp_tools' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY],
     ['function' + Constants.GEN_AI_RESPONSE_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_EVENT_CONTENT],
     ['function' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_TOOL_ARGS_KEY],
+    ['generation' + Constants.GEN_AI_RESPONSE_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_OUTPUT_MESSAGES_KEY],
+    ['generation' + Constants.GEN_AI_REQUEST_CONTENT_KEY, OpenTelemetryConstants.GEN_AI_INPUT_MESSAGES_KEY],
   ]);
 
   constructor(tracer: OtelTracer) {
@@ -119,6 +121,10 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
       parentContext
     );
 
+    if (!parentSpan) {
+      this.rootSpans.set(traceId, otelSpan);
+    }
+
     // Store span and activate context
     this.otelSpans.set(spanId, otelSpan);
     this.spanNames.set(otelSpan, spanName);
@@ -269,8 +275,11 @@ export class OpenAIAgentsTraceProcessor implements TracingProcessor {
   private processGenerationSpanData(otelSpan: OtelSpan, data: SpanData, traceId: string): void {
     const attrs = Utils.getAttributesFromGenerationSpanData(data);
     Object.entries(attrs).forEach(([key, value]) => {
-      if (value !== null && value !== undefined) {
-        otelSpan.setAttribute(key, value as string | number | boolean);
+      const shouldExcludeKey = key === OpenTelemetryConstants.GEN_AI_EXECUTION_TYPE_KEY
+        || key === Constants.GEN_AI_EXECUTION_PAYLOAD_KEY;
+      if (value !== null && value !== undefined && !shouldExcludeKey) {
+        const newKey = this.getNewKey(data.type, key);
+        otelSpan.setAttribute(newKey || key, value as string | number | boolean);
       }
     });