Add LLM Observability to ListStreamSource

.apigentools-info

@@ -4,13 +4,13 @@
     "spec_versions": {
         "v1": {
             "apigentools_version": "1.6.6",
-            "regenerated": "2024-10-16 20:07:12.083214",
-            "spec_repo_commit": "86072741"
+            "regenerated": "2024-10-17 14:10:58.758527",
+            "spec_repo_commit": "fb024a45"
         },
         "v2": {
             "apigentools_version": "1.6.6",
-            "regenerated": "2024-10-16 20:07:12.101717",
-            "spec_repo_commit": "86072741"
+            "regenerated": "2024-10-17 14:10:58.777345",
+            "spec_repo_commit": "fb024a45"
         }
     }
 }

.generator/schemas/v1/openapi.yaml

@@ -4796,6 +4796,7 @@ components:
       - logs_transaction_stream
       - event_stream
       - rum_stream
+      - llm_observability_stream
       example: apm_issue_stream
       type: string
       x-enum-varnames:
@@ -4811,6 +4812,7 @@ components:
       - LOGS_TRANSACTION_STREAM
       - EVENT_STREAM
       - RUM_STREAM
+      - LLM_OBSERVABILITY_STREAM
     ListStreamWidgetDefinition:
       description: 'The list stream visualization displays a table of recent events
         in your application that

cassettes/v1/Dashboards_1335235736/Create-a-new-dashboard-with-llm_observability_stream-list_stream-widget_3300240229/frozen.json

@@ -0,0 +1 @@
+"2024-10-15T21:46:06.749Z"

cassettes/v1/Dashboards_1335235736/Create-a-new-dashboard-with-llm_observability_stream-list_stream-widget_3300240229/recording.har

@@ -0,0 +1,110 @@
+{
+  "log": {
+    "_recordingName": "Dashboards/Create a new dashboard with llm_observability_stream list_stream widget",
+    "creator": {
+      "comment": "persister:fs",
+      "name": "Polly.JS",
+      "version": "6.0.5"
+    },
+    "entries": [
+      {
+        "_id": "cf641f4f8e4f7935f5b44afac4518e18",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 759,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            },
+            {
+              "_fromType": "array",
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 559,
+          "httpVersion": "HTTP/1.1",
+          "method": "POST",
+          "postData": {
+            "mimeType": "application/json",
+            "params": [],
+            "text": "{\"layout_type\":\"ordered\",\"title\":\"Test-Create_a_new_dashboard_with_llm_observability_stream_list_stream_widget-1729028766 with list_stream widget\",\"widgets\":[{\"definition\":{\"requests\":[{\"columns\":[{\"field\":\"@status\",\"width\":\"compact\"},{\"field\":\"@content.prompt\",\"width\":\"auto\"},{\"field\":\"@content.response.content\",\"width\":\"auto\"},{\"field\":\"timestamp\",\"width\":\"auto\"},{\"field\":\"@ml_app\",\"width\":\"auto\"},{\"field\":\"service\",\"width\":\"auto\"},{\"field\":\"@meta.evaluations.quality\",\"width\":\"auto\"},{\"field\":\"@meta.evaluations.security\",\"width\":\"auto\"},{\"field\":\"@duration\",\"width\":\"auto\"}],\"query\":{\"data_source\":\"llm_observability_stream\",\"indexes\":[],\"query_string\":\"@event_type:span @parent_id:undefined\"},\"response_format\":\"event_list\"}],\"type\":\"list_stream\"}}]}"
+          },
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/dashboard"
+        },
+        "response": {
+          "bodySize": 1199,
+          "content": {
+            "mimeType": "application/json",
+            "size": 1199,
+            "text": "{\"id\":\"k3w-qcg-ug8\",\"title\":\"Test-Create_a_new_dashboard_with_llm_observability_stream_list_stream_widget-1729028766 with list_stream widget\",\"description\":null,\"author_handle\":\"9919ec9b-ebc7-49ee-8dc8-03626e717cca\",\"author_name\":\"CI Account\",\"layout_type\":\"ordered\",\"url\":\"/dashboard/k3w-qcg-ug8/test-createanewdashboardwithllmobservabilitystreamliststreamwidget-1729028766-wi\",\"is_read_only\":false,\"template_variables\":null,\"widgets\":[{\"definition\":{\"requests\":[{\"columns\":[{\"field\":\"@status\",\"width\":\"compact\"},{\"field\":\"@content.prompt\",\"width\":\"auto\"},{\"field\":\"@content.response.content\",\"width\":\"auto\"},{\"field\":\"timestamp\",\"width\":\"auto\"},{\"field\":\"@ml_app\",\"width\":\"auto\"},{\"field\":\"service\",\"width\":\"auto\"},{\"field\":\"@meta.evaluations.quality\",\"width\":\"auto\"},{\"field\":\"@meta.evaluations.security\",\"width\":\"auto\"},{\"field\":\"@duration\",\"width\":\"auto\"}],\"query\":{\"data_source\":\"llm_observability_stream\",\"indexes\":[],\"query_string\":\"@event_type:span @parent_id:undefined\"},\"response_format\":\"event_list\"}],\"type\":\"list_stream\"},\"id\":8221646523831060}],\"notify_list\":null,\"created_at\":\"2024-10-15T21:46:06.954265+00:00\",\"modified_at\":\"2024-10-15T21:46:06.954265+00:00\",\"restricted_roles\":[]}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 658,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2024-10-15T21:46:06.757Z",
+        "time": 272
+      },
+      {
+        "_id": "a917abb5e3048f7de7b8bd4e3f095748",
+        "_order": 0,
+        "cache": {},
+        "request": {
+          "bodySize": 0,
+          "cookies": [],
+          "headers": [
+            {
+              "_fromType": "array",
+              "name": "accept",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 520,
+          "httpVersion": "HTTP/1.1",
+          "method": "DELETE",
+          "queryString": [],
+          "url": "https://api.datadoghq.com/api/v1/dashboard/k3w-qcg-ug8"
+        },
+        "response": {
+          "bodySize": 39,
+          "content": {
+            "mimeType": "application/json",
+            "size": 39,
+            "text": "{\"deleted_dashboard_id\":\"k3w-qcg-ug8\"}\n"
+          },
+          "cookies": [],
+          "headers": [
+            {
+              "name": "content-type",
+              "value": "application/json"
+            }
+          ],
+          "headersSize": 656,
+          "httpVersion": "HTTP/1.1",
+          "redirectURL": "",
+          "status": 200,
+          "statusText": "OK"
+        },
+        "startedDateTime": "2024-10-15T21:46:07.036Z",
+        "time": 222
+      }
+    ],
+    "pages": [],
+    "version": "1.2"
+  }
+}

examples/v1/dashboards/CreateDashboard_3298564989.ts

@@ -0,0 +1,79 @@
+/**
+ * Create a new dashboard with llm_observability_stream list_stream widget
+ */
+
+import { client, v1 } from "@datadog/datadog-api-client";
+
+const configuration = client.createConfiguration();
+const apiInstance = new v1.DashboardsApi(configuration);
+
+const params: v1.DashboardsApiCreateDashboardRequest = {
+  body: {
+    layoutType: "ordered",
+    title: "Example-Dashboard with list_stream widget",
+    widgets: [
+      {
+        definition: {
+          type: "list_stream",
+          requests: [
+            {
+              responseFormat: "event_list",
+              query: {
+                dataSource: "llm_observability_stream",
+                queryString: "@event_type:span @parent_id:undefined",
+                indexes: [],
+              },
+              columns: [
+                {
+                  field: "@status",
+                  width: "compact",
+                },
+                {
+                  field: "@content.prompt",
+                  width: "auto",
+                },
+                {
+                  field: "@content.response.content",
+                  width: "auto",
+                },
+                {
+                  field: "timestamp",
+                  width: "auto",
+                },
+                {
+                  field: "@ml_app",
+                  width: "auto",
+                },
+                {
+                  field: "service",
+                  width: "auto",
+                },
+                {
+                  field: "@meta.evaluations.quality",
+                  width: "auto",
+                },
+                {
+                  field: "@meta.evaluations.security",
+                  width: "auto",
+                },
+                {
+                  field: "@duration",
+                  width: "auto",
+                },
+              ],
+            },
+          ],
+        },
+      },
+    ],
+  },
+};
+
+apiInstance
+  .createDashboard(params)
+  .then((data: v1.Dashboard) => {
+    console.log(
+      "API called successfully. Returned data: " + JSON.stringify(data)
+    );
+  })
+  .catch((error: any) => console.error(error));

features/v1/dashboards.feature

@@ -500,6 +500,14 @@ Feature: Dashboards
     And the response "widgets[0].definition.requests[0].query.sort.column" is equal to "timestamp"
     And the response "widgets[0].definition.requests[0].query.sort.order" is equal to "desc"
 
+  @team:DataDog/dashboards-backend
+  Scenario: Create a new dashboard with llm_observability_stream list_stream widget
+    Given new "CreateDashboard" request
+    And body with value {"layout_type":"ordered","title":"{{ unique }} with list_stream widget","widgets":[{"definition":{"type":"list_stream","requests":[{"response_format":"event_list","query":{"data_source":"llm_observability_stream","query_string":"@event_type:span @parent_id:undefined","indexes":[]},"columns":[{"field":"@status","width":"compact"},{"field":"@content.prompt","width":"auto"},{"field":"@content.response.content","width":"auto"},{"field":"timestamp","width":"auto"},{"field":"@ml_app","width":"auto"},{"field":"service","width":"auto"},{"field":"@meta.evaluations.quality","width":"auto"},{"field":"@meta.evaluations.security","width":"auto"},{"field":"@duration","width":"auto"}]}]}}]}
+    When the request is sent
+    Then the response status is 200 OK
+    And the response "widgets[0].definition.requests[0].query.data_source" is equal to "llm_observability_stream"
+
   @team:DataDog/dashboards-backend
   Scenario: Create a new dashboard with log_stream widget
     Given new "CreateDashboard" request

packages/datadog-api-client-v1/models/ListStreamSource.ts

@@ -23,6 +23,7 @@ export type ListStreamSource =
   | typeof LOGS_TRANSACTION_STREAM
   | typeof EVENT_STREAM
   | typeof RUM_STREAM
+  | typeof LLM_OBSERVABILITY_STREAM
   | UnparsedObject;
 export const LOGS_STREAM = "logs_stream";
 export const AUDIT_STREAM = "audit_stream";
@@ -36,3 +37,4 @@ export const LOGS_PATTERN_STREAM = "logs_pattern_stream";
 export const LOGS_TRANSACTION_STREAM = "logs_transaction_stream";
 export const EVENT_STREAM = "event_stream";
 export const RUM_STREAM = "rum_stream";
+export const LLM_OBSERVABILITY_STREAM = "llm_observability_stream";

packages/datadog-api-client-v1/models/ObjectSerializer.ts

@@ -899,6 +899,7 @@ const enumsMap: { [key: string]: any[] } = {
     "logs_transaction_stream",
     "event_stream",
     "rum_stream",
+    "llm_observability_stream",
   ],
   ListStreamWidgetDefinitionType: ["list_stream"],
   LogStreamWidgetDefinitionType: ["log_stream"],