Add experiment create wip

Author: Yun-Kim

ddtrace/llmobs/_experiment.py

@@ -45,6 +45,7 @@ def __init__(
         evaluators: List[Callable[[NonNoneJSONType, JSONType, JSONType], JSONType]],
         project_name: str,
         description: str = "",
+        tags: Optional[List[str]] = None,
         config: Optional[Dict[str, Any]] = None,
         _llmobs_instance: Optional["LLMObs"] = None,
     ) -> None:
@@ -53,6 +54,7 @@ def __init__(
         self._dataset = dataset
         self._evaluators = evaluators
         self._description = description
+        self._tags = tags or []
         self._config: Dict[str, Any] = config or {}
         self._llmobs_instance = _llmobs_instance
 
@@ -67,6 +69,15 @@ def __init__(
         self._id: Optional[str] = None
 
     def run(self, jobs: int = 1, raise_errors: bool = False, sample_size: Optional[int] = None) -> None:
+        experiment_id = self._llmobs_instance._create_experiment(
+            name=self.name,
+            dataset_id=self._dataset._id,
+            project_name=self._project_name,
+            exp_config=self._config,
+            tags=self._tags,
+            description=self._description,
+        )
+        self._id = experiment_id
         task_results = self._run_task(jobs, raise_errors, sample_size)
         self._run_evaluators(task_results, raise_errors=raise_errors)
         return

ddtrace/llmobs/_llmobs.py

@@ -581,6 +581,24 @@ def create_dataset(cls, name: str, description: str) -> Dataset:
     def _delete_dataset(cls, dataset_id: str) -> None:
         return cls._instance._dne_client.dataset_delete(dataset_id)
 
+    def _create_experiment(
+        self,
+        name: str,
+        dataset_id: str,
+        project_name: str,
+        dataset_version: int = 1,
+        exp_config: Optional[Dict[str, JSONType]] = None,
+        tags: Optional[List[str]] = None,
+        description: Optional[str] = None,
+    ) -> Tuple[str, str]:
+        project_id = self._dne_client.project_get(project_name)
+        if not project_id:
+            project_id = self._dne_client.project_create(project_name)
+        experiment_id, experiment_name = self._dne_client.experiment_create(
+            name, dataset_id, project_id, dataset_version, exp_config, tags, description
+        )
+        return experiment_id, experiment_name
+
     @classmethod
     def experiment(
         cls,

ddtrace/llmobs/_writer.py

@@ -4,7 +4,9 @@
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Tuple
 from typing import Union
+from typing import cast
 from urllib.parse import quote
 
 
@@ -386,6 +388,43 @@ def project_get(self, name: str) -> str:
             raise ValueError(f"Project {name} not found")
         return data[0]["id"]
 
+    def experiment_create(
+        self,
+        name: str,
+        dataset_id: str,
+        project_id: str,
+        dataset_version: int = 1,
+        exp_config: Optional[Dict[str, JSONType]] = None,
+        tags: Optional[List[str]] = None,
+        description: Optional[str] = None,
+    ) -> Tuple[str, str]:
+        path = "/api/unstable/llm-obs/v1/experiments"
+        resp = self.request(
+            "POST",
+            path,
+            body={
+                "data": {
+                    "type": "experiments",
+                    "attributes": {
+                        "name": name,
+                        "description": description or "",
+                        "dataset_id": dataset_id,
+                        "project_id": project_id,
+                        "dataset_version": dataset_version,
+                        "config": exp_config or {},
+                        "metadata": {"tags": cast(JSONType, tags or [])},
+                        "ensure_unique": True,
+                    },
+                }
+            },
+        )
+        if resp.status != 200:
+            raise ValueError(f"Failed to create experiment {name}: {resp.status} {resp.get_json()}")
+        response_data = resp.get_json()
+        experiment_id = response_data["data"]["id"]
+        experiment_name = response_data["data"]["attributes"]["name"]  # API may rename the experiment
+        return experiment_id, experiment_name
+
 
 class LLMObsSpanWriter(BaseLLMObsWriter):
     """Writes span events to the LLMObs Span Endpoint."""