Added evaluation span and event. Added log api for event

Author: wrisa

instrumentation-genai/opentelemetry-genai-sdk/src/opentelemetry/genai/sdk/api.py

@@ -25,6 +25,7 @@
 from opentelemetry.metrics import get_meter
 from opentelemetry.trace import get_tracer
 from opentelemetry._events import get_event_logger
+from opentelemetry._logs import get_logger
 from opentelemetry.semconv.schemas import Schemas
 
 
@@ -49,8 +50,13 @@ def __init__(self, exporter_type_full: bool = True, **kwargs):
             __name__, __version__, event_logger_provider=event_logger_provider, schema_url=Schemas.V1_28_0.value
         )
 
+        logger_provider = kwargs.get("logger_provider")
+        self._logger = get_logger(
+            __name__, __version__, logger_provider=logger_provider, schema_url=Schemas.V1_28_0.value
+        )
+
         self._exporter = (
-            SpanMetricEventExporter(tracer=self._tracer, meter=self._meter, event_logger=self._event_logger)
+            SpanMetricEventExporter(tracer=self._tracer, meter=self._meter, event_logger=self._event_logger, logger=self._event_logger)
             if exporter_type_full
             else SpanMetricExporter(tracer=self._tracer, meter=self._meter)
         )

instrumentation-genai/opentelemetry-genai-sdk/src/opentelemetry/genai/sdk/evals.py

@@ -1,5 +1,15 @@
 from abc import ABC, abstractmethod
+from opentelemetry._events import Event
+
 from .types import LLMInvocation
+from opentelemetry import trace
+from opentelemetry.trace import (
+    Tracer,
+)
+from opentelemetry import _events
+from .deepeval import evaluate_answer_relevancy_metric
+from opentelemetry.trace import SpanContext, Span
+from opentelemetry.trace.span import NonRecordingSpan
 
 
 class EvaluationResult:
@@ -22,20 +32,75 @@ def evaluate(self, invocation: LLMInvocation) -> EvaluationResult:
         """
         pass
 
-class DeepEvalsEvaluator(Evaluator):
+class DeepEvalEvaluator(Evaluator):
     """
     Uses DeepEvals library for LLM-as-judge evaluations.
     """
-    def __init__(self, config: dict = None):
+    def __init__(self, event_logger, tracer: Tracer = None, config: dict = None):
         # e.g. load models, setup API keys
         self.config = config or {}
+        self._tracer = tracer or trace.get_tracer(__name__)
+        self._event_logger = event_logger or _events.get_event_logger(__name__)
 
-    def evaluate(self, invocation: LLMInvocation) -> EvaluationResult:
+    def evaluate(self, invocation: LLMInvocation):
         # stub: integrate with deepevals SDK
         # result = deepevals.judge(invocation.prompt, invocation.response, **self.config)
-        score = 0.0  # placeholder
-        details = {"method": "deepevals"}
-        return EvaluationResult(score=score, details=details)
+        human_message = next((msg for msg in invocation.messages if msg.type == "human"), None)
+        content = invocation.chat_generations[0].content
+        if content is not None and content != "":
+            eval_arm = evaluate_answer_relevancy_metric(human_message.content, invocation.chat_generations[0].content, [])
+            self._do_telemetry(invocation.messages[1].content, invocation.chat_generations[0].content,
+                               invocation.span_id, invocation.trace_id, eval_arm)
+
+    def _do_telemetry(self, query, output, parent_span_id, parent_trace_id, eval_arm):
+
+        # emit event
+        body = {
+            "content": f"query: {query} output: {output}",
+        }
+        attributes = {
+            "gen_ai.evaluation.name": "relevance",
+            "gen_ai.evaluation.score": eval_arm.score,
+            "gen_ai.evaluation.reasoning": eval_arm.reason,
+            "gen_ai.evaluation.cost": eval_arm.evaluation_cost,
+        }
+
+        event = Event(
+            name="gen_ai.evaluation.message",
+            attributes=attributes,
+            body=body if body else None,
+            span_id=parent_span_id,
+            trace_id=parent_trace_id,
+        )
+        self._event_logger.emit(event)
+
+        # create span
+        span_context = SpanContext(
+            trace_id=parent_trace_id,
+            span_id=parent_span_id,
+            is_remote=False,
+        )
+
+        span = NonRecordingSpan(
+            context=span_context,
+        )
+
+        tracer = trace.get_tracer(__name__)
+
+        with tracer.start_as_current_span("evaluation relevance") as span:
+            # do evaluation
+
+            span.add_link(span_context, attributes={
+                "gen_ai.operation.name": "evaluation",
+            })
+            span.set_attribute("gen_ai.operation.name", "evaluation")
+            span.set_attribute("gen_ai.evaluation.name", "relevance")
+            span.set_attribute("gen_ai.evaluation.score", eval_arm.score)
+            span.set_attribute("gen_ai.evaluation.label", "Pass")
+            span.set_attribute("gen_ai.evaluation.reasoning", eval_arm.reason)
+            span.set_attribute("gen_ai.evaluation.model", eval_arm.evaluation_model)
+            span.set_attribute("gen_ai.evaluation.cost", eval_arm.evaluation_cost)
+            #span.set_attribute("gen_ai.evaluation.verdict", eval_arm.verdicts)
 
 
 class OpenLitEvaluator(Evaluator):
@@ -54,16 +119,16 @@ def evaluate(self, invocation: LLMInvocation) -> EvaluationResult:
 
 # Registry for easy lookup
 EVALUATORS = {
-    "deepevals": DeepEvalsEvaluator,
+    "deepeval": DeepEvalEvaluator,
     "openlit": OpenLitEvaluator,
 }
 
 
-def get_evaluator(name: str, config: dict = None) -> Evaluator:
+def get_evaluator(name: str, event_logger = None, tracer: Tracer = None, config: dict = None) -> Evaluator:
     """
     Factory: return an evaluator by name.
     """
     cls = EVALUATORS.get(name.lower())
     if not cls:
         raise ValueError(f"Unknown evaluator: {name}")
-    return cls(config)
+    return cls(event_logger, tracer, config)