[evaluation] fix: Remove "complex" type hints for AsyncEvaluators (#3…

…7964) * fix: Remove "complex" type hints from AsyncEvaluators This introduced a regression where promptflow rejected the evaluators because they were of types promptflow doesn't support. * style: Run isort
Azure · Oct 22, 2024 · 0e400d0 · 0e400d0
1 parent 3a43719
commit 0e400d0
Show file tree

Hide file tree

Showing 17 changed files with 23 additions and 28 deletions.
diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py
@@ -14,8 +14,8 @@
 from promptflow.entities import Run
 
 from azure.ai.evaluation._common.math import list_sum
-from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._common.utils import validate_azure_ai_project
+from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 
 from .._constants import (
     CONTENT_SAFETY_DEFECT_RATE_THRESHOLD_DEFAULT,

diff --git a/...luation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py b/...luation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py
@@ -1,7 +1,7 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from typing import Dict, Optional, Union, TypeVar
+from typing import Dict, Optional, TypeVar, Union
 
 from typing_extensions import override
 

diff --git a/...on/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py b/...on/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_content_safety.py
@@ -2,13 +2,13 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from concurrent.futures import as_completed
-from typing import Callable, Dict, List, Union, Optional
-from typing_extensions import override
+from typing import Callable, Dict, List, Optional, Union
 
 from promptflow.tracing import ThreadPoolExecutorWithContext as ThreadPoolExecutor
+from typing_extensions import override
 
-from azure.ai.evaluation._evaluators._common import EvaluatorBase
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._evaluators._common import EvaluatorBase
 
 from ._hate_unfairness import HateUnfairnessEvaluator
 from ._self_harm import SelfHarmEvaluator

diff --git a/...n/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py b/...n/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_hate_unfairness.py
@@ -5,8 +5,8 @@
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 

diff --git a/...luation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py b/...luation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_self_harm.py
@@ -5,8 +5,8 @@
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 

diff --git a/...evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py b/...evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_sexual.py
@@ -5,8 +5,8 @@
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 

diff --git a/...aluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py b/...aluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_content_safety/_violence.py
@@ -5,8 +5,8 @@
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_eci/_eci.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_eci/_eci.py
@@ -3,8 +3,8 @@
 # ---------------------------------------------------------
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import _InternalEvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import _InternalEvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 

diff --git a/...-ai-evaluation/azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py b/...-ai-evaluation/azure/ai/evaluation/_evaluators/_protected_material/_protected_material.py
@@ -6,8 +6,8 @@
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_retrieval/_retrieval.py
@@ -7,13 +7,10 @@
 import math
 import os
 import re
-from typing import Union
 
 from promptflow._utils.async_utils import async_run_allowing_running_loop
 from promptflow.core import AsyncPrompty
 
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
-
 from ..._common.math import list_mean_nan_safe
 from ..._common.utils import construct_prompty_model_config, validate_model_config
 
@@ -31,9 +28,9 @@ class _AsyncRetrievalScoreEvaluator:
     _LLM_CALL_TIMEOUT = 600
     _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
-    def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
+    def __init__(self, model_config: dict):
         prompty_model_config = construct_prompty_model_config(
-            model_config,
+            validate_model_config(model_config),
             self._DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
@@ -138,7 +135,7 @@ class RetrievalEvaluator:
     """
 
     def __init__(self, model_config):
-        self._async_evaluator = _AsyncRetrievalScoreEvaluator(validate_model_config(model_config))
+        self._async_evaluator = _AsyncRetrievalScoreEvaluator(model_config)
 
     def __call__(self, *, conversation, **kwargs):
         """Evaluates retrieval score chat scenario.

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_rouge/_rouge.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_rouge/_rouge.py
@@ -4,8 +4,8 @@
 from enum import Enum
 
 from promptflow._utils.async_utils import async_run_allowing_running_loop
-from azure.ai.evaluation._vendor.rouge_score import rouge_scorer
 
+from azure.ai.evaluation._vendor.rouge_score import rouge_scorer
 from azure.core import CaseInsensitiveEnumMeta
 
 

diff --git a/...evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py b/...evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_similarity/_similarity.py
@@ -5,13 +5,11 @@
 import math
 import os
 import re
-from typing import Union
 
 from promptflow._utils.async_utils import async_run_allowing_running_loop
 from promptflow.core import AsyncPrompty
 
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
-from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 
 from ..._common.utils import construct_prompty_model_config, validate_model_config
 
@@ -27,9 +25,9 @@ class _AsyncSimilarityEvaluator:
     _LLM_CALL_TIMEOUT = 600
     _DEFAULT_OPEN_API_VERSION = "2024-02-15-preview"
 
-    def __init__(self, model_config: Union[AzureOpenAIModelConfiguration, OpenAIModelConfiguration]):
+    def __init__(self, model_config: dict):
         prompty_model_config = construct_prompty_model_config(
-            model_config,
+            validate_model_config(model_config),
             self._DEFAULT_OPEN_API_VERSION,
             USER_AGENT,
         )
@@ -108,7 +106,7 @@ class SimilarityEvaluator:
     """
 
     def __init__(self, model_config):
-        self._async_evaluator = _AsyncSimilarityEvaluator(validate_model_config(model_config))
+        self._async_evaluator = _AsyncSimilarityEvaluator(model_config)
 
     def __call__(self, *, query: str, response: str, ground_truth: str, **kwargs):
         """

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_xpia/xpia.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_xpia/xpia.py
@@ -6,8 +6,8 @@
 
 from typing_extensions import override
 
-from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.constants import EvaluationMetrics
 from azure.ai.evaluation._evaluators._common import RaiServiceEvaluatorBase
 
 logger = logging.getLogger(__name__)

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_adversarial_simulator.py
@@ -10,8 +10,8 @@
 
 from tqdm import tqdm
 
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import get_async_http_client
 from azure.ai.evaluation.simulator import AdversarialScenario

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_direct_attack_simulator.py
@@ -7,8 +7,8 @@
 from random import randint
 from typing import Callable, Optional, cast
 
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.core.credentials import TokenCredential

diff --git a/...valuation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py b/...valuation/azure-ai-evaluation/azure/ai/evaluation/simulator/_indirect_attack_simulator.py
@@ -6,8 +6,8 @@
 import logging
 from typing import Callable, cast
 
-from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.utils import validate_azure_ai_project
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation.simulator import AdversarialScenario
 from azure.core.credentials import TokenCredential

diff --git a/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py b/sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py
@@ -14,8 +14,8 @@
 from promptflow.core import AsyncPrompty
 from tqdm import tqdm
 
-from azure.ai.evaluation._common.utils import construct_prompty_model_config
 from azure.ai.evaluation._common._experimental import experimental
+from azure.ai.evaluation._common.utils import construct_prompty_model_config
 from azure.ai.evaluation._model_configurations import AzureOpenAIModelConfiguration, OpenAIModelConfiguration
 
 from .._exceptions import ErrorBlame, ErrorCategory, EvaluationException