fix: replace bare NotImplementedError with CLIError for unimplemented test types

src/inference_endpoint/config/schema.py

@@ -43,6 +43,7 @@
 from .. import metrics
 from ..core.types import APIType
 from ..endpoint_client.config import HTTPClientConfig
+from ..exceptions import CLIError
 from ..utils import WithUpdatesMixin
 from .ruleset_base import BenchmarkSuiteRuleset
 from .utils import parse_dataset_string, resolve_env_vars
@@ -714,7 +715,7 @@ def create_default_config(test_type: TestType) -> BenchmarkConfig:
             BenchmarkConfig (or subclass) instance
 
         Raises:
-            NotImplementedError: If test_type is EVAL or SUBMISSION
+            CLIError: If test_type is EVAL or SUBMISSION (not yet implemented)
             ValueError: If test_type is invalid
         """
         _common = {
@@ -734,14 +735,14 @@ def create_default_config(test_type: TestType) -> BenchmarkConfig:
                 ),
             )
         if test_type == TestType.EVAL:
-            raise NotImplementedError(
+            raise CLIError(
                 "Default EVAL config not yet implemented. "
-                "EVAL templates will be added in future release."
+                "Track progress at: https://github.com/mlcommons/endpoints/issues/4"
             )
         if test_type == TestType.SUBMISSION:
-            raise NotImplementedError(
+            raise CLIError(
                 "Default SUBMISSION config not yet implemented. "
-                "SUBMISSION templates will be added in future release."
+                "Track progress at: https://github.com/mlcommons/endpoints/issues/5"
             )
         raise ValueError(f"Unknown test type: {test_type}")
 

src/inference_endpoint/main.py

@@ -116,7 +116,10 @@ def eval(
     judge: str | None = None,
 ):
     """Run accuracy evaluation."""
-    raise NotImplementedError("Accuracy evaluation not yet implemented")
+    raise CLIError(
+        "Accuracy evaluation is not yet implemented. "
+        "Track progress at: https://github.com/mlcommons/endpoints/issues/4"
+    )
 
 
 def run() -> None:

tests/unit/commands/test_eval.py

@@ -16,6 +16,7 @@
 """Tests for eval command."""
 
 import pytest
+from inference_endpoint.exceptions import CLIError
 from inference_endpoint.main import app
 
 
@@ -24,6 +25,6 @@ class TestRunEvalCommand:
 
     @pytest.mark.unit
     def test_eval_not_implemented(self):
-        """Test that eval raises NotImplementedError."""
-        with pytest.raises(NotImplementedError, match="not yet implemented"):
+        """Test that eval raises CLIError."""
+        with pytest.raises(CLIError, match="not yet implemented"):
             app(["eval", "http://test.com"])

tests/unit/config/test_schema.py

@@ -30,6 +30,7 @@
     SubmissionReference,
     TestType,
 )
+from inference_endpoint.exceptions import CLIError
 
 
 class TestOSLDistribution:
@@ -332,12 +333,12 @@ def test_create_default_online(self):
 
     @pytest.mark.unit
     def test_create_default_eval_not_implemented(self):
-        with pytest.raises(NotImplementedError):
+        with pytest.raises(CLIError, match="EVAL config not yet implemented"):
             BenchmarkConfig.create_default_config(TestType.EVAL)
 
     @pytest.mark.unit
     def test_create_default_submission_not_implemented(self):
-        with pytest.raises(NotImplementedError):
+        with pytest.raises(CLIError, match="SUBMISSION config not yet implemented"):
             BenchmarkConfig.create_default_config(TestType.SUBMISSION)
 
     @pytest.mark.unit

tests/unit/config/test_yaml_loader.py

@@ -20,6 +20,7 @@
 import pytest
 from inference_endpoint.config.schema import BenchmarkConfig, LoadPatternType
 from inference_endpoint.config.schema import TestType as BenchmarkTestType
+from inference_endpoint.exceptions import CLIError
 from pydantic import ValidationError
 
 
@@ -189,11 +190,11 @@ def test_create_default_online_config(self):
         assert config.settings.runtime.min_duration_ms == 600000
 
     def test_create_default_eval_not_implemented(self):
-        with pytest.raises(NotImplementedError, match="EVAL"):
+        with pytest.raises(CLIError, match="EVAL"):
             BenchmarkConfig.create_default_config(BenchmarkTestType.EVAL)
 
     def test_create_default_submission_not_implemented(self):
-        with pytest.raises(NotImplementedError, match="SUBMISSION"):
+        with pytest.raises(CLIError, match="SUBMISSION"):
             BenchmarkConfig.create_default_config(BenchmarkTestType.SUBMISSION)