From 524b812d363f315f3067048cac4bef84f9625bc5 Mon Sep 17 00:00:00 2001 From: Sherif Akoush Date: Mon, 11 Oct 2021 10:00:06 +0100 Subject: [PATCH] fmt --- .../mlserver_alibi_explain/__init__.py | 2 +- .../mlserver_alibi_explain/common.py | 26 +++++++----- .../explainers/black_box_runtime.py | 13 +++--- .../explainers/integrated_gradients.py | 12 ++---- .../explainers/white_box_runtime.py | 6 ++- .../mlserver_alibi_explain/runtime.py | 39 +++++++++++------ runtimes/alibi-explain/setup.py | 5 +-- runtimes/alibi-explain/tests/conftest.py | 38 ++++++++--------- .../alibi-explain/tests/test_black_box.py | 31 +++++++++----- .../tests/test_integrated_gradients.py | 15 ++++--- runtimes/alibi-explain/tests/test_runtime.py | 42 ++++++++++++------- 11 files changed, 132 insertions(+), 97 deletions(-) diff --git a/runtimes/alibi-explain/mlserver_alibi_explain/__init__.py b/runtimes/alibi-explain/mlserver_alibi_explain/__init__.py index 43dd5bc78..d7364da1d 100644 --- a/runtimes/alibi-explain/mlserver_alibi_explain/__init__.py +++ b/runtimes/alibi-explain/mlserver_alibi_explain/__init__.py @@ -1,3 +1,3 @@ from .runtime import AlibiExplainRuntime -__all__ = ["AlibiExplainRuntime"] \ No newline at end of file +__all__ = ["AlibiExplainRuntime"] diff --git a/runtimes/alibi-explain/mlserver_alibi_explain/common.py b/runtimes/alibi-explain/mlserver_alibi_explain/common.py index f3bf8d4e0..479c8fc79 100644 --- a/runtimes/alibi-explain/mlserver_alibi_explain/common.py +++ b/runtimes/alibi-explain/mlserver_alibi_explain/common.py @@ -20,15 +20,18 @@ _INTEGRATED_GRADIENTS_TAG = "integrated_gradients" _TAG_TO_RT_IMPL = { - _ANCHOR_IMAGE_TAG: - ("mlserver_alibi_explain.explainers.black_box_runtime.AlibiExplainBlackBoxRuntime", - "alibi.explainers.AnchorImage"), - _ANCHOR_TEXT_TAG: - ("mlserver_alibi_explain.explainers.black_box_runtime.AlibiExplainBlackBoxRuntime", - "alibi.explainers.AnchorText"), + _ANCHOR_IMAGE_TAG: ( + "mlserver_alibi_explain.explainers.black_box_runtime.AlibiExplainBlackBoxRuntime", + "alibi.explainers.AnchorImage", + ), + _ANCHOR_TEXT_TAG: ( + "mlserver_alibi_explain.explainers.black_box_runtime.AlibiExplainBlackBoxRuntime", + "alibi.explainers.AnchorText", + ), _INTEGRATED_GRADIENTS_TAG: ( "mlserver_alibi_explain.explainers.integrated_gradients.IntegratedGradientsWrapper", - "alibi.explainers.IntegratedGradients") + "alibi.explainers.IntegratedGradients", + ), } @@ -53,11 +56,14 @@ def convert_from_bytes(output: ResponseOutput, ty: Optional[Type]) -> Any: else: py_str = bytearray(output.data).decode("UTF-8") from ast import literal_eval + return literal_eval(py_str) # TODO: add retry -def remote_predict(v2_payload: InferenceRequest, predictor_url: str) -> InferenceResponse: +def remote_predict( + v2_payload: InferenceRequest, predictor_url: str +) -> InferenceResponse: response_raw = requests.post(predictor_url, json=v2_payload.dict()) if response_raw.status_code != 200: raise ValueError(f"{response_raw.status_code} / {response_raw.reason}") @@ -66,7 +72,7 @@ def remote_predict(v2_payload: InferenceRequest, predictor_url: str) -> Inferenc # TODO: this is very similar to `asyncio.to_thread` (python 3.9+), so lets use it at some point. def execute_async( - loop: Optional[AbstractEventLoop], fn: Callable, *args, **kwargs + loop: Optional[AbstractEventLoop], fn: Callable, *args, **kwargs ) -> Awaitable: if loop is None: loop = asyncio.get_running_loop() @@ -102,5 +108,5 @@ class Config: def import_and_get_class(class_path: str) -> type: last_dot = class_path.rfind(".") - klass = getattr(import_module(class_path[:last_dot]), class_path[last_dot + 1:]) + klass = getattr(import_module(class_path[:last_dot]), class_path[last_dot + 1 :]) return klass diff --git a/runtimes/alibi-explain/mlserver_alibi_explain/explainers/black_box_runtime.py b/runtimes/alibi-explain/mlserver_alibi_explain/explainers/black_box_runtime.py index 45929a529..ef0cb05cb 100644 --- a/runtimes/alibi-explain/mlserver_alibi_explain/explainers/black_box_runtime.py +++ b/runtimes/alibi-explain/mlserver_alibi_explain/explainers/black_box_runtime.py @@ -31,16 +31,15 @@ async def load(self) -> bool: self._model = self._explainer_class(**init_parameters) # type: ignore else: # load the model from disk - self._model = load_explainer(self.settings.parameters.uri, predictor=self._infer_impl) + self._model = load_explainer( + self.settings.parameters.uri, predictor=self._infer_impl + ) self.ready = True return self.ready def _explain_impl(self, input_data: Any, explain_parameters: Dict) -> Explanation: - return self._model.explain( - input_data, - **explain_parameters # type: ignore - ) + return self._model.explain(input_data, **explain_parameters) # type: ignore def _infer_impl(self, input_data: np.ndarray) -> np.ndarray: # The contract is that alibi-explain would input/output ndarray @@ -56,8 +55,8 @@ def _infer_impl(self, input_data: np.ndarray) -> np.ndarray: # TODO add some exception handling here v2_response = remote_predict( - v2_payload=v2_request, - predictor_url=self.alibi_explain_settings.infer_uri) + v2_payload=v2_request, predictor_url=self.alibi_explain_settings.infer_uri + ) # TODO: do we care about more than one output? return np_codec.decode_response_output(v2_response.outputs[0]) diff --git a/runtimes/alibi-explain/mlserver_alibi_explain/explainers/integrated_gradients.py b/runtimes/alibi-explain/mlserver_alibi_explain/explainers/integrated_gradients.py index 7c9e02a8f..95ea0a402 100644 --- a/runtimes/alibi-explain/mlserver_alibi_explain/explainers/integrated_gradients.py +++ b/runtimes/alibi-explain/mlserver_alibi_explain/explainers/integrated_gradients.py @@ -3,21 +3,17 @@ import tensorflow as tf from alibi.api.interfaces import Explanation -from mlserver_alibi_explain.explainers.white_box_runtime import AlibiExplainWhiteBoxRuntime +from mlserver_alibi_explain.explainers.white_box_runtime import ( + AlibiExplainWhiteBoxRuntime, +) class IntegratedGradientsWrapper(AlibiExplainWhiteBoxRuntime): def _explain_impl(self, input_data: Any, explain_parameters: Dict) -> Explanation: # TODO: how are we going to deal with that? predictions = self._inference_model(input_data).numpy().argmax(axis=1) - return self._model.explain( - input_data, - target=predictions, - **explain_parameters - ) + return self._model.explain(input_data, target=predictions, **explain_parameters) async def _get_inference_model(self) -> Any: inference_model_path = self.alibi_explain_settings.infer_uri return tf.keras.models.load_model(inference_model_path) - - diff --git a/runtimes/alibi-explain/mlserver_alibi_explain/explainers/white_box_runtime.py b/runtimes/alibi-explain/mlserver_alibi_explain/explainers/white_box_runtime.py index fafee9f7a..4dd63a38e 100644 --- a/runtimes/alibi-explain/mlserver_alibi_explain/explainers/white_box_runtime.py +++ b/runtimes/alibi-explain/mlserver_alibi_explain/explainers/white_box_runtime.py @@ -14,6 +14,7 @@ class AlibiExplainWhiteBoxRuntime(AlibiExplainRuntimeBase): White box alibi explain requires access to the full inference model to compute gradients etc. usually in the same domain as the explainer itself. e.g. `IntegratedGradients` """ + def __init__(self, settings: ModelSettings, explainer_class: Type[Explainer]): self._inference_model = None self._explainer_class = explainer_class @@ -33,7 +34,9 @@ async def load(self) -> bool: else: # load the model from disk # full model is passed as `predictor` - self._model = load_explainer(self.settings.parameters.uri, predictor=self._inference_model) + self._model = load_explainer( + self.settings.parameters.uri, predictor=self._inference_model + ) self.ready = True return self.ready @@ -43,4 +46,3 @@ def _explain_impl(self, input_data: Any, explain_parameters: Dict) -> Explanatio async def _get_inference_model(self) -> Any: raise NotImplementedError - diff --git a/runtimes/alibi-explain/mlserver_alibi_explain/runtime.py b/runtimes/alibi-explain/mlserver_alibi_explain/runtime.py index 2b6e564e2..3125d3a48 100644 --- a/runtimes/alibi-explain/mlserver_alibi_explain/runtime.py +++ b/runtimes/alibi-explain/mlserver_alibi_explain/runtime.py @@ -5,11 +5,24 @@ from mlserver.codecs import NumpyCodec, InputCodec, StringCodec from mlserver.model import MLModel from mlserver.settings import ModelSettings -from mlserver.types import InferenceRequest, InferenceResponse, RequestInput, MetadataModelResponse, Parameters, \ - MetadataTensor, ResponseOutput -from mlserver_alibi_explain.common import execute_async, AlibiExplainSettings, \ - get_mlmodel_class_as_str, \ - get_alibi_class_as_str, import_and_get_class, EXPLAIN_PARAMETERS_TAG, EXPLAINER_TYPE_TAG +from mlserver.types import ( + InferenceRequest, + InferenceResponse, + RequestInput, + MetadataModelResponse, + Parameters, + MetadataTensor, + ResponseOutput, +) +from mlserver_alibi_explain.common import ( + execute_async, + AlibiExplainSettings, + get_mlmodel_class_as_str, + get_alibi_class_as_str, + import_and_get_class, + EXPLAIN_PARAMETERS_TAG, + EXPLAINER_TYPE_TAG, +) class AlibiExplainRuntimeBase(MLModel): @@ -17,7 +30,9 @@ class AlibiExplainRuntimeBase(MLModel): Base class for Alibi-Explain models """ - def __init__(self, settings: ModelSettings, explainer_settings: AlibiExplainSettings): + def __init__( + self, settings: ModelSettings, explainer_settings: AlibiExplainSettings + ): self.alibi_explain_settings = explainer_settings super().__init__(settings) @@ -37,7 +52,9 @@ async def predict(self, payload: InferenceRequest) -> InferenceResponse: outputs=[output_data], ) - async def _async_explain_impl(self, input_data: Any, settings: Parameters) -> ResponseOutput: + async def _async_explain_impl( + self, input_data: Any, settings: Parameters + ) -> ResponseOutput: """run async""" explain_parameters = dict() if EXPLAIN_PARAMETERS_TAG in settings: @@ -47,7 +64,7 @@ async def _async_explain_impl(self, input_data: Any, settings: Parameters) -> Re loop=None, fn=self._explain_impl, input_data=input_data, - explain_parameters=explain_parameters + explain_parameters=explain_parameters, ) # TODO: Convert alibi-explain output to v2 protocol, for now we use to_json return StringCodec.encode(payload=[explanation.to_json()], name="explain") @@ -109,7 +126,7 @@ def ready(self, value: bool): self._rt.ready = value def decode( - self, request_input: RequestInput, default_codec: Optional[InputCodec] = None + self, request_input: RequestInput, default_codec: Optional[InputCodec] = None ) -> Any: return self._rt.decode(request_input, default_codec) @@ -124,7 +141,3 @@ async def load(self) -> bool: async def predict(self, payload: InferenceRequest) -> InferenceResponse: return await self._rt.predict(payload) - - - - diff --git a/runtimes/alibi-explain/setup.py b/runtimes/alibi-explain/setup.py index 763c31a58..d729f5212 100644 --- a/runtimes/alibi-explain/setup.py +++ b/runtimes/alibi-explain/setup.py @@ -33,10 +33,7 @@ def _load_description() -> str: author_email="hello@seldon.io", description="Alibi-Explain runtime for MLServer", packages=find_packages(), - install_requires=[ - "mlserver", - "alibi" - ], + install_requires=["mlserver", "alibi"], long_description=_load_description(), long_description_content_type="text/markdown", license="Apache 2.0", diff --git a/runtimes/alibi-explain/tests/conftest.py b/runtimes/alibi-explain/tests/conftest.py index c95dc7690..38885d4e2 100644 --- a/runtimes/alibi-explain/tests/conftest.py +++ b/runtimes/alibi-explain/tests/conftest.py @@ -81,10 +81,7 @@ async def custom_runtime_tf() -> MLModel: @pytest.fixture def settings() -> Settings: - return Settings( - debug=True, - host="127.0.0.1" - ) + return Settings(debug=True, host="127.0.0.1") @pytest.fixture @@ -108,7 +105,7 @@ def model_repository(tmp_path, runtime_pytorch) -> ModelRepository: "parallel_workers": 0, "parameters": { "uri": runtime_pytorch.settings.parameters.uri, - } + }, } model_settings_path.write_text(json.dumps(model_settings_dict, indent=4)) @@ -156,21 +153,27 @@ def rest_client(rest_app: FastAPI) -> TestClient: @pytest.fixture async def anchor_image_runtime_with_remote_predict_patch( - custom_runtime_tf: MLModel, - remote_predict_mock_path: str = "mlserver_alibi_explain.common.remote_predict") -> AlibiExplainRuntime: + custom_runtime_tf: MLModel, + remote_predict_mock_path: str = "mlserver_alibi_explain.common.remote_predict", +) -> AlibiExplainRuntime: with patch(remote_predict_mock_path) as remote_predict: + def mock_predict(*args, **kwargs): # note: sometimes the event loop is not running and in this case we create a new one otherwise # we use the existing one. # this mock implementation is required as we dont want to spin up a server, we just use MLModel.predict try: loop = asyncio.get_event_loop() - res = loop.run_until_complete(custom_runtime_tf.predict(kwargs["v2_payload"])) + res = loop.run_until_complete( + custom_runtime_tf.predict(kwargs["v2_payload"]) + ) return res except Exception: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - res = loop.run_until_complete(custom_runtime_tf.predict(kwargs["v2_payload"])) + res = loop.run_until_complete( + custom_runtime_tf.predict(kwargs["v2_payload"]) + ) return res remote_predict.side_effect = mock_predict @@ -181,10 +184,9 @@ def mock_predict(*args, **kwargs): parameters=ModelParameters( uri=f"{TESTS_PATH}/data/mnist_anchor_image", extra=AlibiExplainSettings( - explainer_type="anchor_image", - infer_uri=f"dummy_call" - ) - ) + explainer_type="anchor_image", infer_uri=f"dummy_call" + ), + ), ) ) await rt.load() @@ -199,17 +201,13 @@ async def integrated_gradients_runtime() -> AlibiExplainRuntime: parallel_workers=1, parameters=ModelParameters( extra=AlibiExplainSettings( - init_parameters={ - "n_steps": 50, - "method": "gausslegendre" - }, + init_parameters={"n_steps": 50, "method": "gausslegendre"}, explainer_type="integrated_gradients", - infer_uri=f"{TESTS_PATH}/data/tf_mnist/model.h5" + infer_uri=f"{TESTS_PATH}/data/tf_mnist/model.h5", ) - ) + ), ) ) await rt.load() return rt - diff --git a/runtimes/alibi-explain/tests/test_black_box.py b/runtimes/alibi-explain/tests/test_black_box.py index a1430c253..e1dbab3e7 100644 --- a/runtimes/alibi-explain/tests/test_black_box.py +++ b/runtimes/alibi-explain/tests/test_black_box.py @@ -34,8 +34,8 @@ def payload() -> InferenceRequest: async def test_predict_impl( - anchor_image_runtime_with_remote_predict_patch: AlibiExplainRuntime, - custom_runtime_tf: MLModel + anchor_image_runtime_with_remote_predict_patch: AlibiExplainRuntime, + custom_runtime_tf: MLModel, ): # note: custom_runtime_tf is the underlying inference runtime # we want to test that the underlying impl predict is functionally correct @@ -58,7 +58,9 @@ async def test_predict_impl( ], ) expected_result = await custom_runtime_tf.predict(inference_request) - expected_result_numpy = NumpyCodec.decode_response_output(expected_result.outputs[0]) + expected_result_numpy = NumpyCodec.decode_response_output( + expected_result.outputs[0] + ) assert_array_equal(actual_result, expected_result_numpy) @@ -71,15 +73,22 @@ def alibi_anchor_image_model(): async def test_end_2_end( - anchor_image_runtime_with_remote_predict_patch: AlibiExplainRuntime, - alibi_anchor_image_model, - payload: InferenceRequest + anchor_image_runtime_with_remote_predict_patch: AlibiExplainRuntime, + alibi_anchor_image_model, + payload: InferenceRequest, ): # in this test we are getting explanation and making sure that it the same one as returned by alibi # directly - runtime_result = await anchor_image_runtime_with_remote_predict_patch.predict(payload) - decoded_runtime_results = json.loads(convert_from_bytes(runtime_result.outputs[0], ty=str)) - alibi_result = alibi_anchor_image_model.explain(NumpyCodec.decode(payload.inputs[0])) - - assert_array_equal(np.array(decoded_runtime_results["data"]["anchor"]), alibi_result.data["anchor"]) + runtime_result = await anchor_image_runtime_with_remote_predict_patch.predict( + payload + ) + decoded_runtime_results = json.loads( + convert_from_bytes(runtime_result.outputs[0], ty=str) + ) + alibi_result = alibi_anchor_image_model.explain( + NumpyCodec.decode(payload.inputs[0]) + ) + assert_array_equal( + np.array(decoded_runtime_results["data"]["anchor"]), alibi_result.data["anchor"] + ) diff --git a/runtimes/alibi-explain/tests/test_integrated_gradients.py b/runtimes/alibi-explain/tests/test_integrated_gradients.py index 7213b1deb..74a12de7b 100644 --- a/runtimes/alibi-explain/tests/test_integrated_gradients.py +++ b/runtimes/alibi-explain/tests/test_integrated_gradients.py @@ -40,14 +40,16 @@ def alibi_integrated_gradients_model() -> Tuple: async def test_end_2_end( - integrated_gradients_runtime, - alibi_integrated_gradients_model, - payload: InferenceRequest + integrated_gradients_runtime, + alibi_integrated_gradients_model, + payload: InferenceRequest, ): # in this test we are getting explanation and making sure that it the same one as returned by alibi # directly runtime_result = await integrated_gradients_runtime.predict(payload) - decoded_runtime_results = json.loads(convert_from_bytes(runtime_result.outputs[0], ty=str)) + decoded_runtime_results = json.loads( + convert_from_bytes(runtime_result.outputs[0], ty=str) + ) # get the data as numpy from the inference request payload infer_model, ig_model = alibi_integrated_gradients_model @@ -55,4 +57,7 @@ async def test_end_2_end( predictions = infer_model(input_data_np).numpy().argmax(axis=1) alibi_result = ig_model.explain(input_data_np, target=predictions) - assert_array_equal(np.array(decoded_runtime_results["data"]["attributions"]), alibi_result.data["attributions"]) + assert_array_equal( + np.array(decoded_runtime_results["data"]["attributions"]), + alibi_result.data["attributions"], + ) diff --git a/runtimes/alibi-explain/tests/test_runtime.py b/runtimes/alibi-explain/tests/test_runtime.py index e6e1e85dd..ef14e10f3 100644 --- a/runtimes/alibi-explain/tests/test_runtime.py +++ b/runtimes/alibi-explain/tests/test_runtime.py @@ -13,7 +13,9 @@ """ -async def test_integrated_gradients__smoke(integrated_gradients_runtime: AlibiExplainRuntime): +async def test_integrated_gradients__smoke( + integrated_gradients_runtime: AlibiExplainRuntime, +): # TODO: there is an inherit batch as first dimension data = np.random.randn(10, 28, 28, 1) * 255 inference_request = InferenceRequest( @@ -21,7 +23,7 @@ async def test_integrated_gradients__smoke(integrated_gradients_runtime: AlibiEx content_type=NumpyCodec.ContentType, explain_parameters={ "baselines": None, - } + }, ), inputs=[ RequestInput( @@ -36,7 +38,9 @@ async def test_integrated_gradients__smoke(integrated_gradients_runtime: AlibiEx _ = convert_from_bytes(response.outputs[0], ty=str) -async def test_anchors__smoke(anchor_image_runtime_with_remote_predict_patch: AlibiExplainRuntime): +async def test_anchors__smoke( + anchor_image_runtime_with_remote_predict_patch: AlibiExplainRuntime, +): data = np.random.randn(28, 28, 1) * 255 inference_request = InferenceRequest( parameters=Parameters( @@ -45,7 +49,7 @@ async def test_anchors__smoke(anchor_image_runtime_with_remote_predict_patch: Al "threshold": 0.95, "p_sample": 0.5, "tau": 0.25, - } + }, ), inputs=[ RequestInput( @@ -56,7 +60,9 @@ async def test_anchors__smoke(anchor_image_runtime_with_remote_predict_patch: Al ) ], ) - response = await anchor_image_runtime_with_remote_predict_patch.predict(inference_request) + response = await anchor_image_runtime_with_remote_predict_patch.predict( + inference_request + ) _ = convert_from_bytes(response.outputs[0], ty=str) @@ -79,15 +85,14 @@ def test_remote_predict__smoke(runtime_pytorch, rest_client): endpoint = f"v2/models/{runtime_pytorch.settings.name}/infer" - _ = remote_predict( - inference_request, - predictor_url=endpoint) + _ = remote_predict(inference_request, predictor_url=endpoint) async def test_alibi_runtime_wrapper(custom_runtime_tf: MLModel): """ Checks that the wrappers returns back the expected valued from the underlying runtime """ + class _MockInit(AlibiExplainRuntime): def __init__(self, settings: ModelSettings): self._rt = custom_runtime_tf @@ -116,14 +121,18 @@ def __init__(self, settings: ModelSettings): assert wrapper.ready == custom_runtime_tf.ready assert await wrapper.metadata() == await custom_runtime_tf.metadata() - assert await wrapper.predict(inference_request) == await custom_runtime_tf.predict(inference_request) + assert await wrapper.predict(inference_request) == await custom_runtime_tf.predict( + inference_request + ) # check setters - dummy_shape_metadata = [MetadataTensor( - name="dummy", - datatype="FP32", - shape=[1, 2], - )] + dummy_shape_metadata = [ + MetadataTensor( + name="dummy", + datatype="FP32", + shape=[1, 2], + ) + ] wrapper.inputs = dummy_shape_metadata custom_runtime_tf.inputs = dummy_shape_metadata assert wrapper.inputs == custom_runtime_tf.inputs @@ -133,7 +142,8 @@ def __init__(self, settings: ModelSettings): assert wrapper.outputs == custom_runtime_tf.outputs wrapper_public_funcs = list(filter(lambda x: not x.startswith("_"), dir(wrapper))) - expected_public_funcs = list(filter(lambda x: not x.startswith("_"), dir(custom_runtime_tf))) + expected_public_funcs = list( + filter(lambda x: not x.startswith("_"), dir(custom_runtime_tf)) + ) assert wrapper_public_funcs == expected_public_funcs -