Refactor inference endpoints

python/hsml/client/__init__.py

@@ -18,11 +18,13 @@
 from hsml.client.hopsworks import internal as hw_internal
 from hsml.client.hopsworks import external as hw_external
 
-from hsml.client import istio
 from hsml.client.istio import base as ist_base
 from hsml.client.istio import internal as ist_internal
 from hsml.client.istio import external as ist_external
 
+
+_client_type = None
+
 _hopsworks_client = None
 _istio_client = None
 
@@ -37,6 +39,9 @@ def init(
     api_key_file=None,
     api_key_value=None,
 ):
+    global _client_type
+    _client_type = client_type
+
     global _hopsworks_client
     if not _hopsworks_client:
         if client_type == "internal":
@@ -52,22 +57,6 @@ def init(
                 api_key_value,
             )
 
-    global _istio_client
-    if not _istio_client:
-        if client_type == "internal":
-            if istio.is_available():
-                _istio_client = ist_internal.Client()
-        elif client_type == "external":
-            _istio_client = ist_external.Client(
-                host,
-                "",  # set at request time
-                project,
-                hostname_verification,
-                trust_store_path,
-                api_key_file,
-                api_key_value,
-            )
-
 
 def get_instance() -> hw_base.Client:
     global _hopsworks_client
@@ -76,13 +65,29 @@ def get_instance() -> hw_base.Client:
     raise Exception("Couldn't find client. Try reconnecting to Hopsworks.")
 
 
+def set_istio_client(host, port, project=None, api_key_value=None):
+    global _client_type
+    global _istio_client
+
+    if not _istio_client:
+        if _client_type == "internal":
+            _istio_client = ist_internal.Client(host, port)
+        elif _client_type == "external":
+            _istio_client = ist_external.Client(host, port, project, api_key_value)
+
+
 def get_istio_instance() -> ist_base.Client:
     global _istio_client
     if _istio_client:
         return _istio_client
     raise Exception("Couldn't find the istio client. Try reconnecting to Hopsworks.")
 
 
+def get_client_type() -> str:
+    global _client_type
+    return _client_type
+
+
 def stop():
     global _hopsworks_client, _istio_client
     _hopsworks_client._close()

python/hsml/client/auth.py

@@ -14,8 +14,11 @@
 #   limitations under the License.
 #
 
+import os
 import requests
 
+from hsml.client import exceptions
+
 
 class BearerAuth(requests.auth.AuthBase):
     """Class to encapsulate a Bearer token."""
@@ -37,3 +40,25 @@ def __init__(self, token):
     def __call__(self, r):
         r.headers["Authorization"] = "ApiKey " + self._token
         return r
+
+
+def get_api_key(api_key_value, api_key_file):
+    if api_key_value is not None:
+        return api_key_value
+    elif api_key_file is not None:
+        file = None
+        if os.path.exists(api_key_file):
+            try:
+                file = open(api_key_file, mode="r")
+                return file.read()
+            finally:
+                file.close()
+        else:
+            raise IOError(
+                "Could not find api key file on path: {}".format(api_key_file)
+            )
+    else:
+        raise exceptions.ExternalClientError(
+            "Either api_key_file or api_key_value must be set when connecting to"
+            " hopsworks from an external environment."
+        )

python/hsml/client/hopsworks/external.py

@@ -14,7 +14,6 @@
 #   limitations under the License.
 #
 
-import os
 import requests
 
 from hsml.client import auth, exceptions
@@ -49,26 +48,7 @@ def __init__(
         self._base_url = "https://" + self._host + ":" + str(self._port)
         self._project_name = project
 
-        if api_key_value is not None:
-            api_key = api_key_value
-        elif api_key_file is not None:
-            file = None
-            if os.path.exists(api_key_file):
-                try:
-                    file = open(api_key_file, mode="r")
-                    api_key = file.read()
-                finally:
-                    file.close()
-            else:
-                raise IOError(
-                    "Could not find api key file on path: {}".format(api_key_file)
-                )
-        else:
-            raise exceptions.ExternalClientError(
-                "Either api_key_file or api_key_value must be set when connecting to"
-                " hopsworks from an external environment."
-            )
-
+        api_key = auth.get_api_key(api_key_value, api_key_file)
         self._auth = auth.ApiKeyAuth(api_key)
 
         self._session = requests.session()

python/hsml/client/istio/__init__.py

@@ -13,12 +13,3 @@
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
 #
-
-import os
-
-from hsml.client.istio import base
-
-
-def is_available():
-    """Whether Istio is available or not"""
-    return base.Client.ISTIO_ENDPOINT in os.environ

python/hsml/client/istio/external.py

@@ -14,10 +14,9 @@
 #   limitations under the License.
 #
 
-import os
 import requests
 
-from hsml.client import auth, exceptions
+from hsml.client import auth
 from hsml.client.istio import base as istio
 
 
@@ -27,53 +26,21 @@ def __init__(
         host,
         port,
         project,
-        hostname_verification,
-        trust_store_path,
-        api_key_file,
         api_key_value,
+        hostname_verification=None,
+        trust_store_path=None,
     ):
         """Initializes a client in an external environment such as AWS Sagemaker."""
-        if not host:
-            raise exceptions.ExternalClientError(
-                "host cannot be of type NoneType, host is a non-optional "
-                "argument to connect to hopsworks from an external environment."
-            )
-        if not project:
-            raise exceptions.ExternalClientError(
-                "project cannot be of type NoneType, project is a non-optional "
-                "argument to connect to hopsworks from an external environment."
-            )
-
         self._host = host
         self._port = port
         self._base_url = "http://" + self._host + ":" + str(self._port)
         self._project_name = project
 
-        if api_key_value is not None:
-            api_key = api_key_value
-        elif api_key_file is not None:
-            file = None
-            if os.path.exists(api_key_file):
-                try:
-                    file = open(api_key_file, mode="r")
-                    api_key = file.read()
-                finally:
-                    file.close()
-            else:
-                raise IOError(
-                    "Could not find api key file on path: {}".format(api_key_file)
-                )
-        else:
-            raise exceptions.ExternalClientError(
-                "Either api_key_file or api_key_value must be set when connecting to"
-                " hopsworks from an external environment."
-            )
-
-        self._auth = auth.ApiKeyAuth(api_key)
+        self._auth = auth.ApiKeyAuth(api_key_value)
 
         self._session = requests.session()
         self._connected = True
-        self._verify = self._get_verify(self._host, trust_store_path)
+        self._verify = self._get_verify(hostname_verification, trust_store_path)
 
         self._cert_key = None
 

python/hsml/client/istio/internal.py

@@ -48,10 +48,12 @@ class Client(istio.Client):
     MATERIAL_PWD = "material_passwd"
     SECRETS_DIR = "SECRETS_DIR"
 
-    def __init__(self):
+    def __init__(self, host, port):
         """Initializes a client being run from a job/notebook directly on Hopsworks."""
-        self._base_url = self._get_istio_endpoint()
-        self._host, self._port = self._get_host_port_pair()
+        self._host = host
+        self._port = port
+        self._base_url = "http://" + self._host + ":" + str(self._port)
+
         trust_store_path = self._get_trust_store_path()
         hostname_verification = (
             os.environ[self.REQUESTS_VERIFY]
@@ -66,10 +68,6 @@ def __init__(self):
 
         self._connected = True
 
-    def _get_istio_endpoint(self):
-        """Get the istio endpoint for making requests to the ingress gateway."""
-        return os.environ[self.ISTIO_ENDPOINT]
-
     def _project_name(self):
         try:
             return os.environ[self.PROJECT_NAME]

python/hsml/constants.py

@@ -69,3 +69,13 @@ class PREDICTOR_STATE:
     STATUS_STOPPING = "STOPPING"
     STATUS_STOPPED = "STOPPED"
     STATUS_UPDATING = "UPDATING"
+
+
+class INFERENCE_ENDPOINTS:
+    ENDPOINT_TYPE_NODE = "NODE"
+    ENDPOINT_TYPE_KUBE_CLUSTER = "KUBE_CLUSTER"
+    ENDPOINT_TYPE_LOAD_BALANCER = "LOAD_BALANCER"
+    PORT_NAME_HTTP = "HTTP"
+    PORT_NAME_HTTPS = "HTTPS"
+    PORT_NAME_STATUS_PORT = "STATUS"
+    PORT_NAME_TLS = "TLS"

python/hsml/core/model_serving_api.py

@@ -16,13 +16,16 @@
 
 from hsml import client
 from hsml.model_serving import ModelServing
-from hsml.core import dataset_api
+from hsml.core import dataset_api, serving_api
+from hsml.constants import INFERENCE_ENDPOINTS
+from hsml.inference_endpoint import get_endpoint_by_type
 from hsml.client.exceptions import ModelRegistryException
 
 
 class ModelServingApi:
     def __init__(self):
         self._dataset_api = dataset_api.DatasetApi()
+        self._serving_api = serving_api.ServingApi()
 
     def get(self):
         """Get model serving for specific project.
@@ -31,8 +34,34 @@ def get(self):
         :return: the model serving metadata
         :rtype: ModelServing
         """
+
         _client = client.get_instance()
 
+        # check kserve installed
+        if self._serving_api.is_kserve_installed():
+            # if kserve is installed, setup istio client
+            inference_endpoints = self._serving_api.get_inference_endpoints()
+            if client.get_client_type() == "internal":
+                endpoint = get_endpoint_by_type(
+                    inference_endpoints, INFERENCE_ENDPOINTS.ENDPOINT_TYPE_NODE
+                )
+                if endpoint is not None:
+                    client.set_istio_client(
+                        endpoint.get_any_host(),
+                        endpoint.get_port(INFERENCE_ENDPOINTS.PORT_NAME_HTTP).number,
+                    )
+            else:
+                endpoint = get_endpoint_by_type(
+                    inference_endpoints, INFERENCE_ENDPOINTS.ENDPOINT_TYPE_LOAD_BALANCER
+                )
+                if endpoint is not None:
+                    client.set_istio_client(
+                        endpoint.get_any_host(),
+                        endpoint.get_port(INFERENCE_ENDPOINTS.PORT_NAME_HTTP).number,
+                        _client._project_name,
+                        _client._auth._token,  # reuse hopsworks client token
+                    )
+
         # Validate that there is a Models dataset in the connected project
         if not self._dataset_api.path_exists("Models"):
             raise ModelRegistryException(

python/hsml/core/serving_api.py

@@ -17,6 +17,7 @@
 import json
 
 from hsml import client, deployment, predictor_state
+from hsml import inference_endpoint
 
 
 class ServingApi:
@@ -69,6 +70,18 @@ def get_all(self):
         deployments_json = _client._send_request("GET", path_params)
         return deployment.Deployment.from_response_json(deployments_json)
 
+    def get_inference_endpoints(self):
+        """Get inference endpoints.
+
+        :return: inference endpoints for the current project.
+        :rtype: List[InferenceEndpoint]
+        """
+
+        _client = client.get_instance()
+        path_params = ["project", _client._project_id, "inference", "endpoints"]
+        endpoints_json = _client._send_request("GET", path_params)
+        return inference_endpoint.InferenceEndpoint.from_response_json(endpoints_json)
+
     def put(self, deployment_instance, query_params: dict):
         """Save deployment metadata to model serving.
 
@@ -174,16 +187,25 @@ def send_inference_request(
                 _client._project_name, deployment_instance.name
             )
 
-            if _client._base_url.endswith(":"):
-                # if the istio ingress port is not set, use the one in the deployment metadata
-                _client._base_url += str(deployment_instance.get_state().internal_port)
-
         return _client._send_request(
             "POST", path_params, headers=headers, data=json.dumps(data)
         )
 
-    def _get_inference_request_host_header(self, project_name: str, serving_name: str):
-        return "{}.{}.hopsworks.ai".format(serving_name, project_name.replace("_", "-"))
+    def is_kserve_installed(self):
+        _client = client.get_instance()
+        path_params = ["variables", "kube_kserve_installed"]
+        kserve_installed = _client._send_request("GET", path_params)
+        return (
+            "successMessage" in kserve_installed
+            and kserve_installed["successMessage"] == "true"
+        )
+
+    def _get_inference_request_host_header(
+        self, project_name: str, deployment_name: str
+    ):
+        return "{}.{}.hopsworks.ai".format(
+            deployment_name, project_name.replace("_", "-")
+        )
 
     def _get_hopsworks_inference_path(self, project_id: int, deployment_instance):
         return [