[HOPSWORKS-2827] Add serving module

python/hsml/client/__init__.py

@@ -1,5 +1,5 @@
 #
-#   Copyright 2021 Logical Clocks AB
+#   Copyright 2022 Logical Clocks AB
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -14,9 +14,15 @@
 #   limitations under the License.
 #
 
-from hsml.client import external, hopsworks
+from hsml.client.hopsworks import base as hw_base
+from hsml.client.hopsworks import internal as hw_internal
+from hsml.client.hopsworks import external as hw_external
 
-_client = None
+from hsml.client.istio import base as ist_base
+from hsml.client.istio import internal as ist_internal
+
+_hopsworks_client = None
+_istio_client = None
 
 
 def init(
@@ -31,12 +37,12 @@ def init(
     api_key_file=None,
     api_key_value=None,
 ):
-    global _client
-    if not _client:
-        if client_type == "hopsworks":
-            _client = hopsworks.Client()
+    global _hopsworks_client
+    if not _hopsworks_client:
+        if client_type == "internal":
+            _hopsworks_client = hw_internal.Client()
         elif client_type == "external":
-            _client = external.Client(
+            _hopsworks_client = hw_external.external.Client(
                 host,
                 port,
                 project,
@@ -48,15 +54,29 @@ def init(
                 api_key_value,
             )
 
+    global _istio_client
+    if not _istio_client and client_type == "internal":
+        _istio_client = (
+            ist_internal.Client()
+        )  # TODO: Add external Istio client after adding support for AKS, EKS
+
 
-def get_instance():
-    global _client
-    if _client:
-        return _client
+def get_instance() -> hw_base.Client:
+    global _hopsworks_client
+    if _hopsworks_client:
+        return _hopsworks_client
     raise Exception("Couldn't find client. Try reconnecting to Hopsworks.")
 
 
+def get_istio_instance() -> ist_base.Client:
+    global _istio_client
+    if _istio_client:
+        return _istio_client
+    raise Exception("Couldn't find the istio client. Try reconnecting to Hopsworks.")
+
+
 def stop():
-    global _client
-    _client._close()
-    _client = None
+    global _hopsworks_client, _istio_client
+    _hopsworks_client._close()
+    _istio_client.close()
+    _hopsworks_client = _istio_client = None

python/hsml/client/base.py

@@ -1,5 +1,5 @@
 #
-#   Copyright 2021 Logical Clocks AB
+#   Copyright 2022 Logical Clocks AB
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -14,14 +14,13 @@
 #   limitations under the License.
 #
 
-import os
 import furl
 from abc import ABC, abstractmethod
 
 import requests
 import urllib3
 
-from hsml.client import exceptions, auth
+from hsml.client import exceptions
 from hsml.decorators import connected
 
 
@@ -30,57 +29,25 @@
 
 
 class Client(ABC):
-    TOKEN_FILE = "token.jwt"
-    REST_ENDPOINT = "REST_ENDPOINT"
-    DEFAULT_DATABRICKS_ROOT_VIRTUALENV_ENV = "DEFAULT_DATABRICKS_ROOT_VIRTUALENV_ENV"
-
     @abstractmethod
     def __init__(self):
         """To be implemented by clients."""
         pass
 
+    @abstractmethod
     def _get_verify(self, verify, trust_store_path):
-        """Get verification method for sending HTTP requests to Hopsworks.
-
-        Credit to https://gist.github.com/gdamjan/55a8b9eec6cf7b771f92021d93b87b2c
-
-        :param verify: perform hostname verification, 'true' or 'false'
-        :type verify: str
-        :param trust_store_path: path of the truststore locally if it was uploaded manually to
-            the external environment such as AWS Sagemaker
-        :type trust_store_path: str
-        :return: if verify is true and the truststore is provided, then return the trust store location
-                 if verify is true but the truststore wasn't provided, then return true
-                 if verify is false, then return false
-        :rtype: str or boolean
-        """
-        if verify == "true":
-            if trust_store_path is not None:
-                return trust_store_path
-            else:
-                return True
+        """To be implemented by clients."""
+        pass
 
-        return False
+    @abstractmethod
+    def _get_retry(self, session, request, response):
+        """To be implemented by clients."""
+        pass
 
+    @abstractmethod
     def _get_host_port_pair(self):
-        """
-        Removes "http or https" from the rest endpoint and returns a list
-        [endpoint, port], where endpoint is on the format /path.. without http://
-
-        :return: a list [endpoint, port]
-        :rtype: list
-        """
-        endpoint = self._base_url
-        if "http" in endpoint:
-            last_index = endpoint.rfind("/")
-            endpoint = endpoint[last_index + 1 :]
-        host, port = endpoint.split(":")
-        return host, port
-
-    def _read_jwt(self):
-        """Retrieve jwt from local container."""
-        with open(os.path.join(self._secrets_dir, self.TOKEN_FILE), "r") as jwt:
-            return jwt.read()
+        """To be implemented by clients."""
+        pass
 
     @connected
     def _send_request(
@@ -93,7 +60,7 @@ def _send_request(
         stream=False,
         files=None,
     ):
-        """Send REST request to Hopsworks.
+        """Send REST request to a REST endpoint.
 
         Uses the client it is executed from. Path parameters are url encoded automatically.
 
@@ -117,9 +84,8 @@ def _send_request(
         :return: Response json
         :rtype: dict
         """
-        base_path_params = ["hopsworks-api", "api"]
         f_url = furl.furl(self._base_url)
-        f_url.path.segments = base_path_params + path_params
+        f_url.path.segments = self.BASE_PATH_PARAMS + path_params
         url = str(f_url)
 
         request = requests.Request(
@@ -135,11 +101,7 @@ def _send_request(
         prepped = self._session.prepare_request(request)
         response = self._session.send(prepped, verify=self._verify, stream=stream)
 
-        if response.status_code == 401 and self.REST_ENDPOINT in os.environ:
-            # refresh token and retry request - only on hopsworks
-            self._auth = auth.BearerAuth(self._read_jwt())
-            # Update request with the new token
-            request.auth = self._auth
+        if self._get_retry(request, response):
             prepped = self._session.prepare_request(request)
             response = self._session.send(prepped, verify=self._verify, stream=stream)
 

python/hsml/client/exceptions.py

@@ -45,6 +45,10 @@ class ModelRegistryException(Exception):
     """Generic model registry exception"""
 
 
+class ModelServingException(Exception):
+    """Generic model serving exception"""
+
+
 class ExternalClientError(TypeError):
     """Raised when external client cannot be initialized due to missing arguments."""
 

python/hsml/client/hopsworks/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2022 Logical Clocks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#

python/hsml/client/hopsworks/base.py

@@ -0,0 +1,96 @@
+#
+#   Copyright 2022 Logical Clocks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+
+import os
+from abc import abstractmethod
+
+from hsml.client import base, auth
+
+
+class Client(base.Client):
+    TOKEN_FILE = "token.jwt"
+    REST_ENDPOINT = "REST_ENDPOINT"
+
+    BASE_PATH_PARAMS = ["hopsworks-api", "api"]
+
+    @abstractmethod
+    def __init__(self):
+        """To be extended by clients."""
+        pass
+
+    def _get_verify(self, verify, trust_store_path):
+        """Get verification method for sending HTTP requests to Hopsworks.
+
+        Credit to https://gist.github.com/gdamjan/55a8b9eec6cf7b771f92021d93b87b2c
+
+        :param verify: perform hostname verification, 'true' or 'false'
+        :type verify: str
+        :param trust_store_path: path of the truststore locally if it was uploaded manually to
+            the external environment such as AWS Sagemaker
+        :type trust_store_path: str
+        :return: if verify is true and the truststore is provided, then return the trust store location
+                 if verify is true but the truststore wasn't provided, then return true
+                 if verify is false, then return false
+        :rtype: str or boolean
+        """
+        if verify == "true":
+            if trust_store_path is not None:
+                return trust_store_path
+            else:
+                return True
+
+        return False
+
+    def _get_retry(self, request, response):
+        """Get retry method for resending HTTP requests to Hopsworks
+
+        :param request: original HTTP request already sent
+        :type request: requests.Request
+        :param response: response of the original HTTP request
+        :type response: requests.Response
+        """
+        if response.status_code == 401 and self.REST_ENDPOINT in os.environ:
+            # refresh token and retry request - only on hopsworks
+            self._auth = auth.BearerAuth(self._read_jwt())
+            # Update request with the new token
+            request.auth = self._auth
+            # retry request
+            return True
+        return False
+
+    def _get_host_port_pair(self):
+        """
+        Removes "http or https" from the rest endpoint and returns a list
+        [endpoint, port], where endpoint is on the format /path.. without http://
+
+        :return: a list [endpoint, port]
+        :rtype: list
+        """
+        endpoint = self._base_url
+        if endpoint.startswith("http"):
+            last_index = endpoint.rfind("/")
+            endpoint = endpoint[last_index + 1 :]
+        host, port = endpoint.split(":")
+        return host, port
+
+    def _read_jwt(self):
+        """Retrieve jwt from local container."""
+        with open(os.path.join(self._secrets_dir, self.TOKEN_FILE), "r") as jwt:
+            return jwt.read()
+
+    def _close(self):
+        """Closes a client. Can be implemented for clean up purposes, not mandatory."""
+        self._connected = False

python/hsml/client/external.py → python/hsml/client/hopsworks/external.py

@@ -20,10 +20,11 @@
 import json
 import requests
 
-from hsml.client import base, auth, exceptions
+from hsml.client import auth, exceptions
+from hsml.client.hopsworks import base as hopsworks
 
 
-class Client(base.Client):
+class Client(hopsworks.Client):
     DEFAULT_REGION = "default"
     SECRETS_MANAGER = "secretsmanager"
     PARAMETER_STORE = "parameterstore"

python/hsml/client/hopsworks.py → python/hsml/client/hopsworks/internal.py

@@ -20,15 +20,16 @@
 import base64
 
 from pathlib import Path
-from hsml.client import base, auth
+from hsml.client import auth
+from hsml.client.hopsworks import base as hopsworks
 
 try:
     import jks
 except ImportError:
     pass
 
 
-class Client(base.Client):
+class Client(hopsworks.Client):
     REQUESTS_VERIFY = "REQUESTS_VERIFY"
     DOMAIN_CA_TRUSTSTORE_PEM = "DOMAIN_CA_TRUSTSTORE_PEM"
     PROJECT_ID = "HOPSWORKS_PROJECT_ID"

python/hsml/client/istio/__init__.py

@@ -0,0 +1,15 @@
+#
+#   Copyright 2022 Logical Clocks AB
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#