Use swiftclient.Connection to handle operation retries natively (#93)

Author: leo-naeka

swift/storage.py

@@ -145,6 +145,7 @@ class SwiftStorage(Storage):
     _token = ''
     name_prefix = setting('SWIFT_NAME_PREFIX', '')
     full_listing = setting('SWIFT_FULL_LISTING', True)
+    max_retries = setting('SWIFT_MAX_RETRIES', 5)
 
     def __init__(self, **settings):
         # check if some of the settings provided as class attributes
@@ -169,21 +170,19 @@ def __init__(self, **settings):
         }
         self.os_options.update(self.os_extra_options)
 
-        # Get authentication token
-        self.storage_url, self.token = swiftclient.get_auth(
-            self.api_auth_url,
-            self.api_username,
-            self.api_key,
-            auth_version=self.auth_version,
-            os_options=self.os_options)
-        self.http_conn = swiftclient.http_connection(self.storage_url)
+        # Get Connection wrapper
+        self.swift_conn = swiftclient.Connection(
+            authurl=self.api_auth_url,
+            user=self.api_username,
+            key=self.api_key,
+            retries=self.max_retries,
+            tenant_name=self.tenant_name,
+            os_options=self.os_options,
+            auth_version=self.auth_version)
 
         # Check container
         try:
-            swiftclient.head_container(self.storage_url,
-                                       self.token,
-                                       self.container_name,
-                                       http_conn=self.http_conn)
+            self.swift_conn.head_container(self.container_name)
         except swiftclient.ClientException:
             headers = {}
             if self.auto_create_container:
@@ -192,11 +191,8 @@ def __init__(self, **settings):
                 if self.auto_create_container_allow_orgin:
                     headers['X-Container-Meta-Access-Control-Allow-Origin'] = \
                         self.auto_create_container_allow_orgin
-                swiftclient.put_container(self.storage_url,
-                                          self.token,
-                                          self.container_name,
-                                          http_conn=self.http_conn,
-                                          headers=headers)
+                self.swift_conn.put_container(self.container_name,
+                                              headers=headers)
             else:
                 raise ImproperlyConfigured(
                     "Container %s does not exist." % self.container_name)
@@ -205,7 +201,7 @@ def __init__(self, **settings):
             # Derive a base URL based on the authentication information from
             # the server, optionally overriding the protocol, host/port and
             # potentially adding a path fragment before the auth information.
-            self.base_url = self.storage_url + '/'
+            self.base_url = self.swift_conn.url + '/'
             if self.override_base_url is not None:
                 # override the protocol and host, append any path fragments
                 split_derived = urlparse.urlsplit(self.base_url)
@@ -222,32 +218,11 @@ def __init__(self, **settings):
         else:
             self.base_url = self.override_base_url
 
-    def get_token(self):
-        if time() - self._token_creation_time >= self.auth_token_duration:
-            new_token = swiftclient.get_auth(
-                self.api_auth_url,
-                self.api_username,
-                self.api_key,
-                auth_version=self.auth_version,
-                os_options=self.os_options)[1]
-            self.token = new_token
-        return self._token
-
-    def set_token(self, new_token):
-        self._token_creation_time = time()
-        self._token = new_token
-
-    token = property(get_token, set_token)
-
     def _open(self, name, mode='rb'):
         original_name = name
         name = self.name_prefix + name
 
-        headers, content = swiftclient.get_object(self.storage_url,
-                                                  self.token,
-                                                  self.container_name,
-                                                  name,
-                                                  http_conn=self.http_conn)
+        headers, content = self.swift_conn.get_object(self.container_name, name)
         buf = BytesIO(content)
         buf.name = os.path.basename(original_name)
         buf.mode = mode
@@ -273,15 +248,12 @@ def _save(self, name, content, headers=None):
         else:
             content_type = mimetypes.guess_type(name)[0]
         content_length = content.size
-        swiftclient.put_object(self.storage_url,
-                               self.token,
-                               self.container_name,
-                               name,
-                               content,
-                               http_conn=self.http_conn,
-                               content_type=content_type,
-                               content_length=content_length,
-                               headers=headers)
+        self.swift_conn.put_object(self.container_name,
+                                   name,
+                                   content,
+                                   content_length=content_length,
+                                   content_type=content_type,
+                                   headers=headers)
         return original_name
 
     def get_headers(self, name):
@@ -294,12 +266,8 @@ def get_headers(self, name):
         """
         if name != self.last_headers_name:
             # miss -> update
-            self.last_headers_value = swiftclient.head_object(
-                self.storage_url,
-                self.token,
-                self.container_name,
-                name,
-                http_conn=self.http_conn)
+            self.last_headers_value = self.swift_conn.head_object(
+                self.container_name, name)
             self.last_headers_name = name
         return self.last_headers_value
 
@@ -314,11 +282,7 @@ def exists(self, name):
     @prepend_name_prefix
     def delete(self, name):
         try:
-            swiftclient.delete_object(self.storage_url,
-                                      self.token,
-                                      self.container_name,
-                                      name,
-                                      http_conn=self.http_conn)
+            self.swift_conn.delete_object(self.container_name, name)
         except swiftclient.ClientException:
             pass
 
@@ -386,9 +350,8 @@ def isdir(self, name):
 
     @prepend_name_prefix
     def listdir(self, path):
-        container = swiftclient.get_container(self.storage_url, self.token,
-                                              self.container_name, prefix=path,
-                                              full_listing=self.full_listing)
+        container = self.swift_conn.get_container(
+            self.container_name, prefix=path, full_listing=self.full_listing)
         files = []
         dirs = []
         for obj in container[1]:
@@ -404,23 +367,18 @@ def listdir(self, path):
 
     @prepend_name_prefix
     def makedirs(self, dirs):
-        swiftclient.put_object(self.storage_url,
-                               token=self.token,
-                               container=self.container_name,
-                               name='%s/.' % (self.name_prefix + dirs),
-                               contents='')
+        self.swift_conn.put_object(self.container_name,
+                                   '%s/.' % (self.name_prefix + dirs),
+                                   contents='')
 
     @prepend_name_prefix
     def rmtree(self, abs_path):
-        container = swiftclient.get_container(self.storage_url, self.token,
-                                              self.container_name)
+        container = self.swift_conn.get_container(self.container_name)
 
         for obj in container[1]:
             if obj['name'].startswith(abs_path):
-                swiftclient.delete_object(self.storage_url,
-                                          token=self.token,
-                                          container=self.container_name,
-                                          name=obj['name'])
+                self.swift_conn.delete_object(self.container_name,
+                                              obj['name'])
 
 
 class StaticSwiftStorage(SwiftStorage):

tests/tests.py

@@ -160,18 +160,18 @@ def test_illegal_extra_opts(self):
             self.default_storage('v3', os_extra_options="boom!")
 
 
-@patch('swift.storage.swiftclient', new=FakeSwift)
-class TokenTest(SwiftStorageTestCase):
-
-    def test_get_token(self):
-        """Renewing token"""
-        backend = self.default_storage('v3', auth_token_duration=0)
-        backend.get_token()
-
-    def test_set_token(self):
-        """Set token manually"""
-        backend = self.default_storage('v3', auth_token_duration=0)
-        backend.set_token('token')
+# @patch('swift.storage.swiftclient', new=FakeSwift)
+# class TokenTest(SwiftStorageTestCase):
+
+#     def test_get_token(self):
+#         """Renewing token"""
+#         backend = self.default_storage('v3', auth_token_duration=0)
+#         backend.get_token()
+
+#     def test_set_token(self):
+#         """Set token manually"""
+#         backend = self.default_storage('v3', auth_token_duration=0)
+#         backend.set_token('token')
 
 
 @patch('swift.storage.swiftclient', new=FakeSwift)

tests/utils.py

@@ -90,6 +90,71 @@ class FakeSwift(object):
     objects = CONTAINER_CONTENTS
     containers = ['container']
 
+    class Connection(object):
+        def __init__(self, authurl=None, user=None, key=None, retries=5,
+                     preauthurl=None, preauthtoken=None, snet=False,
+                     starting_backoff=1, max_backoff=64, tenant_name=None,
+                     os_options=None, auth_version="1", cacert=None,
+                     insecure=False, cert=None, cert_key=None,
+                     ssl_compression=True, retry_on_ratelimit=False,
+                     timeout=None, session=None):
+            self.service_token = None
+
+        def _retry(self, reset_func, func, *args, **kwargs):
+            self.url, self.token = self.get_auth()
+            self.http_conn = None
+            return func(self.url, self.token, *args,
+                        service_token=self.service_token, **kwargs)
+
+        def get_auth(self):
+            return base_url(), TOKEN
+
+        def head_container(self, container, headers=None):
+            return self._retry(None, FakeSwift.head_container, container,
+                               headers=headers)
+
+        def put_container(self, container, headers=None, response_dict=None,
+                          query_string=None):
+            return self._retry(None, FakeSwift.put_container, container,
+                               headers=headers, response_dict=response_dict,
+                               query_string=query_string)
+
+        def head_object(self, container, obj, headers=None):
+            return self._retry(None, FakeSwift.head_object, container, obj,
+                               headers=headers)
+
+        def get_object(self, container, obj, resp_chunk_size=None,
+                       query_string=None, response_dict=None, headers=None):
+            return self._retry(None, FakeSwift.get_object, container, obj,
+                               resp_chunk_size=resp_chunk_size,
+                               query_string=query_string,
+                               response_dict=response_dict, headers=headers)
+
+        def get_container(self, container, marker=None, limit=None, prefix=None,
+                          delimiter=None, end_marker=None, path=None,
+                          full_listing=False, headers=None, query_string=None):
+            return self._retry(None, FakeSwift.get_container, container,
+                               marker=marker, limit=limit, prefix=prefix,
+                               delimiter=delimiter, end_marker=end_marker,
+                               path=path, full_listing=full_listing,
+                               headers=headers, query_string=query_string)
+
+        def delete_object(self, container, obj, query_string=None,
+                          response_dict=None, headers=None):
+            return self._retry(None, FakeSwift.delete_object, container, obj,
+                               query_string=query_string,
+                               response_dict=response_dict, headers=headers)
+
+        def put_object(self, container, obj, contents, content_length=None,
+                       etag=None, chunk_size=None, content_type=None,
+                       headers=None, query_string=None, response_dict=None):
+            return self._retry(None, FakeSwift.put_object, container, obj,
+                               contents, content_length=content_length,
+                               etag=etag, chunk_size=chunk_size,
+                               content_type=content_type, headers=headers,
+                               query_string=query_string,
+                               response_dict=response_dict)
+
     @classmethod
     def get_auth(cls, auth_url, user, passwd, **kwargs):
         return base_url(), TOKEN
@@ -138,7 +203,7 @@ def delete_object(cls, url, token, container, name, **kwargs):
     @classmethod
     def put_object(cls, url, token, container, name=None, contents=None,
                    http_conn=None, content_type=None, content_length=None,
-                   headers=None):
+                   headers=None, **kwargs):
         if not name:
             raise ValueError("Attempting to add an object with no name/path")
         FakeSwift.objects.append(create_object(name))

tox.ini

@@ -4,8 +4,9 @@ args_are_paths = false
 envlist =
     flake8,
     isort,
-    py27-{1.9,1.10,master}
-    py{34,35}-{1.9,1.10,master}
+    py27-{1.9,1.10}
+    py34-{1.9,1.10}
+    py35-{1.9,1.10,master}
 
 [testenv]
 usedevelop = true