Add exponential backoff and retry functionality to rest_client

.gitignore

@@ -121,4 +121,4 @@ venv_/
 .vscode
 
 Pipfile
-Pipfile.lock
+Pipfile.lock

atlassian/rest_client.py

@@ -2,8 +2,8 @@
 import logging
 from json import dumps
 
+import random
 import requests
-
 try:
     from oauthlib.oauth1.rfc5849 import SIGNATURE_RSA_SHA512 as SIGNATURE_RSA
 except ImportError:
@@ -60,7 +60,46 @@ def __init__(
         cloud=False,
         proxies=None,
         token=None,
+        backoff_and_retry=False,
+        retry_error_matches=[(429, "Too Many Requests"),
+                                         (429, "Unknown Status Code")],
+        max_backoff_seconds=1800,
+        max_backoff_retries=1000,
     ):
+        """
+        init function for the AtlassianRestAPI object.
+
+        Args:
+            url (str): The url to be used in the request.
+            username (str, optional): Username Defaults to None.
+            password (sstr, optional): Password. Defaults to None.
+            timeout (int, optional): Request timeout. Defaults to 75.
+            api_root (str, optional): Root for the api requests. Defaults to "rest/api".
+            api_version (str, optional): Version of the API to use. Defaults to "latest".
+            verify_ssl (bool, optional): Turn on / off SSL verification. Defaults to True.
+            session ([type], optional): Pass an existing Python requests session object. Defaults to None.
+            oauth ([type], optional): oauth. Defaults to None.
+            oauth2 ([type], optional): oauth2. Defaults to None.
+            cookies ([type], optional): Cookies to send with the request. Defaults to None.
+            advanced_mode ([type], optional): Return results in advanced mode. Defaults to None.
+            kerberos ([type], optional): Kerberos. Defaults to None.
+            cloud (bool, optional): Specify if using Atlassian Cloud. Defaults to False.
+            proxies ([type], optional): Specify proxies to use. Defaults to None.
+            token ([type], optional): Atlassian / Jira auth token. Defaults to None.
+            backoff_and_retry (bool, optional): Enable exponential backoff and retry.
+                This will retry the request if there is a predefined failure. Primarily
+                designed for Atlassian Cloud where API limits are commonly hit if doing
+                operations on many issues, and the limits require a cooling off period.
+                The wait period before the next request increases exponentially with each
+                failed retry. Defaults to False.
+            retry_error_matches (list, optional): Errors to match, passed as a list of tuples
+                containing the response code and the response text to match (exact match).
+                Defaults to the rate limit error from Atlassian Cloud - [(429, 'Too Many Requests')].
+            max_backoff_seconds (int, optional): Max backoff seconds. When backing off, requests won't
+                wait any longer than this. Defaults to 1800.
+            max_backoff_retries (int, optional): Maximum number of retries to try before
+                continuing. Defaults to 1000.
+        """
         self.url = url
         self.username = username
         self.password = password
@@ -72,6 +111,10 @@ def __init__(
         self.advanced_mode = advanced_mode
         self.cloud = cloud
         self.proxies = proxies
+        self.backoff_and_retry = backoff_and_retry
+        self.retry_error_matches = retry_error_matches
+        self.max_backoff_seconds = max_backoff_seconds
+        self.max_backoff_retries = max_backoff_retries
         if session is None:
             self._session = requests.Session()
         else:
@@ -236,17 +279,35 @@ def request(
             data=data if data else json_dump,
         )
         headers = headers or self.default_headers
-        response = self._session.request(
-            method=method,
-            url=url,
-            headers=headers,
-            data=data,
-            json=json,
-            timeout=self.timeout,
-            verify=self.verify_ssl,
-            files=files,
-            proxies=self.proxies,
-        )
+
+        backoff = 1
+        retries = 0
+        while True:
+            response = self._session.request(
+                method=method,
+                url=url,
+                headers=headers,
+                data=data,
+                json=json,
+                timeout=self.timeout,
+                verify=self.verify_ssl,
+                files=files,
+                proxies=self.proxies,
+            )
+            if self.backoff_and_retry:
+                for em in self.retry_error_matches:
+                    if retries > self.max_backoff_retries:
+                        log.warning("Hit max backoff retry limit of {0}, no more retries.".format(self.max_backoff_retries))
+                        responseloop = False
+                        break
+                    if response.status_code == em[0] and response.reason == em[1]:
+                        log.warning('Backing off due to error "{0}: {1}" for {2}s'.format(em[0], em[1], backoff))
+                        time.sleep(backoff + (random.random() * backoff / 10))
+                        backoff = min(2 * backoff, self.max_backoff_seconds)
+                        retries += 1
+            else:
+                break
+
         response.encoding = "utf-8"
 
         log.debug("HTTP: %s %s -> %s %s", method, path, response.status_code, response.reason)