Cp 7117/add retries to api calls (#124)

* Automatically retry failed 5xx api calls

* Add additional tests for retries

* Fix test failing on CI by using different way to compare json response

* Increase test case speed by modifying backoff_factor to 0 for tests which deal with error responses

* Allow test_retries tests to pass even when user has toggled use_retries to False

* Dont wait on retry-after headers to keep current functionality intact

* Add documentation for retry configuration

* Update failing datatable export test following merge

* Reduce default retry settings so retries will be less aggressive

* Uncaptialize retry status codes config

* Allow retries on 429's

* Update readme retry examples to have default values

* Bump version

* Update changelog

* Remove debug statements

Author: jjmar

CHANGELOG.md

@@ -1,6 +1,14 @@
 ### unreleased
 * Remove dependency on unittest2, use unittest instead (#113)
 
+### 3.4.4 - 2018-10-24
+
+* Add functionality to automatically retry failed API calls https://github.com/quandl/quandl-python/pull/124
+
+### 3.4.3 - 2018-10-19
+
+* Allow for exporting of datatables https://github.com/quandl/quandl-python/pull/120
+
 ### 3.4.2 - 2018-08-21
 
 * Fix typos in our warning messages https://github.com/quandl/quandl-python/pull/114

README.md

@@ -28,7 +28,12 @@ pip3 install quandl
 |---|---|---|
 | api_key | Your access key | `tEsTkEy123456789` | Used to identify who you are and provide full access. |
 | api_version | The API version you wish to use | 2015-04-09 | Can be used to test your code against the latest version without committing to it. |
-
+| use_retries | Whether API calls which return statuses in `retry_status_codes` should be automatically retried | True
+| number_of_retries | Maximum number of retries that should be attempted. Only used if `use_retries` is True | 5
+| max_wait_between_retries | Maximum amount of time in seconds that should be waited before attempting a retry. Only used if `use_retries` is True | 8
+| retry_backoff_factor | Determines the amount of time in seconds that should be waited before attempting another retry. Note that this factor is exponential so a `retry_backoff_factor` of 0.5 will cause waits of [0.5, 1, 2, 4, etc]. Only used if `use_retries` is True | 0.5
+| retry_status_codes | A list of HTTP status codes which will trigger a retry to occur. Only used if `use_retries` is True| [429, 500, 501, 502, 503, 504, 505, 506, 507, 508, 509, 510, 511]
+    
 ```python
 import quandl
 quandl.ApiConfig.api_key = 'tEsTkEy123456789'

quandl/api_config.py

@@ -1,9 +1,16 @@
 class ApiConfig:
     api_key = None
-    api_base = 'https://www.quandl.com/api/v3'
+    api_protocol = 'https://'
+    api_base = '{}www.quandl.com/api/v3'.format(api_protocol)
     api_version = None
     page_limit = 100
 
+    use_retries = True
+    number_of_retries = 5
+    retry_backoff_factor = 0.5
+    max_wait_between_retries = 8
+    retry_status_codes = [429] + list(range(500, 512))
+
 
 def save_key(apikey, filename=None):
     if filename is None:

quandl/connection.py

@@ -1,6 +1,8 @@
 import re
 
 import requests
+from urllib3.util.retry import Retry
+from requests.adapters import HTTPAdapter
 
 from .util import Util
 from .version import VERSION
@@ -37,9 +39,10 @@ def request(cls, http_verb, url, **options):
 
     @classmethod
     def execute_request(cls, http_verb, url, **options):
+        session = cls.get_session()
+
         try:
-            func = getattr(requests, http_verb)
-            response = func(url, **options)
+            response = session.request(method=http_verb, url=url, **options)
             if response.status_code < 200 or response.status_code >= 300:
                 cls.handle_api_error(response)
             else:
@@ -49,6 +52,29 @@ def execute_request(cls, http_verb, url, **options):
                 cls.handle_api_error(e.response)
             raise e
 
+    @classmethod
+    def get_session(cls):
+        session = requests.Session()
+        adapter = HTTPAdapter(max_retries=cls.get_retries())
+        session.mount(ApiConfig.api_protocol, adapter)
+
+        return session
+
+    @classmethod
+    def get_retries(cls):
+        if not ApiConfig.use_retries:
+            return Retry(total=0)
+
+        Retry.BACKOFF_MAX = ApiConfig.max_wait_between_retries
+        retries = Retry(total=ApiConfig.number_of_retries,
+                        connect=ApiConfig.number_of_retries,
+                        read=ApiConfig.number_of_retries,
+                        status_forcelist=ApiConfig.retry_status_codes,
+                        backoff_factor=ApiConfig.retry_backoff_factor,
+                        raise_on_status=False)
+
+        return retries
+
     @classmethod
     def parse(cls, response):
         try:

quandl/version.py

@@ -1 +1 @@
-VERSION = '3.4.3'
+VERSION = '3.4.4'

test/test_connection.py

@@ -4,17 +4,18 @@
     QuandlError, LimitExceededError, InternalServerError,
     AuthenticationError, ForbiddenError, InvalidRequestError,
     NotFoundError, ServiceUnavailableError)
-import unittest
+from test.test_retries import ModifyRetrySettingsTestCase
 from test.helpers.httpretty_extension import httpretty
 import json
 from mock import patch, call
 from quandl.version import VERSION
 
 
-class ConnectionTest(unittest.TestCase):
+class ConnectionTest(ModifyRetrySettingsTestCase):
 
     @httpretty.activate
-    def test_quandl_exceptions(self):
+    def test_quandl_exceptions_no_retries(self):
+        ApiConfig.use_retries = False
         quandl_errors = [('QELx04', 429, LimitExceededError),
                          ('QEMx01', 500, InternalServerError),
                          ('QEAx01', 400, AuthenticationError),
@@ -38,6 +39,7 @@ def test_quandl_exceptions(self):
 
     @httpretty.activate
     def test_parse_error(self):
+        ApiConfig.retry_backoff_factor = 0
         httpretty.register_uri(httpretty.GET,
                                "https://www.quandl.com/api/v3/databases",
                                body="not json", status=500)
@@ -46,6 +48,7 @@ def test_parse_error(self):
 
     @httpretty.activate
     def test_non_quandl_error(self):
+        ApiConfig.retry_backoff_factor = 0
         httpretty.register_uri(httpretty.GET,
                                "https://www.quandl.com/api/v3/databases",
                                body=json.dumps(
@@ -70,6 +73,4 @@ def test_build_request(self, mock):
                                  'request-source': 'python',
                                  'request-source-version': VERSION},
                         params={'per_page': 10, 'page': 2})
-        print(mock.call_args)
-        print(expected)
         self.assertEqual(mock.call_args, expected)

test/test_database.py

@@ -14,6 +14,7 @@
 from quandl.api_config import ApiConfig
 from quandl.model.database import Database
 from quandl.connection import Connection
+from test.test_retries import ModifyRetrySettingsTestCase
 from mock import patch, call, mock_open
 from test.factories.database import DatabaseFactory
 from test.factories.meta import MetaFactory
@@ -110,10 +111,9 @@ def test_databases_has_more(self):
         self.assertTrue(results.has_more_results())
 
 
-class BulkDownloadDatabaseTest(unittest.TestCase):
+class BulkDownloadDatabaseTest(ModifyRetrySettingsTestCase):
 
-    @classmethod
-    def setUpClass(cls):
+    def setUp(self):
         httpretty.enable()
         httpretty.register_uri(httpretty.GET,
                                re.compile(
@@ -125,17 +125,15 @@ def setUpClass(cls):
         httpretty.register_uri(httpretty.GET,
                                re.compile('https://www.blah.com/'), body='{}')
 
-    @classmethod
-    def tearDownClass(cls):
-        httpretty.disable()
-        httpretty.reset()
-
-    def setUp(self):
         database = {'database': DatabaseFactory.build(database_code='NSE')}
         self.database = Database(database['database']['database_code'], database['database'])
         ApiConfig.api_key = 'api_token'
         ApiConfig.api_version = '2015-04-09'
 
+    def tearDown(self):
+        httpretty.disable()
+        httpretty.reset()
+
     def test_get_bulk_downnload_url_with_download_type(self):
         url = self.database.bulk_download_url(params={'download_type': 'partial'})
         parsed_url = urlparse(url)
@@ -179,12 +177,15 @@ def test_bulk_download_raises_exception_when_no_path(self):
             QuandlError, lambda: self.database.bulk_download_to_file(None))
 
     def test_bulk_download_raises_exception_when_error_response(self):
+        ApiConfig.retry_backoff_factor = 0
+        httpretty.reset()
         httpretty.register_uri(httpretty.GET,
                                re.compile(
                                    'https://www.quandl.com/api/v3/databases/*'),
                                body=json.dumps(
                                    {'quandl_error':
                                     {'code': 'QEMx01', 'message': 'something went wrong'}}),
                                status=500)
+
         self.assertRaises(
             InternalServerError, lambda: self.database.bulk_download_to_file('.'))

test/test_datatable.py

@@ -11,11 +11,12 @@
 from quandl.model.datatable import Datatable
 from mock import patch, call, mock_open
 from test.factories.datatable import DatatableFactory
+from test.test_retries import ModifyRetrySettingsTestCase
 from quandl.api_config import ApiConfig
 from quandl.errors.quandl_error import (InternalServerError, QuandlError)
 
 
-class GetDatatableDatasetTest(unittest.TestCase):
+class GetDatatableDatasetTest(ModifyRetrySettingsTestCase):
 
     @classmethod
     def setUpClass(cls):
@@ -114,12 +115,17 @@ def test_bulk_download_raises_exception_when_no_path(self):
                 QuandlError, lambda: self.datatable.download_file(None))
 
     def test_bulk_download_table_raises_exception_when_error_response(self):
+        httpretty.reset()
+        ApiConfig.number_of_retries = 2
+        error_responses = [httpretty.Response(
+            body=json.dumps({'quandl_error': {'code': 'QEMx01',
+                                              'message': 'something went wrong'}}),
+            status=500)]
+
         httpretty.register_uri(httpretty.GET,
                                re.compile(
                                    'https://www.quandl.com/api/v3/datatables/*'),
-                               body=json.dumps(
-                                   {'quandl_error':
-                                    {'code': 'QEMx01', 'message': 'something went wrong'}}),
-                               status=500)
+                               responses=error_responses)
+
         self.assertRaises(
             InternalServerError, lambda: self.datatable.download_file('.'))

test/test_retries.py

@@ -0,0 +1,112 @@
+import unittest
+import json
+
+from quandl.connection import Connection
+from quandl.api_config import ApiConfig
+from test.factories.datatable import DatatableFactory
+from test.helpers.httpretty_extension import httpretty
+from quandl.errors.quandl_error import InternalServerError
+
+
+class ModifyRetrySettingsTestCase(unittest.TestCase):
+
+    def setUp(self):
+        self.default_use_retries = ApiConfig.use_retries
+        self.default_number_of_retries = ApiConfig.number_of_retries
+        self.default_retry_backoff_factor = ApiConfig.retry_backoff_factor
+        self.default_max_wait_between_retries = ApiConfig.max_wait_between_retries
+        self.default_retry_status_codes = ApiConfig.retry_status_codes
+
+    def tearDown(self):
+        ApiConfig.use_retries = self.default_use_retries
+        ApiConfig.number_of_retries = self.default_number_of_retries
+        ApiConfig.retry_backoff_factor = self.default_retry_backoff_factor
+        ApiConfig.max_wait_between_retries = self.default_max_wait_between_retries
+        ApiConfig.retry_status_codes = self.default_retry_status_codes
+
+
+class TestRetries(ModifyRetrySettingsTestCase):
+
+    def setUp(self):
+        ApiConfig.use_retries = True
+        super(TestRetries, self).setUp()
+
+    @classmethod
+    def setUpClass(cls):
+        cls.datatable = {'datatable': DatatableFactory.build(
+            vendor_code='ZACKS',
+            datatable_code='FC')}
+
+        cls.error_response = httpretty.Response(
+            body=json.dumps({'quandl_error': {'code': 'QEMx01',
+                                              'message': 'something went wrong'}}),
+            status=500)
+        cls.success_response = httpretty.Response(body=json.dumps(cls.datatable), status=200)
+
+    def test_modifying_use_retries(self):
+        ApiConfig.use_retries = False
+
+        retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
+        self.assertEqual(retries.total, 0)
+
+    def test_modifying_number_of_retries(self):
+        ApiConfig.number_of_retries = 3000
+
+        retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
+
+        self.assertEqual(retries.total, ApiConfig.number_of_retries)
+        self.assertEqual(retries.connect, ApiConfig.number_of_retries)
+        self.assertEqual(retries.read, ApiConfig.number_of_retries)
+
+    def test_modifying_retry_backoff_factor(self):
+        ApiConfig.retry_backoff_factor = 3000
+
+        retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
+        self.assertEqual(retries.backoff_factor, ApiConfig.retry_backoff_factor)
+
+    def test_modifying_retry_status_codes(self):
+        ApiConfig.retry_status_codes = [1, 2, 3]
+
+        retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
+        self.assertEqual(retries.status_forcelist, ApiConfig.retry_status_codes)
+
+    def test_modifying_max_wait_between_retries(self):
+        ApiConfig.max_wait_between_retries = 3000
+
+        retries = Connection.get_session().get_adapter(ApiConfig.api_protocol).max_retries
+        self.assertEqual(retries.BACKOFF_MAX, ApiConfig.max_wait_between_retries)
+
+    @httpretty.activate
+    def test_correct_response_returned_if_retries_succeed(self):
+        ApiConfig.number_of_retries = 3
+        ApiConfig.retry_status_codes = [self.error_response.status]
+
+        mock_responses = [self.error_response] + [self.error_response] + [self.success_response]
+        httpretty.register_uri(httpretty.GET,
+                               "https://www.quandl.com/api/v3/databases",
+                               responses=mock_responses)
+
+        response = Connection.request('get', 'databases')
+        self.assertEqual(response.json(), self.datatable)
+        self.assertEqual(response.status_code, self.success_response.status)
+
+    @httpretty.activate
+    def test_correct_response_exception_raised_if_retries_fail(self):
+        ApiConfig.number_of_retries = 2
+        ApiConfig.retry_status_codes = [self.error_response.status]
+        mock_responses = [self.error_response] * 3
+        httpretty.register_uri(httpretty.GET,
+                               "https://www.quandl.com/api/v3/databases",
+                               responses=mock_responses)
+
+        self.assertRaises(InternalServerError, Connection.request, 'get', 'databases')
+
+    @httpretty.activate
+    def test_correct_response_exception_raised_for_errors_not_in_retry_status_codes(self):
+        ApiConfig.retry_status_codes = []
+        mock_responses = [self.error_response]
+        httpretty.register_uri(httpretty.GET,
+                               "https://www.quandl.com/api/v3/databases",
+                               responses=mock_responses)
+
+        self.assertRaises(InternalServerError, Connection.request, 'get', 'databases')