TDL-19745 unittests and retrylogic for 500 exceptions

.circleci/config.yml

@@ -23,6 +23,18 @@ jobs:
             source /usr/local/share/virtualenvs/tap-dixa/bin/activate
             pip install pylint==2.14.1
             pylint tap_dixa --disable C,W,R,no-member
+      - add_ssh_keys
+      - run:
+          name: 'Unit Tests'
+          command: |
+            source /usr/local/share/virtualenvs/tap-dixa/bin/activate
+            pip install nose coverage
+            nosetests --with-coverage --cover-erase --cover-package=tap_dixa --cover-html-dir=htmlcov tests/unittests
+            coverage html
+      - store_test_results:
+          path: test_output/report.xml
+      - store_artifacts:
+          path: htmlcov
 workflows:
   version: 2
   commit:

tap_dixa/client.py

@@ -3,8 +3,9 @@
 import backoff
 import requests
 
-from tap_dixa.exceptions import (DixaClient429Error,DixaClient400Error, 
-                                 raise_for_error, retry_after_wait_gen)
+from tap_dixa.exceptions import (DixaClient429Error, DixaClient408Error, 
+                                DixaClient5xxError, raise_for_error,
+                                retry_after_wait_gen)
 from tap_dixa.helpers import DixaURL
 
 class Client:
@@ -59,9 +60,8 @@ def _post(self, url, headers=None, params=None, data=None):
         """
         return self._make_request(url, method="POST", headers=headers, params=params, data=data)
 
-    # Added retry logic for 3 times when bad request happens
-    @backoff.on_exception(retry_after_wait_gen, DixaClient400Error, jitter=None, max_tries=3)
-    @backoff.on_exception(retry_after_wait_gen, DixaClient429Error, jitter=None, max_tries=3)
+    # Added retry logic for 3 times when bad request or server error or rate limit happens
+    @backoff.on_exception(retry_after_wait_gen, (DixaClient429Error, DixaClient5xxError,DixaClient408Error), jitter=None, max_tries=3)
     def _make_request(self, url, method, headers=None, params=None, data=None) -> dict:
         """
         Makes the API request.

tap_dixa/exceptions.py

@@ -17,39 +17,33 @@ def __init__(self, message=None, response=None):
         self.response = response
 
 
-class DixaClient5xxError(DixaClientError):
+class DixaClient400Error(DixaClientError):
     pass
 
-
 class DixaClient401Error(DixaClientError):
     pass
 
-
-class DixaClient400Error(DixaClientError):
+class DixaClient408Error(DixaClientError):
     pass
 
-
 class DixaClient429Error(DixaClientError):
     pass
 
 
 class DixaClient422Error(DixaClientError):
     pass
 
+class DixaClient5xxError(DixaClientError):
+    pass
 
 ERROR_CODE_EXCEPTION_MAPPING = {
-    500: {
-        "raise_exception": DixaClient5xxError,
-        "message": "Server Error",
-    },
-    503: {
-        "raise_exception": DixaClient5xxError,
-        "message": "Server Error",
-    },
-    401: {"raise_exception": DixaClient401Error, "message": "Invalid or missing credentials"},
     400: {"raise_exception": DixaClient400Error, "message": "Invalid query parameters"},
-    429: {"raise_exception": DixaClient429Error, "message": "API limit has been reached"},
+    401: {"raise_exception": DixaClient401Error, "message": "Invalid or missing credentials"},
+    408: {"raise_exception": DixaClient408Error, "message": "Request Timeout"},
     422: {"raise_exception": DixaClient422Error, "message": "Exceeded max allowed 10 csids per request"},
+    429: {"raise_exception": DixaClient429Error, "message": "API limit has been reached"},
+    500: {"raise_exception": DixaClient5xxError,"message": "Dixa Server Error",},
+    503: {"raise_exception": DixaClient5xxError,"message": "Dixa Server Unavailable",},
 }
 
 

tap_dixa/helpers.py

@@ -7,11 +7,7 @@
 from typing import Iterator
 from urllib.parse import parse_qsl, urlparse
 
-from singer import get_logger, utils
-
-LOGGER = get_logger()
-
-LOGGER = get_logger()
+from singer import  utils
 
 def unix_ms_to_date(timestamp_ms: int) -> str:
     """

tap_dixa/streams/abstracts.py

@@ -29,7 +29,7 @@ def __init__(self, client: Client):
         self.client = client
 
     @abstractmethod
-    def get_records(self, start_date: datetime.datetime = None) -> list:
+    def get_records(self, start_date: datetime.datetime = None, config: dict = {}) -> list:
         """
         Returns a list of records for that stream.
 
@@ -125,8 +125,7 @@ def sync(self, state: dict, stream_schema: dict, stream_metadata: dict, config:
 
         with singer.metrics.record_counter(self.tap_stream_id) as counter:
             for record in self.get_records(bookmark_datetime):
-                transformed_record = transformer.transform(
-                    record, stream_schema, stream_metadata)
+                transformed_record = transformer.transform(record, stream_schema, stream_metadata)
                 record_datetime = transformed_record[self.replication_key]
                 if record_datetime >= bookmark_datetime:
                     singer.write_record(self.tap_stream_id, transformed_record)

tap_dixa/streams/activitylogs.py

@@ -38,10 +38,8 @@ def sync(self, state: dict, stream_schema: dict, stream_metadata: dict, config:
 
         with metrics.record_counter(self.tap_stream_id) as counter:
             for record in self.get_records(bookmark_datetime, config=config):
-                transformed_record = transformer.transform(
-                    record, stream_schema, stream_metadata)
-                record_datetime = singer.utils.strptime_to_utc(
-                    transformed_record[self.replication_key])
+                transformed_record = transformer.transform(record, stream_schema, stream_metadata)
+                record_datetime = singer.utils.strptime_to_utc(transformed_record[self.replication_key])
                 if record_datetime >= bookmark_datetime:
                     singer.write_record(self.tap_stream_id, transformed_record)
                     counter.increment()

tests/unittests/test_backoff.py

@@ -0,0 +1,80 @@
+from unittest.mock import patch
+import unittest
+import requests
+from datetime import datetime
+
+from tap_dixa.client import Client, DixaClient429Error
+from tap_dixa.exceptions import DixaClient5xxError
+
+
+class Mockresponse:
+    def __init__(self, resp, status_code, content=[], headers=None, raise_error=False):
+        self.json_data = resp
+        self.status_code = status_code
+        self.content = content
+        self.headers = headers
+        self.raise_error = raise_error
+
+    def raise_for_status(self):
+        if not self.raise_error:
+            return self.status_code
+
+        raise requests.HTTPError("sample message")
+
+def mocked_failed_429_request(*args, **kwargs):
+    return Mockresponse('', 429, headers={}, raise_error=True)
+
+class Test_backoff(unittest.TestCase):
+
+    @patch("time.sleep")
+    @patch("requests.Session.request", side_effect=mocked_failed_429_request)
+    def test_too_many_requests_429_error(self, mocked_send, mocked_sleep):
+        client = Client(api_token="test")
+
+        try:
+            """
+            Verifying if the custom exception 'DixaClient429Error' 
+            is raised on receiving status code 429
+            """
+            _ = client.get("https://test.com", "/test")
+        except DixaClient429Error as api_error:
+            pass
+
+            """
+            Verifying the retry is happening thrice for the 429 exception
+            """
+        self.assertEquals(mocked_send.call_count, 3)
+
+    @patch("requests.Session.request", side_effect=mocked_failed_429_request)
+    def test_request_timeout_and_backoff(self, mock_send):
+        """
+        Check whether the request backoffs properly for get call for more than a minute for Server429Error.
+        """
+        mock_send.side_effect = DixaClient429Error
+        client = Client(api_token="test")
+        before_time = datetime.now()
+        with self.assertRaises(DixaClient429Error):
+            _ = client.get("https://test.com", "/test")
+        after_time = datetime.now()
+        # verify that the tap backoff for more than 60 seconds
+        time_difference = (after_time - before_time).total_seconds()
+        self.assertTrue(60 <= time_difference <= 121)
+
+    @patch("time.sleep")
+    @patch("requests.Session.request", side_effect= lambda *args, **kwargs : Mockresponse('', 500, headers={}, raise_error=True))
+    def test_500_server_error(self, mocked_send, mocked_sleep):
+        client = Client(api_token="test")
+
+        try:
+            """
+            Verifying if the custom exception 'DixaClient5xxError' 
+            is raised on receiving status code 500
+            """
+            _ = client.get("https://test.com", "/test")
+        except DixaClient5xxError as api_error:
+            pass
+
+            """
+            Verifying the retry is happening thrice for the 500 server error exception
+            """
+        self.assertEquals(mocked_send.call_count, 3)

tests/unittests/test_config_interval.py

@@ -0,0 +1,29 @@
+import unittest
+from tap_dixa.streams.abstracts import IncrementalStream
+from tap_dixa.exceptions import InvalidInterval
+from tap_dixa.helpers import Interval
+
+
+class test_increemtalstream(unittest.TestCase):
+    """
+    Testing that valid value is passed in the interval attribute of config.json
+    Allowed values in interval are Hour,Day, Week and Month
+    """
+    def test_invalid_value_of_interval(self):
+        try :
+            config = {'interval': "INVALID"}
+            IncrementalStream.set_interval(self,value=config['interval'])
+            response = IncrementalStream.get_interval(self)
+        except InvalidInterval as e :
+            expected_error_message = "invalid interval provided"
+            self.assertEquals(str(e), expected_error_message)
+
+    def test_valid_value_of_interval(self):
+        try :
+            config = {'interval': "MONTH"}
+            IncrementalStream.set_interval(self,value=config['interval'])
+            response = IncrementalStream.get_interval(self)
+        except InvalidInterval as e :
+            expected_error_message = "invalid interval provided"
+        self.assertEquals(response,Interval.MONTH.value)
+