Persistent sessions between API-calls

Author: dariosky

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Changelog
 
+## v1.0.39 (2020-09-22)
+* Persistent sessions between API-calls
 
 ## v1.0.38 (2020-09-04)
 * Text-Analytics API additional functions: `get_analytics` and `get_annotated`

ravenpackapi/core.py

@@ -1,8 +1,6 @@
 import logging
 import os
 
-import requests
-
 from ravenpackapi import Dataset
 from ravenpackapi.exceptions import APIException
 from ravenpackapi.models.dataset_list import DatasetList
@@ -13,9 +11,10 @@
 from ravenpackapi.util import to_curl
 from ravenpackapi.utils.constants import ENTITY_TYPES
 from ravenpackapi.utils.date_formats import as_datetime_str
+from ravenpackapi.utils.dynamic_sessions import DynamicSession
 
 _VALID_METHODS = ('get', 'post', 'put', 'delete', 'patch')
-VERSION = '1.0.38'
+VERSION = '1.0.39'
 
 logger = logging.getLogger("ravenpack.core")
 
@@ -45,6 +44,7 @@ def __init__(self, api_key=None):
             )
         self.api_key = api_key
         self.log_curl_commands = True
+        self.session = DynamicSession()
         self.upload = UploadApi(self)
 
     @property
@@ -62,7 +62,7 @@ def request(self, endpoint, data=None, json=None,
             'Method {used} not accepted. Please choose one of {valid_methods}'.format(
                 used=method, valid_methods=", ".join(_VALID_METHODS)
             )
-        requests_call = getattr(requests, method)
+        requests_call = getattr(self.session, method)
         logger.debug("Request {method} to {endpoint}".format(method=method,
                                                              endpoint=endpoint))
         if endpoint.startswith("http://") or endpoint.startswith("https://"):

ravenpackapi/models/dataset.py

@@ -298,11 +298,11 @@ def request_realtime(self):
                                                            dataset_id=self.id)
         logger.debug("Connecting with RT feed: %s" % endpoint)
         try:
-            response = requests.get(endpoint,
-                                    headers=api.headers,
-                                    stream=True,
-                                    **api.common_request_params
-                                    )
+            response = api.session.get(endpoint,
+                                       headers=api.headers,
+                                       stream=True,
+                                       **api.common_request_params
+                                       )
             if response.status_code != 200:
                 logger.error("Error calling the API, we tried: %s" % to_curl(response.request))
                 raise APIException(

ravenpackapi/models/job.py

@@ -2,8 +2,6 @@
 import logging
 from time import sleep
 
-import requests
-
 from ravenpackapi.exceptions import (api_method,
                                      APIException,
                                      DataFileTimeout,
@@ -112,11 +110,11 @@ def save_to_file(self, filename):
 
             # this is a different request than the normal API
             # streaming the file in chunks
-            response = requests.get(job.url,
-                                    headers=api.headers,
-                                    stream=True,
-                                    **api.common_request_params
-                                    )
+            response = api.session.get(job.url,
+                                       headers=api.headers,
+                                       stream=True,
+                                       **api.common_request_params
+                                       )
             if response.status_code != 200:
                 logger.error("Error calling the API, we tried: %s" % to_curl(response.request))
                 raise APIException(
@@ -133,22 +131,21 @@ def iterate_results(self, include_headers=False):
         job = self  # just to be clear
         job.wait_for_completion()
 
-        with requests.Session() as s:
-            r = s.get(job.url,
-                      headers=api.headers,
-                      stream=True,
-                      **api.common_request_params
-                      )
-            iterator = r.iter_lines(chunk_size=self._CHUNK_SIZE)
+        r = api.session.get(job.url,
+                            headers=api.headers,
+                            stream=True,
+                            **api.common_request_params
+                            )
+        iterator = r.iter_lines(chunk_size=self._CHUNK_SIZE)
 
-            headers = next(iterator)  # discard the headers
+        headers = next(iterator)  # discard the headers
 
-            if include_headers:
-                yield parse_csv_line(headers)
+        if include_headers:
+            yield parse_csv_line(headers)
 
-            for line in iterator:
-                fields = parse_csv_line(line)
-                yield fields
+        for line in iterator:
+            fields = parse_csv_line(line)
+            yield fields
 
     def __iter__(self):
         # this will be yield from in Py3

ravenpackapi/tests/test_job_cancellation.py

@@ -37,15 +37,15 @@ def test_job_cancel(self):
             job.cancel()
         except APIException as exception:
             # cancel raised an exception, means that we were already processing it
-            assert status == 'processing'
-            assert exception.response.status_code == 400
-        else:
-            assert status == 'enqueued'
-            assert job.get_status() == 'cancelled'
-
-            assert job.is_processing is False
-            with pytest.raises(JobNotProcessing):
-                job.wait_for_completion()
+            if status == 'processing':
+                assert exception.response.status_code == 400
+            else:
+                assert status == 'enqueued'
+                assert job.get_status() == 'cancelled'
+
+                assert job.is_processing is False
+                with pytest.raises(JobNotProcessing):
+                    job.wait_for_completion()
 
     @classmethod
     def teardown_class(cls):

ravenpackapi/utils/dynamic_sessions.py

@@ -0,0 +1,47 @@
+from collections import defaultdict
+from functools import partial
+
+import requests
+from six.moves.urllib.parse import urlparse
+
+_REQUESTS_METHODS = ('get', 'post', 'put', 'delete', 'patch')
+
+
+# see test_dynamic_sessions.py for how to use it
+
+class DynamicSession(object):
+    """ This looks intricate but its a way to make requests.session work transparently creating different
+        session depending on the hostname of the urls
+    """
+    _session_by_host = defaultdict(requests.session)
+
+    @staticmethod
+    def get_session_from_args(args, kwargs):
+        url = kwargs.get('url', None)
+        if url is None:
+            url = args[0]  # get the positional url
+        parsed_uri = urlparse(url)
+        session_key = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
+        if 'verify' in kwargs:
+            session_key += "-noverify"  # unverified calls get a different session
+        session = DynamicSession._session_by_host[session_key]
+        return session
+
+    @staticmethod
+    def get_session_method(method, *args, **kwargs):
+        # this will be returned as a partial - it will get a session for the url in the call
+        # that will get the proper session for that host
+        session = DynamicSession.get_session_from_args(args, kwargs)
+        return getattr(session, method)(*args, **kwargs)
+
+    @staticmethod
+    def __getattr__(method):
+        # when we do DynamicSession.post - it will return a partial - that will ask to get
+        if method in _REQUESTS_METHODS:
+            return partial(DynamicSession.get_session_method, method)
+        return getattr(requests, method)
+
+    @staticmethod
+    def get_session(*args, **kwargs):
+        """ This is the only method that is only from here - everything else is taken from requests """
+        return DynamicSession.get_session_from_args(args, kwargs)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '1.0.38'
+VERSION = '1.0.39'
 
 with open('README.rst') as readme_file:
     readme = readme_file.read()