Tox, Pytest, API methods for models and json endpoint support

Author: Dario Varotto

.gitignore

@@ -7,3 +7,4 @@
 /*.sh
 /dist
 /build
+/.*/

ravenpackapi/__init__.py

@@ -1 +1,2 @@
 from .models.dataset import Dataset
+from .core import RPApi

ravenpackapi/core.py

@@ -1,36 +1,35 @@
 import json
 import logging
 import os
-from time import sleep
 
 import requests
 
 from ravenpackapi import Dataset
 from ravenpackapi.exceptions import APIException
-from ravenpackapi.models.dataset import DatasetList
-from ravenpackapi.models.job import Job
+from ravenpackapi.models.dataset_list import DatasetList
+from ravenpackapi.models.results import Results
 from ravenpackapi.util import to_curl
+from ravenpackapi.utils.constants import JSON_AVAILABLE_FIELDS
 
 _VALID_METHODS = ('get', 'post', 'put', 'delete')
 
 logger = logging.getLogger("ravenpack.core")
 
 
 class RPApi(object):
-    _BASE_URL = os.environ.get('RP_API_ENDPOINT', 'https://api.ravenpack.com/1.0')
-    _FILE_AVAILABILIY_SECONDS_DELAY = 5.0
-    _CHUNK_SIZE = 1024 * 32
+    _BASE_URL = os.environ.get('RP_API_ENDPOINT',
+                               'https://api.ravenpack.com/1.0')
 
-    def __init__(self, api_key):
+    def __init__(self, api_key=None):
         api_key = api_key or os.environ.get('RP_API_KEY')
         if api_key is None:
             raise ValueError(
                 "Please initialize with an api_key "
-                "or set your environment RP_API_KEY with a permanent token"
+                "or set your environment RP_API_KEY with your API KEY."
             )
         self.api_key = api_key
 
-    def request(self, endpoint, data=None, method='get'):
+    def request(self, endpoint, data=None, params=None, method='get'):
         assert method in _VALID_METHODS, 'Method {used} not accepted. Please use {valid_methods}'
         logger.debug("Request to %s" % endpoint)
         requests_call = getattr(requests, method)
@@ -40,79 +39,70 @@ def request(self, endpoint, data=None, method='get'):
             url=self._BASE_URL + endpoint,
             headers=dict(API_KEY=self.api_key),
             data=json.dumps(data) if data else None,
+            params=params,
         )
         if response.status_code != 200:
             logger.error("Error calling the API, we tried: %s" % to_curl(response.request))
-            raise APIException('Got an error {status}: {error_message}'.format(
-                status=response.status_code, error_message=response.text
-            ), response=response)
+            raise APIException(
+                'Got an error {status}: body was \'{error_message}\''.format(
+                    status=response.status_code, error_message=response.text
+                ), response=response)
         return response
 
-    def list_datasets(self, tags=None, scope=None):
+    def list_datasets(self, scope=None, tags=None):
         """ Return a DataSetList of datasets in the scope """
-        response = self.request('/datasets', data=dict(
-            tags=tags or [],
+        response = self.request('/datasets', params=dict(
+            tags=tags or None,
             scope=scope or 'private',
         ))
-        return DatasetList(map(Dataset.from_dict, response.json()['datasets']))
+        return DatasetList(
+            map(lambda item: Dataset.from_dict(item, api=self),
+                response.json()['datasets'])
+        )
 
     def create_dataset(self, dataset):
         response = self.request(endpoint="/datasets",
                                 data=dataset.as_dict(),
                                 method='post')
         dataset_id = response.json()['dataset_uuid']
         logger.info("Created dataset %s" % dataset_id)
-        return dataset_id
 
-    def download_dataset(self, dataset_id, start_date, end_date,
-                         output_format='csv', compressed=False, notify=False):
-        response = self.request(
-            endpoint="/datafile/%s" % dataset_id,
-            data={
-                "start_date": start_date,
-                "end_date": end_date,
-                "format": output_format,
-                "compressed": compressed,
-                "notify": notify,
-            },
-            method='post',
+        # we return the Dataset object just created
+        dataset.api = self
+        new_dataset_data = dataset.as_dict()
+        new_dataset_data['uuid'] = dataset_id
+        new_dataset = Dataset(api=self,
+                              **new_dataset_data)
+        return new_dataset
+
+    def get_dataset(self, dataset_id):
+        return Dataset(
+            api=self,
+            uuid=dataset_id,
         )
-        job = Job(response.json()['token'])  # an undefined job, has just the token
-        return job
 
-    def get_job_status(self, job):
-        token = job.token
+    def json(self,
+             start_date,
+             end_date,
+             fields,
+             filters=None,
+             time_zone=None,
+             frequency='granular',
+             having=None,
+             product='rpa',
+             product_version='1.0',
+             ):
+        # let's build the body, with all the defined fields
+        body = {}
+        for k in JSON_AVAILABLE_FIELDS:
+            if locals().get(k) is not None:
+                body[k] = locals().get(k)
+
         response = self.request(
-            endpoint="/jobs/%s" % token,
-            data={
-                "token": token,
-            },
-            method='get',
+            endpoint="/json",
+            method='post',
+            data=body,
         )
-        return Job(token, **response.json())
-
-    def wait_job_to_be_ready(self, job):
-        logger.info("Waiting for the job to be ready...")
-        while True:
-            try:
-                job = self.get_job_status(job)
-            except APIException:  # keep waiting if API raises exceptions
-                sleep(self._FILE_AVAILABILIY_SECONDS_DELAY)
-                continue
-            if not job.is_processing:
-                return job
-            sleep(self._FILE_AVAILABILIY_SECONDS_DELAY)
-
-    def save_job_to_file(self, job, filename):
-        with open(filename, 'wb') as output:
-            job = self.wait_job_to_be_ready(job)
-            logger.info(u"Writing to %s" % filename)
-
-            r = requests.get(job.url,
-                             headers=dict(API_KEY=self.api_key),
-                             stream=True,
-                             )
-
-            for chunk in r.iter_content(chunk_size=self._CHUNK_SIZE):
-                if chunk:
-                    output.write(chunk)
+        data = response.json()
+        return Results(data['records'],
+                       name='Ad-hoc JSON query')

ravenpackapi/examples/__init__.py

@@ -0,0 +1,29 @@
+from ravenpackapi import RPApi
+import logging
+
+logging.basicConfig(level=logging.DEBUG)
+# initialize the API (here we use the RP_API_KEY in os.environ)
+api = RPApi()
+
+# query the json endpoint for a dataset ***
+# use the public dataset `us30`
+ds = api.get_dataset(dataset_id='us500')
+# query the dataset analytics with the json endpoint
+data = ds.json(
+    start_date='2018-01-05 18:00:00',
+    end_date='2018-01-05 18:01:00',
+)
+
+for record in data:
+    print(record)
+
+# query the ad-hoc json endpoint ***
+adhoc_data = api.json(
+    start_date='2018-01-05 18:00:00',
+    end_date='2018-01-05 18:01:00',
+    fields=ds.fields,
+    filters=ds.filters,
+)
+print(adhoc_data)
+for record in adhoc_data:
+    print(record)

ravenpackapi/examples/query_with_json.py

@@ -1,5 +1,28 @@
-class APIException(Exception):
+from functools import wraps
+
 
-    def __init__(self, *args: object, response=None) -> None:
+class APIException(Exception):
+    def __init__(self, *args, **kwargs):
+        response = kwargs.pop('response', None)
         super(APIException, self).__init__(*args)
         self.response = response
+
+
+class MissingAPIException(Exception):
+    def __str__(self):
+        return "Define the api parameter with your own APIKEY " \
+               "to access the API methods from a model"
+
+
+class DataFileTimeout(Exception):
+    pass
+
+
+def api_method(func):
+    @wraps(func)
+    def decorated_func(instance, *args, **kwargs):
+        if not getattr(instance, 'api'):
+            raise MissingAPIException()
+        return func(instance, *args, **kwargs)
+
+    return decorated_func