Version 1.0.29: Explicit parameters and major rewrites

+ support for custom_fields and conditions

Author: dariosky

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## v1.0.29 (2019-05-21)
+**dataset creation explicit parameters**
+
+The Dataset parameters are not explictly passed in the constructor
+instead of being hidden in the kwargs.
+
+This allows also to clearly support custom_fields and conditions.
+
+A few new examples have been added or updated:
+[get historical flat files](ravenpackapi/examples/get_historical_flat_files.py) and
+[create a dataset with custom_fields and conditions](ravenpackapi/examples/indicator_datasets.py).
+
 ## v1.0.28 (2019-05-15)
 **dataset.count method**
 

conftest.py

@@ -0,0 +1,7 @@
+import logging
+import os
+
+# set the DEBUG environ variable to enable verbose logging
+if 'DEBUG' in os.environ:
+    logging.basicConfig()
+    logging.getLogger('ravenpack').setLevel(logging.DEBUG)

pytest.ini

@@ -11,3 +11,4 @@ markers=
 	slow: marks tests as slow (deselect with '-m "not slow"')
 	datafile: the async datafile requests
 	json: the sync json requests
+	datasets: CRUD operations over datasets

ravenpackapi/core.py

@@ -14,7 +14,7 @@
 from ravenpackapi.utils.constants import JSON_AVAILABLE_FIELDS, ENTITY_TYPES
 
 _VALID_METHODS = ('get', 'post', 'put', 'delete')
-VERSION = '1.0.28'
+VERSION = '1.0.29'
 
 logger = logging.getLogger("ravenpack.core")
 

ravenpackapi/examples/get_historical_flat_files.py

@@ -0,0 +1,50 @@
+# Download the historical compressed flat files (with all entities or just companies)
+# they are decompressed and combined in a single csv file per year
+
+import os
+import zipfile
+
+import requests
+
+from ravenpackapi import RPApi
+from ravenpackapi.util import parse_csv_line
+
+api_key = os.environ['RP_API_KEY']  # set your API KEY here
+api = RPApi(api_key)
+
+flat_type = 'companies'  # can be 'companies' or 'full'
+full_list_api_url = 'https://app.ravenpack.com/downloads/history-list/%s' % flat_type
+response = requests.get(
+    full_list_api_url,
+    params=dict(token=api_key)
+)
+response.raise_for_status()
+for flat_file in response.json():
+    local_filename = flat_file['name']
+    output_filename = '%s.combined.csv' % local_filename
+    if not os.path.isfile(output_filename):
+        with open(output_filename, 'wb') as output:
+            headers_written = False
+            with requests.get(
+                'https://app.ravenpack.com/history/getfile',
+                dict(token=api_key, id=flat_file['id'], type=flat_type),
+                stream=True,
+            ) as flatzip:
+                flatzip.raise_for_status()
+                if not os.path.isfile(local_filename):
+                    print("Downloading", local_filename, flat_file['size'])
+                    with open(local_filename, 'wb') as f:
+                        for chunk in flatzip.iter_content(chunk_size=8192):
+                            f.write(chunk)
+                with zipfile.ZipFile(local_filename) as zipped:
+                    for fileinfo in zipped.namelist():
+                        print(fileinfo)
+                        with zipped.open(fileinfo) as csv:
+                            header_line = next(csv)
+                            headers = parse_csv_line(header_line)
+                            if not headers_written:
+                                output.write(header_line)
+                                headers_written = True
+                            for line in csv:
+                                row = parse_csv_line(line)
+                                output.write(line)

ravenpackapi/examples/indicator_datasets.py

@@ -6,6 +6,7 @@
 # Begin creating a dataset with your desired filters (see the RPA user guide for syntax)
 # You can then add functions (https://app.ravenpack.com/api-documentation/#indicator-syntax)
 # Alternatively you can also create the dataset via the query builder and just use the dataset_uuid
+print("Creating a dataset with a few functions...")
 dataset = Dataset(api,
                   name='My Indicator dataset',
                   filters={"relevance": {"$gt": 90}},
@@ -22,17 +23,67 @@
 dataset.save()
 
 # you can also change the fields, (remember to save afterward)
+print("Updating fields...")
 dataset.fields = [
     {"avg": {"avg": {"field": "EVENT_SENTIMENT_SCORE", "lookback": 365}}},
 ]
 dataset.save()
 
 # Following this, you can then generate a datafile (for your desired date range)
+print("Requesting a datafile in the CSV format...")
 job = dataset.request_datafile(
     start_date='2018-04-10', end_date='2018-04-11',
     output_format='csv'
 )
 job.save_to_file('output.csv')  # This will poll until the file is ready for download
+print("Saved to output.csv")
 
 # a convenience function to delete all the dataset given a name
 # delete_all_datasets_by_name(api, 'My Indicator dataset')
+
+# here's an example of another dataset with custom_fields and conditions
+print("Creating a new dataset with functions and conditions...")
+dataset = api.create_dataset(Dataset.from_dict(
+    {
+        "name": "Dataset with functions and conditions",
+        "fields": [
+            "timestamp_utc",
+            "rp_entity_id",
+            "entity_name",
+            "AVG_REL"
+        ],
+        "filters": {
+        },
+        "custom_fields": [
+            {
+                "AVG_REL": {
+                    "avg": {
+                        "field": "RELEVANCE",
+                        "mode": "daily"
+                    }
+                }
+            }
+        ],
+        "conditions": {
+            "$and": [
+                {
+                    "AVG_REL": {
+                        "$gt": 30
+                    }
+                },
+                {
+                    "rp_entity_id": {
+                        "$in": [
+                            "ROLLUP"
+                        ]
+                    }
+                }
+            ]
+        },
+        "frequency": "daily",
+        "tags": []
+    }
+))
+
+dataset.save()
+print("Dataset created:", dataset.id)