Download datafile chucks: more chunk options

SPLIT_WEEKLY and SPLIT_DAILY

Author: Dario Varotto

ravenpackapi/core.py

@@ -14,7 +14,7 @@
 from ravenpackapi.utils.constants import JSON_AVAILABLE_FIELDS
 
 _VALID_METHODS = ('get', 'post', 'put', 'delete')
-VERSION = '1.0.18'
+VERSION = '1.0.19'
 
 logger = logging.getLogger("ravenpack.core")
 

ravenpackapi/examples/get_datafile_chunked.py

@@ -8,22 +8,23 @@
 import os
 
 from ravenpackapi import RPApi
-from ravenpackapi.util import time_intervals, SPLIT_YEARLY
+from ravenpackapi.util import time_intervals, SPLIT_WEEKLY
 
 api = RPApi(api_key='YOUR_API_KEY')
 ds = api.get_dataset('YOUR_DATASET_ID')
 
-start_date = '2000-01-01'
+start_date = '2018-01-01'
 end_date = '2018-01-10'
 GET_COMPRESSED = True
 
 output_folder = './output'
 
 os.makedirs(output_folder, exist_ok=True)  # create folder for output
 for range_start, range_end in time_intervals(start_date, end_date,
-                                             split=SPLIT_YEARLY,
-                                             # another option is split=SPLIT_MONTHLY
-                                             # (that is the default)
+                                             split=SPLIT_WEEKLY,
+                                             # available splits:
+                                             # SPLIT_YEARLY, SPLIT_WEEKLY, SPLIT_DAILY
+                                             # or SPLIT_MONTHLY (the default)
                                              ):
     job = ds.request_datafile(
         start_date=range_start,
@@ -35,7 +36,7 @@
         continue
     filename = os.path.join(output_folder,
                             "datafile-{datestr}.{ext}".format(
-                                datestr=range_start.strftime('%Y'),
+                                datestr=range_start.strftime('%Y-%m-%d'),
                                 ext='zip' if GET_COMPRESSED else 'csv')
                             )
     print("Saving", range_start, "-", range_end, "=>", filename)

ravenpackapi/tests/test_date_intervals.py

@@ -1,6 +1,8 @@
 import datetime
 
-from ravenpackapi.util import time_intervals, SPLIT_YEARLY, SPLIT_MONTHLY
+import pytest
+
+from ravenpackapi.util import time_intervals, SPLIT_YEARLY, SPLIT_MONTHLY, SPLIT_WEEKLY, SPLIT_DAILY
 
 
 class TestTimeRanges(object):
@@ -66,12 +68,41 @@ def test_minutes_years(self):
             ('2016-01-01 00:00', '2016-01-10 11:30'),
         ]
 
-    def test_very_short_interval(self):
+    def test_weekly(self):
+        start = '2017-12-20 15:00'
+        end = '2018-01-08 11:30'
+        intervals = [tuple(map(lambda d: d.strftime("%Y-%m-%d %H:%M"), rng))
+                     for rng in time_intervals(start, end,
+                                               split=SPLIT_WEEKLY)]
+        assert intervals == [
+            ('2017-12-20 15:00', '2017-12-25 00:00'),
+            ('2017-12-25 00:00', '2018-01-01 00:00'),
+            ('2018-01-01 00:00', '2018-01-08 00:00'),
+            ('2018-01-08 00:00', '2018-01-08 11:30'),
+        ]
+
+    def test_daily(self):
+        start = '2017-12-29 15:00'
+        end = '2018-01-02 11:30'
+        intervals = [tuple(map(lambda d: d.strftime("%Y-%m-%d %H:%M"), rng))
+                     for rng in time_intervals(start, end,
+                                               split=SPLIT_DAILY)]
+        assert intervals == [
+            ('2017-12-29 15:00', '2017-12-30 00:00'),
+            ('2017-12-30 00:00', '2017-12-31 00:00'),
+            ('2017-12-31 00:00', '2018-01-01 00:00'),
+            ('2018-01-01 00:00', '2018-01-02 00:00'),
+            ('2018-01-02 00:00', '2018-01-02 11:30'),
+        ]
+
+    @pytest.mark.parametrize("split", [SPLIT_MONTHLY, SPLIT_YEARLY,
+                                       SPLIT_WEEKLY, SPLIT_DAILY])
+    def test_very_short_interval(self, split):
         start = '2004-02-29 15:00'
         end = '2004-02-29 16:00'
         intervals = [tuple(map(lambda d: d.strftime("%Y-%m-%d %H:%M"), rng))
                      for rng in time_intervals(start, end,
-                                               split=SPLIT_YEARLY)]
+                                               split=split)]
         assert intervals == [
             (start, end),
         ]

ravenpackapi/util.py

@@ -1,9 +1,12 @@
-import datetime
+from dateutil.relativedelta import relativedelta
+from dateutil.rrule import MO
 
 from ravenpackapi.utils.date_formats import as_datetime
 
 SPLIT_YEARLY = 'yearly'
 SPLIT_MONTHLY = 'monthly'
+SPLIT_WEEKLY = 'weekly'
+SPLIT_DAILY = 'daily'
 
 
 def parts_to_curl(method, endpoint, headers, data=None):
@@ -35,21 +38,43 @@ def to_curl(request):
 
 
 def time_intervals(date_start, date_end, split=SPLIT_MONTHLY):
-    assert split in (SPLIT_MONTHLY, SPLIT_YEARLY)
+    assert split in (
+        SPLIT_YEARLY,
+        SPLIT_MONTHLY,
+        SPLIT_WEEKLY,
+        SPLIT_DAILY)
     start = as_datetime(date_start)
     date_end = as_datetime(date_end)
 
     def get_end(get_next_end):
-        result = get_next_end
         if split == SPLIT_MONTHLY:
             # up to beginning of next month
-            result = result.replace(day=1) + datetime.timedelta(days=32)
-            return result.replace(day=1,
-                                  hour=0, minute=0, second=0, microsecond=0)
+            return get_next_end + \
+                   relativedelta(
+                       months=+1,
+                       day=1, hour=0, minute=0, second=0, microsecond=0
+                   )
         elif split == SPLIT_YEARLY:
             # up to beginning of next year
-            return result.replace(result.year + 1, month=1, day=1,
-                                  hour=0, minute=0, second=0, microsecond=0)
+            return get_next_end + \
+                   relativedelta(
+                       years=+1,
+                       month=1, day=1, hour=0, minute=0, second=0, microsecond=0
+                   )
+        elif split == SPLIT_WEEKLY:
+            # will break the time on weeks starting on Mondays
+            return get_next_end + \
+                   relativedelta(
+                       days=+1, weekday=MO,
+                       hour=0, minute=0, second=0, microsecond=0
+                   )
+        elif split == SPLIT_DAILY:
+            # will break the time on weeks starting on Mondays
+            return get_next_end + \
+                   relativedelta(
+                       days=+1,
+                       hour=0, minute=0, second=0, microsecond=0
+                   )
 
     while True:
         # some datetime trick to get the beginning of next month

requirements.txt

@@ -1,2 +1,3 @@
 requests[security]
 future
+python-dateutil

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '1.0.18'
+VERSION = '1.0.19'
 
 with open('README.rst') as readme_file:
     readme = readme_file.read()
@@ -35,5 +35,5 @@
     ],
 
     keywords='python analytics api rest news data',
-    install_requires=['requests[security]', 'future'],
+    install_requires=['requests[security]', 'future', 'python-dateutil'],
 )

tox.ini

@@ -6,6 +6,7 @@ deps =
 	pytest
 	pytest-xdist
 	future
+	python-dateutil
 	requests
 commands = pytest -n 4
 passenv = RP_API_KEY RP_API_ENDPOINT