v1.0.36 - Text-Analytics updates

Author: dariosky

CHANGELOG.md

@@ -1,7 +1,11 @@
 # Changelog
 
+## v1.0.36 (2020-05-14)
+* Text-Analytics API endpoint updated: folders & richer metadata
+* Extended error handling to support Feed disconnection problems
+
 ## v1.0.35 (2020-02-22)
-Initial support for the Upload-API endpoints  
+Initial support for the Text-Analytics API endpoints
 
 ## v1.0.34 (2019-11-11)
 Retrieve a lazy-loaded dataset when setting one of its paramters.

ravenpackapi/core.py

@@ -14,8 +14,8 @@
 from ravenpackapi.utils.constants import ENTITY_TYPES
 from ravenpackapi.utils.date_formats import as_datetime_str
 
-_VALID_METHODS = ('get', 'post', 'put', 'delete')
-VERSION = '1.0.35'
+_VALID_METHODS = ('get', 'post', 'put', 'delete', 'patch')
+VERSION = '1.0.36'
 
 logger = logging.getLogger("ravenpack.core")
 

ravenpackapi/examples/text_extraction.py

@@ -9,12 +9,13 @@
 
 # upload a file to access the analytics
 f = api.upload.file("_orig.doc",
-                    # properties={"primary_entity": "Ravenpack"}
+                    # properties={"primary_entity": "RavenPack"}
                     )
+f.wait_for_completion()
+
 # we can also get it if we know the id
 # f = api.upload.get('XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX')
 
-f.wait_for_completion()
 # get back the analytics found in the document
 f.save_analytics("_analytics.json")
 
@@ -25,7 +26,7 @@
 f.save_original("_orig.doc")
 
 # given a file we can set tags
-# f.set_tags(['file tag'])
+# f.set_metadata(tags=['file tag'])
 
 # ... or delete it
 # f.delete()

ravenpackapi/upload/models.py

@@ -1,31 +1,63 @@
-import json
 from time import sleep
 
 from ravenpackapi.exceptions import api_method
 
+FILE_FIELDS = (
+    'file_id', 'file_name', 'folder_id',
+    'status',
+    'upload_ts', 'raw_size', 'starred', 'trashed',
+    'tags',
+)
+
+FOLDER_FIELDS = (
+    'folder_id', 'parent_folder_id',
+    'folder_name',
+    'starred', 'trashed',
+)
+
 
 class File(object):
     """ A promise to get a file """
 
+    # check FILE_FIELD for the supported fields
     def __init__(self, file_id,
+                 file_name=None, folder_id=None,
                  status=None,
-                 name=None,
+                 upload_ts=None, raw_size=None, starred=None, trashed=None,
+                 tags=None,
                  api=None,
                  ):
+        self.api = api
         self.file_id = file_id
+        self.folder_id = folder_id
+        self.tags = tags or []
         self.status = status
-        self.name = name
-        self.api = api
+
+        self.upload_ts = upload_ts
+        self.starred = starred
+        self.trashed = trashed
+        self.raw_size = raw_size
+        self.file_name = file_name
 
     def __str__(self):
-        return "File: %(file_id)s - %(name)s - status: %(status)s" % self.__dict__
+        self.get_metadata()  # be sure to have the metadata (this is called only once per file)
+        return "File: %(file_id)s - %(file_name)s - status: %(status)s" % self.__dict__
 
     @api_method
     def get_status(self):
         response = self.api.request('%s/files/%s/status' % (self.api._UPLOAD_BASE_URL, self.file_id))
         self.status = response.json()['status']
         return self.status
 
+    @api_method
+    def get_metadata(self, force_refresh=False):
+        if self.file_name and not force_refresh:  # we already have the file metadata
+            return
+        response = self.api.request('%s/files/%s/metadata' % (self.api._UPLOAD_BASE_URL, self.file_id))
+        metadata = response.json()
+        for field in FILE_FIELDS:
+            setattr(self, field, metadata.get(field))
+
     @api_method
     def save_original(self, filename):
         response = self.api.request('%s/files/%s' % (self.api._UPLOAD_BASE_URL, self.file_id),
@@ -61,12 +93,64 @@ def delete(self):
         return response
 
     @api_method
-    def set_tags(self, tags):
-        self.api.request('%s/files/%s/tags' % (self.api._UPLOAD_BASE_URL, self.file_id),
-                         data=json.dumps(tags),
-                         method='put')
+    def set_metadata(self, file_name=None,
+                     folder_id=None,
+                     trashed=None, starred=None,
+                     tags=None
+                     ):
+        metadata = {k: v
+                    for k, v in dict(file_name=file_name, folder_id=folder_id,
+                                     trashed=trashed, starred=starred, tags=tags,
+                                     ).items()
+                    if v is not None}
+        self.api.request('%s/files/%s/metadata' % (self.api._UPLOAD_BASE_URL, self.file_id),
+                         json=metadata,
+                         method='patch')
 
     def wait_for_completion(self):
         while self.status not in {"COMPLETED", "DELETED"}:
             sleep(1)
             self.get_status()
+
+
+class Folder(object):
+    """ A Folder containing files """
+
+    def __init__(self, folder_id,
+                 folder_name=None,
+                 parent_folder_id=None,
+                 starred=None, trashed=None,
+                 api=None,
+                 ):
+        self.api = api
+        self.folder_id = folder_id
+        self.parent_folder_id = parent_folder_id
+        self.folder_name = folder_name
+
+        self.starred = starred
+        self.trashed = trashed
+
+    def __str__(self):
+        return "Folder: %(folder_id)s - %(folder_name)s" % self.__dict__
+
+    @api_method
+    def delete(self):
+        response = self.api.request('%s/folder/%s' % (self.api._UPLOAD_BASE_URL, self.folder_id),
+                                    method='delete')
+        return response
+
+    @api_method
+    def set_metadata(self,
+                     folder_name=None,
+                     parent_folder_id=None,
+                     trashed=None, starred=None,
+                     ):
+        metadata = {k: v
+                    for k, v in dict(folder_name=folder_name,
+                                     parent_folder_id=parent_folder_id,
+                                     trashed=trashed, starred=starred,
+                                     ).items()
+                    if v is not None}
+        self.api.request('%s/folders/%s' % (self.api._UPLOAD_BASE_URL, self.folder_id),
+                         json=metadata,
+                         method='patch')

ravenpackapi/upload/module.py

@@ -1,7 +1,7 @@
-import json
 import os
 
-from ravenpackapi.upload.models import File
+from ravenpackapi.upload.models import (File, FILE_FIELDS,
+                                        Folder, FOLDER_FIELDS)
 from ravenpackapi.utils.date_formats import as_datetime_str
 
 
@@ -15,6 +15,7 @@ def list(self,
              tags=None,
              status=None,
              filename=None,
+             page_size=50,
              ):
         params = dict(
             start_date=as_datetime_str(start_date),
@@ -25,23 +26,28 @@ def list(self,
         )
 
         # the list of files is splitted in pages - let's collect them
+        get_next, offset = False, 0
         while True:
             response = self.api.request('%s/files' % self.api._UPLOAD_BASE_URL,
                                         params={k: v
                                                 for k, v in params.items()
                                                 if v is not None})
             data = response.json()
             if 'results' in data:
-                for r in data['results']:
-                    yield File(r['file_id'],
-                               status=r.get('status'),
-                               name=r.get('name'),
-                               api=self.api,
-                               )
-            if data.get('next_page_key'):  # next page
-                params['next_page_key'] = json.dumps(data['next_page_key'])
-            else:
+                results = data['results']
+                for r in results:
+                    get_next = len(results) == page_size
+                    file_params = {
+                        field: r.get(field) for field in FILE_FIELDS
+                    }
+                    yield File(
+                        api=self.api,
+                        **file_params
+                    )
+            if not get_next:
                 break
+            offset += page_size
+            params['offset'] = offset
 
     def file(self, name_or_file_handler, properties=None):
         """ Upload a file - file can be either a filename or a file handler """
@@ -53,10 +59,10 @@ def file(self, name_or_file_handler, properties=None):
         else:
             filepath = name_or_file_handler.name
             fh = name_or_file_handler
-        filename = os.path.basename(filepath)
+        file_name = os.path.basename(filepath)
 
         params = dict(
-            filename=filename,
+            filename=file_name,
             properties=properties
         )
 
@@ -85,7 +91,19 @@ def file(self, name_or_file_handler, properties=None):
             fh.close()
         return File(file_id,
                     api=self.api,
-                    name=filename)
+                    file_name=file_name)
 
     def get(self, file_id):
         return File(file_id, api=self.api)
+
+    def list_folders(self):
+        response = self.api.request('%s/folders' % self.api._UPLOAD_BASE_URL)
+        data = response.json()
+        for r in data:
+            folder_params = {
+                field: r.get(field) for field in FOLDER_FIELDS
+            }
+            yield Folder(**folder_params)
+
+    def folder_get(self, folder_id):
+        return Folder(folder_id, api=self.api)

ravenpackapi/util.py

@@ -1,4 +1,5 @@
 import csv
+import logging
 import sys
 
 import six
@@ -11,6 +12,7 @@
 SPLIT_MONTHLY = 'monthly'
 SPLIT_WEEKLY = 'weekly'
 SPLIT_DAILY = 'daily'
+logger = logging.getLogger("ravenpack.util")
 
 
 def parts_to_curl(method, endpoint, headers, data=None):
@@ -37,10 +39,15 @@ def parts_to_curl(method, endpoint, headers, data=None):
 def to_curl(request):
     if not request:
         return 'No request'
+    try:
+        data = request.body.decode() if getattr(request, 'body') else None
+    except Exception as e:
+        logger.debug("Cannot convert data to curl: %s" % e)
+        data = "?"
     return parts_to_curl(request.method,
                          request.url,
                          request.headers,
-                         request.body if getattr(request, 'body') else None)
+                         data=data)
 
 
 def time_intervals(date_start, date_end, split=SPLIT_MONTHLY):

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-VERSION = '1.0.35'
+VERSION = '1.0.36'
 
 with open('README.rst') as readme_file:
     readme = readme_file.read()
@@ -24,7 +24,7 @@
         #   3 - Alpha
         #   4 - Beta
         #   5 - Production/Stable
-        'Development Status :: 5 - Production',
+        'Development Status :: 5 - Production/Stable',
 
         # Indicate who your project is intended for
         'Intended Audience :: Developers',