Merge pull request #3032 from WikiWatershed/tt/hydroshare-export-bulk…

…-upload

HydroShare Bulk Upload

Connects #3016

src/mmw/apps/export/hydroshare.py

@@ -4,8 +4,9 @@
 from __future__ import division
 
 import json
-import StringIO
 
+from cStringIO import StringIO
+from zipfile import ZipFile
 from rauth import OAuth2Service
 from urlparse import urljoin, urlparse
 from hs_restclient import HydroShare, HydroShareAuthOAuth2, HydroShareNotFound
@@ -81,40 +82,34 @@ class HydroShareClient(HydroShare):
     Helper class for utility methods for HydroShare
     """
 
-    def add_files(self, resource_id, files, overwrite=False):
+    def add_files(self, resource_id, files):
         """
         Helper method that will add an array of files to a resource.
 
         :param resource_id: ID of the resource to add files to
         :param files: List of dicts in the format
-                      {'name': 'String', 'contents': 'String', 'object': False}
-                      or
-                      {'name': 'String', 'contents': file_like_object, 'object': True}  # NOQA
-        :param overwrite: Whether to overwrite files or not. False by default.
+                      {'name': 'String', 'contents': 'String'}
         """
-
-        for f in files:
-            fobject = f.get('object', False)
-            fcontents = f.get('contents')
-            fname = f.get('name')
-            if fcontents and fname:
-                # Overwrite files if specified
-                if overwrite:
-                    try:
-                        # Delete the resource file if it already exists
-                        self.deleteResourceFile(resource_id, fname)
-                    except HydroShareNotFound:
-                        # File didn't already exists, move on
-                        pass
-
-                if fobject:
-                    fio = fcontents
-                else:
-                    fio = StringIO.StringIO()
-                    fio.write(fcontents)
-
-                # Add the new file
-                self.addResourceFile(resource_id, fio, fname)
+        zippath = resource_id + '.zip'
+        stream = StringIO()
+
+        # Zip all given files into the stream
+        with ZipFile(stream, 'w') as zf:
+            for f in files:
+                fcontents = f.get('contents')
+                fname = f.get('name')
+                if fname and fcontents:
+                    zf.writestr(fname, fcontents.encode('utf-8'))
+
+        # Send zip file to HydroShare
+        self.addResourceFile(resource_id, stream, zippath)
+
+        # Unzip file in HydroShare and replace any existing files
+        self.resource(resource_id).functions.unzip({
+            'zip_with_rel_path': zippath,
+            'remove_original_zip': True,
+            'overwrite': True,
+        })
 
     def check_resource_exists(self, resource_id):
         try:
@@ -133,7 +128,7 @@ def get_project_snapshot(self, resource_id):
         snapshot_path = 'mmw_project_snapshot.json'
         try:
             stream = self.getResourceFile(resource_id, snapshot_path)
-            fio = StringIO.StringIO()
+            fio = StringIO()
             for chunk in stream:
                 fio.write(chunk)
 

src/mmw/apps/export/tasks.py

@@ -55,8 +55,7 @@ def update_resource(user_id, project_id, params):
                 mdata = json.loads(job.result)
                 files.append({
                     'name': md.get('name'),
-                    'contents': to_gms_file(mdata),
-                    'object': True,
+                    'contents': to_gms_file(mdata).read(),
                 })
             except (Job.DoesNotExist, ValueError):
                 # Either the job wasn't found, or its content isn't JSON
@@ -65,7 +64,7 @@ def update_resource(user_id, project_id, params):
     # Except the existing analyze files
     files = [f for f in files if f['name'] not in current_analyze_files]
 
-    hs.add_files(hsresource.resource, files, overwrite=True)
+    hs.add_files(hsresource.resource, files)
 
     hsresource.exported_at = now()
     hsresource.save()
@@ -111,8 +110,7 @@ def create_resource(user_id, project_id, params):
                 mdata = json.loads(job.result)
                 files.append({
                     'name': md.get('name'),
-                    'contents': to_gms_file(mdata),
-                    'object': True,
+                    'contents': to_gms_file(mdata).read(),
                 })
             except (Job.DoesNotExist, ValueError):
                 # Either the job wasn't found, or its content isn't JSON