cleaned up multipart file streaming a lot, cleaner Client.add_* helper methods, and added compress option to file downloads.

Author: amstocker

ipfsApi/client.py

@@ -4,6 +4,7 @@
 from __future__ import absolute_import
 
 from . import http
+from . import multipart
 from . import utils
 from .commands import Command, ArgCommand, FileCommand, DownloadCommand
 
@@ -502,7 +503,14 @@ def add_str(self, string, **kwargs):
         >> ipfs_client.add_str('Mary had a little lamb')
         u'QmZfF6C9j4VtoCsTp4KSrhYH47QMd3DNXVZBKaxJdhaPab'
         """
-        return self.add(utils.make_string_buffer(string), **kwargs)
+        chunk_size = kwargs.pop('chunk_size',
+                                multipart.default_chunk_size)
+        body, headers = multipart.stream_text(string,
+                                              chunk_size=chunk_size)
+        return self._client.request('/add',
+                                    data=body,
+                                    headers=headers,
+                                    **kwargs)
 
     @utils.return_field('Hash')
     def add_json(self, json_obj, **kwargs):
@@ -512,7 +520,14 @@ def add_json(self, json_obj, **kwargs):
         >> ipfs_client.add_json({'one': 1, 'two': 2, 'three': 3})
         u'QmVz9g7m5u3oHiNKHj2CJX1dbG1gtismRS3g9NaPBBLbob'
         """
-        return self.add(utils.make_json_buffer(json_obj), **kwargs)
+        chunk_size = kwargs.pop('chunk_size',
+                                multipart.default_chunk_size)
+        body, headers = multipart.stream_text(utils.encode_json(json_obj),
+                                              chunk_size=chunk_size)
+        return self._client.request('/add',
+                                    data=body,
+                                    headers=headers,
+                                    **kwargs)
 
     def get_json(self, multihash, **kwargs):
         """
@@ -531,7 +546,14 @@ def add_pyobj(self, py_obj, **kwargs):
         >> c.add_pyobj([0, 1.0, 2j, '3', 4e5])
         u'QmdCWFLDXqgdWQY9kVubbEHBbkieKd3uo7MtCm7nTZZE9K'
         """
-        return self.add(utils.make_pyobj_buffer(py_obj), **kwargs)
+        chunk_size = kwargs.pop('chunk_size',
+                                multipart.default_chunk_size)
+        body, headers = multipart.stream_text(utils.encode_pyobj(py_obj),
+                                              chunk_size=chunk_size)
+        return self._client.request('/add',
+                                    data=body,
+                                    headers=headers,
+                                    **kwargs)
 
     def get_pyobj(self, multihash, **kwargs):
         """

ipfsApi/commands.py

@@ -3,6 +3,7 @@
 import os
 
 from . import multipart
+from .multipart import default_chunk_size
 from .exceptions import InvalidArguments, FileCommandException
 
 import six
@@ -37,6 +38,12 @@ def __init__(self, path, accept_multiple=True):
         self.accept_multiple = accept_multiple
 
     def request(self, client, f, **kwargs):
+        """
+        Takes either a file object, a filename, an iterable of filenames, an
+        iterable of file objects, or a homogenous iterable of file objects and
+        filenames.  Can only take one directory at a time, which will be
+        traversed (optionally recursive).
+        """
         if kwargs.pop('recursive', False):
             return self.directory(client, f, recursive=True, **kwargs)
         if isinstance(f, (list, tuple)):
@@ -46,25 +53,28 @@ def request(self, client, f, **kwargs):
         else:
             return self.single(client, f, **kwargs)
 
-    def single(self, client, _file, **kwargs):
+    def single(self, client, _file, chunk_size=default_chunk_size, **kwargs):
         """
         Adds a single file-like object to IPFS.
         """
-        body, headers = multipart.stream_file(_file)
+        body, headers = multipart.stream_file(_file,
+                                              chunk_size=chunk_size)
         return client.request(self.path, data=body, headers=headers, **kwargs)
 
-    def multiple(self, client, files, **kwargs):
+    def multiple(self, client, files, chunk_size=default_chunk_size, **kwargs):
         """
         Adds multiple file-like objects as a multipart request to IPFS.
         """
         if not self.accept_multiple:
             raise FileCommandException(
                 "[%s] does not accept multiple files." % self.path)
-        body, headers = multipart.stream_files(files)
+        body, headers = multipart.stream_files(files,
+                                               chunk_size=chunk_size)
         return client.request(self.path, data=body, headers=headers, **kwargs)
 
     def directory(self, client, dirname,
-                  fnpattern='*', recursive=False, **kwargs):
+                  fnpattern='*', recursive=False,
+                  chunk_size=default_chunk_size, **kwargs):
         """
         Loads a directory recursively into IPFS, files are matched against the
         given pattern.
@@ -74,7 +84,8 @@ def directory(self, client, dirname,
                 "[%s] does not accept multiple files." % self.path)
         body, headers = multipart.stream_directory(dirname,
                                                    fnpattern=fnpattern,
-                                                   recursive=recursive)
+                                                   recursive=recursive,
+                                                   chunk_size=chunk_size)
         return client.request(self.path, data=body, headers=headers, **kwargs)
 
 

ipfsApi/http.py

@@ -94,17 +94,20 @@ def request(self, path,
         return ret
 
     @pass_defaults
-    def download(self, path, filepath=None, args=[], opts={}, **kwargs):
+    def download(self, path, filepath=None,
+                 args=[], opts={},
+                 compress=True, **kwargs):
         """
         Downloads a file or files from IPFS into the current working
         directory, or the directory given by :filepath:.
         """
         url = self.base + path
+        wd = filepath or '.'
 
         params = []
         params.append(('stream-channels', 'true'))
-        params.append(('archive', 'true'))
-        params.append(('compress', 'true'))
+        if compress:
+            params.append(('compress', 'true'))
 
         for opt in opts.items():
             params.append(opt)
@@ -122,9 +125,10 @@ def download(self, path, filepath=None, args=[], opts={}, **kwargs):
 
         res.raise_for_status()
 
-        # try to stream download as a gzipped tar file stream
-        wd = filepath or '.'
-        with tarfile.open(fileobj=res.raw, mode='r|gz') as tf:
+        # try to stream download as a tar file stream
+        mode = 'r|gz' if compress else 'r|'
+
+        with tarfile.open(fileobj=res.raw, mode=mode) as tf:
             tf.extractall(path=wd)
 
     @contextlib.contextmanager