Dropbox: large file support and a regression fix (#379)

* Fix .save method of the Dropbox backend (#378)

* Dropbox large file support (#301)

* Make linter happy

AUTHORS

@@ -31,6 +31,7 @@ By order of apparition, thanks:
     * Jody McIntyre (Google Cloud Storage native support)
     * Stanislav Kaledin (Bug fixes in SFTPStorage)
     * Filip Vavera (Google Cloud MIME types support)
+    * Max Malysh (Dropbox large file support)
 
 Extra thanks to Marty for adding this in Django,
 you can buy his very interesting book (Pro Django).

storages/backends/dropbox.py

@@ -21,6 +21,7 @@
 from django.utils.deconstruct import deconstructible
 from dropbox import Dropbox
 from dropbox.exceptions import ApiError
+from dropbox.files import CommitInfo, UploadSessionCursor
 
 from storages.utils import setting
 
@@ -50,6 +51,8 @@ def file(self):
 class DropBoxStorage(Storage):
     """DropBox Storage class for Django pluggable storage system."""
 
+    CHUNK_SIZE = 4 * 1024 * 1024
+
     def __init__(self, oauth2_access_token=None, root_path=None):
         oauth2_access_token = oauth2_access_token or setting('DROPBOX_OAUTH2_TOKEN')
         self.root_path = root_path or setting('DROPBOX_ROOT_PATH', '/')
@@ -108,5 +111,31 @@ def _open(self, name, mode='rb'):
         return remote_file
 
     def _save(self, name, content):
-        self.client.files_upload(content, self._full_path(name))
+        content.open()
+        if content.size <= self.CHUNK_SIZE:
+            self.client.files_upload(content.read(), self._full_path(name))
+        else:
+            self._chunked_upload(content, self._full_path(name))
+        content.close()
         return name
+
+    def _chunked_upload(self, content, dest_path):
+        upload_session = self.client.files_upload_session_start(
+            content.read(self.CHUNK_SIZE)
+        )
+        cursor = UploadSessionCursor(
+            session_id=upload_session.session_id,
+            offset=content.tell()
+        )
+        commit = CommitInfo(path=dest_path)
+
+        while content.tell() < content.size:
+            if (content.size - content.tell()) <= self.CHUNK_SIZE:
+                self.client.files_upload_session_finish(
+                    content.read(self.CHUNK_SIZE), cursor, commit
+                )
+            else:
+                self.client.files_upload_session_append_v2(
+                    content.read(self.CHUNK_SIZE), cursor
+                )
+                cursor.offset = content.tell()

tests/test_dropbox.py

@@ -5,6 +5,7 @@
 )
 from django.core.files.base import ContentFile, File
 from django.test import TestCase
+from django.utils.six import BytesIO
 
 from storages.backends import dropbox
 
@@ -118,8 +119,22 @@ def test_open(self, *args):
 
     @mock.patch('dropbox.Dropbox.files_upload',
                 return_value='foo')
-    def test_save(self, *args):
-        self.storage._save('foo', b'bar')
+    def test_save(self, files_upload, *args):
+        self.storage._save('foo', File(BytesIO(b'bar'), 'foo'))
+        self.assertTrue(files_upload.called)
+
+    @mock.patch('dropbox.Dropbox.files_upload')
+    @mock.patch('dropbox.Dropbox.files_upload_session_finish')
+    @mock.patch('dropbox.Dropbox.files_upload_session_append_v2')
+    @mock.patch('dropbox.Dropbox.files_upload_session_start',
+                return_value=mock.MagicMock(session_id='foo'))
+    def test_chunked_upload(self, start, append, finish, upload):
+        large_file = File(BytesIO(b'bar' * self.storage.CHUNK_SIZE), 'foo')
+        self.storage._save('foo', large_file)
+        self.assertTrue(start.called)
+        self.assertTrue(append.called)
+        self.assertTrue(finish.called)
+        self.assertFalse(upload.called)
 
     @mock.patch('dropbox.Dropbox.files_get_temporary_link',
                 return_value=FILE_MEDIA_FIXTURE)