S3Boto3Storage: fix for file-like objects without name, multipart upload support

pytest.ini

@@ -0,0 +1,7 @@
+[pytest]
+django_find_project = false
+norecursedirs = .tox .git .idea docs storages
+python_files=test_*.py tests.py
+env =
+    DJANGO_SETTINGS_MODULE=tests.settings
+    PYTHONDONTWRITEBYTECODE=1

requirements-tests.txt

@@ -1,4 +1,5 @@
 Django>=1.7
+pytest-env
 pytest-cov==2.2.1
 boto>=2.32.0
 boto3>=1.2.3

storages/backends/s3boto3.py

@@ -15,6 +15,7 @@
 
 try:
     from boto3 import resource
+    from boto3.s3.transfer import TransferConfig
     from boto3 import __version__ as boto3_version
     from botocore.client import Config
     from botocore.exceptions import ClientError
@@ -206,6 +207,7 @@ class S3Boto3Storage(Storage):
     file_class = S3Boto3StorageFile
     # If config provided in init, signature_version and addressing_style settings/args are ignored.
     config = None
+    transfer_config = None
 
     # used for looking up the access and secret key from env vars
     access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID']
@@ -278,6 +280,9 @@ def __init__(self, acl=None, bucket=None, **settings):
             self.config = Config(s3={'addressing_style': self.addressing_style},
                                  signature_version=self.signature_version)
 
+        if not self.transfer_config:
+            self.transfer_config = TransferConfig()
+
     @property
     def connection(self):
         # TODO: Support host, port like in s3boto
@@ -447,6 +452,17 @@ def _save(self, name, content):
         if self.preload_metadata:
             self._entries[encoded_name] = obj
 
+        # If both `name` and `content.name` are empty or None, your request
+        # can be rejected with `XAmzContentSHA256Mismatch` error, because in
+        # `django.core.files.storage.Storage.save` method your file-like object
+        # will be wrapped in `django.core.files.File` if no `chunks` method
+        # provided. `File.__bool__`  method is Django-specific and depends on
+        # file name, for this reason`botocore.handlers.calculate_md5` can fail
+        # even if wrapped file-like object exists. To avoid Django-specific
+        # logic, pass internal file-like object if `content` is `File`
+        # class instance.
+        if type(content) is File:
+            content = content.file
         self._save_content(obj, content, parameters=parameters)
         # Note: In boto3, after a put, last_modified is automatically reloaded
         # the next time it is accessed; no need to specifically reload it.
@@ -462,7 +478,8 @@ def _save_content(self, obj, content, parameters):
         if self.default_acl:
             put_parameters['ACL'] = self.default_acl
         content.seek(0, os.SEEK_SET)
-        obj.put(Body=content, **put_parameters)
+        obj.upload_fileobj(content, ExtraArgs=put_parameters,
+                           Config=self.transfer_config)
 
     def delete(self, name):
         name = self._normalize_name(self._clean_name(name))

tests/test_s3boto3.py

@@ -1,12 +1,15 @@
 from datetime import datetime
 import gzip
+
+from io import BytesIO
+
 try:
     from unittest import mock
 except ImportError:  # Python 3.2 and below
     import mock
 
 from django.test import TestCase
-from django.core.files.base import ContentFile
+from django.core.files.base import ContentFile, File
 from django.utils.six.moves.urllib import parse as urlparse
 from django.utils.timezone import is_aware, utc
 
@@ -118,10 +121,52 @@ def test_storage_save(self):
         self.storage.bucket.Object.assert_called_once_with(name)
 
         obj = self.storage.bucket.Object.return_value
-        obj.put.assert_called_with(
-            Body=content,
-            ContentType='text/plain',
-            ACL=self.storage.default_acl,
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
+                "ContentType": 'text/plain',
+                "ACL": self.storage.default_acl
+            },
+            Config=self.storage.transfer_config
+        )
+
+    def test_storage_save_file_like_object(self):
+        """
+        Test saving a python file-like object
+        """
+        name = 'test_storage_save.txt'
+        content = BytesIO(b'new content')
+        self.storage.save(name, content)
+        self.storage.bucket.Object.assert_called_once_with(name)
+
+        obj = self.storage.bucket.Object.return_value
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
+                "ContentType": 'text/plain',
+                "ACL": self.storage.default_acl
+            },
+            Config=self.storage.transfer_config
+        )
+
+    def test_storage_save_file_like_object_wrapped(self):
+        """
+        Test saving a python file-like object manually wrapped in
+        Django's File object.
+        """
+        name = 'test_storage_save.txt'
+        content = BytesIO(b'new content')
+        self.storage.save(name, File(content))
+        self.storage.bucket.Object.assert_called_once_with(name)
+
+        obj = self.storage.bucket.Object.return_value
+        obj.upload_fileobj.assert_called_with(
+            content,
+            ExtraArgs={
+                "ContentType": 'text/plain',
+                "ACL": self.storage.default_acl
+            },
+            Config=self.storage.transfer_config
         )
 
     def test_storage_save_gzip(self):
@@ -133,13 +178,16 @@ def test_storage_save_gzip(self):
         content = ContentFile("I should be gzip'd")
         self.storage.save(name, content)
         obj = self.storage.bucket.Object.return_value
-        obj.put.assert_called_with(
-            Body=mock.ANY,
-            ContentType='text/css',
-            ContentEncoding='gzip',
-            ACL=self.storage.default_acl
+        obj.upload_fileobj.assert_called_with(
+            mock.ANY,
+            ExtraArgs={
+                "ContentType": 'text/css',
+                "ContentEncoding": 'gzip',
+                "ACL": self.storage.default_acl
+            },
+            Config=self.storage.transfer_config
         )
-        body = obj.put.call_args[1]['Body']
+        body = obj.upload_fileobj.call_args[0][0]
         zfile = gzip.GzipFile(mode='rb', fileobj=body)
         self.assertEquals(zfile.read(), b"I should be gzip'd")