Merge remote-tracking branch 'upstream/master'

CHANGELOG.rst

@@ -1,6 +1,38 @@
 django-storages CHANGELOG
 =========================
 
+1.12.3 (2021-10-29)
+*******************
+
+General
+-------
+
+- Add support for Python 3.10 (`#1078`_)
+
+S3
+--
+
+- Re-raise non-404 errors in ``.exists()`` (`#1084`_, `#1085`_)
+
+Azure
+-----
+
+- Fix using ``AZURE_CUSTOM_DOMAIN`` with an account key credential (`#1082`_, `#1083`_)
+
+SFTP
+----
+
+- Catch ``FileNotFoundError`` instead of ``OSerror`` in ``.exists()`` to prevent swallowing ``socket.timeout`` exceptions (`#1064`_, `#1087`_)
+
+
+.. _#1078: https://github.com/jschneier/django-storages/pull/1078
+.. _#1084: https://github.com/jschneier/django-storages/issues/1084
+.. _#1085: https://github.com/jschneier/django-storages/pull/1085
+.. _#1082: https://github.com/jschneier/django-storages/issues/1082
+.. _#1083: https://github.com/jschneier/django-storages/pull/1083
+.. _#1064: https://github.com/jschneier/django-storages/issues/1064
+.. _#1087: https://github.com/jschneier/django-storages/pull/1087
+
 1.12.2 (2021-10-16)
 *******************
 
@@ -31,10 +63,11 @@ S3
 1.12 (2021-10-06)
 *****************
 
+General
+-------
 - Add support for Django 3.2 (`#1046`_, `#1042`_, `#1005`_)
 - Replace Travis CI with GitHub actions (`#1051`_)
 
-
 S3
 --
 

docs/backends/gcloud.rst

@@ -84,10 +84,12 @@ Your Google Cloud project ID. If unset, falls back to the default
 inferred from the environment.
 
 ``GS_IS_GZIPPED`` (optional: default is ``False``)
-    Whether or not to enable gzipping of content types specified by ``GZIP_CONTENT_TYPES``
+
+Whether or not to enable gzipping of content types specified by ``GZIP_CONTENT_TYPES``
 
 ``GZIP_CONTENT_TYPES`` (optional: default is ``text/css``, ``text/javascript``, ``application/javascript``, ``application/x-javascript``, ``image/svg+xml``)
-    When ``GS_IS_GZIPPED`` is set to ``True`` the content types which will be gzipped
+
+When ``GS_IS_GZIPPED`` is set to ``True`` the content types which will be gzipped
 
 .. _gs-creds:
 

storages/__init__.py

@@ -1 +1 @@
-__version__ = '1.12.2'
+__version__ = '1.12.3'

storages/backends/azure_storage.py

@@ -158,7 +158,10 @@ def _get_service_client(self):
 
         credential = None
         if self.account_key:
-            credential = self.account_key
+            credential = {
+                "account_name": self.account_name,
+                "account_key": self.account_key,
+            }
         elif self.sas_token:
             credential = self.sas_token
         elif self.token_credential:

storages/backends/s3boto3.py

@@ -465,18 +465,15 @@ def exists(self, name):
             self.connection.meta.client.head_object(Bucket=self.bucket_name, Key=name)
             return True
         except ClientError as error:
-            if error.response.get('Error', {}).get('Code') == '404':
+            if error.response['ResponseMetadata']['HTTPStatusCode'] == 404:
                 return False
 
-            # Some other error was encountered. As `get_available_name` calls this,
-            # we have to assume the filename is unavailable. If we return true due to some
-            # other error, we'd overwrite a file.
-            return True
+            # Some other error was encountered. Re-raise it.
+            raise
 
     def listdir(self, name):
         path = self._normalize_name(self._clean_name(name))
-        # The path needs to end with a slash, but if the root is empty, leave
-        # it.
+        # The path needs to end with a slash, but if the root is empty, leave it.
         if path and not path.endswith('/'):
             path += '/'
 

storages/backends/sftpstorage.py

@@ -150,7 +150,7 @@ def exists(self, name):
         try:
             self.sftp.stat(self._remote_path(name))
             return True
-        except OSError:
+        except FileNotFoundError:
             return False
 
     def _isdir_attr(self, item):

tests/test_azure.py

@@ -232,7 +232,7 @@ def test_container_client_params_account_key(self):
             self.assertEqual(storage.client, client_mock)
             bsc_mocked.assert_called_once_with(
                 'https://foo_domain',
-                credential='foo_key')
+                credential={'account_name': 'foo_name', 'account_key': 'foo_key'})
 
     def test_container_client_params_sas_token(self):
         storage = azure_storage.AzureStorage()

tests/test_s3boto3.py

@@ -477,15 +477,25 @@ def test_storage_exists(self):
 
     def test_storage_exists_false(self):
         self.storage.connection.meta.client.head_object.side_effect = ClientError(
-            {'Error': {'Code': '404', 'Message': 'Not Found'}},
+            {'Error': {}, 'ResponseMetadata': {'HTTPStatusCode': 404}},
             'HeadObject',
         )
-        self.assertFalse(self.storage.exists("file.txt"))
+        self.assertFalse(self.storage.exists('file.txt'))
         self.storage.connection.meta.client.head_object.assert_called_with(
             Bucket=self.storage.bucket_name,
             Key='file.txt',
         )
 
+    def test_storage_exists_other_error_reraise(self):
+        self.storage.connection.meta.client.head_object.side_effect = ClientError(
+            {'Error': {}, 'ResponseMetadata': {'HTTPStatusCode': 403}},
+            'HeadObject',
+        )
+        with self.assertRaises(ClientError) as cm:
+            self.storage.exists('file.txt')
+
+        self.assertEqual(cm.exception.response['ResponseMetadata']['HTTPStatusCode'], 403)
+
     def test_storage_delete(self):
         self.storage.delete("path/to/file.txt")
         self.storage.bucket.Object.assert_called_with('path/to/file.txt')

tests/test_sftp.py

@@ -1,5 +1,6 @@
 import io
 import os
+import socket
 import stat
 from datetime import datetime
 from unittest.mock import MagicMock, patch
@@ -56,7 +57,7 @@ def test_mkdir(self, mock_sftp):
         self.assertEqual(mock_sftp.mkdir.call_args[0], ('foo',))
 
     @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{
-        'stat.side_effect': (IOError(), True)
+        'stat.side_effect': (FileNotFoundError(), True)
     })
     def test_mkdir_parent(self, mock_sftp):
         self.storage._mkdir('bar/foo')
@@ -69,7 +70,7 @@ def test_save(self, mock_sftp):
         self.assertTrue(mock_sftp.open.return_value.write.called)
 
     @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{
-        'stat.side_effect': (IOError(), True)
+        'stat.side_effect': (FileNotFoundError(), True)
     })
     def test_save_in_subdir(self, mock_sftp):
         self.storage._save('bar/foo', File(io.BytesIO(b'foo'), 'foo'))
@@ -86,11 +87,18 @@ def test_exists(self, mock_sftp):
         self.assertTrue(self.storage.exists('foo'))
 
     @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{
-        'stat.side_effect': IOError()
+        'stat.side_effect': FileNotFoundError()
     })
     def test_not_exists(self, mock_sftp):
         self.assertFalse(self.storage.exists('foo'))
 
+    @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{
+        'stat.side_effect': socket.timeout()
+    })
+    def test_not_exists_timeout(self, mock_sftp):
+        with self.assertRaises(socket.timeout):
+            self.storage.exists('foo')
+
     @patch('storages.backends.sftpstorage.SFTPStorage.sftp', **{
         'listdir_attr.return_value':
             [MagicMock(filename='foo', st_mode=stat.S_IFDIR),