[azure] Update to use azure-storage-blob>=12.0 (jschneier#805)

* Update azure storage version

* remove integration tests for azure

* remove extra azure filepath_to_uri call

* cleanup conf, settings

CHANGELOG.rst

@@ -273,9 +273,9 @@ Azure
   ``azure-storage-blob`` (`#680`_, `#684`_)
 - Fix compatability with ``generate_blob_shared_access_signature`` updated signature (`#705`_, `#723`_)
 - Fetching a file now uses the configured timeout rather than hardcoding one (`#727`_)
-- Add support for configuring all blobservice options: ``AZURE_EMULATED_MODE``, ``AZURE_ENDPOINT_SUFFIX``,
-  ``AZURE_CUSTOM_DOMAIN``, ``AZURE_CONNECTION_STRING``, ``AZURE_CUSTOM_CONNECTION_STRING``,
-  ``AZURE_TOKEN_CREDENTIAL``. See the docs for more info. Huge thanks once again to @nitely. (`#750`_)
+- Add support for configuring all blobservice options: ``AZURE_ENDPOINT_SUFFIX``,
+  ``AZURE_CUSTOM_DOMAIN``, ``AZURE_CONNECTION_STRING``, ``AZURE_TOKEN_CREDENTIAL``.
+  See the docs for more info. Huge thanks once again to @nitely. (`#750`_)
 - Fix filename handling to not strip special characters (`#609`_, `#752`_)
 
 

docs/backends/azure.rst

@@ -124,37 +124,17 @@ The following settings are available:
 
     Default location for the uploaded files. This is a path that gets prepended to every file name.
 
-``AZURE_EMULATED_MODE``
-
-    Whether to use the emulator (i.e Azurite). Defaults to False.
-
-``AZURE_ENDPOINT_SUFFIX``
-
-    The host base component of the url, minus the account name. Defaults
-    to Azure (``core.windows.net``). Override this to use the China cloud
-    (``core.chinacloudapi.cn``).
-
 ``AZURE_CUSTOM_DOMAIN``
 
     The custom domain to use. This can be set in the Azure Portal. For
     example, ``www.mydomain.com`` or ``mycdn.azureedge.net``.
 
-    It may contain a ``host:port`` when using the emulator
-    (``AZURE_EMULATED_MODE = True``).
-
 ``AZURE_CONNECTION_STRING``
 
     If specified, this will override all other parameters.
     See http://azure.microsoft.com/en-us/documentation/articles/storage-configure-connection-string/
     for the connection string format.
 
-``AZURE_CUSTOM_CONNECTION_STRING``
-
-    This is similar to ``AZURE_CONNECTION_STRING``, but it's used
-    when generating the file's URL. A custom domain or CDN may be
-    specified here instead of within ``AZURE_CONNECTION_STRING``.
-    Defaults to ``AZURE_CONNECTION_STRING``'s value.
-
 ``AZURE_TOKEN_CREDENTIAL``
 
     A token credential used to authenticate HTTPS requests. The token value

setup.cfg

@@ -38,7 +38,7 @@ packages =
 
 [options.extras_require]
 azure =
-    azure-storage-blob >=1.3.1,<12.0.0
+    azure-storage-blob >= 12.0.0
 boto3 =
     boto3 >= 1.4.4
 dropbox =

storages/backends/azure_storage.py

@@ -2,14 +2,16 @@
 from datetime import datetime, timedelta
 from tempfile import SpooledTemporaryFile
 
-from azure.common import AzureMissingResourceHttpError
-from azure.storage.blob import BlobPermissions, ContentSettings
-from azure.storage.blob.blockblobservice import BlockBlobService
+from azure.core.exceptions import ResourceNotFoundError
+from azure.storage.blob import (
+    BlobClient, BlobSasPermissions, ContainerClient, ContentSettings,
+    generate_blob_sas,
+)
 from django.core.exceptions import SuspiciousOperation
 from django.core.files.base import File
 from django.utils import timezone
 from django.utils.deconstruct import deconstructible
-from django.utils.encoding import filepath_to_uri, force_bytes
+from django.utils.encoding import force_bytes
 
 from storages.base import BaseStorage
 from storages.utils import (
@@ -40,12 +42,9 @@ def _get_file(self):
         if 'r' in self._mode or 'a' in self._mode:
             # I set max connection to 1 since spooledtempfile is
             # not seekable which is required if we use max_connections > 1
-            self._storage.service.get_blob_to_stream(
-                container_name=self._storage.azure_container,
-                blob_name=self._path,
-                stream=file,
-                max_connections=1,
-                timeout=self._storage.timeout)
+            download_stream = self._storage.client.download_blob(
+                self._path, timeout=self._storage.timeout)
+            download_stream.download_to_stream(file, max_concurrency=1)
         if 'r' in self._mode:
             file.seek(0)
 
@@ -122,8 +121,7 @@ def _get_valid_path(s):
 class AzureStorage(BaseStorage):
     def __init__(self, **settings):
         super().__init__(**settings)
-        self._service = None
-        self._custom_service = None
+        self._client = None
 
     def get_default_settings(self):
         return {
@@ -140,51 +138,41 @@ def get_default_settings(self):
             "location": setting('AZURE_LOCATION', ''),
             "default_content_type": 'application/octet-stream',
             "cache_control": setting("AZURE_CACHE_CONTROL"),
-            "is_emulated": setting('AZURE_EMULATED_MODE', False),
-            "endpoint_suffix": setting('AZURE_ENDPOINT_SUFFIX'),
             "sas_token": setting('AZURE_SAS_TOKEN'),
             "custom_domain": setting('AZURE_CUSTOM_DOMAIN'),
             "connection_string": setting('AZURE_CONNECTION_STRING'),
-            "custom_connection_string": setting(
-                'AZURE_CUSTOM_CONNECTION_STRING',
-                setting('AZURE_CONNECTION_STRING'),
-            ),
             "token_credential": setting('AZURE_TOKEN_CREDENTIAL'),
         }
 
-    def _blob_service(self, custom_domain=None, connection_string=None):
-        # This won't open a connection or anything,
-        # it's akin to a client
-        return BlockBlobService(
-            account_name=self.account_name,
-            account_key=self.account_key,
-            sas_token=self.sas_token,
-            is_emulated=self.is_emulated,
-            protocol=self.azure_protocol,
-            custom_domain=custom_domain,
-            connection_string=connection_string,
-            token_credential=self.token_credential,
-            endpoint_suffix=self.endpoint_suffix)
-
-    @property
-    def service(self):
-        if self._service is None:
-            custom_domain = None
-            if self.is_emulated:
-                custom_domain = self.custom_domain
-            self._service = self._blob_service(
-                custom_domain=custom_domain,
-                connection_string=self.connection_string)
-        return self._service
+    def _container_client(self, custom_domain=None, connection_string=None):
+        if custom_domain is None:
+            account_domain = "blob.core.windows.net"
+        else:
+            account_domain = custom_domain
+        if connection_string is None:
+            connection_string = "{}://{}.{}".format(
+                self.azure_protocol,
+                self.account_name,
+                account_domain)
+        credential = None
+        if self.account_key:
+            credential = self.account_key
+        elif self.sas_token:
+            credential = self.sas_token
+        elif self.token_credential:
+            credential = self.token_credential
+        return ContainerClient(
+            connection_string,
+            self.azure_container,
+            credential=credential)
 
     @property
-    def custom_service(self):
-        """This is used to generate the URL"""
-        if self._custom_service is None:
-            self._custom_service = self._blob_service(
+    def client(self):
+        if self._client is None:
+            self._client = self._container_client(
                 custom_domain=self.custom_domain,
-                connection_string=self.custom_connection_string)
-        return self._custom_service
+                connection_string=self.connection_string)
+        return self._client
 
     @property
     def azure_protocol(self):
@@ -219,26 +207,25 @@ def get_available_name(self, name, max_length=_AZURE_NAME_MAX_LEN):
         return super().get_available_name(name, max_length)
 
     def exists(self, name):
-        return self.service.exists(
-            self.azure_container,
-            self._get_valid_path(name),
-            timeout=self.timeout)
+        blob_client = self.client.get_blob_client(self._get_valid_path(name))
+        try:
+            blob_client.get_blob_properties()
+            return True
+        except ResourceNotFoundError:
+            return False
 
     def delete(self, name):
         try:
-            self.service.delete_blob(
-                container_name=self.azure_container,
-                blob_name=self._get_valid_path(name),
+            self.client.delete_blob(
+                self._get_valid_path(name),
                 timeout=self.timeout)
-        except AzureMissingResourceHttpError:
+        except ResourceNotFoundError:
             pass
 
     def size(self, name):
-        properties = self.service.get_blob_properties(
-            self.azure_container,
-            self._get_valid_path(name),
-            timeout=self.timeout).properties
-        return properties.content_length
+        blob_client = self.client.get_blob_client(self._get_valid_path(name))
+        properties = blob_client.get_blob_properties(timeout=self.timeout)
+        return properties.size
 
     def _save(self, name, content):
         cleaned_name = clean_name(name)
@@ -250,13 +237,13 @@ def _save(self, name, content):
             content = content.file
 
         content.seek(0)
-        self.service.create_blob_from_stream(
-            container_name=self.azure_container,
-            blob_name=name,
-            stream=content,
+        self.client.upload_blob(
+            name,
+            content,
             content_settings=ContentSettings(**params),
-            max_connections=self.upload_max_conn,
-            timeout=self.timeout)
+            max_concurrency=self.upload_max_conn,
+            timeout=self.timeout,
+            overwrite=self.overwrite_files)
         return cleaned_name
 
     def _expire_at(self, expire):
@@ -269,17 +256,19 @@ def url(self, name, expire=None):
         if expire is None:
             expire = self.expiration_secs
 
-        make_blob_url_kwargs = {}
+        credential = None
         if expire:
-            sas_token = self.custom_service.generate_blob_shared_access_signature(
-                self.azure_container, name, permission=BlobPermissions.READ, expiry=self._expire_at(expire))
-            make_blob_url_kwargs['sas_token'] = sas_token
+            sas_token = generate_blob_sas(
+                self.account_name,
+                self.azure_container,
+                name,
+                account_key=self.account_key,
+                permission=BlobSasPermissions(read=True),
+                expiry=self._expire_at(expire))
+            credential = sas_token
 
-        return self.custom_service.make_blob_url(
-            container_name=self.azure_container,
-            blob_name=filepath_to_uri(name),
-            protocol=self.azure_protocol,
-            **make_blob_url_kwargs)
+        container_blob_url = self.client.get_blob_client(name).url
+        return BlobClient.from_blob_url(container_blob_url, credential=credential).url
 
     def _get_content_settings_parameters(self, name, content=None):
         params = {}
@@ -311,10 +300,9 @@ def get_modified_time(self, name):
         Returns an (aware) datetime object containing the last modified time if
         USE_TZ is True, otherwise returns a naive datetime in the local timezone.
         """
-        properties = self.service.get_blob_properties(
-            self.azure_container,
+        properties = self.client.get_blob_properties(
             self._get_valid_path(name),
-            timeout=self.timeout).properties
+            timeout=self.timeout)
         if not setting('USE_TZ', False):
             return timezone.make_naive(properties.last_modified)
 
@@ -342,9 +330,8 @@ def list_all(self, path=''):
         # XXX make generator, add start, end
         return [
             blob.name
-            for blob in self.service.list_blobs(
-                self.azure_container,
-                prefix=path,
+            for blob in self.client.list_blobs(
+                name_starts_with=path,
                 timeout=self.timeout)]
 
     def listdir(self, path=''):