Support passing boto3 config, stop using internal API for url

storages/backends/s3boto3.py

@@ -11,8 +11,8 @@
 try:
     from boto3 import resource
     from boto3 import __version__ as boto3_version
-    from botocore.exceptions import ClientError, UnknownClientMethodError
-    from botocore.awsrequest import prepare_request_dict
+    from botocore.client import Config
+    from botocore.exceptions import ClientError
 except ImportError:
     raise ImproperlyConfigured("Could not load Boto3's S3 bindings.\n"
                                "See https://github.com/boto/boto3")
@@ -202,6 +202,8 @@ class S3Boto3Storage(Storage):
     default_content_type = 'application/octet-stream'
     connection_response_error = ClientError
     file_class = S3Boto3StorageFile
+    # If config provided in init, signature_version and addressing_style settings/args are ignored.
+    config = None
 
     # used for looking up the access and secret key from env vars
     access_key_names = ['AWS_S3_ACCESS_KEY_ID', 'AWS_ACCESS_KEY_ID']
@@ -217,11 +219,12 @@ class S3Boto3Storage(Storage):
     bucket_acl = setting('AWS_BUCKET_ACL', default_acl)
     querystring_auth = setting('AWS_QUERYSTRING_AUTH', True)
     querystring_expire = setting('AWS_QUERYSTRING_EXPIRE', 3600)
+    signature_version = setting('AWS_S3_SIGNATURE_VERSION')
     reduced_redundancy = setting('AWS_REDUCED_REDUNDANCY', False)
     location = setting('AWS_LOCATION', '')
     encryption = setting('AWS_S3_ENCRYPTION', False)
     custom_domain = setting('AWS_S3_CUSTOM_DOMAIN')
-    # calling_format = setting('AWS_S3_CALLING_FORMAT', SubdomainCallingFormat())
+    addressing_style = setting('AWS_S3_ADDRESSING_STYLE')
     secure_urls = setting('AWS_S3_SECURE_URLS', True)
     file_name_charset = setting('AWS_S3_FILE_NAME_CHARSET', 'utf-8')
     gzip = setting('AWS_IS_GZIPPED', False)
@@ -268,17 +271,25 @@ def __init__(self, acl=None, bucket=None, **settings):
         if not self.access_key and not self.secret_key:
             self.access_key, self.secret_key = self._get_access_keys()
 
+        if not self.config:
+            self.config = Config(s3={'addressing_style': self.addressing_style},
+                                 signature_version=self.signature_version)
+
     @property
     def connection(self):
-        # TODO(mbarrien): Support alternate calling formats, host, port, proxy
+        # TODO(mbarrien): Support host, port
+        # Note that proxies are handled by environment variables that the underlying
+        # urllib/requests libraries read. See https://github.com/boto/boto3/issues/338
+        # and http://docs.python-requests.org/en/latest/user/advanced/#proxies
         if self._connection is None:
             self._connection = self.connection_class(
                 self.connection_service_name,
                 aws_access_key_id=self.access_key,
                 aws_secret_access_key=self.secret_key,
                 region_name=self.region_name,
                 use_ssl=self.use_ssl,
-                endpoint_url=self.endpoint_url
+                endpoint_url=self.endpoint_url,
+                config=self.config
             )
         return self._connection
 
@@ -501,57 +512,6 @@ def modified_time(self, name):
             entry = self.bucket.Object(self._encode_name(name))
         return entry.last_modified
 
-    # Have to access boto3's internals because this isn't exposed.
-    def _generate_unsigned_url(self, ClientMethod, Params=None, HttpMethod=None):
-        """Generate an unsigned url given a client, its method, and arguments
-
-        :type ClientMethod: string
-        :param ClientMethod: The client method to generate url for
-
-        :type Params: dict
-        :param Params: The parameters normally passed to
-            ``ClientMethod``.
-
-        :type HttpMethod: string
-        :param HttpMethod: The http method to use on the generated url. By
-            default, the http method is whatever is used in the method's model.
-
-        :returns: The presigned url
-        """
-        client = self.bucket.meta.client
-        client_method = ClientMethod
-        params = Params
-        http_method = HttpMethod
-
-        serializer = client._serializer
-
-        try:
-            operation_name = client._PY_TO_OP_NAME[client_method]
-        except KeyError:
-            raise UnknownClientMethodError(method_name=client_method)
-
-        operation_model = client.meta.service_model.operation_model(
-            operation_name)
-
-        # Create a request dict based on the params to serialize.
-        request_dict = serializer.serialize_to_request(
-            params, operation_model)
-
-        # Switch out the http method if user specified it.
-        if http_method is not None:
-            request_dict['method'] = http_method
-
-        # Prepare the request dict by including the client's endpoint url.
-        prepare_request_dict(
-            request_dict, endpoint_url=client.meta.endpoint_url)
-
-        # Generate the unsigned url.
-        # TODO(mbarrien): Actually pass operation_model? Only seems to be used for
-        # event signalling internally in boto3, which we may not want.
-        request = client._endpoint.create_request(
-            request_dict, operation_model=operation_model)
-        return request.url
-
     def url(self, name, parameters=None):
         # Preserve the trailing slash after normalizing the path.
         # TODO(mbarrien): Handle force_http=not self.secure_urls
@@ -562,11 +522,28 @@ def url(self, name, parameters=None):
         params = parameters.copy() if parameters else {}
         params['Bucket'] = self.bucket.name
         params['Key'] = self._encode_name(name)
+        url = self.bucket.meta.client.generate_presigned_url('get_object', Params=params,
+                                                             ExpiresIn=self.querystring_expire)
         if self.querystring_auth:
-            return self.bucket.meta.client.generate_presigned_url('get_object', Params=params,
-                                                                  ExpiresIn=self.querystring_expire)
-        else:
-            return self._generate_unsigned_url('get_object', Params=params)
+            return url
+
+        # Boto3 does not currently support generating URLs that are unsigned. Instead we
+        # take the signed URLs and strip any querystring params related to signing and expiration.
+        # Note that this may end up with URLs that are still invalid, especially if params are
+        # passed in that only work with signed URLs, e.g. response header params.
+        # The code attempts to strip all query parameters that match names of known parameters
+        # from v2 and v4 signatures, regardless of the actual signature version used.
+        split_url = urlparse.urlsplit(url)
+        qs = urlparse.parse_qsl(split_url.query, keep_blank_values=True)
+        blacklist = set(['x-amz-algorithm', 'x-amz-credential', 'x-amz-date',
+                         'x-amz-expires', 'x-amz-signedheaders', 'x-amz-signature',
+                         'awsaccesskeyid', 'expires', 'signature'])
+        filtered_qs = ((key, val) for key, val in qs if key.lower() not in blacklist)
+        # Note: Parameters that did not have a value in the original query string will have
+        # an '=' sign appended to it, e.g ?foo&bar becomes ?foo=&bar=
+        joined_qs = ('='.join(keyval) for keyval in filtered_qs)
+        split_url = split_url._replace(query="&".join(joined_qs))
+        return split_url.geturl()
 
     def get_available_name(self, name, max_length=None):
         """Overwrite existing file with the same name."""