Added default location configuration

This is useful for when the S3 Cloud provider
does not return location header and is used as a
fallback by thumbor-aws

README.md

@@ -62,6 +62,17 @@ As usual for thumbor, you don't need to use both at the same time. Feel free to
 
 thumbor-aws allows you to configure each storage independently, so there are configuration keys for each.
 
+#### General
+
+Some S3 providers fail to return a valid location header when uploading a new object. For that scenario, `thumbor-aws` allows users to set the location template to be used.
+
+```
+## Default location to use if S3 does not return location header.
+## Can use {bucket_name} var.
+## Defaults to: 'https://{bucket_name}.s3.amazonaws.com'
+AWS_DEFAULT_LOCATION = "https://{bucket_name}.s3.amazonaws.com"
+```
+
 #### Storage
 
 Below you can see the result of running thumbor's config generation after importing thumbor-aws:

tests/test_storage.py

@@ -8,6 +8,7 @@
 # http://www.opensource.org/licenses/mit-license
 # Copyright (c) 2021 Bernardo Heynemann heynemann@gmail.com
 
+from unittest.mock import patch
 from uuid import uuid4
 
 import pytest
@@ -105,15 +106,35 @@ async def test_can_handle_expired_data(self):
         expect(status).to_equal(410)
         expect(data).to_equal(b"")
 
+    @gen_test
+    async def test_can_upload_with_valid_location(self):
+        """Verifies that uploading with valid location returns location"""
+        storage = await self.ensure_bucket()
+        filepath = f"/test/can_put_file_{uuid4()}"
+
+        with patch.object(storage.__class__, "get_location", return_value=None):
+            response = await storage.upload(
+                storage.normalize_path(filepath),
+                b"ACME-SEC2",
+                "application/text",
+                "https://my-site.com/{bucket_name}",
+            )
+
+            expect(response).to_equal(
+                f"https://my-site.com/{self.bucket_name}/st{filepath}"
+            )
+
     @gen_test
     async def test_can_get_crypto_from_s3(self):
         """Verifies that security information can be loaded from S3 using Storage"""
         storage = await self.ensure_bucket()
         filepath = f"/test/can_put_file_{uuid4()}"
+
         await storage.upload(
             storage.normalize_path(filepath + ".txt"),
             b"ACME-SEC2",
             "application/text",
+            "http://my-site.com",
         )
 
         data = await storage.get_crypto(filepath)
@@ -129,6 +150,7 @@ async def test_can_get_detector_data_from_s3(self):
             storage.normalize_path(filepath + ".detectors.txt"),
             b'{"some": "data"}',
             "application/text",
+            "",
         )
 
         data = await storage.get_detector_data(filepath)

thumbor.conf

@@ -749,6 +749,11 @@ RESULT_STORAGE_STORES_UNSAFE = True
 
 ################################# AWS Storage ##################################
 
+## Default location to use if S3 does not return location header.
+## Can use {bucket_name} var.
+## Defaults to: 'https://{bucket_name}.s3.amazonaws.com'
+AWS_DEFAULT_LOCATION = "https://{bucket_name}.s3.amazonaws.com"
+
 ## Region where thumbor's objects are going to be stored.
 ## Defaults to: 'us-east-1'
 AWS_STORAGE_REGION_NAME = 'local'

thumbor_aws/result_storage.py

@@ -13,10 +13,10 @@
 from urllib.parse import unquote
 
 from deprecated import deprecated
-
 from thumbor.engines import BaseEngine
 from thumbor.result_storages import BaseStorage, ResultStorageResult
 from thumbor.utils import logger
+
 from thumbor_aws.s3_client import S3Client
 
 
@@ -54,7 +54,12 @@ async def put(self, image_bytes: bytes) -> str:
         file_abspath = self.normalize_path(self.context.request.url)
         logger.debug("[RESULT_STORAGE] putting at %s", file_abspath)
         content_type = BaseEngine.get_mimetype(image_bytes)
-        response = await self.upload(file_abspath, image_bytes, content_type)
+        response = await self.upload(
+            file_abspath,
+            image_bytes,
+            content_type,
+            self.context.config.AWS_DEFAULT_LOCATION,
+        )
         logger.info("[RESULT_STORAGE] Image uploaded successfully to %s", file_abspath)
         return response
 

thumbor_aws/s3_client.py

@@ -13,11 +13,20 @@
 
 from aiobotocore.client import AioBaseClient
 from aiobotocore.session import AioSession, get_session
-
 from thumbor.config import Config, config
 from thumbor.context import Context
 from thumbor.utils import logger
 
+Config.define(
+    "AWS_DEFAULT_LOCATION",
+    "https://{bucket_name}.s3.amazonaws.com",
+    (
+        "Default location to use if S3 does not return location header."
+        " Can use {bucket_name} var."
+    ),
+    "AWS Storage",
+)
+
 Config.define(
     "AWS_STORAGE_REGION_NAME",
     "us-east-1",
@@ -173,6 +182,7 @@ async def upload(
         path: str,
         data: bytes,
         content_type,
+        default_location,
     ) -> str:
         """Uploads a File to S3"""
 
@@ -198,14 +208,15 @@ async def upload(
                 msg = f"Unable to upload image to {path}: Status Code {status_code}"
                 logger.error(msg)
                 raise RuntimeError(msg)
+
             location = self.get_location(response)
             if location is None:
                 msg = (
                     f"Unable to process response from AWS to {path}: "
                     "Location Headers was not found in response"
                 )
-                logger.error(msg)
-                raise RuntimeError(msg)
+                logger.warning(msg)
+                location = default_location.format(bucket_name=self.bucket_name)
 
             return f"{location.rstrip('/')}/{path.lstrip('/')}"
 

thumbor_aws/storage.py

@@ -15,6 +15,7 @@
 from thumbor import storages
 from thumbor.engines import BaseEngine
 from thumbor.utils import logger
+
 from thumbor_aws.s3_client import S3Client
 
 
@@ -28,7 +29,12 @@ async def put(self, path: str, file_bytes: bytes) -> str:
         content_type = BaseEngine.get_mimetype(file_bytes)
         normalized_path = self.normalize_path(path)
         logger.debug("[STORAGE] putting at %s", normalized_path)
-        path = await self.upload(normalized_path, file_bytes, content_type)
+        path = await self.upload(
+            normalized_path,
+            file_bytes,
+            content_type,
+            self.context.config.AWS_DEFAULT_LOCATION,
+        )
         return path
 
     async def put_crypto(self, path: str) -> str:
@@ -44,7 +50,12 @@ async def put_crypto(self, path: str) -> str:
         normalized_path = self.normalize_path(path)
         crypto_path = f"{normalized_path}.txt"
         key = self.context.server.security_key.encode()
-        s3_path = await self.upload(crypto_path, key, "application/text")
+        s3_path = await self.upload(
+            crypto_path,
+            key,
+            "application/text",
+            self.context.config.AWS_DEFAULT_LOCATION,
+        )
 
         logger.debug("Stored crypto at %s", crypto_path)
 
@@ -54,7 +65,12 @@ async def put_detector_data(self, path: str, data: Any) -> str:
         normalized_path = self.normalize_path(path)
         filepath = f"{normalized_path}.detectors.txt"
         details = dumps(data)
-        return await self.upload(filepath, details, "application/json")
+        return await self.upload(
+            filepath,
+            details,
+            "application/json",
+            self.context.config.AWS_DEFAULT_LOCATION,
+        )
 
     async def get(self, path: str) -> bytes:
         normalized_path = self.normalize_path(path)