See scrapy#1778 - change default S3 ACL to "private" and allow customization via settings

Author: lopuhin

docs/topics/feed-exports.rst

@@ -185,6 +185,9 @@ passed through the following settings:
  * :setting:`AWS_ACCESS_KEY_ID`
  * :setting:`AWS_SECRET_ACCESS_KEY`
 
+Default access policy for uploaded files is ``private``, it can be changed
+(for example, to ``public-read``) via :setting:`S3_STORE_ACL`.
+
 .. _topics-feed-storage-stdout:
 
 Standard output

docs/topics/settings.rst

@@ -926,6 +926,15 @@ If enabled, Scrapy will respect robots.txt policies. For more information see
     this option is enabled by default in settings.py file generated
     by ``scrapy startproject`` command.
 
+.. setting:: S3_STORE_ACL
+
+S3_STORE_ACL
+------------
+
+Default: ``'private'``
+
+S3-specific access control policy (ACL) for uploaded files.
+
 .. setting:: SCHEDULER
 
 SCHEDULER

scrapy/pipelines/files.py

@@ -82,7 +82,8 @@ class S3FilesStore(object):
     AWS_ACCESS_KEY_ID = None
     AWS_SECRET_ACCESS_KEY = None
 
-    POLICY = 'public-read'
+    POLICY = 'private'  # Overriden from settings.S3_STORE_ACL in
+                        # FilesPipeline.from_settings.
     HEADERS = {
         'Cache-Control': 'max-age=172800',
     }
@@ -232,6 +233,7 @@ def from_settings(cls, settings):
         s3store = cls.STORE_SCHEMES['s3']
         s3store.AWS_ACCESS_KEY_ID = settings['AWS_ACCESS_KEY_ID']
         s3store.AWS_SECRET_ACCESS_KEY = settings['AWS_SECRET_ACCESS_KEY']
+        s3store.POLICY = settings['S3_STORE_ACL']
 
         cls.FILES_URLS_FIELD = settings.get('FILES_URLS_FIELD', cls.DEFAULT_FILES_URLS_FIELD)
         cls.FILES_RESULT_FIELD = settings.get('FILES_RESULT_FIELD', cls.DEFAULT_FILES_RESULT_FIELD)

scrapy/settings/default_settings.py

@@ -231,6 +231,8 @@
 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
 
+S3_STORE_ACL = 'private'
+
 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
 
 SPIDER_MIDDLEWARES = {}