Skip to content

Commit c7229f2

Browse files
authored
feat: add additional_blob_attributes to upload_many_from_filenames (#1162)
Fixes #996 🦕
1 parent c5a983d commit c7229f2

File tree

3 files changed

+68
-0
lines changed

3 files changed

+68
-0
lines changed

google/cloud/storage/transfer_manager.py

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -427,6 +427,8 @@ def upload_many_from_filenames(
427427
raise_exception=False,
428428
worker_type=PROCESS,
429429
max_workers=DEFAULT_MAX_WORKERS,
430+
*,
431+
additional_blob_attributes=None,
430432
):
431433
"""Upload many files concurrently by their filenames.
432434
@@ -557,6 +559,17 @@ def upload_many_from_filenames(
557559
and the default is a conservative number that should work okay in most
558560
cases without consuming excessive resources.
559561
562+
:type additional_blob_attributes: dict
563+
:param additional_blob_attributes:
564+
A dictionary of blob attribute names and values. This allows the
565+
configuration of blobs beyond what is possible with
566+
blob_constructor_kwargs. For instance, {"cache_control": "no-cache"}
567+
would set the cache_control attribute of each blob to "no-cache".
568+
569+
As with blob_constructor_kwargs, this affects the creation of every
570+
blob identically. To fine-tune each blob individually, use `upload_many`
571+
and create the blobs as desired before passing them in.
572+
560573
:raises: :exc:`concurrent.futures.TimeoutError` if deadline is exceeded.
561574
562575
:rtype: list
@@ -567,13 +580,17 @@ def upload_many_from_filenames(
567580
"""
568581
if blob_constructor_kwargs is None:
569582
blob_constructor_kwargs = {}
583+
if additional_blob_attributes is None:
584+
additional_blob_attributes = {}
570585

571586
file_blob_pairs = []
572587

573588
for filename in filenames:
574589
path = os.path.join(source_directory, filename)
575590
blob_name = blob_name_prefix + filename
576591
blob = bucket.blob(blob_name, **blob_constructor_kwargs)
592+
for prop, value in additional_blob_attributes.items():
593+
setattr(blob, prop, value)
577594
file_blob_pairs.append((path, blob))
578595

579596
return upload_many(

tests/system/test_transfer_manager.py

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -102,6 +102,25 @@ def test_upload_many_skip_if_exists(
102102
assert len(blobs_to_delete) == 1
103103

104104

105+
def test_upload_many_from_filenames_with_attributes(
106+
listable_bucket, listable_filenames, file_data, blobs_to_delete
107+
):
108+
SOURCE_DIRECTORY, FILENAME = os.path.split(file_data["logo"]["path"])
109+
110+
transfer_manager.upload_many_from_filenames(
111+
listable_bucket,
112+
[FILENAME],
113+
source_directory=SOURCE_DIRECTORY,
114+
additional_blob_attributes={"cache_control": "no-cache"},
115+
raise_exception=True,
116+
)
117+
118+
blob = listable_bucket.blob(FILENAME)
119+
blob.reload()
120+
blobs_to_delete.append(blob)
121+
assert blob.cache_control == "no-cache"
122+
123+
105124
def test_download_many(listable_bucket):
106125
blobs = list(listable_bucket.list_blobs())
107126
with tempfile.TemporaryDirectory() as tempdir:

tests/unit/test_transfer_manager.py

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -482,6 +482,38 @@ def test_upload_many_from_filenames_minimal_args():
482482
bucket.blob.assert_any_call(FILENAMES[1])
483483

484484

485+
def test_upload_many_from_filenames_additional_properties():
486+
bucket = mock.Mock()
487+
blob = mock.Mock()
488+
bucket_blob = mock.Mock(return_value=blob)
489+
blob.cache_control = None
490+
bucket.blob = bucket_blob
491+
492+
FILENAME = "file_a.txt"
493+
ADDITIONAL_BLOB_ATTRIBUTES = {"cache_control": "no-cache"}
494+
EXPECTED_FILE_BLOB_PAIRS = [(FILENAME, mock.ANY)]
495+
496+
with mock.patch(
497+
"google.cloud.storage.transfer_manager.upload_many"
498+
) as mock_upload_many:
499+
transfer_manager.upload_many_from_filenames(
500+
bucket, [FILENAME], additional_blob_attributes=ADDITIONAL_BLOB_ATTRIBUTES
501+
)
502+
503+
mock_upload_many.assert_called_once_with(
504+
EXPECTED_FILE_BLOB_PAIRS,
505+
skip_if_exists=False,
506+
upload_kwargs=None,
507+
deadline=None,
508+
raise_exception=False,
509+
worker_type=transfer_manager.PROCESS,
510+
max_workers=8,
511+
)
512+
513+
for attrib, value in ADDITIONAL_BLOB_ATTRIBUTES.items():
514+
assert getattr(blob, attrib) == value
515+
516+
485517
def test_download_many_to_path():
486518
bucket = mock.Mock()
487519

0 commit comments

Comments
 (0)