From 5e60991c634bf5f5f3ddd78a6bd192f03b858a15 Mon Sep 17 00:00:00 2001 From: Gurov Ilya Date: Mon, 15 Jun 2020 23:44:09 +0300 Subject: [PATCH 1/2] feat: add if*generation*match args into Bucket.delete_blobs() (#130) Towards #127 --- google/cloud/storage/bucket.py | 93 +++++++++++++++++++++++++++++++++- tests/unit/test_bucket.py | 76 +++++++++++++++++++++++++++ 2 files changed, 167 insertions(+), 2 deletions(-) diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 228c0e2aa..2e88f5a84 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -1455,7 +1455,17 @@ def delete_blob( timeout=timeout, ) - def delete_blobs(self, blobs, on_error=None, client=None, timeout=_DEFAULT_TIMEOUT): + def delete_blobs( + self, + blobs, + on_error=None, + client=None, + timeout=_DEFAULT_TIMEOUT, + if_generation_match=None, + if_generation_not_match=None, + if_metageneration_match=None, + if_metageneration_not_match=None, + ): """Deletes a list of blobs from the current bucket. Uses :meth:`delete_blob` to delete each individual blob. @@ -1484,15 +1494,74 @@ def delete_blobs(self, blobs, on_error=None, client=None, timeout=_DEFAULT_TIMEO Can also be passed as a tuple (connect_timeout, read_timeout). See :meth:`requests.Session.request` documentation for details. + :type if_generation_match: list of long + :param if_generation_match: (Optional) Make the operation conditional on whether + the blob's current generation matches the given value. + Setting to 0 makes the operation succeed only if there + are no live versions of the blob. The list must match + ``blobs`` item-to-item. + + :type if_generation_not_match: list of long + :param if_generation_not_match: (Optional) Make the operation conditional on whether + the blob's current generation does not match the given + value. If no live blob exists, the precondition fails. + Setting to 0 makes the operation succeed only if there + is a live version of the blob. The list must match + ``blobs`` item-to-item. + + :type if_metageneration_match: list of long + :param if_metageneration_match: (Optional) Make the operation conditional on whether the + blob's current metageneration matches the given value. + The list must match ``blobs`` item-to-item. + + :type if_metageneration_not_match: list of long + :param if_metageneration_not_match: (Optional) Make the operation conditional on whether the + blob's current metageneration does not match the given value. + The list must match ``blobs`` item-to-item. + :raises: :class:`~google.cloud.exceptions.NotFound` (if `on_error` is not passed). + + Example: + Delete blobs using generation match preconditions. + + >>> from google.cloud import storage + + >>> client = storage.Client() + >>> bucket = client.bucket("bucket-name") + + >>> blobs = [bucket.blob("blob-name-1"), bucket.blob("blob-name-2")] + >>> if_generation_match = [None] * len(blobs) + >>> if_generation_match[0] = "123" # precondition for "blob-name-1" + + >>> bucket.delete_blobs(blobs, if_generation_match=if_generation_match) """ + _raise_if_len_differs( + len(blobs), + if_generation_match=if_generation_match, + if_generation_not_match=if_generation_not_match, + if_metageneration_match=if_metageneration_match, + if_metageneration_not_match=if_metageneration_not_match, + ) + if_generation_match = iter(if_generation_match or []) + if_generation_not_match = iter(if_generation_not_match or []) + if_metageneration_match = iter(if_metageneration_match or []) + if_metageneration_not_match = iter(if_metageneration_not_match or []) + for blob in blobs: try: blob_name = blob if not isinstance(blob_name, six.string_types): blob_name = blob.name - self.delete_blob(blob_name, client=client, timeout=timeout) + self.delete_blob( + blob_name, + client=client, + timeout=timeout, + if_generation_match=next(if_generation_match, None), + if_generation_not_match=next(if_generation_not_match, None), + if_metageneration_match=next(if_metageneration_match, None), + if_metageneration_not_match=next(if_metageneration_not_match, None), + ) except NotFound: if on_error is not None: on_error(blob) @@ -2980,3 +3049,23 @@ def generate_signed_url( headers=headers, query_parameters=query_parameters, ) + + +def _raise_if_len_differs(expected_len, **generation_match_args): + """ + Raise an error if any generation match argument + is set and its len differs from the given value. + + :type expected_len: int + :param expected_len: Expected argument length in case it's set. + + :type generation_match_args: dict + :param generation_match_args: Lists, which length must be checked. + + :raises: :exc:`ValueError` if any argument set, but has an unexpected length. + """ + for name, value in generation_match_args.items(): + if value is not None and len(value) != expected_len: + raise ValueError( + "'{}' length must be the same as 'blobs' length".format(name) + ) diff --git a/tests/unit/test_bucket.py b/tests/unit/test_bucket.py index 27bd94f1a..3c5f2e68d 100644 --- a/tests/unit/test_bucket.py +++ b/tests/unit/test_bucket.py @@ -1146,6 +1146,82 @@ def test_delete_blobs_hit_w_user_project(self): self.assertEqual(kw[0]["query_params"], {"userProject": USER_PROJECT}) self.assertEqual(kw[0]["timeout"], 42) + def test_delete_blobs_w_generation_match(self): + NAME = "name" + BLOB_NAME = "blob-name" + BLOB_NAME2 = "blob-name2" + GENERATION_NUMBER = 6 + GENERATION_NUMBER2 = 9 + + connection = _Connection({}, {}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + bucket.delete_blobs( + [BLOB_NAME, BLOB_NAME2], + timeout=42, + if_generation_match=[GENERATION_NUMBER, GENERATION_NUMBER2], + ) + kw = connection._requested + self.assertEqual(len(kw), 2) + + self.assertEqual(kw[0]["method"], "DELETE") + self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw[0]["timeout"], 42) + self.assertEqual( + kw[0]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER} + ) + self.assertEqual(kw[1]["method"], "DELETE") + self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME2)) + self.assertEqual(kw[1]["timeout"], 42) + self.assertEqual( + kw[1]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER2} + ) + + def test_delete_blobs_w_generation_match_wrong_len(self): + NAME = "name" + BLOB_NAME = "blob-name" + BLOB_NAME2 = "blob-name2" + GENERATION_NUMBER = 6 + + connection = _Connection() + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + with self.assertRaises(ValueError): + bucket.delete_blobs( + [BLOB_NAME, BLOB_NAME2], + timeout=42, + if_generation_not_match=[GENERATION_NUMBER], + ) + + def test_delete_blobs_w_generation_match_none(self): + NAME = "name" + BLOB_NAME = "blob-name" + BLOB_NAME2 = "blob-name2" + GENERATION_NUMBER = 6 + GENERATION_NUMBER2 = None + + connection = _Connection({}, {}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + bucket.delete_blobs( + [BLOB_NAME, BLOB_NAME2], + timeout=42, + if_generation_match=[GENERATION_NUMBER, GENERATION_NUMBER2], + ) + kw = connection._requested + self.assertEqual(len(kw), 2) + + self.assertEqual(kw[0]["method"], "DELETE") + self.assertEqual(kw[0]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME)) + self.assertEqual(kw[0]["timeout"], 42) + self.assertEqual( + kw[0]["query_params"], {"ifGenerationMatch": GENERATION_NUMBER} + ) + self.assertEqual(kw[1]["method"], "DELETE") + self.assertEqual(kw[1]["path"], "/b/%s/o/%s" % (NAME, BLOB_NAME2)) + self.assertEqual(kw[1]["timeout"], 42) + self.assertEqual(kw[1]["query_params"], {}) + def test_delete_blobs_miss_no_on_error(self): from google.cloud.exceptions import NotFound From 62d1543e18040b286b23464562aa6eb998074c54 Mon Sep 17 00:00:00 2001 From: HemangChothani <50404902+HemangChothani@users.noreply.github.com> Date: Tue, 16 Jun 2020 04:42:10 +0530 Subject: [PATCH 2/2] docs(storage): fix indent in code blocks (#171) Fixes #170 --- google/cloud/storage/__init__.py | 1 + google/cloud/storage/acl.py | 5 +++++ google/cloud/storage/bucket.py | 8 ++++++++ google/cloud/storage/client.py | 4 ++++ 4 files changed, 18 insertions(+) diff --git a/google/cloud/storage/__init__.py b/google/cloud/storage/__init__.py index 2b643fc80..2a9629dfb 100644 --- a/google/cloud/storage/__init__.py +++ b/google/cloud/storage/__init__.py @@ -19,6 +19,7 @@ .. literalinclude:: snippets.py :start-after: [START storage_get_started] :end-before: [END storage_get_started] + :dedent: 4 The main concepts with this API are: diff --git a/google/cloud/storage/acl.py b/google/cloud/storage/acl.py index fb07faba9..765590f94 100644 --- a/google/cloud/storage/acl.py +++ b/google/cloud/storage/acl.py @@ -21,6 +21,7 @@ .. literalinclude:: snippets.py :start-after: [START client_bucket_acl] :end-before: [END client_bucket_acl] + :dedent: 4 Adding and removing permissions can be done with the following methods @@ -52,6 +53,7 @@ .. literalinclude:: snippets.py :start-after: [START acl_user_settings] :end-before: [END acl_user_settings] + :dedent: 4 After that, you can save any changes you make with the :func:`google.cloud.storage.acl.ACL.save` method: @@ -59,6 +61,7 @@ .. literalinclude:: snippets.py :start-after: [START acl_save] :end-before: [END acl_save] + :dedent: 4 You can alternatively save any existing :class:`google.cloud.storage.acl.ACL` object (whether it was created by a factory method or not) from a @@ -67,6 +70,7 @@ .. literalinclude:: snippets.py :start-after: [START acl_save_bucket] :end-before: [END acl_save_bucket] + :dedent: 4 To get the list of ``entity`` and ``role`` for each unique pair, the :class:`ACL` class is iterable: @@ -74,6 +78,7 @@ .. literalinclude:: snippets.py :start-after: [START acl_print] :end-before: [END acl_print] + :dedent: 4 This list of tuples can be used as the ``entity`` and ``role`` fields when sending metadata for ACLs to the API. diff --git a/google/cloud/storage/bucket.py b/google/cloud/storage/bucket.py index 2e88f5a84..c82226cc3 100644 --- a/google/cloud/storage/bucket.py +++ b/google/cloud/storage/bucket.py @@ -968,6 +968,7 @@ def get_blob( .. literalinclude:: snippets.py :start-after: [START get_blob] :end-before: [END get_blob] + :dedent: 4 If :attr:`user_project` is set, bills the API request to that project. @@ -1381,6 +1382,7 @@ def delete_blob( .. literalinclude:: snippets.py :start-after: [START delete_blob] :end-before: [END delete_blob] + :dedent: 4 If :attr:`user_project` is set, bills the API request to that project. @@ -1431,6 +1433,7 @@ def delete_blob( .. literalinclude:: snippets.py :start-after: [START delete_blobs] :end-before: [END delete_blobs] + :dedent: 4 """ client = self._require_client(client) @@ -2105,6 +2108,7 @@ def add_lifecycle_delete_rule(self, **kw): .. literalinclude:: snippets.py :start-after: [START add_lifecycle_delete_rule] :end-before: [END add_lifecycle_delete_rule] + :dedent: 4 :type kw: dict :params kw: arguments passed to :class:`LifecycleRuleConditions`. @@ -2122,6 +2126,7 @@ def add_lifecycle_set_storage_class_rule(self, storage_class, **kw): .. literalinclude:: snippets.py :start-after: [START add_lifecycle_set_storage_class_rule] :end-before: [END add_lifecycle_set_storage_class_rule] + :dedent: 4 :type storage_class: str, one of :attr:`STORAGE_CLASSES`. :param storage_class: new storage class to assign to matching items. @@ -2465,12 +2470,14 @@ def configure_website(self, main_page_suffix=None, not_found_page=None): .. literalinclude:: snippets.py :start-after: [START configure_website] :end-before: [END configure_website] + :dedent: 4 You probably should also make the whole bucket public: .. literalinclude:: snippets.py :start-after: [START make_public] :end-before: [END make_public] + :dedent: 4 This says: "Make the bucket public, and all the stuff already in the bucket, and anything else I add to the bucket. Just make it @@ -2804,6 +2811,7 @@ def generate_upload_policy(self, conditions, expiration=None, client=None): .. literalinclude:: snippets.py :start-after: [START policy_document] :end-before: [END policy_document] + :dedent: 4 .. _policy documents: https://cloud.google.com/storage/docs/xml-api\ diff --git a/google/cloud/storage/client.py b/google/cloud/storage/client.py index 4b23de04e..2fb7fb75c 100644 --- a/google/cloud/storage/client.py +++ b/google/cloud/storage/client.py @@ -329,6 +329,7 @@ def get_bucket( .. literalinclude:: snippets.py :start-after: [START get_bucket] :end-before: [END get_bucket] + :dedent: 4 Get a bucket using a resource. @@ -367,6 +368,7 @@ def lookup_bucket( .. literalinclude:: snippets.py :start-after: [START lookup_bucket] :end-before: [END lookup_bucket] + :dedent: 4 :type bucket_name: str :param bucket_name: The name of the bucket to get. @@ -461,6 +463,7 @@ def create_bucket( .. literalinclude:: snippets.py :start-after: [START create_bucket] :end-before: [END create_bucket] + :dedent: 4 Create a bucket using a resource. @@ -702,6 +705,7 @@ def list_buckets( .. literalinclude:: snippets.py :start-after: [START list_buckets] :end-before: [END list_buckets] + :dedent: 4 This implements "storage.buckets.list".