Merge branch 'master' into dss-params

* master:
  Revert "Bump httpie from 1.0.2 to 1.0.3 (#2397)" (#2401)
  Bump httpie from 1.0.2 to 1.0.3 (#2397)
  Fix flaky test: TestFileApi.test_file_get_checkout . (#2396)
  Add retries to verification in _test_gs_cache(). (#2372)
  Added details of notify intervals (#2368)

Author: chmreid

daemons/dss-notify-v2/Readme.md

@@ -18,8 +18,8 @@ or via SQS queue:
 * Lambda-triggered via SQS `dss-notify-v2-{DSS_DEPLOYMENT_STAGE}`
 
 If notification delivery fails, a notification record is made in the queue. Delivery will be attempted once
-after 15 minutes, and again every hour for 7 days. Delivery delay and duration is configured via SQS DelaySeconds
-and VisibilityTimeout.
+after 15 minutes, and again every hour for 7 days. Delivery delay and duration is 
+configured via SQS attribute DelaySeconds.
 
 Event handlers in the dss-notify-v2 daemon use utility functions in
 [dss.events.handlers.notify_v2](../../dss/events/handlers/notify_v2.py).
@@ -66,6 +66,11 @@ of the subscription owner, and the sort key is the subscription uuid. There is o
 Subscriptions are accessed via owner for API actions. When notifications are triggered during an object storage event,
 subscriptions are fetched from the backend via `scan`.
 
+If notification delivery fails, a notification record is made in the queue.Delivery will be attempted immediately
+after failure and at time intervals of one minute, ten minutes, one hour, six hours, and 16 hours
+after the first failure. Delivery would then continue for the next 6 days at 24 hour intervals. After day 7, no further notification will be delivered and the subscription will be
+removed from the delivery service. The old subscription will remain and the user would need to delete it and create a new subscription in order to receive new notifications.
+
 ### Bundle Metadata Document
 
 The bundle metadata document is constructed from json files present in the bundle manifest, and

tests/test_checkout_caching.py

@@ -4,6 +4,7 @@
 import sys
 import unittest
 import tempfile
+import time
 from google.cloud import storage
 from unittest import mock
 
@@ -186,8 +187,14 @@ def _test_gs_cache(self, src_data, content_type, checkout_bucket):
         # parameters of copy_worker are arbitrary, only passed because required.
         event = gscopyclient.implementation.copy_worker(event, spoof_context)
         # verify
-        bucket = client.get_bucket(checkout_bucket)
-        blob_class = bucket.get_blob(test_dst_key).storage_class
+        for retry in [1, 1, 1]:
+            try:
+                bucket = client.get_bucket(checkout_bucket)
+                blob_class = bucket.get_blob(test_dst_key).storage_class
+            except AttributeError:
+                time.sleep(retry)
+            else:
+                break
         # cleanup
         gs_blobstore.delete(replica.bucket, test_src_key)
         gs_blobstore.delete(checkout_bucket, test_dst_key)

tests/test_file.py

@@ -704,12 +704,8 @@ def test_checkout():
             test_checkout()
 
         with self.subTest(f"{replica}: Initiates checkout and returns 302 immediately for GET on stale checkout file."):
-            @eventually(30, 1)
-            def test_creation_date_updated(key, prev_creation_date):
-                self.assertTrue(prev_creation_date < handle.get_creation_date(replica.checkout_bucket, key))
-
             now = datetime.datetime.now(datetime.timezone.utc)
-            old_creation_date = handle.get_creation_date(replica.checkout_bucket, file_key)
+            creation_date = handle.get_creation_date(replica.checkout_bucket, file_key)
             creation_date_fn = ("cloud_blobstore.s3.S3BlobStore.get_creation_date"
                                 if replica.name == "aws"
                                 else "cloud_blobstore.gs.GSBlobStore.get_creation_date")
@@ -718,7 +714,9 @@ def test_creation_date_updated(key, prev_creation_date):
                 blob_ttl_days = int(os.environ['DSS_BLOB_PUBLIC_TTL_DAYS'])
                 mock_creation_date.return_value = now - datetime.timedelta(days=blob_ttl_days + 1)
                 self.assertGetResponse(url, requests.codes.found, headers=get_auth_header(), redirect_follow_retries=0)
-            test_creation_date_updated(file_key, old_creation_date)
+                self.assertTrue(creation_date > handle.get_creation_date(replica.checkout_bucket, file_key),
+                                f'\ncurr_creation_date: {creation_date}'
+                                f'\nprev_creation_date: {handle.get_creation_date(replica.checkout_bucket)}')
 
         handle.delete(test_bucket, f"files/{file_uuid}.{version}")
         handle.delete(replica.checkout_bucket, file_key)