Skip to content

Commit 11dfa31

Browse files
chmreidchmreid
committed
Merge branch 'master' into dss-params
* master: Adding a document to explain the action and resource relationship for… (#2395) Requirements Update (#2410) Increase dss notify workers. (#2408)
2 parents 052eb6c + fbb0735 commit 11dfa31

File tree

6 files changed

+281
-18
lines changed

6 files changed

+281
-18
lines changed

‎docs/resource_def.md

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
|Actions| Resource|
2+
|-------|-----------|
3+
|dss:GetCheckout |arn:hca:dss:{stage}:{replica}:checkout/{checkout_job_id}
4+
|dss:DeleteBundle |arn:hca:dss:{stage}:{replica}:bundles/{uuid}.{version}
5+
|dss:GetBundle |arn:hca:dss:{stage}:{replica}:bundles/{uuid}.{version}
6+
|dss:PatchBundle |arn:hca:dss:{stage}:{replica}:bundles/{uuid}.{version}
7+
|dss:PutBundle |arn:hca:dss:{stage}:{replica}:bundles/{uuid}.{version}
8+
|dss:PostCheckout |arn:hca:dss:{stage}:{replica}:bundles/{uuid}.{version}
9+
|dss:GetCollections |arn:hca:dss:{stage}:{replica}:{user}/collections
10+
|dss:PutCollection |arn:hca:dss:{stage}:{replica}:collections/{uuid}.{version}, arn:hca:dss:{stage}:{replica}:{user}/collections/{uuid}
11+
|dss:DeleteCollection |arn:hca:dss:{stage}:{replica}:collections/{uuid}.{version}, arn:hca:dss:{stage}:{replica}:{user}/collections/{uuid}
12+
|dss:GetCollection |arn:hca:dss:{stage}:{replica}:collections/{uuid}.{version}, arn:hca:dss:{stage}:{replica}:{user}/collections/{uuid}
13+
|dss:PatchCollection |arn:hca:dss:{stage}:{replica}:collections/{uuid}.{version}, arn:hca:dss:{stage}:{replica}:{user}/collections/{uuid}
14+
|dss:GetFiles |arn:hca:dss:{stage}:{replica}:files/{uuid}.{version}
15+
|dss:HeadFiles |arn:hca:dss:{stage}:{replica}:files/{uuid}.{version}
16+
|dss:PutFiles |arn:hca:dss:{stage}:{replica}:files/{uuid}.{version}
17+
|dss:PostSearch |arn:hca:dss:{stage}:{replica}:query
18+
|dss:GetSubscriptions |arn:hca:dss:{stage}:{replica}:{user}/subscriptions
19+
|dss:PutSubscriptions |arn:hca:dss:{stage}:{replica}:subscription/{uuid}, arn:hca:dss:{stage}:{replica}:{user}/subscription/{uuid}
20+
|dss:GetSubscription |arn:hca:dss:{stage}:{replica}:subscription/{uuid}, arn:hca:dss:{stage}:{replica}:{user}/subscription/{uuid}
21+
|dss:DeleteSubscription |arn:hca:dss:{stage}:{replica}:subscription/{uuid}, arn:hca:dss:{stage}:{replica}:{user}/subscription/{uuid}

‎dss/notify/notifier.py

+8-2
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
SQS_MAX_VISIBILITY_TIMEOUT = 43200
2020

21+
2122
class Notifier:
2223

2324
@classmethod
@@ -85,9 +86,14 @@ def __init__(self,
8586

8687
def deploy(self) -> None:
8788
sqs = boto3.client('sqs')
89+
two_weeks = '1209600'
8890
for queue_index in self._queue_indices:
89-
sqs.create_queue(QueueName=self._queue_name(queue_index), Attributes={'FifoQueue': "true"})
90-
sqs.create_queue(QueueName=self._queue_name(None), Attributes={'FifoQueue': "true"})
91+
sqs.create_queue(QueueName=self._queue_name(queue_index),
92+
Attributes={'FifoQueue': 'true',
93+
'MessageRetentionPeriod': two_weeks})
94+
sqs.create_queue(QueueName=self._queue_name(None),
95+
Attributes={'FifoQueue': 'true',
96+
'MessageRetentionPeriod': two_weeks})
9197

9298
def destroy(self, purge_only=False) -> None:
9399
if self._deployment_stage.startswith('test-'):

‎environment

+8
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ EXPORT_ENV_VARS_TO_LAMBDA_ARRAY=(
2929
DSS_NOTIFICATION_SENDER
3030
DSS_NOTIFY_DELAYS
3131
DSS_NOTIFY_TIMEOUT
32+
DSS_NOTIFY_WORKERS
3233
DSS_S3_BUCKET
3334
DSS_S3_CHECKOUT_BUCKET
3435
DSS_VERSION
@@ -133,6 +134,13 @@ DSS_NOTIFY_TIMEOUT=60 # This may seem excessive but the default of 10s is not e
133134
# and would cause the notification to be retried, causing Lira to make a duplicate
134135
# submission.
135136

137+
# Touch base with Lira (Saman and Rex currently) on this timeout once:
138+
# - they have scale-tested their new queue component
139+
# - all subscribers (not only Green Box) agree on the shorter timeout
140+
# https://github.com/HumanCellAtlas/data-store/issues/2398
141+
142+
DSS_NOTIFY_WORKERS=24 # unset to automatically determine the number of workers
143+
136144
# Enables X-Ray profiling of daemons running in AWS Lambdas. A value of 0 disables profiling. A value >1 will enable
137145
# profiling.
138146
DSS_XRAY_TRACE=0

‎requirements-dev.txt

+12-10
Original file line numberDiff line numberDiff line change
@@ -2,16 +2,16 @@
22
asn1crypto==0.24.0
33
attrs==19.1.0
44
aws-xray-sdk==2.4.2
5-
awscli==1.16.217
5+
awscli==1.16.229
66
azure-common==1.1.23
77
azure-nspkg==3.0.2
88
azure-storage==0.36.0
9-
boto3==1.9.207
10-
botocore==1.12.207
9+
boto3==1.9.219
10+
botocore==1.12.219
1111
cachetools==3.1.1
1212
certifi==2019.6.16
1313
cffi==1.12.3
14-
chalice==1.10.0
14+
chalice==1.11.0
1515
chardet==3.0.4
1616
click==6.7
1717
clickclick==1.2.2
@@ -22,26 +22,27 @@ coverage==4.5.4
2222
crc32c==1.7
2323
crcmod==1.7
2424
cryptography==2.7
25-
dcplib==3.2.0
26-
docutils==0.14
27-
domovoi==2.0.1
25+
dcplib==3.3.0
26+
docutils==0.15.2
27+
domovoi==2.0.2
2828
elasticsearch==5.5.3
2929
elasticsearch-dsl==5.4.0
3030
entrypoints==0.3
3131
enum-compat==0.0.2
32-
Faker==2.0.0
32+
Faker==2.0.1
3333
fasteners==0.15
3434
flake8==3.7.8
3535
Flask==1.1.1
36+
furl==2.0.0
3637
future==0.17.1
3738
google-api-core==1.14.2
3839
google-api-python-client==1.7.11
3940
google-apitools==0.5.30
4041
google-auth==1.6.3
4142
google-auth-httplib2==0.0.3
4243
google-cloud-core==1.0.3
43-
google-cloud-storage==1.18.0
44-
google-resumable-media==0.3.2
44+
google-cloud-storage==1.19.0
45+
google-resumable-media==0.3.3
4546
googleapis-common-protos==1.6.0
4647
httpie==1.0.2
4748
httplib2==0.13.1
@@ -61,6 +62,7 @@ mypy==0.720
6162
mypy-extensions==0.4.1
6263
nestedcontext==0.0.4
6364
oauth2client==4.1.3
65+
orderedmultidict==1.0.1
6466
protobuf==3.9.1
6567
puremagic==1.4
6668
pyasn1==0.4.6

‎requirements.txt

+8-6
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ aws-xray-sdk==2.4.2
44
azure-common==1.1.23
55
azure-nspkg==3.0.2
66
azure-storage==0.36.0
7-
boto3==1.9.207
8-
botocore==1.12.207
7+
boto3==1.9.219
8+
botocore==1.12.219
99
cachetools==3.1.1
1010
certifi==2019.6.16
1111
cffi==1.12.3
@@ -16,21 +16,22 @@ cloud-blobstore==3.1.0
1616
connexion==1.5.3
1717
crc32c==1.7
1818
cryptography==2.7
19-
dcplib==3.2.0
20-
docutils==0.14
19+
dcplib==3.3.0
20+
docutils==0.15.2
2121
elasticsearch==5.5.3
2222
elasticsearch-dsl==5.4.0
2323
fasteners==0.15
2424
Flask==1.1.1
25+
furl==2.0.0
2526
future==0.17.1
2627
google-api-core==1.14.2
2728
google-api-python-client==1.7.11
2829
google-apitools==0.5.30
2930
google-auth==1.6.3
3031
google-auth-httplib2==0.0.3
3132
google-cloud-core==1.0.3
32-
google-cloud-storage==1.18.0
33-
google-resumable-media==0.3.2
33+
google-cloud-storage==1.19.0
34+
google-resumable-media==0.3.3
3435
googleapis-common-protos==1.6.0
3536
httplib2==0.13.1
3637
idna==2.8
@@ -46,6 +47,7 @@ MarkupSafe==1.1.1
4647
monotonic==1.5
4748
nestedcontext==0.0.4
4849
oauth2client==4.1.3
50+
orderedmultidict==1.0.1
4951
protobuf==3.9.1
5052
puremagic==1.4
5153
pyasn1==0.4.6

0 commit comments

Comments
 (0)