-
Notifications
You must be signed in to change notification settings - Fork 6.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add Migration Guide Snippets for Cloud Tasks #2316
Changes from 5 commits
38d0ff7
5b8f559
7dd0ac5
faa3997
f6c9045
0d333d3
4cd4404
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,285 @@ | ||
# Copyright 2019 Google LLC All Rights Reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
from google.cloud import tasks | ||
|
||
|
||
def create_queue(project, location, queue_blue_name, queue_red_name): | ||
# [START taskqueues_using_yaml] | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue_blue_name = 'queue-blue' | ||
# queue_red_name = 'queue-red' | ||
|
||
parent = client.location_path(project, location) | ||
|
||
queue_blue = { | ||
'name': client.queue_path(project, location, queue_blue_name), | ||
'rate_limits': { | ||
'max_dispatches_per_second': 5 | ||
}, | ||
'app_engine_routing_override': { | ||
'version': 'v2', | ||
'service': 'task-module' | ||
} | ||
} | ||
|
||
queue_red = { | ||
'name': client.queue_path(project, location, queue_red_name), | ||
'rate_limits': { | ||
'max_dispatches_per_second': 1 | ||
} | ||
} | ||
|
||
queues = [queue_blue, queue_red] | ||
for queue in queues: | ||
response = client.create_queue(parent, queue) | ||
print(response) | ||
# [END taskqueues_using_yaml] | ||
return response | ||
|
||
|
||
def update_queue(project, location, queue): | ||
# [START taskqueues_processing_rate] | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue = 'queue-blue' | ||
|
||
# Get queue object | ||
queue_path = client.queue_path(project, location, queue) | ||
queue = client.get_queue(queue_path) | ||
|
||
# Update queue object | ||
queue.rate_limits.max_dispatches_per_second = 20 | ||
queue.rate_limits.max_concurrent_dispatches = 10 | ||
|
||
response = client.update_queue(queue) | ||
print(response) | ||
# [END taskqueues_processing_rate] | ||
return response | ||
|
||
|
||
def create_task(project, location, queue): | ||
# [START taskqueues_new_task] | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue = 'default' | ||
amount = 10 | ||
|
||
parent = client.queue_path(project, location, queue) | ||
|
||
task = { | ||
'app_engine_http_request': { | ||
'http_method': 'POST', | ||
'relative_uri': '/update_counter', | ||
'app_engine_routing': { | ||
'service': 'worker' | ||
}, | ||
'body': str(amount).encode() | ||
} | ||
} | ||
|
||
response = client.create_task(parent, task) | ||
eta = response.schedule_time.ToDatetime().strftime("%m/%d/%Y, %H:%M:%S") | ||
print('Task {} enqueued, ETA {}.'.format(response.name, eta)) | ||
# [END taskqueues_new_task] | ||
return response | ||
|
||
|
||
def create_tasks_with_data(project, location, queue): | ||
# [START taskqueues_passing_data] | ||
import json | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue = 'default' | ||
|
||
parent = client.queue_path(project, location, queue) | ||
|
||
task1 = { | ||
'app_engine_http_request': { | ||
'http_method': 'POST', | ||
'relative_uri': '/update_counter?key=blue', | ||
'app_engine_routing': { | ||
'service': 'worker' | ||
} | ||
} | ||
} | ||
|
||
task2 = { | ||
'app_engine_http_request': { | ||
'http_method': 'POST', | ||
'relative_uri': '/update_counter', | ||
'app_engine_routing': { | ||
'service': 'worker' | ||
}, | ||
'headers': { | ||
'Content-Type': 'application/json' | ||
}, | ||
'body': json.dumps({'key': 'blue'}).encode() | ||
} | ||
} | ||
|
||
response = client.create_task(parent, task1) | ||
print(response) | ||
response = client.create_task(parent, task2) | ||
print(response) | ||
# [END taskqueues_passing_data] | ||
return response | ||
|
||
|
||
def create_task_with_name(project, location, queue, task_name): | ||
# [START taskqueues_naming_tasks] | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue = 'default' | ||
# task_name = 'first-try' | ||
|
||
parent = client.queue_path(project, location, queue) | ||
|
||
task = { | ||
'name': client.task_path(project, location, queue, task_name), | ||
'app_engine_http_request': { | ||
'http_method': 'GET', | ||
'relative_uri': '/url/path' | ||
} | ||
} | ||
response = client.create_task(parent, task) | ||
print(response) | ||
# [END taskqueues_naming_tasks] | ||
return response | ||
|
||
|
||
def delete_task(project, location, queue): | ||
# [START taskqueues_setup] | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue = 'queue1' | ||
# [START taskqueues_setup] | ||
|
||
# [START taskqueues_deleting_tasks] | ||
task_path = client.task_path(project, location, queue, 'foo') | ||
response = client.delete_task(task_path) | ||
# [END taskqueues_deleting_tasks] | ||
|
||
# [START taskqueues_purging_tasks] | ||
queue_path = client.queue_path(project, location, queue) | ||
response = client.purge_queue(queue_path) | ||
# [END taskqueues_purging_tasks] | ||
|
||
# [START taskqueues_pause_queue] | ||
queue_path = client.queue_path(project, location, queue) | ||
response = client.pause_queue(queue_path) | ||
# [END taskqueues_pause_queues] | ||
return response | ||
|
||
|
||
def delete_queue(project, location, queue): | ||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# queue = 'queue1' | ||
|
||
# [START taskqueues_deleting_queues] | ||
queue_path = client.queue_path(project, location, queue) | ||
response = client.delete_queue(queue_path) | ||
# [END taskqueues_deleting_queues] | ||
return response | ||
|
||
|
||
def retry_task(project, location, fooqueue, barqueue, bazqueue): | ||
# [START taskqueues_retrying_tasks] | ||
from google.protobuf import duration_pb2 | ||
|
||
client = tasks.CloudTasksClient() | ||
|
||
# TODO(developer): Uncomment these lines and replace with your values. | ||
# project = 'my-project-id' | ||
# location = 'us- central1' | ||
# fooqueue = 'fooqueue' | ||
# barqueue = 'barqueue' | ||
# bazqueue = 'bazqueue' | ||
|
||
parent = client.location_path(project, location) | ||
|
||
max_retry = duration_pb2.Duration() | ||
max_retry.seconds = 2*60*60*24 | ||
|
||
foo = { | ||
'name': client.queue_path(project, location, fooqueue), | ||
'rate_limits': { | ||
'max_dispatches_per_second': 1 | ||
}, | ||
'retry_config': { | ||
'max_attempts': 7, | ||
'max_retry_duration': max_retry | ||
} | ||
} | ||
|
||
min = duration_pb2.Duration() | ||
min.seconds = 10 | ||
|
||
max = duration_pb2.Duration() | ||
max.seconds = 200 | ||
|
||
bar = { | ||
'name': client.queue_path(project, location, barqueue), | ||
'rate_limits': { | ||
'max_dispatches_per_second': 1 | ||
}, | ||
'retry_config': { | ||
'min_backoff': min, | ||
'max_backoff': max, | ||
'max_doublings': 0 | ||
} | ||
} | ||
|
||
max.seconds = 300 | ||
baz = { | ||
'name': client.queue_path(project, location, bazqueue), | ||
'rate_limits': { | ||
'max_dispatches_per_second': 1 | ||
}, | ||
'retry_config': { | ||
'min_backoff': min, | ||
'max_backoff': max, | ||
'max_doublings': 3 | ||
} | ||
} | ||
|
||
queues = [foo, bar, baz] | ||
for queue in queues: | ||
response = client.create_queue(parent, queue) | ||
print(response) | ||
# [END taskqueues_retrying_tasks] | ||
return response |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
# Copyright 2019 Google LLC All Rights Reserved. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import os | ||
import pytest | ||
import uuid | ||
|
||
import snippets | ||
|
||
TEST_PROJECT_ID = os.getenv('GCLOUD_PROJECT') | ||
TEST_LOCATION = os.getenv('TEST_QUEUE_LOCATION', 'us-central1') | ||
QUEUE_NAME_1 = "queue-{}".format(uuid.uuid4()) | ||
QUEUE_NAME_2 = "queue-{}".format(uuid.uuid4()) | ||
|
||
|
||
@pytest.mark.order1 | ||
def test_create_queue(): | ||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_2) | ||
result = snippets.create_queue( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1, QUEUE_NAME_2) | ||
assert name in result.name | ||
|
||
|
||
@pytest.mark.order2 | ||
def test_update_queue(): | ||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
result = snippets.update_queue( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
assert name in result.name | ||
|
||
|
||
@pytest.mark.order3 | ||
def test_create_task(): | ||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
result = snippets.create_task( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
assert name in result.name | ||
|
||
|
||
@pytest.mark.order4 | ||
def test_create_task_with_data(): | ||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
result = snippets.create_tasks_with_data( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
assert name in result.name | ||
|
||
|
||
@pytest.mark.order5 | ||
def test_create_task_with_name(): | ||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
result = snippets.create_task_with_name( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1, 'foo') | ||
assert name in result.name | ||
|
||
|
||
@pytest.mark.order6 | ||
def test_delete_task(): | ||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
result = snippets.delete_task( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
assert name in result.name | ||
|
||
|
||
@pytest.mark.order8 | ||
def test_delete_queue(): | ||
result = snippets.delete_queue( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_1) | ||
assert result is None | ||
|
||
result = snippets.delete_queue( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME_2) | ||
assert result is None | ||
|
||
|
||
@pytest.mark.order7 | ||
def test_retry_task(): | ||
QUEUE_NAME = [] | ||
for i in range(3): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Like Otherwise, we may break the tests if we change L95 without changing L105 vice versa. |
||
QUEUE_NAME.append("queue-{}".format(uuid.uuid4())) | ||
|
||
name = "projects/{}/locations/{}/queues/{}".format( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME[2]) | ||
result = snippets.retry_task( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME[0], QUEUE_NAME[1], | ||
QUEUE_NAME[2]) | ||
assert name in result.name | ||
|
||
for i in range(3): | ||
snippets.delete_queue( | ||
TEST_PROJECT_ID, TEST_LOCATION, QUEUE_NAME[i]) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this line could be more readable with named parameters project=TEST_PROJECT_ID, location=TEST_LOCATION, queue=QUEUE_NAME[i] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,4 +1,2 @@ | ||
Flask==1.0.2 | ||
gunicorn==19.9.0 | ||
google-cloud-tasks==1.1.0 | ||
googleapis-common-protos==1.6.0 |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Nit: It would make sense if
@pytest.mark.order7
came before@pytest.mark.order8
if possible.