forked from googleapis/google-cloud-python
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add google.api.core.retry with base retry functionality (googleapis#3819
) Add google.api.core.retry with base retry functionality Additionally: * Add google.api.core.exceptions.RetryError * Add google.api.core.helpers package * Add google.api.core.helpers.datetime_helpers module
- Loading branch information
Showing
7 changed files
with
344 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# Copyright 2017 Google Inc. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
"""Helpers for :mod:`datetime`.""" | ||
|
||
import datetime | ||
|
||
|
||
def utcnow(): | ||
"""A :meth:`datetime.datetime.utcnow()` alias to allow mocking in tests.""" | ||
return datetime.datetime.utcnow() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,148 @@ | ||
# Copyright 2017 Google Inc. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
"""Helpers for retrying functions with exponential back-off.""" | ||
|
||
import datetime | ||
import logging | ||
import random | ||
import time | ||
|
||
import six | ||
|
||
from google.api.core import exceptions | ||
from google.api.core.helpers import datetime_helpers | ||
|
||
_LOGGER = logging.getLogger(__name__) | ||
_DEFAULT_MAX_JITTER = 0.2 | ||
|
||
|
||
def if_exception_type(*exception_types): | ||
"""Creates a predicate to check if the exception is of a given type. | ||
Args: | ||
exception_types (Sequence[type]): The exception types to check for. | ||
Returns: | ||
Callable[Exception]: A predicate that returns True if the provided | ||
exception is of the given type(s). | ||
""" | ||
def inner(exception): | ||
"""Bound predicate for checking an exception type.""" | ||
return isinstance(exception, exception_types) | ||
return inner | ||
|
||
|
||
# pylint: disable=invalid-name | ||
# Pylint sees this as a constant, but it is also an alias that should be | ||
# considered a function. | ||
if_transient_error = if_exception_type(( | ||
exceptions.InternalServerError, | ||
exceptions.TooManyRequests)) | ||
"""A predicate that checks if an exception is a transient API error. | ||
The following server errors are considered transient: | ||
- :class:`google.api.core.exceptions.InternalServerError` - HTTP 500, gRPC | ||
``INTERNAL(13)`` and its subclasses. | ||
- :class:`google.api.core.exceptions.TooManyRequests` - HTTP 429 | ||
- :class:`google.api.core.exceptions.ResourceExhausted` - gRPC | ||
``RESOURCE_EXHAUSTED(8)`` | ||
""" | ||
# pylint: enable=invalid-name | ||
|
||
|
||
def exponential_sleep_generator( | ||
initial, maximum, multiplier=2, jitter=_DEFAULT_MAX_JITTER): | ||
"""Generates sleep intervals based on the exponential back-off algorithm. | ||
This implements the `Truncated Exponential Back-off`_ algorithm. | ||
.. _Truncated Exponential Back-off: | ||
https://cloud.google.com/storage/docs/exponential-backoff | ||
Args: | ||
initial (float): The minimum about of time to delay. This must | ||
be greater than 0. | ||
maximum (float): The maximum about of time to delay. | ||
multiplier (float): The multiplier applied to the delay. | ||
jitter (float): The maximum about of randomness to apply to the delay. | ||
Yields: | ||
float: successive sleep intervals. | ||
""" | ||
delay = initial | ||
while True: | ||
yield delay | ||
delay = min( | ||
delay * multiplier + random.uniform(0, jitter), maximum) | ||
|
||
|
||
def retry_target(target, predicate, sleep_generator, deadline): | ||
"""Call a function and retry if it fails. | ||
This is the lowest-level retry helper. Generally, you'll use the | ||
higher-level retry helper :class:`Retry`. | ||
Args: | ||
target(Callable): The function to call and retry. This must be a | ||
nullary function - apply arguments with `functools.partial`. | ||
predicate (Callable[Exception]): A callable used to determine if an | ||
exception raised by the target should be considered retryable. | ||
It should return True to retry or False otherwise. | ||
sleep_generator (Iterator[float]): An infinite iterator that determines | ||
how long to sleep between retries. | ||
deadline (float): How long to keep retrying the target. | ||
Returns: | ||
Any: the return value of the target function. | ||
Raises: | ||
google.api.core.RetryError: If the deadline is exceeded while retrying. | ||
ValueError: If the sleep generator stops yielding values. | ||
Exception: If the target raises a method that isn't retryable. | ||
""" | ||
if deadline is not None: | ||
deadline_datetime = ( | ||
datetime_helpers.utcnow() + datetime.timedelta(seconds=deadline)) | ||
else: | ||
deadline_datetime = None | ||
|
||
last_exc = None | ||
|
||
for sleep in sleep_generator: | ||
try: | ||
return target() | ||
|
||
# pylint: disable=broad-except | ||
# This function explicitly must deal with broad exceptions. | ||
except Exception as exc: | ||
if not predicate(exc): | ||
raise | ||
last_exc = exc | ||
|
||
now = datetime_helpers.utcnow() | ||
if deadline_datetime is not None and deadline_datetime < now: | ||
six.raise_from( | ||
exceptions.RetryError( | ||
'Deadline of {:.1f}s exceeded while calling {}'.format( | ||
deadline, target), | ||
last_exc), | ||
last_exc) | ||
|
||
_LOGGER.debug('Retrying due to {}, sleeping {:.1f}s ...'.format( | ||
last_exc, sleep)) | ||
time.sleep(sleep) | ||
|
||
raise ValueError('Sleep generator stopped yielding sleep values.') |
Empty file.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
# Copyright 2017, Google Inc. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import datetime | ||
|
||
from google.api.core.helpers import datetime_helpers | ||
|
||
|
||
def test_utcnow(): | ||
result = datetime_helpers.utcnow() | ||
assert isinstance(result, datetime.datetime) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,129 @@ | ||
# Copyright 2017 Google Inc. | ||
# | ||
# Licensed under the Apache License, Version 2.0 (the "License"); | ||
# you may not use this file except in compliance with the License. | ||
# You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, software | ||
# distributed under the License is distributed on an "AS IS" BASIS, | ||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
# See the License for the specific language governing permissions and | ||
# limitations under the License. | ||
|
||
import datetime | ||
import itertools | ||
|
||
import mock | ||
import pytest | ||
|
||
from google.api.core import exceptions | ||
from google.api.core import retry | ||
|
||
|
||
def test_if_exception_type(): | ||
predicate = retry.if_exception_type(ValueError) | ||
|
||
assert predicate(ValueError()) | ||
assert not predicate(TypeError()) | ||
|
||
|
||
def test_if_exception_type_multiple(): | ||
predicate = retry.if_exception_type(ValueError, TypeError) | ||
|
||
assert predicate(ValueError()) | ||
assert predicate(TypeError()) | ||
assert not predicate(RuntimeError()) | ||
|
||
|
||
def test_if_transient_error(): | ||
assert retry.if_transient_error(exceptions.InternalServerError('')) | ||
assert retry.if_transient_error(exceptions.TooManyRequests('')) | ||
assert not retry.if_transient_error(exceptions.InvalidArgument('')) | ||
|
||
|
||
def test_exponential_sleep_generator_base_2(): | ||
gen = retry.exponential_sleep_generator( | ||
1, 60, 2, jitter=0.0) | ||
|
||
result = list(itertools.islice(gen, 8)) | ||
assert result == [1, 2, 4, 8, 16, 32, 60, 60] | ||
|
||
|
||
@mock.patch('random.uniform') | ||
def test_exponential_sleep_generator_jitter(uniform): | ||
uniform.return_value = 1 | ||
gen = retry.exponential_sleep_generator( | ||
1, 60, 2, jitter=2.2) | ||
|
||
result = list(itertools.islice(gen, 7)) | ||
assert result == [1, 3, 7, 15, 31, 60, 60] | ||
uniform.assert_called_with(0.0, 2.2) | ||
|
||
|
||
@mock.patch('time.sleep') | ||
@mock.patch( | ||
'google.api.core.helpers.datetime_helpers.utcnow', | ||
return_value=datetime.datetime.min) | ||
def test_retry_target_success(utcnow, sleep): | ||
predicate = retry.if_exception_type(ValueError) | ||
call_count = [0] | ||
|
||
def target(): | ||
call_count[0] += 1 | ||
if call_count[0] < 3: | ||
raise ValueError() | ||
return 42 | ||
|
||
result = retry.retry_target(target, predicate, range(10), None) | ||
|
||
assert result == 42 | ||
assert call_count[0] == 3 | ||
sleep.assert_has_calls([mock.call(0), mock.call(1)]) | ||
|
||
|
||
@mock.patch('time.sleep') | ||
@mock.patch( | ||
'google.api.core.helpers.datetime_helpers.utcnow', | ||
return_value=datetime.datetime.min) | ||
def test_retry_target_non_retryable_error(utcnow, sleep): | ||
predicate = retry.if_exception_type(ValueError) | ||
exception = TypeError() | ||
target = mock.Mock(side_effect=exception) | ||
|
||
with pytest.raises(TypeError) as exc_info: | ||
retry.retry_target(target, predicate, range(10), None) | ||
|
||
assert exc_info.value == exception | ||
sleep.assert_not_called() | ||
|
||
|
||
@mock.patch('time.sleep') | ||
@mock.patch( | ||
'google.api.core.helpers.datetime_helpers.utcnow') | ||
def test_retry_target_deadline_exceeded(utcnow, sleep): | ||
predicate = retry.if_exception_type(ValueError) | ||
exception = ValueError('meep') | ||
target = mock.Mock(side_effect=exception) | ||
# Setup the timeline so that the first call takes 5 seconds but the second | ||
# call takes 6, which puts the retry over the deadline. | ||
utcnow.side_effect = [ | ||
# The first call to utcnow establishes the start of the timeline. | ||
datetime.datetime.min, | ||
datetime.datetime.min + datetime.timedelta(seconds=5), | ||
datetime.datetime.min + datetime.timedelta(seconds=11)] | ||
|
||
with pytest.raises(exceptions.RetryError) as exc_info: | ||
retry.retry_target(target, predicate, range(10), deadline=10) | ||
|
||
assert exc_info.value.cause == exception | ||
assert exc_info.match('Deadline of 10.0s exceeded') | ||
assert exc_info.match('last exception: meep') | ||
assert target.call_count == 2 | ||
|
||
|
||
def test_retry_target_bad_sleep_generator(): | ||
with pytest.raises(ValueError, match='Sleep generator'): | ||
retry.retry_target( | ||
mock.sentinel.target, mock.sentinel.predicate, [], None) |