-
Notifications
You must be signed in to change notification settings - Fork 2
Expand file tree
/
Copy pathretry.py
More file actions
116 lines (93 loc) · 3.3 KB
/
Copy pathretry.py
File metadata and controls
116 lines (93 loc) · 3.3 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
"""Retry strategy for API requests with exponential backoff and jitter."""
import logging
import random
from typing import List, Optional
logger = logging.getLogger(__name__)
class RetryStrategy:
"""
Shared retry logic for both sync and async clients.
Implements exponential backoff with configurable retry conditions.
"""
def __init__(
self,
max_retries: int = 3,
retry_on: Optional[List[int]] = None,
jitter: bool = True,
):
"""
Initialize retry strategy.
Args:
max_retries: Maximum number of retry attempts
retry_on: HTTP status codes to retry on (default: [500, 502, 503, 504])
jitter: Add randomized jitter to backoff to prevent thundering herd (default: True)
"""
self.max_retries = max_retries
self.retry_on = retry_on or [500, 502, 503, 504]
self.jitter = jitter
def should_retry(self, attempt: int, status_code: int) -> bool:
"""
Determine if request should be retried.
Args:
attempt: Current attempt number (0-indexed)
status_code: HTTP status code from response
Returns:
True if request should be retried, False otherwise
"""
return (
status_code in self.retry_on
and attempt < self.max_retries - 1
)
def should_retry_on_exception(self, attempt: int) -> bool:
"""
Determine if request should be retried on exception.
Args:
attempt: Current attempt number (0-indexed)
Returns:
True if request should be retried, False otherwise
"""
return attempt < self.max_retries - 1
def calculate_wait_time(self, attempt: int) -> float:
"""
Calculate exponential backoff wait time with optional jitter.
Jitter prevents thundering herd problem where many clients retry
simultaneously after an outage, potentially overwhelming the recovered service.
Args:
attempt: Current attempt number (0-indexed)
Returns:
Wait time in seconds (capped at 60 seconds)
Examples:
Without jitter:
- Attempt 0: 1.0s
- Attempt 1: 2.0s
- Attempt 2: 4.0s
With jitter (adds 0-30% randomization):
- Attempt 0: 1.0-1.3s
- Attempt 1: 2.0-2.6s
- Attempt 2: 4.0-5.2s
"""
base_wait = min(2 ** attempt, 60)
if self.jitter:
# Add 0-30% random jitter to prevent synchronized retries
jitter_amount = random.uniform(0, 0.3 * base_wait)
return base_wait + jitter_amount
return base_wait
def log_retry(
self,
attempt: int,
reason: str,
wait_time: float,
is_async: bool = False
) -> None:
"""
Log retry attempt.
Args:
attempt: Current attempt number (0-indexed)
reason: Reason for retry
wait_time: Wait time before retry
is_async: Whether this is an async client
"""
client_type = "Async" if is_async else "Sync"
logger.warning(
f"[{client_type}] {reason}, retrying in {wait_time}s "
f"(attempt {attempt + 1}/{self.max_retries})"
)