Skip to content

Commit 58649f1

Browse files
authored
Use tenacity.retry for merge_tags (#2285)
* Use tenacity.retry for merge_tags * Add type: ignore * Tenacity raises its own error
1 parent 0ab0a2f commit 58649f1

File tree

3 files changed

+31
-32
lines changed

3 files changed

+31
-32
lines changed

.pre-commit-config.yaml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ repos:
4545
"numpy",
4646
"pytest",
4747
"requests",
48+
"tenacity",
4849
"urllib3",
4950
"types-beautifulsoup4",
5051
"types-python-dateutil",

requirements-dev.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,4 @@ pytest-xdist
99
python-dateutil
1010
requests
1111
tabulate
12+
tenacity

tagging/apps/merge_tags.py

Lines changed: 29 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,10 +3,14 @@
33
# Distributed under the terms of the Modified BSD License.
44
import logging
55
import os
6-
import time
7-
from collections.abc import Callable
86

97
import plumbum
8+
from tenacity import ( # type: ignore
9+
RetryError,
10+
retry,
11+
stop_after_attempt,
12+
wait_exponential,
13+
)
1014

1115
from tagging.apps.common_cli_arguments import common_arguments_parser
1216
from tagging.apps.config import Config
@@ -44,20 +48,11 @@ def read_local_tags_from_files(config: Config) -> tuple[list[str], set[str]]:
4448
return all_local_tags, merged_local_tags
4549

4650

47-
def run_with_retries(func: Callable[[], None]) -> None:
48-
ATTEMPTS = 3
49-
SLEEP_BACKOFF = 2
50-
51-
for attempt in range(ATTEMPTS):
52-
try:
53-
func()
54-
break
55-
except Exception as e:
56-
LOGGER.warning(f"Attempt {attempt + 1} failed: {e}")
57-
if attempt + 1 == ATTEMPTS:
58-
LOGGER.error(f"Failed after {ATTEMPTS} attempts")
59-
raise
60-
time.sleep(SLEEP_BACKOFF * (attempt + 1))
51+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))
52+
def pull_tag(tag: str) -> None:
53+
LOGGER.info(f"Pulling tag: {tag}")
54+
docker["pull", tag] & plumbum.FG
55+
LOGGER.info(f"Tag {tag} pulled successfully")
6156

6257

6358
def pull_missing_tags(merged_tag: str, all_local_tags: list[str]) -> list[str]:
@@ -74,27 +69,35 @@ def pull_missing_tags(merged_tag: str, all_local_tags: list[str]) -> list[str]:
7469

7570
LOGGER.warning(f"Trying to pull: {platform_tag} from registry")
7671
try:
77-
run_with_retries(lambda: docker["pull", platform_tag] & plumbum.FG)
72+
pull_tag(platform_tag)
7873
existing_platform_tags.append(platform_tag)
7974
LOGGER.info(f"Tag {platform_tag} pulled successfully")
80-
except plumbum.ProcessExecutionError:
75+
except RetryError:
8176
LOGGER.warning(f"Pull failed, tag {platform_tag} doesn't exist")
8277

8378
return existing_platform_tags
8479

8580

86-
def push_manifest(merged_tag: str, existing_platform_tags: list[str]) -> None:
81+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))
82+
def create_manifest(merged_tag: str, existing_platform_tags: list[str]) -> None:
83+
# This allows to rerun the script without having to remove the manifest manually
84+
try:
85+
docker["manifest", "rm", merged_tag] & plumbum.FG
86+
LOGGER.warning(f"Manifest {merged_tag} was present locally, removed it")
87+
except plumbum.ProcessExecutionError:
88+
pass
89+
8790
LOGGER.info(f"Creating manifest for tag: {merged_tag}")
8891
# Unfortunately, `docker manifest create` requires images to have been already pushed to the registry
8992
# which is not true for new tags in PRs
90-
run_with_retries(
91-
lambda: docker["manifest", "create", merged_tag][existing_platform_tags]
92-
& plumbum.FG
93-
)
93+
docker["manifest", "create", merged_tag][existing_platform_tags] & plumbum.FG
9494
LOGGER.info(f"Successfully created manifest for tag: {merged_tag}")
9595

96+
97+
@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4))
98+
def push_manifest(merged_tag: str) -> None:
9699
LOGGER.info(f"Pushing manifest for tag: {merged_tag}")
97-
run_with_retries(lambda: docker["manifest", "push", merged_tag] & plumbum.FG)
100+
docker["manifest", "push", merged_tag] & plumbum.FG
98101
LOGGER.info(f"Successfully pushed manifest for tag: {merged_tag}")
99102

100103

@@ -103,16 +106,10 @@ def merge_tags(
103106
) -> None:
104107
LOGGER.info(f"Trying to merge tag: {merged_tag}")
105108

106-
# This allows to rerun the script without having to remove the manifest manually
107-
try:
108-
docker["manifest", "rm", merged_tag] & plumbum.FG
109-
LOGGER.warning(f"Manifest {merged_tag} was present locally, removed it")
110-
except plumbum.ProcessExecutionError:
111-
pass
112-
113109
existing_platform_tags = pull_missing_tags(merged_tag, all_local_tags)
114110
if push_to_registry:
115-
push_manifest(merged_tag, existing_platform_tags)
111+
create_manifest(merged_tag, existing_platform_tags)
112+
push_manifest(merged_tag)
116113
else:
117114
LOGGER.info(f"Skipping push for tag: {merged_tag}")
118115

0 commit comments

Comments
 (0)