Add a retry loop around manifest-tool push to workaround notary int…

…ermittent failures (#5396)

The retry loop was introduced for `docker push` with #5040.

We still have some issues with `manifest-tool push` like:
```
Inspect of image "datadog/agent-amd64:7.19.0-rc.8-jmx-win1909" failed with error: manifest unknown: manifest unknown
```

Relaunching the job that produced that error was enough to get it fixed.
So, that was well an intermittent failure.

tasks/docker.py

@@ -16,6 +16,24 @@
 
 from .dogstatsd import DOGSTATSD_TAG
 
+def retry_run(ctx, *args, **kwargs):
+    remaining_retries = 5
+    while True:
+        warn = True
+        if remaining_retries == 0:
+            warn = False
+
+        r = ctx.run(*args, warn=warn, **kwargs)
+
+        if r.ok:
+            return r
+
+        # Pause between retries. Hope it helps.
+        time.sleep(5)
+
+        remaining_retries -= 1
+
+    return r
 
 @task
 def test(ctx):
@@ -142,26 +160,9 @@ def publish(ctx, src, dst, signed_pull=False, signed_push=False):
     push_env = {}
     if signed_push:
         push_env["DOCKER_CONTENT_TRUST"] = "1"
-    remaining_retries = 5
-    while True:
-        warn = True
-        if remaining_retries == 0:
-            warn = False
-
-        r = ctx.run(
-            "docker push {dst}".format(dst=dst),
-            env=push_env,
-            warn=warn
-        )
-
-        if r.ok:
-            break
-
-        # docker push might have failed because of a notary temporary error.
-        # Let give some time to the server to recover by itself before making a new attempt.
-        time.sleep(5)
-
-        remaining_retries -= 1
+    retry_run(ctx, "docker push {dst}".format(dst=dst),
+              env=push_env,
+    )
 
     ctx.run("docker rmi {src} {dst}".format(src=src, dst=dst))
 
@@ -227,7 +228,7 @@ def publish_manifest(ctx, name, tag, image, signed_push=False):
     ctx.run("cat {}".format(temp_file_path))
 
     try:
-        result = ctx.run("manifest-tool push from-spec {}".format(temp_file_path))
+        result = retry_run(ctx, "manifest-tool push from-spec {}".format(temp_file_path))
         if result.stdout:
             out = result.stdout.split('\n')[0]
             fields = out.split(" ")
@@ -251,7 +252,7 @@ def publish_manifest(ctx, name, tag, image, signed_push=False):
                 -p docker.io/{name} {tag} {length} --sha256 {sha256} \
                 -r targets/releases
             """
-            ctx.run(cmd.format(home=os.path.expanduser("~"), name=name, tag=tag, length=length, sha256=digest))
+            retry_run(ctx, cmd.format(home=os.path.expanduser("~"), name=name, tag=tag, length=length, sha256=digest))
     finally:
         os.remove(temp_file_path)