ci: retry testmo requests (#9727)

Author: nikitka

.github/actions/test_ya/action.yml

@@ -101,7 +101,9 @@ runs:
       shell: bash
       run: |
         set -x
-
+        
+        echo "$(pwd)/ydb/ci/scripts" >> $GITHUB_PATH
+        
         export TMP_DIR=$(pwd)/tmp
         rm -rf $TMP_DIR
         mkdir -p $TMP_DIR
@@ -356,7 +358,7 @@ runs:
             TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-link --name build --url "$TESTMO_RUN_URL" --resources $CURRENT_PUBLIC_DIR/testmo.json
             TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:resources:add-field --name git-sha --type string --value "${GITHUB_SHA:0:7}" --resources $CURRENT_PUBLIC_DIR/testmo.json
             TESTMO_RUN_ID=$(
-              TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
+              TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:create --instance "https://$TESTMO_PROXY_ADDR" --project-id ${{ inputs.testman_project_id }} \
                 --name "$TESTMO_RUN_NAME" --source "$TESTMO_SOURCE" --resources $CURRENT_PUBLIC_DIR/testmo.json \
                 --tags "$TESTMO_BRANCH_TAG" --tags "$TESTMO_EXTRA_TAG"
             )
@@ -471,11 +473,11 @@ runs:
             # archive unitest reports (transformed)
             tar -C $TESTMO_JUNIT_REPORT_PARTS/.. -czf $PUBLIC_DIR/junit_parts.xml.tar.gz $(basename $TESTMO_JUNIT_REPORT_PARTS)
 
-            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:submit-thread \
+            TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:submit-thread \
               --instance "https://$TESTMO_PROXY_ADDR" --run-id "$TESTMO_RUN_ID" \
               --results "$TESTMO_JUNIT_REPORT_PARTS/*.xml"
 
-            TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
+            TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id $TESTMO_RUN_ID || true
             echo "runid=" >> $GITHUB_OUTPUT
           fi
 
@@ -512,7 +514,7 @@ runs:
       shell: bash
       run: |
         if [ ${{ steps.build.outputs.runid }} ]; then
-          TESTMO_TOKEN=${{ inputs.testman_token }} testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
+          TESTMO_TOKEN=${{ inputs.testman_token }} retry.sh -- testmo automation:run:complete --instance "https://$TESTMO_PROXY_ADDR" --run-id ${{ steps.build.outputs.runid }} || true
         fi
         if [ ${{ inputs.testman_token }} ]; then
           kill $TESTMO_PROXY_PID

ydb/ci/scripts/retry.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+usage() {
+  echo "$0 -r [max_attempts] -t [timeout] -s [sleep] -- cmd"
+}
+if [ $# -lt 2 ]; then
+  usage
+  exit 1
+fi
+
+max_attempts=0
+timeout=1800
+sleep_time=30
+
+while getopts 'r:t:s:' opt; do
+  case "$opt" in
+    r)
+      max_attempts=$OPTARG
+      ;;
+    t)
+      timeout=$OPTARG
+      ;;
+    s)
+      sleep_time=$OPTARG
+      ;;
+    *)
+      usage
+      exit 1
+      ;;
+  esac
+done
+
+
+shift $((OPTIND - 2))
+
+if [[ "$1" == "--" ]]; then
+  shift  # Shift past the double dash
+else
+  echo "Error: Missing -- before command"
+  exit 1
+fi
+
+attempt_num=1
+start_time=$(date +%s)
+
+while true; do
+  elapsed=$(($(date +%s) - start_time))
+
+  if [ "$max_attempts" -ne 0 ] && [ "$attempt_num" -ge "$max_attempts" ]; then
+    echo "maximum attempts reached, exit" >&2
+    exit 10
+  fi
+
+  if [ "$timeout" -ne 0 ] && [ $elapsed -ge "$timeout" ]; then
+    echo "timeout reached, exit" >&2
+    exit 11
+  fi
+
+  if "$@"; then
+    exit
+  else
+    attempt_num=$(( attempt_num + 1 ))
+    if [ "$sleep_time" != "0" ]; then
+      sleep "$sleep_time"
+    fi
+  fi
+done

ydb/ci/testmo-proxy/testmo-proxy.py

@@ -15,10 +15,10 @@ class Handler(http.server.BaseHTTPRequestHandler):
     error_message_format = '{"status":"error", "code":"%(code)d", "message": "%(message)s"}\n'
 
     # noinspection PyMissingConstructor
-    def __init__(self, target_url: str, timeout: Tuple[int, int], max_reqest_time: int):
+    def __init__(self, target_url: str, timeout: Tuple[int, int], max_request_time: int):
         self._target_url = target_url
         self._timeout = timeout
-        self._max_request_time = max_reqest_time
+        self._max_request_time = max_request_time
 
     def __call__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)