MLE-21148: Add retry for group config

Author: Peng Zhou

charts/templates/configmap-scripts.yaml

@@ -9,8 +9,6 @@ metadata:
 data:
   copy-certs.sh: |
     #!/bin/bash
-    MARKLOGIC_ADMIN_USERNAME="$(< /run/secrets/ml-secrets/username)"            
-    MARKLOGIC_ADMIN_PASSWORD="$(< /run/secrets/ml-secrets/username)"
     log () {
         local TIMESTAMP=$(date +"%Y-%m-%d %T.%3N")
         echo "${TIMESTAMP}  $@"
@@ -176,6 +174,22 @@ data:
         echo $message >> /tmp/script.log
     }
 
+    retry() {
+        local retries=$1
+        shift
+        local count=0
+        until "$@"; do
+            exit_code=$?
+            count=$((count + 1))
+            if [ $count -ge $retries ]; then
+            echo "Command failed after $retries attempts."
+            return $exit_code
+            fi
+            echo "Attempt $count failed. Retrying..."
+            sleep 5
+        done
+    }
+
     ###############################################################
     # Function to get the current host protocol
     # $1: The host name
@@ -559,9 +573,11 @@ data:
                     info "group \"${current_group}\" updated and a restart of all hosts in the group was triggered"
                 else
                     info "unexpected response when updating group \"${current_group}\": ${response_code}"
+                    return 1
                 fi
             else
                 info "failed to get current group, response code: ${response_code}"
+                return 1
             fi
 
             if [[ "$MARKLOGIC_CLUSTER_TYPE" == "non-bootstrap" ]]; then
@@ -584,7 +600,7 @@ data:
         else
             info "not bootstrap host. Skip group configuration"
         fi
-
+        return 0
     }
 
     function configure_tls {
@@ -813,10 +829,10 @@ data:
        if [[ "${MARKLOGIC_CLUSTER_TYPE}" == "bootstrap" ]]; then
             log "Info:  bootstrap host is ready"
             init_security_db
-            configure_group
+            retry 5 configure_group
         else 
             log "Info:  bootstrap host is ready"
-            configure_group
+            retry 5 configure_group
             join_cluster $HOST_FQDN
         fi
         configure_path_based_routing