chart(add): Grid scaler use trigger auth to secure GraphQL endpoint

Base/check-grid.sh

@@ -5,6 +5,7 @@ set -e
 
 HOST="localhost"
 PORT="4444"
+BASIC_AUTH="$(echo -n "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64)"
 
 echoerr() { echo "$@" 1>&2; }
 
@@ -26,4 +27,4 @@ while [[ $# -gt 0 ]]; do
   esac
 done
 
-curl -skSL ${SE_SERVER_PROTOCOL:-"http"}://${HOST}:${PORT}/wd/hub/status | jq -r '.value.ready' | grep -q "true" || exit 1
+curl -skSL -H "Authorization: Basic ${BASIC_AUTH}" ${SE_SERVER_PROTOCOL:-"http"}://${HOST}:${PORT}/wd/hub/status | jq -r '.value.ready' | grep -q "true" || exit 1

Video/video.sh

@@ -203,8 +203,15 @@ else
     if [[ "$session_id" != "null" && "$session_id" != "" && "$session_id" != "reserved" && "$recording_started" = "false" ]]; then
       echo "$(date +%FT%T%Z) [${process_name}] - Session: $session_id is created"
       return_list=($(bash ${VIDEO_CONFIG_DIRECTORY}/video_graphQLQuery.sh "$session_id"))
-      caps_se_video_record=${return_list[0]}
+      caps_se_video_record="${return_list[0]}"
       video_file_name="${return_list[1]}.mp4"
+      endpoint_status="${return_list[2]}"
+      endpoint_url="${return_list[3]}"
+      if [[ "${endpoint_status}" = "401" ]]; then
+        echo "$(date +%FT%T%Z) [${process_name}] - GraphQL endpoint requires authentication, please set env variables SE_ROUTER_USERNAME and SE_ROUTER_PASSWORD"
+      elif [[ "${endpoint_status}" = "404" ]]; then
+        echo "$(date +%FT%T%Z) [${process_name}] -GraphQL endpoint could not be found, please check the endpoint ${endpoint_url}"
+      fi
       echo "$(date +%FT%T%Z) [${process_name}] - Start recording: $caps_se_video_record, video file name: $video_file_name"
       log_node_response
     fi

Video/video_graphQLQuery.sh

@@ -14,6 +14,8 @@ if [[ -n ${GRAPHQL_ENDPOINT} ]] && [[ ! ${GRAPHQL_ENDPOINT} == */graphql ]]; the
   GRAPHQL_ENDPOINT="${GRAPHQL_ENDPOINT}/graphql"
 fi
 
+BASIC_AUTH="$(echo -n "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64)"
+
 VIDEO_CAP_NAME=${VIDEO_CAP_NAME:-"se:recordVideo"}
 TEST_NAME_CAP=${TEST_NAME_CAP:-"se:name"}
 VIDEO_NAME_CAP=${VIDEO_NAME_CAP:-"se:videoName"}
@@ -27,12 +29,15 @@ if [ -n "${GRAPHQL_ENDPOINT}" ]; then
     # Send GraphQL query
     endpoint_checks=$(curl --noproxy "*" -m ${max_time} -k -X POST \
       -H "Content-Type: application/json" \
+      -H "Authorization: Basic ${BASIC_AUTH}" \
       --data '{"query":"{ session (id: \"'${SESSION_ID}'\") { id, capabilities, startTime, uri, nodeId, nodeUri, sessionDurationMillis, slot { id, stereotype, lastStarted } } } "}' \
       -s "${GRAPHQL_ENDPOINT}" -o "/tmp/graphQL_${SESSION_ID}.json" -w "%{http_code}")
     current_check=$((current_check + 1))
     # Check if the response contains "capabilities"
     if [[ "$endpoint_checks" = "404" ]] || [[ $current_check -eq $retry_time ]]; then
       break
+    elif [[ "$endpoint_checks" = "401" ]] || [[ $current_check -eq $retry_time ]]; then
+      break
     elif [[ "$endpoint_checks" = "200" ]] && [[ $(jq -e '.data.session.capabilities | fromjson | ."'se:vncEnabled'"' /tmp/graphQL_${SESSION_ID}.json >/dev/null) -eq 0 ]]; then
       break
     fi
@@ -73,7 +78,7 @@ fi
 # Normalize the video file name
 TEST_NAME="$(echo "${TEST_NAME}" | tr ' ' '_' | tr -dc "${VIDEO_FILE_NAME_TRIM}" | cut -c 1-251)"
 
-return_array=("${RECORD_VIDEO}" "${TEST_NAME}")
+return_array=("${RECORD_VIDEO}" "${TEST_NAME}" "${endpoint_checks}" "${GRAPHQL_ENDPOINT}")
 
 # stdout the values for other scripts consuming
 echo "${return_array[@]}"

Video/video_gridUrl.sh

@@ -2,18 +2,15 @@
 
 max_time=3
 
-if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then
-  BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@"
-fi
 if [ "${SE_SUB_PATH}" = "/" ]; then
   SE_SUB_PATH=""
 fi
 
 grid_url="${SE_NODE_GRID_URL}"
 if [ -n "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] && [ -n "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then
-  grid_url=${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
+  grid_url=${SE_SERVER_PROTOCOL}://${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
 elif [ -n "${DISPLAY_CONTAINER_NAME}" ] && [ "${SE_VIDEO_RECORD_STANDALONE}" = "true" ]; then
-  grid_url="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT:-4444}${SE_SUB_PATH}" # For standalone mode
+  grid_url="${SE_SERVER_PROTOCOL}://${DISPLAY_CONTAINER_NAME}:${SE_NODE_PORT:-4444}${SE_SUB_PATH}" # For standalone mode
 fi
 
 if [[ ${grid_url} == */ ]]; then

charts/selenium-grid/CONFIGURATION.md

@@ -66,9 +66,12 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | tls.disableHostnameVerification | bool | `true` | Disable verification the hostname included in the server's TLS/SSL certificates matches the hostnames provided |
 | registrationSecret.enabled | bool | `false` | Enable feature node registration secret to make sure that the node is one you control and not a rouge node |
 | registrationSecret.value | string | `"HappyTesting"` | The secret value to be used for node registration |
+| basicAuth.nameOverride | string | `""` | External secret containing the basic auth username and password for reference |
 | basicAuth.enabled | bool | `false` | Enable or disable basic auth for the Hub/Router |
 | basicAuth.username | string | `"admin"` | Username for basic auth |
 | basicAuth.password | string | `"admin"` | Password for basic auth |
+| basicAuth.embeddedUrl | bool | `false` | Embed the basic auth "username:password@" in few URLs e.g. SE_NODE_GRID_URL |
+| basicAuth.annotations | object | `{}` | Annotations for basic auth secret resource |
 | isolateComponents | bool | `false` | Deploy Router, Distributor, EventBus, SessionMap and Nodes separately |
 | serviceAccount.create | bool | `true` | Create a service account for all components |
 | serviceAccount.nameOverride | string | `nil` | Override to use an external service account |
@@ -300,6 +303,7 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | autoscaling.enabled | bool | `false` | Enable autoscaling. Implies installing KEDA |
 | autoscaling.enableWithExistingKEDA | bool | `false` | Enable autoscaling without automatically installing KEDA |
 | autoscaling.scalingType | string | `"job"` | Which type of KEDA scaling to use: job or deployment |
+| autoscaling.authenticationRef | object | `{"name":""}` | Specify an external KEDA TriggerAuthentication resource is used for scaler triggers config. Apply for all browser nodes |
 | autoscaling.annotations | object | `{"helm.sh/hook":"post-install,post-upgrade,post-rollback","helm.sh/hook-weight":"1"}` | Annotations for KEDA resources: ScaledObject and ScaledJob |
 | autoscaling.patchObjectFinalizers.nameOverride | string | `nil` | Override the name of the patch job |
 | autoscaling.patchObjectFinalizers.enabled | bool | `true` | Enable patching finalizers for KEDA scaled resources. Workaround for Hook post-upgrade selenium-grid/templates/x-node-hpa.yaml failed: object is being deleted: scaledobjects.keda.sh "x" already exists |
@@ -361,7 +365,6 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | chromeNode.scaledOptions | string | `nil` | Override the scaled options for chrome nodes |
 | chromeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for chrome nodes |
 | chromeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for chrome nodes |
-| chromeNode.hpa.url | string | `"{{ template \"seleniumGrid.graphqlURL\" . }}"` | Graphql endpoint for the HPA to fetch metrics |
 | chromeNode.hpa.browserName | string | `"chrome"` | browserName from the capability |
 | chromeNode.hpa.sessionBrowserName | string | `"chrome"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
 | chromeNode.hpa.platformName | string | `"linux"` | platformName from the capability |
@@ -411,7 +414,6 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | firefoxNode.scaledOptions | string | `nil` | Override the scaled options for firefox nodes |
 | firefoxNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for firefox nodes |
 | firefoxNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for firefox nodes |
-| firefoxNode.hpa.url | string | `"{{ template \"seleniumGrid.graphqlURL\" . }}"` | Graphql endpoint for the HPA to fetch metrics |
 | firefoxNode.hpa.browserName | string | `"firefox"` | browserName from the capability |
 | firefoxNode.hpa.sessionBrowserName | string | `"firefox"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
 | firefoxNode.hpa.platformName | string | `"linux"` | platformName from the capability |
@@ -461,7 +463,6 @@ A Helm chart for creating a Selenium Grid Server in Kubernetes
 | edgeNode.scaledOptions | string | `nil` | Override the scaled options for edge nodes |
 | edgeNode.scaledJobOptions | string | `nil` | Override the scaledJobOptions for edge nodes |
 | edgeNode.scaledObjectOptions | string | `nil` | Override the scaledObjectOptions for edge nodes |
-| edgeNode.hpa.url | string | `"{{ template \"seleniumGrid.graphqlURL\" . }}"` | Graphql endpoint for the HPA to fetch metrics |
 | edgeNode.hpa.browserName | string | `"MicrosoftEdge"` | browserName from the capability |
 | edgeNode.hpa.sessionBrowserName | string | `"msedge"` | sessionBrowserName if the browserName is different from the sessionBrowserName |
 | edgeNode.hpa.platformName | string | `"linux"` | platformName from the capability |

charts/selenium-grid/README.md

@@ -11,6 +11,7 @@ This chart enables the creation of a Selenium Grid Server in Kubernetes.
     * [Installing the Nightly chart](#installing-the-nightly-chart)
     * [Chart Release Name convention](#chart-release-name-convention)
   * [Enable Selenium Grid Autoscaling](#enable-selenium-grid-autoscaling)
+    * [Preview new changes in Selenium Grid Scaler implementation](#preview-new-changes-in-selenium-grid-scaler-implementation)
     * [Settings common for both `job` and `deployment` scalingType](#settings-common-for-both-job-and-deployment-scalingtype)
     * [Settings when scalingType with `deployment`](#settings-when-scalingtype-with-deployment-)
     * [Settings when scalingType with `job`](#settings-when-scalingtype-with-job)
@@ -127,7 +128,9 @@ or [jobs](https://keda.sh/docs/latest/concepts/scaling-jobs/) and the charts sup
 chart support both modes.  It is controlled with `autoscaling.scalingType` that can be set to either
 job (default) or deployment.
 
-Preview new changes in Selenium Grid scaler implementation. Refer to [README](../../.keda/README.md)
+### Preview new changes in Selenium Grid Scaler implementation
+
+Refer to [README](../../.keda/README.md)
 
 ### Settings common for both `job` and `deployment` scalingType
 
@@ -338,8 +341,13 @@ hub:
 ```
 # Source: selenium-grid/templates/node-configmap.yaml
 
-SE_NODE_GRID_URL: 'http://admin:admin@10.10.10.10/selenium'
+SE_NODE_GRID_URL: 'http://10.10.10.10/selenium'
 ```
+
+For security reasons, it is not recommend to put the credentials in the URL in env variable `SE_NODE_GRID_URL`. For any utilities that need to access the Grid, basic auth should get from env variables `SE_ROUTER_USERNAME` and `SE_ROUTER_PASSWORD`.
+
+If you want to keep basic auth credential is embedded in few URLs (in case Websocket URLs construct in session capabilities), you can set `basicAuth.embeddedUrl` to `true`. By default, it is `false`.
+
 Besides that, from the outside of the cluster, you can access via NodePort `http://10.10.10.10:30444/selenium`
 
 ### Configuration of Nodes

charts/selenium-grid/configs/distributor/distributorProbe.sh

@@ -7,27 +7,28 @@ ts_format=${SE_LOG_TIMESTAMP_FORMAT:-"+%T.%3N"}
 ROUTER_CONFIG_DIRECTORY=${ROUTER_CONFIG_DIRECTORY:-"/opt/bin"}
 
 GRID_GRAPHQL_URL=$(bash ${ROUTER_CONFIG_DIRECTORY}/routerGraphQLUrl.sh)
+BASIC_AUTH="$(echo -n "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64)"
 
 if [ -z "${GRID_GRAPHQL_URL}" ]; then
   echo "$(date ${ts_format}) DEBUG [${probe_name}] - Could not construct GraphQL endpoint, please provide SE_HUB_HOST (or SE_ROUTER_HOST) and SE_HUB_PORT (or SE_ROUTER_PORT). Bypass the probe checks for now."
   exit 0
 fi
 
-GRAPHQL_PRE_CHECK=$(curl --noproxy "*" -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { sessionCount } }"}' -s -o /dev/null -w "%{http_code}" ${GRID_GRAPHQL_URL})
+GRAPHQL_PRE_CHECK=$(curl --noproxy "*" -m ${max_time} -k -X POST -H "Authorization: Basic ${BASIC_AUTH}" -H "Content-Type: application/json" --data '{"query":"{ grid { sessionCount } }"}' -s -o /dev/null -w "%{http_code}" ${GRID_GRAPHQL_URL})
 
 if [ ${GRAPHQL_PRE_CHECK} -ne 200 ]; then
   echo "$(date ${ts_format}) DEBUG [${probe_name}] - GraphQL endpoint ${GRID_GRAPHQL_URL} is not reachable. Status code: ${GRAPHQL_PRE_CHECK}."
   exit 1
 fi
 
-SESSION_QUEUE_SIZE=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { sessionQueueSize } }"}' -s ${GRID_GRAPHQL_URL} | jq -r '.data.grid.sessionQueueSize')
+SESSION_QUEUE_SIZE=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Authorization: Basic ${BASIC_AUTH}" -H "Content-Type: application/json" --data '{"query":"{ grid { sessionQueueSize } }"}' -s ${GRID_GRAPHQL_URL} | jq -r '.data.grid.sessionQueueSize')
 
-SESSION_COUNT=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query": "{ grid { sessionCount } }"}' -s ${GRID_GRAPHQL_URL} | jq -r '.data.grid.sessionCount')
+SESSION_COUNT=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Authorization: Basic ${BASIC_AUTH}" -H "Content-Type: application/json" --data '{"query": "{ grid { sessionCount } }"}' -s ${GRID_GRAPHQL_URL} | jq -r '.data.grid.sessionCount')
 
-MAX_SESSION=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { maxSession } }"}' -s ${GRID_GRAPHQL_URL} | jq -r '.data.grid.maxSession')
+MAX_SESSION=$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Authorization: Basic ${BASIC_AUTH}" -H "Content-Type: application/json" --data '{"query":"{ grid { maxSession } }"}' -s ${GRID_GRAPHQL_URL} | jq -r '.data.grid.maxSession')
 
 if [ "${SE_LOG_LISTEN_GRAPHQL:-"false"}" = "true" ]; then
-  echo "$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Content-Type: application/json" --data '{"query":"{ grid { sessionCount, maxSession, totalSlots }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, slot { id, stereotype } } } }, sessionsInfo { sessionQueueRequests } }"}' -s ${GRID_GRAPHQL_URL} )"
+  echo "$(curl --noproxy "*" --retry ${retry_time} -m ${max_time} -k -X POST -H "Authorization: Basic ${BASIC_AUTH}" -H "Content-Type: application/json" --data '{"query":"{ grid { sessionCount, maxSession, totalSlots }, nodesInfo { nodes { id, status, sessionCount, maxSession, slotCount, stereotypes, sessions { id, capabilities, slot { id, stereotype } } } }, sessionsInfo { sessionQueueRequests } }"}' -s ${GRID_GRAPHQL_URL} )"
 fi
 
 if [ ${SESSION_QUEUE_SIZE} -gt 0 ] && [ ${SESSION_COUNT} -eq 0 ]; then

charts/selenium-grid/configs/node/nodeGridUrl.sh

@@ -2,33 +2,29 @@
 
 max_time=3
 
-if [ -n "${SE_ROUTER_USERNAME}" ] && [ -n "${SE_ROUTER_PASSWORD}" ]; then
-  BASIC_AUTH="${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}@"
-fi
+BASIC_AUTH="$(echo -n "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64)"
+
 if [ "${SE_SUB_PATH}" = "/" ]; then
   SE_SUB_PATH=""
 fi
 
 if [ -z "${SE_HUB_HOST:-$SE_ROUTER_HOST}" ] || [ -z "${SE_HUB_PORT:-$SE_ROUTER_PORT}" ]; then
   grid_url=""
 else
-  grid_url=${SE_SERVER_PROTOCOL}://${BASIC_AUTH}${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
+  grid_url=${SE_SERVER_PROTOCOL}://${SE_HUB_HOST:-$SE_ROUTER_HOST}:${SE_HUB_PORT:-$SE_ROUTER_PORT}${SE_SUB_PATH}
 fi
 
 if [ -z "${grid_url}" ]; then
   grid_url="${SE_NODE_GRID_URL}"
 fi
 
 if [ -z "${grid_url}" ]; then
-  grid_url="${SE_SERVER_PROTOCOL}://${BASIC_AUTH}127.0.0.1:4444${SE_SUB_PATH}" # For standalone mode
+  grid_url="${SE_SERVER_PROTOCOL}://127.0.0.1:4444${SE_SUB_PATH}" # For standalone mode
 fi
 
-grid_url_checks=$(curl --noproxy "*" -m ${max_time} -s -k -o /dev/null -w "%{http_code}" ${grid_url})
-if [ "${grid_url_checks}" = "401" ]; then
-  return ${grid_url_checks}
-fi
-if [ "${grid_url_checks}" = "404" ]; then
-  return ${grid_url_checks}
-fi
+grid_url_checks=$(curl --noproxy "*" -H "Authorization: Basic ${BASIC_AUTH}" -m ${max_time} -s -k -o /dev/null -w "%{http_code}" ${grid_url})
+
+return_array=("${grid_url}" "${grid_url_checks}")
 
-echo "${grid_url}"
+# stdout the values for other scripts consuming
+echo "${return_array[@]}"

charts/selenium-grid/configs/node/nodePreStop.sh

@@ -12,7 +12,7 @@ tmp_node_file="/tmp/nodeProbe${ID}"
 
 function on_exit() {
   rm -rf ${tmp_node_file}
-  echo "$(date +%FT%T%Z) [${probe_name}] - Exiting Node preStop..."
+  echo "$(date +%FT%T%Z) [${probe_name}] - Exiting Node ${probe_name}..."
   exit 0
 }
 trap on_exit EXIT
@@ -30,12 +30,20 @@ else
 fi
 
 function signal_hub_to_drain_node() {
-  grid_url=$(bash ${NODE_CONFIG_DIRECTORY}/nodeGridUrl.sh)
+  return_list=($(bash ${NODE_CONFIG_DIRECTORY}/nodeGridUrl.sh))
+  grid_url=${return_list[0]}
+  grid_check=${return_list[1]}
+  BASIC_AUTH="$(echo -n "${SE_ROUTER_USERNAME}:${SE_ROUTER_PASSWORD}" | base64)"
   if [ -n "${grid_url}" ]; then
+    if [ "${grid_check}" = "401" ]; then
+      echo "$(date +%FT%T%Z) [${probe_name}] - Hub/Router requires authentication. Please check SE_ROUTER_USERNAME and SE_ROUTER_PASSWORD."
+    elif [ "${grid_check}" = "404" ]; then
+      echo "$(date +%FT%T%Z) [${probe_name}] - Hub/Router endpoint could not be found. Please check the endpoint ${grid_url}"
+    fi
     echo "$(date +%FT%T%Z) [${probe_name}] - Signaling Hub/Router to drain node"
-    curl --noproxy "*" -m ${max_time} -k -X POST ${grid_url}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
+    curl --noproxy "*" -m ${max_time} -k -X POST -H "Authorization: Basic ${BASIC_AUTH}" ${grid_url}/se/grid/distributor/node/${NODE_ID}/drain --header "${HEADERS}"
   else
-    echo "$(date +%FT%T%Z) [${probe_name}] - There is no configured HUB/ROUTER host or SE_NODE_GRID_URL isn't set. preStop ignores to send drain request to upstream."
+    echo "$(date +%FT%T%Z) [${probe_name}] - There is no configured HUB/ROUTER host or SE_NODE_GRID_URL isn't set. ${probe_name} ignores to send drain request to upstream."
   fi
 }
 
@@ -66,7 +74,7 @@ if curl --noproxy "*" -m ${max_time} -sfk ${SE_SERVER_PROTOCOL}://127.0.0.1:${SE
         echo
         exit 0
       else
-        echo "$(date +%FT%T%Z) [${probe_name}] - Node preStop is waiting for current session on slot ${SLOT_HAS_SESSION} to be finished. Node details: message: $(jq -r '.value.message' ${tmp_node_file} || "unknown"), availability: $(jq -r '.value.node.availability' ${tmp_node_file} || "unknown")"
+        echo "$(date +%FT%T%Z) [${probe_name}] - Node ${probe_name} is waiting for current session on slot ${SLOT_HAS_SESSION} to be finished. Node details: message: $(jq -r '.value.message' ${tmp_node_file} || "unknown"), availability: $(jq -r '.value.node.availability' ${tmp_node_file} || "unknown")"
         sleep 2;
       fi