chore: check if custom login endpoint is reachable (#4780)

Azure · Dec 8, 2021 · 6af8e6d · 6af8e6d
1 parent 3abb929
commit 6af8e6d
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 55 deletions.
diff --git a/parts/k8s/cloud-init/artifacts/cse_customcloud.sh b/parts/k8s/cloud-init/artifacts/cse_customcloud.sh
@@ -83,6 +83,15 @@ configureK8sCustomCloud() {
   fi
   set -x
 
+  {{/* Log whether the custom login endpoint is reachable to simplify troubleshooting. */}}
+  {{/* CSE will finish successfully but kubelet will error out if not reachable. */}}
+  LOGIN_ENDPOINT=$(jq -r .activeDirectoryEndpoint /etc/kubernetes/azurestackcloud.json)
+  LOGIN_ENDPOINT=${LOGIN_ENDPOINT#'https://'}
+  LOGIN_ENDPOINT=${LOGIN_ENDPOINT%'/'}
+  timeout 10 nc -vz ${LOGIN_ENDPOINT} 443 \
+  && echo "login endpoint reachable: ${LOGIN_ENDPOINT}" \
+  || echo "error: login endpoint not reachable: ${LOGIN_ENDPOINT}"
+
   {{- if not IsAzureCNI}}
   # Decrease eth0 MTU to mitigate Azure Stack's NRP issue
   echo "iface eth0 inet dhcp" | sudo tee -a /etc/network/interfaces
@@ -131,33 +140,19 @@ ensureAzureStackCertificates() {
 }
 
 configureAzureStackInterfaces() {
-  set +x
-
   NETWORK_INTERFACES_FILE="/etc/kubernetes/network_interfaces.json"
   AZURE_CNI_CONFIG_FILE="/etc/kubernetes/interfaces.json"
   AZURESTACK_ENVIRONMENT_JSON_PATH="/etc/kubernetes/azurestackcloud.json"
   SERVICE_MANAGEMENT_ENDPOINT=$(jq -r '.serviceManagementEndpoint' ${AZURESTACK_ENVIRONMENT_JSON_PATH})
   ACTIVE_DIRECTORY_ENDPOINT=$(jq -r '.activeDirectoryEndpoint' ${AZURESTACK_ENVIRONMENT_JSON_PATH})
   RESOURCE_MANAGER_ENDPOINT=$(jq -r '.resourceManagerEndpoint' ${AZURESTACK_ENVIRONMENT_JSON_PATH})
+  TOKEN_URL="${ACTIVE_DIRECTORY_ENDPOINT}${TENANT_ID}/oauth2/token"
 
   if [[ ${IDENTITY_SYSTEM,,} == "adfs" ]]; then
     TOKEN_URL="${ACTIVE_DIRECTORY_ENDPOINT}adfs/oauth2/token"
-  else
-    TOKEN_URL="${ACTIVE_DIRECTORY_ENDPOINT}${TENANT_ID}/oauth2/token"
   fi
 
-  echo "Generating token for Azure Resource Manager"
-  echo "------------------------------------------------------------------------"
-  echo "Parameters"
-  echo "------------------------------------------------------------------------"
-  echo "SERVICE_PRINCIPAL_CLIENT_ID:     ..."
-  echo "SERVICE_PRINCIPAL_CLIENT_SECRET: ..."
-  echo "SERVICE_MANAGEMENT_ENDPOINT:     $SERVICE_MANAGEMENT_ENDPOINT"
-  echo "ACTIVE_DIRECTORY_ENDPOINT:       $ACTIVE_DIRECTORY_ENDPOINT"
-  echo "TENANT_ID:                       $TENANT_ID"
-  echo "IDENTITY_SYSTEM:                 $IDENTITY_SYSTEM"
-  echo "TOKEN_URL:                       $TOKEN_URL"
-  echo "------------------------------------------------------------------------"
+  set +x
 
   TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \
     -H "Content-Type: application/x-www-form-urlencoded" \
@@ -172,16 +167,6 @@ configureAzureStackInterfaces() {
     exit 120
   fi
 
-  echo "Fetching network interface configuration for node"
-  echo "------------------------------------------------------------------------"
-  echo "Parameters"
-  echo "------------------------------------------------------------------------"
-  echo "RESOURCE_MANAGER_ENDPOINT: $RESOURCE_MANAGER_ENDPOINT"
-  echo "SUBSCRIPTION_ID:           $SUBSCRIPTION_ID"
-  echo "RESOURCE_GROUP:            $RESOURCE_GROUP"
-  echo "NETWORK_API_VERSION:       $NETWORK_API_VERSION"
-  echo "------------------------------------------------------------------------"
-
   curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X GET \
     -H "Authorization: Bearer $TOKEN" \
     -H "Content-Type: application/json" \

diff --git a/pkg/engine/templates_generated.go b/pkg/engine/templates_generated.go
diff --git a/scripts/collect-logs.sh b/scripts/collect-logs.sh
@@ -1,11 +1,14 @@
 #!/bin/bash
 
+set -o pipefail
 source /opt/azure/containers/provision_source.sh
 
 clusterInfo() {
-    FIRST_MASTER_READY=$(kubectl get nodes | grep k8s-master | grep Ready | sort | head -n 1 | cut -d ' ' -f 1)
-    if [[ "${FIRST_MASTER_READY}" == "${HOSTNAME}" ]]; then
-        retrycmd_no_stats 3 5 120 kubectl cluster-info dump --namespace=kube-system --output-directory=${OUTDIR}/cluster-info
+    if [ -f .kube/config ]; then
+        FIRST_MASTER_READY=$(retrycmd_no_stats 3 3 10 kubectl get nodes -l node-role.kubernetes.io/master | grep Ready | sort | head -n 1 | cut -d ' ' -f 1)
+        if [[ "${FIRST_MASTER_READY}" == "${HOSTNAME}" ]]; then
+            retrycmd_no_stats 3 5 300 kubectl cluster-info dump --namespace=kube-system --output-directory=${OUTDIR}/cluster-info
+        fi
     fi
 }
 
@@ -185,6 +188,16 @@ stackfyNetwork() {
     mkdir -p ${DIR}
     # basic name resolution test
     ping ${HOSTNAME} -c 3 &> ${DIR}/k8s-ping.txt
+
+    if [ -f /etc/kubernetes/azurestackcloud.json ]; then
+        LOGIN_ENDPOINT=$(jq -r .activeDirectoryEndpoint /etc/kubernetes/azurestackcloud.json)
+        LOGIN_ENDPOINT=${LOGIN_ENDPOINT#'https://'}
+        LOGIN_ENDPOINT=${LOGIN_ENDPOINT%'/'}
+        LOGIN_ENDPOINT_LOG=${DIR}/k8s-loginEndpoint.txt
+        timeout 10 nc -vz ${LOGIN_ENDPOINT} 443 \
+        && echo "login endpoint reachable: ${LOGIN_ENDPOINT}" &> ${LOGIN_ENDPOINT_LOG} \
+        || echo "error: login endpoint not reachable: ${LOGIN_ENDPOINT}" &> ${LOGIN_ENDPOINT_LOG}
+    fi
 }
 
 stackfyFileNames() {