[Testing] Save pod desc text and stackdriver link as artifacts (#2791)

* Save pod desc text and stackdriver link as artifacts

* Fix artifact location

* Restructure artifact

* Also log pod yaml

test/check-argo-status.sh

@@ -16,7 +16,6 @@
 
 set -e
 
-ARTIFACT_DIR=$WORKSPACE/_artifacts
 WORKFLOW_COMPLETE_KEYWORD="completed=true"
 WORKFLOW_FAILED_KEYWORD="phase=Failed"
 PULL_ARGO_WORKFLOW_STATUS_MAX_ATTEMPT=$(expr $TIMEOUT_SECONDS / 20 )
@@ -70,8 +69,7 @@ fi
 echo "Argo workflow finished successfully."
 if [[ -n "$TEST_RESULT_FOLDER" ]]; then
   echo "Copy test result"
-  mkdir -p "$ARTIFACT_DIR"
-  gsutil cp -r "${TEST_RESULTS_GCS_DIR}"/* "${ARTIFACT_DIR}" || true
+  gsutil cp -r "${TEST_RESULTS_GCS_DIR}"/* "${ARTIFACTS}" || true
 fi
 
 echo "=========Main workflow=============="

test/deploy-cluster.sh

@@ -28,20 +28,25 @@ SHOULD_CLEANUP_CLUSTER=false
 function clean_up {
   set +e # the following clean up commands shouldn't exit on error
 
-  echo "Describe pods with unhealthy status:"
-  UNHEALTHY_PODS=($(kubectl get pods --field-selector=status.phase!=Running,status.phase!=Succeeded -o=custom-columns=:metadata.name -n $NAMESPACE))
-  for POD_NAME in "${UNHEALTHY_PODS[@]}"; do
-    echo ""
-    echo "For pod $POD_NAME:"
-    kubectl describe pod $POD_NAME -n $NAMESPACE
-  done
-  echo "============================================================="
-  echo "The above is output of describing pods with unhealthy status."
-  echo "============================================================="
-
   echo "Status of pods before clean up:"
   kubectl get pods --all-namespaces
 
+  echo "Dumping all pods info as artifacts..."
+  POD_INFO_DIR="$ARTIFACTS/pods_info"
+  mkdir -p "$POD_INFO_DIR"
+  # Refer to https://github.com/kubernetes/test-infra/blob/master/prow/jobs.md#job-environment-variables
+  ALL_PODS=($(kubectl get pods -o=custom-columns=:metadata.name -n $NAMESPACE))
+  for POD_NAME in "${ALL_PODS[@]}"; do
+    pod_info_file="$POD_INFO_DIR/$POD_NAME.txt"
+    echo "Pod name: $POD_NAME" >> "$pod_info_file"
+    echo "Detailed logs: https://console.cloud.google.com/logs/viewer?project=$PROJECT&advancedFilter=resource.type%3D%22k8s_container%22%0Aresource.labels.project_id%3D%22$PROJECT%22%0Aresource.labels.location%3D%22us-east1-b%22%0Aresource.labels.cluster_name%3D%22${TEST_CLUSTER}%22%0Aresource.labels.namespace_name%3D%22$NAMESPACE%22%0Aresource.labels.pod_name%3D%22$POD_NAME%22" \
+      >> "$pod_info_file"
+    echo "--------" >> "$pod_info_file"
+    kubectl describe pod $POD_NAME -n $NAMESPACE >> "$pod_info_file"
+    echo "--------" >> "$pod_info_file"
+    kubectl get pod $POD_NAME -n $NAMESPACE -o yaml >> "$pod_info_file"
+  done
+
   echo "Clean up..."
   if [ $SHOULD_CLEANUP_CLUSTER == true ]; then
     # --async doesn't wait for this operation to complete, so we can get test