Delete Prometheus PVC prior to cluster deletion to delete provisioned disk

Author: Miguel Varela Ramos

cli/cmd/cluster_gcp.go

@@ -263,6 +263,11 @@ var _clusterGCPDownCmd = &cobra.Command{
 
 		fmt.Print("￮ spinning down the cluster ")
 
+		_, _, err = runGCPManagerAccessCommand("/root/uninstall.sh", *accessConfig, nil, nil)
+		if err != nil {
+			exit.Error(err)
+		}
+
 		_, err = gcpClient.DeleteCluster(gkeClusterName)
 		if err != nil {
 			fmt.Print("\n\n")

manager/uninstall.sh

@@ -18,8 +18,36 @@ set -e
 
 EKSCTL_TIMEOUT=45m
 
-echo
+function main() {
+  if [ "$CORTEX_PROVIDER" == "aws" ]; then
+    uninstall_aws
+  elif [ "$CORTEX_PROVIDER" == "gcp" ]; then
+    uninstall_gcp
+  fi
+}
 
-eksctl delete cluster --wait --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --timeout=$EKSCTL_TIMEOUT
+function uninstall_gcp() {
+  gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS 2> /dev/stdout 1> /dev/null | (grep -v "Activated service account credentials" || true)
+  gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE 2> /dev/stdout 1> /dev/null | (grep -v "Fetching cluster" | grep -v "kubeconfig entry generated" || true)
 
-echo -e "\n✓ done spinning down the cluster"
+  uninstall_prometheus
+}
+
+function uninstall_aws() {
+  echo
+
+  uninstall_prometheus
+
+  eksctl delete cluster --wait --name=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION --timeout=$EKSCTL_TIMEOUT
+  echo -e "\n✓ done spinning down the cluster"
+}
+
+function uninstall_prometheus() {
+  kubectl get configmap cluster-config -o jsonpath='{.data.cluster\.yaml}' > ./cluster.yaml
+
+  # delete resources to detach disk
+  python render_template.py ./cluster.yaml manifests/prometheus-monitoring.yaml.j2 | kubectl delete -f - >/dev/null
+  kubectl delete pvc prometheus-prometheus-db-prometheus-prometheus-0 >/dev/null
+}
+
+main