fix rook check to allow manual fix when is upgrading from only

addons/openebs/3.3.0/install.sh

@@ -65,7 +65,7 @@ function openebs_maybe_migrate_from_rook() {
 function openebs_maybe_rook_migration_checks() {
     logStep "Running Rook to OpenEBS migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook to OpenEBS. Rook Ceph is unhealthy."
     fi
 

addons/openebs/3.4.0/install.sh

@@ -65,7 +65,7 @@ function openebs_maybe_migrate_from_rook() {
 function openebs_maybe_rook_migration_checks() {
     logStep "Running Rook to OpenEBS migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook to OpenEBS. Rook Ceph is unhealthy."
     fi
 

addons/openebs/3.5.0/install.sh

@@ -65,7 +65,7 @@ function openebs_maybe_migrate_from_rook() {
 function openebs_maybe_rook_migration_checks() {
     logStep "Running Rook to OpenEBS migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook to OpenEBS. Rook Ceph is unhealthy."
     fi
 

addons/openebs/3.6.0/install.sh

@@ -68,7 +68,7 @@ function openebs_maybe_migrate_from_rook() {
 function openebs_maybe_rook_migration_checks() {
     logStep "Running Rook to OpenEBS migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook to OpenEBS. Rook Ceph is unhealthy."
     fi
 

addons/openebs/3.7.0/install.sh

@@ -68,7 +68,7 @@ function openebs_maybe_migrate_from_rook() {
 function openebs_maybe_rook_migration_checks() {
     logStep "Running Rook to OpenEBS migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook to OpenEBS. Rook Ceph is unhealthy."
     fi
 

addons/openebs/template/base/install.sh

@@ -68,7 +68,7 @@ function openebs_maybe_migrate_from_rook() {
 function openebs_maybe_rook_migration_checks() {
     logStep "Running Rook to OpenEBS migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook to OpenEBS. Rook Ceph is unhealthy."
     fi
 

scripts/common/object_store.sh

@@ -236,7 +236,7 @@ function migrate_between_object_stores() {
 function migrate_rgw_to_minio_checks() {
     logStep "Running Rook Ceph Object Store to Minio migration checks ..."
 
-    if ! rook_is_healthy_to_upgrade; then
+    if ! rook_is_healthy_to_migrate_from; then
         bail "Cannot upgrade from Rook Ceph Object Store to Minio. Rook Ceph is unhealthy."
     fi
 

scripts/common/rook.sh

@@ -368,6 +368,67 @@ function rook_operator_ready() {
     return 0
 }
 
+function rook_is_healthy_to_migrate_from() {
+    log "Awaiting 2 minutes to check Rook Ceph Pod(s) are Running"
+    if ! spinner_until 300 check_for_running_pods "rook-ceph"; then
+        logFail "Rook Ceph has unhealthy Pod(s)"
+        return 1
+    fi
+
+    log "Awaiting Rook Ceph health ..."
+    if ! $DIR/bin/kurl rook wait-for-health 600 ; then
+        logWarn "Rook Ceph is unhealthy"
+
+        output=$(kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph status)
+
+        echo ""
+        echo output
+        echo ""
+
+        if [[ $output == *"Module 'dashboard' has failed"* ]] || [[ $output == *"Module 'prometheus'"* ]]; then
+            echo "Disable modules to try fix status"
+            kubectl -n rook-ceph exec deployment/rook-ceph-tools -- ceph mgr module disable prometheus
+            kubectl -n rook-ceph exec deployment/rook-ceph-tools -- ceph mgr module disable dashboard
+        fi
+
+        log "Verify Rook Ceph health after fix"
+        if ! $DIR/bin/kurl rook wait-for-health 600; then
+            kubectl -n rook-ceph exec deploy/rook-ceph-tools -- ceph status
+            logFail "Rook Ceph is unhealthy"
+            return 1
+        fi
+        return 1
+    fi
+
+    log "Checking Rook Ceph versions and replicas"
+    kubectl -n rook-ceph get deployment -l rook_cluster=rook-ceph -o jsonpath='{range .items[*]}{.metadata.name}{"  \treq/upd/avl: "}{.spec.replicas}{"/"}{.status.updatedReplicas}{"/"}{.status.readyReplicas}{"  \trook-version="}{.metadata.labels.rook-version}{"\n"}{end}'
+    local rook_versions=
+    rook_versions="$(kubectl -n rook-ceph get deployment -l rook_cluster=rook-ceph -o jsonpath='{range .items[*]}{"rook-version="}{.metadata.labels.rook-version}{"\n"}{end}' | sort | uniq)"
+    if [ -n "${rook_versions}" ] && [ "$(echo "${rook_versions}" | wc -l)" -gt "1" ]; then
+        logFail "Multiple Rook versions detected"
+        logFail "${rook_versions}"
+        return 1
+    fi
+
+    log "Checking Ceph versions and replicas"
+    kubectl -n rook-ceph get deployment -l rook_cluster=rook-ceph -o jsonpath='{range .items[*]}{.metadata.name}{"  \treq/upd/avl: "}{.spec.replicas}{"/"}{.status.updatedReplicas}{"/"}{.status.readyReplicas}{"  \tceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}'
+    local ceph_versions_found=
+    ceph_versions_found="$(kubectl -n rook-ceph get deployment -l rook_cluster=rook-ceph -o jsonpath='{range .items[*]}{"ceph-version="}{.metadata.labels.ceph-version}{"\n"}{end}' | sort | uniq)"
+    if [ -n "${ceph_versions_found}" ] && [ "$(echo "${ceph_versions_found}" | wc -l)" -gt "1" ]; then
+        # It is required because an Rook Ceph bug which was sorted out with the release 1.4.8
+        # More info: https://github.com/rook/rook/pull/6610
+        if [ "$(echo "${ceph_versions_found}" | wc -l)" == "2" ] && [ "$(echo "${ceph_versions_found}" | grep "0.0.0-0")" ]; then
+            log "Found two ceph versions but one of them is 0.0.0-0 which will be ignored"
+            echo "${ceph_versions_found}"
+        else
+            logFail "Multiple Ceph versions detected"
+            logFail "${ceph_versions_found}"
+            return 1
+        fi
+    fi
+    return 0
+}
+
 function rook_is_healthy_to_upgrade() {
     log "Awaiting 2 minutes to check Rook Ceph Pod(s) are Running"
     if ! spinner_until 120 check_for_running_pods "rook-ceph"; then