feat: Call multi-node storage migration from kURL installer scripts (#…

…4524) * feat: create function to migrate from openebs to rook this commit creates the function rook_maybe_migrate_from_openebs. this function may migrate data from openebs to rook when all the following conditions are met: - ekco version is >= 0.27.1. - openebs and rook are selected on the installer. - rook's minimum node count is set to a value > 1. - the number of nodes on the cluster is >= than the rook minimum node count. - the 'scaling' storage class exists. * use ekco global envars also new join.sh script * add migrate-to-multinode-storage task * Check for EKCO add-on version >= 0.27.1 * wip * use confirmN * implement and use rook_maybe_migrate_from_openebs_primary * Update scripts/tasks.sh Co-authored-by: Rafael Polanco <6497491+rrpolanco@users.noreply.github.com> * install.sh already has env vars set * fixes * do not require spec versions for tasks.sh * Update scripts/common/rook.sh Co-authored-by: Rafael Polanco <6497491+rrpolanco@users.noreply.github.com> * Update scripts/tasks.sh Co-authored-by: Rafael Polanco <6497491+rrpolanco@users.noreply.github.com> --------- Co-authored-by: Rafael R. Polanco <6497491+rrpolanco@users.noreply.github.com> Co-authored-by: Andrew Lavery <laverya@umich.edu>
replicatedhq · Jun 29, 2023 · b2b4b10 · b2b4b10
1 parent efd0032
commit b2b4b10
Show file tree

Hide file tree

Showing 7 changed files with 163 additions and 8 deletions.
diff --git a/kurl_util/cmd/bashmerge/main.go b/kurl_util/cmd/bashmerge/main.go
@@ -302,6 +302,10 @@ func parseBashFlags(installer *kurlv1beta1.Installer, bashFlags string) error {
 				installer.Spec.Velero = &kurlv1beta1.Velero{}
 			}
 			installer.Spec.Velero.ServerFlags = append(installer.Spec.Velero.ServerFlags, flags...)
+		case "ekco-address":
+			continue
+		case "ekco-auth-token":
+			continue
 		default:
 			return fmt.Errorf("string %s is not a bash flag", split[0])
 		}

diff --git a/scripts/common/common.sh b/scripts/common/common.sh
@@ -1536,3 +1536,36 @@ function node_is_using_docker() {
     node="$(get_local_node_name)"
     kubectl get node "$node" -ojsonpath='{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io/cri-socket}' | grep -q "dockershim.sock"
 }
+
+# get_ekco_addr prints the host address (including port) for reaching the EKCO service to stdout
+function get_ekco_addr() {
+    if [ -n "$EKCO_ADDRESS" ]; then
+        echo "$EKCO_ADDRESS"
+        return
+    fi
+
+    local ekco_addr=
+    local ekco_port=
+    local current_node_ip=
+    current_node_ip=$(kubectl get nodes -o wide | grep "$(get_local_node_name)" | awk '{print $6}')
+    ekco_port="${EKCO_NODE_PORT}"
+
+    if [ -z "${ekco_port}" ]; then
+        ekco_port=$(kubectl get svc ekc-operator -n kurl -o jsonpath='{.spec.ports[?(@.nodePort)].nodePort}')
+    fi
+    ekco_addr="${current_node_ip}:${ekco_port}"
+    echo "$ekco_addr"
+}
+
+# get_ekco_storage_migration_auth_token prints the ekco storage migration authentication token to stdout
+function get_ekco_storage_migration_auth_token() {
+    if [ -n "$EKCO_AUTH_TOKEN" ]; then
+        echo "$EKCO_AUTH_TOKEN"
+        return
+    fi
+
+    local auth_token=
+    auth_token=$(kubectl get cm -n kurl ekco-config -ojsonpath='{.data.config\.yaml}' | grep "storage_migration_auth_token:" | awk '{print $2}')
+
+    echo "$auth_token"
+}
diff --git a/scripts/common/rook.sh b/scripts/common/rook.sh
@@ -514,3 +514,86 @@ function add_rook_store_object_migration_status() {
     kubectl patch configmap kurl-migration-from-rook -n kurl --type merge -p '{"data":{"DID_MIGRATE_ROOK_OBJECT_STORE":"1"}}'
     export DID_MIGRATE_ROOK_OBJECT_STORE=1
 }
+
+# rook_maybe_migrate_from_openebs may migrate data from OpenEBS to Rook when all the following
+# conditions are met:
+# - Ekco version is >= 0.27.1.
+# - OpenEBS and Rook are selected on the Installer.
+# - Rook's minimum node count is set to a value > 1.
+# - The number of nodes on the cluster is >= than the Rook minimum node count.
+# - The 'scaling' storage class exists.
+function rook_maybe_migrate_from_openebs() {
+    semverCompare "$EKCO_VERSION" "0.27.1"
+    if [ "$SEMVER_COMPARE_RESULT" -lt "0" ]; then
+        return 0
+    fi
+    if [ -z "$ROOK_VERSION" ] || [ -z "$OPENEBS_VERSION" ]; then
+        return 0
+    fi
+    if [ -z "$ROOK_MINIMUM_NODE_COUNT" ] || [ "$ROOK_MINIMUM_NODE_COUNT" -le "1" ]; then
+        return 0
+    fi
+    rook_maybe_migrate_from_openebs_internal
+}
+
+# rook_maybe_migrate_from_openebs_internal SHOULD NOT BE CALLED DIRECTLY.
+# it is called by rook_maybe_migrate_from_openebs and rook_maybe_migrate_from_openebs_tasks when all the conditions are met.
+# it will check that the required environment variables (EKCO_AUTH_TOKEN and EKCO_ADDRESS) are set and then
+# check EKCO to see if the migration is available. If it is, it will prompt the user to start it.
+function rook_maybe_migrate_from_openebs_internal() {
+    if [ -z "$EKCO_AUTH_TOKEN" ]; then
+        logFail "Internal Error: an authentication token is required to start the OpenEBS to Rook multi-node migration."
+        return 0
+    fi
+
+    if [ -z "$EKCO_ADDRESS" ]; then
+        logFail "Internal Error: unable to determine network address of the kURL operator."
+        return 0
+    fi
+
+    # are both rook and openebs installed, not just specified?
+    if ! kubectl get ns | grep -q rook-ceph && ! kubectl get ns | grep -q openebs; then
+        bail "Rook and OpenEBS must be installed in order to migrate to multi-node storage"
+    fi
+
+    # check if OpenEBS to Rook multi-node migration is available - if it is, prompt the user to start it
+    if "${DIR}"/bin/kurl cluster migrate-multinode-storage --ekco-address "$EKCO_ADDRESS" --ekco-auth-token "$EKCO_AUTH_TOKEN" --check-status; then
+        printf "    The installer detected both OpenEBS and Rook installations in your cluster. Migration from OpenEBS to Rook\n"
+        printf "    is possible now, but it requires scaling down applications using OpenEBS volumes, causing downtime. You can\n"
+        printf "    choose to run the migration later if preferred.\n"
+        printf "Would you like to continue with the migration now? \n"
+        if ! confirmN ; then
+            printf "Not migrating from OpenEBS to Rook\n"
+            return 0
+        fi
+    else
+        # migration is not available, so exit
+        printf "Migration from OpenEBS to Rook is not available\n"
+        return 0
+    fi
+
+    # Initiate OpenEBS to Rook multi-node migration
+    if ! "${DIR}"/bin/kurl cluster migrate-multinode-storage --ekco-address "$EKCO_ADDRESS" --ekco-auth-token "$EKCO_AUTH_TOKEN" --assume-yes; then
+        logFail "Failed to migrate from OpenEBS to Rook. The installation will move on."
+        logFail "If you would like to run the migration later, run the following command:"
+        logFail "    $DIR/bin/kurl cluster migrate-multinode-storage --ekco-address $EKCO_ADDRESS --ekco-auth-token $EKCO_AUTH_TOKEN"
+        return 0
+    fi
+}
+
+# rook_maybe_migrate_from_openebs_tasks will call rook_maybe_migrate_from_openebs_internal
+# after determining values for EKCO_AUTH_TOKEN and EKCO_ADDRESS from the cluster.
+function rook_maybe_migrate_from_openebs_tasks() {
+    local ekcoAddress=
+    local ekcoAuthToken=
+    ekcoAddress=$(get_ekco_addr)
+    ekcoAuthToken=$(get_ekco_storage_migration_auth_token)
+    if [ -z "$ekcoAddress" ] || [ -z "$ekcoAuthToken" ]; then
+        return 0
+    fi
+
+    export EKCO_ADDRESS="$ekcoAddress"
+    export EKCO_AUTH_TOKEN="$ekcoAuthToken"
+
+    rook_maybe_migrate_from_openebs_internal
+}
diff --git a/scripts/common/utilbinaries.sh b/scripts/common/utilbinaries.sh
@@ -91,6 +91,16 @@ function get_patch_yaml() {
                 ;;
             docker-registry-ip)
                 ;;
+            ekco-address)
+                if [ -z "$EKCO_ADDRESS" ]; then
+                    EKCO_ADDRESS="$_value"
+                fi
+                ;;
+            ekco-auth-token)
+                if [ -z "$EKCO_AUTH_TOKEN" ]; then
+                    EKCO_AUTH_TOKEN="$_value"
+                fi
+                ;;
             ekco-enable-internal-load-balancer)
                 ;;
             ha)

diff --git a/scripts/install.sh b/scripts/install.sh
@@ -476,7 +476,7 @@ function outro() {
         printf "To add worker nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:"
         printf "\n"
         printf "\n"
-        printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION}${common_flags}\n"
+        printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n"
         printf "${NC}"
         printf "\n"
         printf "\n"
@@ -485,7 +485,7 @@ function outro() {
             printf "To add ${GREEN}MASTER${NC} nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:"
             printf "\n"
             printf "\n"
-            printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane${common_flags}\n"
+            printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n"
             printf "${NC}"
             printf "\n"
             printf "\n"
@@ -494,15 +494,15 @@ function outro() {
         printf "\n"
         printf "To add worker nodes to this installation, run the following script on your other nodes:"
         printf "\n"
-        printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION}${common_flags}\n"
+        printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n"
         printf "${NC}"
         printf "\n"
         printf "\n"
         if [ "$HA_CLUSTER" = "1" ]; then
             printf "\n"
             printf "To add ${GREEN}MASTER${NC} nodes to this installation, run the following script on your other nodes:"
             printf "\n"
-            printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=$KUBEADM_TOKEN_CA_HASH kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane${common_flags}\n"
+            printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=$KUBEADM_TOKEN_CA_HASH kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n"
             printf "${NC}"
             printf "\n"
             printf "\n"
@@ -603,6 +603,7 @@ function main() {
     uninstall_docker
     ${K8S_DISTRO}_addon_for_each addon_post_init
     check_proxy_config
+    rook_maybe_migrate_from_openebs
     outro
     package_cleanup
 

diff --git a/scripts/join.sh b/scripts/join.sh
@@ -191,6 +191,7 @@ function main() {
     install_helm
     join
     check_proxy_config
+    rook_maybe_migrate_from_openebs
     outro
     package_cleanup
     popd_install_directory

diff --git a/scripts/tasks.sh b/scripts/tasks.sh
@@ -100,6 +100,9 @@ function tasks() {
         weave-to-flannel-secondary|weave_to_flannel_secondary)
             weave_to_flannel_secondary "$@"
             ;;
+        migrate-to-multinode-storage|migrate_to_multinode_storage)
+            migrate_to_multinode_storage "$@"
+            ;;
         *)
             bail "Unknown task: $1"
             ;;
@@ -386,6 +389,11 @@ function join_token() {
     local prefix=
     prefix="$(build_installer_prefix "${installer_id}" "${KURL_VERSION}" "${kurl_url}" "" "")"
 
+    local ekcoAddress=
+    local ekcoAuthToken=
+    ekcoAddress=$(get_ekco_addr)
+    ekcoAuthToken=$(get_ekco_storage_migration_auth_token)
+
     if [ "$HA_CLUSTER" = "1" ]; then
         printf "Master node join commands expire after two hours, and worker node join commands expire after 24 hours.\n"
         printf "\n"
@@ -409,7 +417,7 @@ function join_token() {
         printf "To add worker nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:"
         printf "\n"
         printf "\n"
-        printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags}\n"
+        printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n"
         printf "${NC}"
         printf "\n"
         printf "\n"
@@ -418,7 +426,7 @@ function join_token() {
             printf "To add ${GREEN}MASTER${NC} nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:"
             printf "\n"
             printf "\n"
-            printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags}\n"
+            printf "${GREEN}    cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n"
             printf "${NC}"
             printf "\n"
             printf "\n"
@@ -427,15 +435,15 @@ function join_token() {
         printf "\n"
         printf "To add worker nodes to this installation, run the following script on your other nodes:"
         printf "\n"
-        printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags}\n"
+        printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n"
         printf "${NC}"
         printf "\n"
         printf "\n"
         if [ "$HA_CLUSTER" = "1" ]; then
             printf "\n"
             printf "To add ${GREEN}MASTER${NC} nodes to this installation, run the following script on your other nodes:"
             printf "\n"
-            printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=$kubeadm_ca_hash kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags}\n"
+            printf "${GREEN}    ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=$kubeadm_ca_hash kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n"
             printf "${NC}"
             printf "\n"
             printf "\n"
@@ -900,6 +908,21 @@ function flannel_images_present() {
     fi
 }
 
+# Determine if the cluster is eligible for a multi-node migration and then initiate the migration
+function migrate_to_multinode_storage() {
+    export KUBECONFIG=/etc/kubernetes/admin.conf
+    download_util_binaries
+
+    # Get Rook.minimumNodeCount option from the configmap
+    local rookMinNodes=
+    rookMinNodes=$(kubectl get cm kurl-current-config -n kurl -ojsonpath='{.data.addons-rook}' | base64 -d | tr "," " "| awk '{print $1}' | cut -d ":" -f 2)
+    if [ -z "$rookMinNodes" ] || [ "$rookMinNodes" -lt "3" ]; then
+        bail "Rook.minimumNodeCount must be greater than or equal to 3"
+    fi
+
+    rook_maybe_migrate_from_openebs_tasks
+}
+
 mkdir -p /var/log/kurl
 LOGFILE="/var/log/kurl/tasks-$(date +"%Y-%m-%dT%H-%M-%S").log"
 tasks "$@" 2>&1 | tee $LOGFILE