From b2b4b10cc94749fab6bf15dea23a3610e640de7b Mon Sep 17 00:00:00 2001 From: Ricardo Maraschini Date: Thu, 29 Jun 2023 22:32:07 +0200 Subject: [PATCH] feat: Call multi-node storage migration from kURL installer scripts (#4524) * feat: create function to migrate from openebs to rook this commit creates the function rook_maybe_migrate_from_openebs. this function may migrate data from openebs to rook when all the following conditions are met: - ekco version is >= 0.27.1. - openebs and rook are selected on the installer. - rook's minimum node count is set to a value > 1. - the number of nodes on the cluster is >= than the rook minimum node count. - the 'scaling' storage class exists. * use ekco global envars also new join.sh script * add migrate-to-multinode-storage task * Check for EKCO add-on version >= 0.27.1 * wip * use confirmN * implement and use rook_maybe_migrate_from_openebs_primary * Update scripts/tasks.sh Co-authored-by: Rafael Polanco <6497491+rrpolanco@users.noreply.github.com> * install.sh already has env vars set * fixes * do not require spec versions for tasks.sh * Update scripts/common/rook.sh Co-authored-by: Rafael Polanco <6497491+rrpolanco@users.noreply.github.com> * Update scripts/tasks.sh Co-authored-by: Rafael Polanco <6497491+rrpolanco@users.noreply.github.com> --------- Co-authored-by: Rafael R. Polanco <6497491+rrpolanco@users.noreply.github.com> Co-authored-by: Andrew Lavery --- kurl_util/cmd/bashmerge/main.go | 4 ++ scripts/common/common.sh | 33 +++++++++++++ scripts/common/rook.sh | 83 +++++++++++++++++++++++++++++++++ scripts/common/utilbinaries.sh | 10 ++++ scripts/install.sh | 9 ++-- scripts/join.sh | 1 + scripts/tasks.sh | 31 ++++++++++-- 7 files changed, 163 insertions(+), 8 deletions(-) diff --git a/kurl_util/cmd/bashmerge/main.go b/kurl_util/cmd/bashmerge/main.go index e325d2e3ee..9351a4c41c 100644 --- a/kurl_util/cmd/bashmerge/main.go +++ b/kurl_util/cmd/bashmerge/main.go @@ -302,6 +302,10 @@ func parseBashFlags(installer *kurlv1beta1.Installer, bashFlags string) error { installer.Spec.Velero = &kurlv1beta1.Velero{} } installer.Spec.Velero.ServerFlags = append(installer.Spec.Velero.ServerFlags, flags...) + case "ekco-address": + continue + case "ekco-auth-token": + continue default: return fmt.Errorf("string %s is not a bash flag", split[0]) } diff --git a/scripts/common/common.sh b/scripts/common/common.sh index eba5486ba9..e0e15f8ec6 100644 --- a/scripts/common/common.sh +++ b/scripts/common/common.sh @@ -1536,3 +1536,36 @@ function node_is_using_docker() { node="$(get_local_node_name)" kubectl get node "$node" -ojsonpath='{.metadata.annotations.kubeadm\.alpha\.kubernetes\.io/cri-socket}' | grep -q "dockershim.sock" } + +# get_ekco_addr prints the host address (including port) for reaching the EKCO service to stdout +function get_ekco_addr() { + if [ -n "$EKCO_ADDRESS" ]; then + echo "$EKCO_ADDRESS" + return + fi + + local ekco_addr= + local ekco_port= + local current_node_ip= + current_node_ip=$(kubectl get nodes -o wide | grep "$(get_local_node_name)" | awk '{print $6}') + ekco_port="${EKCO_NODE_PORT}" + + if [ -z "${ekco_port}" ]; then + ekco_port=$(kubectl get svc ekc-operator -n kurl -o jsonpath='{.spec.ports[?(@.nodePort)].nodePort}') + fi + ekco_addr="${current_node_ip}:${ekco_port}" + echo "$ekco_addr" +} + +# get_ekco_storage_migration_auth_token prints the ekco storage migration authentication token to stdout +function get_ekco_storage_migration_auth_token() { + if [ -n "$EKCO_AUTH_TOKEN" ]; then + echo "$EKCO_AUTH_TOKEN" + return + fi + + local auth_token= + auth_token=$(kubectl get cm -n kurl ekco-config -ojsonpath='{.data.config\.yaml}' | grep "storage_migration_auth_token:" | awk '{print $2}') + + echo "$auth_token" +} diff --git a/scripts/common/rook.sh b/scripts/common/rook.sh index de0a0a6f22..e67a51ad6c 100644 --- a/scripts/common/rook.sh +++ b/scripts/common/rook.sh @@ -514,3 +514,86 @@ function add_rook_store_object_migration_status() { kubectl patch configmap kurl-migration-from-rook -n kurl --type merge -p '{"data":{"DID_MIGRATE_ROOK_OBJECT_STORE":"1"}}' export DID_MIGRATE_ROOK_OBJECT_STORE=1 } + +# rook_maybe_migrate_from_openebs may migrate data from OpenEBS to Rook when all the following +# conditions are met: +# - Ekco version is >= 0.27.1. +# - OpenEBS and Rook are selected on the Installer. +# - Rook's minimum node count is set to a value > 1. +# - The number of nodes on the cluster is >= than the Rook minimum node count. +# - The 'scaling' storage class exists. +function rook_maybe_migrate_from_openebs() { + semverCompare "$EKCO_VERSION" "0.27.1" + if [ "$SEMVER_COMPARE_RESULT" -lt "0" ]; then + return 0 + fi + if [ -z "$ROOK_VERSION" ] || [ -z "$OPENEBS_VERSION" ]; then + return 0 + fi + if [ -z "$ROOK_MINIMUM_NODE_COUNT" ] || [ "$ROOK_MINIMUM_NODE_COUNT" -le "1" ]; then + return 0 + fi + rook_maybe_migrate_from_openebs_internal +} + +# rook_maybe_migrate_from_openebs_internal SHOULD NOT BE CALLED DIRECTLY. +# it is called by rook_maybe_migrate_from_openebs and rook_maybe_migrate_from_openebs_tasks when all the conditions are met. +# it will check that the required environment variables (EKCO_AUTH_TOKEN and EKCO_ADDRESS) are set and then +# check EKCO to see if the migration is available. If it is, it will prompt the user to start it. +function rook_maybe_migrate_from_openebs_internal() { + if [ -z "$EKCO_AUTH_TOKEN" ]; then + logFail "Internal Error: an authentication token is required to start the OpenEBS to Rook multi-node migration." + return 0 + fi + + if [ -z "$EKCO_ADDRESS" ]; then + logFail "Internal Error: unable to determine network address of the kURL operator." + return 0 + fi + + # are both rook and openebs installed, not just specified? + if ! kubectl get ns | grep -q rook-ceph && ! kubectl get ns | grep -q openebs; then + bail "Rook and OpenEBS must be installed in order to migrate to multi-node storage" + fi + + # check if OpenEBS to Rook multi-node migration is available - if it is, prompt the user to start it + if "${DIR}"/bin/kurl cluster migrate-multinode-storage --ekco-address "$EKCO_ADDRESS" --ekco-auth-token "$EKCO_AUTH_TOKEN" --check-status; then + printf " The installer detected both OpenEBS and Rook installations in your cluster. Migration from OpenEBS to Rook\n" + printf " is possible now, but it requires scaling down applications using OpenEBS volumes, causing downtime. You can\n" + printf " choose to run the migration later if preferred.\n" + printf "Would you like to continue with the migration now? \n" + if ! confirmN ; then + printf "Not migrating from OpenEBS to Rook\n" + return 0 + fi + else + # migration is not available, so exit + printf "Migration from OpenEBS to Rook is not available\n" + return 0 + fi + + # Initiate OpenEBS to Rook multi-node migration + if ! "${DIR}"/bin/kurl cluster migrate-multinode-storage --ekco-address "$EKCO_ADDRESS" --ekco-auth-token "$EKCO_AUTH_TOKEN" --assume-yes; then + logFail "Failed to migrate from OpenEBS to Rook. The installation will move on." + logFail "If you would like to run the migration later, run the following command:" + logFail " $DIR/bin/kurl cluster migrate-multinode-storage --ekco-address $EKCO_ADDRESS --ekco-auth-token $EKCO_AUTH_TOKEN" + return 0 + fi +} + +# rook_maybe_migrate_from_openebs_tasks will call rook_maybe_migrate_from_openebs_internal +# after determining values for EKCO_AUTH_TOKEN and EKCO_ADDRESS from the cluster. +function rook_maybe_migrate_from_openebs_tasks() { + local ekcoAddress= + local ekcoAuthToken= + ekcoAddress=$(get_ekco_addr) + ekcoAuthToken=$(get_ekco_storage_migration_auth_token) + if [ -z "$ekcoAddress" ] || [ -z "$ekcoAuthToken" ]; then + return 0 + fi + + export EKCO_ADDRESS="$ekcoAddress" + export EKCO_AUTH_TOKEN="$ekcoAuthToken" + + rook_maybe_migrate_from_openebs_internal +} diff --git a/scripts/common/utilbinaries.sh b/scripts/common/utilbinaries.sh index 0218b19e08..2737f78f72 100644 --- a/scripts/common/utilbinaries.sh +++ b/scripts/common/utilbinaries.sh @@ -91,6 +91,16 @@ function get_patch_yaml() { ;; docker-registry-ip) ;; + ekco-address) + if [ -z "$EKCO_ADDRESS" ]; then + EKCO_ADDRESS="$_value" + fi + ;; + ekco-auth-token) + if [ -z "$EKCO_AUTH_TOKEN" ]; then + EKCO_AUTH_TOKEN="$_value" + fi + ;; ekco-enable-internal-load-balancer) ;; ha) diff --git a/scripts/install.sh b/scripts/install.sh index 5548c85723..c5143c0c0f 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -476,7 +476,7 @@ function outro() { printf "To add worker nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:" printf "\n" printf "\n" - printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION}${common_flags}\n" + printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n" printf "${NC}" printf "\n" printf "\n" @@ -485,7 +485,7 @@ function outro() { printf "To add ${GREEN}MASTER${NC} nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:" printf "\n" printf "\n" - printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane${common_flags}\n" + printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n" printf "${NC}" printf "\n" printf "\n" @@ -494,7 +494,7 @@ function outro() { printf "\n" printf "To add worker nodes to this installation, run the following script on your other nodes:" printf "\n" - printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION}${common_flags}\n" + printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=${KUBEADM_TOKEN_CA_HASH} kubernetes-version=${KUBERNETES_VERSION} ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n" printf "${NC}" printf "\n" printf "\n" @@ -502,7 +502,7 @@ function outro() { printf "\n" printf "To add ${GREEN}MASTER${NC} nodes to this installation, run the following script on your other nodes:" printf "\n" - printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=$KUBEADM_TOKEN_CA_HASH kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane${common_flags}\n" + printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${API_SERVICE_ADDRESS} kubeadm-token=${BOOTSTRAP_TOKEN} kubeadm-token-ca-hash=$KUBEADM_TOKEN_CA_HASH kubernetes-version=${KUBERNETES_VERSION} cert-key=${CERT_KEY} control-plane ekco-address=${EKCO_ADDRESS} ekco-auth-token=${EKCO_AUTH_TOKEN}${common_flags}\n" printf "${NC}" printf "\n" printf "\n" @@ -603,6 +603,7 @@ function main() { uninstall_docker ${K8S_DISTRO}_addon_for_each addon_post_init check_proxy_config + rook_maybe_migrate_from_openebs outro package_cleanup diff --git a/scripts/join.sh b/scripts/join.sh index 8d76f6420b..2e1b1a1b0e 100755 --- a/scripts/join.sh +++ b/scripts/join.sh @@ -191,6 +191,7 @@ function main() { install_helm join check_proxy_config + rook_maybe_migrate_from_openebs outro package_cleanup popd_install_directory diff --git a/scripts/tasks.sh b/scripts/tasks.sh index c12db133c2..708584b28d 100755 --- a/scripts/tasks.sh +++ b/scripts/tasks.sh @@ -100,6 +100,9 @@ function tasks() { weave-to-flannel-secondary|weave_to_flannel_secondary) weave_to_flannel_secondary "$@" ;; + migrate-to-multinode-storage|migrate_to_multinode_storage) + migrate_to_multinode_storage "$@" + ;; *) bail "Unknown task: $1" ;; @@ -386,6 +389,11 @@ function join_token() { local prefix= prefix="$(build_installer_prefix "${installer_id}" "${KURL_VERSION}" "${kurl_url}" "" "")" + local ekcoAddress= + local ekcoAuthToken= + ekcoAddress=$(get_ekco_addr) + ekcoAuthToken=$(get_ekco_storage_migration_auth_token) + if [ "$HA_CLUSTER" = "1" ]; then printf "Master node join commands expire after two hours, and worker node join commands expire after 24 hours.\n" printf "\n" @@ -409,7 +417,7 @@ function join_token() { printf "To add worker nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:" printf "\n" printf "\n" - printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags}\n" + printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n" printf "${NC}" printf "\n" printf "\n" @@ -418,7 +426,7 @@ function join_token() { printf "To add ${GREEN}MASTER${NC} nodes to this installation, copy and unpack this bundle on your other nodes, and run the following:" printf "\n" printf "\n" - printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags}\n" + printf "${GREEN} cat ./join.sh | sudo bash -s airgap kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n" printf "${NC}" printf "\n" printf "\n" @@ -427,7 +435,7 @@ function join_token() { printf "\n" printf "To add worker nodes to this installation, run the following script on your other nodes:" printf "\n" - printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags}\n" + printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=${kubeadm_ca_hash} kubernetes-version=${kubernetes_version}${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n" printf "${NC}" printf "\n" printf "\n" @@ -435,7 +443,7 @@ function join_token() { printf "\n" printf "To add ${GREEN}MASTER${NC} nodes to this installation, run the following script on your other nodes:" printf "\n" - printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=$kubeadm_ca_hash kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags}\n" + printf "${GREEN} ${prefix}join.sh | sudo bash -s kubernetes-master-address=${api_service_address} kubeadm-token=${bootstrap_token} kubeadm-token-ca-hash=$kubeadm_ca_hash kubernetes-version=${kubernetes_version} cert-key=${cert_key} control-plane${common_flags} ekco-address=${ekcoAddress} ekco-auth-token=${ekcoAuthToken}\n" printf "${NC}" printf "\n" printf "\n" @@ -900,6 +908,21 @@ function flannel_images_present() { fi } +# Determine if the cluster is eligible for a multi-node migration and then initiate the migration +function migrate_to_multinode_storage() { + export KUBECONFIG=/etc/kubernetes/admin.conf + download_util_binaries + + # Get Rook.minimumNodeCount option from the configmap + local rookMinNodes= + rookMinNodes=$(kubectl get cm kurl-current-config -n kurl -ojsonpath='{.data.addons-rook}' | base64 -d | tr "," " "| awk '{print $1}' | cut -d ":" -f 2) + if [ -z "$rookMinNodes" ] || [ "$rookMinNodes" -lt "3" ]; then + bail "Rook.minimumNodeCount must be greater than or equal to 3" + fi + + rook_maybe_migrate_from_openebs_tasks +} + mkdir -p /var/log/kurl LOGFILE="/var/log/kurl/tasks-$(date +"%Y-%m-%dT%H-%M-%S").log" tasks "$@" 2>&1 | tee $LOGFILE