From 0f8d231808dbc1713cfb0cb6b975b3e5625ecb21 Mon Sep 17 00:00:00 2001 From: root Date: Sun, 16 Jun 2024 08:34:35 -0400 Subject: [PATCH] MGMT-8357: Deploy assisted-service for e2e tests in Kind --- Makefile | 7 +- scripts/bring_assisted_service.sh | 6 +- scripts/create_full_environment.sh | 5 +- scripts/deploy_assisted_service.sh | 59 +++++--------- scripts/deploy_ui.sh | 11 ++- scripts/hub-cluster/create.sh | 24 ------ scripts/hub-cluster/delete.sh | 24 ------ scripts/hub-cluster/kind/kind-config.yaml | 9 --- scripts/hub-cluster/kind/kind.sh | 60 -------------- scripts/hub-cluster/minikube.sh | 81 ------------------- scripts/utils.sh | 16 ++++ .../test_infra/utils/utils.py | 8 +- 12 files changed, 63 insertions(+), 247 deletions(-) delete mode 100755 scripts/hub-cluster/create.sh delete mode 100755 scripts/hub-cluster/delete.sh delete mode 100644 scripts/hub-cluster/kind/kind-config.yaml delete mode 100755 scripts/hub-cluster/kind/kind.sh delete mode 100755 scripts/hub-cluster/minikube.sh diff --git a/Makefile b/Makefile index 7255c9e47d8..675c7150e3e 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,7 @@ SERVICE := $(or $(SERVICE), quay.io/edge-infrastructure/assisted-service:latest) SERVICE_NAME := $(or $(SERVICE_NAME),assisted-service) INDEX_IMAGE := $(or ${INDEX_IMAGE},quay.io/edge-infrastructure/assisted-service-index:latest) REMOTE_SERVICE_URL := $(or $(REMOTE_SERVICE_URL), "") +USE_LOCAL_SERVICE := $(or $(USE_LOCAL_SERVICE), false) # terraform TF_LOG_PATH=$(REPORTS)/terraform_$(TEST_SESSION_ID).log @@ -99,7 +100,7 @@ endif SSO_URL := $(or $(SSO_URL), https://sso.redhat.com/auth/realms/redhat-external/protocol/openid-connect/token) OCM_BASE_URL := $(or $(OCM_BASE_URL), https://api.integration.openshift.com/) -DEPLOY_TARGET := $(or $(DEPLOY_TARGET),minikube) +DEPLOY_TARGET := $(or $(DEPLOY_TARGET),kind) OCP_KUBECONFIG := $(or $(OCP_KUBECONFIG),build/kubeconfig) IPV6_SUPPORT := $(or ${IPV6_SUPPORT},true) @@ -147,10 +148,10 @@ image_build: bring_assisted_service generate_python_client $(CONTAINER_COMMAND) tag $(IMAGE_NAME) test-infra:latest # For backwards computability create_hub_cluster: - scripts/hub-cluster/create.sh + TARGET=${DEPLOY_TARGET} assisted-service/hack/hub_cluster.sh create delete_hub_cluster: - scripts/hub-cluster/delete.sh + (cd assisted-service && TARGET=${DEPLOY_TARGET} ROOT_DIR=${ROOT_DIR}/assisted-service hack/hub_cluster.sh delete) clean: -rm -rf build assisted-service test_infra.log diff --git a/scripts/bring_assisted_service.sh b/scripts/bring_assisted_service.sh index 2e60af470da..28609c8f264 100755 --- a/scripts/bring_assisted_service.sh +++ b/scripts/bring_assisted_service.sh @@ -22,8 +22,10 @@ export PULL_BASE_REF=${PULL_BASE_REF:-master} if ! [[ -d "assisted-service" ]]; then echo "Can't find assisted-service source locally, cloning ${SERVICE_REPO}" retry -- git clone "${SERVICE_REPO}" -elif - [[ ${SERVICE_REPO} != $(cd assisted-service && git remote get-url origin) ]]; then +elif [[ "${USE_LOCAL_SERVICE}" == "true" ]]; then + echo "Using local assisted-service" + exit 0 +elif [[ ${SERVICE_REPO} != $(cd assisted-service && git remote get-url origin) ]]; then echo "assisted-service repository found locally but with a different origin, replacing with ${SERVICE_REPO}" rm -rf ./assisted-service retry -- git clone "${SERVICE_REPO}" diff --git a/scripts/create_full_environment.sh b/scripts/create_full_environment.sh index 81b83322f66..24aa8be8aac 100755 --- a/scripts/create_full_environment.sh +++ b/scripts/create_full_environment.sh @@ -34,11 +34,12 @@ scripts/install_environment.sh echo "Done installing" echo "Installing kind" -scripts/hub-cluster/kind/kind.sh install +make bring_assisted_service +assisted-service/hack/kind/kind.sh install echo "Done installing kind" echo "Installing minikube" -scripts/hub-cluster/minikube.sh install +assisted-service/hack/minikube/minikube.sh install echo "Done installing minikube" echo "Installing oc and kubectl" diff --git a/scripts/deploy_assisted_service.sh b/scripts/deploy_assisted_service.sh index c7f1d747f05..40241ed7eab 100755 --- a/scripts/deploy_assisted_service.sh +++ b/scripts/deploy_assisted_service.sh @@ -8,9 +8,9 @@ export SERVICE_NAME=assisted-service case ${DEPLOY_TARGET} in kind) - export SERVICE_URL=${SERVICE_URL:-$(hostname)} - export SERVICE_PORT=80 - export IMAGE_SERVICE_PORT=80 + export SERVICE_URL=${SERVICE_URL:-$(get_main_ip)} + export SERVICE_PORT=8090 + export IMAGE_SERVICE_PORT=8080 export EXTERNAL_PORT=false ;; *) @@ -165,51 +165,36 @@ else firewall-cmd --zone=libvirt-routed --add-forward fi - if [ "${DEBUG_SERVICE}" == "true" ]; then - # Change the registry service type to LoadBalancer to be able to access it from outside the cluster - kubectl patch service $REGISTRY_SERVICE_NAME -n $REGISTRY_SERVICE_NAMESPACE --type json -p='[{"op": "replace", "path": "/spec/type", "value":"LoadBalancer"}]' - # Forward the minikube registry addon k8s service to the host to push the debug image using localhost:5000 - spawn_port_forwarding_command $REGISTRY_SERVICE_NAME $REGISTRY_SERVICE_HOST_PORT $REGISTRY_SERVICE_NAMESPACE 999 $KUBECONFIG minikube undeclaredip $REGISTRY_SERVICE_PORT $REGISTRY_SERVICE_NAME - # Set the local registry to the minikube registry (used by the assisted-service update-local-image target) - export SUBSYSTEM_LOCAL_REGISTRY=localhost:5000 - - print_log "Patching assisted service image with a debuggable code " - (cd assisted-service/ && skipper --env-file ../skipper.env make update-local-image -e CONTAINER_COMMAND=${CONTAINER_RUNTIME_COMMAND+x} ) - DEBUG_DEPLOY_AI_PARAMS="REPLICAS_COUNT=1" - # Override the SERVICE environment variable with the local registry debug image - export SERVICE="${SUBSYSTEM_LOCAL_REGISTRY}/assisted-service:latest" - - # Force removing the debugable image before re-deploying the service - # Change replicas to 0 so when applying the deployment it will identify the changes and allow deleting the - # image if it's already in use in the current pods - kubectl scale -n assisted-installer --replicas=0 deployment assisted-service || true - kubectl wait --for=delete -n assisted-installer pod $(kubectl -n assisted-installer get pods | grep assisted-service | awk '{print $1}') - minikube image rm "${SERVICE}" || true - fi - skipper run src/update_assisted_service_cm.py - (cd assisted-service/ && skipper --env-file ../skipper.env run "make deploy-all" ${SKIPPER_PARAMS} $ENABLE_KUBE_API_CMD TARGET=$DEPLOY_TARGET DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} AUTH_TYPE=${AUTH_TYPE} ${DEBUG_DEPLOY_AI_PARAMS:-}) + (cd assisted-service/ && skipper --env-file ../skipper.env run "make deploy-all" ${SKIPPER_PARAMS} $ENABLE_KUBE_API_CMD TARGET=$DEPLOY_TARGET DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} AUTH_TYPE=${AUTH_TYPE} ${DEBUG_DEPLOY_AI_PARAMS:-} IP=$(get_main_ip)) add_firewalld_port $SERVICE_PORT - print_log "Starting port forwarding for deployment/${SERVICE_NAME} on port $SERVICE_PORT" - wait_for_url_and_run ${SERVICE_BASE_URL} "spawn_port_forwarding_command $SERVICE_NAME $SERVICE_PORT $NAMESPACE $NAMESPACE_INDEX $KUBECONFIG minikube undeclared $SERVICE_INTERNAL_PORT" + if [ "${DEPLOY_TARGET}" == "minikube" ]; then + print_log "Starting port forwarding for deployment/${SERVICE_NAME} on port $SERVICE_PORT" + wait_for_url_and_run ${SERVICE_BASE_URL} "spawn_port_forwarding_command $SERVICE_NAME $SERVICE_PORT $NAMESPACE $NAMESPACE_INDEX $KUBECONFIG minikube undeclared $SERVICE_INTERNAL_PORT" + fi - if [ "${DEBUG_SERVICE}" == "true" ]; then - # delve stops all the thread/goroutines once a breakpoint hits which cause the health call to fail and kubernetes reboots the containers. - # see: https://github.com/go-delve/delve/issues/777 - print_log "Removing liveness Probe to prevent rebooting while debugging" - kubectl patch deployment assisted-service -n $NAMESPACE --type json -p='[{"op": "remove", "path": "/spec/template/spec/containers/0/livenessProbe"}]' + if [[ "${DEBUG_SERVICE}" == "true" || "${USE_LOCAL_SERVICE}" == "true" ]]; then + (cd assisted-service/ && make patch-service ROOT_DIR=$ROOT_DIR/assisted-service TARGET=$DEPLOY_TARGET) + fi + if [ "${DEBUG_SERVICE}" == "true" ]; then add_firewalld_port ${DEBUG_SERVICE_PORT} - print_log "Starting port forwarding for deployment/${SERVICE_NAME} on debug port $DEBUG_SERVICE_PORT" - spawn_port_forwarding_command $SERVICE_NAME $DEBUG_SERVICE_PORT $NAMESPACE $NAMESPACE_INDEX $KUBECONFIG minikube undeclaredip $DEBUG_SERVICE_PORT $DEBUG_SERVICE_NAME + + if [ "${DEPLOY_TARGET}" == "minikube" ]; then + print_log "Starting port forwarding for deployment/${SERVICE_NAME} on debug port $DEBUG_SERVICE_PORT" + spawn_port_forwarding_command $SERVICE_NAME $DEBUG_SERVICE_PORT $NAMESPACE $NAMESPACE_INDEX $KUBECONFIG minikube undeclaredip $DEBUG_SERVICE_PORT $DEBUG_SERVICE_NAME + fi fi add_firewalld_port ${IMAGE_SERVICE_PORT} - print_log "Starting port forwarding for deployment/${IMAGE_SERVICE_NAME} on debug port $IMAGE_SERVICE_PORT" - spawn_port_forwarding_command $IMAGE_SERVICE_NAME $IMAGE_SERVICE_PORT $NAMESPACE $NAMESPACE_INDEX $KUBECONFIG minikube undeclaredip $IMAGE_SERVICE_INTERNAL_PORT $IMAGE_SERVICE_NAME + + if [ "${DEPLOY_TARGET}" == "minikube" ]; then + print_log "Starting port forwarding for deployment/${IMAGE_SERVICE_NAME} on debug port $IMAGE_SERVICE_PORT" + spawn_port_forwarding_command $IMAGE_SERVICE_NAME $IMAGE_SERVICE_PORT $NAMESPACE $NAMESPACE_INDEX $KUBECONFIG minikube undeclaredip $IMAGE_SERVICE_INTERNAL_PORT $IMAGE_SERVICE_NAME + fi print_log "${SERVICE_NAME} can be reached at ${SERVICE_BASE_URL} " print_log "Done" diff --git a/scripts/deploy_ui.sh b/scripts/deploy_ui.sh index ed295151b7f..a3b7dbe009c 100755 --- a/scripts/deploy_ui.sh +++ b/scripts/deploy_ui.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +set -x source scripts/utils.sh @@ -17,6 +18,9 @@ mkdir -p build print_log "Starting ui" skipper run "make -C assisted-service/ deploy-ui" ${SKIPPER_PARAMS} TARGET=${DEPLOY_TARGET} DEPLOY_TAG=${DEPLOY_TAG} DEPLOY_MANIFEST_PATH=${DEPLOY_MANIFEST_PATH} DEPLOY_MANIFEST_TAG=${DEPLOY_MANIFEST_TAG} NAMESPACE=${NAMESPACE} +ui_pod=$(get_pod_name app=assisted-installer-ui ${NAMESPACE}) +retry -- is_pod_ready $ui_pod ${NAMESPACE} + case ${DEPLOY_TARGET} in minikube) node_ip=$(get_main_ip) @@ -25,10 +29,9 @@ case ${DEPLOY_TARGET} in ;; kind) - EXTERNAL_PORT=false - node_ip=$(hostname) - ui_port=80 - ui_url="http://${node_ip}/" + node_ip=$(get_main_ip) + ui_port=8060 + ui_url="http://${node_ip}:${ui_port}" ;; *) echo "Non-supported deploy target ${DEPLOY_TARGET}!"; diff --git a/scripts/hub-cluster/create.sh b/scripts/hub-cluster/create.sh deleted file mode 100755 index c7f955d5a98..00000000000 --- a/scripts/hub-cluster/create.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o pipefail -set -o errexit -set -o xtrace - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -case ${DEPLOY_TARGET} in - minikube) - $__dir/minikube.sh create - ;; - kind) - $__dir/kind/kind.sh create - ;; - onprem) - echo "onprem/podman requires no special setup" - ;; - *) - echo "Unknown deploy target ${DEPLOY_TARGET}!"; - exit 1 - ;; -esac diff --git a/scripts/hub-cluster/delete.sh b/scripts/hub-cluster/delete.sh deleted file mode 100755 index 604fbefbdde..00000000000 --- a/scripts/hub-cluster/delete.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o pipefail -set -o errexit -set -o xtrace - -__dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -case ${DEPLOY_TARGET} in - minikube) - $__dir/minikube.sh delete - ;; - kind) - $__dir/kind/kind.sh delete - ;; - onprem) - ROOT_DIR=$(realpath assisted-service/) make -C assisted-service/ clean-onprem - ;; - *) - echo "Unknown deploy target ${DEPLOY_TARGET}!"; - exit 1 - ;; -esac diff --git a/scripts/hub-cluster/kind/kind-config.yaml b/scripts/hub-cluster/kind/kind-config.yaml deleted file mode 100644 index 1bc6643d8e2..00000000000 --- a/scripts/hub-cluster/kind/kind-config.yaml +++ /dev/null @@ -1,9 +0,0 @@ -kind: Cluster -apiVersion: kind.x-k8s.io/v1alpha4 -nodes: -- role: control-plane - # port forward 80 on the host to 80 on this node - extraPortMappings: - - containerPort: 80 - hostPort: 80 - protocol: TCP diff --git a/scripts/hub-cluster/kind/kind.sh b/scripts/hub-cluster/kind/kind.sh deleted file mode 100755 index 34a62075152..00000000000 --- a/scripts/hub-cluster/kind/kind.sh +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env bash - -set -o nounset -set -o errexit -set -o pipefail - -KIND_VERSION="0.17.0" - -__dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) - -function check() { - if [ "$(kind --version)" == "kind version $KIND_VERSION" ]; then - return 0 - else - echo "Does not have 'kind' with version $KIND_VERSION!" - return 1 - fi -} - -function install() { - if check; then - return 0 - fi - - echo "Installing kind $KIND_VERSION..." - sudo curl --retry 5 --connect-timeout 30 -L https://kind.sigs.k8s.io/dl/v$KIND_VERSION/kind-linux-amd64 -o /usr/local/bin/kind - sudo chmod u+x /usr/local/bin/kind - echo "Installed successfully!" -} - -function setup_contour() { - # based on https://kind.sigs.k8s.io/docs/user/ingress/#contour - # for more information about contour, see: https://projectcontour.io - kubectl apply -f https://projectcontour.io/quickstart/contour.yaml - kubectl rollout status -n projectcontour daemonset envoy --timeout 2m - kubectl rollout status -n projectcontour deployment contour --timeout 2m -} - -function create() { - check - - if ! kind export kubeconfig &> /dev/null ; then - KIND_EXPERIMENTAL_PROVIDER=podman kind create cluster --config $__dir/kind-config.yaml - else - echo "Cluster already existing. Skipping creation" - fi - - setup_contour -} - -function delete() { - kind delete cluster -} - -if [ $# -eq 0 ]; then - echo "Usage: $__dir/kind.sh (install|check|create)" - exit 1 -else - $@ -fi diff --git a/scripts/hub-cluster/minikube.sh b/scripts/hub-cluster/minikube.sh deleted file mode 100755 index aefca041db8..00000000000 --- a/scripts/hub-cluster/minikube.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -source scripts/utils.sh - -set -o nounset -set -o pipefail -set -o errexit -set -o xtrace - -export MINIKUBE_DRIVER="${MINIKUBE_DRIVER:-kvm2}" -export MINIKUBE_CPUS="${MINIKUBE_CPUS:-4}" -export MINIKUBE_MEMORY="${MINIKUBE_MEMORY:-8192}" -export MINIKUBE_DISK_SIZE="${MINIKUBE_DISK_SIZE:-50g}" -export MINIKUBE_REGISTRY_IMAGE="${MINIKUBE_REGISTRY_IMAGE:-quay.io/libpod/registry:2.8}" - -export SUDO=$(if [ -x "$(command -v sudo)" ]; then echo "sudo"; else echo ""; fi) - -__dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) - -function _configure_minikube() { - echo "Configuring minikube..." - minikube config set WantUpdateNotification false -} - -function _init_minikube() { - #If the vm exists, it has already been initialized - for p in $(virsh -c qemu:///system list --name ); do - if [[ $p == minikube ]]; then - return - fi - done - - for i in {1..5} - do - minikube delete - minikube start --force --wait-timeout=15m0s || true - - if minikube status ; then - break - else - minikube logs || true - systemctl restart libvirtd.service || true - fi - done - - minikube status - minikube addons enable registry --images="Registry=${MINIKUBE_REGISTRY_IMAGE}" - minikube update-context - minikube tunnel --cleanup &> /dev/null & -} - -function install() { - minikube_version=v1.25.2 - curl --retry 3 --connect-timeout 30 -Lo minikube https://storage.googleapis.com/minikube/releases/${minikube_version}/minikube-linux-amd64 - ${SUDO} install minikube /usr/local/bin/ - minikube version - rm -f minikube -} - -function create() { - _configure_minikube - as_singleton _init_minikube - - # Change the registry service type to LoadBalancer to be able to access it from outside the cluster - kubectl patch service registry -n kube-system --type json -p='[{"op": "replace", "path": "/spec/type", "value":"LoadBalancer"}]' - # Forward the minikube registry addon k8s service to the host to push the debug image using localhost:5000 - spawn_port_forwarding_command registry 5000 kube-system 999 $KUBECONFIG minikube undeclaredip 80 registry - - eval $(minikube docker-env) -} - -function delete() { - minikube delete --all --purge -} - -if [ $# -eq 0 ]; then - echo "Usage: $__dir/minikube.sh (install|create)" - exit 1 -else - $@ -fi diff --git a/scripts/utils.sh b/scripts/utils.sh index 275f6ee4280..6d51fc6965b 100755 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -277,4 +277,20 @@ function retry() { return ${rc} } +function get_pod_name() { + local pod_label=$1 + local namespace=$2 + + kubectl get pods -n $namespace -l $pod_label -o name +} + +function is_pod_ready() { + local pod_name=$1 + local namespace=$2 + + while [ $(kubectl get -n $namespace $pod_name -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') != "True" ]; do + sleep 3 + done +} + "$@" diff --git a/src/assisted_test_infra/test_infra/utils/utils.py b/src/assisted_test_infra/test_infra/utils/utils.py index fbe03b13bf5..849ab901434 100644 --- a/src/assisted_test_infra/test_infra/utils/utils.py +++ b/src/assisted_test_infra/test_infra/utils/utils.py @@ -233,11 +233,17 @@ def get_remote_assisted_service_url(oc, namespace, service, scheme): def get_local_assisted_service_url(namespace, service, deploy_target): + res = subprocess.check_output("ip route get 1", shell=True).split()[6] + ip = str(res, "utf-8") if deploy_target == "onprem": assisted_hostname_or_ip = os.environ["ASSISTED_SERVICE_HOST"] return f"http://{assisted_hostname_or_ip}:8090" elif deploy_target == "kind": - return f"http://{socket.gethostname()}" + url = f"http://{ip}:8090" + if is_assisted_service_reachable(url): + return url + + raise RuntimeError(f"The parsed url {url} to service {service} in {namespace} namespace was not reachable.]") elif deploy_target == "ocp": res = subprocess.check_output("ip route get 1", shell=True).split()[6] ip = str(res, "utf-8")