diff --git a/tests/performance/api-load-tests.sh b/tests/performance/api-load-tests.sh new file mode 100755 index 00000000000..4683d485690 --- /dev/null +++ b/tests/performance/api-load-tests.sh @@ -0,0 +1,120 @@ +#!/bin/bash +set -e + +while getopts "i:n:c:p:o:b:u:s:" opt; do + case $opt in + i) export TEST_IMAGE=$OPTARG + ;; + n) export USER_COUNT=$OPTARG + ;; + c) export COMPLETITIONS_COUNT=$OPTARG + ;; + o) export OCP_URL=$OPTARG + ;; + b) export BASE_URL=$OPTARG + ;; + u) export USERNAME=$OPTARG + ;; + p) export PASSWORD=$OPTARG + ;; + s) export TEST_SUITE=$OPTARG + ;; + \?) # invalid option + exit 1 + ;; + :) + echo "Option \"$opt\" needs an argument." + exit 1 + ;; + esac +done + +if [ -z $COMPLETITIONS_COUNT ]; then + echo "Parameter -t wasn't set, setting completitions count to 1." + export COMPLETITIONS_COUNT=1 +fi + +if [ -z $USER_COUNT ]; then + echo "Parameter -t wasn't set, setting users number to 1." + export USER_COUNT=1 +fi + +if [ -z $TEST_SUITE ]; then + echo "Parameter -s wasn't set, setting testing suite to php." + export TEST_SUITE="php" +fi + +echo "Clean up" +oc delete jobs -l group=load-tests --all-namespaces || true +oc delete pods -l group=load-tests --all-namespaces || true + +# Delete all workspaces in test users namespaces +for ((i=1; i<=$USER_COUNT; i++)); do + oc delete dw --all -n user$i-devspaces || true +done + +# Set common variables to template.yaml +apiTest="https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-test-$TEST_SUITE.sh" +rm template.yaml || true +cp api-pod.yaml template.yaml +sed -i "s/REPLACE_COMPLETITIONS/$COMPLETITIONS_COUNT/g" template.yaml +sed -i "s#REPLACE_OCP_SERVER_URL#$OCP_URL#g" template.yaml +sed -i "s#REPLACE_BASE_URL#$BASE_URL#g" template.yaml +sed -i "s#REPLACE_API_TEST#$apiTest#g" template.yaml + +oc project load-tests || oc new-project load-tests + +# Set variables specific for each pod and create pods +users_assigned=0 +executor_context=$(oc whoami -c) +if [ ! -z $USER_COUNT ]; then + while [ $users_assigned -lt $USER_COUNT ] + do + users_assigned=$((users_assigned+1)) + cp template.yaml final.yaml + sed -i "s/REPLACE_NAME/load-test-$users_assigned/g" final.yaml + sed -i "s/REPLACE_USERNAME/$USERNAME$users_assigned/g" final.yaml + sed -i "s/REPLACE_PASSWORD/$PASSWORD/g" final.yaml + oc create -f final.yaml -n load-tests + done +fi + +echo "-- Waiting for all pods to be completed." +start=$(date +%s) +# Waiting for jobs to be completed +all_completed=false +while [ $all_completed == false ] +do + sleep 5 + all_completed=true + for job_name in $(oc get jobs -o name ) + do + if [ $(oc get $job_name -o json | jq .status.completionTime) == null ]; then + echo "Some jobs are still not completed. Waiting for 5 seconds." + all_completed=false + break + fi + done +done + +echo "All jobs are completed!" +end=$(date +%s) + +rm -rf reports || true +mkdir reports +statuses="" +for p in $(oc get pods -l group=load-tests -o name) +do + name=$(oc get $p | awk '{print $1}' | tail -n 1) + oc logs $name >> ./reports/$name.txt + status=$(oc get $p | awk '{print $3}' | tail -n 1) + statuses="$statuses $status" +done +echo "Pods ended with those statuses: $statuses" + +# ----------- GATHERING LOGS ----------- # +./api-process-logs.sh $(($end - $start)) + +echo "Clean up" +oc delete jobs -l group=load-tests --all-namespaces || true +oc delete pods -l group=load-tests --all-namespaces || true diff --git a/tests/performance/api-pod.yaml b/tests/performance/api-pod.yaml new file mode 100644 index 00000000000..2654756df4b --- /dev/null +++ b/tests/performance/api-pod.yaml @@ -0,0 +1,36 @@ +apiVersion: batch/v1 +kind: Job +metadata: + labels: + group: load-tests + name: REPLACE_NAME +spec: + completions: REPLACE_COMPLETITIONS + backoffLimit: 0 + template: + metadata: + name: load-tests-template + labels: + group: load-tests + spec: + containers: + - name: mycontainer + env: + - name: TS_SELENIUM_BASE_URL + value: REPLACE_URL + - name: USERSTORY + value: REPLACE_USERSTORY + - name: OCP_SERVER_URL + value: REPLACE_OCP_SERVER_URL + - name: BASE_URL + value: REPLACE_BASE_URL + - name: OCP_USERNAME + value: REPLACE_USERNAME + - name: OCP_PASSWORD + value: REPLACE_PASSWORD + - name: KUBECONFIG + value: /tmp/.kube/config + image: quay.io/openshift/origin-cli + command: ["/bin/bash"] + args: ["-c", "curl -sSL REPLACE_API_TEST | bash"] + restartPolicy: Never diff --git a/tests/performance/api-process-logs.sh b/tests/performance/api-process-logs.sh new file mode 100755 index 00000000000..dadbae01667 --- /dev/null +++ b/tests/performance/api-process-logs.sh @@ -0,0 +1,62 @@ +#!/bin/bash +set -e + +function average() { + local arr=("$@") # copy arguments to local array + local sum=0 + for i in "${arr[@]}"; do + sum=$((sum + i)) + done + echo "$((sum / ${#arr[@]}))" +} + +# Create sum up file for gathering logs +results="./reports/results.txt" +rm -rf $results || true + +wst_array=() # workspace starting time +bt_array=() # build time +passed=0 # number of passed tests +failed=0 # number of failed tests + +for d in reports/load-test*.txt ; do + workspaceStartingTime=$(cat $d | grep "Workspace started" | grep -o '[0-9]\+') + if [ -n $workspaceStartingTime ]; then + passed=$((passed+1)) + wst_array+=($workspaceStartingTime) + else + failed=$((failed+1)) + fi + + buildTime=$(cat $d | grep "Command succeeded in" | grep -o '[0-9]\+') + if [ -n $workspaceStartingTime ]; then + bt_array+=($buildTime) + fi +done + +sorted_array=($(echo "${wst_array[@]}" | tr ' ' '\n' | sort -n | tr '\n' ' ')) +wsStartMin=${sorted_array[0]} +wsStartMax=${sorted_array[${#sorted_array[@]}-1]} +wsStartAvr=$(average "${sorted_array[@]}") + +# Fill out load testing report +echo "$TEST_SUITE load testing for $USER_COUNT users x $COMPLETITIONS_COUNT workspaces took $1 seconds" >> $results +echo "$passed tests passed" >> $results +echo "$failed tests failed" >> $results + +echo "Workspace startup time: " >> $results +echo "min: $wsStartMin second" >> $results +echo "avr: $wsStartAvr second" >> $results +echo "max: $wsStartMax second" >> $results + +sorted_array=($(echo "${bt_array[@]}" | tr ' ' '\n' | sort -n | tr '\n' ' ')) +buildTimeMin=${sorted_array[0]} +buildTimeMax=${sorted_array[${#sorted_array[@]}-1]} +buildTimeAvr=$(average "${sorted_array[@]}") + +echo "Project build time:" >> $results +echo "min: $buildTimeMin second" >> $results +echo "avr: $buildTimeAvr second" >> $results +echo "max: $buildTimeMax second" >> $results + +cat $results diff --git a/tests/performance/api-tests/api-test-cpp.sh b/tests/performance/api-tests/api-test-cpp.sh new file mode 100644 index 00000000000..845329d2c73 --- /dev/null +++ b/tests/performance/api-tests/api-test-cpp.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh +source /tmp/api-utils.sh + +export TEST_DEVFILE_PATH="devfile-registry/devfiles/TP__cpp__c-plus-plus/devworkspace-che-code-latest.yaml" +export WORKSPACE_NAME="cpp" +export projectName="c-plus-plus/strings" +export expectedCommandOutput="Found" +export containerName="tools" +export commandToTest="cd /projects/$projectName && rm -f bin.out && g++ -g "knuth_morris_pratt.cpp" -o bin.out; ./bin.out >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;" + +oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify +cd /tmp + +startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME} + +testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName} + +testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}" + +oc delete dw $WORKSPACE_NAME || true diff --git a/tests/performance/api-tests/api-test-lombok.sh b/tests/performance/api-tests/api-test-lombok.sh new file mode 100755 index 00000000000..b85684e3b63 --- /dev/null +++ b/tests/performance/api-tests/api-test-lombok.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh +source /tmp/api-utils.sh + +export TEST_DEVFILE_PATH="devfile-registry/devfiles/java11-maven-lombok__lombok-project-sample/devworkspace-che-code-latest.yaml" +export WORKSPACE_NAME="java-lombok" +export projectName="lombok-project-sample" +export expectedCommandOutput="BUILD SUCCESS" +export containerName="tools" +export commandToTest="cd '$projectName'; mvn clean install >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;" + +oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify +cd /tmp + +startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME} + +testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName} + +testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}" + +oc delete dw $WORKSPACE_NAME || true diff --git a/tests/performance/api-tests/api-test-php.sh b/tests/performance/api-tests/api-test-php.sh new file mode 100644 index 00000000000..443b7bb8ed1 --- /dev/null +++ b/tests/performance/api-tests/api-test-php.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh +source /tmp/api-utils.sh + +export TEST_DEVFILE_PATH="devfile-registry/devfiles/TP__php__php-hello-world/devworkspace-che-code-latest.yaml" +export WORKSPACE_NAME="php-hello-world" +export projectName="php-hello-world" +export expectedCommandOutput="Hello, world!" +export containerName="tools" +export commandToTest="cd /projects/$projectName; php hello-world.php >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;" + +oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify +cd /tmp + +startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME} + +testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName} + +testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}" + +oc delete dw $WORKSPACE_NAME || true diff --git a/tests/performance/api-tests/api-test-python.sh b/tests/performance/api-tests/api-test-python.sh new file mode 100644 index 00000000000..56aa4f7e3aa --- /dev/null +++ b/tests/performance/api-tests/api-test-python.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh +source /tmp/api-utils.sh + +export TEST_DEVFILE_PATH="devfile-registry/devfiles/python__python-hello-world/devworkspace-che-code-latest.yaml" +export WORKSPACE_NAME="python-hello-world" +export projectName="python-hello-world" +export expectedCommandOutput="Hello, world!" +export containerName="python" +export commandToTest="cd /projects/$projectName; pwd && ls -la && python -m venv .venv && . .venv/bin/activate; python hello-world.py >> command_log.txt; cat command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;" + +oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify +cd /tmp + +startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME} + +testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName} + +testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}" + +oc delete dw $WORKSPACE_NAME || true diff --git a/tests/performance/api-tests/api-test-quarkus.sh b/tests/performance/api-tests/api-test-quarkus.sh new file mode 100644 index 00000000000..5eb9dac97df --- /dev/null +++ b/tests/performance/api-tests/api-test-quarkus.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh +source /tmp/api-utils.sh + +export TEST_DEVFILE_PATH="devfile-registry/devfiles/java11-maven-quarkus__quarkus-quickstarts/devworkspace-che-code-latest.yaml" +export WORKSPACE_NAME="quarkus-quickstart" +export projectName="quarkus-quickstarts/getting-started" +export expectedCommandOutput="BUILD SUCCESS" +export containerName="tools" +export commandToTest="cd /projects/$projectName && mvn package >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;" + +oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify +cd /tmp + +startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME} + +testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName} + +testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}" + +oc delete dw $WORKSPACE_NAME || true diff --git a/tests/performance/api-tests/api-utils.sh b/tests/performance/api-tests/api-utils.sh new file mode 100644 index 00000000000..e2076e38b32 --- /dev/null +++ b/tests/performance/api-tests/api-utils.sh @@ -0,0 +1,61 @@ +#!/bin/bash +set -e + +# This function starts workspace. +# +# Arguments: +# $1: DevSpaces URL +# $2: path to devfile in devfile-registry +# $3: workspace name +function startWorkspace() { + curl --insecure $1 -o devfile.yaml + echo " routingClass: che" >> devfile.yaml + cat devfile.yaml + oc apply -f devfile.yaml + + start=$(date +%s) + oc wait --for=condition=Ready dw $2 --timeout=360s + end=$(date +%s) + echo "Workspace started in $(($end - $start)) seconds" +} + +# This function checks that project was imported. +# +# Arguments: +# $1: workspace name +# $2: container name +# $3: project name +function testProjectImported() { + export WS_ID=$(oc get dw $1 --template='{{.status.devworkspaceId}}') + export POD_NAME=$(oc get pods | grep $WS_ID | awk '{print $1}') + + # Test if workspace project is imported into workspace + echo "---- Test project is imported ----" + oc exec $POD_NAME -c $2 -- sh -c "pwd && ls -la && ls -la $3" +} + +# This function starts command in workspace pod. +# +# Arguments: +# $1: workspace name +# $2: container name +# $3: command to execute +# $4: expected command output +function testCommand() { + export WS_ID=$(oc get dw $1 --template='{{.status.devworkspaceId}}') + export POD_NAME=$(oc get pods | grep $WS_ID | awk '{print $1}') + + echo "---- Test #$3# command ----" + start=$(date +%s) + export LOG=$(oc exec $POD_NAME -c $2 -- /bin/bash -c "$3") + end=$(date +%s) + + echo $LOG + + if echo $LOG | grep -q "$4"; then + echo "Command succeeded in $(($end - $start)) seconds" + else + echo "Command failed." + exit 1 + fi +}