[Test] Estimate time of java project build and run inside Che Code Ed…

…itor in performance tests (#22127)

* create api load tests scripts

tests/performance/api-load-tests.sh

@@ -0,0 +1,120 @@
+#!/bin/bash
+set -e
+
+while getopts "i:n:c:p:o:b:u:s:" opt; do 
+  case $opt in
+    i) export TEST_IMAGE=$OPTARG
+      ;;
+    n) export USER_COUNT=$OPTARG
+      ;;
+    c) export COMPLETITIONS_COUNT=$OPTARG
+      ;;
+    o) export OCP_URL=$OPTARG
+      ;;
+    b) export BASE_URL=$OPTARG
+      ;;
+    u) export USERNAME=$OPTARG
+      ;;
+    p) export PASSWORD=$OPTARG
+      ;;      
+    s) export TEST_SUITE=$OPTARG
+      ;;
+    \?) # invalid option
+      exit 1
+      ;;
+    :)
+      echo "Option \"$opt\" needs an argument."
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z $COMPLETITIONS_COUNT ]; then
+  echo "Parameter -t wasn't set, setting completitions count to 1."
+  export COMPLETITIONS_COUNT=1
+fi
+
+if [ -z $USER_COUNT ]; then
+  echo "Parameter -t wasn't set, setting users number to 1."
+  export USER_COUNT=1
+fi
+
+if [ -z $TEST_SUITE ]; then
+  echo "Parameter -s wasn't set, setting testing suite to php."
+  export TEST_SUITE="php"
+fi
+
+echo "Clean up"
+oc delete jobs -l group=load-tests --all-namespaces || true
+oc delete pods -l group=load-tests --all-namespaces || true
+
+# Delete all workspaces in test users namespaces
+for ((i=1; i<=$USER_COUNT; i++)); do
+  oc delete dw --all -n user$i-devspaces || true
+done
+
+# Set common variables to template.yaml
+apiTest="https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-test-$TEST_SUITE.sh"
+rm template.yaml || true
+cp api-pod.yaml template.yaml
+sed -i "s/REPLACE_COMPLETITIONS/$COMPLETITIONS_COUNT/g" template.yaml
+sed -i "s#REPLACE_OCP_SERVER_URL#$OCP_URL#g" template.yaml  
+sed -i "s#REPLACE_BASE_URL#$BASE_URL#g" template.yaml
+sed -i "s#REPLACE_API_TEST#$apiTest#g" template.yaml
+
+oc project load-tests || oc new-project load-tests
+
+# Set variables specific for each pod and create pods
+users_assigned=0
+executor_context=$(oc whoami -c)
+if [ ! -z $USER_COUNT ]; then
+  while [ $users_assigned -lt $USER_COUNT ] 
+  do
+    users_assigned=$((users_assigned+1))
+    cp template.yaml final.yaml
+    sed -i "s/REPLACE_NAME/load-test-$users_assigned/g" final.yaml
+    sed -i "s/REPLACE_USERNAME/$USERNAME$users_assigned/g" final.yaml
+    sed -i "s/REPLACE_PASSWORD/$PASSWORD/g" final.yaml
+    oc create -f final.yaml -n load-tests
+  done
+fi
+
+echo "-- Waiting for all pods to be completed."
+start=$(date +%s)
+# Waiting for jobs to be completed
+all_completed=false
+while [ $all_completed == false ] 
+do
+  sleep 5
+  all_completed=true
+  for job_name in $(oc get jobs -o name )
+  do
+    if [ $(oc get $job_name -o json | jq .status.completionTime) == null ]; then
+      echo "Some jobs are still not completed. Waiting for 5 seconds."
+      all_completed=false
+      break
+    fi
+  done
+done
+
+echo "All jobs are completed!"
+end=$(date +%s)
+
+rm -rf reports || true
+mkdir reports
+statuses=""
+for p in $(oc get pods -l group=load-tests -o name)
+do
+  name=$(oc get $p | awk '{print $1}' | tail -n 1)
+  oc logs $name >> ./reports/$name.txt
+  status=$(oc get $p | awk '{print $3}' | tail -n 1)
+  statuses="$statuses $status"
+done
+echo "Pods ended with those statuses: $statuses"
+
+# ----------- GATHERING LOGS ----------- #
+./api-process-logs.sh $(($end - $start))
+
+echo "Clean up"
+oc delete jobs -l group=load-tests --all-namespaces || true
+oc delete pods -l group=load-tests --all-namespaces || true

tests/performance/api-pod.yaml

@@ -0,0 +1,36 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    group: load-tests
+  name: REPLACE_NAME
+spec:
+  completions: REPLACE_COMPLETITIONS
+  backoffLimit: 0
+  template:
+    metadata:
+      name: load-tests-template
+      labels:
+        group: load-tests
+    spec:
+      containers:
+        - name: mycontainer
+          env:                                 
+            - name: TS_SELENIUM_BASE_URL
+              value: REPLACE_URL
+            - name: USERSTORY
+              value: REPLACE_USERSTORY
+            - name: OCP_SERVER_URL
+              value: REPLACE_OCP_SERVER_URL    
+            - name: BASE_URL
+              value: REPLACE_BASE_URL                  
+            - name: OCP_USERNAME
+              value: REPLACE_USERNAME
+            - name: OCP_PASSWORD
+              value: REPLACE_PASSWORD
+            - name: KUBECONFIG
+              value: /tmp/.kube/config
+          image: quay.io/openshift/origin-cli
+          command: ["/bin/bash"]
+          args: ["-c", "curl -sSL REPLACE_API_TEST | bash"]
+      restartPolicy: Never

tests/performance/api-process-logs.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+set -e
+
+function average() {
+  local arr=("$@")  # copy arguments to local array
+  local sum=0
+  for i in "${arr[@]}"; do
+    sum=$((sum + i))
+  done
+  echo "$((sum / ${#arr[@]}))"
+}
+
+# Create sum up file for gathering logs
+results="./reports/results.txt"
+rm -rf $results || true
+
+wst_array=() # workspace starting time
+bt_array=()  # build time
+passed=0     # number of passed tests
+failed=0     # number of failed tests
+
+for d in reports/load-test*.txt ; do
+  workspaceStartingTime=$(cat $d | grep "Workspace started" | grep -o '[0-9]\+')
+  if [ -n $workspaceStartingTime ]; then
+    passed=$((passed+1))
+    wst_array+=($workspaceStartingTime)
+  else
+    failed=$((failed+1))
+  fi
+
+  buildTime=$(cat $d | grep "Command succeeded in" | grep -o '[0-9]\+')
+  if [ -n $workspaceStartingTime ]; then
+    bt_array+=($buildTime)
+  fi    
+done
+
+sorted_array=($(echo "${wst_array[@]}" | tr ' ' '\n' | sort -n | tr '\n' ' '))
+wsStartMin=${sorted_array[0]}
+wsStartMax=${sorted_array[${#sorted_array[@]}-1]}
+wsStartAvr=$(average "${sorted_array[@]}")
+
+# Fill out load testing report
+echo "$TEST_SUITE load testing for $USER_COUNT users x $COMPLETITIONS_COUNT workspaces took $1 seconds" >> $results
+echo "$passed tests passed" >> $results
+echo "$failed tests failed" >> $results
+
+echo "Workspace startup time: " >> $results
+echo "min: $wsStartMin second" >> $results
+echo "avr: $wsStartAvr second" >> $results
+echo "max: $wsStartMax second" >> $results
+
+sorted_array=($(echo "${bt_array[@]}" | tr ' ' '\n' | sort -n | tr '\n' ' '))
+buildTimeMin=${sorted_array[0]}
+buildTimeMax=${sorted_array[${#sorted_array[@]}-1]}
+buildTimeAvr=$(average "${sorted_array[@]}")
+
+echo "Project build time:" >> $results
+echo "min: $buildTimeMin second" >> $results
+echo "avr: $buildTimeAvr second" >> $results
+echo "max: $buildTimeMax second" >> $results
+
+cat $results

tests/performance/api-tests/api-test-cpp.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh
+source /tmp/api-utils.sh
+
+export TEST_DEVFILE_PATH="devfile-registry/devfiles/TP__cpp__c-plus-plus/devworkspace-che-code-latest.yaml"
+export WORKSPACE_NAME="cpp"
+export projectName="c-plus-plus/strings"
+export expectedCommandOutput="Found"
+export containerName="tools"
+export commandToTest="cd /projects/$projectName && rm -f bin.out && g++ -g "knuth_morris_pratt.cpp" -o bin.out; ./bin.out >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;"
+
+oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify
+cd /tmp
+
+startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME}
+
+testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName}
+
+testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}"
+
+oc delete dw $WORKSPACE_NAME || true

tests/performance/api-tests/api-test-lombok.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh
+source /tmp/api-utils.sh
+
+export TEST_DEVFILE_PATH="devfile-registry/devfiles/java11-maven-lombok__lombok-project-sample/devworkspace-che-code-latest.yaml"
+export WORKSPACE_NAME="java-lombok"
+export projectName="lombok-project-sample"
+export expectedCommandOutput="BUILD SUCCESS"
+export containerName="tools"
+export commandToTest="cd '$projectName'; mvn clean install >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;"
+
+oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify
+cd /tmp
+
+startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME}
+
+testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName}
+
+testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}"
+
+oc delete dw $WORKSPACE_NAME || true

tests/performance/api-tests/api-test-php.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh
+source /tmp/api-utils.sh
+
+export TEST_DEVFILE_PATH="devfile-registry/devfiles/TP__php__php-hello-world/devworkspace-che-code-latest.yaml"
+export WORKSPACE_NAME="php-hello-world"
+export projectName="php-hello-world"
+export expectedCommandOutput="Hello, world!"
+export containerName="tools"
+export commandToTest="cd /projects/$projectName; php hello-world.php >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;"
+
+oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify
+cd /tmp
+
+startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME}
+
+testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName}
+
+testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}"
+
+oc delete dw $WORKSPACE_NAME || true

tests/performance/api-tests/api-test-python.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh
+source /tmp/api-utils.sh
+
+export TEST_DEVFILE_PATH="devfile-registry/devfiles/python__python-hello-world/devworkspace-che-code-latest.yaml"
+export WORKSPACE_NAME="python-hello-world"
+export projectName="python-hello-world"
+export expectedCommandOutput="Hello, world!"
+export containerName="python"
+export commandToTest="cd /projects/$projectName; pwd && ls -la && python -m venv .venv && . .venv/bin/activate; python hello-world.py >> command_log.txt; cat command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;"
+
+oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify
+cd /tmp
+
+startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME}
+
+testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName}
+
+testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}"
+
+oc delete dw $WORKSPACE_NAME || true

tests/performance/api-tests/api-test-quarkus.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+wget -O /tmp/api-utils.sh https://raw.githubusercontent.com/eclipse/che/main/tests/performance/api-tests/api-utils.sh
+source /tmp/api-utils.sh
+
+export TEST_DEVFILE_PATH="devfile-registry/devfiles/java11-maven-quarkus__quarkus-quickstarts/devworkspace-che-code-latest.yaml"
+export WORKSPACE_NAME="quarkus-quickstart"
+export projectName="quarkus-quickstarts/getting-started"
+export expectedCommandOutput="BUILD SUCCESS"
+export containerName="tools"
+export commandToTest="cd /projects/$projectName && mvn package >> command_log.txt; grep '$expectedCommandOutput' ./command_log.txt;"
+
+oc login -u $OCP_USERNAME -p $OCP_PASSWORD --server=$OCP_SERVER_URL --insecure-skip-tls-verify
+cd /tmp
+
+startWorkspace ${BASE_URL}/${TEST_DEVFILE_PATH} ${WORKSPACE_NAME}
+
+testProjectImported ${WORKSPACE_NAME} ${containerName} ${projectName}
+
+testCommand ${WORKSPACE_NAME} ${containerName} "${commandToTest}" "${expectedCommandOutput}"
+
+oc delete dw $WORKSPACE_NAME || true