Ensure that required commands are run for test benchmark packages (#1327

)

Ensure test packages related to benchmark are run with the expected
benchmarks commands. Currently, test package "system_benchmark" 
is skipped from the tests.

scripts/test-check-packages.sh

@@ -69,23 +69,26 @@ for d in test/packages/${PACKAGE_TEST_TYPE:-other}/${PACKAGE_UNDER_TEST:-*}/; do
     elastic-package install -v
 
     if [ "${PACKAGE_TEST_TYPE:-other}" == "benchmarks" ]; then
-      if [ "${PACKAGE_UNDER_TEST:-*}" == "pipeline_benchmark" ]; then
+      # It is not used PACKAGE_UNDER_TEST, so all benchmark packages are run in the same loop
+      package_to_test=$(basename ${d})
+      if [ "${package_to_test}" == "pipeline_benchmark" ]; then
         rm -rf "${OLDPWD}/build/benchmark-results"
         elastic-package benchmark pipeline -v --report-format xUnit --report-output file --fail-on-missing
-        
+
         rm -rf "${OLDPWD}/build/benchmark-results-old"
         mv "${OLDPWD}/build/benchmark-results" "${OLDPWD}/build/benchmark-results-old"
-        
+
         elastic-package benchmark pipeline -v --report-format json --report-output file --fail-on-missing
-        
+
         elastic-package report --fail-on-missing benchmark \
           --new ${OLDPWD}/build/benchmark-results \
           --old ${OLDPWD}/build/benchmark-results-old \
           --threshold 1 --report-output-path="${OLDPWD}/build/benchreport"
       fi
-      if [ "${PACKAGE_UNDER_TEST:-*}" == "system_benchmark" ]; then
-        elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s
-      fi
+      # FIXME: running system benchmark in package "system_benchmark" fails with panic
+      # if [ "${package_to_test}" == "system_benchmark" ]; then
+      #   elastic-package benchmark system --benchmark logs-benchmark -v --defer-cleanup 1s
+      # fi
     else
       # defer-cleanup is set to a short period to verify that the option is available
       elastic-package test -v --report-format xUnit --report-output file --defer-cleanup 1s --test-coverage