Add expected errors for benchmark test package [false positive test] (#…

…2017)

Force to fail rally_benchmark_malformed_template test package
by updating the template.ndjson file as suggested in the README.md.
Added an error in the template file as well as the expected errors for
the rally_benchmark_malformed_template test package.

.buildkite/pipeline.yml

@@ -6,6 +6,7 @@ env:
   DOCKER_VERSION: "26.1.2"
   KIND_VERSION: 'v0.20.0'
   K8S_VERSION: 'v1.30.0'
+  YQ_VERSION: 'v4.44.3'
   LINUX_AGENT_IMAGE: "golang:${GO_VERSION}"
   # Using this one instead of family/core-windows-2022 because of conflicts with the pre-installed docker.
   WINDOWS_AGENT_IMAGE: "family/platform-ingest-beats-windows-2022"

.buildkite/scripts/install_deps.sh

@@ -153,6 +153,23 @@ with_jq() {
     jq --version
 }
 
+with_yq() {
+    create_bin_folder
+    check_platform_architecture
+    local binary="yq_${platform_type_lowercase}_${arch_type}"
+    local bin_folder="${WORKSPACE}/bin"
+
+    retry 5 curl -sSL -o "${bin_folder}/yq.tar.gz" "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/${binary}.tar.gz"
+
+    tar -C "${bin_folder}" -xpf "${bin_folder}/yq.tar.gz" "./${binary}"
+
+    mv "${bin_folder}/${binary}" "${bin_folder}/yq"
+    chmod +x "${bin_folder}/yq"
+    yq --version
+
+    rm -rf "${bin_folder}/yq.tar.gz"
+}
+
 with_aws_cli() {
     check_platform_architecture
 

.buildkite/scripts/integration_tests.sh

@@ -87,6 +87,12 @@ if [[ "$SERVERLESS" == "false" ]]; then
     with_docker_compose_plugin
 fi
 
+if [[ "${TARGET}" == "${FALSE_POSITIVES_TARGET}" ]]; then
+    # just required for now while testing false positive targets
+    echo "--- install yq"
+    with_yq
+fi
+
 if [[ "${TARGET}" == "${KIND_TARGET}" || "${TARGET}" == "${SYSTEM_TEST_FLAGS_TARGET}" ]]; then
     echo "--- install kubectl & kind"
     with_kubernetes

scripts/test-check-false-positives.sh

@@ -26,9 +26,11 @@ function cleanup() {
 function check_expected_errors() {
   local package_root=$1
   local package_name=""
+  local package_name_manifest=""
   package_name=$(basename "$1")
+  package_name_manifest=$(cat "$package_root/manifest.yml" | yq -r '.name')
   local expected_errors_file="${package_root%/}.expected_errors"
-  local result_tests="build/test-results/${package_name}-*.xml"
+  local result_tests="build/test-results/${package_name_manifest}-*.xml"
   local results_no_spaces="build/test-results-no-spaces.xml"
 
   if [ ! -f "${expected_errors_file}" ]; then
@@ -43,7 +45,7 @@ function check_expected_errors() {
 
   # check number of expected errors
   local number_errors
-  number_errors=$(cat ${result_tests} | grep "<failure>" | wc -l)
+  number_errors=$(cat ${result_tests} | grep -E '<failure>|<error>' | wc -l)
   local expected_errors
   expected_errors=$(cat ${expected_errors_file} | wc -l)
 

test/packages/false_positives/rally_benchmark_malformed_template.expected_errors

@@ -0,0 +1 @@
+<error>failed to initialize backfill generator for scenario &#34;logs-benchmark&#34;: template: generator:12: unexpected &#34;}&#34; in operand</error>

test/packages/false_positives/rally_benchmark_malformed_template/_dev/benchmark/rally/logs-benchmark/template.ndjson

@@ -1,7 +1,7 @@
 {{- $timestamp := generate "timestamp" -}}
 {
 "@timestamp": "{{ $timestamp.Format "2006-01-02T15:04:05.999999Z07:00" }}",
-"data_stream.type": "logs",
+data_stream.type": "logs",
 "data_stream.dataset": "rally_benchmarks.testds",
 "data_stream.namespace": "ep",
 "container.id": "{{ generate "container.id" }}",
@@ -11,4 +11,4 @@
 "log.offset": {{ generate "log.offset" }},
 "tags": ["rally_benchmark.testds", "forwarded", "{{ generate "tags" }" ],
 "message": "{{ generate "IP" }} - - [{{ $timestamp.Format "02/Jan/2006:15:04:05.999999 -0700" }}] \"GET /favicon.ico HTTP/1.1\" {{ generate "StatusCode" }} {{ generate "Size" }} \"http://{{ generate "Hostname" }}:{{ generate "Port" }}/\" \"skip-this-one/5.0 (Macintosh; Intel Mac OS X 10_12_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.59 Safari/537.36\""
-}
+}