[DO NOT MERGE] Retry upgrade integration tests multiple times to test flaky tests

Author: pchila

.buildkite/bk.integration.pipeline.yml

@@ -158,6 +158,31 @@ steps:
           - upgrade-flavor
           - install-uninstall
 
+      - label: "Win2022:sudo: repeated-{{matrix}}"
+        depends_on:
+          - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_WIN_2022}"
+        retry:
+          automatic:
+            limit: 10
+            # this is not a typo we want to retry successful run to "stress" flaky test execution
+            exit_status: 0
+          manual:
+            permit_on_passed: true
+        matrix:
+          - upgrade
+
       - label: "Win2022:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
@@ -207,6 +232,31 @@ steps:
           - upgrade-flavor
           - install-uninstall
 
+      - label: "Win2025:sudo:repeated-{{matrix}}"
+        depends_on:
+          - packaging-windows
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step 'packaging-windows'
+          .buildkite/scripts/steps/integration_tests_tf.ps1 {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_WIN_2025}"
+        retry:
+          automatic:
+            limit: 10
+            # this is not a typo we want to retry successful run to "stress" flaky test execution
+            exit_status: 0
+          manual:
+            permit_on_passed: true
+        matrix:
+          - upgrade
+
       - label: "Win2025:non-sudo:{{matrix}}"
         depends_on:
           - packaging-windows
@@ -291,6 +341,32 @@ steps:
           - deb
           - container
 
+      - label: "x86_64:sudo: repeated {{matrix}}"
+        depends_on:
+          - packaging-ubuntu-x86-64
+        # due to deb group present in matrix tar.gz and deb packages artifacts are required
+        env:
+          TEST_PACKAGE: "github.com/elastic/elastic-agent/testing/integration/ess"
+        command: |
+          buildkite-agent artifact download build/distributions/** . --step packaging-ubuntu-x86-64
+          .buildkite/scripts/steps/integration_tests_tf.sh {{matrix}} true
+        artifact_paths:
+          - build/**
+          - build/diagnostics/**
+        retry:
+          automatic:
+            limit: 10
+            # this is not a typo we want to retry successful run to "stress" flaky test execution
+            exit_status: 0
+          manual:
+            permit_on_passed: true
+        agents:
+          provider: "gcp"
+          machineType: "n2-standard-8"
+          image: "${IMAGE_UBUNTU_2404_X86_64}"
+        matrix:
+          - upgrade
+
       - label: "arm:sudo: {{matrix}}"
         depends_on:
           - packaging-ubuntu-arm64