[9.1] [ci] fix flaky-test-runner reporter for scout/synthetics (#236616) (#237662)

# Backport

This will backport the following commits from `main` to `9.1`:
- [[ci] fix flaky-test-runner reporter for scout/synthetics
(#236616)](#236616)

<!--- Backport version: 10.0.2 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sorenlouv/backport)

<!--BACKPORT [{"author":{"name":"Dzmitry
Lemechko","email":"dzmitry.lemechko@elastic.co"},"sourceCommit":{"committedDate":"2025-09-26T19:46:53Z","message":"[ci]
fix flaky-test-runner reporter for scout/synthetics (#236616)\n\n##
Summary\n\nWe saw `🎉 All tests passed!` reported for Scout tests, even
when they\nfailed. This PR adjusts `post_stats_on_pr.ts` to properly
handle Scout\ntests (Synthetics as
well?)","sha":"ffdbdc23088daebdf99dfc1f161a8c66d7725bb0","branchLabelMapping":{"^v9.2.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","backport
missing","backport:all-open","test:scout","v9.2.0"],"title":"[ci] fix
flaky-test-runner reporter for
scout/synthetics","number":236616,"url":"https://github.com/elastic/kibana/pull/236616","mergeCommit":{"message":"[ci]
fix flaky-test-runner reporter for scout/synthetics (#236616)\n\n##
Summary\n\nWe saw `🎉 All tests passed!` reported for Scout tests, even
when they\nfailed. This PR adjusts `post_stats_on_pr.ts` to properly
handle Scout\ntests (Synthetics as
well?)","sha":"ffdbdc23088daebdf99dfc1f161a8c66d7725bb0"}},"sourceBranch":"main","suggestedTargetBranches":[],"targetPullRequestStates":[{"branch":"main","label":"v9.2.0","branchLabelMappingKey":"^v9.2.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/236616","number":236616,"mergeCommit":{"message":"[ci]
fix flaky-test-runner reporter for scout/synthetics (#236616)\n\n##
Summary\n\nWe saw `🎉 All tests passed!` reported for Scout tests, even
when they\nfailed. This PR adjusts `post_stats_on_pr.ts` to properly
handle Scout\ntests (Synthetics as
well?)","sha":"ffdbdc23088daebdf99dfc1f161a8c66d7725bb0"}}]}]
BACKPORT-->

Author: dmlemeshko

.buildkite/pipelines/flaky_tests/constants.ts

@@ -0,0 +1,17 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+export enum TestSuiteType {
+  FTR = 'ftr-suite',
+  SCOUT = 'scout-suite',
+  CYPRESS = 'cypress-suite',
+  SYNTHETICS = 'synthetics-suite',
+}
+
+export const TEST_SUITE_TYPES = Object.values(TestSuiteType);

.buildkite/pipelines/flaky_tests/pipeline.ts

@@ -8,7 +8,9 @@
  */
 
 import { groups } from './groups.json';
-import { BuildkiteStep, expandAgentQueue, collectEnvFromLabels } from '#pipeline-utils';
+import { TestSuiteType } from './constants';
+import type { BuildkiteStep } from '#pipeline-utils';
+import { expandAgentQueue, collectEnvFromLabels } from '#pipeline-utils';
 
 const configJson = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG;
 if (!configJson) {
@@ -170,7 +172,7 @@ for (const testSuite of testSuites) {
       env: {
         FTR_CONFIG: testSuite.ftrConfig,
       },
-      key: `ftr-suite-${suiteIndex++}`,
+      key: `${TestSuiteType.FTR}-${suiteIndex++}`,
       label: `${testSuite.ftrConfig}`,
       parallelism: testSuite.count,
       concurrency,
@@ -197,7 +199,7 @@ for (const testSuite of testSuites) {
         SCOUT_CONFIG: testSuite.scoutConfig,
         SCOUT_CONFIG_GROUP_TYPE: scoutConfigGroupType!,
       },
-      key: `scout-suite-${suiteIndex++}`,
+      key: `${TestSuiteType.SCOUT}-${suiteIndex++}`,
       label: `${testSuite.scoutConfig}`,
       parallelism: testSuite.count,
       concurrency,
@@ -228,7 +230,7 @@ for (const testSuite of testSuites) {
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: group.name,
         agents: expandAgentQueue(agentQueue),
-        key: `cypress-suite-${suiteIndex++}`,
+        key: `${TestSuiteType.CYPRESS}-${suiteIndex++}`,
         depends_on: 'build',
         timeout_in_minutes: 150,
         parallelism: testSuite.count,
@@ -261,7 +263,7 @@ for (const testSuite of testSuites) {
         command: `.buildkite/scripts/steps/functional/${suiteName}.sh`,
         label: synthGroup.name,
         agents: expandAgentQueue('n2-4-spot'),
-        key: `synthetics-suite-${suiteIndex++}`,
+        key: `${TestSuiteType.SYNTHETICS}-${suiteIndex++}`,
         depends_on: 'build',
         timeout_in_minutes: 30,
         parallelism: testSuite.count,

.buildkite/pipelines/flaky_tests/post_stats_on_pr.ts

@@ -7,6 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
+import { TEST_SUITE_TYPES } from './constants';
 import { BuildkiteClient, getGithubClient } from '#pipeline-utils';
 
 interface TestSuiteResult {
@@ -28,7 +29,7 @@ async function main() {
   // Calculate success metrics
   const jobs = buildkiteBuild.jobs;
   const testSuiteRuns = jobs.filter((step) => {
-    return step.step_key?.includes('ftr-suite') || step.step_key?.includes('cypress-suite');
+    return TEST_SUITE_TYPES.some((testType) => step.step_key?.includes(testType));
   });
   const testSuiteGroups = groupBy('name', testSuiteRuns);