diff --git a/.buildkite/pipelines/flaky_tests/groups.json b/.buildkite/pipelines/flaky_tests/groups.json
deleted file mode 100644
index b47ccf16a0184..0000000000000
--- a/.buildkite/pipelines/flaky_tests/groups.json
+++ /dev/null
@@ -1,34 +0,0 @@
-{
-  "groups": [
-    {
-      "key": "oss/cigroup",
-      "name": "OSS CI Group",
-      "ciGroups": 12
-    },
-    {
-      "key": "oss/firefox",
-      "name": "OSS Firefox"
-    },
-    {
-      "key": "oss/accessibility",
-      "name": "OSS Accessibility"
-    },
-    {
-      "key": "xpack/cigroup",
-      "name": "Default CI Group",
-      "ciGroups": 27
-    },
-    {
-      "key": "xpack/cigroup/Docker",
-      "name": "Default CI Group Docker"
-    },
-    {
-      "key": "xpack/firefox",
-      "name": "Default Firefox"
-    },
-    {
-      "key": "xpack/accessibility",
-      "name": "Default Accessibility"
-    }
-  ]
-}
diff --git a/.buildkite/pipelines/flaky_tests/pipeline.js b/.buildkite/pipelines/flaky_tests/pipeline.js
index 68e19c8e275d7..bf4abb9ff4c89 100644
--- a/.buildkite/pipelines/flaky_tests/pipeline.js
+++ b/.buildkite/pipelines/flaky_tests/pipeline.js
@@ -1,7 +1,3 @@
-const groups = /** @type {Array<{key: string, name: string, ciGroups: number }>} */(
-  require('./groups.json').groups
-)
-
 const stepInput = (key, nameOfSuite) => {
   return {
     key: `ftsr-suite/${key}`,
@@ -11,31 +7,38 @@ const stepInput = (key, nameOfSuite) => {
   };
 };
 
+const OSS_CI_GROUPS = 12;
+const XPACK_CI_GROUPS = 27;
+
 const inputs = [
   {
     key: 'ftsr-override-count',
     text: 'Override for all suites',
-    default: '0',
+    default: 0,
     required: true,
   },
 ];
 
-for (const group of groups) {
-  if (!group.ciGroups) {
-    inputs.push(stepInput(group.key, group.name))
-  } else {
-    for (let i = 1; i <= group.ciGroups; i++) {
-      inputs.push(stepInput(`${group.key}/${i}`, `${group.name} ${i}`))
-    }
-  }
+for (let i = 1; i <= OSS_CI_GROUPS; i++) {
+  inputs.push(stepInput(`oss/cigroup/${i}`, `OSS CI Group ${i}`));
 }
 
+inputs.push(stepInput(`oss/firefox`, 'OSS Firefox'));
+inputs.push(stepInput(`oss/accessibility`, 'OSS Accessibility'));
+
+for (let i = 1; i <= XPACK_CI_GROUPS; i++) {
+  inputs.push(stepInput(`xpack/cigroup/${i}`, `Default CI Group ${i}`));
+}
+
+inputs.push(stepInput(`xpack/cigroup/Docker`, 'Default CI Group Docker'));
+inputs.push(stepInput(`xpack/firefox`, 'Default Firefox'));
+inputs.push(stepInput(`xpack/accessibility`, 'Default Accessibility'));
+
 const pipeline = {
   steps: [
     {
       input: 'Number of Runs - Click Me',
       fields: inputs,
-      if: `build.env('KIBANA_FLAKY_TEST_RUNNER_CONFIG') == null`
     },
     {
       wait: '~',
diff --git a/.buildkite/pipelines/flaky_tests/runner.js b/.buildkite/pipelines/flaky_tests/runner.js
index 54da6890b2b07..0c2db5c724f7b 100644
--- a/.buildkite/pipelines/flaky_tests/runner.js
+++ b/.buildkite/pipelines/flaky_tests/runner.js
@@ -1,85 +1,37 @@
 const { execSync } = require('child_process');
 
-const concurrency = 25;
-const defaultCount = concurrency * 2;
-const initialJobs = 3;
-
-function getTestSuitesFromMetadata() {
-  const keys = execSync('buildkite-agent meta-data keys')
-    .toString()
-    .split('\n')
-    .filter((k) => k.startsWith('ftsr-suite/'));
-
-  const overrideCount = execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim();
-
-  const testSuites = [];
-  for (const key of keys) {
-    if (!key) {
-      continue;
-    }
+const keys = execSync('buildkite-agent meta-data keys')
+  .toString()
+  .split('\n')
+  .filter((k) => k.startsWith('ftsr-suite/'));
 
-    const value =
-      overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
+const overrideCount = parseInt(
+  execSync(`buildkite-agent meta-data get 'ftsr-override-count'`).toString().trim()
+);
 
-    const count = value === '' ? defaultCount : parseInt(value);
-    totalJobs += count;
-
-    testSuites.push({
-      key: key.replace('ftsr-suite/', ''),
-      count: count,
-    });
-  }
-
-  return testSuites
-}
+const concurrency = 25;
+const initialJobs = 3;
 
-function getTestSuitesFromJson(json) {
-  const fail = (errorMsg) => {
-    console.error('+++ Invalid test config provided')
-    console.error(`${errorMsg}: ${json}`);
-    process.exit(1);
-  }
+let totalJobs = initialJobs;
 
-  let parsed;
-  try {
-    parsed = JSON.parse(json)
-  } catch (error) {
-    fail(`JSON test config did not parse correctly`)
+const testSuites = [];
+for (const key of keys) {
+  if (!key) {
+    continue;
   }
 
-  if (!Array.isArray(parsed)) {
-    fail(`JSON test config must be an array`)
-  }
+  const value =
+    overrideCount || execSync(`buildkite-agent meta-data get '${key}'`).toString().trim();
 
-  /** @type {Array<{ key: string, count: number }>} */
-  const testSuites = []
-  for (const item of parsed) {
-    if (typeof item !== 'object' || item === null) {
-      fail(`testSuites must be objects`)
-    }
-    const key = item.key
-    if (typeof key !== 'string') {
-      fail(`testSuite.key must be a string`)
-    }
-    const count = item.count;
-    if (typeof count !== 'number') {
-      fail(`testSuite.count must be a number`)
-    }
-    testSuites.push({
-      key,
-      count,
-    })
-  }
+  const count = value === '' ? defaultCount : parseInt(value);
+  totalJobs += count;
 
-  return testSuites
+  testSuites.push({
+    key: key.replace('ftsr-suite/', ''),
+    count: count,
+  });
 }
 
-const testSuites = process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG
-  ? getTestSuitesFromJson(process.env.KIBANA_FLAKY_TEST_RUNNER_CONFIG)
-  : getTestSuitesFromMetadata();
-
-let totalJobs = testSuites.reduce((acc, t) => acc + t.count, initialJobs);
-
 if (totalJobs > 500) {
   console.error('+++ Too many tests');
   console.error(