Merge branch 'main' into backfill-metrics

Author: elasticmachine

.buildkite/ftr_platform_stateful_configs.yml

@@ -6,12 +6,10 @@ disabled:
   # Base config files, only necessary to inform config finding script
   - src/platform/test/functional/config.base.js
   - src/platform/test/functional/firefox/config.base.ts
-  - x-pack/test/functional/config.base.js
   - x-pack/platform/test/functional/config.base.ts
   - x-pack/platform/test/localization/config.base.ts
   - src/platform/test/server_integration/config.base.js
   - x-pack/platform/test/functional_with_es_ssl/config.base.ts
-  - x-pack/test/api_integration/config.ts
   - x-pack/platform/test/api_integration/config.ts
   - x-pack/platform/test/fleet_api_integration/config.base.ts
   - x-pack/platform/test/functional_basic/apps/ml/config.base.ts
@@ -146,6 +144,8 @@ enabled:
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group4/config.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group5/config.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group6/config.ts
+  - x-pack/platform/test/alerting_api_integration/security_and_spaces/group7/config.ts
+  - x-pack/platform/test/alerting_api_integration/security_and_spaces/group8/config.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group3/config_with_schedule_circuit_breaker.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group2/config_non_dedicated_task_runner.ts
   - x-pack/platform/test/alerting_api_integration/security_and_spaces/group4/config_non_dedicated_task_runner.ts
@@ -172,6 +172,7 @@ enabled:
   - x-pack/platform/test/fleet_api_integration/config.fleet.ts
   - x-pack/platform/test/fleet_api_integration/config.package_policy.ts
   - x-pack/platform/test/fleet_api_integration/config.space_awareness.ts
+  - x-pack/platform/test/fleet_api_integration/config.cloud_connector.ts
   - x-pack/platform/test/fleet_functional/config.ts
   - x-pack/platform/test/fleet_tasks/config.ts
   - x-pack/platform/test/ftr_apis/security_and_spaces/config.ts
@@ -333,6 +334,7 @@ enabled:
   - x-pack/platform/test/ui_capabilities/security_and_spaces/config.ts
   - x-pack/platform/test/ui_capabilities/spaces_only/config.ts
   - x-pack/platform/test/upgrade_assistant_integration/config.ts
+  - x-pack/platform/test/reindex_service/config.ts
   - x-pack/platform/test/usage_collection/config.ts
   - x-pack/performance/journeys_e2e/aiops_log_rate_analysis.ts
   - x-pack/performance/journeys_e2e/ecommerce_dashboard.ts
@@ -399,7 +401,7 @@ enabled:
   - x-pack/platform/test/encrypted_saved_objects_api_integration/config.ts
   - x-pack/platform/test/fleet_multi_cluster/config.ts
   - x-pack/platform/test/monitoring_api_integration/config.ts
-  - x-pack/platform/test/onechat_api_integration/config.ts
+  - x-pack/platform/test/onechat_api_integration/configs/config.stateful.ts
   - x-pack/platform/test/plugin_api_integration/config.ts
   - x-pack/platform/test/saved_object_api_integration/security_and_spaces/config_basic.ts
   - x-pack/platform/test/saved_object_api_integration/security_and_spaces/config_trial.ts

.buildkite/ftr_security_serverless_configs.yml

@@ -17,6 +17,9 @@ disabled:
   # Playwright
   - x-pack/solutions/security/test/security_solution_playwright/serverless_config.ts
 
+  # QA suites that are run out-of-band
+  - x-pack/solutions/security/test/serverless/functional/configs/config.cloud_security_posture.cloud.ts
+
   # MKI only configs files
   - x-pack/solutions/security/test/serverless/functional/configs/config.mki_only.ts
 
@@ -84,7 +87,9 @@ enabled:
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/rule_update/basic_license_essentials_tier/configs/serverless.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/configs/serverless_essentials_tier.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_disabled/configs/serverless_essentials_tier.config.ts
-  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/configs/serverless.config.ts
+  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/configs/serverless.config.ts
+  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_prebuilt_rules/configs/serverless.config.ts
+  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_notifications/configs/serverless.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_prebuilt_rules/diffable_rule_fields/common_fields/configs/serverless.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_prebuilt_rules/diffable_rule_fields/type_specific_fields/configs/serverless.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/ml_disabled/configs/serverless_essentials_tier.config.ts

.buildkite/ftr_security_stateful_configs.yml

@@ -8,7 +8,6 @@ disabled:
   - x-pack/solutions/security/test/security_solution_api_integration/config/ess/config.base.frozen.trial.ts
   - x-pack/solutions/security/test/security_solution_endpoint/configs/config.base.ts
   - x-pack/solutions/security/test/security_solution_endpoint/config.base.ts
-  - x-pack/test/security_solution_endpoint_api_int/config.base.ts
   - x-pack/solutions/security/test/api_integration/config.ts
   - x-pack/solutions/security/test/functional/config.base.ts
 
@@ -74,7 +73,9 @@ enabled:
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/configs/edge_cases/ess_air_gapped_with_bundled_large_package.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_disabled/configs/ess_basic_license.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/common/configs/edge_cases/ess_trial_license.config.ts
-  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/configs/ess.config.ts
+  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/customization/configs/ess.config.ts
+  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_prebuilt_rules/configs/ess.config.ts
+  - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_notifications/configs/ess.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_prebuilt_rules/diffable_rule_fields/common_fields/configs/ess.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/customization_enabled/upgrade_prebuilt_rules/diffable_rule_fields/type_specific_fields/configs/ess.config.ts
   - x-pack/solutions/security/test/security_solution_api_integration/test_suites/detections_response/rules_management/prebuilt_rules/ml_disabled/configs/ess_basic_license.config.ts

.buildkite/package-lock.json

@@ -15,6 +15,7 @@
     "js-yaml": "^4.1.0",
     "minimatch": "^5.0.1",
     "minimist": "^1.2.8",
+    "p-limit": "^3.1.0",
     "tslib": "*"
   },
   "devDependencies": {
@@ -23,7 +24,7 @@
     "@types/jscodeshift": "^0.12.0",
     "@types/minimatch": "^3.0.5",
     "@types/minimist": "^1.2.5",
-    "@types/node": "^15.12.2",
+    "@types/node": "^22.17.1",
     "jest": "^30.0.3",
     "jscodeshift": "^17.1.2",
     "nock": "^12.0.2",

.buildkite/package.json

@@ -573,6 +573,15 @@ export async function pickTestGroupRunOrder() {
   );
 }
 
+// copied from src/platform/packages/shared/kbn-scout/src/config/discovery/search_configs.ts
+interface ScoutTestDiscoveryConfig {
+  group: string;
+  path: string;
+  usesParallelWorkers: boolean;
+  configs: string[];
+  type: 'plugin' | 'package';
+}
+
 export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
   const bk = new BuildkiteClient();
   const envFromlabels: Record<string, string> = collectEnvFromLabels();
@@ -581,33 +590,37 @@ export async function pickScoutTestGroupRunOrder(scoutConfigsPath: string) {
     throw new Error(`Scout configs file not found at ${scoutConfigsPath}`);
   }
 
-  const rawScoutConfigs = JSON.parse(Fs.readFileSync(scoutConfigsPath, 'utf-8'));
-  const pluginsWithScoutConfigs: string[] = Object.keys(rawScoutConfigs);
+  const rawScoutConfigs = JSON.parse(Fs.readFileSync(scoutConfigsPath, 'utf-8')) as Record<
+    string,
+    ScoutTestDiscoveryConfig
+  >;
+  const pluginsOrPackagesWithScoutTests: string[] = Object.keys(rawScoutConfigs);
 
-  if (pluginsWithScoutConfigs.length === 0) {
+  if (pluginsOrPackagesWithScoutTests.length === 0) {
     // no scout configs found, nothing to need to upload steps
     return;
   }
 
-  const scoutGroups = pluginsWithScoutConfigs.map((plugin) => ({
-    title: plugin,
-    key: plugin,
-    usesParallelWorkers: rawScoutConfigs[plugin].usesParallelWorkers,
-    group: rawScoutConfigs[plugin].group,
+  const scoutCiRunGroups = pluginsOrPackagesWithScoutTests.map((name) => ({
+    label: `Scout: [ ${rawScoutConfigs[name].group} / ${name} ] ${rawScoutConfigs[name].type}`,
+    key: name,
+    agents: expandAgentQueue(rawScoutConfigs[name].usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+    group: rawScoutConfigs[name].group,
   }));
 
   // upload the step definitions to Buildkite
   bk.uploadSteps(
     [
       {
         group: 'Scout Configs',
-        depends_on: ['build'],
-        steps: scoutGroups.map(
-          ({ title, key, group, usesParallelWorkers }): BuildkiteStep => ({
-            label: `Scout: [ ${group} / ${title} ] plugin`,
+        key: 'scout-configs',
+        depends_on: ['build_scout_tests'],
+        steps: scoutCiRunGroups.map(
+          ({ label, key, group, agents }): BuildkiteStep => ({
+            label,
             command: getRequiredEnv('SCOUT_CONFIGS_SCRIPT'),
             timeout_in_minutes: 60,
-            agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+            agents,
             env: {
               SCOUT_CONFIG_GROUP_KEY: key,
               SCOUT_CONFIG_GROUP_TYPE: group,

.buildkite/pipeline-utils/ci-stats/pick_test_group_run_order.ts

@@ -61,6 +61,21 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
+    agents:
+      machineType: n2-standard-4
+    key: build_scout_tests
+    timeout_in_minutes: 10
+    depends_on:
+      - build
+    env:
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh
     label: 'Serverless Entity Analytics - Security Cypress Tests'
     agents:

.buildkite/pipelines/chrome_forward_testing.yml

@@ -56,6 +56,24 @@ steps:
         - exit_status: '*'
           limit: 1
 
+  - command: .buildkite/scripts/steps/test/scout_test_run_builder.sh
+    label: 'Scout Test Run Builder'
+    agents:
+      image: family/kibana-ubuntu-2404
+      imageProject: elastic-images-prod
+      provider: gcp
+      machineType: n2-standard-4
+    key: build_scout_tests
+    timeout_in_minutes: 10
+    depends_on:
+      - build
+    env:
+      SCOUT_CONFIGS_SCRIPT: '.buildkite/scripts/steps/test/scout_configs.sh'
+    retry:
+      automatic:
+        - exit_status: '*'
+          limit: 1
+
   - command: .buildkite/scripts/steps/es_snapshots/trigger_promote.sh
     label: Trigger promotion
     timeout_in_minutes: 10

.buildkite/pipelines/es_snapshots/verify.yml

@@ -34,6 +34,19 @@ if (Number.isNaN(concurrency)) {
 const BASE_JOBS = 1;
 const MAX_JOBS = 500;
 
+function getScoutConfigGroupType(configPath: string): string | null {
+  // Match platform paths: x-pack/platform/... or src/platform/...
+  if (/^(x-pack|src)\/platform\//.test(configPath)) {
+    return 'platform';
+  }
+  // Match solution paths: x-pack/solutions/<solution>/plugins/...
+  const match = configPath.match(/^x-pack\/solutions\/([^/]+)\/plugins\//);
+  if (match) {
+    return match[1];
+  }
+  return null;
+}
+
 function getTestSuitesFromJson(json: string) {
   const fail = (errorMsg: string) => {
     console.error('+++ Invalid test config provided');
@@ -55,6 +68,7 @@ function getTestSuitesFromJson(json: string) {
   const testSuites: Array<
     | { type: 'group'; key: string; count: number }
     | { type: 'ftrConfig'; ftrConfig: string; count: number }
+    | { type: 'scoutConfig'; scoutConfig: string; count: number }
   > = [];
   for (const item of parsed) {
     if (typeof item !== 'object' || item === null) {
@@ -160,6 +174,33 @@ for (const testSuite of testSuites) {
     continue;
   }
 
+  if (testSuite.type === 'scoutConfig') {
+    const usesParallelWorkers = testSuite.scoutConfig.endsWith('parallel.playwright.config.ts');
+    const scoutConfigGroupType = getScoutConfigGroupType(testSuite.scoutConfig);
+
+    steps.push({
+      command: `.buildkite/scripts/steps/test/scout_configs.sh`,
+      env: {
+        SCOUT_CONFIG: testSuite.scoutConfig,
+        SCOUT_CONFIG_GROUP_TYPE: scoutConfigGroupType!,
+      },
+      key: `scout-suite-${suiteIndex++}`,
+      label: `${testSuite.scoutConfig}`,
+      parallelism: testSuite.count,
+      concurrency,
+      concurrency_group: process.env.UUID,
+      concurrency_method: 'eager',
+      agents: expandAgentQueue(usesParallelWorkers ? 'n2-8-spot' : 'n2-4-spot'),
+      depends_on: 'build',
+      timeout_in_minutes: 30,
+      cancel_on_build_failing: true,
+      retry: {
+        automatic: [{ exit_status: '-1', limit: 3 }],
+      },
+    });
+    continue;
+  }
+
   const [category, suiteName] = testSuite.key.split('/');
   switch (category) {
     case 'cypress':