[ftr] rework ciGroup validation to remove JOBS.yml and avoid duplication (#109149) (#109324)

Co-authored-by: spalger <spalger@users.noreply.github.com>
# Conflicts:
#	.ci/jobs.yml
#	test/scripts/jenkins_code_coverage.sh

Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com>

Author: Spencer

.ci/ci_groups.yml

@@ -0,0 +1,29 @@
+root:
+  - ciGroup1
+  - ciGroup2
+  - ciGroup3
+  - ciGroup4
+  - ciGroup5
+  - ciGroup6
+  - ciGroup7
+  - ciGroup8
+  - ciGroup9
+  - ciGroup10
+  - ciGroup11
+  - ciGroup12
+
+xpack:
+  - ciGroup1
+  - ciGroup2
+  - ciGroup3
+  - ciGroup4
+  - ciGroup5
+  - ciGroup6
+  - ciGroup7
+  - ciGroup8
+  - ciGroup9
+  - ciGroup10
+  - ciGroup11
+  - ciGroup12
+  - ciGroup13
+  - ciGroupDocker

.ci/jobs.yml

@@ -71,6 +71,7 @@ const defaultRelativeToConfigPath = (path: string) => {
 
 export const schema = Joi.object()
   .keys({
+    rootTags: Joi.array().items(Joi.string()),
     testFiles: Joi.array().items(Joi.string()),
     testRunner: Joi.func(),
 

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -21,7 +21,7 @@ function split(arr, fn) {
   return [a, b];
 }
 
-export function decorateMochaUi(log, lifecycle, context, { isDockerGroup }) {
+export function decorateMochaUi(log, lifecycle, context, { isDockerGroup, rootTags }) {
   // incremented at the start of each suite, decremented after
   // so that in each non-suite call we can know if we are within
   // a suite, or that when a suite is defined it is within a suite
@@ -62,7 +62,13 @@ export function decorateMochaUi(log, lifecycle, context, { isDockerGroup }) {
           });
 
           const relativeFilePath = relative(REPO_ROOT, this.file);
-          this._tags = isDockerGroup ? ['ciGroupDocker', relativeFilePath] : [relativeFilePath];
+          this._tags = [
+            ...(isDockerGroup ? ['ciGroupDocker', relativeFilePath] : [relativeFilePath]),
+            // we attach the "root tags" to all the child suites of the root suite, so that if they
+            // need to be excluded they can be removed from the root suite without removing the entire
+            // root suite
+            ...(this.parent.root ? [...(rootTags ?? [])] : []),
+          ];
           this.suiteTag = relativeFilePath; // The tag that uniquely targets this suite/file
           this.tags = (tags) => {
             const newTags = Array.isArray(tags) ? tags : [tags];

packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -62,6 +62,7 @@ export const loadTestFiles = ({
 
       const context = decorateMochaUi(log, lifecycle, global, {
         isDockerGroup,
+        rootTags: config.get('rootTags'),
       });
       mocha.suite.emit('pre-require', context, path, mocha);
 

packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js

@@ -52,9 +52,9 @@ export async function assertNoneExcluded({ configPath, options }) {
     throw new CliError(`
       ${stats.excludedTests.length} tests in the ${configPath} config
       are excluded when filtering by the tags run on CI. Make sure that all suites are
-      tagged with one of the following tags, or extend the list of tags in test/scripts/jenkins_xpack.sh
+      tagged with one of the following tags:
 
-      tags: ${JSON.stringify(options.suiteTags)}
+      ${JSON.stringify(options.suiteTags)}
 
       - ${stats.excludedTests.join('\n      - ')}
     `);

packages/kbn-test/src/functional_tests/lib/run_ftr.js

@@ -55,7 +55,7 @@ const makeSuccessMessage = (options) => {
  * @property {string} options.esFrom         Optionally run from source instead of snapshot
  */
 export async function runTests(options) {
-  if (!process.env.KBN_NP_PLUGINS_BUILT) {
+  if (!process.env.KBN_NP_PLUGINS_BUILT && !options.assertNoneExcluded) {
     const log = options.createLogger();
     log.warning('❗️❗️❗️');
     log.warning('❗️❗️❗️');

packages/kbn-test/src/functional_tests/tasks.js

@@ -7,4 +7,4 @@
  */
 
 require('../src/setup_node_env');
-require('../src/dev/run_ensure_all_tests_in_ci_group');
+require('../src/dev/ensure_all_tests_in_ci_group').runEnsureAllTestsInCiGroupsCli();

scripts/ensure_all_tests_in_ci_group.js

@@ -0,0 +1,52 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import Path from 'path';
+import Fs from 'fs/promises';
+
+import execa from 'execa';
+import { safeLoad } from 'js-yaml';
+
+import { run, REPO_ROOT } from '@kbn/dev-utils';
+import { schema } from '@kbn/config-schema';
+
+const RELATIVE_JOBS_YAML_PATH = '.ci/ci_groups.yml';
+const JOBS_YAML_PATH = Path.resolve(REPO_ROOT, RELATIVE_JOBS_YAML_PATH);
+const SCHEMA = schema.object({
+  root: schema.arrayOf(schema.string()),
+  xpack: schema.arrayOf(schema.string()),
+});
+
+export function runEnsureAllTestsInCiGroupsCli() {
+  run(async ({ log }) => {
+    const { root, xpack } = SCHEMA.validate(safeLoad(await Fs.readFile(JOBS_YAML_PATH, 'utf-8')));
+
+    log.info(
+      'validating root tests directory contains all "root" ciGroups from',
+      RELATIVE_JOBS_YAML_PATH
+    );
+    await execa(process.execPath, [
+      'scripts/functional_tests',
+      ...root.map((tag) => `--include-tag=${tag}`),
+      '--include-tag=runOutsideOfCiGroups',
+      '--assert-none-excluded',
+    ]);
+
+    log.info(
+      'validating x-pack/tests directory contains all "xpack" ciGroups from',
+      RELATIVE_JOBS_YAML_PATH
+    );
+    await execa(process.execPath, [
+      'x-pack/scripts/functional_tests',
+      ...xpack.map((tag) => `--include-tag=${tag}`),
+      '--assert-none-excluded',
+    ]);
+
+    log.success('all tests are in a valid ciGroup');
+  });
+}