[FTR] Add support for --include and --exclude files via tags (#60123) (#60383)

Author: brianseeders

packages/kbn-test/src/functional_test_runner/cli.ts

@@ -48,12 +48,15 @@ export function runFtrCli() {
           kbnTestServer: {
             installDir: parseInstallDir(flags),
           },
+          suiteFiles: {
+            include: toArray(flags.include as string | string[]).map(makeAbsolutePath),
+            exclude: toArray(flags.exclude as string | string[]).map(makeAbsolutePath),
+          },
           suiteTags: {
             include: toArray(flags['include-tag'] as string | string[]),
             exclude: toArray(flags['exclude-tag'] as string | string[]),
           },
           updateBaselines: flags.updateBaselines,
-          excludeTestFiles: flags.exclude || undefined,
         }
       );
 
@@ -96,7 +99,15 @@ export function runFtrCli() {
     },
     {
       flags: {
-        string: ['config', 'grep', 'exclude', 'include-tag', 'exclude-tag', 'kibana-install-dir'],
+        string: [
+          'config',
+          'grep',
+          'include',
+          'exclude',
+          'include-tag',
+          'exclude-tag',
+          'kibana-install-dir',
+        ],
         boolean: ['bail', 'invert', 'test-stats', 'updateBaselines'],
         default: {
           config: 'test/functional/config.js',
@@ -107,7 +118,8 @@ export function runFtrCli() {
         --bail             stop tests after the first failure
         --grep <pattern>   pattern used to select which tests to run
         --invert           invert grep to exclude tests
-        --exclude=file     path to a test file that should not be loaded
+        --include=file     a test file to be included, pass multiple times for multiple files
+        --exclude=file     a test file to be excluded, pass multiple times for multiple files
         --include-tag=tag  a tag to be included, pass multiple times for multiple tags
         --exclude-tag=tag  a tag to be excluded, pass multiple times for multiple tags
         --test-stats       print the number of tests (included and excluded) to STDERR

packages/kbn-test/src/functional_test_runner/lib/config/schema.ts

@@ -64,9 +64,16 @@ export const schema = Joi.object()
     testFiles: Joi.array().items(Joi.string()),
     testRunner: Joi.func(),
 
-    excludeTestFiles: Joi.array()
-      .items(Joi.string())
-      .default([]),
+    suiteFiles: Joi.object()
+      .keys({
+        include: Joi.array()
+          .items(Joi.string())
+          .default([]),
+        exclude: Joi.array()
+          .items(Joi.string())
+          .default([]),
+      })
+      .default(),
 
     suiteTags: Joi.object()
       .keys({

packages/kbn-test/src/functional_test_runner/lib/mocha/decorate_mocha_ui.js

@@ -16,7 +16,8 @@
  * specific language governing permissions and limitations
  * under the License.
  */
-
+import { relative } from 'path';
+import { REPO_ROOT } from '@kbn/dev-utils';
 import { createAssignmentProxy } from './assignment_proxy';
 import { wrapFunction } from './wrap_function';
 import { wrapRunnableArgs } from './wrap_runnable_args';
@@ -65,6 +66,10 @@ export function decorateMochaUi(lifecycle, context) {
             this._tags = [].concat(this._tags || [], tags);
           };
 
+          const relativeFilePath = relative(REPO_ROOT, this.file);
+          this.tags(relativeFilePath);
+          this.suiteTag = relativeFilePath; // The tag that uniquely targets this suite/file
+
           provider.call(this);
 
           after(async () => {

packages/kbn-test/src/functional_test_runner/lib/mocha/load_test_files.js

@@ -31,28 +31,12 @@ import { decorateMochaUi } from './decorate_mocha_ui';
  *  @param  {String} path
  *  @return {undefined} - mutates mocha, no return value
  */
-export const loadTestFiles = ({
-  mocha,
-  log,
-  lifecycle,
-  providers,
-  paths,
-  excludePaths,
-  updateBaselines,
-}) => {
-  const pendingExcludes = new Set(excludePaths.slice(0));
-
+export const loadTestFiles = ({ mocha, log, lifecycle, providers, paths, updateBaselines }) => {
   const innerLoadTestFile = path => {
     if (typeof path !== 'string' || !isAbsolute(path)) {
       throw new TypeError('loadTestFile() only accepts absolute paths');
     }
 
-    if (pendingExcludes.has(path)) {
-      pendingExcludes.delete(path);
-      log.warning('Skipping test file %s', path);
-      return;
-    }
-
     loadTracer(path, `testFile[${path}]`, () => {
       log.verbose('Loading test file %s', path);
 
@@ -94,13 +78,4 @@ export const loadTestFiles = ({
   };
 
   paths.forEach(innerLoadTestFile);
-
-  if (pendingExcludes.size) {
-    throw new Error(
-      `After loading all test files some exclude paths were not consumed:${[
-        '',
-        ...pendingExcludes,
-      ].join('\n  -')}`
-    );
-  }
 };

packages/kbn-test/src/functional_test_runner/lib/mocha/setup_mocha.js

@@ -18,6 +18,8 @@
  */
 
 import Mocha from 'mocha';
+import { relative } from 'path';
+import { REPO_ROOT } from '@kbn/dev-utils';
 
 import { loadTestFiles } from './load_test_files';
 import { filterSuitesByTags } from './filter_suites_by_tags';
@@ -50,10 +52,20 @@ export async function setupMocha(lifecycle, log, config, providers) {
     lifecycle,
     providers,
     paths: config.get('testFiles'),
-    excludePaths: config.get('excludeTestFiles'),
     updateBaselines: config.get('updateBaselines'),
   });
 
+  // Each suite has a tag that is the path relative to the root of the repo
+  // So we just need to take input paths, make them relative to the root, and use them as tags
+  // Also, this is a separate filterSuitesByTags() call so that the test suites will be filtered first by
+  //  files, then by tags. This way, you can target tags (like smoke) in a specific file.
+  filterSuitesByTags({
+    log,
+    mocha,
+    include: config.get('suiteFiles.include').map(file => relative(REPO_ROOT, file)),
+    exclude: config.get('suiteFiles.exclude').map(file => relative(REPO_ROOT, file)),
+  });
+
   filterSuitesByTags({
     log,
     mocha,

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/args.test.js.snap

@@ -46,6 +46,14 @@ const options = {
   updateBaselines: {
     desc: 'Replace baseline screenshots with whatever is generated from the test.',
   },
+  include: {
+    arg: '<file>',
+    desc: 'Files that must included to be run, can be included multiple times.',
+  },
+  exclude: {
+    arg: '<file>',
+    desc: 'Files that must NOT be included to be run, can be included multiple times.',
+  },
   'include-tag': {
     arg: '<tag>',
     desc: 'Tags that suites must include to be run, can be included multiple times.',
@@ -115,6 +123,13 @@ export function processOptions(userOptions, defaultConfigPaths) {
     delete userOptions['kibana-install-dir'];
   }
 
+  userOptions.suiteFiles = {
+    include: [].concat(userOptions.include || []),
+    exclude: [].concat(userOptions.exclude || []),
+  };
+  delete userOptions.include;
+  delete userOptions.exclude;
+
   userOptions.suiteTags = {
     include: [].concat(userOptions['include-tag'] || []),
     exclude: [].concat(userOptions['exclude-tag'] || []),

packages/kbn-test/src/functional_tests/cli/run_tests/__snapshots__/cli.test.js.snap

@@ -22,7 +22,7 @@ import { CliError } from './run_cli';
 
 async function createFtr({
   configPath,
-  options: { installDir, log, bail, grep, updateBaselines, suiteTags },
+  options: { installDir, log, bail, grep, updateBaselines, suiteFiles, suiteTags },
 }) {
   const config = await readConfigFile(log, configPath);
 
@@ -37,6 +37,10 @@ async function createFtr({
         installDir,
       },
       updateBaselines,
+      suiteFiles: {
+        include: [...suiteFiles.include, ...config.get('suiteFiles.include')],
+        exclude: [...suiteFiles.exclude, ...config.get('suiteFiles.exclude')],
+      },
       suiteTags: {
         include: [...suiteTags.include, ...config.get('suiteTags.include')],
         exclude: [...suiteTags.exclude, ...config.get('suiteTags.exclude')],