Code coverage: skip failing tests for coverage run (#59121)

* filter out failing tests

* more skipped tests

* fix tests

* move json files to save space

Author: dmlemeshko

test/functional/apps/management/_scripted_fields.js

@@ -53,7 +53,7 @@ export default function({ getService, getPageObjects }) {
   ]);
 
   describe('scripted fields', function() {
-    this.tags(['skipFirefox']);
+    this.tags(['skipFirefox', 'skipCoverage']);
 
     before(async function() {
       await browser.setWindowSize(1200, 800);

test/functional/apps/visualize/_tile_map.js

@@ -58,6 +58,7 @@ export default function({ getService, getPageObjects }) {
     });
 
     describe('complete config', function describeIndexTests() {
+      this.tags(['skipCoverage']);
       before(async function() {
         await browser.setWindowSize(1280, 1000);
 

test/functional/config.coverage.js

@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+export default async function({ readConfigFile }) {
+  const defaultConfig = await readConfigFile(require.resolve('./config'));
+
+  return {
+    ...defaultConfig.getAll(),
+
+    suiteTags: {
+      exclude: ['skipCoverage'],
+    },
+
+    junit: {
+      reportName: 'Code Coverage for Functional Tests',
+    },
+  };
+}

test/scripts/jenkins_ci_group.sh

@@ -21,13 +21,14 @@ else
   cd "kibana${CI_GROUP}"
 
   echo " -> running tests from the clone folder"
-  yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
+  #yarn run grunt "run:functionalTests_ciGroup${CI_GROUP}";
+  node scripts/functional_tests --debug --include-tag "ciGroup$CI_GROUP"  --config test/functional/config.coverage.js;
 
   if [[ -d target/kibana-coverage/functional ]]; then
     echo " -> replacing kibana${CI_GROUP} with kibana in json files"
     sed -i "s|kibana${CI_GROUP}|kibana|g" target/kibana-coverage/functional/*.json
     echo " -> copying coverage to the original folder"
     mkdir -p ../kibana/target/kibana-coverage/functional
-    cp -R target/kibana-coverage/functional/. ../kibana/target/kibana-coverage/functional/
+    mv target/kibana-coverage/functional/* ../kibana/target/kibana-coverage/functional/
   fi
 fi

test/scripts/jenkins_xpack_ci_group.sh

@@ -23,13 +23,13 @@ else
   cd "kibana${CI_GROUP}/x-pack"
 
   echo " -> running tests from the clone folder"
-  node scripts/functional_tests --debug --include-tag "ciGroup$CI_GROUP"
+  node scripts/functional_tests --debug --include-tag "ciGroup$CI_GROUP"  --config test/functional/config.coverage.js;
 
   if [[ -d ../target/kibana-coverage/functional ]]; then
     echo " -> replacing kibana${CI_GROUP} with kibana in json files"
     sed -i "s|kibana${CI_GROUP}|kibana|g" ../target/kibana-coverage/functional/*.json
     echo " -> copying coverage to the original folder"
     mkdir -p ../../kibana/target/kibana-coverage/functional
-    cp -R ../target/kibana-coverage/functional/. ../../kibana/target/kibana-coverage/functional/
+    mv ../target/kibana-coverage/functional/* ../../kibana/target/kibana-coverage/functional/
   fi
 fi

x-pack/test/functional/apps/advanced_settings/feature_controls/advanced_settings_spaces.ts

@@ -57,7 +57,8 @@ export default function({ getPageObjects, getService }: FtrProviderContext) {
       });
     });
 
-    describe('space with Advanced Settings disabled', () => {
+    describe('space with Advanced Settings disabled', function() {
+      this.tags('skipCoverage');
       before(async () => {
         // we need to load the following in every situation as deleting
         // a space deletes all of the associated saved objects

x-pack/test/functional/apps/dev_tools/feature_controls/index.ts

@@ -7,7 +7,7 @@ import { FtrProviderContext } from '../../../ftr_provider_context';
 
 export default function({ loadTestFile }: FtrProviderContext) {
   describe('feature controls', function() {
-    this.tags('skipFirefox');
+    this.tags(['skipFirefox', 'skipCoverage']);
     loadTestFile(require.resolve('./dev_tools_security'));
     loadTestFile(require.resolve('./dev_tools_spaces'));
   });

x-pack/test/functional/apps/visualize/index.ts

@@ -8,7 +8,7 @@ import { FtrProviderContext } from '../../ftr_provider_context';
 
 export default function visualize({ loadTestFile }: FtrProviderContext) {
   describe('Visualize', function visualizeTestSuite() {
-    this.tags(['ciGroup4', 'skipFirefox']);
+    this.tags(['ciGroup4', 'skipFirefox', 'skipCoverage']);
 
     loadTestFile(require.resolve('./feature_controls/visualize_security'));
     loadTestFile(require.resolve('./feature_controls/visualize_spaces'));

x-pack/test/functional/config.coverage.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+export default async function({ readConfigFile }) {
+  const chromeConfig = await readConfigFile(require.resolve('./config'));
+
+  return {
+    ...chromeConfig.getAll(),
+
+    suiteTags: {
+      exclude: ['skipCoverage'],
+    },
+
+    junit: {
+      reportName: 'Code Coverage for Functional Tests',
+    },
+  };
+}