Merge remote-tracking branch 'upstream/master' into pipeline-refactoring

.ci/Jenkinsfile_coverage

@@ -9,13 +9,7 @@ kibanaPipeline(timeoutMinutes: 180) {
       'CODE_COVERAGE=1', // Needed for multiple ci scripts, such as remote.ts, test/scripts/*.sh, schema.js, etc.
     ]) {
       parallel([
-        'kibana-intake-agent': {
-          withEnv([
-            'NODE_ENV=test' // Needed for jest tests only
-          ]) {
-            workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh')()
-          }
-        },
+        'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
         'x-pack-intake-agent': {
           withEnv([
             'NODE_ENV=test' // Needed for jest tests only

.ci/end2end.groovy

@@ -13,7 +13,7 @@ pipeline {
     BASE_DIR = 'src/github.com/elastic/kibana'
     HOME = "${env.WORKSPACE}"
     APM_ITS = 'apm-integration-testing'
-    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/cypress'
+    CYPRESS_DIR = 'x-pack/legacy/plugins/apm/e2e'
     PIPELINE_LOG_LEVEL = 'DEBUG'
   }
   options {
@@ -25,7 +25,7 @@ pipeline {
     durabilityHint('PERFORMANCE_OPTIMIZED')
   }
   triggers {
-    issueCommentTrigger('(?i).*jenkins\\W+run\\W+(?:the\\W+)?e2e(?:\\W+please)?.*')
+    issueCommentTrigger('(?i)(retest|.*jenkins\\W+run\\W+(?:the\\W+)?e2e?.*)')
   }
   parameters {
     booleanParam(name: 'FORCE', defaultValue: false, description: 'Whether to force the run.')
@@ -60,8 +60,14 @@ pipeline {
         }
       }
       steps {
+        notifyStatus('Starting services', 'PENDING')
         dir("${APM_ITS}"){
-          sh './scripts/compose.py start master --no-kibana --no-xpack-secure'
+          sh './scripts/compose.py start master --no-kibana'
+        }
+      }
+      post {
+        unsuccessful {
+          notifyStatus('Environmental issue', 'FAILURE')
         }
       }
     }
@@ -77,10 +83,16 @@ pipeline {
         JENKINS_NODE_COOKIE = 'dontKillMe'
       }
       steps {
+        notifyStatus('Preparing kibana', 'PENDING')
         dir("${BASE_DIR}"){
           sh script: "${CYPRESS_DIR}/ci/prepare-kibana.sh"
         }
       }
+      post {
+        unsuccessful {
+          notifyStatus('Kibana warm up failed', 'FAILURE')
+        }
+      }
     }
     stage('Smoke Tests'){
       options { skipDefaultCheckout() }
@@ -91,10 +103,11 @@ pipeline {
         }
       }
       steps{
+        notifyStatus('Running smoke tests', 'PENDING')
         dir("${BASE_DIR}"){
           sh '''
             jobs -l
-            docker build --tag cypress ${CYPRESS_DIR}/ci
+            docker build --tag cypress --build-arg NODE_VERSION=$(cat .node-version) ${CYPRESS_DIR}/ci
             docker run --rm -t --user "$(id -u):$(id -g)" \
                     -v `pwd`:/app --network="host" \
                     --name cypress cypress'''
@@ -103,15 +116,21 @@ pipeline {
       post {
         always {
           dir("${BASE_DIR}"){
-            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/screenshots/**,${CYPRESS_DIR}/videos/**,${CYPRESS_DIR}/*e2e-tests.xml")
-            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/*e2e-tests.xml")
+            archiveArtifacts(allowEmptyArchive: false, artifacts: "${CYPRESS_DIR}/**/screenshots/**,${CYPRESS_DIR}/**/videos/**,${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
+            junit(allowEmptyResults: true, testResults: "${CYPRESS_DIR}/**/test-results/*e2e-tests.xml")
           }
           dir("${APM_ITS}"){
             sh 'docker-compose logs > apm-its.log || true'
             sh 'docker-compose down -v || true'
             archiveArtifacts(allowEmptyArchive: false, artifacts: 'apm-its.log')
           }
         }
+        unsuccessful {
+          notifyStatus('Test failures', 'FAILURE')
+        }
+        success {
+          notifyStatus('Tests passed', 'SUCCESS')
+        }
       }
     }
   }
@@ -123,3 +142,7 @@ pipeline {
     }
   }
 }
+
+def notifyStatus(String description, String status) {
+  withGithubNotify.notify('end2end-for-apm-ui', description, status, getBlueoceanDisplayURL())
+}

.eslintignore

@@ -39,7 +39,7 @@ src/legacy/core_plugins/vis_type_vislib/public/vislib/__tests__/lib/fixtures/moc
 /x-pack/legacy/plugins/infra/common/graphql/types.ts
 /x-pack/legacy/plugins/infra/public/graphql/types.ts
 /x-pack/legacy/plugins/infra/server/graphql/types.ts
-/x-pack/legacy/plugins/apm/cypress/**/snapshots.js
+/x-pack/legacy/plugins/apm/e2e/cypress/**/snapshots.js
 /src/legacy/plugin_discovery/plugin_pack/__tests__/fixtures/plugins/broken
 **/graphql/types.ts
 **/*.js.snap

.eslintrc.js

@@ -82,12 +82,6 @@ module.exports = {
         'react-hooks/exhaustive-deps': 'off',
       },
     },
-    {
-      files: ['src/legacy/core_plugins/vis_type_table/**/*.{js,ts,tsx}'],
-      rules: {
-        'react-hooks/exhaustive-deps': 'off',
-      },
-    },
     {
       files: [
         'src/legacy/core_plugins/vis_default_editor/public/components/controls/**/*.{ts,tsx}',

.github/CODEOWNERS

@@ -54,6 +54,7 @@
 /src/plugins/ui_actions/ @elastic/kibana-app-arch
 /src/plugins/visualizations/ @elastic/kibana-app-arch
 /x-pack/plugins/advanced_ui_actions/ @elastic/kibana-app-arch
+/x-pack/plugins/drilldowns/ @elastic/kibana-app-arch
 
 # APM
 /x-pack/legacy/plugins/apm/  @elastic/apm-ui
@@ -72,8 +73,10 @@
 # Observability UIs
 /x-pack/legacy/plugins/infra/ @elastic/logs-metrics-ui
 /x-pack/plugins/infra/ @elastic/logs-metrics-ui
-/x-pack/legacy/plugins/integrations_manager/ @elastic/epm
+/x-pack/plugins/ingest_manager/ @elastic/ingest
+/x-pack/legacy/plugins/ingest_manager/ @elastic/ingest
 /x-pack/plugins/observability/ @elastic/logs-metrics-ui @elastic/apm-ui @elastic/uptime @elastic/ingest
+/x-pack/legacy/plugins/monitoring/ @elastic/stack-monitoring-ui
 
 # Machine Learning
 /x-pack/legacy/plugins/ml/  @elastic/ml-ui
@@ -149,13 +152,15 @@
 # Kibana Alerting Services
 /x-pack/legacy/plugins/alerting/ @elastic/kibana-alerting-services
 /x-pack/legacy/plugins/actions/ @elastic/kibana-alerting-services
+/x-pack/plugins/alerting/ @elastic/kibana-alerting-services
 /x-pack/plugins/actions/ @elastic/kibana-alerting-services
 /x-pack/plugins/event_log/ @elastic/kibana-alerting-services
 /x-pack/plugins/task_manager/ @elastic/kibana-alerting-services
 /x-pack/test/alerting_api_integration/ @elastic/kibana-alerting-services
 /x-pack/test/plugin_api_integration/plugins/task_manager/ @elastic/kibana-alerting-services
 /x-pack/test/plugin_api_integration/test_suites/task_manager/ @elastic/kibana-alerting-services
 /x-pack/legacy/plugins/triggers_actions_ui/ @elastic/kibana-alerting-services
+/x-pack/plugins/triggers_actions_ui/ @elastic/kibana-alerting-services
 /x-pack/test/functional_with_es_ssl/apps/triggers_actions_ui/ @elastic/kibana-alerting-services
 /x-pack/test/functional_with_es_ssl/fixtures/plugins/alerts/ @elastic/kibana-alerting-services
 
@@ -170,8 +175,27 @@
 /x-pack/legacy/plugins/index_lifecycle_management/  @elastic/es-ui
 /x-pack/legacy/plugins/index_management/  @elastic/es-ui
 /x-pack/legacy/plugins/license_management/  @elastic/es-ui
-/x-pack/legacy/plugins/remote_clusters/  @elastic/es-ui
+/x-pack/plugins/remote_clusters/  @elastic/es-ui
 /x-pack/legacy/plugins/rollup/  @elastic/es-ui
 /x-pack/plugins/searchprofiler/  @elastic/es-ui
 /x-pack/legacy/plugins/snapshot_restore/  @elastic/es-ui
+/x-pack/legacy/plugins/upgrade_assistant/  @elastic/es-ui
+/x-pack/plugins/upgrade_assistant/  @elastic/es-ui
 /x-pack/plugins/watcher/  @elastic/es-ui
+
+# Endpoint
+/x-pack/plugins/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/api_integration/apis/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/functional/apps/endpoint/ @elastic/endpoint-app-team
+/x-pack/test/functional/es_archives/endpoint/ @elastic/endpoint-app-team
+
+# SIEM
+/x-pack/legacy/plugins/siem/ @elastic/siem
+/x-pack/plugins/siem/ @elastic/siem
+/x-pack/test/detection_engine_api_integration @elastic/siem
+/x-pack/test/api_integration/apis/siem @elastic/siem
+/x-pack/plugins/case @elastic/siem
+
+# Security Intelligence And Analytics
+/x-pack/legacy/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics
+/x-pack/plugins/siem/server/lib/detection_engine/rules/prepackaged_rules @elastic/security-intelligence-analytics

.github/paths-labeller.yml

@@ -0,0 +1,3 @@
+---
+  - "Feature:Drilldowns":
+    - "x-pack/plugins/drilldowns/**/*.*"

.github/workflows/pr-project-assigner.yml

@@ -8,7 +8,7 @@ jobs:
     name: Assign a PR to project based on label
     steps:
       - name: Assign to project
-        uses: elastic/github-actions/project-assigner@v1.0.2
+        uses: elastic/github-actions/project-assigner@v1.0.3
         id: project_assigner
         with:
           issue-mappings: |
@@ -17,4 +17,4 @@ jobs:
               { "label": "Feature:Lens", "projectName": "Lens", "columnId": 6219362 },
               { "label": "Team:Canvas", "projectName": "canvas", "columnId": 6187580 }
             ]
-          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          ghToken: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}

.github/workflows/project-assigner.yml

@@ -8,10 +8,10 @@ jobs:
     name: Assign issue or PR to project based on label
     steps:
       - name: Assign to project
-        uses: elastic/github-actions/project-assigner@v1.0.2
+        uses: elastic/github-actions/project-assigner@v1.0.3
         id: project_assigner
         with:
           issue-mappings: '[{"label": "Team:AppArch", "projectName": "kibana-app-arch", "columnId": 6173895}, {"label": "Feature:Lens", "projectName": "Lens", "columnId": 6219363}, {"label": "Team:Canvas", "projectName": "canvas", "columnId": 6187593}]'
-          ghToken: ${{ secrets.GITHUB_TOKEN }}
+          ghToken: ${{ secrets.PROJECT_ASSIGNER_TOKEN }}
 
 

CONTRIBUTING.md

@@ -55,11 +55,9 @@ Granted that you share your thoughts, we might even be able to come up with crea
 
 First of all, **sorry about that!** We want you to have a great time with Kibana.
 
-Hosting meaningful discussions on GitHub can be challenging. For that reason, we'll sometimes ask that you join us on IRC _([#kibana](https://kiwiirc.com/client/irc.freenode.net/?#kibana) on freenode)_ to chat about your issues. You may also experience **faster response times** when engaging us via IRC.
-
 There's hundreds of open issues and prioritizing what to work on is an important aspect of our daily jobs. We prioritize issues according to impact and difficulty, so some issues can be neglected while we work on more pressing issues.
 
-Feel free to bump your issues if you think they've been neglected for a prolonged period, or just jump on IRC and let us have it!
+Feel free to bump your issues if you think they've been neglected for a prolonged period.
 
 ### "I want to help!"
 
@@ -471,11 +469,11 @@ The following table outlines possible test file locations and how to invoke them
 
 | Test runner        | Test location                                                                                                                                           | Runner command (working directory is kibana root)                                       |
 | -----------------  | ------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------- |
-| Jest               | `src/**/*.test.js`<br>`src/**/*.test.ts`                                                                                                                | `node scripts/jest -t regexp [test path]`                                               |
-| Jest (integration) | `**/integration_tests/**/*.test.js`                                                                                                                     | `node scripts/jest_integration -t regexp [test path]`                                   |
+| Jest               | `src/**/*.test.js`<br>`src/**/*.test.ts`                                                                                                                | `yarn test:jest -t regexp [test path]`                                               |
+| Jest (integration) | `**/integration_tests/**/*.test.js`                                                                                                                     | `yarn test:jest_integration -t regexp [test path]`                                   |
 | Mocha              | `src/**/__tests__/**/*.js`<br>`!src/**/public/__tests__/*.js`<br>`packages/kbn-datemath/test/**/*.js`<br>`packages/kbn-dev-utils/src/**/__tests__/**/*.js`<br>`tasks/**/__tests__/**/*.js` | `node scripts/mocha --grep=regexp [test path]`       |
-| Functional         | `test/*integration/**/config.js`<br>`test/*functional/**/config.js`<br>`test/accessibility/config.js`                                                                                    | `node scripts/functional_tests_server --config test/[directory]/config.js`<br>`node scripts/functional_test_runner --config test/[directory]/config.js --grep=regexp`       |
-| Karma              | `src/**/public/__tests__/*.js`                                                                                                                          | `npm run test:dev`                                                                      |
+| Functional         | `test/*integration/**/config.js`<br>`test/*functional/**/config.js`<br>`test/accessibility/config.js`                                                                                    | `yarn test:ftr:server --config test/[directory]/config.js`<br>`yarn test:ftr:runner --config test/[directory]/config.js --grep=regexp`       |
+| Karma              | `src/**/public/__tests__/*.js`                                                                                                                          | `yarn test:karma:debug`                                                                      |
 
 For X-Pack tests located in `x-pack/` see [X-Pack Testing](x-pack/README.md#testing)
 
@@ -486,56 +484,38 @@ Test runner arguments:
  Examples:
   - Run the entire elasticsearch_service test suite:
     ```
-    node scripts/jest src/core/server/elasticsearch/elasticsearch_service.test.ts
+    yarn test:jest src/core/server/elasticsearch/elasticsearch_service.test.ts
     ```
   - Run the jest test case whose description matches `stops both admin and data clients`:
     ```
-    node scripts/jest -t 'stops both admin and data clients' src/core/server/elasticsearch/elasticsearch_service.test.ts
+    yarn test:jest -t 'stops both admin and data clients' src/core/server/elasticsearch/elasticsearch_service.test.ts
     ```
   - Run the api integration test case whose description matches the given string:
     ```
-    node scripts/functional_tests_server --config test/api_integration/config.js
-    node scripts/functional_test_runner --config test/api_integration/config.js --grep='should return 404 if id does not match any sample data sets'
+    yarn test:ftr:server --config test/api_integration/config.js
+    yarn test:ftr:runner --config test/api_integration/config.js --grep='should return 404 if id does not match any sample data sets'
     ```
 
 ### Debugging Unit Tests
 
 The standard `yarn test` task runs several sub tasks and can take several minutes to complete, making debugging failures pretty painful. In order to ease the pain specialized tasks provide alternate methods for running the tests.
 
-To execute both server and browser tests, but skip linting, use `yarn test:quick`.
-
-```bash
-yarn test:quick
-```
-
-Use `yarn test:mocha` when you want to run the mocha tests.
-
-```bash
-yarn test:mocha
-```
-
-When you'd like to execute individual server-side test files, you can use the command below. Note that this command takes care of configuring Mocha with Babel compilation for you, and you'll be better off avoiding a globally installed `mocha` package. This command is great for development and for quickly identifying bugs.
-
-```bash
-node scripts/mocha <file>
-```
-
 You could also add the `--debug` option so that `node` is run using the `--debug-brk` flag. You'll need to connect a remote debugger such as [`node-inspector`](https://github.com/node-inspector/node-inspector) to proceed in this mode.
 
 ```bash
 node scripts/mocha --debug <file>
 ```
 
-With `yarn test:browser`, you can run only the browser tests. Coverage reports are available for browser tests by running `yarn test:coverage`. You can find the results under the `coverage/` directory that will be created upon completion.
+With `yarn test:karma`, you can run only the browser tests. Coverage reports are available for browser tests by running `yarn test:coverage`. You can find the results under the `coverage/` directory that will be created upon completion.
 
 ```bash
-yarn test:browser
+yarn test:karma
 ```
 
-Using `yarn test:dev` initializes an environment for debugging the browser tests. Includes an dedicated instance of the kibana server for building the test bundle, and a karma server. When running this task the build is optimized for the first time and then a karma-owned instance of the browser is opened. Click the "debug" button to open a new tab that executes the unit tests.
+Using `yarn test:karma:debug` initializes an environment for debugging the browser tests. Includes an dedicated instance of the kibana server for building the test bundle, and a karma server. When running this task the build is optimized for the first time and then a karma-owned instance of the browser is opened. Click the "debug" button to open a new tab that executes the unit tests.
 
 ```bash
-yarn test:dev
+yarn test:karma:debug
 ```
 
 In the screenshot below, you'll notice the URL is `localhost:9876/debug.html`. You can append a `grep` query parameter to this URL and set it to a string value which will be used to exclude tests which don't match. For example, if you changed the URL to `localhost:9876/debug.html?query=my test` and then refreshed the browser, you'd only see tests run which contain "my test" in the test description.
@@ -551,7 +531,7 @@ To run the tests for just your particular plugin run the following command from
 
 ```bash
 yarn test:mocha
-yarn test:browser --dev # remove the --dev flag to run them once and close
+yarn test:karma:debug # remove the debug flag to run them once and close
 ```
 
 ### Automated Accessibility Testing

Jenkinsfile

@@ -11,7 +11,7 @@ kibanaPipeline(timeoutMinutes: 135) {
         'kibana-intake-agent': workers.intake('kibana-intake', './test/scripts/jenkins_unit.sh'),
         'x-pack-intake-agent': workers.intake('x-pack-intake', './test/scripts/jenkins_xpack.sh'),
         'kibana-oss-agent': workers.functional('kibana-oss-tests', { kibanaPipeline.buildOss() }, [
-          'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
+          // 'oss-firefoxSmoke': kibanaPipeline.functionalTestProcess('kibana-firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh'),
           'oss-ciGroup1': kibanaPipeline.ossCiGroupProcess(1),
           'oss-ciGroup2': kibanaPipeline.ossCiGroupProcess(2),
           'oss-ciGroup3': kibanaPipeline.ossCiGroupProcess(3),
@@ -28,7 +28,7 @@ kibanaPipeline(timeoutMinutes: 135) {
           // 'oss-visualRegression': kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh'),
         ]),
         'kibana-xpack-agent': workers.functional('kibana-xpack-tests', { kibanaPipeline.buildXpack() }, [
-          'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
+          // 'xpack-firefoxSmoke': kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh'),
           'xpack-ciGroup1': kibanaPipeline.xpackCiGroupProcess(1),
           'xpack-ciGroup2': kibanaPipeline.xpackCiGroupProcess(2),
           'xpack-ciGroup3': kibanaPipeline.xpackCiGroupProcess(3),