Merge branch 'main' into im/indices_list/add_columns

.buildkite/pipelines/security_solution/security_solution_cypress.yml

@@ -1,16 +1,4 @@
 steps:
-  - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless
-    label: 'Serverless MKI QA Security Cypress Tests'
-    agents:
-      queue: n2-4-spot
-    # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
-    timeout_in_minutes: 300
-    parallelism: 6
-    retry:
-      automatic:
-        - exit_status: '*'
-          limit: 1
-
   - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:explore
     label: 'Serverless MKI QA Explore - Security Solution Cypress Tests'
     agents:
@@ -53,7 +41,7 @@ steps:
       queue: n2-4-spot
     # TODO : Revise the timeout when the pipeline will be officially integrated with the quality gate.
     timeout_in_minutes: 300
-    parallelism: 6
+    parallelism: 2
     retry:
       automatic:
         - exit_status: '*'
@@ -93,7 +81,7 @@ steps:
     retry:
       automatic:
         - exit_status: '*'
-          limit: 1   
+          limit: 1
 
   - command: .buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh cypress:run:qa:serverless:entity_analytics
     label: 'Serverless MKI QA Entity Analytics - Security Solution Cypress Tests'
@@ -105,4 +93,4 @@ steps:
     retry:
       automatic:
         - exit_status: '*'
-          limit: 1                    
+          limit: 1

.buildkite/scripts/common/util.sh

@@ -173,29 +173,29 @@ download_artifact() {
 
 
 vault_get() {
-  path=$1
+  key_path=$1
   field=$2
 
-  fullPath="secret/ci/elastic-kibana/$path"
+  fullPath="secret/ci/elastic-kibana/$key_path"
   if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    fullPath="secret/kibana-issues/dev/$path"
+    fullPath="secret/kibana-issues/dev/$key_path"
   fi
 
-  if [[ -z "${2:-}" ]]; then
-    retry 5 5 vault read "$fullPath"
+  if [[ -z "${2:-}" || "${2:-}" =~ ^-.* ]]; then
+    retry 5 5 vault read "$fullPath" "${@:2}"
   else
-    retry 5 5 vault read -field="$field" "$fullPath"
+    retry 5 5 vault read -field="$field" "$fullPath" "${@:3}"
   fi
 }
 
 vault_set() {
-  path=$1
+  key_path=$1
   shift
   fields=("$@")
 
-  fullPath="secret/ci/elastic-kibana/$path"
+  fullPath="secret/ci/elastic-kibana/$key_path"
   if [[ "$VAULT_ADDR" == *"secrets.elastic.co"* ]]; then
-    fullPath="secret/kibana-issues/dev/$path"
+    fullPath="secret/kibana-issues/dev/$key_path"
   fi
 
   # shellcheck disable=SC2068

.buildkite/scripts/pipelines/pull_request/pipeline.ts

@@ -88,7 +88,10 @@ const uploadPipeline = (pipelineContent: string | object) => {
     }
 
     if (
-      (await doAnyChangesMatch([/^x-pack\/plugins\/observability_onboarding/])) ||
+      (await doAnyChangesMatch([
+        /^x-pack\/plugins\/observability_onboarding/,
+        /^x-pack\/plugins\/fleet/,
+      ])) ||
       GITHUB_PR_LABELS.includes('ci:all-cypress-suites')
     ) {
       pipeline.push(

.buildkite/scripts/pipelines/security_solution_quality_gate/api-integration-tests.sh

@@ -14,7 +14,7 @@ echo "--- Serverless Security Second Quality Gate"
 cd x-pack/test/security_solution_api_integration
 set +e
 
-QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)
+QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key)
 
 # Generate a random 5-digit number
 random_number=$((10000 + $RANDOM % 90000))

.buildkite/scripts/pipelines/security_solution_quality_gate/security_solution_cypress/mki_security_solution_cypress.sh

@@ -17,11 +17,12 @@ export JOB=kibana-security-solution-chrome
 buildkite-agent meta-data set "${BUILDKITE_JOB_ID}_is_test_execution_step" "true"
 
 mkdir .ftr
-retry 5 5 vault kv get -format=json -field=data secret/kibana-issues/dev/security-quality-gate/role-users > .ftr/role_users.json
+vault_get security-quality-gate/role-users data -format=json > .ftr/role_users.json
 
 cd x-pack/test/security_solution_cypress
 set +e
 
-QA_API_KEY=$(vault_get security-solution-qg-enc-key qa_api_key)
+QA_API_KEY=$(vault_get security-solution-quality-gate qa_api_key)
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-quality-gate serverless-sec-sol-cypress-bk-api-key)
 
-CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY CLOUD_QA_API_KEY=$QA_API_KEY yarn $1; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh

@@ -6,5 +6,5 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Elastic Assistant OpenAPI Code Generation
 
-(cd x-pack/plugins/elastic_assistant && yarn openapi:generate)
+(cd x-pack/packages/kbn-elastic-assistant-common && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true

.buildkite/scripts/steps/functional/defend_workflows.sh

@@ -13,4 +13,6 @@ echo "--- Defend Workflows Cypress tests"
 cd x-pack/plugins/security_solution
 
 set +e
-yarn cypress:dw:run; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci defend-workflows-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:run; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/defend_workflows_serverless.sh

@@ -13,4 +13,6 @@ echo "--- Defend Workflows Cypress tests on Serverless"
 cd x-pack/plugins/security_solution
 
 set +e
-yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci defend-workflows-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:dw:serverless:run; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_ai_assistant.sh

@@ -13,4 +13,6 @@ echo "--- AI Assistant Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:ai_assistant:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:ai_assistant:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_detection_engine.sh

@@ -13,4 +13,6 @@ echo "--- Detection Engine Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:detection_engine:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_detection_engine_exceptions.sh

@@ -13,4 +13,6 @@ echo "--- Detection Engine - Exceptions - Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:detection_engine:exceptions:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:exceptions:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_entity_analytics.sh

@@ -13,4 +13,6 @@ echo "--- Entity Analytics Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:entity_analytics:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:entity_analytics:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_explore.sh

@@ -13,4 +13,6 @@ echo "--- Explore - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:explore:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:explore:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_investigations.sh

@@ -13,4 +13,6 @@ echo "--- Investigations Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:investigations:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:investigations:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_rule_management.sh

@@ -13,4 +13,6 @@ echo "--- Rule Management Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:rule_management:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:rule_management:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_serverless_rule_management_prebuilt_rules.sh

@@ -13,4 +13,6 @@ echo "--- Rule Management - Prebuilt Rules - Cypress Tests on Serverless"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:rule_management:prebuilt_rules:run:serverless; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:rule_management:prebuilt_rules:run:serverless; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_ai_assistant.sh

@@ -13,4 +13,6 @@ echo "--- AI Assistant - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:ai_assistant:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:ai_assistant:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_detection_engine.sh

@@ -13,4 +13,6 @@ echo "--- Detection Engine - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:detection_engine:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh

@@ -13,4 +13,6 @@ echo "--- Detection Engine - Exceptions - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:detection_engine:exceptions:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:detection_engine:exceptions:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_entity_analytics.sh

@@ -13,4 +13,6 @@ echo "--- Entity Analytics - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:entity_analytics:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:entity_analytics:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_explore.sh

@@ -13,4 +13,6 @@ echo "--- Explore Cypress Tests on Security Solution"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:explore:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:explore:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_investigations.sh

@@ -13,4 +13,6 @@ echo "--- Investigations - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:investigations:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:investigations:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_rule_management.sh

@@ -13,4 +13,6 @@ echo "--- Rule Management - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:rule_management:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:rule_management:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh

@@ -13,4 +13,6 @@ echo "--- Rule Management - Prebuilt Rules - Security Solution Cypress Tests"
 cd x-pack/test/security_solution_cypress
 
 set +e
-yarn cypress:rule_management:prebuilt_rules:run:ess; status=$?; yarn junit:merge || :; exit $status
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci sec-sol-cypress-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn cypress:rule_management:prebuilt_rules:run:ess; status=$?; yarn junit:merge || :; exit $status

.buildkite/scripts/steps/functional/threat_intelligence.sh

@@ -10,4 +10,6 @@ export KIBANA_INSTALL_DIR=${KIBANA_BUILD_LOCATION}
 
 echo "--- Threat Intelligence Cypress tests (Chrome)"
 
-yarn --cwd x-pack/plugins/threat_intelligence cypress:run
+BK_ANALYTICS_API_KEY=$(vault_get security-solution-ci threat-intelligence-bk-api-key)
+
+BK_ANALYTICS_API_KEY=$BK_ANALYTICS_API_KEY yarn --cwd x-pack/plugins/threat_intelligence cypress:run

.github/CODEOWNERS

@@ -587,10 +587,15 @@ packages/kbn-peggy @elastic/kibana-operations
 packages/kbn-peggy-loader @elastic/kibana-operations
 packages/kbn-performance-testing-dataset-extractor @elastic/kibana-performance-testing
 packages/kbn-picomatcher @elastic/kibana-operations
+packages/kbn-plugin-check @elastic/appex-sharedux
 packages/kbn-plugin-generator @elastic/kibana-operations
 packages/kbn-plugin-helpers @elastic/kibana-operations
 examples/portable_dashboards_example @elastic/kibana-presentation
 examples/preboot_example @elastic/kibana-security @elastic/kibana-core
+packages/presentation/presentation_containers @elastic/kibana-presentation
+packages/presentation/presentation_library @elastic/kibana-presentation
+src/plugins/presentation_panel @elastic/kibana-presentation
+packages/presentation/presentation_publishing @elastic/kibana-presentation
 src/plugins/presentation_util @elastic/kibana-presentation
 x-pack/plugins/profiling_data_access @elastic/obs-ux-infra_services-team
 x-pack/plugins/profiling @elastic/obs-ux-infra_services-team

.i18nrc.json

@@ -30,6 +30,7 @@
     "discover": ["src/plugins/discover", "packages/kbn-discover-utils"],
     "savedSearch": "src/plugins/saved_search",
     "embeddableApi": "src/plugins/embeddable",
+    "presentationPanel": "src/plugins/presentation_panel",
     "embeddableExamples": "examples/embeddable_examples",
     "esQuery": "packages/kbn-es-query/src",
     "esUi": "src/plugins/es_ui_shared",

config/serverless.oblt.yml

@@ -34,6 +34,10 @@ xpack.observability.createO11yGenericFeatureId: true
 ## APM Serverless Onboarding flow
 xpack.apm.serverlessOnboarding: true
 
+# Synthetics mTLS cert locations
+xpack.uptime.service.tls.certificate: /mnt/elastic-internal/http-certs/tls.crt
+xpack.uptime.service.tls.key: /mnt/elastic-internal/http-certs/tls.key
+
 # Fleet specific configuration
 xpack.fleet.internal.registry.capabilities: ['apm', 'observability']
 xpack.fleet.internal.registry.kibanaVersionCheckEnabled: false
@@ -46,10 +50,6 @@ xpack.fleet.internal.registry.excludePackages: [
     'beaconing',
     'osquery_manager',
 
-    # synthetics is not enabled yet
-    'synthetics',
-    'synthetics_dashboards',
-
     # Removed in 8.11 integrations
     'cisco',
     'microsoft',

config/serverless.security.yml

@@ -16,6 +16,7 @@ xpack.securitySolutionServerless.productTypes:
   [
     { product_line: 'security', product_tier: 'complete' },
     { product_line: 'endpoint', product_tier: 'complete' },
+    { product_line: 'cloud', product_tier: 'complete' },
   ]
 
 xpack.securitySolution.offeringSettings: {

docs/CHANGELOG.asciidoc

@@ -187,7 +187,7 @@ Discover::
 Elastic Security::
 For the Elastic Security 8.12.0 release information, refer to {security-guide}/release-notes.html[_Elastic Security Solution Release Notes_].
 Elastic Search::
-*  Split details panel from model selection list ({kibana-pull}173434[#173434]).
+*  Trained models can now be deployed and started directly from the Machine Learning inference pipeline configuration flyout ({kibana-pull}173434[#173434]).
 Fleet::
 * Adds support for Elasticsearch output performance presets ({kibana-pull}172359[#172359]).
 * Adds a new `keep_monitoring_alive` flag to agent policies ({kibana-pull}168865[#168865]).

docs/api/osquery-manager/live-queries/create.asciidoc

@@ -49,6 +49,8 @@ experimental[] Create live queries.
 
 `metadata`:: (Optional, object) Custom metadata object associated to the live query.
 
+`timeout`:: (Optional, number) A timeout period, in seconds, after which the query will stop running. Overwriting the default timeout allows you to support queries that require more time to complete. The default and minimum supported value is `60`. The maximum supported value is `900`.
+
 
 [[osquery-manager-live-queries-api-create-request-codes]]
 ==== Response code
@@ -69,13 +71,13 @@ Run a live query on all supported agents:
 $ curl -X POST api/osquery/live_queries \
 {
   "query": "select * from uptime;",
-
   "ecs_mapping": {
     "host.uptime": {
       "field": "total_seconds"
     }
   },
   "agent_all": true,
+  "timeout": 120
 }
 
 --------------------------------------------------
@@ -110,6 +112,7 @@ The API returns the live query object:
         "action_id": "609c4c66-ba3d-43fa-afdd-53e244577aa0", # unique ID of the query, use it when querying the live query API to get the single query results
         "id": "6724a474-cbba-41ef-a1aa-66aebf0879e2", # ID of the query, doesn't have to be unique
         "query": "select * from uptime;",
+        "timeout": 120,
         "ecs_mapping": {
           "host.uptime": {
             "field": "total_seconds"