From cc4e428d17e6bfbb4e865561a3d61e554bd994e4 Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Tue, 11 Jun 2024 11:49:44 +0200 Subject: [PATCH] [8.14][BK] Migrate kibana on merge 8.14 (#184513) ## Summary Backport of #184391 to 8.14 Migration staging runs: - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/119 - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/122 Was flaky on Cypress tests, but after a retry it went away. Maybe it's the usual cypress flakiness? --- .../kibana-on-merge.yml | 49 +++++ .../locations.yml | 1 + .buildkite/pipelines/on_merge.yml | 169 +++++++++++++++--- 3 files changed, 190 insertions(+), 29 deletions(-) create mode 100644 .buildkite/pipeline-resource-definitions/kibana-on-merge.yml diff --git a/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml b/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml new file mode 100644 index 00000000000000..da2086bacb3ee2 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml @@ -0,0 +1,49 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-on-merge + description: 'Runs for each commit of Kibana, i.e. each time a PR is merged' + links: + - url: 'https://buildkite.com/elastic/kibana-on-merge' + title: Pipeline link +spec: + type: buildkite-pipeline + owner: 'group:kibana-operations' + system: buildkite + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / on merge + description: 'Runs for each commit of Kibana, i.e. each time a PR is merged' + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' + GITHUB_BUILD_COMMIT_STATUS_ENABLED: 'true' + GITHUB_COMMIT_STATUS_CONTEXT: buildkite/on-merge + REPORT_FAILED_TESTS_TO_GITHUB: 'true' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: true + branch_configuration: main 7.17 8.* + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/on_merge.yml + skip_intermediate_builds: false + provider_settings: + build_branches: true + build_pull_requests: false + publish_commit_status: false + trigger_mode: code + build_tags: false + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: false + teams: + everyone: + access_level: BUILD_AND_READ + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index 442445b9c0b937..5ab8fa644d3915 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -21,6 +21,7 @@ spec: - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-fleet-packages-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-migration-staging.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge-unsupported-ftrs.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-on-merge.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml diff --git a/.buildkite/pipelines/on_merge.yml b/.buildkite/pipelines/on_merge.yml index 61739d554176db..16dd398cea9755 100644 --- a/.buildkite/pipelines/on_merge.yml +++ b/.buildkite/pipelines/on_merge.yml @@ -6,7 +6,10 @@ steps: label: Pre-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 retry: automatic: - exit_status: '*' @@ -16,7 +19,11 @@ steps: - label: 'Triggering changes-based pipelines' branches: main agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + # TODO: this can probably be deleted after the migration https://github.com/elastic/kibana-operations/issues/15 plugins: - chronotc/monorepo-diff#v2.0.4: watch: @@ -26,12 +33,19 @@ steps: command: 'ts-node .buildkite/scripts/steps/trigger_pipeline.ts kibana-buildkite-pipelines-deploy main' label: 'Trigger pipeline deploy' agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - command: .buildkite/scripts/steps/on_merge_build_and_metrics.sh label: Build Kibana Distribution and Plugins agents: - queue: n2-16-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-16 + preemptible: true key: build timeout_in_minutes: 60 retry: @@ -42,7 +56,11 @@ steps: - command: .buildkite/scripts/steps/quick_checks.sh label: 'Quick Checks' agents: - queue: n2-2-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + preemptible: true key: quick_checks timeout_in_minutes: 60 retry: @@ -56,7 +74,11 @@ steps: label: Check Public API Docs key: public-api-docs agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 80 retry: automatic: @@ -68,7 +90,10 @@ steps: - command: .buildkite/scripts/steps/ci_stats_ready.sh label: Mark CI Stats as ready agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10 depends_on: - build @@ -81,7 +106,10 @@ steps: - command: .buildkite/scripts/steps/test/pick_test_group_run_order.sh label: 'Pick Test Group Run Order' agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 timeout_in_minutes: 10 env: JEST_UNIT_SCRIPT: '.buildkite/scripts/steps/test/jest.sh' @@ -95,7 +123,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_rule_management.sh label: 'Rule Management - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -109,7 +141,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_rule_management_prebuilt_rules.sh label: 'Rule Management - Prebuilt Rules - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -123,7 +159,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_detection_engine.sh label: 'Detection Engine - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -137,7 +177,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_detection_engine_exceptions.sh label: 'Detection Engine - Exceptions - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -151,7 +195,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_ai_assistant.sh label: 'AI Assistant - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -165,7 +213,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_entity_analytics.sh label: 'Entity Analytics - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -179,7 +231,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_explore.sh label: 'Explore - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -193,7 +249,11 @@ steps: - command: .buildkite/scripts/steps/functional/security_solution_investigations.sh label: 'Investigations - Security Solution Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -207,7 +267,11 @@ steps: - command: .buildkite/scripts/steps/functional/threat_intelligence.sh label: 'Threat Intelligence Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -221,7 +285,11 @@ steps: - command: .buildkite/scripts/steps/functional/osquery_cypress.sh label: 'Osquery Cypress Tests' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true depends_on: - build - quick_checks @@ -235,7 +303,13 @@ steps: - command: .buildkite/scripts/steps/functional/defend_workflows.sh label: 'Defend Workflows Cypress Tests' agents: - queue: n2-4-virt + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + enableNestedVirtualization: true + localSsds: 1 + localSsdInterface: nvme + machineType: n2-standard-4 depends_on: - build - quick_checks @@ -253,12 +327,19 @@ steps: - build - quick_checks agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - command: .buildkite/scripts/steps/lint.sh label: 'Linting' agents: - queue: n2-8-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true key: linting timeout_in_minutes: 60 retry: @@ -269,7 +350,11 @@ steps: - command: .buildkite/scripts/steps/lint_with_types.sh label: 'Linting (with types)' agents: - queue: n2-16-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-16 + preemptible: true key: linting_with_types timeout_in_minutes: 90 retry: @@ -280,7 +365,11 @@ steps: - command: .buildkite/scripts/steps/checks.sh label: 'Checks' agents: - queue: n2-2-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 + preemptible: true timeout_in_minutes: 60 retry: automatic: @@ -290,7 +379,11 @@ steps: - command: .buildkite/scripts/steps/check_types.sh label: 'Check Types' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 60 retry: automatic: @@ -300,7 +393,11 @@ steps: - command: .buildkite/scripts/steps/storybooks/build_and_upload.sh label: 'Build Storybooks' agents: - queue: n2-8-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-8 + preemptible: true key: storybooks timeout_in_minutes: 80 retry: @@ -311,7 +408,11 @@ steps: - command: .buildkite/scripts/steps/bazel_cache/bootstrap_linux.sh label: 'Populate local dev bazel cache (Linux)' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true timeout_in_minutes: 30 retry: automatic: @@ -321,7 +422,11 @@ steps: - command: .buildkite/scripts/steps/archive_so_migration_snapshot.sh target/plugin_so_types_snapshot.json label: 'Extract Saved Object migration plugin types' agents: - queue: n2-4-spot + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-4 + preemptible: true artifact_paths: "target/plugin_so_types_snapshot.json" @@ -332,7 +437,10 @@ steps: label: Post-Build timeout_in_minutes: 10 agents: - queue: kibana-default + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2 - wait @@ -340,4 +448,7 @@ steps: label: Trigger container image build timeout_in_minutes: 10 agents: - queue: 'kibana-default' + image: family/kibana-ubuntu-2004 + imageProject: elastic-images-prod + provider: gcp + machineType: n2-standard-2