Add serverless emergency release quality gate pipeline - take 2

.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml

@@ -0,0 +1,30 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-kibana-emergency-release
+  description: Emergency release
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-emergency-release
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-operations
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana / emergency release
+    spec:
+      repository: elastic/kibana
+      provider_settings:
+        trigger_mode: none
+      pipeline_file: ".buildkite/pipelines/emergency_release.yml"
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml

@@ -0,0 +1,33 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-tests-emergency-pipeline
+  description: Definition of the kibana pipeline
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-tests-emergency
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-tech-leads
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-tests-emergency
+      description: Pipeline that tests the service integration in various environments
+    spec:
+      repository: elastic/kibana
+      pipeline_file: ./.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        cloud-tooling:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml

@@ -0,0 +1,33 @@
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: kibana-tests-pipeline
+  description: Definition of the kibana pipeline
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/kibana-tests
+spec:
+  type: buildkite-pipeline
+  owner: group:kibana-tech-leads
+  system: buildkite
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      name: kibana-tests
+      description: Pipeline that tests the service integration in various environments
+    spec:
+      repository: elastic/kibana
+      pipeline_file: ./.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
+      provider_settings:
+        trigger_mode: none
+      teams:
+        kibana-operations:
+          access_level: MANAGE_BUILD_AND_READ
+        kibana-release-operators:
+          access_level: BUILD_AND_READ
+        cloud-tooling:
+          access_level: BUILD_AND_READ
+        everyone:
+          access_level: READ_ONLY

.buildkite/pipeline-resource-definitions/locations.yml

@@ -27,6 +27,9 @@ spec:
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-performance-data-set-extraction-daily.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-pr.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-purge-cloud-deployments.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-emergency-release.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates.yml
+    - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-quality-gates-emergency.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release-testing.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-serverless-release.yml
     - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/scalability_testing-daily.yml

.buildkite/pipelines/quality-gates/emergency/pipeline.emergency.kibana-tests.yaml

@@ -0,0 +1,33 @@
+# This pipeline serves as the entry point for your service's quality gates definitions. When
+# properly configured, it will be invoked automatically as part of the automated
+# promotion process once a new version was rolled out in one of the various cloud stages.
+#
+# The updated environment is provided via ENVIRONMENT variable. The seedling
+# step will branch and execute pipeline snippets at the following location:
+# pipeline.tests-qa.yaml
+# pipeline.tests-staging.yaml
+# pipeline.tests-production.yaml
+#
+# Docs: https://docs.elastic.dev/serverless/qualitygates
+
+agents:
+  cpu: 2
+  ephemeralStorage: "20G"
+  memory: "8G"
+
+env:
+  SKIP_NODE_SETUP: true
+  TEAM_CHANNEL: "#kibana-mission-control"
+  ENVIRONMENT: ${ENVIRONMENT?}
+
+steps:
+  - label: ":pipeline::grey_question::seedling: Trigger Kibana Tests for ${ENVIRONMENT}"
+    env:
+      QG_PIPELINE_LOCATION: ".buildkite/pipelines/quality-gates/emergency"
+    command: "make -C /agent run-environment-tests" # will trigger https://buildkite.com/elastic/kibana-tests-emergency
+    agents:
+      image: "docker.elastic.co/ci-agent-images/quality-gate-seedling:0.0.4"
+
+notify:
+  - slack: "${TEAM_CHANNEL?}"
+    if: build.branch == "main" && build.state == "failed"

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-production.yaml

@@ -0,0 +1,37 @@
+# These pipeline steps constitute the quality gate for your service within the production environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":kibana: SLO check"
+    trigger: "serverless-quality-gates" # https://buildkite.com/elastic/serverless-quality-gates
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+      env:
+        TARGET_ENV: production
+        CHECK_SLO: true
+        CHECK_SLO_TAG: kibana
+        CHECK_SLO_WAITING_PERIOD: 15m
+        CHECK_SLO_BURN_RATE_THRESHOLD: 0.1
+    soft_fail: true
+
+  - label: ":rocket: control-plane e2e tests"
+    if: build.env("ENVIRONMENT") == "production-canary"
+    trigger: "ess-k8s-production-e2e-tests" # https://buildkite.com/elastic/ess-k8s-production-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-production.yaml)"
+
+  - label: ":cookie: 24h bake time before continuing promotion"
+    if: build.env("ENVIRONMENT") == "production-canary"
+    command: "sleep 86400"
+    soft_fail:
+      # A manual cancel of that step produces return code 255.
+      # We're treating this case as a soft fail to allow manual bake time skipping.
+      # To stop the promotion entirely, instead click the "Cancel" button at the top of the page
+      - exit_status: 255
+    agents:
+      # How long can this agent live for in minutes - 25 hours
+      instanceMaxAge: 1500

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-qa.yaml

@@ -0,0 +1,12 @@
+# These pipeline steps constitute the quality gate for your service within the QA environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":rocket: control-plane e2e tests"
+    trigger: "ess-k8s-qa-e2e-tests-daily" # https://buildkite.com/elastic/ess-k8s-qa-e2e-tests-daily
+    build:
+      env:
+        REGION_ID: aws-eu-west-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-qa.yaml)"

.buildkite/pipelines/quality-gates/emergency/pipeline.tests-staging.yaml

@@ -0,0 +1,45 @@
+# These pipeline steps constitute the quality gate for your service within the staging environment.
+# Incorporate any necessary additional logic to validate the service's integrity.
+# A failure in this pipeline build will prevent further progression to the subsequent stage.
+
+steps:
+  - label: ":rocket: control-plane e2e tests"
+    trigger: "ess-k8s-staging-e2e-tests" # https://buildkite.com/elastic/ess-k8s-staging-e2e-tests
+    build:
+      env:
+        REGION_ID: aws-us-east-1
+        NAME_PREFIX: ci_test_kibana-promotion_
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
+  - label: ":kibana: Kibana Serverless Tests for ${ENVIRONMENT}"
+    trigger: appex-qa-serverless-kibana-ftr-tests # https://buildkite.com/elastic/appex-qa-serverless-kibana-ftr-tests
+    soft_fail: true # Remove when tests stabilize
+    build:
+      env:
+        ENVIRONMENT: ${ENVIRONMENT}
+        EC_ENV: staging
+        EC_REGION: aws-us-east-1
+        RETRY_TESTS_ON_FAIL: "true"
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+
+  - label: ":rocket: Fleet synthetic monitor to check the long standing project"
+    trigger: "serverless-quality-gates"
+    build:
+      message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
+      env:
+        TARGET_ENV: staging
+        CHECK_SYNTHETICS: true
+        CHECK_SYNTHETICS_TAG: "fleet"
+        CHECK_SYNTHETICS_MINIMUM_RUNS: 3
+        MAX_FAILURES: 2
+        CHECK_SYNTHETIC_MAX_POLL: 50
+    soft_fail: true
+
+  - wait: ~
+
+  - group: "Kibana Release Manager"
+    steps:
+      - label: ":judge::seedling: Trigger Manual Tests Phase"
+        command: "make -C /agent trigger-manual-verification-phase"
+        agents:
+          image: "docker.elastic.co/ci-agent-images/manual-verification-agent:0.0.6"

.buildkite/pipelines/quality-gates/pipeline.tests-staging.yaml

@@ -21,7 +21,7 @@ steps:
         EC_REGION: aws-us-east-1
         RETRY_TESTS_ON_FAIL: "true"
       message: "${BUILDKITE_MESSAGE} (triggered by pipeline.tests-staging.yaml)"
-  
+
   - label: ":rocket: Fleet synthetic monitor to check the long standing project"
     trigger: "serverless-quality-gates"
     build:

catalog-info.yaml

@@ -39,41 +39,6 @@ spec:
   type: url
   target: https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/locations.yml
 
----
-
-apiVersion: backstage.io/v1alpha1
-kind: Resource
-metadata:
-  name: kibana-tests-pipeline
-  description: Definition of the kibana pipeline
-  links:
-    - title: Pipeline
-      url: https://buildkite.com/elastic/kibana-tests
-spec:
-  type: buildkite-pipeline
-  owner: group:kibana-tech-leads
-  system: buildkite
-  implementation:
-    apiVersion: buildkite.elastic.dev/v1
-    kind: Pipeline
-    metadata:
-      name: kibana-tests
-      description: Pipeline that tests the service integration in various environments
-    spec:
-      repository: elastic/kibana
-      pipeline_file: ./.buildkite/pipelines/quality-gates/pipeline.kibana-tests.yaml
-      provider_settings:
-        trigger_mode: none
-      teams:
-        kibana-operations:
-          access_level: MANAGE_BUILD_AND_READ
-        kibana-release-operators:
-          access_level: BUILD_AND_READ
-        cloud-tooling:
-          access_level: BUILD_AND_READ
-        everyone:
-          access_level: READ_ONLY
-
 ---
 apiVersion: backstage.io/v1alpha1
 kind: Component
@@ -189,37 +154,6 @@ spec:
          access_level: MANAGE_BUILD_AND_READ
         everyone:
           access_level: READ_ONLY
----
-# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json
-apiVersion: backstage.io/v1alpha1
-kind: Resource
-metadata:
-  name: buildkite-pipeline-kibana-emergency-release
-  description: Emergency release
-  links:
-    - title: Pipeline
-      url: https://buildkite.com/elastic/kibana-emergency-release
-spec:
-  type: buildkite-pipeline
-  owner: group:kibana-operations
-  system: buildkite
-  implementation:
-    apiVersion: buildkite.elastic.dev/v1
-    kind: Pipeline
-    metadata:
-      name: kibana / emergency release
-    spec:
-      repository: elastic/kibana
-      provider_settings:
-        trigger_mode: none
-      pipeline_file: ".buildkite/pipelines/emergency_release.yml"
-      teams:
-        kibana-operations:
-          access_level: MANAGE_BUILD_AND_READ
-        kibana-release-operators:
-          access_level: BUILD_AND_READ
-        everyone:
-          access_level: READ_ONLY
 
 # Please avoid creating new kibana pipelines in this file to avoid bloating.
 # Instead, create a new file in the pipeline-resource-definitions directory, and wire it in through the locations.yml file.