From bfdbb0cf7118fe0e5e3fb5c0b08ef7ccba40943d Mon Sep 17 00:00:00 2001 From: Alex Szabo Date: Wed, 10 Jul 2024 12:08:16 +0200 Subject: [PATCH] [BK] Migrate es-forward (+add versions.json dependent triggering) (#184018) ## Goal We'd like to introduce a way to run pipelines that have a dependency on the currently active branch set (managed in [versions.json](./versions.json)). With this, we'd like to migrate over the `es-forward` pipelines (currently: [this](https://buildkite.com/elastic/kibana-7-dot-17-es-8-dot-15-forward-compatibility), and [this](https://buildkite.com/elastic/kibana-7-dot-17-es-8-dot-14-forward-compatibility)) to the new buildkite infra. ## Summary This PR introduces a new pipeline: https://buildkite.com/elastic/kibana-trigger-version-dependent-jobs (through [trigger-version-dependent-jobs.yml](.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml)). The purpose of this new pipeline is to take the name of a "pipelineSet" that refers to a pipeline, and based on the `versions.json` file, work out what are the branches on which the referred pipeline should be triggered. ### Example: `Trigger ES forward compatibility tests` - a scheduled run on [kibana-trigger-version-dependent-jobs](https://buildkite.com/elastic/kibana-trigger-version-dependent-jobs) with the env var `TRIGGER_PIPELINE_SET=es-forward` runs - the pipeline implementation for `kibana-trigger-version-dependent-jobs` works out (looking at `versions.json`), that the `es-forward` set should trigger https://buildkite.com/elastic/kibana-es-forward (doesn't exist prior to the PR) for (7.17+8.14) and (7.17+8.15) - the pipeline implementation uploads two trigger steps, running https://buildkite.com/elastic/kibana-es-forward in two instances with the relevant parameterization. Since the trigger parameters are derived from the `versions.json` file, if we move on and close `8.14`, and open up `8.16`, this will follow, without having to update the pipeline resources or schedules. ## Changes - 2 pipelines created: [trigger-version-dependent-jobs.yml](.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml), [kibana-es-forward.yml](.buildkite/pipeline-resource-definitions/kibana-es-forward.yml) - [x] add kibana-es-forward.yml - implementation for `trigger-version-dependent-jobs` added - branch configuration removed from pipelines (kibana-artifacts-staging, kibana-artifacts-snapshot, kibana-artifacts-trigger) - added a script for checking RREs validity (moved a few files) ## Verification I've used the migration staging pipeline (*) to run this: - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/130 - Env: `TRIGGER_PIPELINE_SET="artifacts-trigger"` - Result: [(success):](https://buildkite.com/elastic/kibana-artifacts-trigger/builds/10806) it triggered for 8.14 only (as expected) - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/131 - Env: `TRIGGER_PIPELINE_SET="es-forward"` - Result: (success): it generated 2 trigger steps, but since the es-forward pipeline doesn't exist, the upload step failed - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/132 - Env: `TRIGGER_PIPELINE_SET="artifacts-snapshot"` - Result: (success): it triggered jobs for all 3 open branches (main/8.14/7.17) - https://buildkite.com/elastic/kibana-migration-pipeline-staging/builds/134 - Env: `TRIGGER_PIPELINE_SET="artifacts-staging"` - Result: (success): it triggered 8.14 / 7.14, but not for main (*note: this migration staging pipeline will come in handy even after the migration, to stage newly created pipelines without creating the resource up-front) (cherry picked from commit ea2509914f2238c646bb63eb5bbbb240365fd04d) # Conflicts: # .buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml # .buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml # .buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml --- .../pipeline-resource-definitions/README.md | 28 +++ .../_new_pipeline.yml} | 0 .../kibana-artifacts-snapshot.yml | 14 -- .../kibana-artifacts-staging.yml | 10 - .../kibana-artifacts-trigger.yml | 6 - .../kibana-es-forward-testing.yml | 41 ++++ .../locations.yml | 2 + .../{ => scripts}/fix-location-collection.ts | 12 +- .../scripts/validate-pipeline-definition.sh | 28 +++ .../trigger-version-dependent-jobs.yml | 71 +++++++ .buildkite/pipeline-utils/utils.test.ts | 45 +++++ .buildkite/pipeline-utils/utils.ts | 42 +++- .buildkite/scripts/common/util.sh | 6 + .../pipeline.sh | 5 + .../pipeline.test.ts | 77 ++++++++ .../pipeline.ts | 179 ++++++++++++++++++ .buildkite/scripts/steps/artifacts/cloud.sh | 5 + .buildkite/scripts/steps/artifacts/publish.sh | 9 + 18 files changed, 543 insertions(+), 37 deletions(-) create mode 100644 .buildkite/pipeline-resource-definitions/README.md rename .buildkite/pipeline-resource-definitions/{_template/template.yml => _templates/_new_pipeline.yml} (100%) create mode 100644 .buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml rename .buildkite/pipeline-resource-definitions/{ => scripts}/fix-location-collection.ts (83%) create mode 100755 .buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh create mode 100644 .buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml create mode 100644 .buildkite/pipeline-utils/utils.test.ts create mode 100755 .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh create mode 100644 .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts create mode 100755 .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts diff --git a/.buildkite/pipeline-resource-definitions/README.md b/.buildkite/pipeline-resource-definitions/README.md new file mode 100644 index 00000000000000..07a3e54fd93bde --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/README.md @@ -0,0 +1,28 @@ +# Buildkite pipeline resource definitions + +## Overview +The pipeline resources are "RRE" (real resource entities) that are used to create/maintain buildkite pipelines. + +The resources described in these files are parsed and loaded to Backstage (https://backstage.elastic.dev). +From there, [Terrazzo](https://buildkite.com/elastic/terrazzo/) is generating and updating the buildkite pipelines. + +These pipelines are referenced indirectly through the root's [`catalog-info.yaml`](../../catalog-info.yaml) file in order to reduce bloat in the main resources file. +There's a location file that collects files defined in this folder ([locations.yml](locations.yml)), this file needs to be updated in order to keep track of local files. + +Available parameters and further help can be found here: https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic + +## Creating a new pipeline resource definition +The easiest way to create a new pipeline is either by copying and editing a similar pipeline, +or by copying a blank template (see [_new_pipeline.yml](_templates/_new_pipeline.yml)) and editing that. + +You can validate your pipeline's structural integrity, and it's conformity to baseline rules by running the following command: +```bash +.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh +``` + +Once you've added the file, you should update the [locations.yml](locations.yml) file to include the new pipeline, or run the following command to update it: +```bash +.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts +``` + +Add your pipeline implementation, commit & push & merge. The pipeline resource will appear in Backstage within minutes, then the pipeline will be added to Buildkite within ~10 minutes. \ No newline at end of file diff --git a/.buildkite/pipeline-resource-definitions/_template/template.yml b/.buildkite/pipeline-resource-definitions/_templates/_new_pipeline.yml similarity index 100% rename from .buildkite/pipeline-resource-definitions/_template/template.yml rename to .buildkite/pipeline-resource-definitions/_templates/_new_pipeline.yml diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml index 3c28f5de81662a..e1c40f690f4ec7 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-snapshot.yml @@ -22,7 +22,6 @@ spec: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' allow_rebuilds: true - branch_configuration: main 8.14 7.17 default_branch: main repository: elastic/kibana pipeline_file: .buildkite/pipelines/artifacts.yml @@ -44,16 +43,3 @@ spec: access_level: MANAGE_BUILD_AND_READ kibana-tech-leads: access_level: MANAGE_BUILD_AND_READ - schedules: - Daily build (main): - cronline: 0 7 * * * America/New_York - message: Daily build - branch: main - Daily build (8.14): - cronline: 0 7 * * * America/New_York - message: Daily build - branch: '8.14' - Daily build (7.17): - cronline: 0 7 * * * America/New_York - message: Daily build - branch: '7.17' diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml index def68475815a71..71bcc4079c50d8 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml @@ -23,7 +23,6 @@ spec: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' allow_rebuilds: true - branch_configuration: 7.17 8.14 repository: elastic/kibana pipeline_file: .buildkite/pipelines/artifacts.yml skip_intermediate_builds: false @@ -44,12 +43,3 @@ spec: access_level: MANAGE_BUILD_AND_READ kibana-tech-leads: access_level: MANAGE_BUILD_AND_READ - schedules: - Daily build (8.14): - cronline: 0 7 * * * America/New_York - message: Daily build - branch: '8.14' - Daily build (7.17): - cronline: 0 7 * * * America/New_York - message: Daily build - branch: '7.17' diff --git a/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml b/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml index c51728226f9b4e..8a9585762de83e 100644 --- a/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml +++ b/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml @@ -23,7 +23,6 @@ spec: ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' allow_rebuilds: true - branch_configuration: '8.14' default_branch: main repository: elastic/kibana pipeline_file: .buildkite/pipelines/artifacts_trigger.yml @@ -45,8 +44,3 @@ spec: access_level: MANAGE_BUILD_AND_READ kibana-tech-leads: access_level: MANAGE_BUILD_AND_READ - schedules: - Daily build (8.14): - cronline: 0 */2 * * * America/New_York - message: Daily build - branch: '8.14' diff --git a/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml b/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml new file mode 100644 index 00000000000000..dea4426e60e1b9 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml @@ -0,0 +1,41 @@ +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-es-forward-compatibility-testing + description: Forward compatibility testing between Kibana 7.17 and ES 8+ + links: + - url: 'https://buildkite.com/elastic/kibana-es-forward-compatibility-testing' + title: Pipeline link +spec: + type: buildkite-pipeline + system: buildkite + owner: 'group:kibana-operations' + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / ES Forward Compatibility Testing + description: Forward compatibility testing between Kibana 7.17 and ES 8+ + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + allow_rebuilds: false + branch_configuration: main + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/pipelines/es_forward.yml # Note: this file exists in 7.17 only + skip_intermediate_builds: false + provider_settings: + prefix_pull_request_fork_branch_names: false + trigger_mode: none + teams: + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ diff --git a/.buildkite/pipeline-resource-definitions/locations.yml b/.buildkite/pipeline-resource-definitions/locations.yml index ccbc41c60ece1b..4967252f2e6075 100644 --- a/.buildkite/pipeline-resource-definitions/locations.yml +++ b/.buildkite/pipeline-resource-definitions/locations.yml @@ -14,6 +14,7 @@ spec: - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-staging.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-artifacts-trigger.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-coverage-daily.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-forward-testing.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-serverless-snapshots.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-es-snapshots.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/kibana-esql-grammar-sync.yml @@ -39,3 +40,4 @@ spec: - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-gen-ai.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-investigations.yml - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/security-solution-quality-gate/kibana-serverless-security-solution-quality-gate-rule-management.yml + - https://github.com/elastic/kibana/blob/main/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml diff --git a/.buildkite/pipeline-resource-definitions/fix-location-collection.ts b/.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts similarity index 83% rename from .buildkite/pipeline-resource-definitions/fix-location-collection.ts rename to .buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts index d4e36f2559a898..1173cddeb15aa2 100755 --- a/.buildkite/pipeline-resource-definitions/fix-location-collection.ts +++ b/.buildkite/pipeline-resource-definitions/scripts/fix-location-collection.ts @@ -11,7 +11,7 @@ import jsYaml from 'js-yaml'; import path from 'path'; import { execSync } from 'child_process'; -const EXCLUDE_LIST = ['locations.yml', '_template/template.yml']; +const EXCLUDE_LIST = ['locations.yml', '_templates']; const REPO_FILES_BASE = 'https://github.com/elastic/kibana/blob/main'; type BackstageLocationResource = object & { @@ -20,19 +20,19 @@ type BackstageLocationResource = object & { async function main() { const repoRoot = execSync('git rev-parse --show-toplevel').toString().trim(); - const resourceDefinitionsFolder = path.resolve( + const resourceDefinitionsRoot = path.resolve( repoRoot, '.buildkite', 'pipeline-resource-definitions' ); const resourceDefinitionsBaseUrl = `${REPO_FILES_BASE}/.buildkite/pipeline-resource-definitions`; - const locationFile = path.resolve(resourceDefinitionsFolder, 'locations.yml'); + const locationFile = path.resolve(resourceDefinitionsRoot, 'locations.yml'); const locationFileLines = fs.readFileSync(locationFile, 'utf8').split('\n'); - const pipelines = readDirRecursively(resourceDefinitionsFolder) + const pipelines = readDirRecursively(resourceDefinitionsRoot) .filter((file) => file.endsWith('.yml')) - .map((file) => file.replace(`${resourceDefinitionsFolder}/`, '')) - .filter((f) => !EXCLUDE_LIST.includes(f)); + .map((file) => file.replace(`${resourceDefinitionsRoot}/`, '')) + .filter((f) => EXCLUDE_LIST.every((excludeExpr) => !f.match(excludeExpr))); const preamble = locationFileLines.slice(0, 1); diff --git a/.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh b/.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh new file mode 100755 index 00000000000000..cd153b23f10ab8 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/scripts/validate-pipeline-definition.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# This script is used to validate a single RRE for a pipeline definition. + +TARGET_FILE=$1 + +if [ -z "$TARGET_FILE" ]; then + echo "Usage: $0 " + exit 1 +fi + +echo "Validating $TARGET_FILE..." +ABSOLUTE_PATH=$(realpath "$TARGET_FILE") +FILE_NAME=$(basename "$ABSOLUTE_PATH") +FOLDER_NAME=$(dirname "$ABSOLUTE_PATH") + +docker run -it \ + --mount type=bind,source="$FOLDER_NAME",target=/home/app/ \ + docker.elastic.co/ci-agent-images/pipelib \ + rre validate --backstage-entity-aware "/home/app/$FILE_NAME" + +if [ $? -ne 0 ]; then + echo "$FILE_NAME invalid ❌" + exit 1 +else + echo "$FILE_NAME valid ✅" + exit 0 +fi diff --git a/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml b/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml new file mode 100644 index 00000000000000..ea474356b137d1 --- /dev/null +++ b/.buildkite/pipeline-resource-definitions/trigger-version-dependent-jobs.yml @@ -0,0 +1,71 @@ +### +# For more information on authoring pipeline definitions, +# follow the guides at https://docs.elastic.dev/ci/getting-started-with-buildkite-at-elastic +### +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: bk-kibana-trigger-version-dependent-jobs + description: 'Trigger version-dependent jobs' + links: + - url: 'https://buildkite.com/elastic/kibana-trigger-version-dependent-jobs' + title: Pipeline link +spec: + type: buildkite-pipeline + system: buildkite + owner: 'group:kibana-operations' + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: kibana / trigger version-dependent jobs + description: 'Trigger version-dependent jobs' + spec: + env: + SLACK_NOTIFICATIONS_CHANNEL: '#kibana-operations-alerts' + ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' + + allow_rebuilds: false + branch_configuration: main + default_branch: main + repository: elastic/kibana + pipeline_file: .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh + skip_intermediate_builds: false + provider_settings: + prefix_pull_request_fork_branch_names: false + skip_pull_request_builds_for_existing_commits: true + trigger_mode: none + teams: + kibana-operations: + access_level: MANAGE_BUILD_AND_READ + appex-qa: + access_level: MANAGE_BUILD_AND_READ + kibana-tech-leads: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: BUILD_AND_READ + schedules: + Trigger ES forward compatibility tests: + cronline: 0 5 * * * + message: Trigger ES forward compatibility tests + env: + TRIGGER_PIPELINE_SET: es-forward + Trigger artifact staging builds: + cronline: 0 7 * * * America/New_York + message: Trigger artifact staging builds + env: + TRIGGER_PIPELINE_SET: artifacts-staging + MESSAGE: Daily staging build + Trigger artifact snapshot builds: + cronline: 0 7 * * * America/New_York + message: Trigger artifact snapshot builds + env: + TRIGGER_PIPELINE_SET: artifacts-snapshot + MESSAGE: Daily snapshot build + Run kibana-artifacts-trigger: + cronline: 0 */2 * * * America/New_York + message: Trigger 'kibana-artifacts-trigger' + env: + TRIGGER_PIPELINE_SET: artifacts-trigger + MESSAGE: Daily build diff --git a/.buildkite/pipeline-utils/utils.test.ts b/.buildkite/pipeline-utils/utils.test.ts new file mode 100644 index 00000000000000..2fece6082bc5c7 --- /dev/null +++ b/.buildkite/pipeline-utils/utils.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable @typescript-eslint/no-unused-expressions */ + +import { expect } from 'chai'; +import { getKibanaDir, getVersionsFile } from './utils'; +import fs from 'fs'; + +// TODO: replace mocha with jest, and write tests that mock FS + +describe('getKibanaDir', () => { + it('should return the kibana directory', () => { + const kibanaDir = getKibanaDir(); + + expect(kibanaDir).to.be.ok; + expect(fs.existsSync(kibanaDir)).to.be.true; + }); +}); + +describe('getVersionsFile', () => { + it('should return the versions file', () => { + const versionsFile = getVersionsFile(); + + expect(versionsFile).to.be.ok; + expect(versionsFile.versions).to.be.an('array'); + }); + + it('should correctly find prevMajor and prevMinor versions', () => { + const versionsFile = getVersionsFile(); + + expect(versionsFile.prevMajors).to.be.an('array'); + expect(versionsFile.prevMajors.length).to.eql(1); + expect(versionsFile.prevMajors[0].branch).to.eql('7.17'); + + expect(versionsFile.prevMinors).to.be.an('array'); + }); + + // TODO: write more tests with mocking... +}); diff --git a/.buildkite/pipeline-utils/utils.ts b/.buildkite/pipeline-utils/utils.ts index e9a5cf91933342..9f7a27d6e1518d 100644 --- a/.buildkite/pipeline-utils/utils.ts +++ b/.buildkite/pipeline-utils/utils.ts @@ -7,6 +7,8 @@ */ import { execSync } from 'child_process'; +import fs from 'fs'; +import path from 'path'; const getKibanaDir = (() => { let kibanaDir: string | undefined; @@ -21,4 +23,42 @@ const getKibanaDir = (() => { }; })(); -export { getKibanaDir }; +export interface Version { + branch: string; + version: string; +} +export interface VersionsFile { + versions: Array< + { + previousMajor?: boolean; + previousMinor?: boolean; + currentMajor?: boolean; + currentMinor?: boolean; + } & Version + >; +} +const getVersionsFile = (() => { + let versions: VersionsFile & { + prevMinors: Version[]; + prevMajors: Version[]; + current: Version; + }; + const versionsFileName = 'versions.json'; + try { + const versionsJSON = JSON.parse( + fs.readFileSync(path.join(getKibanaDir(), versionsFileName)).toString() + ); + versions = { + versions: versionsJSON.versions, + prevMinors: versionsJSON.versions.filter((v: any) => v.previousMinor), + prevMajors: versionsJSON.versions.filter((v: any) => v.previousMajor), + current: versionsJSON.versions.find((v: any) => v.currentMajor && v.currentMinor), + }; + } catch (error) { + throw new Error(`Failed to read ${versionsFileName}: ${error}`); + } + + return () => versions; +})(); + +export { getKibanaDir, getVersionsFile }; diff --git a/.buildkite/scripts/common/util.sh b/.buildkite/scripts/common/util.sh index 5630fed40bf93b..bc5983e2496691 100755 --- a/.buildkite/scripts/common/util.sh +++ b/.buildkite/scripts/common/util.sh @@ -172,3 +172,9 @@ npm_install_global() { download_artifact() { retry 3 1 timeout 3m buildkite-agent artifact download "$@" } + +print_if_dry_run() { + if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then + echo "DRY_RUN is enabled." + fi +} diff --git a/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh new file mode 100755 index 00000000000000..500372eb915960 --- /dev/null +++ b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +ts-node .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts diff --git a/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts new file mode 100644 index 00000000000000..45475301f1c49d --- /dev/null +++ b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +/* eslint-disable @typescript-eslint/no-unused-expressions */ + +import { getVersionsFile } from '#pipeline-utils'; +import { expect } from 'chai'; + +import { + getArtifactBuildTriggers, + getArtifactSnapshotPipelineTriggers, + getESForwardPipelineTriggers, + getArtifactStagingPipelineTriggers, +} from './pipeline'; + +const versionsFile = getVersionsFile(); + +describe('pipeline trigger combinations', () => { + it('should trigger the correct pipelines for "es-forward"', () => { + const esForwardTriggers = getESForwardPipelineTriggers(); + // tests 7.17 against 8.x versions + const targets = versionsFile.versions.filter((v) => v.currentMajor === true); + + expect(esForwardTriggers.length).to.eql(targets.length); + + expect(esForwardTriggers.every((trigger) => trigger.build?.branch === '7.17')).to.be.true; + + const targetedManifests = esForwardTriggers.map((t) => t.build?.env?.ES_SNAPSHOT_MANIFEST); + targets.forEach((t) => + expect(targetedManifests).to.include( + `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${t.version}/manifest-latest-verified.json` + ) + ); + }); + + it('should trigger the correct pipelines for "artifacts-snapshot"', () => { + const snapshotTriggers = getArtifactSnapshotPipelineTriggers(); + // triggers for all open branches + const branches = versionsFile.versions.map((v) => v.branch); + + expect(snapshotTriggers.length).to.eql(branches.length); + + branches.forEach((b) => { + expect(snapshotTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true; + }); + }); + + it('should trigger the correct pipelines for "artifacts-trigger"', () => { + const triggerTriggers = getArtifactBuildTriggers(); + // all currentMajor+prevMinor branches + const branches = versionsFile.versions + .filter((v) => v.currentMajor === true && v.previousMinor === true) + .map((v) => v.branch); + + expect(triggerTriggers.length).to.eql(branches.length); + branches.forEach((b) => { + expect(triggerTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true; + }); + }); + + it('should trigger the correct pipelines for "artifacts-staging"', () => { + const stagingTriggers = getArtifactStagingPipelineTriggers(); + // all branches that are not currentMajor+currentMinor + const branches = versionsFile.versions + .filter((v) => !v.currentMajor || !v.currentMinor) + .map((v) => v.branch); + + expect(stagingTriggers.length).to.eql(branches.length); + branches.forEach((b) => { + expect(stagingTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true; + }); + }); +}); diff --git a/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts new file mode 100755 index 00000000000000..314e3d21646882 --- /dev/null +++ b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts @@ -0,0 +1,179 @@ +#!/usr/bin/env ts-node +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getVersionsFile, BuildkiteTriggerStep } from '#pipeline-utils'; + +const pipelineSets = { + 'es-forward': 'kibana-es-forward-compatibility-testing', + 'artifacts-snapshot': 'kibana-artifacts-snapshot', + 'artifacts-staging': 'kibana-artifacts-staging', + 'artifacts-trigger': 'kibana-artifacts-trigger', +}; + +/** + * This pipeline is used to emit trigger steps onto different pipelines, based on dynamic parameters retrieved from the repository. + */ +async function main() { + const pipelineSetNames = Object.keys(pipelineSets); + const pipelineSetName: string | undefined = process.env.TRIGGER_PIPELINE_SET; + const pipelineSteps: BuildkiteTriggerStep[] = []; + + if (!pipelineSetName) { + throw new Error( + `Env var TRIGGER_PIPELINE_SET is required, and can be one of: ${pipelineSetNames}` + ); + } else if (!pipelineSetNames.includes(pipelineSetName)) { + throw new Error( + `Invalid value for TRIGGER_PIPELINE_SET (${pipelineSetName}), can be one of: ${pipelineSetNames}` + ); + } + + switch (pipelineSetName) { + case 'es-forward': { + pipelineSteps.push(...getESForwardPipelineTriggers()); + break; + } + case 'artifacts-snapshot': { + pipelineSteps.push(...getArtifactSnapshotPipelineTriggers()); + break; + } + case 'artifacts-staging': { + pipelineSteps.push(...getArtifactStagingPipelineTriggers()); + break; + } + case 'artifacts-trigger': { + pipelineSteps.push(...getArtifactBuildTriggers()); + break; + } + default: { + throw new Error(`Unknown pipeline set: ${pipelineSetName}`); + } + } + + emitPipeline(pipelineSteps); +} + +/** + * This pipeline is testing the forward compatibility of Kibana with different versions of Elasticsearch. + * Should be triggered for combinations of (Kibana@7.17 + ES@8.x {current open branches on the same major}) + */ +export function getESForwardPipelineTriggers(): BuildkiteTriggerStep[] { + const versions = getVersionsFile(); + const kibanaPrevMajor = versions.prevMajors[0]; + const targetESVersions = [versions.prevMinors, versions.current].flat(); + + return targetESVersions.map(({ version }) => { + return { + trigger: pipelineSets['es-forward'], + async: true, + label: `Triggering Kibana ${kibanaPrevMajor.version} + ES ${version} forward compatibility`, + build: { + message: process.env.MESSAGE || `ES forward-compatibility test for ES ${version}`, + branch: kibanaPrevMajor.branch, + commit: 'HEAD', + env: { + ES_SNAPSHOT_MANIFEST: `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${version}/manifest-latest-verified.json`, + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep; + }); +} + +/** + * This pipeline creates Kibana artifact snapshots for all open branches. + * Should be triggered for all open branches in the versions.json: 7.x, 8.x + */ +export function getArtifactSnapshotPipelineTriggers() { + // Trigger for all named branches + const versions = getVersionsFile(); + const targetVersions = [versions.prevMajors, versions.prevMinors, versions.current].flat(); + + return targetVersions.map(({ branch }) => { + return { + trigger: pipelineSets['artifacts-snapshot'], + async: true, + label: `Triggering snapshot artifact builds for ${branch}`, + build: { + message: process.env.MESSAGE || `Snapshot artifact build for ${branch}`, + branch, + commit: 'HEAD', + env: { + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep; + }); +} + +/** + * This pipeline creates Kibana artifacts for branches that are not the current main. + * Should be triggered for all open branches in the versions.json: 7.x, 8.x, but not main. + */ +export function getArtifactStagingPipelineTriggers() { + // Trigger for all branches, that are not current minor+major + const versions = getVersionsFile(); + const targetVersions = [versions.prevMajors, versions.prevMinors].flat(); + + return targetVersions.map(({ branch }) => { + return { + trigger: pipelineSets['artifacts-staging'], + async: true, + label: `Triggering staging artifact builds for ${branch}`, + build: { + message: process.env.MESSAGE || `Staging artifact build for ${branch}`, + branch, + commit: 'HEAD', + env: { + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep; + }); +} + +/** + * This pipeline checks if there are any changes in the incorporated $BEATS_MANIFEST_LATEST_URL (beats version) + * and triggers a staging artifact build. + * Should be triggered only for the active minor versions that are not `main` and not `7.17`. + * + * TODO: we could basically do the check logic of .buildkite/scripts/steps/artifacts/trigger.sh in here, and remove kibana-artifacts-trigger + */ +export function getArtifactBuildTriggers() { + const versions = getVersionsFile(); + const targetVersions = versions.prevMinors; + + return targetVersions.map( + ({ branch }) => + ({ + trigger: pipelineSets['artifacts-trigger'], + async: true, + label: `Triggering artifact build for ${branch}`, + build: { + message: process.env.MESSAGE || `Artifact build for ${branch}`, + branch, + commit: 'HEAD', + env: { + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep) + ); +} + +function emitPipeline(pipelineSteps: BuildkiteTriggerStep[]) { + console.log(JSON.stringify(pipelineSteps, null, 2)); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index e12cf7958c86ec..86fa86f37bd3cf 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -7,6 +7,11 @@ set -euo pipefail source "$(dirname "$0")/../../common/util.sh" source .buildkite/scripts/steps/artifacts/env.sh +if [[ "${DRY_RUN:-}" =~ ^(true|1)$ ]]; then + echo "--- Nothing to do in DRY_RUN mode" + exit 0 +fi + echo "--- Push docker image" mkdir -p target diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 40ea04fc33fea5..c272acac22614c 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -5,6 +5,8 @@ set -euo pipefail source .buildkite/scripts/common/util.sh source .buildkite/scripts/steps/artifacts/env.sh +print_if_dry_run + echo "--- Download and verify artifacts" function download { download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" @@ -60,6 +62,7 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json) + PUBLISH_CMD=$(cat < EOF docker run --rm \ --name release-manager \ -e VAULT_ADDR \ @@ -76,6 +79,12 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then --qualifier "$VERSION_QUALIFIER" \ --dependency "beats:$BEATS_MANIFEST_URL" \ --artifact-set main +EOF) + if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then + PUBLISH_CMD+=(" --dry-run") + fi + + "${PUBLISH_CMD[@]}" KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url')