diff --git a/.buildkite/pipeline-utils/utils.test.ts b/.buildkite/pipeline-utils/utils.test.ts new file mode 100644 index 00000000000000..2fece6082bc5c7 --- /dev/null +++ b/.buildkite/pipeline-utils/utils.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* eslint-disable @typescript-eslint/no-unused-expressions */ + +import { expect } from 'chai'; +import { getKibanaDir, getVersionsFile } from './utils'; +import fs from 'fs'; + +// TODO: replace mocha with jest, and write tests that mock FS + +describe('getKibanaDir', () => { + it('should return the kibana directory', () => { + const kibanaDir = getKibanaDir(); + + expect(kibanaDir).to.be.ok; + expect(fs.existsSync(kibanaDir)).to.be.true; + }); +}); + +describe('getVersionsFile', () => { + it('should return the versions file', () => { + const versionsFile = getVersionsFile(); + + expect(versionsFile).to.be.ok; + expect(versionsFile.versions).to.be.an('array'); + }); + + it('should correctly find prevMajor and prevMinor versions', () => { + const versionsFile = getVersionsFile(); + + expect(versionsFile.prevMajors).to.be.an('array'); + expect(versionsFile.prevMajors.length).to.eql(1); + expect(versionsFile.prevMajors[0].branch).to.eql('7.17'); + + expect(versionsFile.prevMinors).to.be.an('array'); + }); + + // TODO: write more tests with mocking... +}); diff --git a/.buildkite/pipeline-utils/utils.ts b/.buildkite/pipeline-utils/utils.ts index e9a5cf91933342..9f7a27d6e1518d 100644 --- a/.buildkite/pipeline-utils/utils.ts +++ b/.buildkite/pipeline-utils/utils.ts @@ -7,6 +7,8 @@ */ import { execSync } from 'child_process'; +import fs from 'fs'; +import path from 'path'; const getKibanaDir = (() => { let kibanaDir: string | undefined; @@ -21,4 +23,42 @@ const getKibanaDir = (() => { }; })(); -export { getKibanaDir }; +export interface Version { + branch: string; + version: string; +} +export interface VersionsFile { + versions: Array< + { + previousMajor?: boolean; + previousMinor?: boolean; + currentMajor?: boolean; + currentMinor?: boolean; + } & Version + >; +} +const getVersionsFile = (() => { + let versions: VersionsFile & { + prevMinors: Version[]; + prevMajors: Version[]; + current: Version; + }; + const versionsFileName = 'versions.json'; + try { + const versionsJSON = JSON.parse( + fs.readFileSync(path.join(getKibanaDir(), versionsFileName)).toString() + ); + versions = { + versions: versionsJSON.versions, + prevMinors: versionsJSON.versions.filter((v: any) => v.previousMinor), + prevMajors: versionsJSON.versions.filter((v: any) => v.previousMajor), + current: versionsJSON.versions.find((v: any) => v.currentMajor && v.currentMinor), + }; + } catch (error) { + throw new Error(`Failed to read ${versionsFileName}: ${error}`); + } + + return () => versions; +})(); + +export { getKibanaDir, getVersionsFile }; diff --git a/.buildkite/scripts/common/util.sh b/.buildkite/scripts/common/util.sh index 818d712fd2aa80..6fd4ac08c9204a 100755 --- a/.buildkite/scripts/common/util.sh +++ b/.buildkite/scripts/common/util.sh @@ -172,3 +172,9 @@ npm_install_global() { download_artifact() { retry 3 1 timeout 3m buildkite-agent artifact download "$@" } + +print_if_dry_run() { + if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then + echo "DRY_RUN is enabled." + fi +} diff --git a/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh new file mode 100755 index 00000000000000..500372eb915960 --- /dev/null +++ b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -euo pipefail + +ts-node .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts diff --git a/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts new file mode 100644 index 00000000000000..45475301f1c49d --- /dev/null +++ b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts @@ -0,0 +1,77 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +/* eslint-disable @typescript-eslint/no-unused-expressions */ + +import { getVersionsFile } from '#pipeline-utils'; +import { expect } from 'chai'; + +import { + getArtifactBuildTriggers, + getArtifactSnapshotPipelineTriggers, + getESForwardPipelineTriggers, + getArtifactStagingPipelineTriggers, +} from './pipeline'; + +const versionsFile = getVersionsFile(); + +describe('pipeline trigger combinations', () => { + it('should trigger the correct pipelines for "es-forward"', () => { + const esForwardTriggers = getESForwardPipelineTriggers(); + // tests 7.17 against 8.x versions + const targets = versionsFile.versions.filter((v) => v.currentMajor === true); + + expect(esForwardTriggers.length).to.eql(targets.length); + + expect(esForwardTriggers.every((trigger) => trigger.build?.branch === '7.17')).to.be.true; + + const targetedManifests = esForwardTriggers.map((t) => t.build?.env?.ES_SNAPSHOT_MANIFEST); + targets.forEach((t) => + expect(targetedManifests).to.include( + `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${t.version}/manifest-latest-verified.json` + ) + ); + }); + + it('should trigger the correct pipelines for "artifacts-snapshot"', () => { + const snapshotTriggers = getArtifactSnapshotPipelineTriggers(); + // triggers for all open branches + const branches = versionsFile.versions.map((v) => v.branch); + + expect(snapshotTriggers.length).to.eql(branches.length); + + branches.forEach((b) => { + expect(snapshotTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true; + }); + }); + + it('should trigger the correct pipelines for "artifacts-trigger"', () => { + const triggerTriggers = getArtifactBuildTriggers(); + // all currentMajor+prevMinor branches + const branches = versionsFile.versions + .filter((v) => v.currentMajor === true && v.previousMinor === true) + .map((v) => v.branch); + + expect(triggerTriggers.length).to.eql(branches.length); + branches.forEach((b) => { + expect(triggerTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true; + }); + }); + + it('should trigger the correct pipelines for "artifacts-staging"', () => { + const stagingTriggers = getArtifactStagingPipelineTriggers(); + // all branches that are not currentMajor+currentMinor + const branches = versionsFile.versions + .filter((v) => !v.currentMajor || !v.currentMinor) + .map((v) => v.branch); + + expect(stagingTriggers.length).to.eql(branches.length); + branches.forEach((b) => { + expect(stagingTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true; + }); + }); +}); diff --git a/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts new file mode 100755 index 00000000000000..314e3d21646882 --- /dev/null +++ b/.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts @@ -0,0 +1,179 @@ +#!/usr/bin/env ts-node +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { getVersionsFile, BuildkiteTriggerStep } from '#pipeline-utils'; + +const pipelineSets = { + 'es-forward': 'kibana-es-forward-compatibility-testing', + 'artifacts-snapshot': 'kibana-artifacts-snapshot', + 'artifacts-staging': 'kibana-artifacts-staging', + 'artifacts-trigger': 'kibana-artifacts-trigger', +}; + +/** + * This pipeline is used to emit trigger steps onto different pipelines, based on dynamic parameters retrieved from the repository. + */ +async function main() { + const pipelineSetNames = Object.keys(pipelineSets); + const pipelineSetName: string | undefined = process.env.TRIGGER_PIPELINE_SET; + const pipelineSteps: BuildkiteTriggerStep[] = []; + + if (!pipelineSetName) { + throw new Error( + `Env var TRIGGER_PIPELINE_SET is required, and can be one of: ${pipelineSetNames}` + ); + } else if (!pipelineSetNames.includes(pipelineSetName)) { + throw new Error( + `Invalid value for TRIGGER_PIPELINE_SET (${pipelineSetName}), can be one of: ${pipelineSetNames}` + ); + } + + switch (pipelineSetName) { + case 'es-forward': { + pipelineSteps.push(...getESForwardPipelineTriggers()); + break; + } + case 'artifacts-snapshot': { + pipelineSteps.push(...getArtifactSnapshotPipelineTriggers()); + break; + } + case 'artifacts-staging': { + pipelineSteps.push(...getArtifactStagingPipelineTriggers()); + break; + } + case 'artifacts-trigger': { + pipelineSteps.push(...getArtifactBuildTriggers()); + break; + } + default: { + throw new Error(`Unknown pipeline set: ${pipelineSetName}`); + } + } + + emitPipeline(pipelineSteps); +} + +/** + * This pipeline is testing the forward compatibility of Kibana with different versions of Elasticsearch. + * Should be triggered for combinations of (Kibana@7.17 + ES@8.x {current open branches on the same major}) + */ +export function getESForwardPipelineTriggers(): BuildkiteTriggerStep[] { + const versions = getVersionsFile(); + const kibanaPrevMajor = versions.prevMajors[0]; + const targetESVersions = [versions.prevMinors, versions.current].flat(); + + return targetESVersions.map(({ version }) => { + return { + trigger: pipelineSets['es-forward'], + async: true, + label: `Triggering Kibana ${kibanaPrevMajor.version} + ES ${version} forward compatibility`, + build: { + message: process.env.MESSAGE || `ES forward-compatibility test for ES ${version}`, + branch: kibanaPrevMajor.branch, + commit: 'HEAD', + env: { + ES_SNAPSHOT_MANIFEST: `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${version}/manifest-latest-verified.json`, + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep; + }); +} + +/** + * This pipeline creates Kibana artifact snapshots for all open branches. + * Should be triggered for all open branches in the versions.json: 7.x, 8.x + */ +export function getArtifactSnapshotPipelineTriggers() { + // Trigger for all named branches + const versions = getVersionsFile(); + const targetVersions = [versions.prevMajors, versions.prevMinors, versions.current].flat(); + + return targetVersions.map(({ branch }) => { + return { + trigger: pipelineSets['artifacts-snapshot'], + async: true, + label: `Triggering snapshot artifact builds for ${branch}`, + build: { + message: process.env.MESSAGE || `Snapshot artifact build for ${branch}`, + branch, + commit: 'HEAD', + env: { + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep; + }); +} + +/** + * This pipeline creates Kibana artifacts for branches that are not the current main. + * Should be triggered for all open branches in the versions.json: 7.x, 8.x, but not main. + */ +export function getArtifactStagingPipelineTriggers() { + // Trigger for all branches, that are not current minor+major + const versions = getVersionsFile(); + const targetVersions = [versions.prevMajors, versions.prevMinors].flat(); + + return targetVersions.map(({ branch }) => { + return { + trigger: pipelineSets['artifacts-staging'], + async: true, + label: `Triggering staging artifact builds for ${branch}`, + build: { + message: process.env.MESSAGE || `Staging artifact build for ${branch}`, + branch, + commit: 'HEAD', + env: { + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep; + }); +} + +/** + * This pipeline checks if there are any changes in the incorporated $BEATS_MANIFEST_LATEST_URL (beats version) + * and triggers a staging artifact build. + * Should be triggered only for the active minor versions that are not `main` and not `7.17`. + * + * TODO: we could basically do the check logic of .buildkite/scripts/steps/artifacts/trigger.sh in here, and remove kibana-artifacts-trigger + */ +export function getArtifactBuildTriggers() { + const versions = getVersionsFile(); + const targetVersions = versions.prevMinors; + + return targetVersions.map( + ({ branch }) => + ({ + trigger: pipelineSets['artifacts-trigger'], + async: true, + label: `Triggering artifact build for ${branch}`, + build: { + message: process.env.MESSAGE || `Artifact build for ${branch}`, + branch, + commit: 'HEAD', + env: { + DRY_RUN: process.env.DRY_RUN, + }, + }, + } as BuildkiteTriggerStep) + ); +} + +function emitPipeline(pipelineSteps: BuildkiteTriggerStep[]) { + console.log(JSON.stringify(pipelineSteps, null, 2)); +} + +if (require.main === module) { + main().catch((error) => { + console.error(error); + process.exit(1); + }); +} diff --git a/.buildkite/scripts/steps/artifacts/cloud.sh b/.buildkite/scripts/steps/artifacts/cloud.sh index e12cf7958c86ec..86fa86f37bd3cf 100644 --- a/.buildkite/scripts/steps/artifacts/cloud.sh +++ b/.buildkite/scripts/steps/artifacts/cloud.sh @@ -7,6 +7,11 @@ set -euo pipefail source "$(dirname "$0")/../../common/util.sh" source .buildkite/scripts/steps/artifacts/env.sh +if [[ "${DRY_RUN:-}" =~ ^(true|1)$ ]]; then + echo "--- Nothing to do in DRY_RUN mode" + exit 0 +fi + echo "--- Push docker image" mkdir -p target diff --git a/.buildkite/scripts/steps/artifacts/publish.sh b/.buildkite/scripts/steps/artifacts/publish.sh index 40ea04fc33fea5..c272acac22614c 100644 --- a/.buildkite/scripts/steps/artifacts/publish.sh +++ b/.buildkite/scripts/steps/artifacts/publish.sh @@ -5,6 +5,8 @@ set -euo pipefail source .buildkite/scripts/common/util.sh source .buildkite/scripts/steps/artifacts/env.sh +print_if_dry_run + echo "--- Download and verify artifacts" function download { download_artifact "$1" . --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" @@ -60,6 +62,7 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then download_artifact beats_manifest.json /tmp --build "${KIBANA_BUILD_ID:-$BUILDKITE_BUILD_ID}" export BEATS_MANIFEST_URL=$(jq -r .manifest_url /tmp/beats_manifest.json) + PUBLISH_CMD=$(cat < EOF docker run --rm \ --name release-manager \ -e VAULT_ADDR \ @@ -76,6 +79,12 @@ if [[ "$BUILDKITE_BRANCH" == "$KIBANA_BASE_BRANCH" ]]; then --qualifier "$VERSION_QUALIFIER" \ --dependency "beats:$BEATS_MANIFEST_URL" \ --artifact-set main +EOF) + if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then + PUBLISH_CMD+=(" --dry-run") + fi + + "${PUBLISH_CMD[@]}" KIBANA_SUMMARY=$(curl -s "$KIBANA_MANIFEST_LATEST" | jq -re '.summary_url')