[8.15] [BK] Migrate es-forward (+add versions.json dependent triggering) (#184018)

.buildkite/pipeline-utils/utils.test.ts

@@ -0,0 +1,45 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+
+import { expect } from 'chai';
+import { getKibanaDir, getVersionsFile } from './utils';
+import fs from 'fs';
+
+// TODO: replace mocha with jest, and write tests that mock FS
+
+describe('getKibanaDir', () => {
+  it('should return the kibana directory', () => {
+    const kibanaDir = getKibanaDir();
+
+    expect(kibanaDir).to.be.ok;
+    expect(fs.existsSync(kibanaDir)).to.be.true;
+  });
+});
+
+describe('getVersionsFile', () => {
+  it('should return the versions file', () => {
+    const versionsFile = getVersionsFile();
+
+    expect(versionsFile).to.be.ok;
+    expect(versionsFile.versions).to.be.an('array');
+  });
+
+  it('should correctly find prevMajor and prevMinor versions', () => {
+    const versionsFile = getVersionsFile();
+
+    expect(versionsFile.prevMajors).to.be.an('array');
+    expect(versionsFile.prevMajors.length).to.eql(1);
+    expect(versionsFile.prevMajors[0].branch).to.eql('7.17');
+
+    expect(versionsFile.prevMinors).to.be.an('array');
+  });
+
+  // TODO: write more tests with mocking...
+});

.buildkite/pipeline-utils/utils.ts

@@ -7,6 +7,8 @@
  */
 
 import { execSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
 
 const getKibanaDir = (() => {
   let kibanaDir: string | undefined;
@@ -21,4 +23,42 @@ const getKibanaDir = (() => {
   };
 })();
 
-export { getKibanaDir };
+export interface Version {
+  branch: string;
+  version: string;
+}
+export interface VersionsFile {
+  versions: Array<
+    {
+      previousMajor?: boolean;
+      previousMinor?: boolean;
+      currentMajor?: boolean;
+      currentMinor?: boolean;
+    } & Version
+  >;
+}
+const getVersionsFile = (() => {
+  let versions: VersionsFile & {
+    prevMinors: Version[];
+    prevMajors: Version[];
+    current: Version;
+  };
+  const versionsFileName = 'versions.json';
+  try {
+    const versionsJSON = JSON.parse(
+      fs.readFileSync(path.join(getKibanaDir(), versionsFileName)).toString()
+    );
+    versions = {
+      versions: versionsJSON.versions,
+      prevMinors: versionsJSON.versions.filter((v: any) => v.previousMinor),
+      prevMajors: versionsJSON.versions.filter((v: any) => v.previousMajor),
+      current: versionsJSON.versions.find((v: any) => v.currentMajor && v.currentMinor),
+    };
+  } catch (error) {
+    throw new Error(`Failed to read ${versionsFileName}: ${error}`);
+  }
+
+  return () => versions;
+})();
+
+export { getKibanaDir, getVersionsFile };

.buildkite/scripts/common/util.sh

@@ -172,3 +172,9 @@ npm_install_global() {
 download_artifact() {
   retry 3 1 timeout 3m buildkite-agent artifact download "$@"
 }
+
+print_if_dry_run() {
+  if [[ "${DRY_RUN:-}" =~ ^(1|true)$ ]]; then
+    echo "DRY_RUN is enabled."
+  fi
+}

.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -euo pipefail
+
+ts-node .buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts

.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.test.ts

@@ -0,0 +1,77 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+/* eslint-disable @typescript-eslint/no-unused-expressions */
+
+import { getVersionsFile } from '#pipeline-utils';
+import { expect } from 'chai';
+
+import {
+  getArtifactBuildTriggers,
+  getArtifactSnapshotPipelineTriggers,
+  getESForwardPipelineTriggers,
+  getArtifactStagingPipelineTriggers,
+} from './pipeline';
+
+const versionsFile = getVersionsFile();
+
+describe('pipeline trigger combinations', () => {
+  it('should trigger the correct pipelines for "es-forward"', () => {
+    const esForwardTriggers = getESForwardPipelineTriggers();
+    // tests 7.17 against 8.x versions
+    const targets = versionsFile.versions.filter((v) => v.currentMajor === true);
+
+    expect(esForwardTriggers.length).to.eql(targets.length);
+
+    expect(esForwardTriggers.every((trigger) => trigger.build?.branch === '7.17')).to.be.true;
+
+    const targetedManifests = esForwardTriggers.map((t) => t.build?.env?.ES_SNAPSHOT_MANIFEST);
+    targets.forEach((t) =>
+      expect(targetedManifests).to.include(
+        `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${t.version}/manifest-latest-verified.json`
+      )
+    );
+  });
+
+  it('should trigger the correct pipelines for "artifacts-snapshot"', () => {
+    const snapshotTriggers = getArtifactSnapshotPipelineTriggers();
+    // triggers for all open branches
+    const branches = versionsFile.versions.map((v) => v.branch);
+
+    expect(snapshotTriggers.length).to.eql(branches.length);
+
+    branches.forEach((b) => {
+      expect(snapshotTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true;
+    });
+  });
+
+  it('should trigger the correct pipelines for "artifacts-trigger"', () => {
+    const triggerTriggers = getArtifactBuildTriggers();
+    // all currentMajor+prevMinor branches
+    const branches = versionsFile.versions
+      .filter((v) => v.currentMajor === true && v.previousMinor === true)
+      .map((v) => v.branch);
+
+    expect(triggerTriggers.length).to.eql(branches.length);
+    branches.forEach((b) => {
+      expect(triggerTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true;
+    });
+  });
+
+  it('should trigger the correct pipelines for "artifacts-staging"', () => {
+    const stagingTriggers = getArtifactStagingPipelineTriggers();
+    // all branches that are not currentMajor+currentMinor
+    const branches = versionsFile.versions
+      .filter((v) => !v.currentMajor || !v.currentMinor)
+      .map((v) => v.branch);
+
+    expect(stagingTriggers.length).to.eql(branches.length);
+    branches.forEach((b) => {
+      expect(stagingTriggers.some((trigger) => trigger.build?.branch === b)).to.be.true;
+    });
+  });
+});

.buildkite/scripts/pipelines/trigger_version_dependent_jobs/pipeline.ts

@@ -0,0 +1,179 @@
+#!/usr/bin/env ts-node
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { getVersionsFile, BuildkiteTriggerStep } from '#pipeline-utils';
+
+const pipelineSets = {
+  'es-forward': 'kibana-es-forward-compatibility-testing',
+  'artifacts-snapshot': 'kibana-artifacts-snapshot',
+  'artifacts-staging': 'kibana-artifacts-staging',
+  'artifacts-trigger': 'kibana-artifacts-trigger',
+};
+
+/**
+ * This pipeline is used to emit trigger steps onto different pipelines, based on dynamic parameters retrieved from the repository.
+ */
+async function main() {
+  const pipelineSetNames = Object.keys(pipelineSets);
+  const pipelineSetName: string | undefined = process.env.TRIGGER_PIPELINE_SET;
+  const pipelineSteps: BuildkiteTriggerStep[] = [];
+
+  if (!pipelineSetName) {
+    throw new Error(
+      `Env var TRIGGER_PIPELINE_SET is required, and can be one of: ${pipelineSetNames}`
+    );
+  } else if (!pipelineSetNames.includes(pipelineSetName)) {
+    throw new Error(
+      `Invalid value for TRIGGER_PIPELINE_SET (${pipelineSetName}), can be one of: ${pipelineSetNames}`
+    );
+  }
+
+  switch (pipelineSetName) {
+    case 'es-forward': {
+      pipelineSteps.push(...getESForwardPipelineTriggers());
+      break;
+    }
+    case 'artifacts-snapshot': {
+      pipelineSteps.push(...getArtifactSnapshotPipelineTriggers());
+      break;
+    }
+    case 'artifacts-staging': {
+      pipelineSteps.push(...getArtifactStagingPipelineTriggers());
+      break;
+    }
+    case 'artifacts-trigger': {
+      pipelineSteps.push(...getArtifactBuildTriggers());
+      break;
+    }
+    default: {
+      throw new Error(`Unknown pipeline set: ${pipelineSetName}`);
+    }
+  }
+
+  emitPipeline(pipelineSteps);
+}
+
+/**
+ * This pipeline is testing the forward compatibility of Kibana with different versions of Elasticsearch.
+ * Should be triggered for combinations of (Kibana@7.17 + ES@8.x {current open branches on the same major})
+ */
+export function getESForwardPipelineTriggers(): BuildkiteTriggerStep[] {
+  const versions = getVersionsFile();
+  const kibanaPrevMajor = versions.prevMajors[0];
+  const targetESVersions = [versions.prevMinors, versions.current].flat();
+
+  return targetESVersions.map(({ version }) => {
+    return {
+      trigger: pipelineSets['es-forward'],
+      async: true,
+      label: `Triggering Kibana ${kibanaPrevMajor.version} + ES ${version} forward compatibility`,
+      build: {
+        message: process.env.MESSAGE || `ES forward-compatibility test for ES ${version}`,
+        branch: kibanaPrevMajor.branch,
+        commit: 'HEAD',
+        env: {
+          ES_SNAPSHOT_MANIFEST: `https://storage.googleapis.com/kibana-ci-es-snapshots-daily/${version}/manifest-latest-verified.json`,
+          DRY_RUN: process.env.DRY_RUN,
+        },
+      },
+    } as BuildkiteTriggerStep;
+  });
+}
+
+/**
+ * This pipeline creates Kibana artifact snapshots for all open branches.
+ * Should be triggered for all open branches in the versions.json: 7.x, 8.x
+ */
+export function getArtifactSnapshotPipelineTriggers() {
+  // Trigger for all named branches
+  const versions = getVersionsFile();
+  const targetVersions = [versions.prevMajors, versions.prevMinors, versions.current].flat();
+
+  return targetVersions.map(({ branch }) => {
+    return {
+      trigger: pipelineSets['artifacts-snapshot'],
+      async: true,
+      label: `Triggering snapshot artifact builds for ${branch}`,
+      build: {
+        message: process.env.MESSAGE || `Snapshot artifact build for ${branch}`,
+        branch,
+        commit: 'HEAD',
+        env: {
+          DRY_RUN: process.env.DRY_RUN,
+        },
+      },
+    } as BuildkiteTriggerStep;
+  });
+}
+
+/**
+ * This pipeline creates Kibana artifacts for branches that are not the current main.
+ * Should be triggered for all open branches in the versions.json: 7.x, 8.x, but not main.
+ */
+export function getArtifactStagingPipelineTriggers() {
+  // Trigger for all branches, that are not current minor+major
+  const versions = getVersionsFile();
+  const targetVersions = [versions.prevMajors, versions.prevMinors].flat();
+
+  return targetVersions.map(({ branch }) => {
+    return {
+      trigger: pipelineSets['artifacts-staging'],
+      async: true,
+      label: `Triggering staging artifact builds for ${branch}`,
+      build: {
+        message: process.env.MESSAGE || `Staging artifact build for ${branch}`,
+        branch,
+        commit: 'HEAD',
+        env: {
+          DRY_RUN: process.env.DRY_RUN,
+        },
+      },
+    } as BuildkiteTriggerStep;
+  });
+}
+
+/**
+ * This pipeline checks if there are any changes in the incorporated $BEATS_MANIFEST_LATEST_URL (beats version)
+ * and triggers a staging artifact build.
+ * Should be triggered only for the active minor versions that are not `main` and not `7.17`.
+ *
+ * TODO: we could basically do the check logic of .buildkite/scripts/steps/artifacts/trigger.sh in here, and remove kibana-artifacts-trigger
+ */
+export function getArtifactBuildTriggers() {
+  const versions = getVersionsFile();
+  const targetVersions = versions.prevMinors;
+
+  return targetVersions.map(
+    ({ branch }) =>
+      ({
+        trigger: pipelineSets['artifacts-trigger'],
+        async: true,
+        label: `Triggering artifact build for ${branch}`,
+        build: {
+          message: process.env.MESSAGE || `Artifact build for ${branch}`,
+          branch,
+          commit: 'HEAD',
+          env: {
+            DRY_RUN: process.env.DRY_RUN,
+          },
+        },
+      } as BuildkiteTriggerStep)
+  );
+}
+
+function emitPipeline(pipelineSteps: BuildkiteTriggerStep[]) {
+  console.log(JSON.stringify(pipelineSteps, null, 2));
+}
+
+if (require.main === module) {
+  main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+  });
+}

.buildkite/scripts/steps/artifacts/cloud.sh

@@ -7,6 +7,11 @@ set -euo pipefail
 source "$(dirname "$0")/../../common/util.sh"
 source .buildkite/scripts/steps/artifacts/env.sh
 
+if [[ "${DRY_RUN:-}" =~ ^(true|1)$ ]]; then
+  echo "--- Nothing to do in DRY_RUN mode"
+  exit 0
+fi
+
 echo "--- Push docker image"
 mkdir -p target