adjusted pipeline

.buildkite/scripts/agentbeat/prepare_env.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import os
+import platform
+import subprocess
+
+
+def get_os() -> str:
+    return platform.system()
+
+
+def get_arch() -> str:
+    return platform.machine()
+
+
+def get_cwd() -> str:
+    return os.getcwd()
+
+
+def download_agentbeat_artifact(os, arch):
+    pattern = "x-pack/agentbeat/build/distributions/agentbeat-9.0.0-SNAPSHOT-linux-x86_64.tar.gz"
+    # pattern = "x-pack/agentbeat/build/distributions/**"
+    # command = f"buildkite-agent artifact download \"{pattern}\" . --step 'agentbeat-package-linux'"
+
+    try:
+        print("--- Downloading agentbeat artifact")
+        result = subprocess.run(
+            ["buildkite-agent", "artifact", "download", pattern, ".",
+             "--build", "01924d2b-b061-45ae-a106-e885584ff26f",
+             "--step", "agentbeat-package-linux"],
+            check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        print(result.stdout)
+    except subprocess.CalledProcessError as e:
+        print("--- Error occurred while downloading agentbeat\n" + e.stderr)
+        exit(1)
+
+
+def install_synthetics():
+    try:
+        print("--- Installing @elastic/synthetics")
+        subprocess.run(
+            ["npm install -g @elastic/synthetics"],
+            check=True
+        )
+    except subprocess.CalledProcessError:
+        print("Failed to install @elastic/synthetics")
+        exit(1)
+
+
+print("--- OS: " + get_os())
+
+print("--- ARCH: " + get_arch())
+
+print("--- CWD: " + get_cwd())
+
+download_agentbeat_artifact(get_os(), get_arch())
+# install_synthetics()

.buildkite/x-pack/pipeline.xpack.agentbeat.yml

@@ -1,95 +1,152 @@
 env:
   ASDF_MAGE_VERSION: 1.15.0
+  ASDF_NODEJS_VERSION: 18.17.1
+
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
 steps:
-  - group: "Check/Update"
-    key: "x-pack-agentbeat-check-update"
-
-    steps:
-      - label: "agentbeat: Run pre-commit"
-        command: "pre-commit run --all-files"
-        agents:
-          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
-          memory: "2Gi"
-          useCustomGlobalHooks: true
-        notify:
-          - github_commit_status:
-              context: "agentbeat: pre-commit"
+#  - group: "Check/Update"
+#    key: "x-pack-agentbeat-check-update"
+#
+#    steps:
+#      - label: "agentbeat: Run pre-commit"
+#        command: "pre-commit run --all-files"
+#        agents:
+#          image: "${IMAGE_BEATS_WITH_HOOKS_LATEST}"
+#          memory: "2Gi"
+#          useCustomGlobalHooks: true
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: pre-commit"
 
-  - wait: ~
-    # with PRs, we want to run mandatory tests only if check/update step succeed
-    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
-    # this allows building DRA artifacts even if there is flakiness in check/update step
-    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
-    depends_on: "x-pack-agentbeat-check-update"
+#  - wait: ~
+#    # with PRs, we want to run mandatory tests only if check/update step succeed
+#    # for other cases, e.g. merge commits, we want to run mundatory test (and publish) independently of other tests
+#    # this allows building DRA artifacts even if there is flakiness in check/update step
+#    if: build.env("BUILDKITE_PULL_REQUEST") != "false"
+#    depends_on: "x-pack-agentbeat-check-update"
 
   - group: "Agentbeat tests"
     key: "agentbeat-mandatory-tests"
 
     steps:
-      - label: ":linux: Agentbeat packaging Linux"
-        key: "agentbeat-package-linux"
-        env:
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
-        command: |
-          set -euo pipefail
-          cd x-pack/agentbeat
-          mage package
-        artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
-        retry:
-          automatic:
-            - limit: 2
-        timeout_in_minutes: 60
-        agents:
-          provider: "gcp"
-          image: "${IMAGE_UBUNTU_X86_64}"
-          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
-          disk_size: 100
-          disk_type: "pd-ssd"
-        notify:
-          - github_commit_status:
-              context: "agentbeat: Packaging"
+#      - label: ":linux: Agentbeat packaging Linux"
+#        key: "agentbeat-package-linux"
+#        env:
+#          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+#          SNAPSHOT: true
+#        command: |
+#          set -euo pipefail
+#          cd x-pack/agentbeat
+#          mage package
+#        artifact_paths:
+#          - x-pack/agentbeat/build/distributions/**/*
+#          - "x-pack/agentbeat/build/*.xml"
+#          - "x-pack/agentbeat/build/*.json"
+#        retry:
+#          automatic:
+#            - limit: 2
+#        timeout_in_minutes: 60
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_UBUNTU_X86_64}"
+#          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+#          disk_size: 100
+#          disk_type: "pd-ssd"
+#        notify:
+#          - github_commit_status:
+#              context: "agentbeat: Packaging"
+
+      #      - label: ":linux: Agentbeat/Integration tests Linux"
+      #        key: "agentbeat-it-linux"
+      #        depends_on:
+      #          - agentbeat-package-linux
+      #        env:
+      #          ASDF_NODEJS_VERSION: 18.17.1
+      #          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
+      #          SNAPSHOT: true
+      #        command: |
+      #          set -euo pipefail
+      #          echo "~~~ Downloading artifacts"
+      #          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
+      #          ls -lah x-pack/agentbeat/build/distributions/
+      #          echo "~~~ Installing @elastic/synthetics with npm"
+      #          npm install -g @elastic/synthetics
+      #          echo "~~~ Running tests"
+      #          cd x-pack/agentbeat
+      #          mage goIntegTest
+      #        artifact_paths:
+      #          - x-pack/agentbeat/build/distributions/**/*
+      #          - "x-pack/agentbeat/build/*.xml"
+      #          - "x-pack/agentbeat/build/*.json"
+      #        retry:
+      #          automatic:
+      #            - limit: 1
+      #        timeout_in_minutes: 60
+      #        agents:
+      #          provider: "gcp"
+      #          image: "${IMAGE_UBUNTU_X86_64}"
+      #          machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
+      #          disk_size: 100
+      #          disk_type: "pd-ssd"
+      #        notify:
+      #          - github_commit_status:
+      #              context: "agentbeat: Integration tests"
 
       - label: ":linux: Agentbeat/Integration tests Linux"
         key: "agentbeat-it-linux"
-        depends_on:
-          - agentbeat-package-linux
-        env:
-          ASDF_NODEJS_VERSION: 18.17.1
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
+#        depends_on:
+#          - agentbeat-package-linux
         command: |
           set -euo pipefail
-          echo "~~~ Downloading artifacts"
-          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
-          ls -lah x-pack/agentbeat/build/distributions/
-          echo "~~~ Installing @elastic/synthetics with npm"
-          npm install -g @elastic/synthetics
-          echo "~~~ Running tests"
-          cd x-pack/agentbeat
-          mage goIntegTest
-        artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
-        retry:
-          automatic:
-            - limit: 1
-        timeout_in_minutes: 60
+#          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --build 01924d2b-b061-45ae-a106-e885584ff26f --step agentbeat-package-linux
+          ./.buildkite/scripts/agentbeat/prepare_env.py
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
           machineType: "${GCP_HI_PERF_MACHINE_TYPE}"
           disk_size: 100
           disk_type: "pd-ssd"
-        notify:
-          - github_commit_status:
-              context: "agentbeat: Integration tests"
+
+#      - label: ":linux: Agentbeat/Integration tests Windows"
+#        key: "agentbeat-it-windows"
+#        depends_on:
+#          - agentbeat-package-linux
+#        commands:
+#          - ./.buildkite/scripts/agentbeat/prepare_env.py
+#        agents:
+#          provider: "gcp"
+#          image: "${IMAGE_WIN_2022}"
+#          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+#          disk_size: 200
+#          disk_type: "pd-ssd"
+#
+#      - label: ":linux: Agentbeat/Integration tests macOS"
+#        key: "agentbeat-it-macos"
+#        depends_on:
+#          - agentbeat-package-linux
+#        command: |
+#          set -euo pipefail
+#          ./.buildkite/scripts/agentbeat/prepare_env.py
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_X86_64}"
+#
+#      - label: ":linux: Agentbeat/Integration tests macOS arm64"
+#        key: "agentbeat-it-macos-arm"
+#        depends_on:
+#          - agentbeat-package-linux
+#        command: |
+#          set -euo pipefail
+#          ./.buildkite/scripts/agentbeat/prepare_env.py
+#        agents:
+#          provider: "orka"
+#          imagePrefix: "${IMAGE_MACOS_ARM}"