Agentbeat tests based on agentbeat.spec.yml

.buildkite/hooks/pre-command

@@ -18,6 +18,13 @@ if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-packetbeat" && "$BUILDKITE_STEP
     export PRIVATE_CI_GCS_CREDENTIALS_SECRET
 fi
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "beats-xpack-agentbeat" && "$BUILDKITE_STEP_KEY" == *"agentbeat-it"* ]]; then
+    out=$(.buildkite/scripts/agentbeat/setup_agentbeat.py)
+    echo "$out"
+    AGENTBEAT_PATH=$(echo "$out" | tail -n 1)
+    export AGENTBEAT_PATH
+fi
+
 if [[ "$BUILDKITE_PIPELINE_SLUG" == "auditbeat" || \
   "$BUILDKITE_PIPELINE_SLUG" == "beats-libbeat" || \
   "$BUILDKITE_PIPELINE_SLUG" == "beats-metricbeat" || \

.buildkite/scripts/agentbeat/setup_agentbeat.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+import platform
+import re
+import subprocess
+import sys
+import tarfile
+
+PATH = 'x-pack/agentbeat/build/distributions'
+PLATFORMS = {
+                'windows': {
+                    'amd64': 'x86_64',
+                },
+                'linux': {
+                    'x86_64': 'x86_64',
+                    'aarch64': 'arm64',
+                },
+                'darwin': {
+                    'x86_64': 'x86_64',
+                    'arm64': 'aarch64',
+                }
+            }
+
+
+class Archive:
+    def __init__(self, os, arch, ext):
+            self.os = os
+            self.arch = arch
+            self.ext = ext
+
+
+def log(msg):
+    sys.stdout.write(f'{msg}\n')
+    sys.stdout.flush()
+
+
+def log_err(msg):
+    sys.stderr.write(f'{msg}\n')
+    sys.stderr.flush()
+
+
+def get_archive_params() -> Archive:
+    system = platform.system().lower()
+    machine = platform.machine().lower()
+    arch = PLATFORMS.get(system, {}).get(machine)
+    ext = get_artifact_extension(system)
+
+    return Archive(system, arch, ext)
+
+
+def get_artifact_extension(system) -> str:
+    if system == 'windows':
+        return 'zip'
+    else:
+        return 'tar.gz'
+
+
+def get_artifact_pattern(archive_obj) -> str:
+    return f'{PATH}/agentbeat-*-{archive_obj.os}-{archive_obj.arch}.{archive_obj.ext}'
+
+
+def download_agentbeat(archive_obj) -> str:
+    pattern = get_artifact_pattern(archive_obj)
+    log('--- Downloading Agentbeat artifact by pattern: ' + pattern)
+    try:
+        subprocess.run(
+            ['buildkite-agent', 'artifact', 'download', pattern, '.',
+             '--step', 'agentbeat-package-linux'],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+
+    except subprocess.CalledProcessError:
+        exit(1)
+
+    return get_full_filename()
+
+
+def get_full_filename() -> str:
+    try:
+        out = subprocess.run(
+            ['ls', '-p', PATH],
+            check=True, capture_output=True, text=True)
+        return out.stdout.strip()
+    except subprocess.CalledProcessError:
+        exit(1)
+
+
+def extract_agentbeat(filename):
+    filepath = PATH + '/' + filename
+    log('Extracting Agentbeat artifact: ' + filepath)
+
+    if filepath.endswith('.zip'):
+        unzip_agentbeat(filepath)
+    else:
+        untar_agentbeat(filepath)
+
+
+def unzip_agentbeat(filepath):
+    try:
+        subprocess.run(
+            ['unzip', '-qq', filepath],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError as e:
+        log_err(e)
+        exit(1)
+
+
+def untar_agentbeat(filepath):
+    try:
+        subprocess.run(
+            ['tar', '-xvf', filepath],
+            check=True, stdout=sys.stdout, stderr=sys.stderr, text=True)
+    except subprocess.CalledProcessError as e:
+        log_err(e)
+        exit(1)
+
+
+def get_path_to_executable(filepath) -> str:
+    pattern = r'(.*)(?=\.zip|.tar\.gz)'
+    match = re.match(pattern, filepath)
+    if match:
+        path = f'../../{match.group(1)}/agentbeat'
+        return path
+    else:
+        log_err('No agentbeat executable found')
+        exit(1)
+
+archive_params = get_archive_params()
+archive = download_agentbeat(archive_params)
+extract_agentbeat(archive)
+log(get_path_to_executable(archive))

.buildkite/x-pack/pipeline.xpack.agentbeat.yml

@@ -1,10 +1,22 @@
 env:
   ASDF_MAGE_VERSION: 1.15.0
+  ASDF_NODEJS_VERSION: 18.17.1
+
   GCP_HI_PERF_MACHINE_TYPE: "c2d-highcpu-16"
   IMAGE_UBUNTU_X86_64: "family/platform-ingest-beats-ubuntu-2204"
 
+  AWS_ARM_INSTANCE_TYPE: "m6g.xlarge"
+  AWS_IMAGE_UBUNTU_ARM_64: "platform-ingest-beats-ubuntu-2204-aarch64"
+
+  IMAGE_MACOS_ARM: "generic-13-ventura-arm"
+  IMAGE_MACOS_X86_64: "generic-13-ventura-x64"
+
+  IMAGE_WIN_2022: "family/platform-ingest-beats-windows-2022"
+
   IMAGE_BEATS_WITH_HOOKS_LATEST: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-beats-ci-with-hooks:latest"
 
+  AGENTBEAT_SPEC: "./agentbeat.spec.yml"
+
 steps:
   - group: "Check/Update"
     key: "x-pack-agentbeat-check-update"
@@ -58,38 +70,15 @@ steps:
           - github_commit_status:
               context: "agentbeat: Packaging"
 
-      - label: ":linux: Agentbeat/Integration tests Linux"
-        key: "agentbeat-it-linux"
+      - label: ":ubuntu: x-pack/agentbeat: Ubuntu x86_64 Spec tests"
+        key: "agentbeat-it-linux-x86-64"
+        env:
+          PLATFORM: "linux/amd64"
         depends_on:
           - agentbeat-package-linux
-        env:
-          ASDF_NODEJS_VERSION: 18.17.1
-          PLATFORMS: "+all linux/amd64 linux/arm64 windows/amd64 darwin/amd64 darwin/arm64"
-          SNAPSHOT: true
         command: |
-          set -euo pipefail
-          echo "~~~ Downloading artifacts"
-          buildkite-agent artifact download x-pack/agentbeat/build/distributions/** . --step 'agentbeat-package-linux'
-          ls -lah x-pack/agentbeat/build/distributions/
-          echo "~~~ Installing @elastic/synthetics with npm"
-          npm install -g @elastic/synthetics
-          echo "~~~ Running tests"
           cd x-pack/agentbeat
-          mage goIntegTest
-        artifact_paths:
-          - x-pack/agentbeat/build/distributions/**/*
-          - "x-pack/agentbeat/build/*.xml"
-          - "x-pack/agentbeat/build/*.json"
-        plugins:
-          - test-collector#v1.10.2:
-              files: "x-pack/agentbeat/build/TEST-*.xml"
-              format: "junit"
-              branches: "main"
-              debug: true
-        retry:
-          automatic:
-            - limit: 1
-        timeout_in_minutes: 60
+          mage -v testWithSpec
         agents:
           provider: "gcp"
           image: "${IMAGE_UBUNTU_X86_64}"
@@ -98,4 +87,76 @@ steps:
           disk_type: "pd-ssd"
         notify:
           - github_commit_status:
-              context: "agentbeat: Integration tests"
+              context: "agentbeat: Ubuntu x86_64 Spec tests"
+
+      - label: ":ubuntu: x-pack/agentbeat: Ubuntu arm64 Spec tests"
+        key: "agentbeat-it-linux-arm64"
+        env:
+          PLATFORM: "linux/arm64"
+        depends_on:
+          - agentbeat-package-linux
+        command: |
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "aws"
+          imagePrefix: "${AWS_IMAGE_UBUNTU_ARM_64}"
+          instanceType: "${AWS_ARM_INSTANCE_TYPE}"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Ubuntu arm64 Spec tests"
+
+      - label: ":windows: x-pack/agentbeat: Windows x86_64 Spec tests"
+        key: "agentbeat-it-windows"
+        env:
+          PLATFORM: "windows/amd64"
+        depends_on:
+          - agentbeat-package-linux
+        command: |
+          Set-Location -Path x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "gcp"
+          image: "${IMAGE_WIN_2022}"
+          machine_type: "${GCP_WIN_MACHINE_TYPE}"
+          disk_size: 200
+          disk_type: "pd-ssd"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: Windows x86_64 Spec tests"
+
+      - label: ":macos: x-pack/agentbeat: macOS x86_64 Spec tests"
+        key: "agentbeat-it-macos-x86-64"
+        depends_on:
+          - agentbeat-package-linux
+        env:
+          PLATFORM: "darwin/amd64"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_X86_64}"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: macOS x86_64 Spec tests"
+
+      - label: ":macos: x-pack/agentbeat: macOS arm64 Spec tests"
+        key: "agentbeat-it-macos-arm64"
+        depends_on:
+          - agentbeat-package-linux
+        env:
+          PLATFORM: "darwin/arm64"
+        command: |
+          set -euo pipefail
+          source .buildkite/scripts/install_macos_tools.sh
+          cd x-pack/agentbeat
+          mage -v testWithSpec
+        agents:
+          provider: "orka"
+          imagePrefix: "${IMAGE_MACOS_ARM}"
+        notify:
+          - github_commit_status:
+              context: "agentbeat: macOS arm64 Spec tests"

dev-tools/mage/spec.go

@@ -0,0 +1,100 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package mage
+
+import (
+	"gopkg.in/yaml.v2"
+
+	"log"
+	"os"
+	"strings"
+)
+
+type spec struct {
+	Inputs []input
+}
+
+type input struct {
+	Name        string
+	Description string
+	Platforms   []string
+	Command     command
+}
+
+func (i *input) GetCommand() string {
+	return strings.Join(i.Command.Args, " ")
+}
+
+type command struct {
+	Name string
+	Args []string
+}
+
+// SpecCommands parses agent.beat.spec.yml and collects commands for tests
+func SpecCommands(specPath string, platform string) []string {
+	spec, _ := parseToObj(specPath)
+
+	filteredInputs := filter(spec.Inputs, func(input input) bool {
+		return contains(input.Platforms, platform)
+	})
+
+	commands := make(map[string]interface{})
+	for _, i := range filteredInputs {
+		commands[i.GetCommand()] = nil
+	}
+	keys := make([]string, 0, len(commands))
+	for k := range commands {
+		keys = append(keys, k)
+	}
+
+	return keys
+}
+
+func parseToObj(path string) (spec, error) {
+	specFile, err := os.ReadFile(path)
+	if err != nil {
+		log.Fatalf("Error opening agentbeat.spec.yml: %v", err)
+		return spec{}, err
+	}
+	var spec spec
+	err = yaml.Unmarshal(specFile, &spec)
+	if err != nil {
+		log.Fatalf("Error parsing agentbeat.spec.yml: %v", err)
+		return spec, err
+	}
+	return spec, nil
+}
+
+func filter[T any](slice []T, condition func(T) bool) []T {
+	var result []T
+	for _, v := range slice {
+		if condition(v) {
+			result = append(result, v)
+		}
+	}
+	return result
+}
+
+func contains(slice []string, item string) bool {
+	for _, v := range slice {
+		if v == item {
+			return true
+		}
+	}
+	return false
+}