Refactor and fix all upgrade integration tests

.buildkite/hooks/pre-exit

@@ -2,6 +2,17 @@
 
 set -eo pipefail
 
+if [[ "$BUILDKITE_PIPELINE_SLUG" == "elastic-agent" && "$BUILDKITE_STEP_KEY" == "integration-tests" ]]; then
+  if [[ -z "${WORKSPACE-""}" ]]; then
+      WORKSPACE=$(git rev-parse --show-toplevel)
+  fi
+  source "${WORKSPACE}/.buildkite/scripts/common.sh"
+
+  # Perform cleanup of integration tests resources
+  echo "--- Cleaning up integration test resources"
+  TEST_INTEG_AUTH_ESS_REGION=azure-eastus2 SNAPSHOT=true mage integration:clean
+fi
+
 if [ -n "$GOOGLE_APPLICATION_CREDENTIALS" ]; then
   if test -f "$GOOGLE_APPLICATION_CREDENTIALS"; then
     rm $GOOGLE_APPLICATION_CREDENTIALS

.buildkite/scripts/common.sh

@@ -92,7 +92,7 @@ google_cloud_auth() {
 
     gcloud auth activate-service-account --key-file ${keyFile} 2> /dev/null
 
-    export GOOGLE_APPLICATIONS_CREDENTIALS=${secretFileLocation}
+    export GOOGLE_APPLICATION_CREDENTIALS=${secretFileLocation}
 }
 
 retry() {

.buildkite/scripts/steps/integration_tests.sh

@@ -3,12 +3,22 @@ set -euo pipefail
 
 source .buildkite/scripts/common.sh
 
+# Override the agent package version using a string with format <major>.<minor>.<patch>
+# NOTE: use only after version bump when the new version is not yet available, for example:
+# OVERRIDE_AGENT_PACKAGE_VERSION="8.10.3"
+OVERRIDE_AGENT_PACKAGE_VERSION=""
+
+if [[ -n "$OVERRIDE_AGENT_PACKAGE_VERSION" ]]; then
+  OVERRIDE_TEST_AGENT_VERSION=${OVERRIDE_AGENT_PACKAGE_VERSION}"-SNAPSHOT"
+else
+ OVERRIDE_TEST_AGENT_VERSION=""
+fi
 # PACKAGE
-DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package
+AGENT_PACKAGE_VERSION="${OVERRIDE_AGENT_PACKAGE_VERSION}" DEV=true EXTERNAL=true SNAPSHOT=true PLATFORMS=linux/amd64,linux/arm64 PACKAGES=tar.gz mage package
 
 # Run integration tests
 set +e
-TEST_INTEG_AUTH_ESS_REGION=azure-eastus2 TEST_INTEG_CLEAN_ON_EXIT=true SNAPSHOT=true mage integration:test
+AGENT_VERSION="${OVERRIDE_TEST_AGENT_VERSION}" TEST_INTEG_CLEAN_ON_EXIT=true SNAPSHOT=true mage integration:test
 TESTS_EXIT_STATUS=$?
 set -e
 

changelog/fragments/1695780865-Enable-tamper-protection-feature-flag-by-default-for-Agent-8.11.0.yaml

@@ -0,0 +1,32 @@
+# Kind can be one of:
+# - breaking-change: a change to previously-documented behavior
+# - deprecation: functionality that is being removed in a later release
+# - bug-fix: fixes a problem in a previous version
+# - enhancement: extends functionality but does not break or fix existing behavior
+# - feature: new functionality
+# - known-issue: problems that we are aware of in a given version
+# - security: impacts on the security of a product or a user’s deployment.
+# - upgrade: important information for someone upgrading from a prior version
+# - other: does not fit into any of the other categories
+kind: feature
+
+# Change summary; a 80ish characters long description of the change.
+summary: Enable tamper protection feature flag by default for Agent 8.11.0
+
+# Long description; in case the summary is not enough to describe the change
+# this field accommodate a description without length limits.
+# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
+#description:
+
+# Affected component; a word indicating the component this changeset affects.
+component:
+
+# PR URL; optional; the PR number that added the changeset.
+# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
+# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
+# Please provide it if you are adding a fragment for a different PR.
+pr: https://github.com/elastic/elastic-agent/pull/3478
+
+# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
+# If not present is automatically filled by the tooling with the issue linked to the PR number.
+#issue: https://github.com/owner/repo/1234

internal/pkg/agent/cmd/install.go

@@ -40,7 +40,7 @@ would like the Agent to operate.
 		},
 	}
 
-	cmd.Flags().BoolP("force", "f", false, "Force overwrite the current and do not prompt for confirmation")
+	cmd.Flags().BoolP("force", "f", false, "Force overwrite the current installation and do not prompt for confirmation")
 	cmd.Flags().BoolP("non-interactive", "n", false, "Install Elastic Agent in non-interactive mode which will not prompt on missing parameters but fails instead.")
 	cmd.Flags().String(flagInstallBasePath, paths.DefaultBasePath, "The path where the Elastic Agent will be installed. It must be an absolute path.")
 	addEnrollFlags(cmd)
@@ -83,7 +83,7 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command) error {
 
 	nonInteractive, _ := cmd.Flags().GetBool("non-interactive")
 	if nonInteractive {
-		fmt.Fprintf(streams.Out, "Installing in non-interactive mode.")
+		fmt.Fprintln(streams.Out, "Installing in non-interactive mode.")
 	}
 
 	if status == install.PackageInstall {

internal/pkg/agent/install/install.go

@@ -28,21 +28,25 @@ func Install(cfgFile, topPath string, pt ProgressTrackerStep) error {
 		return errors.New(err, "failed to discover the source directory for installation", errors.TypeFilesystem)
 	}
 
-	// Uninstall current installation
-	//
-	// There is no uninstall token for "install" command.
-	// Uninstall will fail on protected agent.
-	// The protected Agent will need to be uninstalled first before it can be installed.
-	s := pt.StepStart("Uninstalling current Elastic Agent")
-	err = Uninstall(cfgFile, topPath, "", s)
-	if err != nil {
-		s.Failed()
-		return errors.New(
-			err,
-			fmt.Sprintf("failed to uninstall Agent at (%s)", filepath.Dir(topPath)),
-			errors.M("directory", filepath.Dir(topPath)))
+	// We only uninstall Agent if it is currently installed.
+	status, _ := Status(topPath)
+	if status == Installed {
+		// Uninstall current installation
+		//
+		// There is no uninstall token for "install" command.
+		// Uninstall will fail on protected agent.
+		// The protected Agent will need to be uninstalled first before it can be installed.
+		s := pt.StepStart("Uninstalling current Elastic Agent")
+		err = Uninstall(cfgFile, topPath, "", s)
+		if err != nil {
+			s.Failed()
+			return errors.New(
+				err,
+				fmt.Sprintf("failed to uninstall Agent at (%s)", filepath.Dir(topPath)),
+				errors.M("directory", filepath.Dir(topPath)))
+		}
+		s.Succeeded()
 	}
-	s.Succeeded()
 
 	// ensure parent directory exists
 	err = os.MkdirAll(filepath.Dir(topPath), 0755)
@@ -54,7 +58,7 @@ func Install(cfgFile, topPath string, pt ProgressTrackerStep) error {
 	}
 
 	// copy source into install path
-	s = pt.StepStart("Copying files")
+	s := pt.StepStart("Copying files")
 	err = copy.Copy(dir, topPath, copy.Options{
 		OnSymlink: func(_ string) copy.SymlinkAction {
 			return copy.Shallow

magefile.go

@@ -1495,7 +1495,11 @@ func (Integration) Local(ctx context.Context, testName string) error {
 	params.Tags = append(params.Tags, "local")
 	params.Packages = []string{"github.com/elastic/elastic-agent/testing/integration"}
 
-	goTestFlags := strings.SplitN(os.Getenv("GOTEST_FLAGS"), " ", -1)
+	var goTestFlags []string
+	rawTestFlags := os.Getenv("GOTEST_FLAGS")
+	if rawTestFlags != "" {
+		goTestFlags = strings.Split(rawTestFlags, " ")
+	}
 	params.ExtraFlags = goTestFlags
 
 	if testName == "all" {

pkg/features/features.go

@@ -19,7 +19,7 @@ import (
 // The following was agreed upon for upcoming releases
 // 8.10  - default is disabled
 // 8.11+ - default is enabled
-const defaultTamperProtection = false
+const defaultTamperProtection = true
 
 var (
 	current = Flags{
@@ -182,6 +182,8 @@ func Parse(policy any) (*Flags, error) {
 	// Tamper protection flag is optional, fallback on default value if missing
 	if parsedFlags.Agent.Features.TamperProtection != nil {
 		flags.setTamperProtection(parsedFlags.Agent.Features.TamperProtection.Enabled)
+	} else {
+		flags.setTamperProtection(defaultTamperProtection)
 	}
 
 	if err := flags.setSource(parsedFlags); err != nil {

pkg/testing/fetcher_artifact.go

@@ -78,6 +78,7 @@ func (f *artifactFetcher) Fetch(ctx context.Context, operatingSystem string, arc
 		}
 	}
 
+	version, _ = splitBuildID(version)
 	path := fmt.Sprintf("elastic-agent-%s-%s", version, suffix)
 	downloadSrc := fmt.Sprintf("%s%s", uri, path)
 	return &artifactResult{
@@ -115,6 +116,7 @@ func (r *artifactResult) Fetch(ctx context.Context, l Logger, dir string) error
 }
 
 func findURI(ctx context.Context, doer httpDoer, version string) (string, error) {
+	version, buildID := splitBuildID(version)
 	artifactsURI := fmt.Sprintf("https://artifacts-api.elastic.co/v1/search/%s/elastic-agent", version)
 	req, err := http.NewRequestWithContext(ctx, "GET", artifactsURI, nil)
 	if err != nil {
@@ -168,13 +170,35 @@ func findURI(ctx context.Context, doer httpDoer, version string) (string, error)
 		// https://snapshots.elastic.co/8.7.0-d050210c/downloads/elastic-agent-shipper/elastic-agent-shipper-8.7.0-SNAPSHOT-linux-x86_64.tar.gz
 		index := strings.Index(uri, "/beats/elastic-agent/")
 		if index != -1 {
-			return fmt.Sprintf("%s/beats/elastic-agent/", uri[:index]), nil
+			if buildID == "" {
+				// no build id, first is selected
+				return fmt.Sprintf("%s/beats/elastic-agent/", uri[:index]), nil
+			}
+			if strings.Contains(uri, fmt.Sprintf("%s-%s", stripSnapshot(version), buildID)) {
+				return fmt.Sprintf("%s/beats/elastic-agent/", uri[:index]), nil
+			}
 		}
 	}
 
 	return "", fmt.Errorf("uri not detected")
 }
 
+func splitBuildID(version string) (string, string) {
+	split := strings.SplitN(version, "+", 2)
+	if len(split) == 1 {
+		// no build ID
+		return split[0], ""
+	}
+	return split[0], split[1]
+}
+
+func stripSnapshot(version string) string {
+	if strings.HasSuffix(version, "-SNAPSHOT") {
+		return strings.TrimSuffix(version, "-SNAPSHOT")
+	}
+	return version
+}
+
 func DownloadPackage(ctx context.Context, l Logger, doer httpDoer, downloadPath string, packageFile string) error {
 	l.Logf("Downloading artifact from %s", downloadPath)
 

pkg/testing/fixture_install.go

@@ -5,9 +5,11 @@
 package testing
 
 import (
+	"archive/zip"
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"os"
 	"path/filepath"
@@ -232,12 +234,68 @@ func (f *Fixture) collectDiagnostics() {
 		f.t.Logf("failed to collect diagnostics; failed to create %s: %s", diagPath, err)
 		return
 	}
-	outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", f.t.Name(), time.Now().Format(time.RFC3339)))
+
+	// Sub-test names are separated by "/" characters which are not valid filenames on Linux.
+	sanitizedTestName := strings.ReplaceAll(f.t.Name(), "/", "-")
+	outputPath := filepath.Join(diagPath, fmt.Sprintf("%s-diagnostics-%s.zip", sanitizedTestName, time.Now().Format(time.RFC3339)))
 
 	output, err := f.Exec(ctx, []string{"diagnostics", "-f", outputPath})
 	if err != nil {
 		f.t.Logf("failed to collect diagnostics to %s (%s): %s", outputPath, err, output)
+
+		// If collecting diagnostics fails, zip up the entire installation directory with the hope that it will contain logs.
+		f.t.Logf("creating zip archive of the installation directory: %s", f.workDir)
+		zipPath := filepath.Join(diagPath, fmt.Sprintf("%s-install-directory-%s.zip", sanitizedTestName, time.Now().Format(time.RFC3339)))
+		err = f.archiveInstallDirectory(f.workDir, zipPath)
+		if err != nil {
+			f.t.Logf("failed to zip install directory to %s: %s", zipPath, err)
+		}
+	}
+}
+
+func (f *Fixture) archiveInstallDirectory(installPath string, outputPath string) error {
+	file, err := os.Create(outputPath)
+	if err != nil {
+		return fmt.Errorf("creating zip output file %s: %w", outputPath, err)
+	}
+	defer file.Close()
+
+	w := zip.NewWriter(file)
+	defer w.Close()
+
+	walker := func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		file, err := os.Open(path)
+		if err != nil {
+			f.t.Logf("failed to add %s to zip, continuing: %s", path, err)
+			return nil
+		}
+		defer file.Close()
+
+		f, err := w.Create(path)
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(f, file)
+		if err != nil {
+			return err
+		}
+
+		return nil
 	}
+
+	err = filepath.Walk(f.workDir, walker)
+	if err != nil {
+		return fmt.Errorf("walking %s to create zip: %w", f.workDir, err)
+	}
+
+	return nil
 }
 
 func collectDiagFlag() bool {

sonar-project.properties

@@ -2,7 +2,7 @@ sonar.projectKey=elastic_elastic-agent_AYluowg0xMq8P7b4moiZ
 sonar.host.url=https://sonar.elastic.dev
 
 sonar.sources=.
-sonar.exclusions=**/*_test.go, .git/**, dev-tools/**, /magefile.go, changelog/**, _meta/**, deploy/**, docs/**, img/**, specs/**, pkg/testing/**, pkg/component/fake/**, testing/**
+sonar.exclusions=**/*_test.go, .git/**, dev-tools/**, /magefile.go, changelog/**, _meta/**, deploy/**, docs/**, img/**, specs/**, pkg/testing/**, pkg/component/fake/**, testing/**, **/mocks/*.go
 sonar.tests=.
 sonar.test.inclusions=**/*_test.go
 

specs/cloudbeat.spec.yml

@@ -55,6 +55,11 @@ inputs:
     platforms: *platforms
     outputs: *outputs
     command: *command
+  - name: cloudbeat/cis_azure
+    description: "CIS AZURE monitoring"
+    platforms: *platforms
+    outputs: *outputs
+    command: *command
   - name: cloudbeat/vuln_mgmt_aws
     description: "AWS Vulnerabilities management"
     platforms: *platforms