More convenient local integration testing (#31560)

Eliminate the need to run integration tests from inside a Linux 
container for filebeat and libbeat. Noticeably speeds up tests,
especially on non-Linux hosts systems. Go test and pytest can
be run directly from the host system for integration testing.

Add the mage docker namespace to allow manually starting
and stopping containers for integration tests. Replaces the
make start-environment and make stop-environment targets.

Update the developer testing documentation with the new commands.

.gitignore

@@ -18,6 +18,7 @@ x-pack/functionbeat/pkg
 *.dev.yml
 *.generated.yml
 coverage.out
+docker.env
 .python-version
 beat.db
 *.keystore

auditbeat/magefile.go

@@ -30,13 +30,15 @@ import (
 	devtools "github.com/elastic/beats/v7/dev-tools/mage"
 	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 
-	// mage:import
+	//mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/common"
-	// mage:import
+	//mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
-	// mage:import
+	//mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/integtest"
-	// mage:import
+	//mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/docker"
+	//mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/test"
 )
 

auditbeat/scripts/mage/config.go

@@ -48,7 +48,7 @@ func XPackConfigFileParams() devtools.ConfigFileParams {
 }
 
 func configFileParams(dirs ...string) (devtools.ConfigFileParams, error) {
-	var globs []string
+	globs := make([]string, 0, len(dirs))
 	for _, dir := range dirs {
 		globs = append(globs, filepath.Join(dir, configTemplateGlob))
 	}

auditbeat/scripts/mage/docs.go

@@ -46,7 +46,7 @@ func ModuleDocs() error {
 		configFiles = append(configFiles, files...)
 	}
 
-	var configs []string
+	configs := make([]string, 0, len(configFiles))
 	params := map[string]interface{}{
 		"GOOS":      "linux",
 		"GOARCH":    "amd64",
@@ -58,7 +58,7 @@ func ModuleDocs() error {
 		configs = append(configs, dst)
 		devtools.MustExpandFile(src, dst, params)
 	}
-	defer devtools.Clean(configs)
+	defer devtools.Clean(configs) //nolint:errcheck // Errors can safely be ignored here.
 
 	// Remove old.
 	for _, path := range dirsWithModules {
@@ -71,7 +71,7 @@ func ModuleDocs() error {
 	}
 
 	// Run the docs_collector.py script.
-	ve, err := devtools.PythonVirtualenv()
+	ve, err := devtools.PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}

auditbeat/scripts/mage/package.go

@@ -126,7 +126,7 @@ func generateConfig(pkgFlavor PackagingFlavor, ct devtools.ConfigFileType, spec
 	case XPackPackaging:
 		args = XPackConfigFileParams()
 	default:
-		panic(fmt.Errorf("Invalid packaging flavor (either oss or xpack): %v", pkgFlavor))
+		panic(fmt.Errorf("invalid packaging flavor (either oss or xpack): %v", pkgFlavor))
 	}
 
 	// PackageDir isn't exported but we can grab it's value this way.

dev-tools/mage/docs.go

@@ -32,7 +32,6 @@ import (
 	"syscall"
 
 	"github.com/magefile/mage/sh"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -70,7 +69,7 @@ var Docs = docsBuilder{}
 // FieldDocs generates docs/fields.asciidoc from the specified fields.yml file.
 func (docsBuilder) FieldDocs(fieldsYML string) error {
 	// Run the docs_collector.py script.
-	ve, err := PythonVirtualenv()
+	ve, err := PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}
@@ -140,14 +139,18 @@ func (b docsBuilder) AsciidocBook(opts ...DocsOption) error {
 		srv := b.servePreview(htmlDir)
 		url := "http://" + srv.Addr
 		fmt.Println("Serving docs preview at", url)
-		b.openBrowser(url)
+		if err := b.openBrowser(url); err != nil {
+			return err
+		}
 
 		// Wait
 		fmt.Println("Ctrl+C to stop")
 		sigs := make(chan os.Signal, 1)
 		signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
 		<-sigs
-		srv.Shutdown(context.Background())
+		if err := srv.Shutdown(context.Background()); err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -178,7 +181,7 @@ func (docsBuilder) servePreview(dir string) *http.Server {
 
 	go func() {
 		if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
-			panic(errors.Wrap(err, "failed to start docs preview"))
+			panic(fmt.Errorf("failed to start docs preview: %w", err))
 		}
 	}()
 

dev-tools/mage/fmt.go

@@ -25,7 +25,6 @@ import (
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
-	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
@@ -91,7 +90,7 @@ func PythonAutopep8() error {
 	}
 
 	fmt.Println(">> fmt - autopep8: Formatting Python code")
-	ve, err := PythonVirtualenv()
+	ve, err := PythonVirtualenv(false)
 	if err != nil {
 		return err
 	}
@@ -129,7 +128,7 @@ func AddLicenseHeaders() error {
 	case "Elasticv2", "Elastic License 2.0":
 		license = "Elasticv2"
 	default:
-		return errors.Errorf("unknown license type %v", BeatLicense)
+		return fmt.Errorf("unknown license type %v", BeatLicense)
 	}
 
 	licenser := gotool.Licenser

dev-tools/mage/gotest.go

@@ -19,6 +19,7 @@ package mage
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -32,7 +33,6 @@ import (
 
 	"github.com/magefile/mage/mg"
 	"github.com/magefile/mage/sh"
-	"github.com/pkg/errors"
 
 	"github.com/elastic/beats/v7/dev-tools/mage/gotool"
 )
@@ -64,6 +64,7 @@ func makeGoTestArgs(name string) GoTestArgs {
 		TestName:        name,
 		Race:            RaceDetector,
 		Packages:        []string{"./..."},
+		Env:             make(map[string]string),
 		OutputFile:      fileName + ".out",
 		JUnitReportFile: fileName + ".xml",
 		Tags:            testTagsFromEnv(),
@@ -106,6 +107,17 @@ func DefaultGoTestUnitArgs() GoTestArgs { return makeGoTestArgs("Unit") }
 func DefaultGoTestIntegrationArgs() GoTestArgs {
 	args := makeGoTestArgs("Integration")
 	args.Tags = append(args.Tags, "integration")
+	// Use the non-cachable -count=1 flag to disable test caching when running integration tests.
+	// There are reasons to re-run tests even if the code is unchanged (e.g. Dockerfile changes).
+	args.ExtraFlags = append(args.ExtraFlags, "-count=1")
+	return args
+}
+
+// DefaultGoTestIntegrationFromHostArgs returns a default set of arguments for running
+// all integration tests from the host system (outside the docker network).
+func DefaultGoTestIntegrationFromHostArgs() GoTestArgs {
+	args := DefaultGoTestIntegrationArgs()
+	args.Env = WithGoIntegTestHostEnv(args.Env)
 	return args
 }
 
@@ -158,7 +170,7 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 		passThroughEnvs(env, IntegrationTestEnvVars()...)
 		runners, err := NewIntegrationRunners(path.Join("./module", fi.Name()), env)
 		if err != nil {
-			return errors.Wrapf(err, "test setup failed for module %s", fi.Name())
+			return fmt.Errorf("test setup failed for module %s: %w", fi.Name(), err)
 		}
 		err = runners.Test("goIntegTest", func() error {
 			err := GoTest(ctx, GoTestIntegrationArgsForModule(fi.Name()))
@@ -182,8 +194,8 @@ func GoTestIntegrationForModule(ctx context.Context) error {
 }
 
 // InstallGoTestTools installs additional tools that are required to run unit and integration tests.
-func InstallGoTestTools() {
-	gotool.Install(
+func InstallGoTestTools() error {
+	return gotool.Install(
 		gotool.Install.Package("gotest.tools/gotestsum"),
 	)
 }
@@ -259,7 +271,7 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 	if params.OutputFile != "" {
 		fileOutput, err := os.Create(createDir(params.OutputFile))
 		if err != nil {
-			return errors.Wrap(err, "failed to create go test output file")
+			return fmt.Errorf("failed to create go test output file: %w", err)
 		}
 		defer fileOutput.Close()
 		outputs = append(outputs, fileOutput)
@@ -278,9 +290,9 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 	var goTestErr *exec.ExitError
 	if err != nil {
 		// Command ran.
-		exitErr, ok := err.(*exec.ExitError)
-		if !ok {
-			return errors.Wrap(err, "failed to execute go")
+		var exitErr *exec.ExitError
+		if errors.As(err, &exitErr) {
+			return fmt.Errorf("failed to execute go: %w", err)
 		}
 
 		// Command ran but failed. Process the output.
@@ -289,7 +301,7 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 
 	if goTestErr != nil {
 		// No packages were tested. Probably the code didn't compile.
-		return errors.Wrap(goTestErr, "go test returned a non-zero value")
+		return fmt.Errorf("go test returned a non-zero value: %w", goTestErr)
 	}
 
 	// Generate a HTML code coverage report.
@@ -301,14 +313,14 @@ func GoTest(ctx context.Context, params GoTestArgs) error {
 			"-html="+params.CoverageProfileFile,
 			"-o", htmlCoverReport)
 		if err = coverToHTML(); err != nil {
-			return errors.Wrap(err, "failed to write HTML code coverage report")
+			return fmt.Errorf("failed to write HTML code coverage report: %w", err)
 		}
 	}
 
 	// Return an error indicating that testing failed.
 	if goTestErr != nil {
 		fmt.Println(">> go test:", params.TestName, "Test Failed")
-		return errors.Wrap(goTestErr, "go test returned a non-zero value")
+		return fmt.Errorf("go test returned a non-zero value: %w", goTestErr)
 	}
 
 	fmt.Println(">> go test:", params.TestName, "Test Passed")

dev-tools/mage/integtest.go

@@ -18,14 +18,14 @@
 package mage
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
 
 	"github.com/joeshaw/multierror"
 	"github.com/magefile/mage/mg"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -99,7 +99,7 @@ func (steps IntegrationTestSteps) Setup(env map[string]string) error {
 				// errors ignored
 				_ = steps.teardownFrom(prev, env)
 			}
-			return errors.Wrapf(err, "%s setup failed", step.Name())
+			return fmt.Errorf("%s setup failed: %w", step.Name(), err)
 		}
 	}
 	return nil
@@ -121,7 +121,7 @@ func (steps IntegrationTestSteps) teardownFrom(start int, env map[string]string)
 			fmt.Printf("Teardown %s...\n", steps[i].Name())
 		}
 		if err := steps[i].Teardown(env); err != nil {
-			errs = append(errs, errors.Wrapf(err, "%s teardown failed", steps[i].Name()))
+			errs = append(errs, fmt.Errorf("%s teardown failed: %w", steps[i].Name(), err))
 		}
 	}
 	return errs.Err()
@@ -165,32 +165,28 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 
 	// Load the overall steps to use (skipped inside of test environment, as they are never ran on the inside).
 	// These steps are duplicated per scenario.
-	var steps IntegrationTestSteps
 	if !IsInIntegTestEnv() {
 		for _, step := range globalIntegrationTestSetupSteps {
-			use, err := step.Use(dir)
+			_, err := step.Use(dir)
 			if err != nil {
-				return nil, errors.Wrapf(err, "%s step failed on Use", step.Name())
-			}
-			if use {
-				steps = append(steps, step)
+				return nil, fmt.Errorf("%s step failed on Use: %w", step.Name(), err)
 			}
 		}
 	}
 
 	// Create the runners (can only be multiple).
-	var runners IntegrationRunners
+	runners := make(IntegrationRunners, 0, len(globalIntegrationTesters))
 	for _, t := range globalIntegrationTesters {
 		use, err := t.Use(dir)
 		if err != nil {
-			return nil, errors.Wrapf(err, "%s tester failed on Use", t.Name())
+			return nil, fmt.Errorf("%s tester failed on Use: %w", t.Name(), err)
 		}
 		if !use {
 			continue
 		}
 		runner, err := initRunner(t, dir, passInEnv)
 		if err != nil {
-			return nil, errors.Wrapf(err, "initializing %s runner", t.Name())
+			return nil, fmt.Errorf("initializing %s runner: %w", t.Name(), err)
 		}
 		runners = append(runners, runner)
 	}
@@ -202,11 +198,11 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio
 		}
 		tester, ok := globalIntegrationTesters["docker"]
 		if !ok {
-			return nil, fmt.Errorf("docker integration test runner not registered")
+			return nil, errors.New("docker integration test runner not registered")
 		}
 		runner, err := initRunner(tester, dir, passInEnv)
 		if err != nil {
-			return nil, errors.Wrapf(err, "initializing docker runner")
+			return nil, fmt.Errorf("initializing docker runner: %w", err)
 		}
 		runners = append(runners, runner)
 	}
@@ -269,7 +265,7 @@ func (r *IntegrationRunner) Test(mageTarget string, test func() error) (err erro
 		var enabled bool
 		enabled, err = strconv.ParseBool(testEnvVar)
 		if err != nil {
-			err = errors.Wrap(err, "failed to parse TEST_ENVIRONMENT value")
+			err = fmt.Errorf("failed to parse TEST_ENVIRONMENT value: %w", err)
 			return
 		}
 		if !enabled {
@@ -295,7 +291,7 @@ func (r *IntegrationRunner) Test(mageTarget string, test func() error) (err erro
 	inTeardown := false
 	defer func() {
 		if recoverErr := recover(); recoverErr != nil {
-			err = recoverErr.(error)
+			err = recoverErr.(error) //nolint:errcheck // Assignment to named err return value.
 			if !inTeardown {
 				// ignore errors
 				_ = r.steps.Teardown(r.env)
@@ -320,7 +316,7 @@ func (r *IntegrationRunner) Test(mageTarget string, test func() error) (err erro
 			err = teardownErr
 		}
 	}
-	return
+	return err
 }
 
 // Test runs the test on each runner and collects the errors.