Make components in packages configurable (#7602)

* Redefine ExpectedBinaries as YAML config

* Move ExpectedBinaries closer to package spec file

* Fix error formatting in downloadDRAArtifacts

* add packageName template to ExpectedPackages

* use a relaxed dependencies version for IAR releases

* Remove FIPS hack introduced in PR #7486

* Allow for a looser match on relaxing dependencies versions

* Add debug logging when packaging with EXTERNAL=true

* move package tests to dedicated package

(cherry picked from commit 894ef93)

# Conflicts:
#	dev-tools/mage/checksums.go
#	dev-tools/mage/manifest/manifest.go

Author: pchila

dev-tools/mage/checksums.go

@@ -15,6 +15,7 @@ import (
 	"github.com/otiai10/copy"
 
 	"github.com/elastic/elastic-agent/dev-tools/mage/manifest"
+	"github.com/elastic/elastic-agent/dev-tools/packaging"
 )
 
 const ComponentSpecFileSuffix = ".spec.yml"
@@ -94,30 +95,16 @@ func ChecksumsWithoutManifest(versionedFlatPath string, versionedDropPath string
 }
 
 // This is a helper function for flattenDependencies that's used when building from a manifest
-func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string {
+func ChecksumsWithManifest(platform, dependenciesVersion string, versionedFlatPath string, versionedDropPath string, manifestResponse *manifest.Build) map[string]string {
 	checksums := make(map[string]string)
 	if manifestResponse == nil {
 		return checksums
 	}
 
-	// Iterate over the component projects in the manifest
-	projects := manifestResponse.Projects
-	for componentName := range projects {
-		// Iterate over the individual package files within each component project
-		for pkgName := range projects[componentName].Packages {
-			// Only care about packages that match the required package constraint (os/arch)
-			if strings.Contains(pkgName, requiredPackage) {
-				// Iterate over the external binaries that we care about for packaging agent
-				for _, spec := range manifest.ExpectedBinaries {
-					// If the individual package doesn't match the expected prefix, then continue
-					// FIXME temporarily skip fips packages until elastic-agent FIPS is in place
-					if !strings.HasPrefix(pkgName, spec.BinaryName) || strings.Contains(pkgName, "-fips-") {
-						if mg.Verbose() {
-							log.Printf(">>>>>>> Package [%s] skipped", pkgName)
-						}
-						continue
-					}
+	// Iterate over the external binaries that we care about for packaging agent
+	for _, spec := range packaging.ExpectedBinaries {
 
+<<<<<<< HEAD
 					if mg.Verbose() {
 						log.Printf(">>>>>>> Package [%s] matches requiredPackage [%s]", pkgName, requiredPackage)
 					}
@@ -202,12 +189,73 @@ func ChecksumsWithManifest(requiredPackage string, versionedFlatPath string, ver
 
 					checksums[specName+ComponentSpecFileSuffix] = checksum
 				}
+=======
+		if spec.PythonWheel {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Component %s/%s is a Python wheel, skipping", spec.ProjectName, spec.BinaryName)
+>>>>>>> 894ef9385 (Make components in packages configurable (#7602))
+			}
+			continue
+		}
+
+		if !spec.SupportsPlatform(platform) {
+			log.Printf(">>>>>>> Component %s/%s does not support platform %s, skipping", spec.ProjectName, spec.BinaryName, platform)
+			continue
+		}
+
+		manifestPackage, err := manifest.ResolveManifestPackage(manifestResponse.Projects[spec.ProjectName], spec, dependenciesVersion, platform)
+		if err != nil {
+			if mg.Verbose() {
+				log.Printf(">>>>>>> Error resolving package for [%s/%s]", spec.BinaryName, platform)
 			}
+			continue
+		}
+
+		// Combine the package name w/ the versioned flat path
+		fullPath := filepath.Join(versionedFlatPath, manifestPackage.Name)
+
+		// Eliminate the file extensions to get the proper directory
+		// name that we need to copy
+		var dirToCopy string
+		if strings.HasSuffix(fullPath, ".tar.gz") {
+			dirToCopy = fullPath[:strings.LastIndex(fullPath, ".tar.gz")]
+		} else if strings.HasSuffix(fullPath, ".zip") {
+			dirToCopy = fullPath[:strings.LastIndex(fullPath, ".zip")]
+		} else {
+			dirToCopy = fullPath
+		}
+		if mg.Verbose() {
+			log.Printf(">>>>>>> Calculated directory to copy: [%s]", dirToCopy)
 		}
+
+		// Set copy options
+		options := copy.Options{
+			OnSymlink: func(_ string) copy.SymlinkAction {
+				return copy.Shallow
+			},
+			Sync: true,
+		}
+		if mg.Verbose() {
+			log.Printf("> prepare to copy %s into %s ", dirToCopy, versionedDropPath)
+		}
+
+		// Do the copy
+		err = copy.Copy(dirToCopy, versionedDropPath, options)
+		if err != nil {
+			panic(err)
+		}
+
+		checksum, err := CopyComponentSpecs(spec.BinaryName, versionedDropPath)
+		if err != nil {
+			panic(err)
+		}
+
+		checksums[spec.BinaryName+ComponentSpecFileSuffix] = checksum
 	}
 
 	return checksums
 }
+<<<<<<< HEAD
 
 // This function is used when building with a Manifest.  In that manifest, it's possible
 // for projects in an Independent Agent Release to have different versions since the opted-in
@@ -283,3 +331,5 @@ func fixCloudDefendDirPath(dirPath string, componentVersion string, expectedArch
 
 	return fixedDirPath
 }
+=======
+>>>>>>> 894ef9385 (Make components in packages configurable (#7602))

dev-tools/mage/manifest/manifest.go

@@ -13,13 +13,14 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"time"
 
 	"github.com/magefile/mage/mg"
 	"golang.org/x/sync/errgroup"
 
-	"github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon"
+	"github.com/elastic/elastic-agent/dev-tools/packaging"
 	"github.com/elastic/elastic-agent/pkg/version"
 )
 
@@ -94,6 +95,7 @@ var PlatformPackages = map[string]string{
 	"windows/amd64": "windows-x86_64.zip",
 }
 
+<<<<<<< HEAD
 // ExpectedBinaries  is a map of binaries agent needs to their project in the unified-release manager.
 // The project names are those used in the "projects" list in the unified release manifest.
 // See the sample manifests in the testdata directory.
@@ -161,6 +163,8 @@ func (p Platform) Platform() string {
 
 var AllPlatforms = []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}, {"darwin", "aarch64"}}
 
+=======
+>>>>>>> 894ef9385 (Make components in packages configurable (#7602))
 // DownloadManifest is going to download the given manifest file and return the ManifestResponse
 func DownloadManifest(ctx context.Context, manifest string) (Build, error) {
 	manifestUrl, urlError := url.Parse(manifest)
@@ -211,7 +215,7 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string
 
 	errGrp, downloadsCtx := errgroup.WithContext(ctx)
 	// for project, pkgs := range expectedProjectPkgs() {
-	for _, spec := range ExpectedBinaries {
+	for _, spec := range packaging.ExpectedBinaries {
 		for _, platform := range platforms {
 			targetPath := filepath.Join(dropPath)
 			err := os.MkdirAll(targetPath, 0755)
@@ -225,12 +229,12 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string
 				continue
 			}
 
-			pkgURL, err := resolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform)
+			resolvedPackage, err := ResolveManifestPackage(projects[spec.ProjectName], spec, majorMinorPatchVersion, platform)
 			if err != nil {
 				return err
 			}
 
-			for _, p := range pkgURL {
+			for _, p := range resolvedPackage.URLs {
 				log.Printf(">>>>>>>>> Downloading [%s] [%s] ", spec.BinaryName, p)
 				pkgFilename := path.Base(p)
 				downloadTarget := filepath.Join(targetPath, pkgFilename)
@@ -252,76 +256,114 @@ func DownloadComponents(ctx context.Context, manifest string, platforms []string
 	return nil
 }
 
-func resolveManifestPackage(project Project, spec BinarySpec, version string, platform string) ([]string, error) {
-	var val Package
-	var ok bool
+type ResolvedPackage struct {
+	Name string
+	URLs []string
+}
+
+func ResolveManifestPackage(project Project, spec packaging.BinarySpec, dependencyVersion string, platform string) (*ResolvedPackage, error) {
 
 	// Try the normal/easy case first
-	packageName := spec.GetPackageName(version, platform)
-	val, ok = project.Packages[packageName]
-	if !ok {
-		// If we didn't find it, it may be an Independent Agent Release, where
-		// the opted-in projects will have a patch version one higher than
-		// the rest of the projects, so we need to seek that out
+	packageName := spec.GetPackageName(dependencyVersion, platform)
+	if mg.Verbose() {
+		log.Printf(">>>>>>>>>>> Got packagename [%s], looking for exact match", packageName)
+	}
+
+	if exactMatch, ok := project.Packages[packageName]; ok {
+		// We found the exact filename we are looking for
 		if mg.Verbose() {
-			log.Printf(">>>>>>>>>>> Looking for package [%s] of type [%s]", spec.BinaryName, PlatformPackages[platform])
+			log.Printf(">>>>>>>>>>> Found exact match packageName for [%s, %s]: %s", project.Branch, project.CommitHash, exactMatch)
 		}
 
-		var foundIt bool
-		for pkgName := range project.Packages {
-			if strings.HasPrefix(pkgName, spec.BinaryName) {
-				firstSplit := strings.Split(pkgName, spec.BinaryName+"-")
-				if len(firstSplit) < 2 {
-					continue
-				}
+		return &ResolvedPackage{
+			Name: packageName,
+			URLs: []string{exactMatch.URL, exactMatch.ShaURL, exactMatch.AscURL},
+		}, nil
+	}
 
-				secondHalf := firstSplit[1]
-				// Make sure we're finding one w/ the same required package type
-				if strings.Contains(secondHalf, PlatformPackages[platform]) {
-
-					// Split again after the version with the required package string
-					secondSplit := strings.Split(secondHalf, "-"+PlatformPackages[platform])
-					if len(secondSplit) < 2 {
-						continue
-					}
-
-					// The first element after the split should normally be the version
-					pkgVersion := secondSplit[0]
-					if mg.Verbose() {
-						log.Printf(">>>>>>>>>>> Using derived version for package [%s]: %s ", pkgName, pkgVersion)
-					}
-
-					// Create a project/package key with the package, derived version, and required package
-					foundPkgKey := fmt.Sprintf("%s-%s-%s", spec.BinaryName, pkgVersion, PlatformPackages[platform])
-					if mg.Verbose() {
-						log.Printf(">>>>>>>>>>> Looking for project package key: [%s]", foundPkgKey)
-					}
-
-					// Get the package value, if it exists
-					val, ok = project.Packages[foundPkgKey]
-					if !ok {
-						continue
-					}
-
-					if mg.Verbose() {
-						log.Printf(">>>>>>>>>>> Found package key [%s]", foundPkgKey)
-					}
-
-					foundIt = true
-				}
-			}
-		}
+	// If we didn't find it, it may be an Independent Agent Release, where
+	// the opted-in projects will have a patch version one higher than
+	// the rest of the projects, so we "relax" the version constraint
 
-		if !foundIt {
-			return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL)
-		}
+	// Find the original version in the filename
+	versionIndex := strings.Index(packageName, dependencyVersion)
+	if versionIndex == -1 {
+		return nil, fmt.Errorf("no exact match and filename %q does not seem to contain dependencyVersion %q to try a fallback", packageName, dependencyVersion)
+	}
+
+	// TODO move relaxVersion to the version package so we can rewrite the version like so
+	//parseVersion, _ := version.ParseVersion(dependencyVersion)
+	//parseVersion.GetRelaxedPatchRegexp()
+	relaxedVersion, err := relaxVersion(dependencyVersion)
+	if err != nil {
+		return nil, fmt.Errorf("relaxing dependencyVersion %q: %w", dependencyVersion, err)
+	}
+
+	if mg.Verbose() {
+		log.Printf(">>>>>>>>>>> Couldn't find exact match, relaxing agent dependencyVersion to %s", relaxedVersion)
 	}
 
+	// locate the original version in the filename and substitute the relaxed version regexp, quoting everything around that
+	relaxedPackageName := regexp.QuoteMeta(packageName[:versionIndex])
+	relaxedPackageName += relaxedVersion
+	relaxedPackageName += regexp.QuoteMeta(packageName[versionIndex+len(dependencyVersion):])
+
 	if mg.Verbose() {
-		log.Printf(">>>>>>>>>>> Project branch/commit [%s, %s]", project.Branch, project.CommitHash)
+		log.Printf(">>>>>>>>>>> Attempting to match a filename with %s", relaxedPackageName)
 	}
 
-	return []string{val.URL, val.ShaURL, val.AscURL}, nil
+	relaxedPackageNameRegexp, err := regexp.Compile(relaxedPackageName)
+	if err != nil {
+		return nil, fmt.Errorf("compiling relaxed package name regex %q: %w", relaxedPackageName, err)
+	}
+
+	for pkgName, pkg := range project.Packages {
+		if mg.Verbose() {
+			log.Printf(">>>>>>>>>>> Evaluating filename %s", pkgName)
+		}
+		if relaxedPackageNameRegexp.MatchString(pkgName) {
+			if mg.Verbose() {
+				log.Printf(">>>>>>>>>>> Found matching packageName for [%s, %s]: %s", project.Branch, project.CommitHash, pkgName)
+			}
+			return &ResolvedPackage{
+				Name: pkgName,
+				URLs: []string{pkg.URL, pkg.ShaURL, pkg.AscURL},
+			}, nil
+		}
+	}
+
+	return nil, fmt.Errorf("package [%s] not found in project manifest at %s", packageName, project.ExternalArtifactsManifestURL)
+}
+
+// versionRegexp is taken from https://semver.org/ (see the FAQ section/Is there a suggested regular expression (RegEx) to check a SemVer string?)
+const versionRegexp = `^(?:0|[1-9]\d*)\.(?:0|[1-9]\d*)\.(0|[1-9]\d*)(?:-(?:(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?:[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$`
+const anyPatchVersionRegexp = `(?:0|[1-9]\d*)`
+
+var versionRegExp = regexp.MustCompile(versionRegexp)
+
+func relaxVersion(version string) (string, error) {
+	matchIndices := versionRegExp.FindSubmatchIndex([]byte(version))
+	// Matches index pairs are (0,1) for the whole regexp and (2,3) for the patch group
+	// check that we have matched correctly
+	if len(matchIndices) < 4 {
+		return "", fmt.Errorf("failed to match regexp for version [%s]", version)
+	}
+
+	// take the starting index of the patch version
+	patchStartIndex := matchIndices[2]
+	// copy everything before the patch version escaping the regexp
+	relaxedVersion := regexp.QuoteMeta(version[:patchStartIndex])
+	// add the patch regexp
+	relaxedVersion += anyPatchVersionRegexp
+	// check if there's more characters after the patch version
+	remainderIndex := matchIndices[3]
+	if remainderIndex < len(version) {
+		// This is a looser regexp that allows anything beyond the major version to change (while still enforcing a valid patch version though)
+		// see TestResolveManifestPackage/Independent_Agent_Staging_8.14_apm-server and TestResolveManifestPackage/Independent_Agent_Staging_8.14_endpoint-dev
+		// Be more relaxed and allow for any character sequence after this
+		relaxedVersion += `.*`
+	}
+	return relaxedVersion, nil
 }
 
 func DownloadPackage(ctx context.Context, downloadUrl string, target string) error {