Skip to content

Commit

Permalink
Remove endpoint security from linux containers (#6016)
Browse files Browse the repository at this point in the history 
* enhancement(5495): added package types to expected binaries, updated packaging function

* enhancement(5495): ran mage addLicenseHeader

* enhancement(5495): added changelog fragment

* Update changelog/fragments/1731517501-removed-endpoint-security-from-linux-containers.yaml

Co-authored-by: Shaunak Kashyap <ycombinator@gmail.com>

* enhancement(5495): ran mage clean

---------

Co-authored-by: Shaunak Kashyap <ycombinator@gmail.com>
  • Loading branch information
@kaanyalti @ycombinator
kaanyalti and ycombinator authored Nov 15, 2024
1 parent cbb08c9 commit 9c7552f
Show file tree
Hide file tree
Showing 5 changed files with 123 additions and 49 deletions.
30 changes: 30 additions & 0 deletions changelog/fragments/1731517501-removed-endpoint-security-from-linux-containers.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
# Kind can be one of:
# - breaking-change: a change to previously-documented behavior
# - deprecation: functionality that is being removed in a later release
# - bug-fix: fixes a problem in a previous version
# - enhancement: extends functionality but does not break or fix existing behavior
# - feature: new functionality
# - known-issue: problems that we are aware of in a given version
# - security: impacts on the security of a product or a user’s deployment.
# - upgrade: important information for someone upgrading from a prior version
# - other: does not fit into any of the other categories
kind: enhancement

# Change summary; a 80ish characters long description of the change.
summary: removed endpoint security from linux containers

# Long description; in case the summary is not enough to describe the change
# this field accommodate a description without length limits.
# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment.
description: Removed elastic endpoint security from linux containers as it has a dependency on systemd.

# Affected component; usually one of "elastic-agent", "fleet-server", "filebeat", "metricbeat", "auditbeat", "all", etc.
component: "elastic-agent"
# PR URL; optional; the PR number that added the changeset.
# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added.
# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number.
# Please provide it if you are adding a fragment for a different PR.
pr: https://github.com/elastic/elastic-agent/pull/6016
# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of).
# If not present is automatically filled by the tooling with the issue linked to the PR number.
issue: https://github.com/elastic/elastic-agent/issues/5495
47 changes: 30 additions & 17 deletions dev-tools/mage/manifest/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/magefile/mage/mg"
"golang.org/x/sync/errgroup"

"github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon"
"github.com/elastic/elastic-agent/pkg/version"
)

Expand Down Expand Up @@ -78,8 +79,10 @@ var backoffSchedule = []time.Duration{
10 * time.Second,
}

var errorInvalidManifestURL = errors.New("invalid ManifestURL provided")
var errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not allowed URL")
var (
errorInvalidManifestURL = errors.New("invalid ManifestURL provided")
errorNotAllowedManifestURL = errors.New("the provided ManifestURL is not allowed URL")
)

var AllowedManifestHosts = []string{"snapshots.elastic.co", "staging.elastic.co"}

Expand All @@ -95,22 +98,23 @@ var PlatformPackages = map[string]string{
// The project names are those used in the "projects" list in the unified release manifest.
// See the sample manifests in the testdata directory.
var ExpectedBinaries = []BinarySpec{
{BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms},
{BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}},
{BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true},
{BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms},
{BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms},
{BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}},
{BinaryName: "agentbeat", ProjectName: "beats", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "apm-server", ProjectName: "apm-server", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}, {"windows", "x86_64"}, {"darwin", "x86_64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "cloudbeat", ProjectName: "cloudbeat", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "connectors", ProjectName: "connectors", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PythonWheel: true, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "endpoint-security", ProjectName: "endpoint-dev", Platforms: AllPlatforms, PackageTypes: []pkgcommon.PackageType{pkgcommon.RPM, pkgcommon.Deb, pkgcommon.Zip, pkgcommon.TarGz}},
{BinaryName: "fleet-server", ProjectName: "fleet-server", Platforms: AllPlatforms, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "pf-elastic-collector", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "pf-elastic-symbolizer", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
{BinaryName: "pf-host-agent", ProjectName: "prodfiler", Platforms: []Platform{{"linux", "x86_64"}, {"linux", "arm64"}}, PackageTypes: pkgcommon.AllPackageTypes},
}

type BinarySpec struct {
BinaryName string
ProjectName string
Platforms []Platform
PythonWheel bool
BinaryName string
ProjectName string
Platforms []Platform
PythonWheel bool
PackageTypes []pkgcommon.PackageType
}

func (proj BinarySpec) SupportsPlatform(platform string) bool {
Expand All @@ -122,6 +126,15 @@ func (proj BinarySpec) SupportsPlatform(platform string) bool {
return false
}

func (proj BinarySpec) SupportsPackageType(pkgType pkgcommon.PackageType) bool {
for _, p := range proj.PackageTypes {
if p == pkgType {
return true
}
}
return false
}

func (proj BinarySpec) GetPackageName(version string, platform string) string {
if proj.PythonWheel {
return fmt.Sprintf("%s-%s.zip", proj.BinaryName, version)
Expand Down Expand Up @@ -153,7 +166,7 @@ func DownloadManifest(ctx context.Context, manifest string) (Build, error) {
if urlError != nil {
return Build{}, errorInvalidManifestURL
}
var valid = false
valid := false
for _, manifestHost := range AllowedManifestHosts {
if manifestHost == manifestUrl.Host {
valid = true
Expand Down Expand Up @@ -315,7 +328,7 @@ func DownloadPackage(ctx context.Context, downloadUrl string, target string) err
if errorUrl != nil {
return errorInvalidManifestURL
}
var valid = false
valid := false
for _, manifestHost := range AllowedManifestHosts {
if manifestHost == parsedURL.Host {
valid = true
Expand Down
25 changes: 25 additions & 0 deletions dev-tools/mage/pkgcommon/pkgcommon-types.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License 2.0;
// you may not use this file except in compliance with the Elastic License 2.0.

package pkgcommon

// PackageType defines the file format of the package (e.g. zip, rpm, etc).
type PackageType int

// List of possible package types.
const (
RPM PackageType = iota + 1
Deb
Zip
TarGz
Docker
)

var AllPackageTypes = []PackageType{
RPM,
Deb,
Zip,
TarGz,
Docker,
}
19 changes: 11 additions & 8 deletions dev-tools/mage/pkgtypes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import (
"github.com/magefile/mage/mg"
"github.com/magefile/mage/sh"
"gopkg.in/yaml.v3"

"github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon"
)

const (
Expand Down Expand Up @@ -60,16 +62,17 @@ var (
componentConfigFilePattern = regexp.MustCompile(`.*beat\.spec\.yml$|.*beat\.yml$|apm-server\.yml$|apm-server\.spec\.yml$|elastic-agent\.yml$`)
)

// PackageType defines the file format of the package (e.g. zip, rpm, etc).
type PackageType int
// Alias for pkgcommon.PackageType. This type is moved to the pkgcommon to
// resolve circular dependency problems
type PackageType pkgcommon.PackageType

// List of possible package types.
const (
RPM PackageType = iota + 1
Deb
Zip
TarGz
Docker
var (
RPM PackageType = PackageType(pkgcommon.RPM)
Deb = PackageType(pkgcommon.Deb)
Zip = PackageType(pkgcommon.Zip)
TarGz = PackageType(pkgcommon.TarGz)
Docker = PackageType(pkgcommon.Docker)
)

// OSPackageArgs define a set of package types to build for an operating
Expand Down
51 changes: 27 additions & 24 deletions magefile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ import (
devtools "github.com/elastic/elastic-agent/dev-tools/mage"
"github.com/elastic/elastic-agent/dev-tools/mage/downloads"
"github.com/elastic/elastic-agent/dev-tools/mage/manifest"
"github.com/elastic/elastic-agent/dev-tools/mage/pkgcommon"
"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade/artifact/download"
"github.com/elastic/elastic-agent/pkg/testing/buildkite"
tcommon "github.com/elastic/elastic-agent/pkg/testing/common"
Expand Down Expand Up @@ -475,7 +476,8 @@ func AssembleDarwinUniversal() error {
args := []string{
"build/golang-crossbuild/%s-darwin-universal",
"build/golang-crossbuild/%s-darwin-arm64",
"build/golang-crossbuild/%s-darwin-amd64"}
"build/golang-crossbuild/%s-darwin-amd64",
}

for _, arg := range args {
lipoArgs = append(lipoArgs, fmt.Sprintf(arg, devtools.BeatName))
Expand Down Expand Up @@ -514,7 +516,7 @@ func Package(ctx context.Context) error {
dependenciesVersion = beatVersion
}

packageAgent(ctx, platforms, dependenciesVersion, manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(CrossBuild))
packageAgent(ctx, platforms, dependenciesVersion, manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(CrossBuild), devtools.SelectedPackageTypes)
return nil
}

Expand Down Expand Up @@ -855,7 +857,7 @@ func (Devmachine) Create(instanceName string) error {
if instanceName == "" {
return errors.New(
`instanceName is required.
Example:
Example:
mage devmachine:create "pavel-dev-machine" `)
}
return devmachine.Run(instanceName)
Expand Down Expand Up @@ -916,7 +918,7 @@ func runAgent(ctx context.Context, env map[string]string) error {
// produce docker package
packageAgent(ctx, []string{
"linux/amd64",
}, dependenciesVersion, nil, mg.F(devtools.UseElasticAgentDemoPackaging), mg.F(CrossBuild))
}, dependenciesVersion, nil, mg.F(devtools.UseElasticAgentDemoPackaging), mg.F(CrossBuild), devtools.SelectedPackageTypes)

dockerPackagePath := filepath.Join("build", "package", "elastic-agent", "elastic-agent-linux-amd64.docker", "docker-build")
if err := os.Chdir(dockerPackagePath); err != nil {
Expand Down Expand Up @@ -963,7 +965,7 @@ func runAgent(ctx context.Context, env map[string]string) error {
return sh.Run("docker", dockerCmdArgs...)
}

func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *manifest.Build, agentPackaging, agentBinaryTarget mg.Fn) error {
func packageAgent(ctx context.Context, platforms []string, dependenciesVersion string, manifestResponse *manifest.Build, agentPackaging, agentBinaryTarget mg.Fn, packageTypes []mage.PackageType) error {
fmt.Println("--- Package Elastic-Agent")

platformPackageSuffixes := []string{}
Expand All @@ -975,7 +977,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
}

// download/copy all the necessary dependencies for packaging elastic-agent
archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes)
archivePath, dropPath := collectPackageDependencies(platforms, dependenciesVersion, platformPackageSuffixes, packageTypes)

// cleanup after build
defer os.RemoveAll(archivePath)
Expand Down Expand Up @@ -1009,8 +1011,7 @@ func packageAgent(ctx context.Context, platforms []string, dependenciesVersion s
// NOTE: after the build is done the caller must:
// - delete archivePath and dropPath contents
// - unset AGENT_DROP_PATH environment variable
func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string) (archivePath string, dropPath string) {

func collectPackageDependencies(platforms []string, packageVersion string, platformPackageSuffixes []string, packageTypes []mage.PackageType) (archivePath string, dropPath string) {
dropPath, found := os.LookupEnv(agentDropPath)

// try not to shadow too many variables
Expand Down Expand Up @@ -1055,10 +1056,15 @@ func collectPackageDependencies(platforms []string, packageVersion string, platf
fmt.Printf("--- Binary %s does not support %s, download skipped\n", spec.BinaryName, platform)
continue
}
targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform])
os.MkdirAll(targetPath, 0755)
packageName := spec.GetPackageName(packageVersion, platform)
errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads))
for _, pkgType := range packageTypes {
if !spec.SupportsPackageType(pkgcommon.PackageType(pkgType)) {
continue
}
targetPath := filepath.Join(archivePath, manifest.PlatformPackages[platform])
os.MkdirAll(targetPath, 0755)
packageName := spec.GetPackageName(packageVersion, platform)
errGroup.Go(downloadBinary(ctx, spec.ProjectName, packageName, spec.BinaryName, platform, packageVersion, targetPath, completedDownloads))
}
}
}

Expand Down Expand Up @@ -1160,7 +1166,6 @@ func removePythonWheels(matches []string, version string) []string {
// flattenDependencies will extract all the required packages collected in archivePath and dropPath in flatPath and
// regenerate checksums
func flattenDependencies(requiredPackages []string, packageVersion, archivePath, dropPath, flatPath string, manifestResponse *manifest.Build) {

for _, rp := range requiredPackages {
targetPath := filepath.Join(archivePath, rp)
versionedFlatPath := filepath.Join(flatPath, rp)
Expand Down Expand Up @@ -1239,7 +1244,6 @@ type branchInfo struct {
// FetchLatestAgentCoreStagingDRA is a mage target that will retrieve the elastic-agent-core DRA artifacts and
// place them under build/dra/buildID. It accepts one argument that has to be a release branch present in staging DRA
func FetchLatestAgentCoreStagingDRA(ctx context.Context, branch string) error {

branchInfo, err := findLatestBuildForBranch(ctx, baseURLForStagingDRA, branch)

// Create a dir with the buildID at <root>/build/dra/<buildID>
Expand All @@ -1259,15 +1263,14 @@ func FetchLatestAgentCoreStagingDRA(ctx context.Context, branch string) error {
}

fmt.Println("Downloaded agent core DRAs:")
for k, _ := range artifacts {
for k := range artifacts {
fmt.Println(k)
}
return nil
}

// PackageUsingDRA packages elastic-agent for distribution using Daily Released Artifacts specified in manifest.
func PackageUsingDRA(ctx context.Context) error {

start := time.Now()
defer func() { fmt.Println("package ran for", time.Since(start)) }()

Expand Down Expand Up @@ -1295,7 +1298,7 @@ func PackageUsingDRA(ctx context.Context) error {
return fmt.Errorf("setting agent commit hash %q: %w", agentCoreProject.CommitHash, err)
}

return packageAgent(ctx, platforms, parsedVersion.VersionWithPrerelease(), manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(useDRAAgentBinaryForPackage, devtools.ManifestURL))
return packageAgent(ctx, platforms, parsedVersion.VersionWithPrerelease(), manifestResponse, mg.F(devtools.UseElasticAgentPackaging), mg.F(useDRAAgentBinaryForPackage, devtools.ManifestURL), devtools.SelectedPackageTypes)
}

func downloadManifestAndSetVersion(ctx context.Context, url string) (*manifest.Build, *version.ParsedSemVer, error) {
Expand Down Expand Up @@ -1479,7 +1482,6 @@ func downloadDRAArtifacts(ctx context.Context, manifestUrl string, downloadDir s
}

func useDRAAgentBinaryForPackage(ctx context.Context, manifestUrl string) error {

repositoryRoot, err := findRepositoryRoot()
if err != nil {
return fmt.Errorf("looking up for repository root: %w", err)
Expand Down Expand Up @@ -1874,7 +1876,6 @@ func prepareIronbankBuild() error {
}
return nil
})

if err != nil {
return fmt.Errorf("cannot create templates for the IronBank: %+v", err)
}
Expand Down Expand Up @@ -2109,8 +2110,10 @@ func (Integration) UpdatePackageVersion(ctx context.Context) error {
return nil
}

var stateDir = ".integration-cache"
var stateFile = "state.yml"
var (
stateDir = ".integration-cache"
stateFile = "state.yml"
)

// readFrameworkState reads the state file from the integration test framework
func readFrameworkState() (runner.State, error) {
Expand Down Expand Up @@ -2184,8 +2187,8 @@ func listStacks() (string, error) {
for i, stack := range state.Stacks {
t := table.NewWriter()
t.AppendRows([]table.Row{
table.Row{"#", i},
table.Row{"Type", stack.Provisioner},
{"#", i},
{"Type", stack.Provisioner},
})

switch {
Expand Down Expand Up @@ -3044,7 +3047,7 @@ func authGCP(ctx context.Context) error {
if err := json.Unmarshal(output, &svcList); err != nil {
return fmt.Errorf("unable to parse service accounts: %w", err)
}
var found = false
found := false
for _, svc := range svcList {
if svc.Email == iamAcctName {
found = true
Expand Down

0 comments on commit 9c7552f

Please sign in to comment.