Add spec for shared folder and link files

code/go/internal/validator/semantic/validate_anyof_required_contents.go

@@ -0,0 +1,96 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package semantic
+
+import (
+	"errors"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"github.com/elastic/package-spec/v3/code/go/internal/fspath"
+	"github.com/elastic/package-spec/v3/code/go/pkg/specerrors"
+)
+
+type anyOfCondition struct {
+	path          string
+	anyOfPatterns []string
+}
+
+func (a *anyOfCondition) validate(fsys fspath.FS) specerrors.ValidationErrors {
+	if len(a.anyOfPatterns) == 0 || a.path == "" {
+		return nil
+	}
+
+	var errs specerrors.ValidationErrors
+	if err := a.validatePath(fsys, a.path); err != nil {
+		errs = append(errs, specerrors.NewStructuredErrorf("path %q: %w", a.path, err))
+	}
+
+	dataStreams, err := listDataStreams(fsys)
+	if err != nil {
+		return specerrors.ValidationErrors{specerrors.NewStructuredError(err, specerrors.UnassignedCode)}
+	}
+
+	for _, dataStream := range dataStreams {
+		path := filepath.ToSlash(filepath.Join("data_stream", dataStream, a.path))
+		err := a.validatePath(fsys, path)
+		if err != nil {
+			errs = append(errs, specerrors.NewStructuredErrorf("data stream %q: %w", dataStream, err))
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs
+	}
+	return nil
+}
+
+func (a *anyOfCondition) validatePath(fsys fspath.FS, path string) specerrors.ValidationError {
+	files, err := fs.ReadDir(fsys, path)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			return specerrors.NewStructuredError(err, specerrors.UnassignedCode)
+		}
+		return nil
+	}
+	for _, file := range files {
+		for _, pattern := range a.anyOfPatterns {
+			matched, err := filepath.Match(pattern, file.Name())
+			if err != nil {
+				return specerrors.NewStructuredError(err, specerrors.UnassignedCode)
+			}
+			if matched {
+				return nil
+			}
+		}
+	}
+	return specerrors.NewStructuredErrorf("no file matching any of the patterns %v found in %s", a.anyOfPatterns, path)
+}
+
+// ValidateAnyOfRequiredContents validates that at least one file matching
+// any of the patterns in the given path exists in the package.
+// It checks the following paths:
+// - agent/input
+// - agent/stream
+// - fields
+// - elasticsearch/ingest_pipeline
+func ValidateAnyOfRequiredContents(fsys fspath.FS) specerrors.ValidationErrors {
+	conditions := []anyOfCondition{
+		{path: "agent/input", anyOfPatterns: []string{"*.yml.hbs", "*.yml.hbs.link"}},
+		{path: "agent/stream", anyOfPatterns: []string{"*.yml.hbs", "*.yml.hbs.link"}},
+		{path: "fields", anyOfPatterns: []string{"*.yml", "*.yml.link"}},
+		{path: "elasticsearch/ingest_pipeline", anyOfPatterns: []string{"*.yml", "*.json", "*.yml.link", "*.json.link"}},
+	}
+
+	var errs specerrors.ValidationErrors
+	for _, c := range conditions {
+		cerrs := c.validate(fsys)
+		if cerrs != nil {
+			errs = append(errs, cerrs...)
+		}
+	}
+	return errs
+}

code/go/internal/validator/spec.go

@@ -214,6 +214,7 @@ func (s Spec) rules(pkgType string, rootSpec spectypes.ItemSpec) validationRules
 		{fn: semantic.ValidateDimensionsPresent, types: []string{"integration"}, since: semver.MustParse("3.0.1")},
 		{fn: semantic.ValidateCapabilitiesRequired, since: semver.MustParse("2.10.0")}, // capabilities definition was added in spec version 2.10.0
 		{fn: semantic.ValidateRequiredVarGroups},
+		{fn: semantic.ValidateAnyOfRequiredContents},
 	}
 
 	var validationRules validationRules

code/go/pkg/linkedfiles/fs.go

@@ -0,0 +1,46 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package linkedfiles
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+var _ fs.FS = (*LinksFS)(nil)
+
+// LinksFS is a filesystem that handles linked files.
+// It wraps another filesystem and checks for linked files with the ".link" extension.
+// If a linked file is found, it reads the link file to determine the target file
+// and its checksum. If the target file is up to date, it returns the target file.
+// Otherwise, it returns an error.
+type LinksFS struct {
+	workDir string
+	inner   fs.FS
+}
+
+// NewLinksFS creates a new LinksFS.
+func NewLinksFS(workDir string, inner fs.FS) *LinksFS {
+	return &LinksFS{workDir: workDir, inner: inner}
+}
+
+// Open opens a file in the filesystem.
+func (lfs *LinksFS) Open(name string) (fs.File, error) {
+	if filepath.Ext(name) != LinkExtension {
+		return lfs.inner.Open(name)
+	}
+	pathName := filepath.Join(lfs.workDir, name)
+	l, err := newLinkedFile(pathName)
+	if err != nil {
+		return nil, err
+	}
+	if !l.UpToDate {
+		return nil, fmt.Errorf("linked file %s is not up to date", name)
+	}
+	includedPath := filepath.Join(lfs.workDir, filepath.Dir(name), l.IncludedFilePath)
+	return os.Open(includedPath)
+}

code/go/pkg/linkedfiles/linkedfiles.go

@@ -0,0 +1,100 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package linkedfiles
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// LinkExtension is the file extension for linked files.
+const LinkExtension = ".link"
+
+// A Link represents a linked file.
+// It contains the path to the link file, the checksum of the linked file,
+// the path to the target file, and the checksum of the included file contents.
+// It also contains a boolean indicating whether the link is up to date.
+type Link struct {
+	LinkFilePath string
+	LinkChecksum string
+
+	IncludedFilePath             string
+	IncludedFileContentsChecksum string
+
+	UpToDate bool
+}
+
+func newLinkedFile(linkFilePath string) (Link, error) {
+	var l Link
+	firstLine, err := readFirstLine(linkFilePath)
+	if err != nil {
+		return Link{}, err
+	}
+	l.LinkFilePath = linkFilePath
+
+	fields := strings.Fields(firstLine)
+	l.IncludedFilePath = fields[0]
+	if len(fields) == 2 {
+		l.LinkChecksum = fields[1]
+	}
+
+	pathName := filepath.Join(filepath.Dir(linkFilePath), l.IncludedFilePath)
+	cs, err := getLinkedFileChecksum(pathName)
+	if err != nil {
+		return Link{}, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err)
+	}
+	if l.LinkChecksum == cs {
+		l.UpToDate = true
+	}
+	l.IncludedFileContentsChecksum = cs
+
+	return l, nil
+}
+
+func getLinkedFileChecksum(path string) (string, error) {
+	b, err := os.ReadFile(filepath.FromSlash(path))
+	if err != nil {
+		return "", err
+	}
+	cs, err := checksum(b)
+	if err != nil {
+		return "", err
+	}
+	return cs, nil
+}
+
+func readFirstLine(filePath string) (string, error) {
+	file, err := os.Open(filepath.FromSlash(filePath))
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	if scanner.Scan() {
+		return scanner.Text(), nil
+	}
+
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+
+	return "", fmt.Errorf("file is empty or first line is missing")
+}
+
+func checksum(b []byte) (string, error) {
+	hash := sha256.New()
+	if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}

code/go/pkg/specerrors/filter_test.go

@@ -15,14 +15,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func createValidationErrors(messages []string) ValidationErrors {
-	var allErrors ValidationErrors
-	for _, m := range messages {
-		allErrors = append(allErrors, NewStructuredErrorf(m))
-	}
-	return allErrors
-}
-
 func createValidationError(message, code string) ValidationError {
 	return NewStructuredError(errors.New(message), code)
 }

code/go/pkg/validator/validator.go

@@ -13,12 +13,15 @@ import (
 
 	"github.com/elastic/package-spec/v3/code/go/internal/packages"
 	"github.com/elastic/package-spec/v3/code/go/internal/validator"
+	"github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles"
 )
 
 // ValidateFromPath validates a package located at the given path against the
 // appropriate specification and returns any errors.
 func ValidateFromPath(packageRootPath string) error {
-	return ValidateFromFS(packageRootPath, os.DirFS(packageRootPath))
+	// We wrap the fs.FS with a linkedfiles.LinksFS to handle linked files.
+	linksFS := linkedfiles.NewLinksFS(packageRootPath, os.DirFS(packageRootPath))
+	return ValidateFromFS(packageRootPath, linksFS)
 }
 
 // ValidateFromZip validates a package on its zip format.

code/go/pkg/validator/validator_test.go

@@ -271,6 +271,7 @@ func TestValidateFile(t *testing.T) {
 				`required var "password" in optional group is not defined`,
 			},
 		},
+		"with_links": {},
 	}
 
 	for pkgName, test := range tests {
@@ -813,6 +814,41 @@ func TestValidateForbiddenDataStreamName(t *testing.T) {
 	}
 }
 
+func TestValidateAnyOfContents(t *testing.T) {
+	tests := map[string][]string{
+		"missing_required_files": {
+			"item [.keep] is not allowed in folder [../../../../test/packages/missing_required_files/agent/input]",
+			"item [.keep] is not allowed in folder [../../../../test/packages/missing_required_files/data_stream/foo/agent/stream]",
+			"item [.keep] is not allowed in folder [../../../../test/packages/missing_required_files/data_stream/foo/elasticsearch/ingest_pipeline]",
+			"item [.keep] is not allowed in folder [../../../../test/packages/missing_required_files/data_stream/foo/fields]",
+			`path "agent/input": no file matching any of the patterns [*.yml.hbs *.yml.hbs.link] found in agent/input`,
+			`data stream "foo": no file matching any of the patterns [*.yml.hbs *.yml.hbs.link] found in data_stream/foo/agent/stream`,
+			`data stream "foo": no file matching any of the patterns [*.yml *.yml.link] found in data_stream/foo/fields`,
+			`path "elasticsearch/ingest_pipeline": no file matching any of the patterns [*.yml *.json *.yml.link *.json.link] found in elasticsearch/ingest_pipeline`,
+			`data stream "foo": no file matching any of the patterns [*.yml *.json *.yml.link *.json.link] found in data_stream/foo/elasticsearch/ingest_pipeline`,
+		},
+	}
+
+	for pkgName, expectedErrorMessages := range tests {
+		t.Run(pkgName, func(t *testing.T) {
+			err := ValidateFromPath(path.Join("..", "..", "..", "..", "test", "packages", pkgName))
+			if len(expectedErrorMessages) == 0 {
+				assert.NoError(t, err)
+				return
+			}
+			assert.Error(t, err)
+
+			errs, ok := err.(specerrors.ValidationErrors)
+			require.True(t, ok)
+			assert.Len(t, errs, len(expectedErrorMessages))
+
+			for _, foundError := range errs {
+				require.Contains(t, expectedErrorMessages, foundError.Error())
+			}
+		})
+	}
+}
+
 func requireErrorMessage(t *testing.T, pkgName string, invalidItemsPerFolder map[string][]string, expectedErrorMessage string) {
 	pkgRootPath := filepath.Join("..", "..", "..", "..", "test", "packages", pkgName)
 

spec/changelog.yml

@@ -15,6 +15,12 @@
     - description: Allow security_rule assets in content package.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/885
+    - description: Add support for _dev/shared folder.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/888
+    - description: Add support for *.link files in agent, pipelines, and fields folders.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/888
 - version: 3.3.4
   changes:
     - description: Remove slo assets when spec versions are less than 3.4.0.

spec/input/_dev/shared/spec.yml

@@ -0,0 +1,2 @@
+spec:
+  additionalContents: true

spec/input/_dev/spec.yml

@@ -17,3 +17,7 @@ spec:
       name: test
       required: false
       $ref: "./test/spec.yml"
+    - description: Folder containing shared files.
+      type: folder
+      name: shared
+      $ref: "./shared/spec.yml"

spec/integration/_dev/shared/spec.yml

@@ -0,0 +1,2 @@
+spec:
+  additionalContents: true

spec/integration/_dev/spec.yml

@@ -22,3 +22,8 @@ spec:
       name: test
       required: false
       $ref: "./test/spec.yml"
+    - description: Folder containing shared files.
+      type: folder
+      name: shared
+      required: false
+      $ref: "./shared/spec.yml"

spec/integration/agent/spec.yml

@@ -11,4 +11,8 @@ spec:
       type: file
       sizeLimit: 2MB
       pattern: '^.+.yml.hbs$'
-      required: true
+      required: false
+    - description: Link to a config template file for inputs defined in the policy_templates section of the top level manifest
+      type: file
+      pattern: '^.+\.yml\.hbs\.link$'
+      required: false

spec/integration/data_stream/agent/spec.yml

@@ -7,8 +7,12 @@ spec:
     required: true
     additionalContents: false
     contents:
-    - description: Folder containing agent stream definitions
+    - description: Config template file for inputs defined in the policy_templates section of the top level manifest
       type: file
       sizeLimit: 2MB
       pattern: '^.+.yml.hbs$'
-      required: true
+      required: false
+    - description: Link to a config template file for inputs defined in the policy_templates section of the top level manifest
+      type: file
+      pattern: '^.+\.yml\.hbs\.link$'
+      required: false