Add spec for shared folder and link files

code/go/internal/specschema/folder_item_spec.go

@@ -81,6 +81,11 @@ func (s *ItemSpec) DevelopmentFolder() bool {
 	return s.itemSpec.DevelopmentFolder
 }
 
+// AllowLink returns true if the item allows links.
+func (s *ItemSpec) AllowLink() bool {
+	return s.itemSpec.AllowLink
+}
+
 // ForbiddenPatterns returns the list of forbidden patterns for the name of this item.
 func (s *ItemSpec) ForbiddenPatterns() []string {
 	return s.itemSpec.ForbiddenPatterns
@@ -135,6 +140,7 @@ type folderItemSpec struct {
 	AdditionalContents bool              `json:"additionalContents" yaml:"additionalContents"`
 	Contents           []*folderItemSpec `json:"contents" yaml:"contents"`
 	DevelopmentFolder  bool              `json:"developmentFolder" yaml:"developmentFolder"`
+	AllowLink          bool              `json:"allowLink" yaml:"allowLink"`
 
 	// As it is required to be inline both in yaml and json, this struct must be public embedded field
 	SpecLimits `yaml:",inline"`

code/go/internal/spectypes/item.go

@@ -57,6 +57,9 @@ type ItemSpec interface {
 	// DevelopmentFolder returns true if the item is inside a development folder.
 	DevelopmentFolder() bool
 
+	// AllowLink returns true if the item allows links.
+	AllowLink() bool
+
 	// ForbiddenPatterns returns the list of forbidden patterns for the name of this item.
 	ForbiddenPatterns() []string
 

code/go/internal/validator/folder_item_spec.go

@@ -16,13 +16,15 @@ import (
 func matchingFileExists(spec spectypes.ItemSpec, files []fs.DirEntry) (bool, error) {
 	if spec.Name() != "" {
 		for _, file := range files {
-			if file.Name() == spec.Name() {
+			_, fileName := checkLink(file.Name())
+			if fileName == spec.Name() {
 				return spec.IsDir() == file.IsDir(), nil
 			}
 		}
 	} else if spec.Pattern() != "" {
 		for _, file := range files {
-			isMatch, err := regexp.MatchString(spec.Pattern(), file.Name())
+			_, fileName := checkLink(file.Name())
+			isMatch, err := regexp.MatchString(spec.Pattern(), fileName)
 			if err != nil {
 				return false, fmt.Errorf("invalid folder item spec pattern: %w", err)
 			}

code/go/internal/validator/folder_spec.go

@@ -190,6 +190,7 @@ func (v *validator) Validate() specerrors.ValidationErrors {
 }
 
 func (v *validator) findItemSpec(folderItemName string) (spectypes.ItemSpec, error) {
+	isLink, folderItemName := checkLink(folderItemName)
 	for _, itemSpec := range v.spec.Contents() {
 		if itemSpec.Name() != "" && itemSpec.Name() == folderItemName {
 			return itemSpec, nil
@@ -213,6 +214,9 @@ func (v *validator) findItemSpec(folderItemName string) (spectypes.ItemSpec, err
 				}
 
 				if !isForbidden {
+					if isLink && !itemSpec.AllowLink() {
+						return nil, fmt.Errorf("item [%s] is a link but is not allowed", folderItemName)
+					}
 					return itemSpec, nil
 				}
 			}
@@ -222,3 +226,13 @@ func (v *validator) findItemSpec(folderItemName string) (spectypes.ItemSpec, err
 	// No item spec found
 	return nil, nil
 }
+
+// checkLink checks if an item is a link and returns the item name without the
+// ".link" suffix if it is a link.
+func checkLink(itemName string) (bool, string) {
+	const linkExtension = ".link"
+	if strings.HasSuffix(itemName, linkExtension) {
+		return true, strings.TrimSuffix(itemName, linkExtension)
+	}
+	return false, itemName
+}

code/go/pkg/linkedfiles/fs.go

@@ -0,0 +1,47 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package linkedfiles
+
+import (
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+)
+
+var _ fs.FS = (*LinksFS)(nil)
+
+// LinksFS is a filesystem that handles linked files.
+// It wraps another filesystem and checks for linked files with the ".link" extension.
+// If a linked file is found, it reads the link file to determine the target file
+// and its checksum. If the target file is up to date, it returns the target file.
+// Otherwise, it returns an error.
+type LinksFS struct {
+	workDir string
+	inner   fs.FS
+}
+
+// NewLinksFS creates a new LinksFS.
+func NewLinksFS(workDir string, inner fs.FS) *LinksFS {
+	return &LinksFS{workDir: workDir, inner: inner}
+}
+
+// Open opens a file in the filesystem.
+func (lfs *LinksFS) Open(name string) (fs.File, error) {
+	const linkExtension = ".link"
+	if filepath.Ext(name) != linkExtension {
+		return lfs.inner.Open(name)
+	}
+	pathName := filepath.Join(lfs.workDir, name)
+	l, err := NewLinkedFile(pathName)
+	if err != nil {
+		return nil, err
+	}
+	if !l.UpToDate {
+		return nil, fmt.Errorf("linked file %s is not up to date", name)
+	}
+	includedPath := filepath.Join(lfs.workDir, filepath.Dir(name), l.IncludedFilePath)
+	return os.Open(includedPath)
+}

code/go/pkg/linkedfiles/linkedfiles.go

@@ -0,0 +1,98 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package linkedfiles
+
+import (
+	"bufio"
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// A Link represents a linked file.
+// It contains the path to the link file, the checksum of the linked file,
+// the path to the target file, and the checksum of the included file contents.
+// It also contains a boolean indicating whether the link is up to date.
+type Link struct {
+	LinkFilePath string
+	LinkChecksum string
+
+	IncludedFilePath             string
+	IncludedFileContentsChecksum string
+
+	UpToDate bool
+}
+
+// NewLinkedFile creates a new Link from the given link file path.
+func NewLinkedFile(linkFilePath string) (Link, error) {
+	var l Link
+	firstLine, err := readFirstLine(linkFilePath)
+	if err != nil {
+		return Link{}, err
+	}
+	l.LinkFilePath = linkFilePath
+
+	fields := strings.Fields(firstLine)
+	l.IncludedFilePath = fields[0]
+	if len(fields) == 2 {
+		l.LinkChecksum = fields[1]
+	}
+
+	pathName := filepath.Join(filepath.Dir(linkFilePath), l.IncludedFilePath)
+	cs, err := getLinkedFileChecksum(pathName)
+	if err != nil {
+		return Link{}, fmt.Errorf("could not collect file %v: %w", l.IncludedFilePath, err)
+	}
+	if l.LinkChecksum == cs {
+		l.UpToDate = true
+	}
+	l.IncludedFileContentsChecksum = cs
+
+	return l, nil
+}
+
+func getLinkedFileChecksum(path string) (string, error) {
+	b, err := os.ReadFile(filepath.FromSlash(path))
+	if err != nil {
+		return "", err
+	}
+	cs, err := checksum(b)
+	if err != nil {
+		return "", err
+	}
+	return cs, nil
+}
+
+func readFirstLine(filePath string) (string, error) {
+	file, err := os.Open(filepath.FromSlash(filePath))
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	if scanner.Scan() {
+		return scanner.Text(), nil
+	}
+
+	if err := scanner.Err(); err != nil {
+		return "", err
+	}
+
+	return "", fmt.Errorf("file is empty or first line is missing")
+}
+
+func checksum(b []byte) (string, error) {
+	hash := sha256.New()
+	if _, err := io.Copy(hash, bytes.NewReader(b)); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}

code/go/pkg/specerrors/filter_test.go

@@ -15,14 +15,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func createValidationErrors(messages []string) ValidationErrors {
-	var allErrors ValidationErrors
-	for _, m := range messages {
-		allErrors = append(allErrors, NewStructuredErrorf(m))
-	}
-	return allErrors
-}
-
 func createValidationError(message, code string) ValidationError {
 	return NewStructuredError(errors.New(message), code)
 }

code/go/pkg/validator/validator.go

@@ -13,12 +13,15 @@ import (
 
 	"github.com/elastic/package-spec/v3/code/go/internal/packages"
 	"github.com/elastic/package-spec/v3/code/go/internal/validator"
+	"github.com/elastic/package-spec/v3/code/go/pkg/linkedfiles"
 )
 
 // ValidateFromPath validates a package located at the given path against the
 // appropriate specification and returns any errors.
 func ValidateFromPath(packageRootPath string) error {
-	return ValidateFromFS(packageRootPath, os.DirFS(packageRootPath))
+	// We wrap the fs.FS with a linkedfiles.LinksFS to handle linked files.
+	linksFS := linkedfiles.NewLinksFS(packageRootPath, os.DirFS(packageRootPath))
+	return ValidateFromFS(packageRootPath, linksFS)
 }
 
 // ValidateFromZip validates a package on its zip format.

code/go/pkg/validator/validator_test.go

@@ -271,6 +271,7 @@ func TestValidateFile(t *testing.T) {
 				`required var "password" in optional group is not defined`,
 			},
 		},
+		"with_links": {},
 	}
 
 	for pkgName, test := range tests {

spec/changelog.yml

@@ -18,6 +18,12 @@
     - description: Allow security_rule assets in content package.
       type: enhancement
       link: https://github.com/elastic/package-spec/pull/885
+    - description: Add support for _dev/shared folder.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/888
+    - description: Add support for *.link files in agent, pipelines, and fields folders.
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/888
     - description: Fix regexes used for paths and names in package files.
       type: bugfix
       link: https://github.com/elastic/package-spec/pull/889

spec/input/_dev/spec.yml

@@ -17,3 +17,7 @@ spec:
       name: test
       required: false
       $ref: "./test/spec.yml"
+    - description: Folder containing shared files.
+      type: folder
+      name: shared
+      $ref: "../../integration/spec.yml"

spec/integration/_dev/shared/spec.yml

@@ -0,0 +1,2 @@
+spec:
+  additionalContents: true

spec/integration/_dev/spec.yml

@@ -22,3 +22,8 @@ spec:
       name: test
       required: false
       $ref: "./test/spec.yml"
+    - description: Folder containing shared files.
+      type: folder
+      name: shared
+      required: false
+      $ref: "./shared/spec.yml"

spec/integration/agent/spec.yml

@@ -12,3 +12,4 @@ spec:
       sizeLimit: 2MB
       pattern: '^.+\.yml\.hbs$'
       required: true
+      allowLink: true

spec/integration/data_stream/agent/spec.yml

@@ -7,8 +7,9 @@ spec:
     required: true
     additionalContents: false
     contents:
-    - description: Folder containing agent stream definitions
+    - description: Config template file for inputs defined in the policy_templates section of the top level manifest
       type: file
       sizeLimit: 2MB
       pattern: '^.+\.yml\.hbs$'
       required: true
+      allowLink: true

spec/integration/data_stream/fields/spec.yml

@@ -6,4 +6,5 @@ spec:
     pattern: '^[a-z0-9][a-z0-9_-]+[a-z0-9]\.yml$'
     required: true
     contentMediaType: "application/x-yaml"
+    allowLink: true
     $ref: "./fields.spec.yml"

spec/integration/data_stream/spec.yml

@@ -61,12 +61,14 @@ spec:
           # TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54)
           contentMediaType: "application/x-yaml; require-document-dashes=true"
           required: false
+          allowLink: true
           $ref: "../../integration/elasticsearch/pipeline.spec.yml"
         - description: Supporting ingest pipeline definitions in JSON
           type: file
           pattern: '^.+\.json$'
           contentMediaType: "application/json"
           required: false
+          allowLink: true
           $ref: "../../integration/elasticsearch/pipeline.spec.yml"
     - description: Sample event file
       type: file

spec/integration/elasticsearch/spec.yml

@@ -22,13 +22,15 @@ spec:
           # TODO Determine if special handling of `---` is required (issue: https://github.com/elastic/package-spec/pull/54)
           contentMediaType: "application/x-yaml; require-document-dashes=true"
           required: false
+          allowLink: true
           $ref: "./pipeline.spec.yml"
         - description: Supporting ingest pipeline definitions in JSON
           type: file
           sizeLimit: 3MB
           pattern: '^.+\.json$'
           contentMediaType: "application/json"
           required: false
+          allowLink: true
           $ref: "./pipeline.spec.yml"
     - description: Folder containing Elasticsearch Transforms
       # https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html

test/packages/good_v3/data_stream/foo/fields/base-fields.yml.link

@@ -0,0 +1 @@
+../../../_dev/shared/base-fields.yml 092c60ec1f7725d278aa0564d946f6803736c436239c4ca20049013a4ce8e91c

test/packages/missing_required_files/_dev/build/build.yml

@@ -0,0 +1,3 @@
+dependencies:
+  ecs:
+    reference: git@v8.7.0

test/packages/missing_required_files/changelog.yml

@@ -0,0 +1,5 @@
+- version: 0.1.2
+  changes:
+    - description: initial release
+      type: enhancement
+      link: https://github.com/elastic/package-spec/pull/131

test/packages/missing_required_files/data_stream/foo/elasticsearch/ingest_pipeline/.keep

@@ -0,0 +1,3 @@
+description: No dashes here
+
+processors:

test/packages/missing_required_files/data_stream/foo/manifest.yml

@@ -0,0 +1,23 @@
+title: Nginx access logs
+type: logs
+release: experimental
+streams:
+  - input: logfile
+    vars:
+      - name: paths
+        type: text
+        title: Paths
+        multi: true
+        required: true
+        show_user: true
+        default:
+          - /var/log/nginx/access.log*
+      - name: server_status_path
+        type: text
+        title: Server Status Path
+        multi: false
+        required: true
+        show_user: false
+        default: /server-status
+    title: Nginx access logs
+    description: Collect Nginx access logs

test/packages/missing_required_files/docs/README.md

@@ -0,0 +1 @@
+Main