Initial support for content packages (#2049)

Add archetype for `elastic-package create package`.
Add test package, tests pass, but they do nothing.
Package installation seems to work, we will need to review related code in Kibana.

cmd/create_package.go

@@ -49,7 +49,7 @@ func createPackageCommandAction(cmd *cobra.Command, args []string) error {
 			Name: "type",
 			Prompt: &survey.Select{
 				Message: "Package type:",
-				Options: []string{"input", "integration"},
+				Options: []string{"input", "integration", "content"},
 				Default: "integration",
 			},
 			Validate: survey.Required,

internal/packages/archetype/_static/package-manifest.yml.tmpl

@@ -16,6 +16,10 @@ conditions:
     version: "{{.Manifest.Conditions.Kibana.Version}}"
   elastic:
     subscription: "{{.Manifest.Conditions.Elastic.Subscription}}"
+{{- if eq .Manifest.Type "content" }}
+discovery:
+  fields: []
+{{- end }}
 screenshots:
   - src: /img/sample-screenshot.png
     title: Sample screenshot
@@ -26,6 +30,7 @@ icons:
     title: Sample logo
     size: 32x32
     type: image/svg+xml
+{{- if (or (eq .Manifest.Type "integration") (eq .Manifest.Type "input")) }}
 policy_templates:
 {{- if eq .Manifest.Type "integration" }}
   - name: sample
@@ -72,6 +77,7 @@ policy_templates:
           - localhost
 {{ end }}
 {{ end -}}
+{{ end -}}
 {{ if .Manifest.Elasticsearch }}
 elasticsearch:
 {{ if .Manifest.Elasticsearch.SourceMode }}

internal/packages/archetype/_static/package-validation.yml.tmpl

@@ -0,0 +1,5 @@
+errors:
+  exclude_checks:
+{{- range $check := .ExcludeChecks }}
+    - {{ $check }}
+{{- end }}

internal/packages/archetype/package.go

@@ -20,6 +20,8 @@ type PackageDescriptor struct {
 	Manifest packages.PackageManifest
 
 	InputDataStreamType string
+
+	ExcludeChecks []string
 }
 
 // CreatePackage function bootstraps the new package based on the provided descriptor.
@@ -92,6 +94,14 @@ func createPackageInDir(packageDescriptor PackageDescriptor, cwd string) error {
 
 	}
 
+	if len(packageDescriptor.ExcludeChecks) > 0 {
+		logger.Debugf("Write validation file")
+		err = renderResourceFile(validationBaseTemplate, &packageDescriptor, filepath.Join(baseDir, "validation.yml"))
+		if err != nil {
+			return fmt.Errorf("can't render validation file")
+		}
+	}
+
 	logger.Debugf("Format the entire package")
 	err = formatter.Format(baseDir, false)
 	if err != nil {

internal/packages/archetype/package_test.go

@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/Masterminds/semver/v3"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -29,6 +30,10 @@ func TestPackage(t *testing.T) {
 		pd := createPackageDescriptorForTest("input", "^8.9.0")
 		createAndCheckPackage(t, pd, true)
 	})
+	t.Run("content-package", func(t *testing.T) {
+		pd := createPackageDescriptorForTest("content", "^8.16.0")
+		createAndCheckPackage(t, pd, true)
+	})
 }
 
 func createAndCheckPackage(t *testing.T, pd PackageDescriptor, valid bool) {
@@ -52,17 +57,28 @@ func createPackageDescriptorForTest(packageType, kibanaVersion string) PackageDe
 			},
 		}
 	}
+	version := "1.2.3"
 	specVersion, err := GetLatestStableSpecVersion()
+	excludeChecks := []string{}
 	if err != nil {
 		panic(err)
 	}
+	if packageType == "content" {
+		minSpecVersion := semver.MustParse("3.4.0")
+		if !specVersion.LessThan(minSpecVersion) {
+			panic("this code can be removed")
+		}
+		version = "0.1.0"
+		specVersion = *minSpecVersion
+		excludeChecks = append(excludeChecks, "PSR00002")
+	}
 	return PackageDescriptor{
 		Manifest: packages.PackageManifest{
 			SpecVersion: specVersion.String(),
 			Name:        "go_unit_test_package",
 			Title:       "Go Unit Test Package",
 			Type:        packageType,
-			Version:     "1.2.3",
+			Version:     version,
 			Conditions: packages.Conditions{
 				Kibana: packages.KibanaConditions{
 					Version: kibanaVersion,
@@ -79,12 +95,13 @@ func createPackageDescriptorForTest(packageType, kibanaVersion string) PackageDe
 			Categories:    []string{"aws", "custom"},
 			Elasticsearch: elasticsearch,
 		},
+		ExcludeChecks:       excludeChecks,
 		InputDataStreamType: inputDataStreamType,
 	}
 }
 
 func checkPackage(t *testing.T, packageRoot string, valid bool) {
-	err := validation.ValidateFromPath(packageRoot)
+	err, _ := validation.ValidateAndFilterFromPath(packageRoot)
 	if !valid {
 		assert.Error(t, err)
 		return

internal/packages/archetype/resources.go

@@ -20,6 +20,9 @@ var packageDocsReadme string
 //go:embed _static/fields-base.yml.tmpl
 var fieldsBaseTemplate string
 
+//go:embed _static/package-validation.yml.tmpl
+var validationBaseTemplate string
+
 // Images (logo and screenshot)
 
 //go:embed _static/sampleIcon.svg

internal/packages/packages.go

@@ -12,6 +12,7 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"slices"
 
 	"github.com/elastic/go-ucfg"
 	"github.com/elastic/go-ucfg/yaml"
@@ -96,6 +97,16 @@ type Conditions struct {
 	Elastic ElasticConditions `config:"elastic" json:"elastic" yaml:"elastic"`
 }
 
+// Discovery define indications for the data this package can be useful with.
+type Discovery struct {
+	Fields []DiscoveryField `config:"fields" json:"fields" yaml:"fields"`
+}
+
+// DiscoveryField defines a field used for discovery.
+type DiscoveryField struct {
+	Name string `config:"name" json:"name" yaml:"name"`
+}
+
 // PolicyTemplate is a configuration of inputs responsible for collecting log or metric data.
 type PolicyTemplate struct {
 	Name        string   `config:"name" json:"name" yaml:"name"`                                                       // Name of policy template.
@@ -130,6 +141,7 @@ type PackageManifest struct {
 	Version         string           `config:"version" json:"version" yaml:"version"`
 	Source          Source           `config:"source" json:"source" yaml:"source"`
 	Conditions      Conditions       `config:"conditions" json:"conditions" yaml:"conditions"`
+	Discovery       Discovery        `config:"discovery" json:"discovery" yaml:"discovery"`
 	PolicyTemplates []PolicyTemplate `config:"policy_templates" json:"policy_templates" yaml:"policy_templates"`
 	Vars            []Variable       `config:"vars" json:"vars" yaml:"vars"`
 	Owner           Owner            `config:"owner" json:"owner" yaml:"owner"`
@@ -450,7 +462,12 @@ func isPackageManifest(path string) (bool, error) {
 	if err != nil {
 		return false, fmt.Errorf("reading package manifest failed (path: %s): %w", path, err)
 	}
-	return (m.Type == "integration" || m.Type == "input") && m.Version != "", nil
+	supportedTypes := []string{
+		"content",
+		"input",
+		"integration",
+	}
+	return slices.Contains(supportedTypes, m.Type) && m.Version != "", nil
 }
 
 func isDataStreamManifest(path string) (bool, error) {

internal/testrunner/testrunner.go

@@ -503,7 +503,7 @@ func PackageHasDataStreams(manifest *packages.PackageManifest) (bool, error) {
 	switch manifest.Type {
 	case "integration":
 		return true, nil
-	case "input":
+	case "input", "content":
 		return false, nil
 	default:
 		return false, fmt.Errorf("unexpected package type %q", manifest.Type)

test/packages/parallel/otel_http_server/LICENSE.txt

@@ -0,0 +1,93 @@
+Elastic License 2.0
+
+URL: https://www.elastic.co/licensing/elastic-license
+
+## Acceptance
+
+By using the software, you agree to all of the terms and conditions below.
+
+## Copyright License
+
+The licensor grants you a non-exclusive, royalty-free, worldwide,
+non-sublicensable, non-transferable license to use, copy, distribute, make
+available, and prepare derivative works of the software, in each case subject to
+the limitations and conditions below.
+
+## Limitations
+
+You may not provide the software to third parties as a hosted or managed
+service, where the service provides users with access to any substantial set of
+the features or functionality of the software.
+
+You may not move, change, disable, or circumvent the license key functionality
+in the software, and you may not remove or obscure any functionality in the
+software that is protected by the license key.
+
+You may not alter, remove, or obscure any licensing, copyright, or other notices
+of the licensor in the software. Any use of the licensor’s trademarks is subject
+to applicable law.
+
+## Patents
+
+The licensor grants you a license, under any patent claims the licensor can
+license, or becomes able to license, to make, have made, use, sell, offer for
+sale, import and have imported the software, in each case subject to the
+limitations and conditions in this license. This license does not cover any
+patent claims that you cause to be infringed by modifications or additions to
+the software. If you or your company make any written claim that the software
+infringes or contributes to infringement of any patent, your patent license for
+the software granted under these terms ends immediately. If your company makes
+such a claim, your patent license ends immediately for work on behalf of your
+company.
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you
+also gets a copy of these terms.
+
+If you modify the software, you must include in any modified copies of the
+software prominent notices stating that you have modified the software.
+
+## No Other Rights
+
+These terms do not imply any licenses other than those expressly granted in
+these terms.
+
+## Termination
+
+If you use the software in violation of these terms, such use is not licensed,
+and your licenses will automatically terminate. If the licensor provides you
+with a notice of your violation, and you cease all violation of this license no
+later than 30 days after you receive that notice, your licenses will be
+reinstated retroactively. However, if you violate these terms after such
+reinstatement, any additional violation of these terms will cause your licenses
+to terminate automatically and permanently.
+
+## No Liability
+
+*As far as the law allows, the software comes as is, without any warranty or
+condition, and the licensor will not be liable to you for any damages arising
+out of these terms or the use or nature of the software, under any kind of
+legal claim.*
+
+## Definitions
+
+The **licensor** is the entity offering these terms, and the **software** is the
+software the licensor makes available under these terms, including any portion
+of it.
+
+**you** refers to the individual or entity agreeing to these terms.
+
+**your company** is any legal entity, sole proprietorship, or other kind of
+organization that you work for, plus all organizations that have control over,
+are under the control of, or are under common control with that
+organization. **control** means ownership of substantially all the assets of an
+entity, or the power to direct its management and policies by vote, contract, or
+otherwise. Control can be direct or indirect.
+
+**your licenses** are all the licenses granted to you for the software under
+these terms.
+
+**use** means anything you do with the software requiring one of your licenses.
+
+**trademark** means trademarks, service marks, and similar rights.

test/packages/parallel/otel_http_server/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

test/packages/parallel/otel_http_server/docs/README.md

@@ -0,0 +1,4 @@
+# HTTP Server following OTEL Semconv
+
+This package includes dashboards for data collected from HTTP Servers following
+OTEL Semconv.

test/packages/parallel/otel_http_server/manifest.yml

@@ -0,0 +1,30 @@
+format_version: 3.4.0
+name: otel_http_server
+title: "HTTP Server following OTEL Semconv"
+version: 0.0.1
+source:
+  license: "Elastic-2.0"
+description: "This package provides dashboards for HTTP server metrics that follow OTEL Semantic Conventions."
+type: content
+categories:
+  - web
+conditions:
+  kibana:
+    version: "^8.16.0"
+  elastic:
+    subscription: "basic"
+discovery:
+  fields: []
+screenshots:
+  - src: /img/sample-screenshot.png
+    title: Sample screenshot
+    size: 600x600
+    type: image/png
+icons:
+  - src: /img/sample-logo.svg
+    title: Sample logo
+    size: 32x32
+    type: image/svg+xml
+owner:
+  github: elastic/ecosystem
+  type: elastic

test/packages/parallel/otel_http_server/validation.yml

@@ -0,0 +1,3 @@
+errors:
+  exclude_checks:
+    - PSR00002