Add validation for dataset type (#501)

The valid dataset types at the moment are logs, metrics and events. events will be removed in the future but at the moment it is still used in the endpoint package. As soon as this is removed, the code here will be cleaned up.

With recent changes to the code, the datasets were not validated anymore during build or running the registry. This is now changed by adding validate for datasets every time a package is created.

CHANGELOG.md

@@ -19,6 +19,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Generate index.json file. [#470](https://github.com/elastic/package-registry/pull/470)
 * Stream archived package content. [#472](https://github.com/elastic/package-registry/pull/472)
 * Generate package index.json files. [#479](https://github.com/elastic/package-registry/pull/479)
+* Add validation for dataset type. [#501](https://github.com/elastic/package-registry/pull/501)
 
 ### Deprecated
 

util/dataset.go

@@ -24,6 +24,13 @@ const (
 	DirIngestPipeline = "ingest-pipeline"
 )
 
+var validTypes = map[string]string{
+	"logs":    "Logs",
+	"metrics": "Metrics",
+	// TODO: Remove as soon as endpoint package does not use it anymore
+	"events": "Events",
+}
+
 type DataSet struct {
 	ID             string   `config:"id" json:"id,omitempty" yaml:"id,omitempty"`
 	Title          string   `config:"title" json:"title" validate:"required"`
@@ -140,6 +147,10 @@ func (d *DataSet) Validate() error {
 		return fmt.Errorf("dataset name is not allowed to contain `-`: %s", d.ID)
 	}
 
+	if !d.validType() {
+		return fmt.Errorf("type is not valid: %s", d.Type)
+	}
+
 	if d.IngestPipeline == "" {
 		// Check that no ingest pipeline exists in the directory except default
 		for _, path := range paths {
@@ -191,6 +202,11 @@ func (d *DataSet) Validate() error {
 	return nil
 }
 
+func (d *DataSet) validType() bool {
+	_, exists := validTypes[d.Type]
+	return exists
+}
+
 func validateIngestPipelineFile(pipelinePath string) error {
 	f, err := ioutil.ReadFile(pipelinePath)
 	if err != nil {

util/package.go

@@ -317,7 +317,7 @@ func (p *Package) Validate() error {
 		}
 	}
 
-	return nil
+	return p.ValidateDatasets()
 }
 
 // GetDatasetPaths returns a list with the dataset paths inside this package
@@ -396,6 +396,28 @@ func (p *Package) LoadDataSets() error {
 	return nil
 }
 
+// ValidateDatasets loads all datasets and with it validates them
+func (p *Package) ValidateDatasets() error {
+
+	datasetPaths, err := p.GetDatasetPaths()
+	if err != nil {
+		return err
+	}
+
+	datasetsBasePath := filepath.Join(p.BasePath, "dataset")
+
+	for _, datasetPath := range datasetPaths {
+
+		datasetBasePath := filepath.Join(datasetsBasePath, datasetPath)
+
+		_, err := NewDataset(datasetBasePath, p)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (p *Package) GetPath() string {
 	return p.Name + "/" + p.Version
 }

util/packages.go

@@ -32,6 +32,7 @@ func GetPackages(packagesBasePath string) ([]Package, error) {
 		if err != nil {
 			return nil, err
 		}
+
 		packageList = append(packageList, *p)
 	}