dry-run: validate required fields

CHANGELOG.md

@@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add validation for dataset type. [#501](https://github.com/elastic/package-registry/pull/501)
 * Add -dry-run flag. [#503](https://github.com/elastic/package-registry/pull/503)
 * Encode fields in Kibana objects if not encoded. [#506](https://github.com/elastic/package-registry/pull/506)
+* Validate required fields in datasets. [#507](https://github.com/elastic/package-registry/pull/507)
 * Do not require "config.yml". [#508](https://github.com/elastic/package-registry/pull/508)
 
 ### Deprecated

docs/api/search-all.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -46,9 +46,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

docs/api/search-category-logs.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -28,9 +28,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

docs/api/search-kibana652.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -28,9 +28,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

docs/api/search-kibana721.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -37,9 +37,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

docs/api/search-package-experimental.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -46,9 +46,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

docs/api/search-package-internal.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -47,9 +47,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

docs/api/search.json

@@ -10,9 +10,9 @@
   },
   {
     "description": "Tests if no pipeline is set, it defaults to the default one",
-    "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
-    "name": "default-pipeline",
-    "path": "/package/default-pipeline/0.0.2",
+    "download": "/epr/default_pipeline/default_pipeline-0.0.2.tar.gz",
+    "name": "default_pipeline",
+    "path": "/package/default_pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -37,9 +37,9 @@
   },
   {
     "description": "Tests that multiple can be set to false",
-    "download": "/epr/multiple-false/multiple-false-0.0.1.tar.gz",
-    "name": "multiple-false",
-    "path": "/package/multiple-false/0.0.1",
+    "download": "/epr/multiple_false/multiple_false-0.0.1.tar.gz",
+    "name": "multiple_false",
+    "path": "/package/multiple_false/0.0.1",
     "title": "Multiple false",
     "type": "integration",
     "version": "0.0.1"

testdata/package/default-pipeline/0.0.2/manifest.yml → testdata/package/default_pipeline/0.0.2/manifest.yml

@@ -1,6 +1,6 @@
 format_version: 1.0.0
 
-name: default-pipeline
+name: default_pipeline
 description: Tests if no pipeline is set, it defaults to the default one
 version: 0.0.2
 title: Default pipeline Integration

testdata/package/multiple-false/0.0.1/manifest.yml → testdata/package/multiple_false/0.0.1/manifest.yml

@@ -1,6 +1,6 @@
 format_version: 1.0.0
 
-name: multiple-false
+name: multiple_false
 description: Tests that multiple can be set to false
 version: 0.0.1
 title: Multiple false

testdata/public/package/default-pipeline/0.0.2/manifest.yml → testdata/public/package/default_pipeline/0.0.2/manifest.yml

@@ -1,6 +1,6 @@
 format_version: 1.0.0
 
-name: default-pipeline
+name: default_pipeline
 description: Tests if no pipeline is set, it defaults to the default one
 version: 0.0.2
 title: Default pipeline Integration

testdata/public/package/multiple-false/0.0.1/manifest.yml → testdata/public/package/multiple_false/0.0.1/manifest.yml

@@ -1,6 +1,6 @@
 format_version: 1.0.0
 
-name: multiple-false
+name: multiple_false
 description: Tests that multiple can be set to false
 version: 0.0.1
 title: Multiple false

util/dataset.go

@@ -84,8 +84,12 @@ type Os struct {
 	Windows interface{} `config:"windows" json:"windows,omitempty" yaml:"windows,omitempty"`
 }
 
-func NewDataset(basePath string, p *Package) (*DataSet, error) {
+type fieldEntry struct {
+	name  string
+	aType string
+}
 
+func NewDataset(basePath string, p *Package) (*DataSet, error) {
 	// Check if manifest exists
 	manifestPath := filepath.Join(basePath, "manifest.yml")
 	_, err := os.Stat(manifestPath)
@@ -132,7 +136,6 @@ func NewDataset(basePath string, p *Package) (*DataSet, error) {
 	if !IsValidRelease(d.Release) {
 		return nil, fmt.Errorf("invalid release: %s", d.Release)
 	}
-
 	return d, nil
 }
 
@@ -199,6 +202,11 @@ func (d *DataSet) Validate() error {
 			return fmt.Errorf("defined ingest_pipeline does not exist: %s", pipelineDir+d.IngestPipeline)
 		}
 	}
+
+	err = d.validateRequiredFields()
+	if err != nil {
+		return errors.Wrap(err, "validating required fields failed")
+	}
 	return nil
 }
 
@@ -230,3 +238,98 @@ func validateIngestPipelineFile(pipelinePath string) error {
 	}
 	return err
 }
+
+// validateRequiredFields method loads fields from all files and checks if required fields are present.
+func (d *DataSet) validateRequiredFields() error {
+	fieldsDirPath := filepath.Join(d.BasePath, "fields")
+
+	// Collect fields from all files
+	var allFields []MapStr
+	err := filepath.Walk(fieldsDirPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		relativePath, err := filepath.Rel(fieldsDirPath, path)
+		if err != nil {
+			return errors.Wrapf(err, "cannot find relative path (fieldsDirPath: %s, path: %s)", fieldsDirPath, path)
+		}
+
+		if relativePath == "." {
+			return nil
+		}
+
+		body, err := ioutil.ReadFile(path)
+		if err != nil {
+			return errors.Wrapf(err, "reading file failed (path: %s)", path)
+		}
+
+		var m []MapStr
+		err = yamlv2.Unmarshal(body, &m)
+		if err != nil {
+			return errors.Wrapf(err, "unmarshaling file failed (path: %s)", path)
+		}
+
+		allFields = append(allFields, m...)
+		return nil
+	})
+	if err != nil {
+		return errors.Wrapf(err, "walking through fields files failed")
+	}
+
+	// Flatten all fields
+	for i, fields := range allFields {
+		allFields[i] = fields.Flatten()
+	}
+
+	// Verify required keys
+	err = requireField(allFields, "dataset.type", "constant_keyword", err)
+	err = requireField(allFields, "dataset.name", "constant_keyword", err)
+	err = requireField(allFields, "dataset.namespace", "constant_keyword", err)
+	err = requireField(allFields, "@timestamp", "date", err)
+	return err
+}
+
+func requireField(allFields []MapStr, searchedName, expectedType string, validationErr error) error {
+	if validationErr != nil {
+		return validationErr
+	}
+
+	f, err := findField(allFields, searchedName)
+	if err != nil {
+		return errors.Wrapf(err, "finding field failed (searchedName: %s)", searchedName)
+	}
+
+	if f.aType != expectedType {
+		return fmt.Errorf("wrong field type for '%s' (expected: %s, got: %s)", searchedName, expectedType, f.aType)
+	}
+	return nil
+}
+
+func findField(allFields []MapStr, searchedName string) (*fieldEntry, error) {
+	for _, fields := range allFields {
+		name, err := fields.GetValue("name")
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot get value (key: name)")
+		}
+
+		if name != searchedName {
+			continue
+		}
+
+		aType, err := fields.GetValue("type")
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot get value (key: type)")
+		}
+
+		if aType == "" {
+			return nil, fmt.Errorf("field '%s' found, but type is undefined", searchedName)
+		}
+
+		return &fieldEntry{
+			name:  name.(string),
+			aType: aType.(string),
+		}, nil
+	}
+	return nil, fmt.Errorf("field '%s' not found", searchedName)
+}