Add common spec and run validation against it

CHANGELOG.md

@@ -8,6 +8,7 @@ Changelog for ecszap
 
 ### Bug Fixes
 * Change `stacktrace` to `stack_trace` in output and in json and yaml config option for `EncoderConfig.EnableStacktrace` [pull#21](https://github.com/elastic/ecs-logging-go-zap/pull/21)
+* Do not allow configuration of `@timestamp`, instead always set format to ISO8601 [pull#23](https://github.com/elastic/ecs-logging-go-zap/pull/23)
 
 ## 0.3.0
 

README.md

@@ -14,7 +14,7 @@ Following fields will be added by default:
 ```
 {
     "log.level":"info",
-    "@timestamp":1583748236254129,
+    "@timestamp":"2020-09-13T10:48:03.000Z",
     "message":"some logging info",
     "ecs.version":"1.6.0"
 }
@@ -56,7 +56,7 @@ logger.Info("some logging info",
 	// Log Output:
 	//{
 	//	"log.level":"info",
-	//	"@timestamp":1584716847523456,
+	//	"@timestamp":"2020-09-13T10:48:03.000Z",
 	//	"log.logger":"mylogger",
 	//	"log.origin":{
 	//		"file.name":"main/main.go",
@@ -80,7 +80,7 @@ logger.Error("some error", zap.Error(pkgerrors.Wrap(err, "crash")))
 	// Log Output:
 	//{
 	//	"log.level":"error",
-	//	"@timestamp":1584716847523842,
+	//	"@timestamp":"2020-09-13T10:48:03.000Z",
 	//	"log.logger":"mylogger",
 	//	"log.origin":{
 	//		"file.name":"main/main.go",
@@ -107,7 +107,7 @@ sugar.Infow("some logging info",
 	// Log Output:
 	//{
 	//	"log.level":"info",
-	//	"@timestamp":1584716847523941,
+	//	"@timestamp":"2020-09-13T10:48:03.000Z",
 	//	"log.logger":"mylogger",
 	//	"log.origin":{
 	//		"file.name":"main/main.go",

core_test.go

@@ -35,7 +35,7 @@ func TestCore(t *testing.T) {
 		zap.Error(errors.New("boom")),
 	}
 	assertLogged := func(t *testing.T, out testOutput) {
-		out.requireContains(t, []string{"error", "foo"})
+		out.validate(t, "foo", "error")
 		outErr, ok := out.m["error"].(map[string]interface{})
 		require.True(t, ok, out.m)
 		assert.Equal(t, map[string]interface{}{"message": "boom"}, outErr)

encoder_config.go

@@ -27,14 +27,14 @@ var (
 	defaultEncodeDuration = zapcore.NanosDurationEncoder
 	defaultEncodeLevel    = zapcore.LowercaseLevelEncoder
 	defaultEncodeCaller   = ShortCallerEncoder
-	defaultEncodeTime     = zapcore.ISO8601TimeEncoder
 
 	callerKey     = "log.origin"
 	logLevelKey   = "log.level"
 	logNameKey    = "log.logger"
 	messageKey    = "message"
 	stackTraceKey = "log.origin.stack_trace"
 	timeKey       = "@timestamp"
+	encodeTime    = zapcore.ISO8601TimeEncoder
 )
 
 // EncoderConfig exports all non ECS related configurable settings.
@@ -68,9 +68,6 @@ type EncoderConfig struct {
 	// EncodeCaller defines how an entry caller should be serialized.
 	// It will only be applied if EnableCaller is set to true.
 	EncodeCaller CallerEncoder `json:"callerEncoder" yaml:"callerEncoder"`
-
-	// EncodeTime defines how the log timestamp should be serialized
-	EncodeTime zapcore.TimeEncoder `json:"timeEncoder" yaml:"timeEncoder"`
 }
 
 // NewDefaultEncoderConfig returns an EncoderConfig with default settings.
@@ -94,15 +91,12 @@ func (cfg EncoderConfig) ToZapCoreEncoderConfig() zapcore.EncoderConfig {
 		MessageKey:     messageKey,
 		LevelKey:       logLevelKey,
 		TimeKey:        timeKey,
-		EncodeTime:     cfg.EncodeTime,
+		EncodeTime:     encodeTime,
 		LineEnding:     cfg.LineEnding,
 		EncodeDuration: cfg.EncodeDuration,
 		EncodeName:     cfg.EncodeName,
 		EncodeLevel:    cfg.EncodeLevel,
 	}
-	if encCfg.EncodeTime == nil {
-		encCfg.EncodeTime = defaultEncodeTime
-	}
 	if encCfg.EncodeDuration == nil {
 		encCfg.EncodeDuration = defaultEncodeDuration
 	}
@@ -156,10 +150,9 @@ func ECSCompatibleEncoderConfig(cfg zapcore.EncoderConfig) zapcore.EncoderConfig
 		cfg.CallerKey = callerKey
 		cfg.EncodeCaller = defaultEncodeCaller
 	}
+	// always set the time encoding to the ISO8601 time format
+	cfg.EncodeTime = encodeTime
 	// ensure all required encoders are set
-	if cfg.EncodeTime == nil {
-		cfg.EncodeTime = defaultEncodeTime
-	}
 	if cfg.EncodeDuration == nil {
 		cfg.EncodeDuration = defaultEncodeDuration
 	}

encoder_config_test.go

@@ -38,10 +38,9 @@ func TestJSONEncoder_EncoderConfig(t *testing.T) {
 		expected string
 	}{
 		{name: "defaultConfig",
-			cfg:   NewDefaultEncoderConfig(),
-			input: `{"timeEncoder":"millis"}`,
+			cfg: NewDefaultEncoderConfig(),
 			expected: `{"log.level": "debug",
-						"@timestamp": 1583484083953.468,
+						"@timestamp": "2020-09-13T10:48:03.000Z",
 						"message": "log message",
 						"log.origin": {
 							"file.line": 30,
@@ -52,9 +51,8 @@ func TestJSONEncoder_EncoderConfig(t *testing.T) {
 						"foo": "bar",
 						"dur": 5000000}`},
 		{name: "defaultUnmarshal",
-			input: `{"timeEncoder":"millis"}`,
 			expected: `{"log.level": "debug",
-						"@timestamp": 1583484083953.468,
+						"@timestamp": "2020-09-13T10:48:03.000Z",
 						"message": "log message",
 						"foo": "bar",
 						"dur": 5000000}`},
@@ -63,11 +61,10 @@ func TestJSONEncoder_EncoderConfig(t *testing.T) {
   					 "enableStackTrace": true,
 					 "enableCaller":true,
 					 "levelEncoder": "upper",
-					 "timeEncoder":"nanos",
 				 	 "durationEncoder": "ms",
 					 "callerEncoder": "short"}`,
 			expected: `{"log.level": "debug",
-						"@timestamp": 1583484083953467800,
+						"@timestamp": "2020-09-13T10:48:03.000Z",
 						"message": "log message",
 						"log.origin": {
 							"file.line": 30,
@@ -78,9 +75,9 @@ func TestJSONEncoder_EncoderConfig(t *testing.T) {
 						"foo": "bar",
 						"dur": 5}`},
 		{name: "fullCaller",
-			input: `{"callerEncoder": "full","enableCaller":true,"timeEncoder":"millis"}`,
+			input: `{"callerEncoder": "full","enableCaller":true}`,
 			expected: `{"log.level": "debug",
-						"@timestamp": 1583484083953.468,
+						"@timestamp": "2020-09-13T10:48:03.000Z",
 						"message": "log message",
 						"log.origin": {
 							"file.line": 30,
@@ -93,7 +90,7 @@ func TestJSONEncoder_EncoderConfig(t *testing.T) {
 			// setup
 			entry := zapcore.Entry{
 				Level:      zapcore.DebugLevel,
-				Time:       time.Unix(1583484083, 953467845),
+				Time:       time.Unix(1599994083, 0).UTC(),
 				Message:    "log message",
 				Caller:     caller,
 				Stack:      "frames",
@@ -127,9 +124,9 @@ func TestECSCompatibleEncoderConfig(t *testing.T) {
 		expected string
 	}{
 		{name: "empty config",
-			cfg: zapcore.EncoderConfig{EncodeTime: zapcore.EpochTimeEncoder},
+			cfg: zapcore.EncoderConfig{},
 			expected: `{"log.level": "debug",
-						"@timestamp": 1583484083.9534678,
+						"@timestamp": "2020-09-13T10:48:03.000Z",
 						"message": "log message",
 						"foo": "bar",
 						"count": 8}`},
@@ -140,7 +137,7 @@ func TestECSCompatibleEncoderConfig(t *testing.T) {
 				NameKey: "replaced nameKey", StacktraceKey: "replaced stackTraceKey",
 				CallerKey: "replaced callerKey", EncodeLevel: zapcore.CapitalLevelEncoder},
 			expected: `{"log.level": "DEBUG",
-						"@timestamp": 1583484083953.468,
+						"@timestamp": "2020-09-13T10:48:03.000Z",
 						"message": "log message",
 						"log.origin": {
 							"file.line": 30,
@@ -155,7 +152,7 @@ func TestECSCompatibleEncoderConfig(t *testing.T) {
 			// setup
 			entry := zapcore.Entry{
 				Level:      zapcore.DebugLevel,
-				Time:       time.Unix(1583484083, 953467845).UTC(),
+				Time:       time.Unix(1599994083, 0).UTC(),
 				Message:    "log message",
 				Caller:     caller,
 				Stack:      "frames",

internal/spec/spec.go

@@ -0,0 +1,106 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package spec
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path"
+	"path/filepath"
+	"runtime"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// V1 holds the v1 specification
+	V1 *Spec
+)
+
+// Spec holds the fields specified for a spec version
+type Spec struct {
+	URL string `json:"url"`
+	ECS struct {
+		Version string `json:"version"`
+	} `json:"ecs"`
+	Fields map[string]Field `json:"fields"`
+}
+
+// Field contains requirements for a log field
+type Field struct {
+	Comment       Comment      `json:"comment"`
+	Default       string       `json:"default"`
+	Index         *int         `json:"index"`
+	Required      bool         `json:"required"`
+	Sanitization  Sanitization `json:"sanitization"`
+	TopLevelField bool         `json:"top_level_field"`
+	Type          string       `json:"type"`
+	URL           string       `json:"url"`
+}
+
+// Comment is an array of strings
+type Comment []string
+
+// Sanitization defines a substitute for certain substrings
+type Sanitization struct {
+	Key struct {
+		Replacements []string `json:"replacements"`
+		Substitute   string   `json:"substitute"`
+	} `json:"key"`
+}
+
+// RequiredFields returns all fields that are defined as required
+func (s *Spec) RequiredFields() []string {
+	var requiredKeys []string
+	for name, field := range s.Fields {
+		if field.Required {
+			requiredKeys = append(requiredKeys, name)
+		}
+	}
+	return requiredKeys
+}
+
+// UnmarshalJSON unmarshals the given byte into a Comment.
+// Valid values are strings and arrays of strings
+func (c *Comment) UnmarshalJSON(b []byte) error {
+	var comments []string
+	if err := json.Unmarshal(b, &comments); err != nil {
+		var comment string
+		if err := json.Unmarshal(b, &comment); err != nil {
+			return errors.Wrap(err, "unmarshaling comment(s)")
+		}
+		comments = []string{comment}
+	}
+	*c = comments
+	return nil
+}
+
+func init() {
+	_, filename, _, ok := runtime.Caller(0)
+	if !ok {
+		panic("cannot recover information from runtime.Caller")
+	}
+	f := path.Join(filepath.ToSlash(filepath.Dir(filename)), "v1.json")
+	b, err := ioutil.ReadFile(f)
+	if err != nil {
+		panic(errors.Wrap(err, "reading spec version 1 failed"))
+	}
+	if err := json.Unmarshal(b, &V1); err != nil {
+		panic(errors.Wrap(err, "initializing spec version 1 failed"))
+	}
+}