Force ECS and JSON logging for libbeat/logp (#28573)

Force the logp package to act as if ECS and JSON has been enabled. Attach the
service.name field to entries only when Config.Beat is not empty.
Remove the logp.Config.ECSEnabled field. Log entries will now be
JSON formatted by default.

(cherry picked from commit 987250d)

CHANGELOG.next.asciidoc

@@ -20,6 +20,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
+- Libbeat: logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 - Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https://github.com/golang/go/issues/46123 {pull}27543[27543]
 - Update docker client. {pull}28716[28716]
 - Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671]

auditbeat/auditbeat.reference.yml

@@ -1448,14 +1448,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/tests/system/test_file_integrity.py

@@ -98,7 +98,7 @@ def test_non_recursive(self):
 
             # wait until file1 is reported before deleting. Otherwise the hash
             # might not be calculated
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file1)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file1)), ignore_case=True)
 
             os.unlink(file1)
 
@@ -107,8 +107,9 @@ def test_non_recursive(self):
             file3 = os.path.join(subdir, "other_file.txt")
             self.create_file(file3, "not reported.")
 
-            self.wait_log_contains("\"deleted\"")
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(subdir)), ignore_case=True)
+            # log entries are JSON formatted, this value shows up as an escaped json string.
+            self.wait_log_contains("\\\"deleted\\\"")
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(subdir)), ignore_case=True)
             self.wait_output(3)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir.lower() for obj in self.read_output()))
@@ -157,7 +158,7 @@ def test_recursive(self):
             # wait until the directories to watch are printed in the logs
             # this happens when the file_integrity module starts
             self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True)
-            self.wait_log_contains("\"recursive\": true")
+            self.wait_log_contains("\"recursive\":true")
 
             # auditbeat_test/subdir/
             subdir = os.path.join(dirs[0], "subdir")
@@ -173,7 +174,7 @@ def test_recursive(self):
             file2 = os.path.join(subdir2, "more.txt")
             self.create_file(file2, "")
 
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file2)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file2)), ignore_case=True)
             self.wait_output(4)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output()))

filebeat/filebeat.reference.yml

@@ -2360,14 +2360,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

heartbeat/heartbeat.reference.yml

@@ -1594,14 +1594,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

journalbeat/journalbeat.reference.yml

@@ -1391,14 +1391,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Journalbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -72,11 +72,3 @@ logging.files:
   # renamed during rotation. Or when set to date, the date is added to
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
-
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false

libbeat/docs/loggingconfig.asciidoc

@@ -264,17 +264,6 @@ If the log file already exists on startup, immediately rotate it and start
 writing to a new file instead of appending to the existing one. Defaults to
 true.
 
-[float]
-==== `logging.json`
-
-When true, logs messages in JSON format. The default is false.
-
-[float]
-==== `logging.ecs`
-
-When true, logs messages with minimal required Elastic Common Schema (ECS)
-information.
-
 ifndef::serverless[]
 [float]
 ==== `logging.files.redirect_stderr` experimental[]

libbeat/logp/config.go

@@ -26,11 +26,9 @@ import (
 // Config contains the configuration options for the logger. To create a Config
 // from a common.Config use logp/config.Build.
 type Config struct {
-	Beat       string   `config:",ignore"`        // Name of the Beat (for default file name).
-	JSON       bool     `config:"json"`           // Write logs as JSON.
-	Level      Level    `config:"level"`          // Logging level (error, warning, info, debug).
-	Selectors  []string `config:"selectors"`      // Selectors for debug level logging.
-	ECSEnabled bool     `config:"ecs" yaml:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line
+	Beat      string   `config:",ignore"`   // Name of the Beat (for default file name).
+	Level     Level    `config:"level"`     // Logging level (error, warning, info, debug).
+	Selectors []string `config:"selectors"` // Selectors for debug level logging.
 
 	toObserver  bool
 	toIODiscard bool

libbeat/logp/core.go

@@ -196,7 +196,7 @@ func makeOptions(cfg Config) []zap.Option {
 	if cfg.development {
 		options = append(options, zap.Development())
 	}
-	if cfg.ECSEnabled {
+	if cfg.Beat != "" {
 		fields := []zap.Field{
 			zap.String("service.name", cfg.Beat),
 		}
@@ -254,10 +254,7 @@ func newCore(cfg Config, enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapco
 	return wrappedCore(cfg, zapcore.NewCore(enc, ws, enab))
 }
 func wrappedCore(cfg Config, core zapcore.Core) zapcore.Core {
-	if cfg.ECSEnabled {
-		return ecszap.WrapCore(core)
-	}
-	return core
+	return ecszap.WrapCore(core)
 }
 
 func globalLogger() *zap.Logger {

libbeat/logp/core_test.go

@@ -48,7 +48,7 @@ func TestLogger(t *testing.T) {
 
 	TestingSetup()
 	exerciseLogger()
-	TestingSetup(AsJSON())
+	TestingSetup()
 	exerciseLogger()
 }
 
@@ -152,7 +152,6 @@ func TestLoggingECSFields(t *testing.T) {
 		Beat:        "beat1",
 		Level:       DebugLevel,
 		development: true,
-		ECSEnabled:  true,
 		Files: FileConfig{
 			Name: "beat1.log",
 		},

libbeat/logp/encoding.go

@@ -43,20 +43,15 @@ type encoderCreator func(cfg zapcore.EncoderConfig) zapcore.Encoder
 func buildEncoder(cfg Config) zapcore.Encoder {
 	var encCfg zapcore.EncoderConfig
 	var encCreator encoderCreator
-	if cfg.JSON {
-		encCfg = JSONEncoderConfig()
-		encCreator = zapcore.NewJSONEncoder
-	} else if cfg.ToSyslog {
+	if cfg.ToSyslog {
 		encCfg = SyslogEncoderConfig()
 		encCreator = zapcore.NewConsoleEncoder
 	} else {
-		encCfg = ConsoleEncoderConfig()
-		encCreator = zapcore.NewConsoleEncoder
+		encCfg = JSONEncoderConfig()
+		encCreator = zapcore.NewJSONEncoder
 	}
 
-	if cfg.ECSEnabled {
-		encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
-	}
+	encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
 	return encCreator(encCfg)
 }
 

libbeat/logp/options.go

@@ -52,10 +52,3 @@ func ToDiscardOutput() Option {
 		cfg.ToStderr = false
 	}
 }
-
-// AsJSON specifies to log the output as JSON.
-func AsJSON() Option {
-	return func(cfg *Config) {
-		cfg.JSON = true
-	}
-}

libbeat/tests/system/test_logging.py

@@ -25,43 +25,25 @@ def assert_contains_ecs_log(self, logfile=None):
         assert self.log_contains(ecs_message_log, logfile=logfile)
         assert self.log_contains(ecs_log_level_log, logfile=logfile)
 
-    def assert_not_contains_ecs_log(self, logfile=None):
-        assert not self.log_contains(ecs_version_log, logfile=logfile)
-        assert not self.log_contains(ecs_timestamp_log, logfile=logfile)
-        assert not self.log_contains(ecs_log_level_log, logfile=logfile)
-
-    def test_console_default(self):
-        """
-        logs to console with default format
-        """
-        self.run_beat_with_args("mockbeat start running", logging_args=["-e"])
-        self.assert_not_contains_ecs_log()
-
     def test_console_ecs(self):
         """
         logs to console with ECS format
         """
         self.run_beat_with_args("mockbeat start running",
-                                logging_args=["-e"],
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=true"])
+                                logging_args=["-e"])
         self.assert_contains_ecs_log()
 
     def test_file_default(self):
         """
         logs to file with default format
         """
         self.run_beat_with_args("Mockbeat is alive!",
-                                logging_args=[],
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=false"])
-        self.assert_not_contains_ecs_log(logfile="logs/mockbeat")
+                                logging_args=[])
+        self.assert_contains_ecs_log(logfile="logs/mockbeat")
 
     def test_file_ecs(self):
         """
         logs to file with ECS format
         """
-        self.run_beat_with_args("Mockbeat is alive!",
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=true"])
+        self.run_beat_with_args("Mockbeat is alive!")
         self.assert_contains_ecs_log(logfile="logs/mockbeat")

libbeat/tests/system/test_template.py

@@ -107,7 +107,7 @@ def test_json_template(self):
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
         self.wait_until(lambda: self.log_contains("Loading json template from file"))
-        self.wait_until(lambda: self.log_contains('Template with name "bla" loaded.'))
+        self.wait_until(lambda: self.log_contains('Template with name \\\"bla\\\" loaded.'))
         proc.check_kill_and_wait()
 
         result = es.transport.perform_request('GET', '/_template/' + template_name)
@@ -149,7 +149,7 @@ def test_template_default(self):
         self.render_config()
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
-        self.wait_until(lambda: self.log_contains('Template with name "mockbeat-9.9.9" loaded.'))
+        self.wait_until(lambda: self.log_contains('Template with name \\\"mockbeat-9.9.9\\\" loaded.'))
         self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
         proc.check_kill_and_wait()
 

metricbeat/metricbeat.reference.yml

@@ -2263,14 +2263,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Metricbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

metricbeat/module/elasticsearch/test_elasticsearch.py

@@ -135,7 +135,7 @@ def test_xpack_cluster_stats(self):
             }
         }])
         proc = self.start_beat()
-        self.wait_log_contains('"dataset": "elasticsearch.cluster.stats"')
+        self.wait_log_contains('\\"dataset\\": \\"elasticsearch.cluster.stats\\"')
 
         proc.check_kill_and_wait()
         self.assert_no_logged_warnings()

packetbeat/packetbeat.reference.yml

@@ -1943,14 +1943,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Packetbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

winlogbeat/winlogbeat.reference.yml

@@ -1371,14 +1371,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Winlogbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/auditbeat/auditbeat.reference.yml

@@ -1504,14 +1504,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl

@@ -197,9 +197,4 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 {{template "providers.yml.tmpl" .}}

x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl

@@ -197,9 +197,4 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 {{template "providers.yml.tmpl" .}}

x-pack/elastic-agent/_meta/elastic-agent.yml

@@ -191,8 +191,3 @@ agent.logging.to_stderr: true
 
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false