From 3d110d1b0a1675414627cfcd723c21f1b7d9ad1e Mon Sep 17 00:00:00 2001 From: Antoine de Troostembergh Date: Sun, 3 Nov 2024 20:54:47 +0100 Subject: [PATCH] add socket log appender and basic ECS json formatting --- .../builditem/LogSocketFormatBuildItem.java | 36 +++++++++ .../logging/LoggingResourceProcessor.java | 5 ++ .../runtime/logging/LogRuntimeConfig.java | 71 +++++++++++++++++ .../runtime/logging/LoggingSetupRecorder.java | 76 ++++++++++++++++++- .../asciidoc/centralized-log-management.adoc | 72 ++++++++++++++++++ docs/src/main/asciidoc/logging.adoc | 18 +++++ .../json/deployment/LoggingJsonProcessor.java | 7 ++ .../SocketJsonFormatterCustomConfigTest.java | 72 ++++++++++++++++++ .../SocketJsonFormatterDefaultConfigTest.java | 62 +++++++++++++++ ...on-socket-json-formatter-custom.properties | 16 ++++ ...n-socket-json-formatter-default.properties | 5 ++ .../logging/json/runtime/JsonLogConfig.java | 18 +++++ .../json/runtime/LoggingJsonRecorder.java | 60 +++++++++++++++ .../application-socket-output.properties | 8 ++ .../io/quarkus/logging/SocketHandlerTest.java | 42 ++++++++++ 15 files changed, 565 insertions(+), 3 deletions(-) create mode 100644 core/deployment/src/main/java/io/quarkus/deployment/builditem/LogSocketFormatBuildItem.java create mode 100644 extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterCustomConfigTest.java create mode 100644 extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterDefaultConfigTest.java create mode 100644 extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-custom.properties create mode 100644 extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-default.properties create mode 100644 integration-tests/test-extension/extension/deployment/src/main/resources/application-socket-output.properties create mode 100644 integration-tests/test-extension/extension/deployment/src/test/java/io/quarkus/logging/SocketHandlerTest.java diff --git a/core/deployment/src/main/java/io/quarkus/deployment/builditem/LogSocketFormatBuildItem.java b/core/deployment/src/main/java/io/quarkus/deployment/builditem/LogSocketFormatBuildItem.java new file mode 100644 index 0000000000000..23aeaf109b955 --- /dev/null +++ b/core/deployment/src/main/java/io/quarkus/deployment/builditem/LogSocketFormatBuildItem.java @@ -0,0 +1,36 @@ +package io.quarkus.deployment.builditem; + +import java.util.Optional; +import java.util.logging.Formatter; + +import org.wildfly.common.Assert; + +import io.quarkus.builder.item.MultiBuildItem; +import io.quarkus.runtime.RuntimeValue; + +/** + * The socket format build item. Producing this item will cause the logging subsystem to disregard its + * socket logging formatting configuration and use the formatter provided instead. If multiple formatters + * are enabled at runtime, a warning message is printed and only one is used. + */ +public final class LogSocketFormatBuildItem extends MultiBuildItem { + private final RuntimeValue> formatterValue; + + /** + * Construct a new instance. + * + * @param formatterValue the optional formatter runtime value to use (must not be {@code null}) + */ + public LogSocketFormatBuildItem(final RuntimeValue> formatterValue) { + this.formatterValue = Assert.checkNotNullParam("formatterValue", formatterValue); + } + + /** + * Get the formatter value. + * + * @return the formatter value + */ + public RuntimeValue> getFormatterValue() { + return formatterValue; + } +} diff --git a/core/deployment/src/main/java/io/quarkus/deployment/logging/LoggingResourceProcessor.java b/core/deployment/src/main/java/io/quarkus/deployment/logging/LoggingResourceProcessor.java index a94f8fdc3894f..8507399dea96c 100644 --- a/core/deployment/src/main/java/io/quarkus/deployment/logging/LoggingResourceProcessor.java +++ b/core/deployment/src/main/java/io/quarkus/deployment/logging/LoggingResourceProcessor.java @@ -75,6 +75,7 @@ import io.quarkus.deployment.builditem.LogConsoleFormatBuildItem; import io.quarkus.deployment.builditem.LogFileFormatBuildItem; import io.quarkus.deployment.builditem.LogHandlerBuildItem; +import io.quarkus.deployment.builditem.LogSocketFormatBuildItem; import io.quarkus.deployment.builditem.LogSyslogFormatBuildItem; import io.quarkus.deployment.builditem.NamedLogHandlersBuildItem; import io.quarkus.deployment.builditem.RunTimeConfigurationDefaultBuildItem; @@ -249,6 +250,7 @@ LoggingSetupBuildItem setupLoggingRuntimeInit( final List consoleFormatItems, final List fileFormatItems, final List syslogFormatItems, + final List socketFormatItems, final Optional possibleBannerBuildItem, final List logStreamBuildItems, final BuildProducer shutdownListenerBuildItemBuildProducer, @@ -290,6 +292,8 @@ LoggingSetupBuildItem setupLoggingRuntimeInit( .map(LogFileFormatBuildItem::getFormatterValue).collect(Collectors.toList()); List>> possibleSyslogFormatters = syslogFormatItems.stream() .map(LogSyslogFormatBuildItem::getFormatterValue).collect(Collectors.toList()); + List>> possibleSocketFormatters = socketFormatItems.stream() + .map(LogSocketFormatBuildItem::getFormatterValue).collect(Collectors.toList()); context.registerSubstitution(InheritableLevel.ActualLevel.class, String.class, InheritableLevel.Substitution.class); context.registerSubstitution(InheritableLevel.Inherited.class, String.class, InheritableLevel.Substitution.class); @@ -308,6 +312,7 @@ LoggingSetupBuildItem setupLoggingRuntimeInit( categoryMinLevelDefaults.content, alwaysEnableLogStream, streamingDevUiLogHandler, handlers, namedHandlers, possibleConsoleFormatters, possibleFileFormatters, possibleSyslogFormatters, + possibleSocketFormatters, possibleSupplier, launchModeBuildItem.getLaunchMode(), true))); List additionalLogCleanupFilters = new ArrayList<>(logCleanupFilters.size()); diff --git a/core/runtime/src/main/java/io/quarkus/runtime/logging/LogRuntimeConfig.java b/core/runtime/src/main/java/io/quarkus/runtime/logging/LogRuntimeConfig.java index ed01e44255ffa..09cb613d2ff21 100644 --- a/core/runtime/src/main/java/io/quarkus/runtime/logging/LogRuntimeConfig.java +++ b/core/runtime/src/main/java/io/quarkus/runtime/logging/LogRuntimeConfig.java @@ -10,6 +10,7 @@ import java.util.logging.Level; import org.jboss.logmanager.handlers.AsyncHandler.OverflowAction; +import org.jboss.logmanager.handlers.SocketHandler; import org.jboss.logmanager.handlers.SyslogHandler.Facility; import org.jboss.logmanager.handlers.SyslogHandler.Protocol; import org.jboss.logmanager.handlers.SyslogHandler.SyslogType; @@ -77,6 +78,14 @@ public interface LogRuntimeConfig { @ConfigDocSection SyslogConfig syslog(); + /** + * Socket logging. + *

+ * Logging to a socket is also supported but not enabled by default. + */ + @ConfigDocSection + SocketConfig socket(); + /** * Logging categories. *

@@ -115,6 +124,15 @@ public interface LogRuntimeConfig { @ConfigDocSection Map syslogHandlers(); + /** + * Socket handlers. + *

+ * The named socket handlers configured here can be linked to one or more categories. + */ + @WithName("handler.socket") + @ConfigDocSection + Map socketHandlers(); + /** * Log cleanup filters - internal use. */ @@ -393,6 +411,59 @@ interface SyslogConfig { AsyncConfig async(); } + interface SocketConfig { + + /** + * If socket logging should be enabled + */ + @WithDefault("false") + boolean enable(); + + /** + * + * The IP address and port of the server receiving the logs + */ + @WithDefault("localhost:4560") + @WithConverter(InetSocketAddressConverter.class) + InetSocketAddress endpoint(); + + /** + * Sets the protocol used to connect to the syslog server + */ + @WithDefault("tcp") + SocketHandler.Protocol protocol(); + + /** + * Enables or disables blocking when attempting to reconnect a + * {@link Protocol#TCP + * TCP} or {@link Protocol#SSL_TCP SSL TCP} protocol + */ + @WithDefault("false") + boolean blockOnReconnect(); + + /** + * The log message format + */ + @WithDefault("%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n") + String format(); + + /** + * The log level specifying, which message levels will be logged by socket logger + */ + @WithDefault("ALL") + Level level(); + + /** + * The name of the filter to link to the file handler. + */ + Optional filter(); + + /** + * Socket async logging config + */ + AsyncConfig async(); + } + interface CleanupFilterConfig { /** * The message prefix to match diff --git a/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java b/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java index 6cea3eb817823..2d390ab877db0 100644 --- a/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java +++ b/core/runtime/src/main/java/io/quarkus/runtime/logging/LoggingSetupRecorder.java @@ -46,6 +46,7 @@ import org.jboss.logmanager.handlers.FileHandler; import org.jboss.logmanager.handlers.PeriodicSizeRotatingFileHandler; import org.jboss.logmanager.handlers.SizeRotatingFileHandler; +import org.jboss.logmanager.handlers.SocketHandler; import org.jboss.logmanager.handlers.SyslogHandler; import io.quarkus.bootstrap.logging.InitialConfigurator; @@ -63,6 +64,7 @@ import io.quarkus.runtime.logging.LogRuntimeConfig.CleanupFilterConfig; import io.quarkus.runtime.logging.LogRuntimeConfig.ConsoleConfig; import io.quarkus.runtime.logging.LogRuntimeConfig.FileConfig; +import io.quarkus.runtime.logging.LogRuntimeConfig.SocketConfig; import io.quarkus.runtime.shutdown.ShutdownListener; import io.smallrye.config.SmallRyeConfig; import io.smallrye.config.SmallRyeConfigBuilder; @@ -116,7 +118,7 @@ public String getName() { new LoggingSetupRecorder(new RuntimeValue<>(consoleRuntimeConfig)).initializeLogging(logRuntimeConfig, logBuildTimeConfig, DiscoveredLogComponents.ofEmpty(), emptyMap(), false, null, emptyList(), emptyList(), emptyList(), emptyList(), - emptyList(), banner, LaunchMode.DEVELOPMENT, false); + emptyList(), emptyList(), banner, LaunchMode.DEVELOPMENT, false); } public ShutdownListener initializeLogging( @@ -131,6 +133,7 @@ public ShutdownListener initializeLogging( final List>> possibleConsoleFormatters, final List>> possibleFileFormatters, final List>> possibleSyslogFormatters, + final List>> possibleSocketFormatters, final RuntimeValue>> possibleBannerSupplier, final LaunchMode launchMode, final boolean includeFilters) { @@ -211,6 +214,14 @@ public void close() throws SecurityException { } } + if (config.socket().enable()) { + final Handler socketHandler = configureSocketHandler(config.socket(), errorManager, cleanupFiler, + namedFilters, possibleSocketFormatters, includeFilters); + if (socketHandler != null) { + handlers.add(socketHandler); + } + } + if ((launchMode.isDevOrTest() || enableWebStream) && streamingDevUiConsoleHandler != null && streamingDevUiConsoleHandler.getValue().isPresent()) { @@ -229,7 +240,7 @@ public void close() throws SecurityException { Map namedHandlers = shouldCreateNamedHandlers(config, additionalNamedHandlers) ? createNamedHandlers(config, consoleRuntimeConfig.getValue(), additionalNamedHandlers, - possibleConsoleFormatters, possibleFileFormatters, possibleSyslogFormatters, + possibleConsoleFormatters, possibleFileFormatters, possibleSyslogFormatters, possibleSocketFormatters, errorManager, cleanupFiler, namedFilters, launchMode, shutdownNotifier, includeFilters) : emptyMap(); @@ -328,7 +339,7 @@ public static void initializeBuildTimeLogging( } Map namedHandlers = createNamedHandlers(config, consoleConfig, emptyList(), - emptyList(), emptyList(), emptyList(), errorManager, logCleanupFilter, + emptyList(), emptyList(), emptyList(), emptyList(), errorManager, logCleanupFilter, emptyMap(), launchMode, dummy, false); setUpCategoryLoggers(buildConfig, categoryDefaultMinLevels, categories, logContext, errorManager, namedHandlers); @@ -388,6 +399,7 @@ private static Map createNamedHandlers( List>> possibleConsoleFormatters, List>> possibleFileFormatters, List>> possibleSyslogFormatters, + List>> possibleSocketFormatters, ErrorManager errorManager, LogCleanupFilter cleanupFilter, Map namedFilters, LaunchMode launchMode, ShutdownNotifier shutdownHandler, boolean includeFilters) { @@ -422,6 +434,17 @@ private static Map createNamedHandlers( addToNamedHandlers(namedHandlers, syslogHandler, sysLogConfigEntry.getKey()); } } + for (Entry socketConfigEntry : config.socketHandlers().entrySet()) { + SocketConfig namedSocketConfig = socketConfigEntry.getValue(); + if (!namedSocketConfig.enable()) { + continue; + } + final Handler socketHandler = configureSocketHandler(namedSocketConfig, errorManager, cleanupFilter, + namedFilters, possibleSocketFormatters, includeFilters); + if (socketHandler != null) { + addToNamedHandlers(namedHandlers, socketHandler, socketConfigEntry.getKey()); + } + } Map additionalNamedHandlersMap; if (additionalNamedHandlers.isEmpty()) { @@ -770,6 +793,53 @@ private static Handler configureSyslogHandler(final LogRuntimeConfig.SyslogConfi } } + private static Handler configureSocketHandler(final LogRuntimeConfig.SocketConfig config, + final ErrorManager errorManager, + final LogCleanupFilter logCleanupFilter, + final Map namedFilters, + final List>> possibleSocketFormatters, + final boolean includeFilters) { + try { + final SocketHandler handler = new SocketHandler(config.endpoint().getHostString(), config.endpoint().getPort()); + handler.setProtocol(config.protocol()); + handler.setBlockOnReconnect(config.blockOnReconnect()); + handler.setLevel(config.level()); + + Formatter formatter = null; + boolean formatterWarning = false; + for (RuntimeValue> value : possibleSocketFormatters) { + if (formatter != null) { + formatterWarning = true; + } + final Optional val = value.getValue(); + if (val.isPresent()) { + formatter = val.get(); + } + } + if (formatter == null) { + formatter = new PatternFormatter(config.format()); + } + handler.setFormatter(formatter); + + handler.setErrorManager(errorManager); + handler.setFilter(logCleanupFilter); + applyFilter(includeFilters, errorManager, logCleanupFilter, config.filter(), namedFilters, handler); + + if (formatterWarning) { + handler.getErrorManager().error("Multiple socket formatters were activated", null, + ErrorManager.GENERIC_FAILURE); + } + + if (config.async().enable()) { + return createAsyncHandler(config.async(), config.level(), handler); + } + return handler; + } catch (IOException e) { + errorManager.error("Failed to create socket handler", e, ErrorManager.OPEN_FAILURE); + return null; + } + } + private static AsyncHandler createAsyncHandler(LogRuntimeConfig.AsyncConfig asyncConfig, Level level, Handler handler) { final AsyncHandler asyncHandler = new AsyncHandler(asyncConfig.queueLength()); asyncHandler.setOverflowAction(asyncConfig.overflow()); diff --git a/docs/src/main/asciidoc/centralized-log-management.adoc b/docs/src/main/asciidoc/centralized-log-management.adoc index 663df993637c1..4fbb13b06388c 100644 --- a/docs/src/main/asciidoc/centralized-log-management.adoc +++ b/docs/src/main/asciidoc/centralized-log-management.adoc @@ -236,6 +236,77 @@ networks: Launch your application, you should see your logs arriving inside the Elastic Stack; you can use Kibana available at http://localhost:5601/ to access them. + +[[logstash_ecs]] +== GELF alternative: Send logs to Logstash in the ECS (Elastic Common Schema) format + +You can also send your logs to Logstash using a TCP input in the https://www.elastic.co/guide/en/ecs-logging/overview/current/intro.html[ECS] format. +To achieve this we will use the `quarkus-logging-json` extension to format the logs in JSON format and the socket handler to send them to Logstash. + +For this you can use the same `docker-compose.yml` file as above but with a different Logstash pipeline configuration. + +[source] +---- +input { + tcp { + port => 4560 + coded => json + } +} + +filter { + if ![span][id] and [mdc][spanId] { + mutate { rename => { "[mdc][spanId]" => "[span][id]" } } + } + if ![trace][id] and [mdc][traceId] { + mutate { rename => {"[mdc][traceId]" => "[trace][id]"} } + } +} + +output { + stdout {} + elasticsearch { + hosts => ["http://elasticsearch:9200"] + } +} +---- + +Then configure your application to log in JSON format instead of GELF + +[source,xml,role="primary asciidoc-tabs-target-sync-cli asciidoc-tabs-target-sync-maven"] +.pom.xml +---- + + io.quarkus + quarkus-logging-json + +---- + +[source,gradle,role="secondary asciidoc-tabs-target-sync-gradle"] +.build.gradle +---- +implementation("io.quarkus:quarkus-logging-json") +---- + +and specify the host and port of your Logstash endpoint. To be ECS compliant, specify the log format. + +[source, properties] +---- +# to keep the logs in the usual format in the console +quarkus.log.console.json=false + +quarkus.log.socket.enable=true +quarkus.log.socket.json=true +quarkus.log.socket.endpoint=localhost:4560 + +# to have the exception serialized into a single text element +quarkus.log.socket.json.exception-output-type=formatted + +# specify the format of the produced JSON log +quarkus.log.socket.json.log-format=ECS +---- + + == Send logs to Fluentd (EFK) First, you need to create a Fluentd image with the needed plugins: elasticsearch and input-gelf. @@ -422,6 +493,7 @@ quarkus.log.syslog.hostname=quarkus-test Launch your application, you should see your logs arriving inside EFK: you can use Kibana available at http://localhost:5601/ to access them. + == Elasticsearch indexing consideration Be careful that, by default, Elasticsearch will automatically map unknown fields (if not disabled in the index settings) by detecting their type. diff --git a/docs/src/main/asciidoc/logging.adoc b/docs/src/main/asciidoc/logging.adoc index aa47489dade9a..8d1491466bbde 100644 --- a/docs/src/main/asciidoc/logging.adoc +++ b/docs/src/main/asciidoc/logging.adoc @@ -511,6 +511,24 @@ quarkus.log.category."com.example".use-parent-handlers=false For details about its configuration, see the xref:#quarkus-core_section_quarkus-log-syslog[Syslog logging configuration] reference. +=== Socket log handler + +This handler will send the logs to a socket. +It is disabled by default, so you must first enable it. +When enabled, it sends all log events to a socket, for instance to a Logstash server. + +This will typically be used in conjunction with the `quarkus-logging-json` extension so send logs in ECS format to an Elasticsearch instance. +An example configuration can be found in the xref:centralized-log-management.adoc[Centralized log management] guide. + +* A global configuration example: ++ +[source, properties] +---- +quarkus.log.socket.enable=true +quarkus.log.socket.endpoint=localhost:4560 +---- + + == Add a logging filter to your log handler Log handlers, such as the console log handler, can be linked with a link:https://docs.oracle.com/en/java/javase/17/docs/api/java.logging/java/util/logging/Filter.html[filter] that determines whether a log record should be logged. diff --git a/extensions/logging-json/deployment/src/main/java/io/quarkus/logging/json/deployment/LoggingJsonProcessor.java b/extensions/logging-json/deployment/src/main/java/io/quarkus/logging/json/deployment/LoggingJsonProcessor.java index 47ef0ba81a332..44db47e3c9b7f 100644 --- a/extensions/logging-json/deployment/src/main/java/io/quarkus/logging/json/deployment/LoggingJsonProcessor.java +++ b/extensions/logging-json/deployment/src/main/java/io/quarkus/logging/json/deployment/LoggingJsonProcessor.java @@ -5,6 +5,7 @@ import io.quarkus.deployment.annotations.Record; import io.quarkus.deployment.builditem.LogConsoleFormatBuildItem; import io.quarkus.deployment.builditem.LogFileFormatBuildItem; +import io.quarkus.deployment.builditem.LogSocketFormatBuildItem; import io.quarkus.deployment.builditem.LogSyslogFormatBuildItem; import io.quarkus.logging.json.runtime.JsonLogConfig; import io.quarkus.logging.json.runtime.LoggingJsonRecorder; @@ -28,4 +29,10 @@ public LogFileFormatBuildItem setUpFileFormatter(LoggingJsonRecorder recorder, J public LogSyslogFormatBuildItem setUpSyslogFormatter(LoggingJsonRecorder recorder, JsonLogConfig config) { return new LogSyslogFormatBuildItem(recorder.initializeSyslogJsonLogging(config)); } + + @BuildStep + @Record(ExecutionTime.RUNTIME_INIT) + public LogSocketFormatBuildItem setUpSocketFormatter(LoggingJsonRecorder recorder, JsonLogConfig config) { + return new LogSocketFormatBuildItem(recorder.initializeSocketJsonLogging(config)); + } } diff --git a/extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterCustomConfigTest.java b/extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterCustomConfigTest.java new file mode 100644 index 0000000000000..e6e0d8a9ac0a2 --- /dev/null +++ b/extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterCustomConfigTest.java @@ -0,0 +1,72 @@ +package io.quarkus.logging.json; + +import static io.quarkus.logging.json.SocketJsonFormatterDefaultConfigTest.getJsonFormatter; +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.ZoneId; +import java.util.logging.Level; +import java.util.logging.LogRecord; + +import org.assertj.core.api.Assertions; +import org.jboss.logmanager.formatters.StructuredFormatter; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + +import io.quarkus.logging.json.runtime.AdditionalFieldConfig; +import io.quarkus.logging.json.runtime.JsonFormatter; +import io.quarkus.test.QuarkusUnitTest; + +public class SocketJsonFormatterCustomConfigTest { + + @RegisterExtension + static final QuarkusUnitTest config = new QuarkusUnitTest() + .withApplicationRoot((jar) -> jar.addClasses(SocketJsonFormatterDefaultConfigTest.class)) + .withConfigurationResource("application-socket-json-formatter-custom.properties"); + + @Test + public void jsonFormatterCustomConfigurationTest() { + JsonFormatter jsonFormatter = getJsonFormatter(); + assertThat(jsonFormatter.isPrettyPrint()).isTrue(); + assertThat(jsonFormatter.getDateTimeFormatter().toString()) + .isEqualTo("Value(DayOfMonth)' 'Text(MonthOfYear,SHORT)' 'Value(Year,4,19,EXCEEDS_PAD)"); + assertThat(jsonFormatter.getDateTimeFormatter().getZone()).isEqualTo(ZoneId.of("UTC+05:00")); + assertThat(jsonFormatter.getExceptionOutputType()) + .isEqualTo(StructuredFormatter.ExceptionOutputType.DETAILED_AND_FORMATTED); + assertThat(jsonFormatter.getRecordDelimiter()).isEqualTo("\n;"); + assertThat(jsonFormatter.isPrintDetails()).isTrue(); + assertThat(jsonFormatter.getExcludedKeys()).containsExactly("timestamp", "sequence"); + assertThat(jsonFormatter.getAdditionalFields().size()).isEqualTo(2); + assertThat(jsonFormatter.getAdditionalFields().containsKey("foo")).isTrue(); + assertThat(jsonFormatter.getAdditionalFields().get("foo").type).isEqualTo(AdditionalFieldConfig.Type.INT); + assertThat(jsonFormatter.getAdditionalFields().get("foo").value).isEqualTo("42"); + assertThat(jsonFormatter.getAdditionalFields().containsKey("bar")).isTrue(); + assertThat(jsonFormatter.getAdditionalFields().get("bar").type).isEqualTo(AdditionalFieldConfig.Type.STRING); + assertThat(jsonFormatter.getAdditionalFields().get("bar").value).isEqualTo("baz"); + } + + @Test + public void jsonFormatterOutputTest() throws Exception { + JsonFormatter jsonFormatter = getJsonFormatter(); + String line = jsonFormatter.format(new LogRecord(Level.INFO, "Hello, World!")); + + JsonNode node = new ObjectMapper().readTree(line); + // "level" has been renamed to HEY + Assertions.assertThat(node.has("level")).isFalse(); + Assertions.assertThat(node.has("HEY")).isTrue(); + Assertions.assertThat(node.get("HEY").asText()).isEqualTo("INFO"); + + // excluded fields + Assertions.assertThat(node.has("timestamp")).isFalse(); + Assertions.assertThat(node.has("sequence")).isFalse(); + + // additional fields + Assertions.assertThat(node.has("foo")).isTrue(); + Assertions.assertThat(node.get("foo").asInt()).isEqualTo(42); + Assertions.assertThat(node.has("bar")).isTrue(); + Assertions.assertThat(node.get("bar").asText()).isEqualTo("baz"); + Assertions.assertThat(node.get("message").asText()).isEqualTo("Hello, World!"); + } +} diff --git a/extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterDefaultConfigTest.java b/extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterDefaultConfigTest.java new file mode 100644 index 0000000000000..99d25bfa99805 --- /dev/null +++ b/extensions/logging-json/deployment/src/test/java/io/quarkus/logging/json/SocketJsonFormatterDefaultConfigTest.java @@ -0,0 +1,62 @@ +package io.quarkus.logging.json; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.Arrays; +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.Level; +import java.util.logging.LogManager; +import java.util.logging.Logger; + +import org.jboss.logmanager.formatters.StructuredFormatter; +import org.jboss.logmanager.handlers.SocketHandler; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.quarkus.bootstrap.logging.InitialConfigurator; +import io.quarkus.bootstrap.logging.QuarkusDelayedHandler; +import io.quarkus.logging.json.runtime.JsonFormatter; +import io.quarkus.test.QuarkusUnitTest; + +public class SocketJsonFormatterDefaultConfigTest { + + @RegisterExtension + static final QuarkusUnitTest config = new QuarkusUnitTest() + .withConfigurationResource("application-socket-json-formatter-default.properties"); + + @Test + public void jsonFormatterDefaultConfigurationTest() { + JsonFormatter jsonFormatter = getJsonFormatter(); + assertThat(jsonFormatter.isPrettyPrint()).isFalse(); + assertThat(jsonFormatter.getDateTimeFormatter().toString()) + .isEqualTo(DateTimeFormatter.ISO_OFFSET_DATE_TIME.withZone(ZoneId.systemDefault()).toString()); + assertThat(jsonFormatter.getDateTimeFormatter().getZone()).isEqualTo(ZoneId.systemDefault()); + assertThat(jsonFormatter.getExceptionOutputType()).isEqualTo(StructuredFormatter.ExceptionOutputType.DETAILED); + assertThat(jsonFormatter.getRecordDelimiter()).isEqualTo("\n"); + assertThat(jsonFormatter.isPrintDetails()).isFalse(); + assertThat(jsonFormatter.getExcludedKeys()).isEmpty(); + assertThat(jsonFormatter.getAdditionalFields().entrySet()).isEmpty(); + } + + public static JsonFormatter getJsonFormatter() { + LogManager logManager = LogManager.getLogManager(); + assertThat(logManager).isInstanceOf(org.jboss.logmanager.LogManager.class); + + QuarkusDelayedHandler delayedHandler = InitialConfigurator.DELAYED_HANDLER; + assertThat(Logger.getLogger("").getHandlers()).contains(delayedHandler); + assertThat(delayedHandler.getLevel()).isEqualTo(Level.ALL); + + Handler handler = Arrays.stream(delayedHandler.getHandlers()) + .filter(h -> (h instanceof SocketHandler)) + .findFirst().orElse(null); + assertThat(handler).isNotNull(); + assertThat(handler.getLevel()).isEqualTo(Level.WARNING); + + Formatter formatter = handler.getFormatter(); + assertThat(formatter).isInstanceOf(JsonFormatter.class); + return (JsonFormatter) formatter; + } +} diff --git a/extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-custom.properties b/extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-custom.properties new file mode 100644 index 0000000000000..0441faac79159 --- /dev/null +++ b/extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-custom.properties @@ -0,0 +1,16 @@ +quarkus.log.level=INFO +quarkus.log.socket.enable=true +quarkus.log.socket.level=WARNING +quarkus.log.socket.json=true +quarkus.log.socket.json.pretty-print=true +quarkus.log.socket.json.date-format=d MMM uuuu +quarkus.log.socket.json.record-delimiter=\n; +quarkus.log.socket.json.zone-id=UTC+05:00 +quarkus.log.socket.json.exception-output-type=DETAILED_AND_FORMATTED +quarkus.log.socket.json.print-details=true +quarkus.log.socket.json.key-overrides=level=HEY +quarkus.log.socket.json.excluded-keys=timestamp,sequence +quarkus.log.socket.json.additional-field.foo.value=42 +quarkus.log.socket.json.additional-field.foo.type=int +quarkus.log.socket.json.additional-field.bar.value=baz +quarkus.log.socket.json.additional-field.bar.type=string diff --git a/extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-default.properties b/extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-default.properties new file mode 100644 index 0000000000000..31933e9601d37 --- /dev/null +++ b/extensions/logging-json/deployment/src/test/resources/application-socket-json-formatter-default.properties @@ -0,0 +1,5 @@ +quarkus.log.level=INFO +quarkus.log.socket.enable=true +quarkus.log.socket.level=WARNING +quarkus.log.socket.format=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n +quarkus.log.socket.json=true diff --git a/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/JsonLogConfig.java b/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/JsonLogConfig.java index 35ec19204e441..3f1cd1fffd9d4 100644 --- a/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/JsonLogConfig.java +++ b/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/JsonLogConfig.java @@ -40,6 +40,13 @@ public class JsonLogConfig { @ConfigItem(name = "syslog.json") JsonConfig syslogJson; + /** + * Socket logging. + */ + @ConfigDocSection + @ConfigItem(name = "socket.json") + JsonConfig socketJson; + @ConfigGroup public static class JsonConfig { /** @@ -98,5 +105,16 @@ public static class JsonConfig { @ConfigItem @ConfigDocMapKey("field-name") Map additionalField; + + /** + * Specify the format of the produced JSON + */ + @ConfigItem(defaultValue = "DEFAULT") + LogFormat logFormat; + + public enum LogFormat { + DEFAULT, + ECS + } } } diff --git a/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/LoggingJsonRecorder.java b/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/LoggingJsonRecorder.java index 872f292569dad..d05745e2a76ca 100644 --- a/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/LoggingJsonRecorder.java +++ b/extensions/logging-json/runtime/src/main/java/io/quarkus/logging/json/runtime/LoggingJsonRecorder.java @@ -1,8 +1,17 @@ package io.quarkus.logging.json.runtime; +import java.util.EnumMap; +import java.util.HashSet; import java.util.Optional; +import java.util.Set; import java.util.logging.Formatter; +import org.eclipse.microprofile.config.Config; +import org.eclipse.microprofile.config.ConfigProvider; +import org.jboss.logmanager.PropertyValues; +import org.jboss.logmanager.formatters.StructuredFormatter.Key; + +import io.quarkus.logging.json.runtime.AdditionalFieldConfig.Type; import io.quarkus.logging.json.runtime.JsonLogConfig.JsonConfig; import io.quarkus.runtime.RuntimeValue; import io.quarkus.runtime.annotations.Recorder; @@ -22,7 +31,19 @@ public RuntimeValue> initializeSyslogJsonLogging(JsonLogConf return getFormatter(config.syslogJson); } + public RuntimeValue> initializeSocketJsonLogging(JsonLogConfig config) { + return getFormatter(config.socketJson); + } + private RuntimeValue> getFormatter(JsonConfig config) { + if (config.logFormat == JsonConfig.LogFormat.ECS) { + addEcsFieldOverrides(config); + } + + return getDefaultFormatter(config); + } + + private RuntimeValue> getDefaultFormatter(JsonConfig config) { if (!config.enable) { return new RuntimeValue<>(Optional.empty()); } @@ -43,4 +64,43 @@ private RuntimeValue> getFormatter(JsonConfig config) { } return new RuntimeValue<>(Optional.of(formatter)); } + + private void addEcsFieldOverrides(JsonConfig config) { + EnumMap keyOverrides = PropertyValues.stringToEnumMap(Key.class, config.keyOverrides.orElse(null)); + keyOverrides.putIfAbsent(Key.TIMESTAMP, "@timestamp"); + keyOverrides.putIfAbsent(Key.LOGGER_NAME, "log.logger"); + keyOverrides.putIfAbsent(Key.LEVEL, "log.level"); + keyOverrides.putIfAbsent(Key.PROCESS_ID, "process.pid"); + keyOverrides.putIfAbsent(Key.PROCESS_NAME, "process.name"); + keyOverrides.putIfAbsent(Key.THREAD_NAME, "process.thread.name"); + keyOverrides.putIfAbsent(Key.THREAD_ID, "process.thread.id"); + keyOverrides.putIfAbsent(Key.HOST_NAME, "host.hostname"); + keyOverrides.putIfAbsent(Key.SEQUENCE, "event.sequence"); + keyOverrides.putIfAbsent(Key.EXCEPTION_MESSAGE, "error.message"); + keyOverrides.putIfAbsent(Key.STACK_TRACE, "error.stack_trace"); + config.keyOverrides = Optional.of(PropertyValues.mapToString(keyOverrides)); + + config.additionalField.computeIfAbsent("ecs.version", k -> buildFieldConfig("1.12.2", Type.STRING)); + config.additionalField.computeIfAbsent("data_stream.type", k -> buildFieldConfig("logs", Type.STRING)); + + Config quarkusConfig = ConfigProvider.getConfig(); + quarkusConfig.getOptionalValue("quarkus.application.name", String.class).ifPresent( + s -> config.additionalField.computeIfAbsent("service.name", k -> buildFieldConfig(s, Type.STRING))); + quarkusConfig.getOptionalValue("quarkus.application.version", String.class).ifPresent( + s -> config.additionalField.computeIfAbsent("service.version", k -> buildFieldConfig(s, Type.STRING))); + quarkusConfig.getOptionalValue("quarkus.profile", String.class).ifPresent( + s -> config.additionalField.computeIfAbsent("service.environment", k -> buildFieldConfig(s, Type.STRING))); + + Set excludedKeys = config.excludedKeys.orElseGet(HashSet::new); + excludedKeys.add(Key.LOGGER_CLASS_NAME.getKey()); + excludedKeys.add(Key.RECORD.getKey()); + config.excludedKeys = Optional.of(excludedKeys); + } + + private AdditionalFieldConfig buildFieldConfig(String value, Type type) { + AdditionalFieldConfig field = new AdditionalFieldConfig(); + field.type = type; + field.value = value; + return field; + } } diff --git a/integration-tests/test-extension/extension/deployment/src/main/resources/application-socket-output.properties b/integration-tests/test-extension/extension/deployment/src/main/resources/application-socket-output.properties new file mode 100644 index 0000000000000..92b64a1e93a19 --- /dev/null +++ b/integration-tests/test-extension/extension/deployment/src/main/resources/application-socket-output.properties @@ -0,0 +1,8 @@ +quarkus.log.level=INFO +quarkus.log.socket.enable=true +quarkus.log.socket.endpoint=localhost:5140 +quarkus.log.socket.protocol=TCP +quarkus.log.socket.format=%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n +quarkus.log.socket.level=WARNING +# Resource path to DSAPublicKey base64 encoded bytes +quarkus.root.dsa-key-location=/DSAPublicKey.encoded \ No newline at end of file diff --git a/integration-tests/test-extension/extension/deployment/src/test/java/io/quarkus/logging/SocketHandlerTest.java b/integration-tests/test-extension/extension/deployment/src/test/java/io/quarkus/logging/SocketHandlerTest.java new file mode 100644 index 0000000000000..754acfc58a4c5 --- /dev/null +++ b/integration-tests/test-extension/extension/deployment/src/test/java/io/quarkus/logging/SocketHandlerTest.java @@ -0,0 +1,42 @@ +package io.quarkus.logging; + +import static io.quarkus.logging.LoggingTestsHelper.getHandler; +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.logging.Formatter; +import java.util.logging.Handler; +import java.util.logging.Level; + +import org.jboss.logmanager.formatters.PatternFormatter; +import org.jboss.logmanager.handlers.SocketHandler; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import io.quarkus.test.QuarkusUnitTest; + +class SocketHandlerTest { + + @RegisterExtension + static final QuarkusUnitTest config = new QuarkusUnitTest() + .withConfigurationResource("application-socket-output.properties") + .withApplicationRoot((jar) -> jar + .addClass(LoggingTestsHelper.class) + .addAsManifestResource("application.properties", "microprofile-config.properties")); + + @Test + void socketOutputTest() { + Handler handler = getHandler(SocketHandler.class); + assertThat(handler.getLevel()).isEqualTo(Level.WARNING); + + Formatter formatter = handler.getFormatter(); + assertThat(formatter).isInstanceOf(PatternFormatter.class); + PatternFormatter patternFormatter = (PatternFormatter) formatter; + assertThat(patternFormatter.getPattern()).isEqualTo("%d{yyyy-MM-dd HH:mm:ss,SSS} %-5p [%c{3.}] (%t) %s%e%n"); + + SocketHandler socketHandler = (SocketHandler) handler; + assertThat(socketHandler.getPort()).isEqualTo(5140); + assertThat(socketHandler.getAddress().getHostAddress()).isEqualTo("127.0.0.1"); + assertThat(socketHandler.getProtocol()).isEqualTo(SocketHandler.Protocol.TCP); + assertThat(socketHandler.isBlockOnReconnect()).isFalse(); + } +} \ No newline at end of file