feat(Issue-282): trim whitespace, add option to omit empty labels

docs/docus/docs/configuration.md

@@ -69,6 +69,7 @@ Format settings do not depend on the encoding you use.
 |format.label.pairSeparator|,|Character sequence to use as a separator between labels|
 |format.label.keyValueSeparator|=|Character to use as a separator between label's name and its value|
 |format.label.readMarkers|false|If true, Loki4j scans each log record for the attached LabelMarker to add its values to the record's labels|
+|format.label.omitEmptyFields|false|If true, labels that have empty values will be omitted.|
 |format.label.streamCache|BoundAtomicMapCache|An implementation of a stream cache to use. By default, caches up to 1000 unique label sets|
 |format.staticLabels|false|If you use only a constant label set (e.g., same keys and values) for all log records, you can set this flag to true and save some CPU and RAM|
 

loki-logback-appender/src/main/java/com/github/loki4j/logback/AbstractLoki4jEncoder.java

@@ -2,6 +2,7 @@
 
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map.Entry;
@@ -36,7 +37,7 @@ public abstract class AbstractLoki4jEncoder extends ContextAwareBase implements
     private static final String REGEX_STARTER = "regex:";
     private static final String[] EMPTY_KV_PAIRS = new String[0];
     private static final String DEFAULT_MSG_PATTERN = "l=%level c=%logger{20} t=%thread | %msg %ex";
-    
+
     public static final class LabelCfg {
         /**
          * Logback pattern to use for log record's label
@@ -61,6 +62,10 @@ public static final class LabelCfg {
          * add its values to record's labels.
          */
         boolean readMarkers = false;
+        /**
+         * If true, omits labels when the corresponding value is empty (null, empty string or whitespace-only.
+         */
+        boolean omitEmptyFields = false;
         /**
          * An implementation of a Stream cache to use.
          */
@@ -81,6 +86,9 @@ public void setPairSeparator(String pairSeparator) {
         public void setReadMarkers(boolean readMarkers) {
             this.readMarkers = readMarkers;
         }
+        public void setOmitEmptyFields(boolean omitEmptyFields) {
+            this.omitEmptyFields = omitEmptyFields;
+        }
         @DefaultClass(BoundAtomicMapCache.class)
         public void setStreamCache(Cache<String, LogRecordStream> streamCache) {
             this.streamCache = streamCache;
@@ -165,7 +173,7 @@ public boolean isStarted() {
     public LogRecordStream eventToStream(ILoggingEvent e) {
         if (staticLabels) {
             if (staticLabelStream == null) {
-                staticLabelStream = LogRecordStream.create(mergeKVPairs(labelKeys, labelValuesExtractor.extract(e)));
+                staticLabelStream = LogRecordStream.create(mergeKVPairs(labelKeys, labelValuesExtractor.extract(e), label));
             }
             return staticLabelStream;
         }
@@ -176,7 +184,7 @@ public LogRecordStream eventToStream(ILoggingEvent e) {
             : EMPTY_KV_PAIRS;
         var streamKey = ArrayUtils.join2(labelValues, markerLabels, " !%! ");//.intern();
         return label.streamCache.get(streamKey, () -> {
-            var layoutLabels = mergeKVPairs(labelKeys, labelValues);
+            var layoutLabels = mergeKVPairs(labelKeys, labelValues, label);
             var allLabels = ArrayUtils.concat(layoutLabels, markerLabels);
             return LogRecordStream.create(allLabels);
         });
@@ -193,7 +201,7 @@ public String[] eventToMetadata(ILoggingEvent e) {
         var patternKVs = EMPTY_KV_PAIRS;
         if (metadataValuesExtractor != null) {
             var metadataValues = metadataValuesExtractor.extract(e);
-            patternKVs = mergeKVPairs(metadataKeys, metadataValues);
+            patternKVs = mergeKVPairs(metadataKeys, metadataValues, label);
         }
         var allKVs = ArrayUtils.concat(patternKVs, markerKVs);
         return allKVs;
@@ -264,14 +272,15 @@ private static List<String> extractIndexesMod2(String[] array, int mod2) {
             .collect(Collectors.toList());
     }
 
-    private static String[] mergeKVPairs(List<String> keys, String[] values) {
-        var resultLen = values.length * 2;
-        var result = new String[resultLen];
-        for (int i = 0; i < resultLen; i += 2) {
-            result[i] = keys.get(i / 2);
-            result[i + 1] = values[i / 2];
+    private static String[] mergeKVPairs(List<String> keys, String[] values, LabelCfg label) {
+        var result = new ArrayList<String>();
+        for (int i = 0; i < keys.size(); i++) {
+            if (!label.omitEmptyFields || !StringUtils.isBlank(values[i])) {
+                result.add(keys.get(i));
+                result.add(values[i]);
+            }
         }
-        return result;
+        return result.toArray(new String[0]);
     }
 
     public LabelCfg getLabel() {

loki-logback-appender/src/main/java/com/github/loki4j/logback/MultiPatternExtractor.java

@@ -50,7 +50,7 @@ public String[] extract(E event) {
                 c.write(strBuilder, event);
                 c = c.getNext();
             }
-            result[i] = strBuilder.toString();
+            result[i] = strBuilder.toString().trim();
             strBuilder.setLength(0);
         }
         return result;

loki-logback-appender/src/test/java/com/github/loki4j/logback/AbstractLoki4jEncoderTest.java

@@ -117,6 +117,43 @@ public void testLogRecordStreams() {
         var stream3 = encoder.eventToStream(loggingEvent(108L, Level.INFO, "test.TestApp", "thread-1", "Test message 3", null));
         assertArrayEquals(new String[] { "level", "INFO", "app", "my-app", "thread", "thread-1" }, stream3.labels);
 
+        var stream4 = encoder.eventToStream(loggingEvent(110L, Level.INFO, "test.TestApp", "", "Test message 3", null));
+        assertArrayEquals(new String[] { "level", "INFO", "app", "my-app", "thread", "" }, stream4.labels);
+
+        var stream5 = encoder.eventToStream(loggingEvent(110L, Level.INFO, "test.TestApp", "   ", "Test message 3", null));
+        assertArrayEquals(new String[] { "level", "INFO", "app", "my-app", "thread", "" }, stream5.labels);
+
+        assertTrue("Same labels resolved to one stream", stream1 == stream3);
+        assertFalse("Different labels resolved to different streams", stream1 == stream2);
+
+        encoder.stop();
+    }
+
+    @Test
+    public void testLogRecordStreamsWithEmptyFields() {
+        var encoder = toStringEncoder(
+                labelCfg("level=%level,app=my-app,thread=%thread,msg=%message", ",", "=", false, false, true),
+                plainTextMsgLayout("l=%level | %msg %ex{1}"),
+                false);
+        encoder.setContext(new LoggerContext());
+        encoder.start();
+
+        var stream1 = encoder.eventToStream(loggingEvent(100L, Level.INFO, "test.TestApp", "thread-1", "", null));
+        assertArrayEquals(new String[] { "level", "INFO", "app", "my-app", "thread", "thread-1" }, stream1.labels);
+
+        var stream2 = encoder.eventToStream(loggingEvent(103L, Level.WARN, "test.TestApp", "   ", "Test message 2", null));
+        assertArrayEquals(new String[] { "level", "WARN", "app", "my-app", "msg", "Test message 2" }, stream2.labels);
+
+        var stream3 = encoder.eventToStream(loggingEvent(108L, Level.INFO, "test.TestApp", "thread-1", "  ", null));
+        assertArrayEquals(new String[] { "level", "INFO", "app", "my-app", "thread", "thread-1" }, stream3.labels);
+
+        var stream4 = encoder.eventToStream(loggingEvent(110L, Level.INFO, "test.TestApp", "", "\t ", null));
+        assertArrayEquals(new String[] { "level", "INFO", "app", "my-app" }, stream4.labels);
+
+        var stream5 = encoder.eventToStream(loggingEvent(110L, Level.INFO, "test.TestApp", "   ", " ", null));
+        assertArrayEquals(new String[] { "level", "INFO", "app", "my-app" }, stream5.labels);
+
+
         assertTrue("Same labels resolved to one stream", stream1 == stream3);
         assertFalse("Different labels resolved to different streams", stream1 == stream2);
 

loki-logback-appender/src/test/java/com/github/loki4j/logback/Generators.java

@@ -175,11 +175,22 @@ public static AbstractLoki4jEncoder.LabelCfg labelCfg(
             String keyValueSeparator,
             boolean nopex,
             boolean readMarkers) {
+        return labelCfg(pattern, pairSeparator, keyValueSeparator, nopex, readMarkers, false);
+    }
+
+    public static AbstractLoki4jEncoder.LabelCfg labelCfg(
+            String pattern,
+            String pairSeparator,
+            String keyValueSeparator,
+            boolean nopex,
+            boolean readMarkers,
+            boolean omitEmpty) {
         var label = new AbstractLoki4jEncoder.LabelCfg();
         label.setPattern(pattern);
         label.setPairSeparator(pairSeparator);
         label.setKeyValueSeparator(keyValueSeparator);
         label.setReadMarkers(readMarkers);
+        label.setOmitEmptyFields(omitEmpty);
         return label;
     }