Add support for Extended Log Format (zalando#1304)

* Added support for Extend Log File Format * Update dependencies to fix build * Revert "Update dependencies to fix build" This reverts commit 944df6f. * Update according to comments remove unnecessary comments and add final to variables * Encapsulate local variable logic into enum lambda * Suppress CVE-2022-45688 This relates to jeremylong/DependencyCheck#5502.
SmartBear · Feb 27, 2023 · 32a8ac0 · 32a8ac0
1 parent 527253d
commit 32a8ac0
Show file tree

Hide file tree

Showing 4 changed files with 422 additions and 1 deletion.
diff --git a/README.md b/README.md
@@ -464,6 +464,33 @@ The Common Log Format ([CLF](https://httpd.apache.org/docs/trunk/logs.html#commo
 185.85.220.253 - - [02/Aug/2019:08:16:41 0000] "GET /search?q=zalando HTTP/1.1" 200 -
 ```
 
+##### Extended Log Format
+
+The Extended Log Format ([ELF](https://en.wikipedia.org/wiki/Extended_Log_Format)) is a standardised text file format, like Common Log Format (CLF), that is used by web servers when generating log files, but ELF files provide more information and flexibility. The format is supported via the `ExtendedLogFormatSink`.
+Also see [W3C](https://www.w3.org/TR/WD-logfile.html) document.
+
+Default fields:
+
+```text
+date time c-ip s-dns cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs-protocol cs(User-Agent) cs(Cookie) cs(Referrer)
+```
+
+Default log output example:
+
+```text
+2019-08-02 08:16:41 185.85.220.253 localhost POST /search ?q=zalando 200 21 20 0.125 HTTP/1.1 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0" "name=value" "https://example.com/page?q=123"
+```
+
+Users may override default fields with their custom fields through the constructor of `ExtendedLogFormatSink`:
+
+```java
+new ExtendedLogFormatSink(new DefaultHttpLogWriter(), "date time cs(Custom-Request-Header) sc(Custom-Response-Header)")
+```
+
+For Http header fields: `cs(Any-Header)` and `sc(Any-Header)`, users could specify any headers they want to extract from the request.
+
+Other supported fields are listed in the value of `ExtendedLogFormatSink.Field`, which can be put in the custom field expression.
+
 ##### cURL
 
 *cURL* is an alternative formatting style, provided by the `CurlHttpLogFormatter` which will render requests as 

diff --git a/cve-suppressions.xml b/cve-suppressions.xml
@@ -13,5 +13,12 @@
             <!-- so far jackson-core and json-path don't have bugfix releases yet for that cve -->
             <cve>CVE-2022-45688</cve>
         </suppress>
-
+        <suppress>
+            <notes><![CDATA[
+            suppress CVE-2022-45688 only to pkg:maven/org.json/json
+            ]]></notes>
+            <packageUrl regex="true">^(?!pkg:maven/org\.json/json@).+$</packageUrl>
+            <!-- Suppressing until https://github.com/jeremylong/DependencyCheck/issues/5502 has been solved -->
+            <cve>CVE-2022-45688</cve>
+        </suppress>
 </suppressions>
diff --git a/logbook-core/src/main/java/org/zalando/logbook/ExtendedLogFormatSink.java b/logbook-core/src/main/java/org/zalando/logbook/ExtendedLogFormatSink.java
@@ -0,0 +1,218 @@
+package org.zalando.logbook;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import org.apiguardian.api.API;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.time.Instant;
+import java.time.ZoneId;
+import java.time.ZonedDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.*;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+import static org.apiguardian.api.API.Status.EXPERIMENTAL;
+
+/**
+ * @see <a href="https://en.wikipedia.org/wiki/Extended_Log_Format">Wikipedia: Extended Log Format</a>
+ * @see <a href="https://www.w3.org/TR/WD-logfile.html">W3C Extended Log Format</a>
+ */
+@API(status = EXPERIMENTAL)
+public final class ExtendedLogFormatSink implements Sink {
+
+    private static final String DELIMITER = " ";
+    private static final String HEADER_DELIMITER = ";";
+    private static final String OMITTED_FIELD = "-";
+    private static final String DEFAULT_VERSION = "1.0";
+    private static final String DEFAULT_FIELDS = "date time c-ip s-dns cs-method cs-uri-stem cs-uri-query sc-status sc-bytes cs-bytes time-taken cs-protocol cs(User-Agent) cs(Cookie) cs(Referrer)";
+    private static final Pattern CS_HEADER_REGEX = Pattern.compile("cs\\((.*?)\\)");
+    private static final Pattern SC_HEADER_REGEX = Pattern.compile("sc\\((.*?)\\)");
+
+    private final HttpLogWriter writer;
+    private final ZoneId timeZone;
+    private final List<String> supportedFields;
+    private final List<String> fields;
+
+    public ExtendedLogFormatSink(final HttpLogWriter writer) {
+        this(writer, ZoneId.of("UTC"), DEFAULT_VERSION, DEFAULT_FIELDS);
+    }
+
+    public ExtendedLogFormatSink(final HttpLogWriter writer, final String fields) {
+        this(writer, ZoneId.of("UTC"), DEFAULT_VERSION, fields);
+    }
+
+    public ExtendedLogFormatSink(final HttpLogWriter writer, ZoneId timeZone, final String version,
+                                 final String fields) {
+        this.writer = writer;
+        this.timeZone = timeZone;
+        this.supportedFields = getSupportedFields();
+        this.fields = getFields(fields);
+        logDirectives(version, this.fields);
+    }
+
+    @Override
+    public boolean isActive() {
+        return writer.isActive();
+    }
+
+    @Override
+    public void write(final Precorrelation precorrelation, final HttpRequest request) {
+        // nothing to do...
+    }
+
+    @Override
+    public void write(final Correlation correlation, final HttpRequest request,
+                      final HttpResponse response) throws IOException {
+
+        final ZonedDateTime startTime = correlation.getStart().atZone(timeZone);
+        final byte[] requestBody = request.getBody();
+        final byte[] responseBody = response.getBody();
+        final FieldParameter fieldParameter = new FieldParameter(startTime, correlation, request, response, requestBody,
+                responseBody);
+
+        final Map<String, String> fieldValMap = new HashMap<>();
+        for (Field field : Field.values()) {
+            fieldValMap.put(field.value, field.getExtraction().apply(fieldParameter));
+        }
+
+        final List<String> outputFields = new ArrayList<>(fields);
+        final String output = outputFields.stream().map(outputField -> getFieldOutput(outputField, fieldValMap,
+                        request, response))
+                .reduce((f1, f2) -> String.join(DELIMITER, f1, f2))
+                .orElse("");
+
+        writer.write(correlation, output);
+    }
+
+    private String getFieldOutput(final String outputField, final Map<String, String> fieldValMap,
+                                  final HttpRequest request, final HttpResponse response) {
+        if (supportedFields.contains(outputField)) {
+            return fieldValMap.get(outputField);
+        }
+        Matcher csHeaderMatcher = CS_HEADER_REGEX.matcher(outputField);
+        if (csHeaderMatcher.find()) {
+            final String headerKey = csHeaderMatcher.group(1);
+            return getCsHeader(request, headerKey);
+        }
+        Matcher scHeaderMatcher = SC_HEADER_REGEX.matcher(outputField);
+        if (scHeaderMatcher.find()) {
+            final String headerKey = scHeaderMatcher.group(1);
+            return getScHeader(response, headerKey);
+        }
+        return OMITTED_FIELD;
+    }
+
+    private String getCsHeader(final HttpRequest request, final String key) {
+        return buildHeaderString(request.getHeaders(), key);
+    }
+
+    private String getScHeader(final HttpResponse response, final String key) {
+        return buildHeaderString(response.getHeaders(), key);
+    }
+
+    private String buildHeaderString(final HttpHeaders httpHeaders, final String key) {
+        return Optional.of(httpHeaders)
+                .map(headers -> httpHeaders.get(key))
+                .map(values ->
+                        values.stream().reduce(
+                                        (v1, v2) ->
+                                                String.join(HEADER_DELIMITER, v1, v2))
+                                .map(valueStr -> "\"".concat(valueStr).concat("\""))
+                                .orElse(OMITTED_FIELD))
+                .orElse(OMITTED_FIELD);
+    }
+
+    /**
+     * Common supported fields
+     *
+     * @see <a href="https://docs.oracle.com/cd/A97329_03/bi.902/a90500/admin-05.htm#634823">Oracle analytical tool log
+     * parsing description</a>
+     */
+    private enum Field {
+
+        DATE("date", fieldParameter -> DateTimeFormatter.ISO_LOCAL_DATE.format(fieldParameter.getStartTime())),
+        TIME("time", fieldParameter -> DateTimeFormatter.ISO_LOCAL_TIME.format(fieldParameter.getStartTime())),
+        TIME_TAKEN("time-taken", fieldParameter -> BigDecimal
+                .valueOf(fieldParameter.getCorrelation().getDuration().toMillis())
+                .divide(BigDecimal.valueOf(1000), 3, RoundingMode.HALF_UP)
+                .toString()),
+        CS_PROTOCOL("cs-protocol", fieldParameter -> fieldParameter.getRequest().getProtocolVersion()),
+        SC_BYTES("sc-bytes", fieldParameter -> String.valueOf(fieldParameter.getResponseBody().length)),
+        CS_BYTES("cs-bytes", fieldParameter -> String.valueOf(fieldParameter.getRequestBody().length)),
+        CLIENT_IP("c-ip", fieldParameter -> fieldParameter.getRequest().getRemote()),
+        SERVER_DNS("s-dns", fieldParameter -> fieldParameter.getRequest().getHost()),
+        RESP_STATUS("sc-status", fieldParameter -> String.valueOf(fieldParameter.getResponse().getStatus())),
+        RESP_COMMENT("sc-comment", fieldParameter -> fieldParameter.getResponse().getReasonPhrase()),
+        REQ_METHOD("cs-method", fieldParameter -> fieldParameter.getRequest().getMethod()),
+        REQ_URI("cs-uri", fieldParameter -> fieldParameter.getRequest().getRequestUri()),
+        REQ_URI_STEM("cs-uri-stem", fieldParameter -> fieldParameter.getRequest().getPath()),
+        REQ_URI_QUERY("cs-uri-query", fieldParameter -> {
+            if (!"".equals(fieldParameter.getRequest().getQuery())) {
+                return "?" + fieldParameter.getRequest().getQuery();
+            } else {
+                return OMITTED_FIELD;
+            }
+        });
+
+        private final String value;
+
+        private final Function<FieldParameter, String> extraction;
+
+        Field(String label, Function<FieldParameter, String> extract) {
+            this.value = label;
+            this.extraction = extract;
+        }
+
+        Function<FieldParameter, String> getExtraction() {
+            return extraction;
+        }
+    }
+
+    @Getter
+    @AllArgsConstructor
+    private static class FieldParameter {
+        private final ZonedDateTime startTime;
+        private final Correlation correlation;
+        private final HttpRequest request;
+        private final HttpResponse response;
+        private final byte[] requestBody;
+        private final byte[] responseBody;
+    }
+
+    private List<String> getSupportedFields() {
+        return Arrays.stream(Field.values()).map(field -> field.value)
+                .collect(Collectors.toList());
+    }
+
+    private List<String> getFields(final String fieldExpression) {
+        final List<String> fields = getFieldsFromExpression(fieldExpression);
+        if (fields.isEmpty()) {
+            return getFieldsFromExpression(DEFAULT_FIELDS);
+        }
+        return fields;
+    }
+
+    private List<String> getFieldsFromExpression(final String fieldExpression) {
+        final List<String> fieldList = Arrays.asList(fieldExpression.split(DELIMITER));
+        return fieldList.stream()
+                .filter(field -> !field.equals(""))
+                .collect(Collectors.toList());
+    }
+
+    private void logDirectives(final String version, final List<String> fields) {
+        final String date = DateTimeFormatter.ISO_LOCAL_DATE.format(Instant.now().atZone(timeZone));
+        final Logger log = LoggerFactory.getLogger(Logbook.class);
+        log.trace("#Version: {}", version);
+        log.trace("#Date: {}", date);
+        log.trace("#Fields: {}", fields);
+    }
+
+}