Fix decode_cef handling of escaped LF and CR (#29268)

* Fix decode_cef handling of escaped LF and CR

Escaped LF and CR were not handled inside of CEF extension value.
Those characters are now accepted and replaced with their unescaped
equivalents.

Replace fgoto with fnext to fix generated Go code that had duplicate 'goto' statements.

Fixes #16995

CHANGELOG.next.asciidoc

@@ -196,6 +196,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix in `aws-s3` input regarding provider discovery through endpoint {pull}28963[28963]
 - Fix `threatintel.misp` filters configuration. {issue}27970[27970]
 - Fix opening files on Windows in filestream so open files can be deleted. {issue}29113[29113] {pull}29180[29180]
+- Fix handling of escaped newlines in the `decode_cef` processor. {issue}16995[16995] {pull}29268[29268]
 - Fix `panw` module ingest errors for GLOBALPROTECT logs {pull}29154[29154]
 
 *Heartbeat*

x-pack/filebeat/processors/decode_cef/cef/cef.go

@@ -152,32 +152,42 @@ func (e *Event) Unpack(data string, opts ...Option) error {
 	return multierr.Combine(errs...)
 }
 
-const (
-	backslash        = `\`
-	escapedBackslash = `\\`
-
-	pipe        = `|`
-	escapedPipe = `\|`
-
-	equalsSign        = `=`
-	escapedEqualsSign = `\=`
-)
-
-var (
-	headerEscapes    = strings.NewReplacer(escapedBackslash, backslash, escapedPipe, pipe)
-	extensionEscapes = strings.NewReplacer(escapedBackslash, backslash, escapedEqualsSign, equalsSign)
-)
+// replaceEscapes replaces the escaped characters contained in v with their
+// unescaped value.
+func replaceEscapes(v string, startOffset int, escapes []int) string {
+	if len(escapes) == 0 {
+		return v
+	}
 
-func replaceHeaderEscapes(b string) string {
-	if strings.Index(b, escapedBackslash) != -1 || strings.Index(b, escapedPipe) != -1 {
-		return headerEscapes.Replace(b)
+	// Adjust escape offsets relative to the start offset of v.
+	for i := 0; i < len(escapes); i++ {
+		escapes[i] = escapes[i] - startOffset
 	}
-	return b
-}
 
-func replaceExtensionEscapes(b string) string {
-	if strings.Index(b, escapedBackslash) != -1 || strings.Index(b, escapedEqualsSign) != -1 {
-		return extensionEscapes.Replace(b)
+	var buf strings.Builder
+	var end int
+
+	// Iterate over escapes and replace them.
+	for i := 0; i < len(escapes); i += 2 {
+		start := escapes[i]
+		buf.WriteString(v[end:start])
+
+		end = escapes[i+1]
+		value := v[start:end]
+
+		switch value {
+		case `\n`:
+			buf.WriteByte('\n')
+		case `\r`:
+			buf.WriteByte('\r')
+		default:
+			// Remove leading slash.
+			if len(value) > 0 && value[0] == '\\' {
+				buf.WriteString(value[1:])
+			}
+		}
 	}
-	return b
+	buf.WriteString(v[end:])
+
+	return buf.String()
 }

x-pack/filebeat/processors/decode_cef/cef/cef.rl

@@ -18,7 +18,8 @@ import (
 // unpack unpacks a CEF message.
 func (e *Event) unpack(data string) error {
     cs, p, pe, eof := 0, 0, len(data), len(data)
-    mark := 0
+    mark, mark_slash := 0, 0
+    var escapes []int
 
     // Extension key.
     var extKey string
@@ -37,32 +38,43 @@ func (e *Event) unpack(data string) error {
         action mark {
             mark = p
         }
+        action mark_slash {
+            mark_slash = p
+        }
+        action mark_escape {
+            escapes = append(escapes, mark_slash, p)
+        }
         action version {
             e.Version, _ = strconv.Atoi(data[mark:p])
         }
         action device_vendor {
-            e.DeviceVendor = replaceHeaderEscapes(data[mark:p])
+            e.DeviceVendor = replaceEscapes(data[mark:p], mark, escapes)
+            escapes = escapes[:0]
         }
         action device_product {
-            e.DeviceProduct = replaceHeaderEscapes(data[mark:p])
+            e.DeviceProduct = replaceEscapes(data[mark:p], mark, escapes)
+            escapes = escapes[:0]
         }
         action device_version {
-            e.DeviceVersion = replaceHeaderEscapes(data[mark:p])
+            e.DeviceVersion = replaceEscapes(data[mark:p], mark, escapes)
+            escapes = escapes[:0]
         }
         action device_event_class_id {
-            e.DeviceEventClassID = replaceHeaderEscapes(data[mark:p])
+            e.DeviceEventClassID = replaceEscapes(data[mark:p], mark, escapes)
+            escapes = escapes[:0]
         }
         action name {
-            e.Name = replaceHeaderEscapes(data[mark:p])
+            e.Name = replaceEscapes(data[mark:p], mark, escapes)
+            escapes = escapes[:0]
         }
         action severity {
             e.Severity = data[mark:p]
         }
         action extension_key {
             // A new extension key marks the end of the last extension value.
             if len(extKey) > 0 && extValueStart <= mark - 1 {
-                e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:mark-1]))
-                extKey, extValueStart, extValueEnd = "", 0, 0
+                e.pushExtension(extKey, replaceEscapes(data[extValueStart:mark-1], extValueStart, escapes))
+                extKey, extValueStart, extValueEnd, escapes = "", 0, 0, escapes[:0]
             }
             extKey = data[mark:p]
         }
@@ -76,27 +88,28 @@ func (e *Event) unpack(data string) error {
         action extension_eof {
             // Reaching the EOF marks the end of the final extension value.
             if len(extKey) > 0 && extValueStart <= extValueEnd {
-                e.pushExtension(extKey, replaceExtensionEscapes(data[extValueStart:extValueEnd]))
-                extKey, extValueStart, extValueEnd = "", 0, 0
+                e.pushExtension(extKey, replaceEscapes(data[extValueStart:extValueEnd], extValueStart, escapes))
+                extKey, extValueStart, extValueEnd, escapes = "", 0, 0, escapes[:0]
             }
         }
         action extension_err {
             recoveredErrs = append(recoveredErrs, fmt.Errorf("malformed value for %s at pos %d", extKey, p+1))
-            fhold; fgoto gobble_extension;
+            fhold; fnext gobble_extension;
         }
         action recover_next_extension {
             extKey, extValueStart, extValueEnd = "", 0, 0
             // Resume processing at p, the start of the next extension key.
             p = mark;
-            fgoto extensions;
+            fnext extensions;
         }
 
         # Define what header characters are allowed.
         pipe = "|";
         escape = "\\";
         escape_pipe = escape pipe;
         backslash = "\\\\";
-        device_chars = backslash | escape_pipe | (any -- pipe -- escape);
+        header_escapes = (backslash | escape_pipe) >mark_slash %mark_escape;
+        device_chars = header_escapes | (any -- pipe -- escape);
         severity_chars = ( alpha | digit | "-" );
 
         # Header fields.
@@ -119,12 +132,15 @@ func (e *Event) unpack(data string) error {
         # Define what extension characters are allowed.
         equal = "=";
         escape_equal = escape equal;
+        escape_newline = escape 'n';
+        escape_carriage_return = escape 'r';
+        extension_value_escapes = (escape_equal | backslash | escape_newline | escape_carriage_return) >mark_slash %mark_escape;
         # Only alnum is defined in the CEF spec. The other characters allow
         # non-conforming extension keys to be parsed.
         extension_key_start_chars = alnum | '_';
         extension_key_chars = extension_key_start_chars | '.' | ',' | '[' | ']';
         extension_key_pattern = extension_key_start_chars extension_key_chars*;
-        extension_value_chars_nospace = backslash | escape_equal | (any -- equal -- escape -- space);
+        extension_value_chars_nospace = extension_value_escapes | (any -- equal -- escape -- space);
 
         # Extension fields.
         extension_key = extension_key_pattern >mark %extension_key;

x-pack/filebeat/processors/decode_cef/cef/cef_test.go

@@ -52,6 +52,8 @@ const (
 	tabMessage = "CEF:0|security|threatmanager|1.0|100|message is padded|10|spt=1232 msg=Tabs\tand\rcontrol\ncharacters are preserved\t src=127.0.0.1"
 
 	tabNoSepMessage = "CEF:0|security|threatmanager|1.0|100|message has tabs|10|spt=1232 msg=Tab is not a separator\tsrc=127.0.0.1"
+
+	escapedMessage = `CEF:0|security\\compliance|threat\|->manager|1.0|100|message contains escapes|10|spt=1232 msg=Newlines in messages\nare allowed.\r\nAnd so are carriage feeds\\newlines\\\=.`
 )
 
 var testMessages = []string{
@@ -71,6 +73,7 @@ var testMessages = []string{
 	paddedMessage,
 	crlfMessage,
 	tabMessage,
+	escapedMessage,
 }
 
 func TestGenerateFuzzCorpus(t *testing.T) {
@@ -374,6 +377,18 @@ func TestEventUnpack(t *testing.T) {
 			"spt": IntegerField(1232),
 		}, e.Extensions)
 	})
+
+	t.Run("escapes are replaced", func(t *testing.T) {
+		var e Event
+		err := e.Unpack(escapedMessage)
+		assert.NoError(t, err)
+		assert.Equal(t, `security\compliance`, e.DeviceVendor)
+		assert.Equal(t, `threat|->manager`, e.DeviceProduct)
+		assert.Equal(t, map[string]*Field{
+			"spt": IntegerField(1232),
+			"msg": StringField("Newlines in messages\nare allowed.\r\nAnd so are carriage feeds\\newlines\\=."),
+		}, e.Extensions)
+	})
 }
 
 func TestEventUnpackWithFullExtensionNames(t *testing.T) {