Emit trace.id into audit logs

docs/changelog/82849.yaml

@@ -0,0 +1,5 @@
+pr: 82849
+summary: Emit `trace.id` into audit logs
+area: "Audit"
+type: enhancement
+issues: []

x-pack/docs/en/security/auditing/event-types.asciidoc

@@ -6,7 +6,7 @@ When you are <<enable-audit-logging,auditing security events>>, a single client
 request might generate multiple audit events, across multiple cluster nodes.
 The common `request.id` attribute can be used to correlate the associated events.
 
-Use the <<xpack-sa-lf-events-include,`xpack.security.audit.logfile.events.include`>> 
+Use the <<xpack-sa-lf-events-include,`xpack.security.audit.logfile.events.include`>>
 setting in `elasticsearch.yml` to specify the kind of events you want to include
 in the auditing output.
 
@@ -534,8 +534,8 @@ The following list shows attributes that are common to all audit event types:
 In addition, if `event.type` equals <<security-config-change,`security_config_change`>>,
 the `event.action` attribute takes one of the following values:
 `put_user`, `change_password`, `put_role`, `put_role_mapping`,
-`change_enable_user`, `change_disable_user`, `put_privileges`, `create_apikey`, 
-`delete_user`, `delete_role`, `delete_role_mapping`, `invalidate_apikeys` or 
+`change_enable_user`, `change_disable_user`, `put_privileges`, `create_apikey`,
+`delete_user`, `delete_role`, `delete_role_mapping`, `invalidate_apikeys` or
 `delete_privileges`.
 
 `request.id`      ::    A synthetic identifier that can be used to correlate the events
@@ -557,9 +557,12 @@ show more details about the requesting client:
                         `transport` (request was received on the transport channel),
                         or `local_node` (the local node issued the request).
 `opaque_id`       ::    The value of the `X-Opaque-Id` HTTP header (if present) of
-                        the request associated with this event. This header can
-                        be used freely by the client to mark API calls, as it has
-                        no semantics in Elasticsearch.
+                        the request associated with this event.
+                        See more: <<x-opaque-id, `X-Opaque-Id` HTTP header - API conventions>>
+`trace_id`        ::    The identifier extracted from the `traceparent` HTTP header
+                        (if present) of the request associated with this event.
+                        It allows to surface audit logs into the Trace Logs feature
+                        of Elastic APM.
 `x_forwarded_for` ::    The verbatim value of the `X-Forwarded-For` HTTP request
                         header (if present) of the request associated with the
                         audit event. This header is commonly added by proxies

x-pack/plugin/core/src/main/config/log4j2.properties

@@ -37,6 +37,7 @@ appender.audit_rolling.layout.pattern = {\
                 %varsNotEmpty{, "request.name":"%enc{%map{request.name}}{JSON}"}\
                 %varsNotEmpty{, "indices":%map{indices}}\
                 %varsNotEmpty{, "opaque_id":"%enc{%map{opaque_id}}{JSON}"}\
+                %varsNotEmpty{, "trace.id":"%enc{%map{trace.id}}{JSON}"}\
                 %varsNotEmpty{, "x_forwarded_for":"%enc{%map{x_forwarded_for}}{JSON}"}\
                 %varsNotEmpty{, "transport.profile":"%enc{%map{transport.profile}}{JSON}"}\
                 %varsNotEmpty{, "rule":"%enc{%map{rule}}{JSON}"}\
@@ -76,6 +77,7 @@ appender.audit_rolling.layout.pattern = {\
 # "request.name" if the event is in connection to a transport message this is the name of the request class, similar to how rest requests are identified by the url path (internal)
 # "indices" the array of indices that the "action" is acting upon
 # "opaque_id" opaque value conveyed by the "X-Opaque-Id" request header
+# "trace_id" an identifier conveyed by the part of "traceparent" request header
 # "x_forwarded_for" the addresses from the "X-Forwarded-For" request header, as a verbatim string value (not an array)
 # "transport.profile" name of the transport profile in case this is a "connection_granted" or "connection_denied" event
 # "rule" name of the applied rule if the "origin.type" is "ip_filter"

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/audit/logfile/LoggingAuditTrail.java

@@ -180,6 +180,7 @@ public class LoggingAuditTrail implements AuditTrail, ClusterStateListener {
     public static final String TRANSPORT_PROFILE_FIELD_NAME = "transport.profile";
     public static final String RULE_FIELD_NAME = "rule";
     public static final String OPAQUE_ID_FIELD_NAME = "opaque_id";
+    public static final String TRACE_ID_FIELD_NAME = "trace.id";
     public static final String X_FORWARDED_FOR_FIELD_NAME = "x_forwarded_for";
     // the fields below are used exclusively for "security_config_change" type of events, and show the configuration
     // object taking effect; it could be creating a new, or updating an existing configuration
@@ -457,8 +458,7 @@ public void authenticationSuccess(String requestId, Authentication authenticatio
                 .withAuthentication(authentication)
                 .withRestOrigin(request)
                 .withRequestBody(request)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -486,8 +486,7 @@ public void authenticationSuccess(String requestId, Authentication authenticatio
                     .withAuthentication(authentication)
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -506,8 +505,7 @@ public void anonymousAccessDenied(String requestId, String action, TransportRequ
                     .withRequestId(requestId)
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -523,8 +521,7 @@ public void anonymousAccessDenied(String requestId, RestRequest request) {
                 .withRestOrigin(request)
                 .withRequestBody(request)
                 .withRequestId(requestId)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -543,8 +540,7 @@ public void authenticationFailed(String requestId, AuthenticationToken token, St
                     .withRequestId(requestId)
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext);
+                    .withThreadContext(threadContext);
                 if (token instanceof ServiceAccountToken) {
                     logEntryBuilder.with(SERVICE_TOKEN_NAME_FIELD_NAME, ((ServiceAccountToken) token).getTokenName());
                 }
@@ -562,8 +558,7 @@ public void authenticationFailed(String requestId, RestRequest request) {
                 .withRestOrigin(request)
                 .withRequestBody(request)
                 .withRequestId(requestId)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -581,8 +576,7 @@ public void authenticationFailed(String requestId, String action, TransportReque
                     .withRequestId(requestId)
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -600,8 +594,7 @@ public void authenticationFailed(String requestId, AuthenticationToken token, Re
                 .withRestOrigin(request)
                 .withRequestBody(request)
                 .withRequestId(requestId)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext);
+                .withThreadContext(threadContext);
             if (token instanceof ServiceAccountToken) {
                 logEntryBuilder.with(SERVICE_TOKEN_NAME_FIELD_NAME, ((ServiceAccountToken) token).getTokenName());
             }
@@ -630,8 +623,7 @@ public void authenticationFailed(
                     .withRequestId(requestId)
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -650,8 +642,7 @@ public void authenticationFailed(String requestId, String realm, AuthenticationT
                 .withRestOrigin(request)
                 .withRequestBody(request)
                 .withRequestId(requestId)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -687,8 +678,7 @@ public void accessGranted(
                     .withAuthentication(authentication)
                     .withRestOrTransportOrigin(msg, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .with(authorizationInfo.asMap())
                     .build();
             }
@@ -795,8 +785,7 @@ public void explicitIndexAccessEvent(
                     .withRequestId(requestId)
                     .withAuthentication(authentication)
                     .with(INDICES_FIELD_NAME, indices)
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .with(authorizationInfo.asMap());
                 final InetSocketAddress restAddress = RemoteHostHeader.restRemoteAddress(threadContext);
                 if (restAddress != null) {
@@ -841,8 +830,7 @@ public void accessDenied(
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
                     .with(authorizationInfo.asMap())
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -857,8 +845,7 @@ public void tamperedRequest(String requestId, RestRequest request) {
                 .withRestOrigin(request)
                 .withRequestBody(request)
                 .withRequestId(requestId)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -876,8 +863,7 @@ public void tamperedRequest(String requestId, String action, TransportRequest tr
                     .withRequestId(requestId)
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -906,8 +892,7 @@ public void tamperedRequest(String requestId, Authentication authentication, Str
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .withAuthentication(authentication)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -925,8 +910,7 @@ public void connectionGranted(InetAddress inetAddress, String profile, SecurityI
                 .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress))
                 .with(TRANSPORT_PROFILE_FIELD_NAME, profile)
                 .with(RULE_FIELD_NAME, rule.toString())
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -943,8 +927,7 @@ public void connectionDenied(InetAddress inetAddress, String profile, SecurityIp
                 .with(ORIGIN_ADDRESS_FIELD_NAME, NetworkAddress.format(inetAddress))
                 .with(TRANSPORT_PROFILE_FIELD_NAME, profile)
                 .with(RULE_FIELD_NAME, rule.toString())
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -979,8 +962,7 @@ public void runAsGranted(
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
                     .with(authorizationInfo.asMap())
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -1016,8 +998,7 @@ public void runAsDenied(
                     .withRestOrTransportOrigin(transportRequest, threadContext)
                     .with(INDICES_FIELD_NAME, indices.orElse(null))
                     .with(authorizationInfo.asMap())
-                    .withOpaqueId(threadContext)
-                    .withXForwardedFor(threadContext)
+                    .withThreadContext(threadContext)
                     .build();
             }
         }
@@ -1045,8 +1026,7 @@ public void runAsDenied(String requestId, Authentication authentication, RestReq
                 .withRestOrigin(request)
                 .withRequestBody(request)
                 .withRequestId(requestId)
-                .withOpaqueId(threadContext)
-                .withXForwardedFor(threadContext)
+                .withThreadContext(threadContext)
                 .build();
         }
     }
@@ -1473,20 +1453,18 @@ LogEntryBuilder withRequestId(String requestId) {
             return this;
         }
 
-        LogEntryBuilder withOpaqueId(ThreadContext threadContext) {
-            final String opaqueId = threadContext.getHeader(Task.X_OPAQUE_ID_HTTP_HEADER);
-            if (opaqueId != null) {
-                logEntry.with(OPAQUE_ID_FIELD_NAME, opaqueId);
-            }
+        LogEntryBuilder withThreadContext(ThreadContext threadContext) {
+            setThreadContextField(threadContext, AuditTrail.X_FORWARDED_FOR_HEADER, X_FORWARDED_FOR_FIELD_NAME);
+            setThreadContextField(threadContext, Task.X_OPAQUE_ID_HTTP_HEADER, OPAQUE_ID_FIELD_NAME);
+            setThreadContextField(threadContext, Task.TRACE_ID, TRACE_ID_FIELD_NAME);
             return this;
         }
 
-        LogEntryBuilder withXForwardedFor(ThreadContext threadContext) {
-            final String xForwardedFor = threadContext.getHeader(AuditTrail.X_FORWARDED_FOR_HEADER);
-            if (xForwardedFor != null) {
-                logEntry.with(X_FORWARDED_FOR_FIELD_NAME, xForwardedFor);
+        private void setThreadContextField(ThreadContext threadContext, String threadContextFieldName, String auditLogFieldName) {
+            final String fieldValue = threadContext.getHeader(threadContextFieldName);
+            if (fieldValue != null) {
+                logEntry.with(auditLogFieldName, fieldValue);
             }
-            return this;
         }
 
         LogEntryBuilder withAuthentication(Authentication authentication) {