[auditbeat] Make auditbeat resumeable when config is locked and immut…

…able is set (#32498)

auditbeat/docs/modules/auditd.asciidoc

@@ -164,9 +164,9 @@ newer and no rules have been defined. Otherwise `unicast` will be used.
 This option can only be used with the `socket_type: unicast` since {beatname_uc}
 needs to manage the rules to be able to set it.
 +
-It is important to note that with this setting set, {beatname_uc} should never
-be stopped, as it won't be able to resume processing `auditd` events until the
-system is restarted.
+It is important to note that with this setting enabled, if {beatname_uc} is
+stopped and resumed events will continue to be processed but the
+configuration won't be updated until the system is restarted entirely.
 
 *`resolve_ids`*:: This boolean setting enables the resolution of UIDs and
 GIDs to their associated names. The default value is true.

auditbeat/module/auditd/_meta/docs.asciidoc

@@ -157,9 +157,9 @@ newer and no rules have been defined. Otherwise `unicast` will be used.
 This option can only be used with the `socket_type: unicast` since {beatname_uc}
 needs to manage the rules to be able to set it.
 +
-It is important to note that with this setting set, {beatname_uc} should never
-be stopped, as it won't be able to resume processing `auditd` events until the
-system is restarted.
+It is important to note that with this setting enabled, if {beatname_uc} is
+stopped and resumed events will continue to be processed but the
+configuration won't be updated until the system is restarted entirely.
 
 *`resolve_ids`*:: This boolean setting enables the resolution of UIDs and
 GIDs to their associated names. The default value is true.

auditbeat/module/auditd/audit_linux.go

@@ -168,7 +168,19 @@ func closeAuditClient(client *libaudit.AuditClient, log *logp.Logger) {
 func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 	defer closeAuditClient(ms.client, ms.log)
 
-	if err := ms.addRules(reporter); err != nil {
+	// Don't attempt to change configuration if audit rules are locked (enabled == 2).
+	// Will result in EPERM.
+	status, err := ms.client.GetStatus()
+	if err != nil {
+		err = fmt.Errorf("failed to get audit status before adding rules: %w", err)
+		reporter.Error(err)
+		return
+	}
+
+	if status.Enabled == auditLocked {
+		err := errors.New("Skipping rule configuration: Audit rules are locked")
+		reporter.Error(err)
+	} else if err := ms.addRules(reporter); err != nil {
 		reporter.Error(err)
 		ms.log.Errorw("Failure adding audit rules", "error", err)
 		return
@@ -181,7 +193,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 		return
 	}
 
-	if ms.config.Immutable {
+	if ms.config.Immutable && status.Enabled != auditLocked {
 		if err := ms.client.SetImmutable(libaudit.WaitForReply); err != nil {
 			reporter.Error(err)
 			ms.log.Errorw("Failure setting audit config as immutable", "error", err)
@@ -267,18 +279,6 @@ func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error {
 	}
 	defer closeAuditClient(client, ms.log)
 
-	// Don't attempt to change configuration if audit rules are locked (enabled == 2).
-	// Will result in EPERM.
-	status, err := client.GetStatus()
-	if err != nil {
-		err = fmt.Errorf("failed to get audit status before adding rules: %w", err)
-		reporter.Error(err)
-		return err
-	}
-	if status.Enabled == auditLocked {
-		return errors.New("Skipping rule configuration: Audit rules are locked")
-	}
-
 	// Delete existing rules.
 	n, err := client.DeleteRules()
 	if err != nil {
@@ -333,56 +333,60 @@ func (ms *MetricSet) initClient() error {
 	ms.log.Infow("audit status from kernel at start", "audit_status", status)
 
 	if status.Enabled == auditLocked {
-		return errors.New("failed to configure: The audit system is locked")
-	}
-
-	if fm, _ := ms.config.failureMode(); status.Failure != fm {
-		if err = ms.client.SetFailure(libaudit.FailureMode(fm), libaudit.NoWait); err != nil {
-			return fmt.Errorf("failed to set audit failure mode in kernel: %w", err)
+		if !ms.config.Immutable {
+			return errors.New("failed to configure: The audit system is locked")
 		}
 	}
 
-	if status.BacklogLimit != ms.config.BacklogLimit {
-		if err = ms.client.SetBacklogLimit(ms.config.BacklogLimit, libaudit.NoWait); err != nil {
-			return fmt.Errorf("failed to set audit backlog limit in kernel: %w", err)
+	if status.Enabled != auditLocked {
+		if fm, _ := ms.config.failureMode(); status.Failure != fm {
+			if err = ms.client.SetFailure(libaudit.FailureMode(fm), libaudit.NoWait); err != nil {
+				return fmt.Errorf("failed to set audit failure mode in kernel: %w", err)
+			}
 		}
-	}
 
-	if ms.backpressureStrategy&(bsKernel|bsAuto) != 0 {
-		// "kernel" backpressure mitigation strategy
-		//
-		// configure the kernel to drop audit events immediately if the
-		// backlog queue is full.
-		if status.FeatureBitmap&libaudit.AuditFeatureBitmapBacklogWaitTime != 0 {
-			ms.log.Info("Setting kernel backlog wait time to prevent backpressure propagating to the kernel.")
-			if err = ms.client.SetBacklogWaitTime(0, libaudit.NoWait); err != nil {
-				return fmt.Errorf("failed to set audit backlog wait time in kernel: %w", err)
+		if status.BacklogLimit != ms.config.BacklogLimit {
+			if err = ms.client.SetBacklogLimit(ms.config.BacklogLimit, libaudit.NoWait); err != nil {
+				return fmt.Errorf("failed to set audit backlog limit in kernel: %w", err)
 			}
-		} else {
-			if ms.backpressureStrategy == bsAuto {
-				ms.log.Warn("setting backlog wait time is not supported in this kernel. Enabling workaround.")
-				ms.backpressureStrategy |= bsUserSpace
+		}
+
+		if ms.backpressureStrategy&(bsKernel|bsAuto) != 0 {
+			// "kernel" backpressure mitigation strategy
+			//
+			// configure the kernel to drop audit events immediately if the
+			// backlog queue is full.
+			if status.FeatureBitmap&libaudit.AuditFeatureBitmapBacklogWaitTime != 0 {
+				ms.log.Info("Setting kernel backlog wait time to prevent backpressure propagating to the kernel.")
+				if err = ms.client.SetBacklogWaitTime(0, libaudit.NoWait); err != nil {
+					return fmt.Errorf("failed to set audit backlog wait time in kernel: %w", err)
+				}
 			} else {
-				return errors.New("kernel backlog wait time not supported by kernel, but required by backpressure_strategy")
+				if ms.backpressureStrategy == bsAuto {
+					ms.log.Warn("setting backlog wait time is not supported in this kernel. Enabling workaround.")
+					ms.backpressureStrategy |= bsUserSpace
+				} else {
+					return errors.New("kernel backlog wait time not supported by kernel, but required by backpressure_strategy")
+				}
 			}
 		}
-	}
 
-	if ms.backpressureStrategy&(bsKernel|bsUserSpace) == bsUserSpace && ms.config.RateLimit == 0 {
-		// force a rate limit if the user-space strategy will be used without
-		// corresponding backlog_wait_time setting in the kernel
-		ms.config.RateLimit = 5000
-	}
+		if ms.backpressureStrategy&(bsKernel|bsUserSpace) == bsUserSpace && ms.config.RateLimit == 0 {
+			// force a rate limit if the user-space strategy will be used without
+			// corresponding backlog_wait_time setting in the kernel
+			ms.config.RateLimit = 5000
+		}
 
-	if status.RateLimit != ms.config.RateLimit {
-		if err = ms.client.SetRateLimit(ms.config.RateLimit, libaudit.NoWait); err != nil {
-			return fmt.Errorf("failed to set audit rate limit in kernel: %w", err)
+		if status.RateLimit != ms.config.RateLimit {
+			if err = ms.client.SetRateLimit(ms.config.RateLimit, libaudit.NoWait); err != nil {
+				return fmt.Errorf("failed to set audit rate limit in kernel: %w", err)
+			}
 		}
-	}
 
-	if status.Enabled == 0 {
-		if err = ms.client.SetEnabled(true, libaudit.NoWait); err != nil {
-			return fmt.Errorf("failed to enable auditing in the kernel: %w", err)
+		if status.Enabled == 0 {
+			if err = ms.client.SetEnabled(true, libaudit.NoWait); err != nil {
+				return fmt.Errorf("failed to enable auditing in the kernel: %w", err)
+			}
 		}
 	}
 
@@ -997,7 +1001,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) {
 		"select the most suitable subscription method."
 	switch c.SocketType {
 	case unicast:
-		if isLocked {
+		if isLocked && !c.Immutable {
 			log.Errorf("requested unicast socket_type is not available "+
 				"because audit configuration is locked in the kernel "+
 				"(enabled=2). %s", useAutodetect)
@@ -1022,7 +1026,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) {
 		// attempt to determine the optimal socket_type
 		if hasMulticast {
 			if hasRules {
-				if isLocked {
+				if isLocked && !c.Immutable {
 					log.Warn("Audit rules specified in the configuration " +
 						"cannot be applied because the audit rules have been locked " +
 						"in the kernel (enabled=2). A multicast audit subscription " +
@@ -1033,7 +1037,7 @@ func determineSocketType(c *Config, log *logp.Logger) (string, error) {
 			}
 			return multicast, nil
 		}
-		if isLocked {
+		if isLocked && !c.Immutable {
 			log.Errorf("Cannot continue: audit configuration is locked " +
 				"in the kernel (enabled=2) which prevents using unicast " +
 				"sockets. Multicast audit subscriptions are not available " +

auditbeat/module/auditd/audit_linux_test.go

@@ -81,7 +81,8 @@ func TestImmutable(t *testing.T) {
 		returnACK().returnStatus().
 		// Send expected ACKs for initialization
 		// With one extra for SetImmutable
-		returnACK().returnACK().returnACK().returnACK().returnACK().returnACK().
+		returnACK().returnStatus().returnACK().returnACK().
+		returnACK().returnACK().returnACK().returnACK().
 		// Send one auditd message.
 		returnMessage(userLoginFailMsg)
 
@@ -114,7 +115,8 @@ func TestData(t *testing.T) {
 		// Get Status response for initClient
 		returnACK().returnStatus().
 		// Send expected ACKs for initialization
-		returnACK().returnACK().returnACK().returnACK().returnACK().
+		returnACK().returnStatus().returnACK().returnACK().
+		returnACK().returnACK().returnACK().
 		// Send three auditd messages.
 		returnMessage(userLoginFailMsg).
 		returnMessage(execveMsgs...).
@@ -127,10 +129,10 @@ func TestData(t *testing.T) {
 	auditMetricSet.client = &libaudit.AuditClient{Netlink: mock}
 
 	events := mbtest.RunPushMetricSetV2(10*time.Second, 3, ms)
+	assertNoErrors(t, events)
 	if len(events) != 3 {
 		t.Fatalf("expected 3 events, but received %d", len(events))
 	}
-	assertNoErrors(t, events)
 
 	assertFieldsAreDocumented(t, events)
 
@@ -146,7 +148,8 @@ func TestLoginType(t *testing.T) {
 		// Get Status response for initClient
 		returnACK().returnStatus().
 		// Send expected ACKs for initialization
-		returnACK().returnACK().returnACK().returnACK().returnACK().
+		returnACK().returnStatus().returnACK().returnACK().
+		returnACK().returnACK().returnACK().
 		// Send an authentication failure and a success.
 		returnMessage(userLoginFailMsg).
 		returnMessage(userLoginSuccessMsg).

auditbeat/module/auditd/golden_files_test.go

@@ -196,7 +196,8 @@ func TestGoldenFiles(t *testing.T) {
 				// Get Status response for initClient
 				returnACK().returnStatus().
 				// Send expected ACKs for initialization
-				returnACK().returnACK().returnACK().returnACK().returnACK().
+				returnACK().returnStatus().returnACK().returnACK().
+				returnACK().returnACK().returnACK().
 				// Send audit messages
 				returnMessage(lines...).
 				// Send stream terminator