LogJsonEnvelope settings in config

Author: UgnineSirdis

ydb/core/audit/audit_log_impl.cpp

@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
                 log.length()
             ));
         } catch (const yexception& e) {
-            LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
+            LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
         }
     }
 }

ydb/core/log_backend/log_backend.cpp

@@ -1,9 +1,52 @@
 #include "log_backend.h"
+#include "json_envelope.h"
 #include "log_backend_build.h"
 #include <ydb/core/base/counters.h>
 
+#include <util/system/mutex.h>
+
 namespace NKikimr {
 
+class TLogBackendWithJsonEnvelope : public TLogBackend {
+public:
+    TLogBackendWithJsonEnvelope(const TString& jsonEnvelope, THolder<TLogBackend> logBackend)
+        : JsonEnvelope(jsonEnvelope)
+        , LogBackend(std::move(logBackend))
+    {}
+
+    void WriteData(const TLogRecord& rec) override {
+        TString data;
+        TLogRecord record = rec;
+        with_lock (Mutex) {
+            data = JsonEnvelope.ApplyJsonEnvelope(TStringBuf(record.Data, record.Len));
+        }
+        record.Data = data.data();
+        record.Len = data.size();
+        LogBackend->WriteData(record);
+    }
+
+    void ReopenLog() override {
+        LogBackend->ReopenLog();
+    }
+
+    void ReopenLogNoFlush() override {
+        LogBackend->ReopenLogNoFlush();
+    }
+
+    ELogPriority FiltrationLevel() const override {
+        return LogBackend->FiltrationLevel();
+    }
+
+    size_t QueueSize() const override {
+        return LogBackend->QueueSize();
+    }
+
+private:
+    TJsonEnvelope JsonEnvelope;
+    THolder<TLogBackend> LogBackend;
+    TMutex Mutex;
+};
+
 TAutoPtr<TLogBackend> CreateLogBackendWithUnifiedAgent(
         const TKikimrRunConfig& runConfig,
         NMonitoring::TDynamicCounterPtr counters)
@@ -127,36 +170,46 @@ TAutoPtr<TLogBackend> CreateAuditLogUnifiedAgentBackend(
     return logBackend;
 }
 
+THolder<TLogBackend> MaybeWrapWithJsonEnvelope(THolder<TLogBackend> logBackend, const TString& jsonEnvelope) {
+    if (jsonEnvelope.empty() || !logBackend) {
+        return logBackend;
+    }
+
+    return MakeHolder<TLogBackendWithJsonEnvelope>(jsonEnvelope, std::move(logBackend));
+}
+
 TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> CreateAuditLogBackends(
         const TKikimrRunConfig& runConfig,
         NMonitoring::TDynamicCounterPtr counters) {
     TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> logBackends;
-    if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasStderrBackend()) {
-        auto logBackend = NActors::CreateStderrBackend();
-        auto format = runConfig.AppConfig.GetAuditConfig().GetStderrBackend().GetFormat();
-        logBackends[format].push_back(std::move(logBackend));
-    }
 
-    if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasFileBackend()) {
-        auto logBackend = CreateAuditLogFileBackend(runConfig);
-        if (logBackend) {
-            auto format = runConfig.AppConfig.GetAuditConfig().GetFileBackend().GetFormat();
+    if (runConfig.AppConfig.HasAuditConfig()) {
+        const auto& auditConfig = runConfig.AppConfig.GetAuditConfig();
+        if (auditConfig.HasStderrBackend()) {
+            auto logBackend = MaybeWrapWithJsonEnvelope(NActors::CreateStderrBackend(), auditConfig.GetStderrBackend().GetLogJsonEnvelope());
+            auto format = auditConfig.GetStderrBackend().GetFormat();
             logBackends[format].push_back(std::move(logBackend));
         }
-    }
 
-    if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasUnifiedAgentBackend()) {
-        auto logBackend = CreateAuditLogUnifiedAgentBackend(runConfig, counters);
-        if (logBackend) {
-            auto format = runConfig.AppConfig.GetAuditConfig().GetUnifiedAgentBackend().GetFormat();
-            logBackends[format].push_back(std::move(logBackend));
+        if (auditConfig.HasFileBackend()) {
+            auto logBackend = MaybeWrapWithJsonEnvelope(CreateAuditLogFileBackend(runConfig), auditConfig.GetFileBackend().GetLogJsonEnvelope());
+            if (logBackend) {
+                auto format = auditConfig.GetFileBackend().GetFormat();
+                logBackends[format].push_back(std::move(logBackend));
+            }
         }
-    }
 
+        if (auditConfig.HasUnifiedAgentBackend()) {
+            auto logBackend = MaybeWrapWithJsonEnvelope(CreateAuditLogUnifiedAgentBackend(runConfig, counters), auditConfig.GetUnifiedAgentBackend().GetLogJsonEnvelope());
+            if (logBackend) {
+                auto format = auditConfig.GetUnifiedAgentBackend().GetFormat();
+                logBackends[format].push_back(std::move(logBackend));
+            }
+        }
+    }
 
     return logBackends;
 }
 
 
 } // NKikimr
-

ydb/core/protos/config.proto

@@ -1455,16 +1455,19 @@ message TAuditConfig {
 
     message TStderrBackend {
         optional EFormat Format = 1 [default = JSON];
+        optional string LogJsonEnvelope = 2; // Json template with text field containing %message% placeholder. For example {"my_enveloped_message": "%message%"}. %message% will be replaced with real audit log message
     }
 
     message TFileBackend {
         optional EFormat Format = 1 [default = JSON];
         optional string FilePath = 2;
+        optional string LogJsonEnvelope = 3; // Json template with text field containing %message% placeholder. For example {"my_enveloped_message": "%message%"}. %message% will be replaced with real audit log message
     }
 
     message TUnifiedAgentBackend {
         optional EFormat Format = 1 [default = JSON];
         optional string LogName = 2;
+        optional string LogJsonEnvelope = 3; // Json template with text field containing %message% placeholder. For example {"my_enveloped_message": "%message%"}. %message% will be replaced with real audit log message
     }
 
     optional TStderrBackend StderrBackend = 1;