Merge audit log commits to 24-3

ydb/core/audit/audit_log_impl.cpp

@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
                 log.length()
             ));
         } catch (const yexception& e) {
-            LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
+            LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
         }
     }
 }
@@ -91,6 +91,27 @@ TString GetJsonLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev) {
     return ss.Str();
 }
 
+TString GetJsonLogCompatibleLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev) {
+    const auto* msg = ev->Get();
+    TStringStream ss;
+    NJsonWriter::TBuf json(NJsonWriter::HEM_DONT_ESCAPE_HTML, &ss);
+    {
+        auto obj = json.BeginObject();
+        obj
+            .WriteKey("@timestamp")
+            .WriteString(msg->Time.ToString().data())
+            .WriteKey("@log_type")
+            .WriteString("audit");
+
+        for (auto& [k, v] : msg->Parts) {
+            obj.WriteKey(k).WriteString(v);
+        }
+        json.EndObject();
+    }
+    ss << Endl;
+    return ss.Str();
+}
+
 TString GetTxtLog(const TEvAuditLog::TEvWriteAuditLog::TPtr& ev) {
     const auto* msg = ev->Get();
     TStringStream ss;
@@ -146,6 +167,9 @@ class TAuditLogActor final : public TActor<TAuditLogActor> {
                 case NKikimrConfig::TAuditConfig::TXT:
                     WriteLog(GetTxtLog(ev), logBackends.second);
                     break;
+                case NKikimrConfig::TAuditConfig::JSON_LOG_COMPATIBLE:
+                    WriteLog(GetJsonLogCompatibleLog(ev), logBackends.second);
+                    break;
                 default:
                     WriteLog(GetJsonLog(ev), logBackends.second);
                     break;

ydb/core/audit/ya.make

@@ -13,8 +13,4 @@ PEERDIR(
     ydb/core/base
 )
 
-RESOURCE(
-    ydb/core/kqp/kqp_default_settings.txt kqp_default_settings.txt
-)
-
 END()

ydb/core/client/server/msgbus_blobstorage_config.cpp

@@ -27,6 +27,7 @@ class TMessageBusBlobStorageConfig
     {
         SetSecurityToken(token);
         SetRequireAdminAccess(true);
+        SetPeerName(msg.GetPeerName());
     }
 
     void Handle(TEvBlobStorage::TEvControllerConfigResponse::TPtr &ev, const TActorContext &ctx) {

ydb/core/client/server/msgbus_server.cpp

@@ -16,6 +16,7 @@ class TBusMessageContext::TImpl : public TThrRefBase {
     virtual void SendReplyMove(NBus::TBusMessageAutoPtr response) = 0;
     virtual TVector<TStringBuf> FindClientCert() const = 0;
     virtual THolder<TMessageBusSessionIdentHolder::TImpl> CreateSessionIdentHolder() = 0;
+    virtual TString GetPeerName() const = 0;
 };
 
 class TBusMessageContext::TImplMessageBus
@@ -61,6 +62,13 @@ class TBusMessageContext::TImplMessageBus
         return {};
     }
 
+    TString GetPeerName() const override {
+        TStringBuilder ret;
+        if (IsConnectionAlive()) {
+            ret << GetPeerAddrNetAddr();
+        }
+        return std::move(ret);
+    }
 
     THolder<TMessageBusSessionIdentHolder::TImpl> CreateSessionIdentHolder() override;
 };
@@ -182,6 +190,10 @@ class TBusMessageContext::TImplGRpc
     };
 
     THolder<TMessageBusSessionIdentHolder::TImpl> CreateSessionIdentHolder() override;
+
+    TString GetPeerName() const override {
+        return RequestContext->GetPeer();
+    }
 };
 
 TBusMessageContext::TBusMessageContext()
@@ -228,6 +240,8 @@ void TBusMessageContext::Swap(TBusMessageContext &msg) {
 
 TVector<TStringBuf> TBusMessageContext::FindClientCert() const { return Impl->FindClientCert(); }
 
+TString TBusMessageContext::GetPeerName() const { return Impl->GetPeerName(); }
+
 THolder<TMessageBusSessionIdentHolder::TImpl> TBusMessageContext::CreateSessionIdentHolder() {
     Y_ABORT_UNLESS(Impl);
     return Impl->CreateSessionIdentHolder();

ydb/core/client/server/msgbus_server.h

@@ -87,6 +87,7 @@ class TBusMessageContext {
     void SendReplyMove(NBus::TBusMessageAutoPtr response);
     void Swap(TBusMessageContext& msg);
     TVector<TStringBuf> FindClientCert() const;
+    TString GetPeerName() const;
 
 private:
     friend class TMessageBusSessionIdentHolder;

ydb/core/client/server/msgbus_server_cms.cpp

@@ -31,6 +31,7 @@ class TCmsRequestActor : public TMessageBusSecureRequest<TMessageBusServerReques
         , Request(request)
     {
         TBase::SetSecurityToken(request.GetSecurityToken());
+        TBase::SetPeerName(msg.GetPeerName());
     }
 
     void Bootstrap(const TActorContext &ctx)

ydb/core/client/server/msgbus_server_console.cpp

@@ -44,6 +44,7 @@ class TConsoleRequestActor : public TMessageBusSecureRequest<TMessageBusServerRe
             TBase::SetRequireAdminAccess(true);
         }
 
+        TBase::SetPeerName(msg.GetPeerName());
     }
 
     void Bootstrap(const TActorContext &ctx)

ydb/core/client/server/msgbus_server_drain_node.cpp

@@ -20,6 +20,7 @@ class TMessageBusDrainNode : public TMessageBusSecureRequest<TMessageBusTabletRe
     {
         SetSecurityToken(Request->Record.GetSecurityToken());
         SetRequireAdminAccess(true);
+        SetPeerName(msg.GetPeerName());
     }
 
     std::pair<ui64, TAutoPtr<IEventBase>> MakeReqPair(const TActorContext& ctx) {

ydb/core/client/server/msgbus_server_fill_node.cpp

@@ -20,6 +20,7 @@ class TMessageBusFillNode : public TMessageBusSecureRequest<TMessageBusTabletReq
     {
         SetSecurityToken(Request->Record.GetSecurityToken());
         SetRequireAdminAccess(true);
+        SetPeerName(msg.GetPeerName());
     }
 
     std::pair<ui64, TAutoPtr<IEventBase>> MakeReqPair(const TActorContext& ctx) {

ydb/core/client/server/msgbus_server_local_minikql.cpp

@@ -17,6 +17,7 @@ class TMessageBusLocalMKQL : public TMessageBusSecureRequest<TMessageBusSimpleTa
     {
         SetSecurityToken(static_cast<TBusTabletLocalMKQL*>(msg.GetMessage())->Record.GetSecurityToken());
         SetRequireAdminAccess(true);
+        SetPeerName(msg.GetPeerName());
     }
 
     void Handle(TEvTablet::TEvLocalMKQLResponse::TPtr &ev, const TActorContext &ctx) {

ydb/core/client/server/msgbus_server_local_scheme_tx.cpp

@@ -21,6 +21,7 @@ class TMessageBusLocalSchemeTx : public TMessageBusSecureRequest<TMessageBusSimp
         Request.Swap(&request);
         TBase::SetSecurityToken(Request.GetSecurityToken());
         TBase::SetRequireAdminAccess(true);
+        TBase::SetPeerName(msg.GetPeerName());
     }
 
     void Handle(TEvTablet::TEvLocalSchemeTxResponse::TPtr &ev, const TActorContext &ctx) {

ydb/core/client/server/msgbus_server_node_registration.cpp

@@ -40,6 +40,7 @@ class TNodeRegistrationActor : public TMessageBusSecureRequest<TMessageBusServer
         } else {
             TBase::SetSecurityToken(BUILTIN_ACL_ROOT); // NBS compatibility
         }
+        TBase::SetPeerName(msg.GetPeerName());
     }
 
     void Bootstrap(const TActorContext &ctx)

ydb/core/client/server/msgbus_server_proxy.cpp

@@ -67,6 +67,7 @@ class TMessageBusServerFlatDescribeRequest : public TMessageBusSecureRequest<TMe
         , Request(static_cast<TBusSchemeDescribe*>(msg->MsgContext.ReleaseMessage()))
     {
         TBase::SetSecurityToken(Request->Record.GetSecurityToken());
+        TBase::SetPeerName(msg->MsgContext.GetPeerName());
     }
 
     //STFUNC(StateWork)

ydb/core/client/server/msgbus_server_request.cpp

@@ -55,6 +55,7 @@ class TMessageBusServerRequest : public TMessageBusSecureRequest<TMessageBusServ
     {
         TBase::SetSecurityToken(Request->Record.GetSecurityToken());
         TBase::SetRequireAdminAccess(true); // MiniKQL and ReadTable execution required administative access
+        TBase::SetPeerName(msg->MsgContext.GetPeerName());
     }
 
     //STFUNC(StateWork)
@@ -72,6 +73,7 @@ class TMessageBusServerRequest : public TMessageBusSecureRequest<TMessageBusServ
         ProposalStatus.Reset(new NKikimrTxUserProxy::TEvProposeTransactionStatus());
         Proposal.Reset(new TEvTxUserProxy::TEvProposeTransaction());
         NKikimrTxUserProxy::TEvProposeTransaction &record = Proposal->Record;
+        record.SetPeerName(GetPeerName());
 
         // Transaction protobuf structure might be very heavy (if it has a batch of parameters)
         // so we don't want to copy it, just move its contents

ydb/core/client/server/msgbus_server_scheme_initroot.cpp

@@ -54,6 +54,7 @@ class TMessageBusSchemeInitRoot : public TMessageBusSecureRequest<TMessageBusSer
     {
         TBase::SetSecurityToken(Request->Record.GetSecurityToken());
         TBase::SetRequireAdminAccess(true);
+        TBase::SetPeerName(msg->MsgContext.GetPeerName());
     }
 
     void Bootstrap(const TActorContext &ctx) {

ydb/core/client/server/msgbus_server_scheme_request.cpp

@@ -74,6 +74,7 @@ class TMessageBusServerSchemeRequest : public TMessageBusSecureRequest<TMessageB
     {
         TBase::SetSecurityToken(Request->Record.GetSecurityToken());
         TBase::SetRequireAdminAccess(true);
+        TBase::SetPeerName(msg->MsgContext.GetPeerName());
     }
 
     //STFUNC(StateWork)
@@ -95,6 +96,7 @@ template <>
 void TMessageBusServerSchemeRequest<TBusPersQueue>::SendProposeRequest(const TActorContext &ctx) {
     TAutoPtr<TEvTxUserProxy::TEvProposeTransaction> req(new TEvTxUserProxy::TEvProposeTransaction());
     NKikimrTxUserProxy::TEvProposeTransaction &record = req->Record;
+    record.SetPeerName(GetPeerName());
 
     if (Request->Record.HasMetaRequest() && Request->Record.GetMetaRequest().HasCmdCreateTopic()) {
         const auto& cmd = Request->Record.GetMetaRequest().GetCmdCreateTopic();
@@ -157,6 +159,7 @@ template <>
 void TMessageBusServerSchemeRequest<TBusSchemeOperation>::SendProposeRequest(const TActorContext &ctx) {
     TAutoPtr<TEvTxUserProxy::TEvProposeTransaction> req(new TEvTxUserProxy::TEvProposeTransaction());
     NKikimrTxUserProxy::TEvProposeTransaction &record = req->Record;
+    record.SetPeerName(GetPeerName());
 
     if (!Request->Record.HasTransaction()) {
         return HandleError(MSTATUS_ERROR, TEvTxUserProxy::TResultStatus::Unknown, "Malformed request: no modify scheme transaction provided", ctx);

ydb/core/cms/console/console__replace_yaml_config.cpp

@@ -19,7 +19,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase<TConfigsMa
         , Config(ev->Get()->Record.GetRequest().config())
         , Peer(ev->Get()->Record.GetPeerName())
         , Sender(ev->Sender)
-        , UserSID(NACLib::TUserToken(ev->Get()->Record.GetUserToken()).GetUserSID())
+        , UserToken(ev->Get()->Record.GetUserToken())
         , Force(force)
         , AllowUnknownFields(ev->Get()->Record.GetRequest().allow_unknown_fields())
         , DryRun(ev->Get()->Record.GetRequest().dry_run())
@@ -56,7 +56,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase<TConfigsMa
             oldVolatileConfig.SetConfig(config);
         }
 
-        Self->Logger.DbLogData(UserSID, logData, txc, ctx);
+        Self->Logger.DbLogData(UserToken.GetUserSID(), logData, txc, ctx);
     }
 
     bool Execute(TTransactionContext &txc, const TActorContext &ctx) override
@@ -164,7 +164,8 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase<TConfigsMa
         if (!Error && Modify && !DryRun) {
             AuditLogReplaceConfigTransaction(
                 /* peer = */ Peer,
-                /* userSID = */ UserSID,
+                /* userSID = */ UserToken.GetUserSID(),
+                /* sanitizedToken = */ UserToken.GetSanitizedToken(),
                 /* oldConfig = */ Self->YamlConfig,
                 /* newConfig = */ Config,
                 /* reason = */ {},
@@ -181,7 +182,8 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase<TConfigsMa
         } else if (Error && !DryRun) {
             AuditLogReplaceConfigTransaction(
                 /* peer = */ Peer,
-                /* userSID = */ UserSID,
+                /* userSID = */ UserToken.GetUserSID(),
+                /* sanitizedToken = */ UserToken.GetSanitizedToken(),
                 /* oldConfig = */ Self->YamlConfig,
                 /* newConfig = */ Config,
                 /* reason = */ ErrorReason,
@@ -195,7 +197,7 @@ class TConfigsManager::TTxReplaceYamlConfig : public TTransactionBase<TConfigsMa
     const TString Config;
     const TString Peer;
     const TActorId Sender;
-    const TString UserSID;
+    const NACLib::TUserToken UserToken;
     const bool Force = false;
     const bool AllowUnknownFields = false;
     const bool DryRun = false;

ydb/core/cms/console/console_audit.cpp

@@ -8,6 +8,7 @@ namespace NKikimr::NConsole {
 void AuditLogReplaceConfigTransaction(
     const TString& peer,
     const TString& userSID,
+    const TString& sanitizedToken,
     const TString& oldConfig,
     const TString& newConfig,
     const TString& reason,
@@ -23,6 +24,7 @@ void AuditLogReplaceConfigTransaction(
         AUDIT_PART("component", COMPONENT_NAME)
         AUDIT_PART("remote_address", (!peerName.empty() ? peerName : EMPTY_VALUE))
         AUDIT_PART("subject", (!userSID.empty() ? userSID : EMPTY_VALUE))
+        AUDIT_PART("sanitized_token", (!sanitizedToken.empty() ? sanitizedToken : EMPTY_VALUE))
         AUDIT_PART("status", TString(success ? "SUCCESS" : "ERROR"))
         AUDIT_PART("reason", reason, !reason.empty())
         AUDIT_PART("operation", TString("REPLACE DYNCONFIG"))

ydb/core/cms/console/console_audit.h

@@ -7,6 +7,7 @@ namespace NKikimr::NConsole {
 void AuditLogReplaceConfigTransaction(
     const TString& peer,
     const TString& userSID,
+    const TString& sanitizedToken,
     const TString& oldConfig,
     const TString& newConfig,
     const TString& reason,

ydb/core/cms/console/console_configs_manager.cpp

@@ -979,6 +979,7 @@ void TConfigsManager::HandleUnauthorized(TEvConsole::TEvReplaceYamlConfigRequest
     AuditLogReplaceConfigTransaction(
         /* peer = */ ev->Get()->Record.GetPeerName(),
         /* userSID = */ ev->Get()->Record.GetUserToken(),
+        /* sanitizedToken = */ TString(),
         /* oldConfig = */ YamlConfig,
         /* newConfig = */ ev->Get()->Record.GetRequest().config(),
         /* reason = */ "Unauthorized.",
@@ -989,6 +990,7 @@ void TConfigsManager::HandleUnauthorized(TEvConsole::TEvSetYamlConfigRequest::TP
     AuditLogReplaceConfigTransaction(
         /* peer = */ ev->Get()->Record.GetPeerName(),
         /* userSID = */ ev->Get()->Record.GetUserToken(),
+        /* sanitizedToken = */ TString(),
         /* oldConfig = */ YamlConfig,
         /* newConfig = */ ev->Get()->Record.GetRequest().config(),
         /* reason = */ "Unauthorized.",

ydb/core/grpc_services/audit_dml_operations.cpp

@@ -60,9 +60,11 @@ namespace {
 
 namespace NKikimr::NGRpcService {
 
-void AuditContextStart(IAuditCtx* ctx, const TString& database, const TString& userSID, const std::vector<std::pair<TString, TString>>& databaseAttrs) {
+void AuditContextStart(IAuditCtx* ctx, const TString& database, const TString& userSID, const TString& sanitizedToken, const std::vector<std::pair<TString, TString>>& databaseAttrs) {
     ctx->AddAuditLogPart("remote_address", NKikimr::NAddressClassifier::ExtractAddress(ctx->GetPeerName()));
     ctx->AddAuditLogPart("subject", userSID);
+    static const TString EMPTY_VALUE = "{none}";
+    ctx->AddAuditLogPart("sanitized_token", !sanitizedToken.empty() ? sanitizedToken : EMPTY_VALUE);
     ctx->AddAuditLogPart("database", database);
     ctx->AddAuditLogPart("operation", ctx->GetRequestName());
     ctx->AddAuditLogPart("start_time", TInstant::Now().ToString());

ydb/core/grpc_services/audit_dml_operations.h

@@ -38,7 +38,7 @@ class IAuditCtx;
 // AuditContextAppend() specializations extract specific info from request (and result) protos.
 //
 
-void AuditContextStart(IAuditCtx* ctx, const TString& database, const TString& userSID, const std::vector<std::pair<TString, TString>>& databaseAttrs);
+void AuditContextStart(IAuditCtx* ctx, const TString& database, const TString& userSID, const TString& sanitizedToken, const std::vector<std::pair<TString, TString>>& databaseAttrs);
 void AuditContextEnd(IAuditCtx* ctx);
 
 template <class TProtoRequest>

ydb/core/grpc_services/audit_log.cpp

@@ -9,7 +9,10 @@
 namespace NKikimr {
 namespace NGRpcService {
 
-void AuditLogConn(const IRequestProxyCtx* ctx, const TString& database, const TString& userSID)
+//NOTE: EmptyValue couldn't be an empty string as AUDIT_PART() skips parts with an empty values
+static const TString EmptyValue = "{none}";
+
+void AuditLogConn(const IRequestProxyCtx* ctx, const TString& database, const TString& userSID, const TString& sanitizedToken)
 {
     static const TString GrpcConnComponentName = "grpc-conn";
 
@@ -18,6 +21,7 @@ void AuditLogConn(const IRequestProxyCtx* ctx, const TString& database, const TS
 
         AUDIT_PART("remote_address", NKikimr::NAddressClassifier::ExtractAddress(ctx->GetPeerName()))
         AUDIT_PART("subject", userSID)
+        AUDIT_PART("sanitized_token", (!sanitizedToken.empty() ? sanitizedToken : EmptyValue))
         AUDIT_PART("database", database)
         AUDIT_PART("operation", ctx->GetRequestName())
     );
@@ -35,9 +39,6 @@ void AuditLog(ui32 status, const TAuditLogParts& parts)
 {
     static const TString GrpcProxyComponentName = "grpc-proxy";
 
-    //NOTE: EmptyValue couldn't be an empty string as AUDIT_PART() skips parts with an empty values
-    static const TString EmptyValue = "{none}";
-
     AUDIT_LOG(
         AUDIT_PART("component", GrpcProxyComponentName)
 
@@ -56,4 +57,3 @@ void AuditLog(ui32 status, const TAuditLogParts& parts)
 
 }
 }
-

ydb/core/grpc_services/audit_log.h

@@ -8,7 +8,7 @@ class IRequestProxyCtx;
 class IRequestCtxMtSafe;
 
 // grpc "connections" log
-void AuditLogConn(const IRequestProxyCtx* reqCtx, const TString& database, const TString& userSID);
+void AuditLogConn(const IRequestProxyCtx* reqCtx, const TString& database, const TString& userSID, const TString& sanitizedToken);
 
 using TAuditLogParts = TVector<std::pair<TString, TString>>;