Skip to content

Commit ff9ce74

Browse files
UgnineSirdisUgnineSirdis
authored
Merge 4b49713 into 8c8ed07 
2 parents 8c8ed07 + 4b49713 commit ff9ce74

File tree

8 files changed

+285
-18
lines changed

8 files changed

+285
-18
lines changed

‎ydb/core/audit/audit_log_impl.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
7373
log.length()
7474
));
7575
} catch (const yexception& e) {
76-
LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
76+
LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
7777
}
7878
}
7979
}

‎ydb/core/log_backend/json_envelope.cpp‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#include "json_envelope.h"
2+
3+
#include <util/charset/utf8.h>
4+
5+
namespace NKikimr {
6+
7+
const TStringBuf PLACEHOLDER = "%message%";
8+
9+
void TJsonEnvelope::Parse() {
10+
ReadJsonTree(TemplateString, &Value, true);
11+
Parse(&Value);
12+
}
13+
14+
void TJsonEnvelope::Parse(NJson::TJsonValue* value) {
15+
switch (value->GetType()) {
16+
case NJson::JSON_STRING: {
17+
TReplace replace(value);
18+
if (replace.Parse(value->GetStringSafe())) {
19+
Replaces.emplace_back(std::move(replace));
20+
}
21+
break;
22+
}
23+
case NJson::JSON_ARRAY: {
24+
for (NJson::TJsonValue& el : value->GetArraySafe()) {
25+
Parse(&el);
26+
}
27+
break;
28+
}
29+
case NJson::JSON_MAP: {
30+
for (auto& [key, el] : value->GetMapSafe()) {
31+
Parse(&el);
32+
}
33+
break;
34+
}
35+
default:
36+
break;
37+
}
38+
}
39+
40+
TString TJsonEnvelope::ApplyJsonEnvelope(const TStringBuf& message) {
41+
if (!IsUtf(message)) {
42+
throw std::runtime_error("Attempt to write non utf-8 string");
43+
}
44+
45+
for (TReplace& replace : Replaces) {
46+
replace.Apply(message);
47+
}
48+
49+
TStringStream ss;
50+
NJson::WriteJson(&ss, &Value, NJson::TJsonWriterConfig().SetValidateUtf8(true));
51+
ss << Endl;
52+
return ss.Str();
53+
}
54+
55+
bool TJsonEnvelope::TReplace::Parse(const TString& replace) {
56+
size_t pos = replace.find(PLACEHOLDER);
57+
if (pos == TString::npos) {
58+
return false;
59+
}
60+
61+
if (pos != 0) {
62+
ReplaceSequence.emplace_back(replace.substr(0, pos));
63+
}
64+
while (pos != TString::npos) {
65+
ReplaceSequence.emplace_back(); // placeholder
66+
if (pos + PLACEHOLDER.size() == replace.size()) {
67+
break;
68+
}
69+
size_t next = replace.find(PLACEHOLDER, pos + PLACEHOLDER.size());
70+
ReplaceSequence.emplace_back(replace.substr(pos + PLACEHOLDER.size(), next == TString::npos ? next : next - pos - PLACEHOLDER.size()));
71+
pos = next;
72+
}
73+
return true;
74+
}
75+
76+
void TJsonEnvelope::TReplace::Apply(const TStringBuf& message) {
77+
TString result;
78+
for (const TString& replace : ReplaceSequence) {
79+
if (replace.empty()) {
80+
result += message;
81+
} else {
82+
result += replace;
83+
}
84+
}
85+
*Value = result;
86+
}
87+
88+
} // namespace NKikimr

‎ydb/core/log_backend/json_envelope.h‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
#pragma once
2+
#include <library/cpp/json/json_reader.h>
3+
#include <library/cpp/json/json_writer.h>
4+
5+
#include <util/generic/string.h>
6+
7+
#include <vector>
8+
9+
namespace NKikimr {
10+
11+
class TJsonEnvelope {
12+
public:
13+
explicit TJsonEnvelope(const TString& templateString)
14+
: TemplateString(templateString)
15+
{
16+
Parse(); // can throw
17+
}
18+
19+
TJsonEnvelope() = delete;
20+
TJsonEnvelope(const TJsonEnvelope&) = delete;
21+
TJsonEnvelope(TJsonEnvelope&&) = delete;
22+
23+
TString ApplyJsonEnvelope(const TStringBuf& message);
24+
25+
private:
26+
void Parse();
27+
void Parse(NJson::TJsonValue* value);
28+
29+
private:
30+
struct TReplace {
31+
NJson::TJsonValue* Value = nullptr;
32+
std::vector<TString> ReplaceSequence; // empty string for placeholder
33+
34+
TReplace(NJson::TJsonValue* value)
35+
: Value(value)
36+
{}
37+
38+
bool Parse(const TString& replace);
39+
void Apply(const TStringBuf& message);
40+
};
41+
42+
private:
43+
TString TemplateString;
44+
NJson::TJsonValue Value;
45+
std::vector<TReplace> Replaces;
46+
};
47+
48+
} // namespace NKikimr

‎ydb/core/log_backend/json_envelope_ut.cpp‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
#include "json_envelope.h"
2+
3+
#include <library/cpp/json/json_reader.h>
4+
#include <library/cpp/json/json_writer.h>
5+
#include <library/cpp/testing/unittest/registar.h>
6+
7+
namespace NKikimr {
8+
9+
#define UNIT_ASSERT_JSONS_EQUAL(j1, j2) { \
10+
const TString js1 = (j1), js2 = (j2); \
11+
UNIT_ASSERT(!js1.empty()); \
12+
UNIT_ASSERT_C(js1.back() == '\n', js1); \
13+
NJson::TJsonValue jv1, jv2; \
14+
UNIT_ASSERT(ReadJsonTree(j1, &jv1)); \
15+
UNIT_ASSERT(ReadJsonTree(j2, &jv2)); \
16+
const TString jsn1 = NJson::WriteJson(&jv1, true, true); \
17+
const TString jsn2 = NJson::WriteJson(&jv2, true, true); \
18+
UNIT_ASSERT_VALUES_EQUAL(jsn1, jsn2); \
19+
}
20+
21+
Y_UNIT_TEST_SUITE(JsonEnvelopeTest) {
22+
Y_UNIT_TEST(Replace) {
23+
TJsonEnvelope env(R"json({
24+
"a": "b",
25+
"m": "abc%message%def - %message%!",
26+
"subfield": {
27+
"s": "% message %",
28+
"t": "%Message%",
29+
"m": "%message%",
30+
"x": 42,
31+
"a": [
32+
42,
33+
"42: %message%"
34+
]
35+
}
36+
})json");
37+
38+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsgdef - msg!","subfield":{"s":"% message %","t":"%Message%","m":"msg","x":42,"a":[42,"42: msg"]}})json");
39+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyzdef - xyz!","subfield":{"s":"% message %","t":"%Message%","m":"xyz","x":42,"a":[42,"42: xyz"]}})json");
40+
}
41+
42+
Y_UNIT_TEST(Escape) {
43+
TJsonEnvelope env(R"json({
44+
"a": "%message%"
45+
})json");
46+
47+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"msg"})json");
48+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("\"\n\""), R"json({"a":"\"\n\""})json");
49+
}
50+
51+
Y_UNIT_TEST(BinaryData) {
52+
TJsonEnvelope env(R"json({
53+
"a": "%message%"
54+
})json");
55+
56+
const ui64 binaryData = 0xABCDEFFF87654321;
57+
const TStringBuf data(reinterpret_cast<const char*>(&binaryData), sizeof(binaryData));
58+
UNIT_ASSERT_EXCEPTION(env.ApplyJsonEnvelope(data), std::exception);
59+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("text"), R"json({"a":"text"})json");
60+
}
61+
}
62+
63+
} // namespace NKikimr

‎ydb/core/log_backend/log_backend.cpp‎

Lines changed: 70 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,52 @@
11
#include "log_backend.h"
2+
#include "json_envelope.h"
23
#include "log_backend_build.h"
34
#include <ydb/core/base/counters.h>
45

6+
#include <util/system/mutex.h>
7+
58
namespace NKikimr {
69

10+
class TLogBackendWithJsonEnvelope : public TLogBackend {
11+
public:
12+
TLogBackendWithJsonEnvelope(const TString& jsonEnvelope, THolder<TLogBackend> logBackend)
13+
: JsonEnvelope(jsonEnvelope)
14+
, LogBackend(std::move(logBackend))
15+
{}
16+
17+
void WriteData(const TLogRecord& rec) override {
18+
TString data;
19+
TLogRecord record = rec;
20+
with_lock (Mutex) {
21+
data = JsonEnvelope.ApplyJsonEnvelope(TStringBuf(record.Data, record.Len));
22+
}
23+
record.Data = data.data();
24+
record.Len = data.size();
25+
LogBackend->WriteData(record);
26+
}
27+
28+
void ReopenLog() override {
29+
LogBackend->ReopenLog();
30+
}
31+
32+
void ReopenLogNoFlush() override {
33+
LogBackend->ReopenLogNoFlush();
34+
}
35+
36+
ELogPriority FiltrationLevel() const override {
37+
return LogBackend->FiltrationLevel();
38+
}
39+
40+
size_t QueueSize() const override {
41+
return LogBackend->QueueSize();
42+
}
43+
44+
private:
45+
TJsonEnvelope JsonEnvelope;
46+
THolder<TLogBackend> LogBackend;
47+
TMutex Mutex;
48+
};
49+
750
TAutoPtr<TLogBackend> CreateLogBackendWithUnifiedAgent(
851
const TKikimrRunConfig& runConfig,
952
NMonitoring::TDynamicCounterPtr counters)
@@ -127,36 +170,46 @@ TAutoPtr<TLogBackend> CreateAuditLogUnifiedAgentBackend(
127170
return logBackend;
128171
}
129172

173+
THolder<TLogBackend> MaybeWrapWithJsonEnvelope(THolder<TLogBackend> logBackend, const TString& jsonEnvelope) {
174+
if (jsonEnvelope.empty() || !logBackend) {
175+
return logBackend;
176+
}
177+
178+
return MakeHolder<TLogBackendWithJsonEnvelope>(jsonEnvelope, std::move(logBackend));
179+
}
180+
130181
TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> CreateAuditLogBackends(
131182
const TKikimrRunConfig& runConfig,
132183
NMonitoring::TDynamicCounterPtr counters) {
133184
TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> logBackends;
134-
if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasStderrBackend()) {
135-
auto logBackend = NActors::CreateStderrBackend();
136-
auto format = runConfig.AppConfig.GetAuditConfig().GetStderrBackend().GetFormat();
137-
logBackends[format].push_back(std::move(logBackend));
138-
}
139185

140-
if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasFileBackend()) {
141-
auto logBackend = CreateAuditLogFileBackend(runConfig);
142-
if (logBackend) {
143-
auto format = runConfig.AppConfig.GetAuditConfig().GetFileBackend().GetFormat();
186+
if (runConfig.AppConfig.HasAuditConfig()) {
187+
const auto& auditConfig = runConfig.AppConfig.GetAuditConfig();
188+
if (auditConfig.HasStderrBackend()) {
189+
auto logBackend = MaybeWrapWithJsonEnvelope(NActors::CreateStderrBackend(), auditConfig.GetStderrBackend().GetLogJsonEnvelope());
190+
auto format = auditConfig.GetStderrBackend().GetFormat();
144191
logBackends[format].push_back(std::move(logBackend));
145192
}
146-
}
147193

148-
if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasUnifiedAgentBackend()) {
149-
auto logBackend = CreateAuditLogUnifiedAgentBackend(runConfig, counters);
150-
if (logBackend) {
151-
auto format = runConfig.AppConfig.GetAuditConfig().GetUnifiedAgentBackend().GetFormat();
152-
logBackends[format].push_back(std::move(logBackend));
194+
if (auditConfig.HasFileBackend()) {
195+
auto logBackend = MaybeWrapWithJsonEnvelope(CreateAuditLogFileBackend(runConfig), auditConfig.GetFileBackend().GetLogJsonEnvelope());
196+
if (logBackend) {
197+
auto format = auditConfig.GetFileBackend().GetFormat();
198+
logBackends[format].push_back(std::move(logBackend));
199+
}
153200
}
154-
}
155201

202+
if (auditConfig.HasUnifiedAgentBackend()) {
203+
auto logBackend = MaybeWrapWithJsonEnvelope(CreateAuditLogUnifiedAgentBackend(runConfig, counters), auditConfig.GetUnifiedAgentBackend().GetLogJsonEnvelope());
204+
if (logBackend) {
205+
auto format = auditConfig.GetUnifiedAgentBackend().GetFormat();
206+
logBackends[format].push_back(std::move(logBackend));
207+
}
208+
}
209+
}
156210

157211
return logBackends;
158212
}
159213

160214

161215
} // NKikimr
162-

‎ydb/core/log_backend/ut/ya.make‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
UNITTEST_FOR(ydb/core/log_backend)
2+
3+
SRCS(
4+
json_envelope_ut.cpp
5+
)
6+
7+
END()

‎ydb/core/log_backend/ya.make‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
LIBRARY()
22

33
SRCS(
4+
json_envelope.cpp
45
log_backend.cpp
56
log_backend.h
67
log_backend_build.cpp
@@ -15,3 +16,7 @@ PEERDIR(
1516
YQL_LAST_ABI_VERSION()
1617

1718
END()
19+
20+
RECURSE_FOR_TESTS(
21+
ut
22+
)

‎ydb/core/protos/config.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1455,16 +1455,19 @@ message TAuditConfig {
14551455

14561456
message TStderrBackend {
14571457
optional EFormat Format = 1 [default = JSON];
1458+
optional string LogJsonEnvelope = 2; // Json template with text field containing %message% placeholder. For example {"my_enveloped_message": "%message%"}. %message% will be replaced with real audit log message
14581459
}
14591460

14601461
message TFileBackend {
14611462
optional EFormat Format = 1 [default = JSON];
14621463
optional string FilePath = 2;
1464+
optional string LogJsonEnvelope = 3; // Json template with text field containing %message% placeholder. For example {"my_enveloped_message": "%message%"}. %message% will be replaced with real audit log message
14631465
}
14641466

14651467
message TUnifiedAgentBackend {
14661468
optional EFormat Format = 1 [default = JSON];
14671469
optional string LogName = 2;
1470+
optional string LogJsonEnvelope = 3; // Json template with text field containing %message% placeholder. For example {"my_enveloped_message": "%message%"}. %message% will be replaced with real audit log message
14681471
}
14691472

14701473
optional TStderrBackend StderrBackend = 1;

0 commit comments

Comments
 (0)