Skip to content

Commit 1f29540

Browse files
UgnineSirdisUgnineSirdis
authored
Merge 250a6ac into 2ec8c2c 
2 parents 2ec8c2c + 250a6ac commit 1f29540

File tree

8 files changed

+335
-19
lines changed

8 files changed

+335
-19
lines changed

‎ydb/core/audit/audit_log_impl.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ void WriteLog(const TString& log, const TVector<THolder<TLogBackend>>& logBacken
7373
log.length()
7474
));
7575
} catch (const yexception& e) {
76-
LOG_W("WriteLog: unable to write audit log (error: " << e.what() << ")");
76+
LOG_E("WriteLog: unable to write audit log (error: " << e.what() << ")");
7777
}
7878
}
7979
}

‎ydb/core/log_backend/json_envelope.cpp‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
#include "json_envelope.h"
2+
3+
#include <util/charset/utf8.h>
4+
5+
namespace NKikimr {
6+
7+
const TStringBuf PLACEHOLDER = "%message%";
8+
9+
void TJsonEnvelope::Parse() {
10+
ReadJsonTree(TemplateString, &Value, true);
11+
Parse(&Value);
12+
}
13+
14+
void TJsonEnvelope::Parse(NJson::TJsonValue* value) {
15+
if (Replace) {
16+
return;
17+
}
18+
19+
switch (value->GetType()) {
20+
case NJson::JSON_STRING: {
21+
TReplace replace(value);
22+
if (replace.Parse(value->GetStringSafe())) {
23+
Replace = std::move(replace);
24+
}
25+
break;
26+
}
27+
case NJson::JSON_ARRAY: {
28+
for (NJson::TJsonValue& el : value->GetArraySafe()) {
29+
Parse(&el);
30+
if (Replace) {
31+
break;
32+
}
33+
}
34+
break;
35+
}
36+
case NJson::JSON_MAP: {
37+
for (auto& [key, el] : value->GetMapSafe()) {
38+
Parse(&el);
39+
if (Replace) {
40+
break;
41+
}
42+
}
43+
break;
44+
}
45+
default:
46+
break;
47+
}
48+
}
49+
50+
TString TJsonEnvelope::ApplyJsonEnvelope(const TStringBuf& message) {
51+
if (!IsUtf(message)) {
52+
throw std::runtime_error("Attempt to write non utf-8 string");
53+
}
54+
55+
if (Replace) {
56+
Replace->Apply(message);
57+
}
58+
59+
TStringStream ss;
60+
NJson::WriteJson(&ss, &Value, NJson::TJsonWriterConfig().SetValidateUtf8(true));
61+
ss << Endl;
62+
return ss.Str();
63+
}
64+
65+
bool TJsonEnvelope::TReplace::Parse(const TString& replace) {
66+
size_t pos = replace.find(PLACEHOLDER);
67+
if (pos == TString::npos) {
68+
return false;
69+
}
70+
71+
if (pos != 0) {
72+
Prefix = replace.substr(0, pos);
73+
}
74+
if (pos + PLACEHOLDER.size() < replace.size()) {
75+
Suffix = replace.substr(pos + PLACEHOLDER.size());
76+
}
77+
return true;
78+
}
79+
80+
void TJsonEnvelope::TReplace::Apply(const TStringBuf& message) {
81+
TString result;
82+
result.reserve(Prefix.size() + Suffix.size() + message.size());
83+
if (Prefix) {
84+
result += Prefix;
85+
}
86+
result += message;
87+
if (Suffix) {
88+
result += Suffix;
89+
}
90+
*Value = result;
91+
}
92+
93+
} // namespace NKikimr

‎ydb/core/log_backend/json_envelope.h‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#pragma once
2+
#include <library/cpp/json/json_reader.h>
3+
#include <library/cpp/json/json_writer.h>
4+
5+
#include <util/generic/string.h>
6+
7+
#include <vector>
8+
9+
namespace NKikimr {
10+
11+
class TJsonEnvelope {
12+
public:
13+
explicit TJsonEnvelope(const TString& templateString)
14+
: TemplateString(templateString)
15+
{
16+
Parse(); // can throw
17+
}
18+
19+
TJsonEnvelope() = delete;
20+
TJsonEnvelope(const TJsonEnvelope&) = delete;
21+
TJsonEnvelope(TJsonEnvelope&&) = delete;
22+
23+
TString ApplyJsonEnvelope(const TStringBuf& message);
24+
25+
private:
26+
void Parse();
27+
void Parse(NJson::TJsonValue* value);
28+
29+
private:
30+
struct TReplace {
31+
NJson::TJsonValue* Value = nullptr;
32+
TString Prefix;
33+
TString Suffix;
34+
35+
TReplace(NJson::TJsonValue* value)
36+
: Value(value)
37+
{}
38+
39+
bool Parse(const TString& replace);
40+
void Apply(const TStringBuf& message);
41+
};
42+
43+
private:
44+
TString TemplateString;
45+
NJson::TJsonValue Value;
46+
TMaybe<TReplace> Replace;
47+
};
48+
49+
} // namespace NKikimr

‎ydb/core/log_backend/json_envelope_ut.cpp‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
#include "json_envelope.h"
2+
3+
#include <library/cpp/json/json_reader.h>
4+
#include <library/cpp/json/json_writer.h>
5+
#include <library/cpp/testing/unittest/registar.h>
6+
7+
namespace NKikimr {
8+
9+
#define UNIT_ASSERT_JSONS_EQUAL(j1, j2) { \
10+
const TString js1 = (j1), js2 = (j2); \
11+
UNIT_ASSERT(!js1.empty()); \
12+
UNIT_ASSERT_C(js1.back() == '\n', js1); \
13+
NJson::TJsonValue jv1, jv2; \
14+
UNIT_ASSERT(ReadJsonTree(j1, &jv1)); \
15+
UNIT_ASSERT(ReadJsonTree(j2, &jv2)); \
16+
const TString jsn1 = NJson::WriteJson(&jv1, true, true); \
17+
const TString jsn2 = NJson::WriteJson(&jv2, true, true); \
18+
UNIT_ASSERT_VALUES_EQUAL(jsn1, jsn2); \
19+
}
20+
21+
Y_UNIT_TEST_SUITE(JsonEnvelopeTest) {
22+
Y_UNIT_TEST(Simple) {
23+
TJsonEnvelope env1(R"json({
24+
"a": "b",
25+
"m": "abc%message%def"
26+
})json");
27+
28+
UNIT_ASSERT_JSONS_EQUAL(env1.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsgdef"})json");
29+
UNIT_ASSERT_JSONS_EQUAL(env1.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyzdef"})json");
30+
31+
32+
TJsonEnvelope env2(R"json({
33+
"a": "b",
34+
"m": "%message%def"
35+
})json");
36+
37+
UNIT_ASSERT_JSONS_EQUAL(env2.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"msgdef"})json");
38+
UNIT_ASSERT_JSONS_EQUAL(env2.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"xyzdef"})json");
39+
40+
41+
TJsonEnvelope env3(R"json({
42+
"a": "b",
43+
"m": "abc%message%"
44+
})json");
45+
46+
UNIT_ASSERT_JSONS_EQUAL(env3.ApplyJsonEnvelope("msg"), R"json({"a":"b","m":"abcmsg"})json");
47+
UNIT_ASSERT_JSONS_EQUAL(env3.ApplyJsonEnvelope("xyz"), R"json({"a":"b","m":"abcxyz"})json");
48+
}
49+
50+
Y_UNIT_TEST(NoReplace) {
51+
TJsonEnvelope env(R"json({
52+
"a": "b",
53+
"x": "%y%",
54+
"subfield": {
55+
"s": "% message %",
56+
"t": "%Message%",
57+
"x": 42,
58+
"a": [
59+
42
60+
]
61+
}
62+
})json");
63+
64+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","x":"%y%","subfield":{"s":"% message %","t":"%Message%","x":42,"a":[42]}})json");
65+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","x":"%y%","subfield":{"s":"% message %","t":"%Message%","x":42,"a":[42]}})json");
66+
}
67+
68+
Y_UNIT_TEST(ArrayItem) {
69+
TJsonEnvelope env(R"json({
70+
"a": "b",
71+
"subfield": {
72+
"a": [
73+
42,
74+
"%message%",
75+
53
76+
]
77+
}
78+
})json");
79+
80+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"b","subfield":{"a":[42,"msg",53]}})json");
81+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("xyz"), R"json({"a":"b","subfield":{"a":[42,"xyz",53]}})json");
82+
}
83+
84+
Y_UNIT_TEST(Escape) {
85+
TJsonEnvelope env(R"json({
86+
"a": "%message%"
87+
})json");
88+
89+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("msg"), R"json({"a":"msg"})json");
90+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("\"\n\""), R"json({"a":"\"\n\""})json");
91+
}
92+
93+
Y_UNIT_TEST(BinaryData) {
94+
TJsonEnvelope env(R"json({
95+
"a": "%message%"
96+
})json");
97+
98+
const ui64 binaryData = 0xABCDEFFF87654321;
99+
const TStringBuf data(reinterpret_cast<const char*>(&binaryData), sizeof(binaryData));
100+
UNIT_ASSERT_EXCEPTION(env.ApplyJsonEnvelope(data), std::exception);
101+
UNIT_ASSERT_JSONS_EQUAL(env.ApplyJsonEnvelope("text"), R"json({"a":"text"})json");
102+
}
103+
}
104+
105+
} // namespace NKikimr

‎ydb/core/log_backend/log_backend.cpp‎

Lines changed: 72 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,52 @@
11
#include "log_backend.h"
2+
#include "json_envelope.h"
23
#include "log_backend_build.h"
34
#include <ydb/core/base/counters.h>
45

6+
#include <util/system/mutex.h>
7+
58
namespace NKikimr {
69

10+
class TLogBackendWithJsonEnvelope : public TLogBackend {
11+
public:
12+
TLogBackendWithJsonEnvelope(const TString& jsonEnvelope, THolder<TLogBackend> logBackend)
13+
: JsonEnvelope(jsonEnvelope)
14+
, LogBackend(std::move(logBackend))
15+
{}
16+
17+
void WriteData(const TLogRecord& rec) override {
18+
TString data;
19+
TLogRecord record = rec;
20+
with_lock (Mutex) {
21+
data = JsonEnvelope.ApplyJsonEnvelope(TStringBuf(record.Data, record.Len));
22+
}
23+
record.Data = data.data();
24+
record.Len = data.size();
25+
LogBackend->WriteData(record);
26+
}
27+
28+
void ReopenLog() override {
29+
LogBackend->ReopenLog();
30+
}
31+
32+
void ReopenLogNoFlush() override {
33+
LogBackend->ReopenLogNoFlush();
34+
}
35+
36+
ELogPriority FiltrationLevel() const override {
37+
return LogBackend->FiltrationLevel();
38+
}
39+
40+
size_t QueueSize() const override {
41+
return LogBackend->QueueSize();
42+
}
43+
44+
private:
45+
TJsonEnvelope JsonEnvelope;
46+
THolder<TLogBackend> LogBackend;
47+
TMutex Mutex;
48+
};
49+
750
TAutoPtr<TLogBackend> CreateLogBackendWithUnifiedAgent(
851
const TKikimrRunConfig& runConfig,
952
NMonitoring::TDynamicCounterPtr counters)
@@ -127,36 +170,47 @@ TAutoPtr<TLogBackend> CreateAuditLogUnifiedAgentBackend(
127170
return logBackend;
128171
}
129172

173+
THolder<TLogBackend> MaybeWrapWithJsonEnvelope(THolder<TLogBackend> logBackend, const TString& jsonEnvelope) {
174+
Y_ASSERT(logBackend);
175+
if (jsonEnvelope.empty()) {
176+
return logBackend;
177+
}
178+
179+
return MakeHolder<TLogBackendWithJsonEnvelope>(jsonEnvelope, std::move(logBackend));
180+
}
181+
130182
TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> CreateAuditLogBackends(
131183
const TKikimrRunConfig& runConfig,
132184
NMonitoring::TDynamicCounterPtr counters) {
133185
TMap<NKikimrConfig::TAuditConfig::EFormat, TVector<THolder<TLogBackend>>> logBackends;
134-
if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasStderrBackend()) {
135-
auto logBackend = NActors::CreateStderrBackend();
136-
auto format = runConfig.AppConfig.GetAuditConfig().GetStderrBackend().GetFormat();
137-
logBackends[format].push_back(std::move(logBackend));
138-
}
139186

140-
if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasFileBackend()) {
141-
auto logBackend = CreateAuditLogFileBackend(runConfig);
142-
if (logBackend) {
143-
auto format = runConfig.AppConfig.GetAuditConfig().GetFileBackend().GetFormat();
144-
logBackends[format].push_back(std::move(logBackend));
187+
if (runConfig.AppConfig.HasAuditConfig()) {
188+
const auto& auditConfig = runConfig.AppConfig.GetAuditConfig();
189+
if (auditConfig.HasStderrBackend()) {
190+
auto logBackend = NActors::CreateStderrBackend();
191+
auto format = auditConfig.GetStderrBackend().GetFormat();
192+
logBackends[format].push_back(MaybeWrapWithJsonEnvelope(std::move(logBackend), auditConfig.GetStderrBackend().GetLogJsonEnvelope()));
145193
}
146-
}
147194

148-
if (runConfig.AppConfig.HasAuditConfig() && runConfig.AppConfig.GetAuditConfig().HasUnifiedAgentBackend()) {
149-
auto logBackend = CreateAuditLogUnifiedAgentBackend(runConfig, counters);
150-
if (logBackend) {
151-
auto format = runConfig.AppConfig.GetAuditConfig().GetUnifiedAgentBackend().GetFormat();
152-
logBackends[format].push_back(std::move(logBackend));
195+
if (auditConfig.HasFileBackend()) {
196+
auto logBackend = CreateAuditLogFileBackend(runConfig);
197+
if (logBackend) {
198+
auto format = auditConfig.GetFileBackend().GetFormat();
199+
logBackends[format].push_back(MaybeWrapWithJsonEnvelope(std::move(logBackend), auditConfig.GetFileBackend().GetLogJsonEnvelope()));
200+
}
153201
}
154-
}
155202

203+
if (auditConfig.HasUnifiedAgentBackend()) {
204+
auto logBackend = CreateAuditLogUnifiedAgentBackend(runConfig, counters);
205+
if (logBackend) {
206+
auto format = auditConfig.GetUnifiedAgentBackend().GetFormat();
207+
logBackends[format].push_back(MaybeWrapWithJsonEnvelope(std::move(logBackend), auditConfig.GetUnifiedAgentBackend().GetLogJsonEnvelope()));
208+
}
209+
}
210+
}
156211

157212
return logBackends;
158213
}
159214

160215

161216
} // NKikimr
162-

‎ydb/core/log_backend/ut/ya.make‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
UNITTEST_FOR(ydb/core/log_backend)
2+
3+
SRCS(
4+
json_envelope_ut.cpp
5+
)
6+
7+
END()

0 commit comments

Comments
 (0)