Skip to content

Commit 2020867

Browse files
ivanmorozov333ivanmorozov333
authored
split logging services through huge messages volume in summary logs (#12382)
1 parent 2570d1a commit 2020867

File tree

7 files changed

+21
-18
lines changed

7 files changed

+21
-18
lines changed

‎ydb/core/tx/columnshard/columnshard.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -411,7 +411,7 @@ void TColumnShard::SendPeriodicStats() {
411411
const TInstant now = TAppData::TimeProvider->Now();
412412

413413
if (LastStatsReport + StatsReportInterval > now) {
414-
LOG_S_TRACE("Skip send periodic stats: report interavl = " << StatsReportInterval);
414+
LOG_S_TRACE("Skip send periodic stats: report interval = " << StatsReportInterval);
415415
return;
416416
}
417417
LastStatsReport = now;

‎ydb/core/tx/columnshard/columnshard__write.cpp‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ void TColumnShard::OverloadWriteFail(const EOverloadStatus overloadReason, const
4646
Y_ABORT("invalid function usage");
4747
}
4848

49-
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD)("event", "write_overload")("size", writeSize)("path_id", writeMeta.GetTableId())(
49+
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "write_overload")("size", writeSize)("path_id", writeMeta.GetTableId())(
5050
"reason", overloadReason);
5151

5252
ctx.Send(writeMeta.GetSource(), event.release(), 0, cookie);
@@ -70,17 +70,17 @@ TColumnShard::EOverloadStatus TColumnShard::CheckOverloaded(const ui64 pathId) c
7070
ui64 writesLimit = Settings.OverloadWritesInFlight;
7171
ui64 writesSizeLimit = Settings.OverloadWritesSizeInFlight;
7272
if (txLimit && Executor()->GetStats().TxInFly > txLimit) {
73-
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "shard_overload")("reason", "tx_in_fly")("sum", Executor()->GetStats().TxInFly)(
73+
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "shard_overload")("reason", "tx_in_fly")("sum", Executor()->GetStats().TxInFly)(
7474
"limit", txLimit);
7575
return EOverloadStatus::ShardTxInFly;
7676
}
7777
if (writesLimit && Counters.GetWritesMonitor()->GetWritesInFlight() > writesLimit) {
78-
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "shard_overload")("reason", "writes_in_fly")(
78+
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "shard_overload")("reason", "writes_in_fly")(
7979
"sum", Counters.GetWritesMonitor()->GetWritesInFlight())("limit", writesLimit);
8080
return EOverloadStatus::ShardWritesInFly;
8181
}
8282
if (writesSizeLimit && Counters.GetWritesMonitor()->GetWritesSizeInFlight() > writesSizeLimit) {
83-
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "shard_overload")("reason", "writes_size_in_fly")(
83+
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "shard_overload")("reason", "writes_size_in_fly")(
8484
"sum", Counters.GetWritesMonitor()->GetWritesSizeInFlight())("limit", writesSizeLimit);
8585
return EOverloadStatus::ShardWritesSizeInFly;
8686
}
@@ -90,7 +90,7 @@ TColumnShard::EOverloadStatus TColumnShard::CheckOverloaded(const ui64 pathId) c
9090
void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& ev, const TActorContext& ctx) {
9191
TMemoryProfileGuard mpg("TEvWritePortionResult");
9292
NActors::TLogContextGuard gLogging =
93-
NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvWritePortionResult");
93+
NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletID())("event", "TEvWritePortionResult");
9494
std::vector<TNoDataWrite> noDataWrites = ev->Get()->DetachNoDataWrites();
9595
for (auto&& i : noDataWrites) {
9696
Counters.GetWritesMonitor()->OnFinishWrite(i.GetDataSize(), 1);
@@ -123,7 +123,7 @@ void TColumnShard::Handle(NPrivateEvents::NWrite::TEvWritePortionResult::TPtr& e
123123

124124
void TColumnShard::Handle(TEvPrivate::TEvWriteBlobsResult::TPtr& ev, const TActorContext& ctx) {
125125
NActors::TLogContextGuard gLogging =
126-
NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvWriteBlobsResult");
126+
NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletID())("event", "TEvWriteBlobsResult");
127127

128128
auto& putResult = ev->Get()->GetPutResult();
129129
OnYellowChannels(putResult);
@@ -228,7 +228,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
228228
};
229229

230230
if (!AppDataVerified().ColumnShardConfig.GetWritingEnabled()) {
231-
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_writing")("reason", "disabled");
231+
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "skip_writing")("reason", "disabled");
232232
return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::Disabled);
233233
}
234234

@@ -245,7 +245,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
245245
{
246246
const ui64 minMemoryRead = portionsIndex.GetMinRawMemoryRead();
247247
if (NOlap::TGlobalLimits::DefaultReduceMemoryIntervalLimit < minMemoryRead) {
248-
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "overlimit")("reason", "read_raw_memory")("current", minMemoryRead)(
248+
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "overlimit")("reason", "read_raw_memory")("current", minMemoryRead)(
249249
"limit", NOlap::TGlobalLimits::DefaultReduceMemoryIntervalLimit)("table_id", writeMeta.GetTableId());
250250
return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::OverlimitReadRawMemory);
251251
}
@@ -254,7 +254,7 @@ void TColumnShard::Handle(TEvColumnShard::TEvWrite::TPtr& ev, const TActorContex
254254
{
255255
const ui64 minMemoryRead = portionsIndex.GetMinBlobMemoryRead();
256256
if (NOlap::TGlobalLimits::DefaultBlobsMemoryIntervalLimit < minMemoryRead) {
257-
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD)("event", "overlimit")("reason", "read_blob_memory")("current", minMemoryRead)(
257+
AFL_ERROR(NKikimrServices::TX_COLUMNSHARD_WRITE)("event", "overlimit")("reason", "read_blob_memory")("current", minMemoryRead)(
258258
"limit", NOlap::TGlobalLimits::DefaultBlobsMemoryIntervalLimit)("table_id", writeMeta.GetTableId());
259259
return returnFail(COUNTER_WRITE_FAIL, EWriteFailReason::OverlimitReadBlobMemory);
260260
}
@@ -466,13 +466,13 @@ class TAbortWriteTransaction: public NTabletFlatExecutor::TTransactionBase<TColu
466466
void TColumnShard::Handle(NEvents::TDataEvents::TEvWrite::TPtr& ev, const TActorContext& ctx) {
467467
TMemoryProfileGuard mpg("NEvents::TDataEvents::TEvWrite");
468468
NActors::TLogContextGuard gLogging =
469-
NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD)("tablet_id", TabletID())("event", "TEvWrite");
469+
NActors::TLogContextBuilder::Build(NKikimrServices::TX_COLUMNSHARD_WRITE)("tablet_id", TabletID())("event", "TEvWrite");
470470

471471
const auto& record = ev->Get()->Record;
472472
const auto source = ev->Sender;
473473
const auto cookie = ev->Cookie;
474474
const auto behaviourConclusion = TOperationsManager::GetBehaviour(*ev->Get());
475-
// AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("ev_write", record.DebugString());
475+
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_WRITE)("ev_write", record.DebugString());
476476
if (behaviourConclusion.IsFail()) {
477477
Counters.GetTabletCounters()->IncCounter(COUNTER_WRITE_FAIL);
478478
auto result = NEvents::TDataEvents::TEvWriteResult::BuildError(TabletID(), 0, NKikimrDataEvents::TEvWriteResult::STATUS_BAD_REQUEST,

‎ydb/core/tx/columnshard/engines/changes/compaction/sparsed/logic.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ void TSparsedMerger::TCursor::InitArrays(const ui32 position) {
160160
SparsedCursor = std::make_shared<TSparsedChunkCursor>(sparsedArray, &*CurrentOwnedArray);
161161
PlainCursor = nullptr;
162162
} else {
163+
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_COMPACTION)("event", "plain_merger");
163164
PlainCursor = make_shared<TPlainChunkCursor>(CurrentOwnedArray->GetArray(), &*CurrentOwnedArray);
164165
SparsedCursor = nullptr;
165166
}

‎ydb/core/tx/columnshard/engines/column_engine.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,14 +17,14 @@ const std::shared_ptr<arrow::Schema>& IColumnEngine::GetReplaceKey() const {
1717
ui64 IColumnEngine::GetMetadataLimit() {
1818
static const ui64 MemoryTotal = NSystemInfo::TotalMemorySize();
1919
if (!HasAppData()) {
20-
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("total", MemoryTotal);
20+
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_WRITE)("total", MemoryTotal);
2121
return MemoryTotal * 0.3;
2222
} else if (AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().HasAbsoluteValue()) {
23-
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)(
23+
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_WRITE)(
2424
"value", AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetAbsoluteValue());
2525
return AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetAbsoluteValue();
2626
} else {
27-
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("total", MemoryTotal)(
27+
AFL_TRACE(NKikimrServices::TX_COLUMNSHARD_WRITE)("total", MemoryTotal)(
2828
"kff", AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetTotalRatio());
2929
return MemoryTotal * AppDataVerified().ColumnShardConfig.GetIndexMetadataMemoryLimit().GetTotalRatio();
3030
}

‎ydb/core/tx/columnshard/engines/column_engine_logs.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -437,9 +437,9 @@ std::vector<std::shared_ptr<TTTLColumnEngineChanges>> TColumnEngineForLogs::Star
437437
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_ACTUALIZATION)("event", "StartTtl")("skip", "not_ready_tiers");
438438
}
439439
std::vector<std::shared_ptr<TTTLColumnEngineChanges>> result;
440-
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_ACTUALIZATION)("event", "StartTtl")("rw_tasks_count", context.GetTasks().size());
440+
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_ACTUALIZATION)("event", "StartTtl")("rw_tasks_count", context.GetTasks().size());
441441
for (auto&& i : context.GetTasks()) {
442-
AFL_WARN(NKikimrServices::TX_COLUMNSHARD_ACTUALIZATION)("event", "StartTtl")("rw", i.first.DebugString())("count", i.second.size());
442+
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD_ACTUALIZATION)("event", "StartTtl")("rw", i.first.DebugString())("count", i.second.size());
443443
for (auto&& t : i.second) {
444444
SignalCounters.OnActualizationTask(t.GetTask()->GetPortionsToEvictCount(), t.GetTask()->GetPortionsToRemoveSize());
445445
result.emplace_back(t.GetTask());

‎ydb/core/tx/data_events/events.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ struct TDataEvents {
9696

9797
static std::unique_ptr<TEvWriteResult> BuildError(const ui64 origin, const ui64 txId, const NKikimrDataEvents::TEvWriteResult::EStatus& status, const TString& errorMsg) {
9898
auto result = std::make_unique<TEvWriteResult>();
99-
ACFL_ERROR("event", "ev_write_error")("status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(status))("details", errorMsg)("tx_id", txId);
99+
ACFL_WARN("event", "ev_write_error")("status", NKikimrDataEvents::TEvWriteResult::EStatus_Name(status))("details", errorMsg)("tx_id", txId);
100100
result->Record.SetOrigin(origin);
101101
result->Record.SetTxId(txId);
102102
result->Record.SetStatus(status);

‎ydb/library/services/services.proto‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,8 @@ enum EServiceKikimr {
299299
CONTINUOUS_BACKUP = 804;
300300

301301
TX_COLUMNSHARD_ACTUALIZATION = 850;
302+
TX_COLUMNSHARD_COMPACTION = 851;
303+
TX_COLUMNSHARD_WRITE = 852;
302304

303305
// System views
304306
SYSTEM_VIEWS = 900;

0 commit comments

Comments
 (0)