@@ -159,11 +159,18 @@ void TColumnShard::Handle(TEvPrivate::TEvReadFinished::TPtr& ev, const TActorCon
159
159
InFlightReadsTracker.RemoveInFlightRequest (ev->Get ()->RequestCookie , index);
160
160
161
161
ui64 txId = ev->Get ()->TxId ;
162
+ bool success = ev->Get ()->Success ;
162
163
if (ScanTxInFlight.contains (txId)) {
163
164
TDuration duration = TAppData::TimeProvider->Now () - ScanTxInFlight[txId];
164
165
IncCounter (COUNTER_SCAN_LATENCY, duration);
165
166
ScanTxInFlight.erase (txId);
166
167
SetCounter (COUNTER_SCAN_IN_FLY, ScanTxInFlight.size ());
168
+ IncCounter (COUNTER_IMMEDIATE_TX_COMPLETED);
169
+ if (success) {
170
+ IncCounter (COUNTER_READ_SUCCESS);
171
+ } else {
172
+ IncCounter (COUNTER_READ_FAIL);
173
+ }
167
174
}
168
175
}
169
176
@@ -308,36 +315,103 @@ void TColumnShard::UpdateResourceMetrics(const TActorContext& ctx, const TUsage&
308
315
metrics->TryUpdate (ctx);
309
316
}
310
317
311
- void TColumnShard::ConfigureStats ( const NOlap::TColumnEngineStats& indexStats,
312
- ::NKikimrTableStats::TTableStats* tabletStats ) {
313
- NOlap::TSnapshot lastIndexUpdate = TablesManager. GetPrimaryIndexSafe (). LastUpdate () ;
314
- auto activeIndexStats = indexStats. Active (); // data stats excluding inactive and evicted
318
+ std::optional< TColumnShard::TTableStatsCollection> TColumnShard::CollectTableStats () const {
319
+ if (!TablesManager. HasPrimaryIndex () ) {
320
+ return std::nullopt ;
321
+ }
315
322
316
- if (activeIndexStats.Rows < 0 || activeIndexStats.Bytes < 0 ) {
317
- LOG_S_WARN (" Negative stats counter. Rows: " << activeIndexStats.Rows << " Bytes: " << activeIndexStats.Bytes
318
- << TabletID ());
323
+ const TMap<ui64, std::shared_ptr<NOlap::TColumnEngineStats>>& columnEngineStats =
324
+ TablesManager.GetPrimaryIndexSafe ().GetStats ();
325
+ TTableStatsCollection resultStats;
326
+
327
+ for (const auto & [pathId, tableInfo] : TablesManager.GetTables ()) {
328
+ TColumnTableStats& tableStats = resultStats.StatsByPathId [pathId];
329
+ tableStats.AccessTime = tableInfo.GetLastAccessTime ();
330
+ tableStats.UpdateTime = tableInfo.GetLastUpdateTime ();
331
+ tableStats.LastFullCompaction = BackgroundController.GetLastCompactionFinishInstant (pathId);
332
+
333
+ auto findEngineStats = columnEngineStats.FindPtr (pathId);
334
+ if (findEngineStats && *findEngineStats) {
335
+ NOlap::TColumnEngineStats::TPortionsStats portionsStats =
336
+ (*findEngineStats)->Active (); // data stats excluding inactive and evicted
337
+
338
+ if (portionsStats.Rows < 0 || portionsStats.Bytes < 0 ) {
339
+ LOG_S_WARN (
340
+ " Negative stats counter. Rows: " << portionsStats.Rows << " Bytes: " << portionsStats.Bytes
341
+ << " Portions: " << portionsStats.Portions
342
+ << " Tablet: " << TabletID ()
343
+ );
344
+
345
+ portionsStats.Rows = (portionsStats.Rows < 0 ) ? 0 : portionsStats.Rows ;
346
+ portionsStats.Bytes = (portionsStats.Bytes < 0 ) ? 0 : portionsStats.Bytes ;
347
+ portionsStats.Portions = (portionsStats.Portions < 0 ) ? 0 : portionsStats.Portions ;
348
+ }
349
+
350
+ // TODO: count rows and bytes of data stored in InsertTable
351
+ // TODO: we need row/dataSize counters for evicted data (managed by tablet but stored outside)
352
+ tableStats.RowCount = portionsStats.Rows ;
353
+ tableStats.DataSize = portionsStats.Bytes ;
354
+ tableStats.Portions = portionsStats.Portions ;
355
+ } else {
356
+ LOG_S_ERROR (" CollectTableStats: missing column engine stats for pathId " << pathId);
357
+ }
319
358
320
- activeIndexStats.Rows = (activeIndexStats.Rows < 0 ) ? 0 : activeIndexStats.Rows ;
321
- activeIndexStats.Bytes = (activeIndexStats.Bytes < 0 ) ? 0 : activeIndexStats.Bytes ;
359
+ if (resultStats.TotalStats .AccessTime < tableStats.AccessTime ) {
360
+ resultStats.TotalStats .AccessTime = tableStats.AccessTime ;
361
+ }
362
+ if (resultStats.TotalStats .UpdateTime < tableStats.UpdateTime ) {
363
+ resultStats.TotalStats .UpdateTime = tableStats.UpdateTime ;
364
+ }
365
+ if (resultStats.TotalStats .LastFullCompaction < tableStats.LastFullCompaction ) {
366
+ resultStats.TotalStats .LastFullCompaction = tableStats.LastFullCompaction ;
367
+ }
368
+ resultStats.TotalStats .RowCount += tableStats.RowCount ;
369
+ resultStats.TotalStats .DataSize += tableStats.DataSize ;
322
370
}
323
371
324
- tabletStats->SetRowCount (activeIndexStats.Rows );
325
- tabletStats->SetDataSize (activeIndexStats.Bytes + TabletCounters->Simple ()[COUNTER_COMMITTED_BYTES].Get ());
372
+ return resultStats;
373
+ }
374
+
375
+ void TColumnShard::ConfigureStats (const TColumnTableStats& inputStats, ::NKikimrTableStats::TTableStats* outputStats) {
376
+ Y_ABORT_UNLESS (outputStats);
377
+
378
+ outputStats->SetRowCount (inputStats.RowCount );
379
+ outputStats->SetDataSize (inputStats.DataSize );
380
+
381
+ // tabletStats->SetIndexSize(...); // Not implemented
382
+ // tabletStats->SetInMemSize(...); // Not implemented
326
383
327
- // TODO: we need row/dataSize counters for evicted data (managed by tablet but stored outside)
328
- // tabletStats->SetIndexSize(); // TODO: calc size of internal tables
384
+ outputStats-> SetLastAccessTime (inputStats. AccessTime . MilliSeconds ());
385
+ outputStats-> SetLastUpdateTime (inputStats. UpdateTime . MilliSeconds ());
329
386
330
- tabletStats->SetLastAccessTime (LastAccessTime.MilliSeconds ());
331
- tabletStats->SetLastUpdateTime (lastIndexUpdate.GetPlanStep ());
387
+ outputStats->SetRowUpdates (TabletCounters->Cumulative ()[COUNTER_WRITE_SUCCESS].Get ());
388
+ outputStats->SetRowDeletes (0 ); // manual deletes are not supported
389
+ outputStats->SetRowReads (0 ); // all reads are range reads
390
+ outputStats->SetRangeReads (TabletCounters->Cumulative ()[COUNTER_READ_SUCCESS].Get ());
391
+ outputStats->SetRangeReadRows (TabletCounters->Cumulative ()[COUNTER_READ_INDEX_ROWS].Get ());
392
+
393
+ outputStats->SetPartCount (inputStats.Portions );
394
+ // outputStats->SetSearchHeight(...); // Not implemented
395
+
396
+ outputStats->SetLastFullCompactionTs (inputStats.LastFullCompaction .Seconds ());
397
+ outputStats->SetHasLoanedParts (Executor ()->HasLoanedParts ());
332
398
}
333
399
334
400
void TColumnShard::FillTxTableStats (::NKikimrTableStats::TTableStats* tableStats) const {
401
+ Y_ABORT_UNLESS (tableStats);
402
+ tableStats->SetImmediateTxCompleted (TabletCounters->Cumulative ()[COUNTER_IMMEDIATE_TX_COMPLETED].Get ());
335
403
tableStats->SetTxRejectedByOverload (TabletCounters->Cumulative ()[COUNTER_WRITE_OVERLOAD].Get ());
336
404
tableStats->SetTxRejectedBySpace (TabletCounters->Cumulative ()[COUNTER_OUT_OF_SPACE].Get ());
405
+ tableStats->SetPlannedTxCompleted (TabletCounters->Cumulative ()[COUNTER_PLANNED_TX_COMPLETED].Get ());
406
+ tableStats->SetTxCompleteLagMsec (GetTxCompleteLag ().MilliSeconds ());
337
407
tableStats->SetInFlightTxCount (Executor ()->GetStats ().TxInFly );
338
408
}
339
409
340
- void TColumnShard::FillOlapStats (const TActorContext& ctx, std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
410
+ void TColumnShard::FillOlapStats (
411
+ const TActorContext& ctx,
412
+ const std::optional<TTableStatsCollection>& tableStats,
413
+ std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev
414
+ ) {
341
415
ev->Record .SetShardState (2 ); // NKikimrTxDataShard.EDatashardState.Ready
342
416
ev->Record .SetGeneration (Executor ()->Generation ());
343
417
ev->Record .SetRound (StatsReportRound++);
@@ -346,30 +420,27 @@ void TColumnShard::FillOlapStats(const TActorContext& ctx, std::unique_ptr<TEvDa
346
420
if (auto * resourceMetrics = Executor ()->GetResourceMetrics ()) {
347
421
resourceMetrics->Fill (*ev->Record .MutableTabletMetrics ());
348
422
}
349
- auto * tabletStats = ev->Record .MutableTableStats ();
350
- FillTxTableStats (tabletStats);
351
- if (TablesManager.HasPrimaryIndex ()) {
352
- const auto & indexStats = TablesManager.MutablePrimaryIndex ().GetTotalStats ();
353
- ConfigureStats (indexStats, tabletStats);
423
+ auto * outputTableStats = ev->Record .MutableTableStats ();
424
+ FillTxTableStats (outputTableStats);
425
+ if (tableStats) {
426
+ ConfigureStats (tableStats->TotalStats , outputTableStats);
354
427
}
355
428
}
356
429
357
- void TColumnShard::FillColumnTableStats (const TActorContext& ctx,
358
- std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev) {
359
- if (!TablesManager.HasPrimaryIndex ()) {
430
+ void TColumnShard::FillColumnTableStats (
431
+ const TActorContext& ctx,
432
+ const std::optional<TTableStatsCollection>& tableStats,
433
+ std::unique_ptr<TEvDataShard::TEvPeriodicTableStats>& ev
434
+ ) {
435
+ if (!tableStats) {
360
436
return ;
361
437
}
362
- const auto & tablesIndexStats = TablesManager.MutablePrimaryIndex ().GetStats ();
363
- LOG_S_DEBUG (" There are stats for " << tablesIndexStats.size () << " tables" );
364
- for (const auto & [tableLocalID, columnStats] : tablesIndexStats) {
365
- if (!columnStats) {
366
- LOG_S_ERROR (" SendPeriodicStats: empty stats" );
367
- continue ;
368
- }
369
438
439
+ LOG_S_DEBUG (" There are stats for " << tableStats->StatsByPathId .size () << " tables" );
440
+ for (const auto & [pathId, columnStats] : tableStats->StatsByPathId ) {
370
441
auto * periodicTableStats = ev->Record .AddTables ();
371
442
periodicTableStats->SetDatashardId (TabletID ());
372
- periodicTableStats->SetTableLocalId (tableLocalID );
443
+ periodicTableStats->SetTableLocalId (pathId );
373
444
374
445
periodicTableStats->SetShardState (2 ); // NKikimrTxDataShard.EDatashardState.Ready
375
446
periodicTableStats->SetGeneration (Executor ()->Generation ());
@@ -381,11 +452,11 @@ void TColumnShard::FillColumnTableStats(const TActorContext& ctx,
381
452
resourceMetrics->Fill (*periodicTableStats->MutableTabletMetrics ());
382
453
}
383
454
384
- auto * tableStats = periodicTableStats->MutableTableStats ();
385
- FillTxTableStats (tableStats );
386
- ConfigureStats (* columnStats, tableStats );
455
+ auto * outputTableStats = periodicTableStats->MutableTableStats ();
456
+ FillTxTableStats (outputTableStats );
457
+ ConfigureStats (columnStats, outputTableStats );
387
458
388
- LOG_S_TRACE (" Add stats for table, tableLocalID=" << tableLocalID );
459
+ LOG_S_TRACE (" Add stats for table, tableLocalID=" << pathId );
389
460
}
390
461
}
391
462
@@ -412,10 +483,11 @@ void TColumnShard::SendPeriodicStats() {
412
483
StatsReportPipe = ctx.Register (NTabletPipe::CreateClient (ctx.SelfID , CurrentSchemeShardId, clientConfig));
413
484
}
414
485
486
+ std::optional<TTableStatsCollection> aggregatedStats = CollectTableStats ();
415
487
auto ev = std::make_unique<TEvDataShard::TEvPeriodicTableStats>(TabletID (), OwnerPathId);
416
488
417
- FillOlapStats (ctx, ev);
418
- FillColumnTableStats (ctx, ev);
489
+ FillOlapStats (ctx, aggregatedStats, ev);
490
+ FillColumnTableStats (ctx, aggregatedStats, ev);
419
491
420
492
NTabletPipe::SendData (ctx, StatsReportPipe, ev.release ());
421
493
}
0 commit comments