Skip to content

Commit 4109984

Browse files
zverevgenyzverevgeny
authored
delete tx_kind_ttl (#15771)
This transaction was used in column shard unit tests only
1 parent 75d9ac2 commit 4109984

File tree

6 files changed

+17
-132
lines changed

6 files changed

+17
-132
lines changed

‎ydb/core/protos/tx_columnshard.proto‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -139,7 +139,7 @@ enum ETransactionKind {
139139
TX_KIND_NONE = 0;
140140
TX_KIND_SCHEMA = 1;
141141
TX_KIND_COMMIT = 2;
142-
TX_KIND_TTL = 3; // Immediate (not planned)
142+
reserved 3; // TX_KIND_TTL
143143
TX_KIND_DATA = 4;
144144
TX_KIND_COMMIT_WRITE = 5;
145145
TX_KIND_BACKUP = 6;
@@ -320,13 +320,6 @@ message TSchemaTxBody {
320320
optional NKikimrSchemeOp.TGranuleShardingInfo GranuleShardingInfo = 20;
321321
}
322322

323-
message TTtlTxBody {
324-
optional string TtlColumnName = 1;
325-
optional uint64 UnixTimeSeconds = 2;
326-
optional uint32 TtlPresetId = 3;
327-
optional string TtlPresetName = 4;
328-
repeated uint64 PathIds = 5;
329-
}
330323

331324
message TBlobRange {
332325
optional bytes BlobId = 1;

‎ydb/core/tx/columnshard/columnshard__propose_transaction.cpp‎

Lines changed: 1 addition & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ class TTxProposeTransaction: public NTabletFlatExecutor::TTransactionBase<TColum
2121
AFL_VERIFY(!!Ev);
2222
}
2323

24-
virtual bool Execute(TTransactionContext& txc, const TActorContext& ctx) override {
24+
virtual bool Execute(TTransactionContext& txc, const TActorContext& /* ctx */) override {
2525
txc.DB.NoMoreReadsForTx();
2626
NIceDb::TNiceDb db(txc.DB);
2727

@@ -34,14 +34,6 @@ class TTxProposeTransaction: public NTabletFlatExecutor::TTransactionBase<TColum
3434
NActors::TLogContextGuard lGuard =
3535
NActors::TLogContextBuilder::Build()("tablet_id", Self->TabletID())("tx_id", txId)("this", (ui64)this);
3636

37-
if (txKind == NKikimrTxColumnShard::TX_KIND_TTL) {
38-
auto proposeResult = ProposeTtlDeprecated(txBody);
39-
auto reply = std::make_unique<TEvColumnShard::TEvProposeTransactionResult>(
40-
Self->TabletID(), txKind, txId, proposeResult.GetStatus(), proposeResult.GetStatusMessage());
41-
ctx.Send(Ev->Sender, reply.release());
42-
return true;
43-
}
44-
4537
if (!Self->ProcessingParams && record.HasProcessingParams()) {
4638
Self->ProcessingParams.emplace().CopyFrom(record.GetProcessingParams());
4739
Schema::SaveSpecialProtoValue(db, Schema::EValueIds::ProcessingParams, *Self->ProcessingParams);
@@ -84,9 +76,6 @@ class TTxProposeTransaction: public NTabletFlatExecutor::TTransactionBase<TColum
8476

8577
virtual void Complete(const TActorContext& ctx) override {
8678
auto& record = Proto(Ev->Get());
87-
if (record.GetTxKind() == NKikimrTxColumnShard::TX_KIND_TTL) {
88-
return;
89-
}
9079
AFL_VERIFY(!!TxOperator);
9180
AFL_VERIFY(!!TxInfo);
9281
const ui64 txId = record.GetTxId();
@@ -128,55 +117,6 @@ class TTxProposeTransaction: public NTabletFlatExecutor::TTransactionBase<TColum
128117
TEvColumnShard::TEvProposeTransaction::TPtr Ev;
129118
std::shared_ptr<TTxController::ITransactionOperator> TxOperator;
130119

131-
TTxController::TProposeResult ProposeTtlDeprecated(const TString& txBody) {
132-
/// @note There's no tx guaranties now. For now TX_KIND_TTL is used to trigger TTL in tests only.
133-
/// In future we could trigger TTL outside of tablet. Then we need real tx with complete notification.
134-
// TODO: make real tx: save and progress with tablets restart support
135-
136-
NKikimrTxColumnShard::TTtlTxBody ttlBody;
137-
if (!ttlBody.ParseFromString(txBody)) {
138-
return TTxController::TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "TTL tx cannot be parsed");
139-
}
140-
141-
// If no paths trigger schema defined TTL
142-
THashMap<ui64, NOlap::TTiering> pathTtls;
143-
if (!ttlBody.GetPathIds().empty()) {
144-
auto unixTime = TInstant::Seconds(ttlBody.GetUnixTimeSeconds());
145-
if (!unixTime) {
146-
return TTxController::TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "TTL tx wrong timestamp");
147-
}
148-
149-
TString columnName = ttlBody.GetTtlColumnName();
150-
if (columnName.empty()) {
151-
return TTxController::TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "TTL tx wrong TTL column ''");
152-
}
153-
154-
if (!Self->TablesManager.HasPrimaryIndex()) {
155-
return TTxController::TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "No primary index for TTL");
156-
}
157-
158-
auto schemaSnapshot = Self->TablesManager.GetPrimaryIndexSafe().GetVersionedIndex().GetLastSchema();
159-
auto index = schemaSnapshot->GetColumnIdOptional(columnName);
160-
if (!index) {
161-
return TTxController::TProposeResult(
162-
NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "TTL tx wrong TTL column '" + columnName + "'");
163-
}
164-
auto ttlColumn = schemaSnapshot->GetFieldByColumnIdVerified(*index);
165-
166-
const TInstant now = TlsActivationContext ? AppData()->TimeProvider->Now() : TInstant::Now();
167-
for (ui64 pathId : ttlBody.GetPathIds()) {
168-
NOlap::TTiering tiering;
169-
AFL_VERIFY(tiering.Add(NOlap::TTierInfo::MakeTtl(now - unixTime, columnName)));
170-
pathTtls.emplace(pathId, std::move(tiering));
171-
}
172-
}
173-
if (!Self->SetupTtl(pathTtls)) {
174-
return TTxController::TProposeResult(NKikimrTxColumnShard::EResultStatus::SCHEMA_ERROR, "TTL not started");
175-
}
176-
Self->TablesManager.MutablePrimaryIndex().OnTieringModified(Self->TablesManager.GetTtl());
177-
178-
return TTxController::TProposeResult();
179-
}
180120
};
181121

182122
void TColumnShard::Handle(TEvColumnShard::TEvProposeTransaction::TPtr& ev, const TActorContext& ctx) {

‎ydb/core/tx/columnshard/columnshard_impl.cpp‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -986,21 +986,17 @@ void TColumnShard::SetupMetadata() {
986986
}
987987
}
988988

989-
bool TColumnShard::SetupTtl(const THashMap<ui64, NOlap::TTiering>& pathTtls) {
989+
bool TColumnShard::SetupTtl() {
990990
if (!AppDataVerified().ColumnShardConfig.GetTTLEnabled() ||
991991
!NYDBTest::TControllers::GetColumnShardController()->IsBackgroundEnabled(NYDBTest::ICSController::EBackground::TTL)) {
992992
AFL_WARN(NKikimrServices::TX_COLUMNSHARD)("event", "skip_ttl")("reason", "disabled");
993993
return false;
994994
}
995995
Counters.GetCSCounters().OnSetupTtl();
996-
THashMap<ui64, NOlap::TTiering> eviction = pathTtls;
997-
for (auto&& i : eviction) {
998-
ACFL_DEBUG("background", "ttl")("path", i.first)("info", i.second.GetDebugString());
999-
}
1000996

1001997
const ui64 memoryUsageLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetTieringsMemoryLimit() : ((ui64)512 * 1024 * 1024);
1002998
std::vector<std::shared_ptr<NOlap::TTTLColumnEngineChanges>> indexChanges =
1003-
TablesManager.MutablePrimaryIndex().StartTtl(eviction, DataLocksManager, memoryUsageLimit);
999+
TablesManager.MutablePrimaryIndex().StartTtl({}, DataLocksManager, memoryUsageLimit);
10041000

10051001
if (indexChanges.empty()) {
10061002
ACFL_DEBUG("background", "ttl")("skip_reason", "no_changes");

‎ydb/core/tx/columnshard/columnshard_impl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -608,7 +608,7 @@ class TColumnShard: public TActor<TColumnShard>, public NTabletFlatExecutor::TTa
608608
void StartCompaction(const std::shared_ptr<NPrioritiesQueue::TAllocationGuard>& guard);
609609

610610
void SetupMetadata();
611-
bool SetupTtl(const THashMap<ui64, NOlap::TTiering>& pathTtls = {});
611+
bool SetupTtl();
612612
void SetupCleanupPortions();
613613
void SetupCleanupTables();
614614
void SetupCleanupInsertTable();

‎ydb/core/tx/columnshard/test_helper/columnshard_ut_common.h‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -364,19 +364,6 @@ struct TTestSchema {
364364
return txBody;
365365
}
366366

367-
static TString TtlTxBody(const std::vector<ui64>& pathIds, TString ttlColumnName, ui64 tsSeconds) {
368-
NKikimrTxColumnShard::TTtlTxBody proto;
369-
proto.SetTtlColumnName(ttlColumnName);
370-
proto.SetUnixTimeSeconds(tsSeconds);
371-
for (auto& pathId : pathIds) {
372-
proto.AddPathIds(pathId);
373-
}
374-
375-
TString txBody;
376-
Y_PROTOBUF_SUPPRESS_NODISCARD proto.SerializeToString(&txBody);
377-
return txBody;
378-
}
379-
380367
static std::vector<TString> ExtractNames(const std::vector<NArrow::NTest::TTestColumn>& columns) {
381368
std::vector<TString> out;
382369
out.reserve(columns.size());

‎ydb/core/tx/columnshard/ut_schema/ut_columnshard_schema.cpp‎

Lines changed: 12 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -83,20 +83,6 @@ std::shared_ptr<arrow::RecordBatch> UpdateColumn(std::shared_ptr<arrow::RecordBa
8383
return arrow::RecordBatch::Make(schema, batch->num_rows(), columns);
8484
}
8585

86-
bool TriggerTTL(TTestBasicRuntime& runtime, TActorId& sender, NOlap::TSnapshot snap, const std::vector<ui64>& pathIds,
87-
ui64 tsSeconds, const TString& ttlColumnName) {
88-
TString txBody = TTestSchema::TtlTxBody(pathIds, ttlColumnName, tsSeconds);
89-
auto event = std::make_unique<TEvColumnShard::TEvProposeTransaction>(
90-
NKikimrTxColumnShard::TX_KIND_TTL, sender, snap.GetTxId(), txBody);
91-
92-
ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, event.release());
93-
auto ev = runtime.GrabEdgeEvent<TEvColumnShard::TEvProposeTransactionResult>(sender);
94-
const auto& res = ev->Get()->Record;
95-
UNIT_ASSERT_EQUAL(res.GetTxId(), snap.GetTxId());
96-
UNIT_ASSERT_EQUAL(res.GetTxKind(), NKikimrTxColumnShard::TX_KIND_TTL);
97-
return (res.GetStatus() == NKikimrTxColumnShard::SUCCESS);
98-
}
99-
10086
bool TriggerMetadata(
10187
TTestBasicRuntime& runtime, TActorId& sender, NYDBTest::TControllers::TGuard<NOlap::TWaitCompactionController>& controller) {
10288
auto isDone = [initialCounter = controller->GetTieringMetadataActualizationCount().Val(), &controller]() {
@@ -225,6 +211,8 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
225211
ui32 ttlSec = TAppData::TimeProvider->Now().Seconds(); // disable internal tll
226212
if (internal) {
227213
ttlSec -= (ts[0] + ts[1]) / 2; // enable internal ttl between ts1 and ts2
214+
} else {
215+
ttlSec -= ts[0] + ttlIncSeconds;
228216
}
229217
if (spec.HasTiers()) {
230218
spec.Tiers[0].EvictAfter = TDuration::Seconds(ttlSec);
@@ -242,6 +230,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
242230

243231
auto blobs = MakeData(ts, PORTION_ROWS, PORTION_ROWS / 2, spec.TtlColumn, ydbSchema);
244232
UNIT_ASSERT_EQUAL(blobs.size(), 2);
233+
auto lastTtlFinishedCount = csControllerGuard->GetTTLFinishedCounter().Val();
245234
for (auto& data : blobs) {
246235
std::vector<ui64> writeIds;
247236
UNIT_ASSERT(WriteData(runtime, sender, ++writeId, tableId, data, ydbSchema, true, &writeIds));
@@ -255,12 +244,8 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
255244
RebootTablet(runtime, TTestTxConfig::TxTablet0, sender);
256245
}
257246

258-
if (internal) {
259-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {}, 0, spec.TtlColumn);
260-
} else {
261-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), { tableId }, ts[0] + ttlIncSeconds, spec.TtlColumn);
262-
}
263-
while (csControllerGuard->GetTTLFinishedCounter().Val() != csControllerGuard->GetTTLStartedCounter().Val()) {
247+
ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, new TEvPrivate::TEvPeriodicWakeup(true));
248+
while (csControllerGuard->GetTTLFinishedCounter().Val() == lastTtlFinishedCount) {
264249
runtime.SimulateSleep(TDuration::Seconds(1)); // wait all finished before (ttl especially)
265250
}
266251

@@ -271,7 +256,6 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
271256
}
272257

273258
{
274-
--planStep;
275259
TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, NOlap::TSnapshot(planStep, Max<ui64>()));
276260
reader.SetReplyColumnIds(TTestSchema::GetColumnIds(TTestSchema::YdbSchema(), { spec.TtlColumn }));
277261
auto rb = reader.ReadAll();
@@ -292,18 +276,13 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
292276
if (spec.HasTiers()) {
293277
csControllerGuard->OverrideTierConfigs(runtime, sender, TTestSchema::BuildSnapshot(spec));
294278
}
295-
296-
if (internal) {
297-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {}, 0, spec.TtlColumn);
298-
} else {
299-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {tableId}, ts[1] + ttlIncSeconds, spec.TtlColumn);
300-
}
301-
while (csControllerGuard->GetTTLFinishedCounter().Val() != csControllerGuard->GetTTLStartedCounter().Val()) {
279+
lastTtlFinishedCount = csControllerGuard->GetTTLFinishedCounter().Val();
280+
ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, new TEvPrivate::TEvPeriodicWakeup(true));
281+
while (csControllerGuard->GetTTLFinishedCounter().Val() == lastTtlFinishedCount) {
302282
runtime.SimulateSleep(TDuration::Seconds(1)); // wait all finished before (ttl especially)
303283
}
304284

305285
{
306-
--planStep;
307286
TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, NOlap::TSnapshot(planStep, Max<ui64>()));
308287
auto columnIds = TTestSchema::GetColumnIds(TTestSchema::YdbSchema(), { spec.TtlColumn });
309288
columnIds.emplace_back((ui32)NOlap::IIndexInfo::ESpecialColumn::PLAN_STEP);
@@ -314,6 +293,7 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
314293
}
315294

316295
// Disable TTL
296+
lastTtlFinishedCount = csControllerGuard->GetTTLFinishedCounter().Val();
317297
auto ok = ProposeSchemaTx(runtime, sender,
318298
TTestSchema::AlterTableTxBody(tableId, 3, TTestSchema::TTableSpecials()),
319299
NOlap::TSnapshot(++planStep, ++txId));
@@ -329,17 +309,9 @@ void TestTtl(bool reboots, bool internal, TTestSchema::TTableSpecials spec = {},
329309
ProposeCommit(runtime, sender, ++txId, writeIds);
330310
PlanCommit(runtime, sender, ++planStep, txId);
331311

332-
if (internal) {
333-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {}, 0, spec.TtlColumn);
334-
} else {
335-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {tableId}, ts[0] - ttlIncSeconds, spec.TtlColumn);
336-
}
337-
while (csControllerGuard->GetTTLFinishedCounter().Val() != csControllerGuard->GetTTLStartedCounter().Val()) {
338-
runtime.SimulateSleep(TDuration::Seconds(1)); // wait all finished before (ttl especially)
339-
}
312+
ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, new TEvPrivate::TEvPeriodicWakeup(true));
340313

341314
{
342-
--planStep;
343315
TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, NOlap::TSnapshot(planStep, Max<ui64>()));
344316
reader.SetReplyColumnIds(TTestSchema::GetColumnIds(TTestSchema::YdbSchema(), { spec.TtlColumn }));
345317
auto rb = reader.ReadAll();
@@ -660,7 +632,7 @@ std::vector<std::pair<ui32, ui64>> TestTiers(bool reboots, const std::vector<TSt
660632
reader->Ack();
661633
}
662634
// Eviction
663-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {}, 0, specs[i].TtlColumn);
635+
ForwardToTablet(runtime, TTestTxConfig::TxTablet0, sender, new TEvPrivate::TEvPeriodicWakeup(true));
664636

665637
Cerr << "-- " << (hasColdEviction ? "COLD" : "HOT")
666638
<< " TIERING(" << i << ") num tiers: " << specs[i].Tiers.size() << Endl;
@@ -690,7 +662,7 @@ std::vector<std::pair<ui32, ui64>> TestTiers(bool reboots, const std::vector<TSt
690662
// Read data after eviction
691663
TString columnToRead = specs[i].TtlColumn;
692664

693-
TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, NOlap::TSnapshot(planStep - 1, Max<ui64>()));
665+
TShardReader reader(runtime, TTestTxConfig::TxTablet0, tableId, NOlap::TSnapshot(planStep, Max<ui64>()));
694666
reader.SetReplyColumnIds(TTestSchema::GetColumnIds(TTestSchema::YdbSchema(), { columnToRead }));
695667
auto rb = reader.ReadAll();
696668
if (expectedReadResult == EExpectedResult::ERROR) {
@@ -1121,9 +1093,6 @@ void TestCompaction(std::optional<ui32> numWrites = {}) {
11211093
ProposeCommit(runtime, sender, txId, writeIds);
11221094
PlanCommit(runtime, sender, planStep, txId);
11231095

1124-
if (i % 2 == 0) {
1125-
TriggerTTL(runtime, sender, NOlap::TSnapshot(++planStep, ++txId), {}, 0, spec.TtlColumn);
1126-
}
11271096
}
11281097
}
11291098

0 commit comments

Comments
 (0)