tiers info initialization fix (#5060)

ivanmorozov333 · web-flow · commit 1d8a147046f8 · 2024-06-03T12:04:46.000+03:00
diff --git a/ydb/core/kqp/ut/olap/indexes_ut.cpp b/ydb/core/kqp/ut/olap/indexes_ut.cpp
@@ -136,6 +136,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
             filler(1100000, 300100000, 10000);
 
         }
+        const ui64 initCount = csController->GetActualizationRefreshSchemeCount().Val();
+        AFL_VERIFY(initCount == 3)("started_value", initCount);
 
         for (ui32 i = 0; i < 10; ++i) {
             auto alterQuery = TStringBuilder() <<
@@ -144,8 +146,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
             auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync();
             UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString());
         }
-        const ui64 startCount = csController->GetActualizationRefreshSchemeCount().Val();
-        AFL_VERIFY(startCount == 30);
+        const ui64 updatesCount = csController->GetActualizationRefreshSchemeCount().Val();
+        AFL_VERIFY(updatesCount == 30 + initCount)("after_modification", updatesCount);
 
         for (auto&& i : csController->GetShardActualIds()) {
             kikimr.GetTestServer().GetRuntime()->Send(MakePipePerNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward(
@@ -165,8 +167,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
             CompareYson(result, R"([[20000u;]])");
         }
 
-        AFL_VERIFY(startCount + 3 /*tables count*/ * 3 /*2 * normalizers + main_load*/ == 
-            (ui64)csController->GetActualizationRefreshSchemeCount().Val())("start", startCount)("count", csController->GetActualizationRefreshSchemeCount().Val());
+        AFL_VERIFY(updatesCount + 3 /*tablets count*/ * 3 /*2 * normalizers + main_load*/ ==
+            (ui64)csController->GetActualizationRefreshSchemeCount().Val())("updates", updatesCount)("count", csController->GetActualizationRefreshSchemeCount().Val());
     }
 
     Y_UNIT_TEST(Indexes) {
@@ -308,7 +310,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {
             CompareYson(result, R"([[1u;]])");
         }
 
-        AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() < 0.20 * csController->GetIndexesSkippingOnSelect().Val());
+        AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() < 0.20 * csController->GetIndexesSkippingOnSelect().Val())
+            ("approved", csController->GetIndexesApprovedOnSelect().Val())("skipped", csController->GetIndexesSkippingOnSelect().Val());
 
     }
 
diff --git a/ydb/core/tx/columnshard/columnshard_impl.cpp b/ydb/core/tx/columnshard/columnshard_impl.cpp
@@ -393,7 +393,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl
         tableVerProto.SetSchemaPresetId(preset.GetId());
 
         if (TablesManager.RegisterSchemaPreset(preset, db)) {
-            TablesManager.AddSchemaVersion(tableProto.GetSchemaPreset().GetId(), version, tableProto.GetSchemaPreset().GetSchema(), db);
+            TablesManager.AddSchemaVersion(tableProto.GetSchemaPreset().GetId(), version, tableProto.GetSchemaPreset().GetSchema(), db, Tiers);
         }
     } else {
         Y_ABORT_UNLESS(tableProto.HasSchema(), "Tables has either schema or preset");
@@ -442,7 +442,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP
     NKikimrTxColumnShard::TTableVersionInfo tableVerProto;
     if (alterProto.HasSchemaPreset()) {
         tableVerProto.SetSchemaPresetId(alterProto.GetSchemaPreset().GetId());
-        TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db);
+        TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db, Tiers);
     } else if (alterProto.HasSchema()) {
         *tableVerProto.MutableSchema() = alterProto.GetSchema();
     }
@@ -501,7 +501,7 @@ void TColumnShard::RunAlterStore(const NKikimrTxColumnShard::TAlterStore& proto,
         if (!TablesManager.HasPreset(presetProto.GetId())) {
             continue; // we don't update presets that we don't use
         }
-        TablesManager.AddSchemaVersion(presetProto.GetId(), version, presetProto.GetSchema(), db);
+        TablesManager.AddSchemaVersion(presetProto.GetId(), version, presetProto.GetSchema(), db, Tiers);
     }
 }
 
@@ -751,14 +751,15 @@ bool TColumnShard::SetupTtl(const THashMap<ui64, NOlap::TTiering>& pathTtls) {
         ACFL_DEBUG("background", "ttl")("path", i.first)("info", i.second.GetDebugString());
     }
 
-    auto actualIndexInfo = std::make_shared<NOlap::TVersionedIndex>(TablesManager.GetPrimaryIndex()->GetVersionedIndex());
     const ui64 memoryUsageLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetTieringsMemoryLimit() : ((ui64)512 * 1024 * 1024);
     std::vector<std::shared_ptr<NOlap::TTTLColumnEngineChanges>> indexChanges = TablesManager.MutablePrimaryIndex().StartTtl(eviction, DataLocksManager, memoryUsageLimit);
 
     if (indexChanges.empty()) {
         ACFL_DEBUG("background", "ttl")("skip_reason", "no_changes");
         return false;
     }
+
+    auto actualIndexInfo = std::make_shared<NOlap::TVersionedIndex>(TablesManager.GetPrimaryIndex()->GetVersionedIndex());
     for (auto&& i : indexChanges) {
         const TString externalTaskId = i->GetTaskIdentifier();
         const bool needWrites = i->NeedConstruction();
diff --git a/ydb/core/tx/columnshard/tables_manager.cpp b/ydb/core/tx/columnshard/tables_manager.cpp
@@ -269,7 +269,7 @@ bool TTablesManager::RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIc
     return true;
 }
 
-void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db) {
+void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager) {
     Y_ABORT_UNLESS(SchemaPresetsIds.contains(presetId));
 
     TSchemaPreset::TSchemaPresetVersionInfo versionInfo;
@@ -285,6 +285,9 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho
             for (auto&& i : Tables) {
                 PrimaryIndex->RegisterTable(i.first);
             }
+            if (manager->IsReady()) {
+                PrimaryIndex->OnTieringModified(manager, Ttl, {});
+            }
         } else {
             PrimaryIndex->RegisterSchemaVersion(version, schema);
         }
@@ -310,10 +313,10 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot&
         if (SchemaPresetsIds.empty()) {
             TSchemaPreset fakePreset;
             Y_ABORT_UNLESS(RegisterSchemaPreset(fakePreset, db));
-            AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db);
+            AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db, manager);
         } else {
             Y_ABORT_UNLESS(SchemaPresetsIds.contains(fakePreset.GetId()));
-            AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db);
+            AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db, manager);
         }
     }
 
diff --git a/ydb/core/tx/columnshard/tables_manager.h b/ydb/core/tx/columnshard/tables_manager.h
@@ -239,7 +239,7 @@ class TTablesManager {
     void RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db);
     bool RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIceDb::TNiceDb& db);
 
-    void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db);
+    void AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager);
     void AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager);
     bool FillMonitoringReport(NTabletFlatExecutor::TTransactionContext& txc, NJson::TJsonValue& json);
 

Original file line number	Diff line number	Diff line change
`@@ -136,6 +136,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {`
`136`	`136`	`filler(1100000, 300100000, 10000);`
`137`	`137`
`138`	`138`	`}`
	`139`	`+ const ui64 initCount = csController->GetActualizationRefreshSchemeCount().Val();`
	`140`	`+ AFL_VERIFY(initCount == 3)("started_value", initCount);`
`139`	`141`
`140`	`142`	`for (ui32 i = 0; i < 10; ++i) {`
`141`	`143`	`auto alterQuery = TStringBuilder() <<`
`@@ -144,8 +146,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {`
`144`	`146`	`auto alterResult = session.ExecuteSchemeQuery(alterQuery).GetValueSync();`
`145`	`147`	`UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString());`
`146`	`148`	`}`
`147`		`- const ui64 startCount = csController->GetActualizationRefreshSchemeCount().Val();`
`148`		`- AFL_VERIFY(startCount == 30);`
	`149`	`+ const ui64 updatesCount = csController->GetActualizationRefreshSchemeCount().Val();`
	`150`	`+ AFL_VERIFY(updatesCount == 30 + initCount)("after_modification", updatesCount);`
`149`	`151`
`150`	`152`	`for (auto&& i : csController->GetShardActualIds()) {`
`151`	`153`	`kikimr.GetTestServer().GetRuntime()->Send(MakePipePerNodeCacheID(false), NActors::TActorId(), new TEvPipeCache::TEvForward(`
`@@ -165,8 +167,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {`
`165`	`167`	`CompareYson(result, R"([[20000u;]])");`
`166`	`168`	`}`
`167`	`169`
`168`		`- AFL_VERIFY(startCount + 3 /tables count/ * 3 /2 normalizers + main_load*/ ==`
`169`		`- (ui64)csController->GetActualizationRefreshSchemeCount().Val())("start", startCount)("count", csController->GetActualizationRefreshSchemeCount().Val());`
	`170`	`+ AFL_VERIFY(updatesCount + 3 /tablets count/ * 3 /2 normalizers + main_load*/ ==`
	`171`	`+ (ui64)csController->GetActualizationRefreshSchemeCount().Val())("updates", updatesCount)("count", csController->GetActualizationRefreshSchemeCount().Val());`
`170`	`172`	`}`
`171`	`173`
`172`	`174`	`Y_UNIT_TEST(Indexes) {`
`@@ -308,7 +310,8 @@ Y_UNIT_TEST_SUITE(KqpOlapIndexes) {`
`308`	`310`	`CompareYson(result, R"([[1u;]])");`
`309`	`311`	`}`
`310`	`312`
`311`		`- AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() < 0.20 * csController->GetIndexesSkippingOnSelect().Val());`
	`313`	`+ AFL_VERIFY(csController->GetIndexesApprovedOnSelect().Val() < 0.20 * csController->GetIndexesSkippingOnSelect().Val())`
	`314`	`+ ("approved", csController->GetIndexesApprovedOnSelect().Val())("skipped", csController->GetIndexesSkippingOnSelect().Val());`
`312`	`315`
`313`	`316`	`}`
`314`	`317`
Original file line number	Diff line number	Diff line change
`@@ -393,7 +393,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl`
`393`	`393`	`tableVerProto.SetSchemaPresetId(preset.GetId());`
`394`	`394`
`395`	`395`	`if (TablesManager.RegisterSchemaPreset(preset, db)) {`
`396`		`- TablesManager.AddSchemaVersion(tableProto.GetSchemaPreset().GetId(), version, tableProto.GetSchemaPreset().GetSchema(), db);`
	`396`	`+ TablesManager.AddSchemaVersion(tableProto.GetSchemaPreset().GetId(), version, tableProto.GetSchemaPreset().GetSchema(), db, Tiers);`
`397`	`397`	`}`
`398`	`398`	`} else {`
`399`	`399`	`Y_ABORT_UNLESS(tableProto.HasSchema(), "Tables has either schema or preset");`
`@@ -442,7 +442,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP`
`442`	`442`	`NKikimrTxColumnShard::TTableVersionInfo tableVerProto;`
`443`	`443`	`if (alterProto.HasSchemaPreset()) {`
`444`	`444`	`tableVerProto.SetSchemaPresetId(alterProto.GetSchemaPreset().GetId());`
`445`		`- TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db);`
	`445`	`+ TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db, Tiers);`
`446`	`446`	`} else if (alterProto.HasSchema()) {`
`447`	`447`	`*tableVerProto.MutableSchema() = alterProto.GetSchema();`
`448`	`448`	`}`
`@@ -501,7 +501,7 @@ void TColumnShard::RunAlterStore(const NKikimrTxColumnShard::TAlterStore& proto,`
`501`	`501`	`if (!TablesManager.HasPreset(presetProto.GetId())) {`
`502`	`502`	`continue; // we don't update presets that we don't use`
`503`	`503`	`}`
`504`		`- TablesManager.AddSchemaVersion(presetProto.GetId(), version, presetProto.GetSchema(), db);`
	`504`	`+ TablesManager.AddSchemaVersion(presetProto.GetId(), version, presetProto.GetSchema(), db, Tiers);`
`505`	`505`	`}`
`506`	`506`	`}`
`507`	`507`
`@@ -751,14 +751,15 @@ bool TColumnShard::SetupTtl(const THashMap<ui64, NOlap::TTiering>& pathTtls) {`
`751`	`751`	`ACFL_DEBUG("background", "ttl")("path", i.first)("info", i.second.GetDebugString());`
`752`	`752`	`}`
`753`	`753`
`754`		`- auto actualIndexInfo = std::make_shared<NOlap::TVersionedIndex>(TablesManager.GetPrimaryIndex()->GetVersionedIndex());`
`755`	`754`	`const ui64 memoryUsageLimit = HasAppData() ? AppDataVerified().ColumnShardConfig.GetTieringsMemoryLimit() : ((ui64)512 * 1024 * 1024);`
`756`	`755`	`std::vector<std::shared_ptr<NOlap::TTTLColumnEngineChanges>> indexChanges = TablesManager.MutablePrimaryIndex().StartTtl(eviction, DataLocksManager, memoryUsageLimit);`
`757`	`756`
`758`	`757`	`if (indexChanges.empty()) {`
`759`	`758`	`ACFL_DEBUG("background", "ttl")("skip_reason", "no_changes");`
`760`	`759`	`return false;`
`761`	`760`	`}`
	`761`	`+`
	`762`	`+ auto actualIndexInfo = std::make_shared<NOlap::TVersionedIndex>(TablesManager.GetPrimaryIndex()->GetVersionedIndex());`
`762`	`763`	`for (auto&& i : indexChanges) {`
`763`	`764`	`const TString externalTaskId = i->GetTaskIdentifier();`
`764`	`765`	`const bool needWrites = i->NeedConstruction();`
Original file line number	Diff line number	Diff line change
`@@ -269,7 +269,7 @@ bool TTablesManager::RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIc`
`269`	`269`	`return true;`
`270`	`270`	`}`
`271`	`271`
`272`		`-void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db) {`
	`272`	`+void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager) {`
`273`	`273`	`Y_ABORT_UNLESS(SchemaPresetsIds.contains(presetId));`
`274`	`274`
`275`	`275`	`TSchemaPreset::TSchemaPresetVersionInfo versionInfo;`
`@@ -285,6 +285,9 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho`
`285`	`285`	`for (auto&& i : Tables) {`
`286`	`286`	`PrimaryIndex->RegisterTable(i.first);`
`287`	`287`	`}`
	`288`	`+ if (manager->IsReady()) {`
	`289`	`+ PrimaryIndex->OnTieringModified(manager, Ttl, {});`
	`290`	`+ }`
`288`	`291`	`} else {`
`289`	`292`	`PrimaryIndex->RegisterSchemaVersion(version, schema);`
`290`	`293`	`}`
`@@ -310,10 +313,10 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot&`
`310`	`313`	`if (SchemaPresetsIds.empty()) {`
`311`	`314`	`TSchemaPreset fakePreset;`
`312`	`315`	`Y_ABORT_UNLESS(RegisterSchemaPreset(fakePreset, db));`
`313`		`- AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db);`
	`316`	`+ AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db, manager);`
`314`	`317`	`} else {`
`315`	`318`	`Y_ABORT_UNLESS(SchemaPresetsIds.contains(fakePreset.GetId()));`
`316`		`- AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db);`
	`319`	`+ AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db, manager);`
`317`	`320`	`}`
`318`	`321`	`}`
`319`	`322`