Skip to content

Commit e7d4e27

Browse files
ivanmorozov333ivanmorozov333
authored
add test for GC usage on insert/delete (#6726)
1 parent a5c62d1 commit e7d4e27

File tree

8 files changed

+68
-7
lines changed

8 files changed

+68
-7
lines changed

‎.github/config/muted_ya.txt‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ ydb/core/external_sources *
88
ydb/core/quoter/ut QuoterWithKesusTest.PrefetchCoefficient
99
ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
1010
ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
11-
ydb/core/kqp/ut/olap KqpOlapIndexes.IndexesActualization
1211
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
1312
ydb/core/kqp/ut/olap KqpOlap.ScanQueryOltpAndOlap
1413
ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL

‎ydb/core/kqp/ut/olap/blobs_sharing_ut.cpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
111111
}
112112

113113
void WaitNormalization() {
114+
Controller->SetReadTimeoutClean(TDuration::Seconds(1));
114115
Controller->SetCompactionControl(NYDBTest::EOptimizerCompactionWeightControl::Force);
115116
const auto start = TInstant::Now();
116117
while (!Controller->IsTrivialLinks() && TInstant::Now() - start < TDuration::Seconds(30)) {
@@ -119,9 +120,11 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
119120
}
120121
AFL_VERIFY(Controller->IsTrivialLinks());
121122
Controller->CheckInvariants();
123+
Controller->SetReadTimeoutClean(TDuration::Minutes(5));
122124
}
123125

124126
void Execute(const ui64 destinationIdx, const std::vector<ui64>& sourceIdxs, const bool move, const NOlap::TSnapshot& snapshot, const std::set<ui64>& pathIdxs) {
127+
Controller->SetReadTimeoutClean(TDuration::Seconds(1));
125128
AFL_VERIFY(destinationIdx < ShardIds.size());
126129
const ui64 destination = ShardIds[destinationIdx];
127130
std::vector<ui64> sources;
@@ -189,6 +192,7 @@ Y_UNIT_TEST_SUITE(KqpOlapBlobsSharing) {
189192
CSTransferStatus->Reset();
190193
AFL_VERIFY(!Controller->IsTrivialLinks());
191194
Controller->CheckInvariants();
195+
Controller->SetReadTimeoutClean(TDuration::Minutes(5));
192196
}
193197
};
194198
Y_UNIT_TEST(BlobsSharingSplit1_1) {

‎ydb/core/kqp/ut/olap/write_ut.cpp‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -131,6 +131,63 @@ Y_UNIT_TEST_SUITE(KqpOlapWrite) {
131131
UNIT_ASSERT_VALUES_EQUAL(GetUint64(rows[0].at("count")), 800000);
132132
}
133133

134+
Y_UNIT_TEST(WriteDeleteCleanGC) {
135+
auto csController = NKikimr::NYDBTest::TControllers::RegisterCSControllerGuard<NKikimr::NYDBTest::NColumnShard::TController>();
136+
csController->SetPeriodicWakeupActivationPeriod(TDuration::MilliSeconds(100));
137+
csController->DisableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC);
138+
Singleton<NKikimr::NWrappers::NExternalStorage::TFakeExternalStorage>()->ResetWriteCounters();
139+
140+
NKikimrConfig::TAppConfig appConfig;
141+
appConfig.MutableTableServiceConfig()->SetEnableOlapSink(true);
142+
143+
auto settings = TKikimrSettings().SetAppConfig(appConfig).SetWithSampleTables(false);
144+
TKikimrRunner kikimr(settings);
145+
TLocalHelper(kikimr).CreateTestOlapTable();
146+
Tests::NCommon::TLoggerInit(kikimr).SetComponents({ NKikimrServices::TX_COLUMNSHARD, NKikimrServices::TX_COLUMNSHARD_BLOBS }, "CS").SetPriority(NActors::NLog::PRI_DEBUG).Initialize();
147+
auto tableClient = kikimr.GetTableClient();
148+
149+
auto client = kikimr.GetQueryClient();
150+
151+
{
152+
auto it = client.ExecuteQuery(R"(
153+
INSERT INTO `/Root/olapStore/olapTable` (timestamp, uid, resource_id) VALUES (Timestamp('1970-01-01T00:00:00Z'), 'a', '0');
154+
INSERT INTO `/Root/olapStore/olapTable` (timestamp, uid, resource_id) VALUES (Timestamp('1970-01-01T00:00:01Z'), 'a', 'test');
155+
INSERT INTO `/Root/olapStore/olapTable` (timestamp, uid, resource_id) VALUES (Timestamp('1970-01-01T00:00:02Z'), 'a', 't');
156+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
157+
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
158+
}
159+
160+
while (csController->GetInsertStartedCounter().Val() == 0) {
161+
Cerr << "Wait indexation..." << Endl;
162+
Sleep(TDuration::Seconds(2));
163+
}
164+
{
165+
const TInstant start = TInstant::Now();
166+
while (!Singleton<NWrappers::NExternalStorage::TFakeExternalStorage>()->GetSize() && TInstant::Now() - start < TDuration::Seconds(10)) {
167+
Cerr << "Wait size in memory... " << Singleton<NWrappers::NExternalStorage::TFakeExternalStorage>()->GetSize() << Endl;
168+
Sleep(TDuration::Seconds(2));
169+
}
170+
AFL_VERIFY(Singleton<NWrappers::NExternalStorage::TFakeExternalStorage>()->GetSize());
171+
}
172+
{
173+
auto it = client.ExecuteQuery(R"(
174+
DELETE FROM `/Root/olapStore/olapTable` ON SELECT CAST(0u AS Timestamp) AS timestamp, Unwrap(CAST('a' AS Utf8)) AS uid;
175+
DELETE FROM `/Root/olapStore/olapTable`;
176+
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
177+
UNIT_ASSERT_C(it.IsSuccess(), it.GetIssues().ToString());
178+
}
179+
csController->SetReadTimeoutClean(TDuration::Zero());
180+
csController->EnableBackground(NKikimr::NYDBTest::ICSController::EBackground::GC);
181+
{
182+
const TInstant start = TInstant::Now();
183+
while (Singleton<NWrappers::NExternalStorage::TFakeExternalStorage>()->GetSize() && TInstant::Now() - start < TDuration::Seconds(10)) {
184+
Cerr << "Wait empty... " << Singleton<NWrappers::NExternalStorage::TFakeExternalStorage>()->GetSize() << Endl;
185+
Sleep(TDuration::Seconds(2));
186+
}
187+
AFL_VERIFY(!Singleton<NWrappers::NExternalStorage::TFakeExternalStorage>()->GetSize());
188+
}
189+
}
190+
134191
}
135192

136193
} // namespace

‎ydb/core/tx/columnshard/blobs_action/bs/blob_manager.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -305,11 +305,12 @@ std::shared_ptr<NBlobOperations::NBlobStorage::TGCTask> TBlobManager::BuildGCTas
305305
AFL_VERIFY(!CollectGenStepInFlight);
306306
if (BlobsToKeep.IsEmpty() && BlobsToDelete.IsEmpty() && LastCollectedGenStep == TGenStep{ CurrentGen, CurrentStep }) {
307307
BlobsManagerCounters.GCCounters.SkipCollectionEmpty->Add(1);
308-
ACFL_DEBUG("event", "TBlobManager::BuildGCTask skip")("current_gen", CurrentGen)("current_step", CurrentStep);
308+
ACFL_DEBUG("event", "TBlobManager::BuildGCTask skip")("current_gen", CurrentGen)("current_step", CurrentStep)("reason", "empty");
309309
return nullptr;
310310
}
311311

312312
if (AppData()->TimeProvider->Now() - PreviousGCTime < NYDBTest::TControllers::GetColumnShardController()->GetOverridenGCPeriod(TDuration::Seconds(GC_INTERVAL_SECONDS))) {
313+
ACFL_DEBUG("event", "TBlobManager::BuildGCTask skip")("current_gen", CurrentGen)("current_step", CurrentStep)("reason", "too_often");
313314
BlobsManagerCounters.GCCounters.SkipCollectionThrottling->Add(1);
314315
return nullptr;
315316
}

‎ydb/core/tx/columnshard/engines/changes/general_compaction.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,6 @@ void TGeneralCompactColumnEngineChanges::BuildAppendedPortionsByChunks(
174174
}
175175
batchResults = mergeStream.DrainAllParts(CheckPoints, indexFields);
176176
}
177-
Y_ABORT_UNLESS(batchResults.size());
178177

179178
std::shared_ptr<TSerializationStats> stats = std::make_shared<TSerializationStats>();
180179
for (auto&& i : SwitchedPortions) {

‎ydb/core/tx/columnshard/engines/column_engine_logs.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,8 +368,9 @@ std::shared_ptr<TCleanupPortionsColumnEngineChanges> TColumnEngineForLogs::Start
368368

369369
const TInstant snapshotInstant = snapshot.GetPlanInstant();
370370
for (auto it = CleanupPortions.begin(); !limitExceeded && it != CleanupPortions.end();) {
371-
if (it->first >= snapshotInstant) {
372-
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartCleanupStop")("snapshot", snapshot.DebugString())("current_snapshot_ts", it->first);
371+
if (it->first > snapshotInstant) {
372+
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "StartCleanupStop")("snapshot", snapshot.DebugString())(
373+
"current_snapshot_ts", it->first.MilliSeconds());
373374
break;
374375
}
375376
for (ui32 i = 0; i < it->second.size();) {

‎ydb/core/tx/columnshard/engines/portions/portion_info.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ class TPortionInfo {
197197
SetRemoveSnapshot(TSnapshot(planStep, txId));
198198
}
199199

200-
std::vector<TString> GetIndexInplaceData(const ui32 indexId) const {
200+
std::vector<TString> GetIndexInplaceDataVerified(const ui32 indexId) const {
201201
std::vector<TString> result;
202202
for (auto&& i : Indexes) {
203203
if (i.GetEntityId() == indexId) {

‎ydb/core/tx/columnshard/engines/storage/actualizer/tiering/tiering.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ std::optional<TTieringActualizer::TFullActualizationInfo> TTieringActualizer::Bu
3636
}
3737
} else {
3838
NYDBTest::TControllers::GetColumnShardController()->OnStatisticsUsage(NIndexes::TIndexMetaContainer(indexMeta));
39-
const std::vector<TString> data = portion.GetIndexInplaceData(indexMeta->GetIndexId());
39+
const std::vector<TString> data = portion.GetIndexInplaceDataVerified(indexMeta->GetIndexId());
4040
max = indexMeta->GetMaxScalarVerified(data, portionSchema->GetIndexInfo().GetColumnFieldVerified(*TieringColumnId)->type());
4141
}
4242
auto tieringInfo = Tiering->GetTierToMove(max, now);

0 commit comments

Comments
 (0)