Skip to content

Commit f208fbb

Browse files
nsofyansofya
nsofya
and
nsofya
authored
Tables manager mem (#4692)
Co-authored-by: nsofya <nsofya@yandex.ru>
1 parent e02e0d6 commit f208fbb

File tree

3 files changed

+90
-73
lines changed

3 files changed

+90
-73
lines changed

‎ydb/core/tx/columnshard/columnshard_impl.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -378,7 +378,7 @@ void TColumnShard::RunEnsureTable(const NKikimrTxColumnShard::TCreateTable& tabl
378378
<< " ttl settings: " << tableProto.GetTtlSettings()
379379
<< " at tablet " << TabletID());
380380

381-
TTableInfo::TTableVersionInfo tableVerProto;
381+
NKikimrTxColumnShard::TTableVersionInfo tableVerProto;
382382
tableVerProto.SetPathId(pathId);
383383

384384
// check schema changed
@@ -439,7 +439,7 @@ void TColumnShard::RunAlterTable(const NKikimrTxColumnShard::TAlterTable& alterP
439439
<< " ttl settings: " << alterProto.GetTtlSettings()
440440
<< " at tablet " << TabletID());
441441

442-
TTableInfo::TTableVersionInfo tableVerProto;
442+
NKikimrTxColumnShard::TTableVersionInfo tableVerProto;
443443
if (alterProto.HasSchemaPreset()) {
444444
tableVerProto.SetSchemaPresetId(alterProto.GetSchemaPreset().GetId());
445445
TablesManager.AddSchemaVersion(alterProto.GetSchemaPreset().GetId(), version, alterProto.GetSchemaPreset().GetSchema(), db);

‎ydb/core/tx/columnshard/tables_manager.cpp‎

Lines changed: 42 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -37,13 +37,18 @@ bool TTablesManager::FillMonitoringReport(NTabletFlatExecutor::TTransactionConte
3737
}
3838
}
3939
json.InsertValue("tables_count", Tables.size());
40-
json.InsertValue("presets_count", SchemaPresets.size());
40+
json.InsertValue("presets_count", SchemaPresetsIds.size());
4141
json.InsertValue("to_drop_count", PathsToDrop.size());
4242
return true;
4343
}
4444

4545
bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
46+
using TTableVersionsInfo = TVersionedSchema<NKikimrTxColumnShard::TTableVersionInfo>;
47+
48+
THashMap<ui32, TSchemaPreset> schemaPresets;
49+
THashMap<ui32, TTableVersionsInfo> tableVersions;
4650
{
51+
TMemoryProfileGuard g("TTablesManager/InitFromDB::Tables");
4752
auto rowset = db.Table<Schema::TableInfo>().Select();
4853
if (!rowset.IsReady()) {
4954
return false;
@@ -57,6 +62,8 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
5762
if (table.IsDropped()) {
5863
PathsToDrop.insert(table.GetPathId());
5964
}
65+
66+
AFL_VERIFY(tableVersions.emplace(table.GetPathId(), TTableVersionsInfo()).second);
6067
AFL_VERIFY(Tables.emplace(table.GetPathId(), std::move(table)).second);
6168

6269
if (!rowset.Next()) {
@@ -67,6 +74,7 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
6774

6875
bool isFakePresetOnly = true;
6976
{
77+
TMemoryProfileGuard g("TTablesManager/InitFromDB::SchemaPresets");
7078
auto rowset = db.Table<Schema::SchemaPresetInfo>().Select();
7179
if (!rowset.IsReady()) {
7280
return false;
@@ -82,14 +90,16 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
8290
Y_VERIFY_S(preset.GetName() == "default", "Preset name: " + preset.GetName());
8391
isFakePresetOnly = false;
8492
}
85-
AFL_VERIFY(SchemaPresets.emplace(preset.GetId(), preset).second);
93+
AFL_VERIFY(schemaPresets.emplace(preset.GetId(), preset).second);
94+
AFL_VERIFY(SchemaPresetsIds.emplace(preset.GetId()).second);
8695
if (!rowset.Next()) {
8796
return false;
8897
}
8998
}
9099
}
91100

92101
{
102+
TMemoryProfileGuard g("TTablesManager/InitFromDB::Versions");
93103
auto rowset = db.Table<Schema::TableVersionInfo>().Select();
94104
if (!rowset.IsReady()) {
95105
return false;
@@ -101,13 +111,14 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
101111
Y_ABORT_UNLESS(Tables.contains(pathId));
102112
NOlap::TSnapshot version(
103113
rowset.GetValue<Schema::TableVersionInfo::SinceStep>(),
104-
rowset.GetValue<Schema::TableVersionInfo::SinceTxId>());
114+
rowset.GetValue<Schema::TableVersionInfo::SinceTxId>());
105115

106-
auto& table = Tables.at(pathId);
107-
TTableInfo::TTableVersionInfo versionInfo;
116+
auto& table = Tables[pathId];
117+
auto& versionsInfo = tableVersions[pathId];
118+
NKikimrTxColumnShard::TTableVersionInfo versionInfo;
108119
Y_ABORT_UNLESS(versionInfo.ParseFromString(rowset.GetValue<Schema::TableVersionInfo::InfoProto>()));
109120
AFL_DEBUG(NKikimrServices::TX_COLUMNSHARD)("event", "load_table_version")("path_id", pathId)("snapshot", version)("version", versionInfo.HasSchema() ? versionInfo.GetSchema().GetVersion() : -1);
110-
Y_ABORT_UNLESS(SchemaPresets.contains(versionInfo.GetSchemaPresetId()));
121+
Y_ABORT_UNLESS(schemaPresets.contains(versionInfo.GetSchemaPresetId()));
111122

112123
if (!table.IsDropped()) {
113124
auto& ttlSettings = versionInfo.GetTtlSettings();
@@ -120,23 +131,25 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
120131
}
121132
}
122133
}
123-
table.AddVersion(version, versionInfo);
134+
table.AddVersion(version);
135+
versionsInfo.AddVersion(version, versionInfo);
124136
if (!rowset.Next()) {
125137
return false;
126138
}
127139
}
128140
}
129141

130142
{
143+
TMemoryProfileGuard g("TTablesManager/InitFromDB::PresetVersions");
131144
auto rowset = db.Table<Schema::SchemaPresetVersionInfo>().Select();
132145
if (!rowset.IsReady()) {
133146
return false;
134147
}
135148

136149
while (!rowset.EndOfSet()) {
137150
const ui32 id = rowset.GetValue<Schema::SchemaPresetVersionInfo::Id>();
138-
Y_ABORT_UNLESS(SchemaPresets.contains(id));
139-
auto& preset = SchemaPresets.at(id);
151+
Y_ABORT_UNLESS(schemaPresets.contains(id));
152+
auto& preset = schemaPresets[id];
140153
NOlap::TSnapshot version(
141154
rowset.GetValue<Schema::SchemaPresetVersionInfo::SinceStep>(),
142155
rowset.GetValue<Schema::SchemaPresetVersionInfo::SinceTxId>());
@@ -151,19 +164,20 @@ bool TTablesManager::InitFromDB(NIceDb::TNiceDb& db) {
151164
}
152165
}
153166

154-
for (const auto& [id, preset] : SchemaPresets) {
167+
TMemoryProfileGuard g("TTablesManager/InitFromDB::Other");
168+
for (const auto& [id, preset] : schemaPresets) {
155169
if (isFakePresetOnly) {
156170
Y_ABORT_UNLESS(id == 0);
157171
} else {
158172
Y_ABORT_UNLESS(id > 0);
159173
}
160-
for (const auto& [version, schemaInfo] : preset.GetVersions()) {
174+
for (const auto& [version, schemaInfo] : preset.GetVersionsById()) {
161175
if (schemaInfo.HasSchema()) {
162176
AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "index_schema")("preset_id", id)("snapshot", version)("version", schemaInfo.GetSchema().GetVersion());
163177
if (!PrimaryIndex) {
164-
PrimaryIndex = std::make_unique<NOlap::TColumnEngineForLogs>(TabletId, StoragesManager, version, schemaInfo.GetSchema());
178+
PrimaryIndex = std::make_unique<NOlap::TColumnEngineForLogs>(TabletId, StoragesManager, preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema());
165179
} else {
166-
PrimaryIndex->RegisterSchemaVersion(version, schemaInfo.GetSchema());
180+
PrimaryIndex->RegisterSchemaVersion(preset.GetMinVersionForId(schemaInfo.GetSchema().GetVersion()), schemaInfo.GetSchema());
167181
}
168182
}
169183
}
@@ -199,7 +213,7 @@ bool TTablesManager::IsReadyForWrite(const ui64 pathId) const {
199213
}
200214

201215
bool TTablesManager::HasPreset(const ui32 presetId) const {
202-
return SchemaPresets.contains(presetId);
216+
return SchemaPresetsIds.contains(presetId);
203217
}
204218

205219
const TTableInfo& TTablesManager::GetTable(const ui64 pathId) const {
@@ -211,26 +225,25 @@ ui64 TTablesManager::GetMemoryUsage() const {
211225
ui64 memory =
212226
Tables.size() * sizeof(TTableInfo) +
213227
PathsToDrop.size() * sizeof(ui64) +
214-
Ttl.PathsCount() * sizeof(TTtl::TDescription) +
215-
SchemaPresets.size() * sizeof(TSchemaPreset);
228+
Ttl.PathsCount() * sizeof(TTtl::TDescription);
216229
if (PrimaryIndex) {
217230
memory += PrimaryIndex->MemoryUsage();
218231
}
219232
return memory;
220233
}
221234

222235
void TTablesManager::DropTable(const ui64 pathId, const NOlap::TSnapshot& version, NIceDb::TNiceDb& db) {
223-
auto& table = Tables.at(pathId);
236+
AFL_VERIFY(Tables.contains(pathId));
237+
auto& table = Tables[pathId];
224238
table.SetDropVersion(version);
225239
PathsToDrop.insert(pathId);
226240
Ttl.DropPathTtl(pathId);
227241
Schema::SaveTableDropVersion(db, pathId, version.GetPlanStep(), version.GetTxId());
228242
}
229243

230244
void TTablesManager::DropPreset(const ui32 presetId, const NOlap::TSnapshot& version, NIceDb::TNiceDb& db) {
231-
auto& preset = SchemaPresets.at(presetId);
232-
Y_ABORT_UNLESS(preset.GetName() != "default", "Cannot drop the default preset");
233-
preset.SetDropVersion(version);
245+
AFL_VERIFY(SchemaPresetsIds.contains(presetId));
246+
SchemaPresetsIds.erase(presetId);
234247
Schema::SaveSchemaPresetDropVersion(db, presetId, version);
235248
}
236249

@@ -247,27 +260,24 @@ void TTablesManager::RegisterTable(TTableInfo&& table, NIceDb::TNiceDb& db) {
247260
}
248261

249262
bool TTablesManager::RegisterSchemaPreset(const TSchemaPreset& schemaPreset, NIceDb::TNiceDb& db) {
250-
if (SchemaPresets.contains(schemaPreset.GetId())) {
263+
if (SchemaPresetsIds.contains(schemaPreset.GetId())) {
251264
return false;
252265
}
266+
SchemaPresetsIds.emplace(schemaPreset.GetId());
253267
Schema::SaveSchemaPresetInfo(db, schemaPreset.GetId(), schemaPreset.GetName());
254-
SchemaPresets.emplace(schemaPreset.GetId(), schemaPreset);
255268
return true;
256269
}
257270

258271
void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapshot& version, const NKikimrSchemeOp::TColumnTableSchema& schema, NIceDb::TNiceDb& db) {
259-
Y_ABORT_UNLESS(SchemaPresets.contains(presetId));
260-
auto preset = SchemaPresets.at(presetId);
272+
Y_ABORT_UNLESS(SchemaPresetsIds.contains(presetId));
261273

262274
TSchemaPreset::TSchemaPresetVersionInfo versionInfo;
263275
versionInfo.SetId(presetId);
264276
versionInfo.SetSinceStep(version.GetPlanStep());
265277
versionInfo.SetSinceTxId(version.GetTxId());
266278
*versionInfo.MutableSchema() = schema;
267279

268-
auto& schemaPreset = SchemaPresets.at(presetId);
269280
Schema::SaveSchemaPresetVersionInfo(db, presetId, version, versionInfo);
270-
schemaPreset.AddVersion(version, versionInfo);
271281
if (versionInfo.HasSchema()) {
272282
if (!PrimaryIndex) {
273283
PrimaryIndex = std::make_unique<NOlap::TColumnEngineForLogs>(TabletId, StoragesManager, version, schema);
@@ -283,21 +293,21 @@ void TTablesManager::AddSchemaVersion(const ui32 presetId, const NOlap::TSnapsho
283293
}
284294
}
285295

286-
void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const TTableInfo::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager) {
296+
void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot& version, const NKikimrTxColumnShard::TTableVersionInfo& versionInfo, NIceDb::TNiceDb& db, std::shared_ptr<TTiersManager>& manager) {
287297
auto it = Tables.find(pathId);
288298
AFL_VERIFY(it != Tables.end());
289299
auto& table = it->second;
290300

291301
if (versionInfo.HasSchemaPresetId()) {
292-
Y_ABORT_UNLESS(SchemaPresets.contains(versionInfo.GetSchemaPresetId()));
302+
Y_ABORT_UNLESS(SchemaPresetsIds.contains(versionInfo.GetSchemaPresetId()));
293303
} else if (versionInfo.HasSchema()) {
294304
TSchemaPreset fakePreset;
295-
if (SchemaPresets.empty()) {
305+
if (SchemaPresetsIds.empty()) {
296306
TSchemaPreset fakePreset;
297307
Y_ABORT_UNLESS(RegisterSchemaPreset(fakePreset, db));
298308
AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db);
299309
} else {
300-
Y_ABORT_UNLESS(SchemaPresets.contains(fakePreset.GetId()));
310+
Y_ABORT_UNLESS(SchemaPresetsIds.contains(fakePreset.GetId()));
301311
AddSchemaVersion(fakePreset.GetId(), version, versionInfo.GetSchema(), db);
302312
}
303313
}
@@ -314,7 +324,7 @@ void TTablesManager::AddTableVersion(const ui64 pathId, const NOlap::TSnapshot&
314324
}
315325
}
316326
Schema::SaveTableVersionInfo(db, pathId, version, versionInfo);
317-
table.AddVersion(version, versionInfo);
327+
table.AddVersion(version);
318328
}
319329

320330
TTablesManager::TTablesManager(const std::shared_ptr<NOlap::IStoragesManager>& storagesManager, const ui64 tabletId)
@@ -332,7 +342,7 @@ bool TTablesManager::TryFinalizeDropPathOnExecute(NTable::TDatabase& dbTable, co
332342
const auto& itTable = Tables.find(pathId);
333343
AFL_VERIFY(itTable != Tables.end())("problem", "No schema for path")("path_id", pathId);
334344
for (auto&& tableVersion : itTable->second.GetVersions()) {
335-
NColumnShard::Schema::EraseTableVersionInfo(db, pathId, tableVersion.first);
345+
NColumnShard::Schema::EraseTableVersionInfo(db, pathId, tableVersion);
336346
}
337347
return true;
338348
}

0 commit comments

Comments
 (0)