Skip to content

Commit 6a7c86a

Browse files
jepett0jepett0
committed
Restore indexes from backup with the original partitioning (#7589)
1 parent 461d186 commit 6a7c86a

File tree

7 files changed

+181
-64
lines changed

7 files changed

+181
-64
lines changed

‎ydb/core/protos/flat_scheme_op.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1801,6 +1801,7 @@ message TDescribeOptions {
18011801
optional bool ReturnChannelsBinding = 8 [default = false];
18021802
optional bool ReturnRangeKey = 9 [default = true];
18031803
optional bool ReturnSetVal = 10 [default = false];
1804+
optional bool ReturnIndexTableBoundaries = 11 [default = false];
18041805
}
18051806

18061807
// Request to read scheme for a specific path

‎ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c
7676
opts.SetReturnPartitioningInfo(false);
7777
opts.SetReturnPartitionConfig(true);
7878
opts.SetReturnBoundaries(true);
79+
opts.SetReturnIndexTableBoundaries(true);
7980

8081
auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts);
8182
auto record = desc->GetRecord();

‎ydb/core/tx/schemeshard/schemeshard_path_describer.cpp‎

Lines changed: 81 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,78 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c
7272
FillTableMetrics(pathDescription.MutableTabletMetrics(), stats);
7373
}
7474

75+
static void FillColumns(
76+
const TTableInfo& tableInfo,
77+
google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TColumnDescription>& out
78+
) {
79+
bool familyNamesBuilt = false;
80+
THashMap<ui32, TString> familyNames;
81+
82+
out.Reserve(tableInfo.Columns.size());
83+
for (const auto& col : tableInfo.Columns) {
84+
const auto& cinfo = col.second;
85+
if (cinfo.IsDropped())
86+
continue;
87+
88+
auto* colDescr = out.Add();
89+
colDescr->SetName(cinfo.Name);
90+
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
91+
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
92+
colDescr->SetTypeId(columnType.TypeId);
93+
if (columnType.TypeInfo) {
94+
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
95+
}
96+
colDescr->SetId(cinfo.Id);
97+
colDescr->SetNotNull(cinfo.NotNull);
98+
99+
if (cinfo.Family != 0) {
100+
colDescr->SetFamily(cinfo.Family);
101+
102+
if (!familyNamesBuilt) {
103+
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
104+
if (family.HasName() && family.HasId()) {
105+
familyNames[family.GetId()] = family.GetName();
106+
}
107+
}
108+
familyNamesBuilt = true;
109+
}
110+
111+
auto it = familyNames.find(cinfo.Family);
112+
if (it != familyNames.end() && !it->second.empty()) {
113+
colDescr->SetFamilyName(it->second);
114+
}
115+
}
116+
117+
colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);
118+
119+
switch (cinfo.DefaultKind) {
120+
case ETableColumnDefaultKind::None:
121+
break;
122+
case ETableColumnDefaultKind::FromSequence:
123+
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
124+
break;
125+
case ETableColumnDefaultKind::FromLiteral:
126+
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
127+
cinfo.DefaultValue));
128+
break;
129+
}
130+
}
131+
}
132+
133+
static void FillKeyColumns(
134+
const TTableInfo& tableInfo,
135+
google::protobuf::RepeatedPtrField<TProtoStringType>& names,
136+
google::protobuf::RepeatedField<ui32>& ids
137+
) {
138+
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
139+
names.Reserve(tableInfo.KeyColumnIds.size());
140+
ids.Reserve(tableInfo.KeyColumnIds.size());
141+
for (ui32 keyColId : tableInfo.KeyColumnIds) {
142+
*names.Add() = tableInfo.Columns.at(keyColId).Name;
143+
*ids.Add() = keyColId;
144+
}
145+
}
146+
75147
void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) {
76148
FillChildDescr(descr, pathEl);
77149

@@ -292,6 +364,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
292364
bool returnBoundaries = false;
293365
bool returnRangeKey = true;
294366
bool returnSetVal = Params.GetOptions().GetReturnSetVal();
367+
bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries();
295368
if (Params.HasOptions()) {
296369
returnConfig = Params.GetOptions().GetReturnPartitionConfig();
297370
returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo();
@@ -416,7 +489,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
416489

417490
switch (childPath->PathType) {
418491
case NKikimrSchemeOp::EPathTypeTableIndex:
419-
Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes());
492+
Self->DescribeTableIndex(
493+
childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes()
494+
);
420495
break;
421496
case NKikimrSchemeOp::EPathTypeCdcStream:
422497
Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams());
@@ -1172,67 +1247,10 @@ void TSchemeShard::DescribeTable(
11721247
) const
11731248
{
11741249
Y_UNUSED(typeRegistry);
1175-
THashMap<ui32, TString> familyNames;
1176-
bool familyNamesBuilt = false;
11771250

11781251
entry->SetTableSchemaVersion(tableInfo->AlterVersion);
1179-
entry->MutableColumns()->Reserve(tableInfo->Columns.size());
1180-
for (auto col : tableInfo->Columns) {
1181-
const auto& cinfo = col.second;
1182-
if (cinfo.IsDropped())
1183-
continue;
1184-
1185-
auto colDescr = entry->AddColumns();
1186-
colDescr->SetName(cinfo.Name);
1187-
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
1188-
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
1189-
colDescr->SetTypeId(columnType.TypeId);
1190-
if (columnType.TypeInfo) {
1191-
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
1192-
}
1193-
colDescr->SetId(cinfo.Id);
1194-
colDescr->SetNotNull(cinfo.NotNull);
1195-
1196-
if (cinfo.Family != 0) {
1197-
colDescr->SetFamily(cinfo.Family);
1198-
1199-
if (!familyNamesBuilt) {
1200-
for (const auto& family : tableInfo->PartitionConfig().GetColumnFamilies()) {
1201-
if (family.HasName() && family.HasId()) {
1202-
familyNames[family.GetId()] = family.GetName();
1203-
}
1204-
}
1205-
familyNamesBuilt = true;
1206-
}
1207-
1208-
auto it = familyNames.find(cinfo.Family);
1209-
if (it != familyNames.end() && !it->second.empty()) {
1210-
colDescr->SetFamilyName(it->second);
1211-
}
1212-
}
1213-
1214-
colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);
1215-
1216-
switch (cinfo.DefaultKind) {
1217-
case ETableColumnDefaultKind::None:
1218-
break;
1219-
case ETableColumnDefaultKind::FromSequence:
1220-
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
1221-
break;
1222-
case ETableColumnDefaultKind::FromLiteral:
1223-
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
1224-
cinfo.DefaultValue));
1225-
break;
1226-
}
1227-
}
1228-
Y_ABORT_UNLESS(!tableInfo->KeyColumnIds.empty());
1229-
1230-
entry->MutableKeyColumnNames()->Reserve(tableInfo->KeyColumnIds.size());
1231-
entry->MutableKeyColumnIds()->Reserve(tableInfo->KeyColumnIds.size());
1232-
for (ui32 keyColId : tableInfo->KeyColumnIds) {
1233-
entry->AddKeyColumnNames(tableInfo->Columns[keyColId].Name);
1234-
entry->AddKeyColumnIds(keyColId);
1235-
}
1252+
FillColumns(*tableInfo, *entry->MutableColumns());
1253+
FillKeyColumns(*tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds());
12361254

12371255
if (fillConfig) {
12381256
FillPartitionConfig(tableInfo->PartitionConfig(), *entry->MutablePartitionConfig());
@@ -1296,6 +1314,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name
12961314
FillPartitionConfig(tableInfo->PartitionConfig(), *tableDescription->MutablePartitionConfig());
12971315
}
12981316
if (fillBoundaries) {
1317+
// column info is necessary for split boundary type conversion
1318+
FillColumns(*tableInfo, *tableDescription->MutableColumns());
1319+
FillKeyColumns(*tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds());
12991320
FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo);
13001321
}
13011322
}

‎ydb/core/ydb_convert/table_description.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -839,8 +839,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings,
839839
NKikimrMiniKQL::TType splitKeyType;
840840
Ydb::Table::DescribeTableResult unused;
841841
FillColumnDescription(unused, splitKeyType, indexImplTableDescription);
842-
FillTableBoundaryImpl(
843-
*settings.mutable_partition_at_keys(),
842+
FillTableBoundaryImpl<Ydb::Table::GlobalIndexSettings>(
843+
settings,
844844
indexImplTableDescription,
845845
splitKeyType
846846
);

‎ydb/public/sdk/cpp/client/ydb_table/table.cpp‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,10 @@ class TTableDescription::TImpl {
485485
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns));
486486
}
487487

488+
void AddSecondaryIndex(const TIndexDescription& indexDescription) {
489+
Indexes_.emplace_back(indexDescription);
490+
}
491+
488492
void AddChangefeed(const TString& name, EChangefeedMode mode, EChangefeedFormat format) {
489493
Changefeeds_.emplace_back(name, mode, format);
490494
}
@@ -757,6 +761,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t
757761
Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
758762
}
759763

764+
void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) {
765+
Impl_->AddSecondaryIndex(indexDescription);
766+
}
767+
760768
void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) {
761769
AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns);
762770
}
@@ -1191,6 +1199,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum
11911199
return *this;
11921200
}
11931201

1202+
TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) {
1203+
TableDescription_.AddSecondaryIndex(indexDescription);
1204+
return *this;
1205+
}
1206+
11941207
TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) {
11951208
TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
11961209
return *this;

‎ydb/public/sdk/cpp/client/ydb_table/table.h‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -183,10 +183,10 @@ struct TExplicitPartitions {
183183
using TSelf = TExplicitPartitions;
184184

185185
FLUENT_SETTING_VECTOR(TValue, SplitPoints);
186-
186+
187187
template <typename TProto>
188188
static TExplicitPartitions FromProto(const TProto& proto);
189-
189+
190190
void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
191191
};
192192

@@ -609,6 +609,7 @@ class TTableDescription {
609609
// common
610610
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
611611
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
612+
void AddSecondaryIndex(const TIndexDescription& indexDescription);
612613
// sync
613614
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns);
614615
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
@@ -820,6 +821,7 @@ class TTableBuilder {
820821
TTableBuilder& AddSerialColumn(const TString& name, const EPrimitiveType& type, TSequenceDescription sequenceDescription, const TString& family = TString());
821822

822823
// common
824+
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
823825
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
824826
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
825827
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn);

‎ydb/services/ydb/backup_ut/ydb_backup_ut.cpp‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,68 @@ void TestTableSplitBoundariesArePreserved(
249249
UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges);
250250
}
251251

252+
void TestIndexTableSplitBoundariesArePreserved(
253+
const char* table, const char* index, ui64 indexPartitions, TSession& session,
254+
TBackupFunction&& backup, TRestoreFunction&& restore
255+
) {
256+
const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable");
257+
258+
{
259+
TExplicitPartitions indexPartitionBoundaries;
260+
for (ui32 boundary : {1, 2, 4, 8, 16, 32, 64, 128, 256}) {
261+
indexPartitionBoundaries.AppendSplitPoints(
262+
// split boundary is technically always a tuple
263+
TValueBuilder().BeginTuple().AddElement().OptionalUint32(boundary).EndTuple().Build()
264+
);
265+
}
266+
// By default indexImplTable has auto partitioning by size enabled,
267+
// so you must set min partition count for partitions to not merge immediately after indexImplTable is built.
268+
TPartitioningSettingsBuilder partitioningSettingsBuilder;
269+
partitioningSettingsBuilder
270+
.SetMinPartitionsCount(indexPartitions)
271+
.SetMaxPartitionsCount(indexPartitions);
272+
273+
const auto indexSettings = TGlobalIndexSettings{
274+
.PartitioningSettings = partitioningSettingsBuilder.Build(),
275+
.Partitions = std::move(indexPartitionBoundaries)
276+
};
277+
278+
auto tableBuilder = TTableBuilder()
279+
.AddNullableColumn("Key", EPrimitiveType::Uint32)
280+
.AddNullableColumn("Value", EPrimitiveType::Uint32)
281+
.SetPrimaryKeyColumn("Key")
282+
.AddSecondaryIndex(TIndexDescription("byValue", EIndexType::GlobalSync, { "Value" }, {}, { indexSettings }));
283+
284+
const auto result = session.CreateTable(table, tableBuilder.Build()).ExtractValueSync();
285+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
286+
}
287+
const auto describeSettings = TDescribeTableSettings()
288+
.WithTableStatistics(true)
289+
.WithKeyShardBoundary(true);
290+
const auto originalIndexTableDescription = GetTableDescription(
291+
session, indexTablePath, describeSettings
292+
);
293+
UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions);
294+
const auto& originalKeyRanges = originalIndexTableDescription.GetKeyRanges();
295+
UNIT_ASSERT_VALUES_EQUAL(originalKeyRanges.size(), indexPartitions);
296+
297+
backup(table);
298+
299+
ExecuteDataDefinitionQuery(session, Sprintf(R"(
300+
DROP TABLE `%s`;
301+
)", table
302+
));
303+
304+
restore(table);
305+
const auto restoredIndexTableDescription = GetTableDescription(
306+
session, indexTablePath, describeSettings
307+
);
308+
UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions);
309+
const auto& restoredKeyRanges = restoredIndexTableDescription.GetKeyRanges();
310+
UNIT_ASSERT_VALUES_EQUAL(restoredKeyRanges.size(), indexPartitions);
311+
UNIT_ASSERT_EQUAL(restoredKeyRanges, originalKeyRanges);
312+
}
313+
252314
}
253315

254316
Y_UNIT_TEST_SUITE(BackupRestore) {
@@ -354,6 +416,7 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
354416
);
355417
}
356418

419+
// TO DO: test index impl table split boundaries restoration from a backup
357420
}
358421

359422
Y_UNIT_TEST_SUITE(BackupRestoreS3) {
@@ -544,4 +607,20 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
544607
);
545608
}
546609

610+
Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) {
611+
TS3TestEnv testEnv;
612+
constexpr const char* table = "/Root/table";
613+
constexpr const char* index = "byValue";
614+
constexpr ui64 indexPartitions = 10;
615+
616+
TestIndexTableSplitBoundariesArePreserved(
617+
table,
618+
index,
619+
indexPartitions,
620+
testEnv.GetSession(),
621+
CreateBackupLambda(testEnv.GetDriver(), testEnv.GetS3Port()),
622+
CreateRestoreLambda(testEnv.GetDriver(), testEnv.GetS3Port())
623+
);
624+
}
625+
547626
}

0 commit comments

Comments
 (0)