Skip to content

Commit e56caba

Browse files
jepett0jepett0
authored
Restore indexes from backup with the original partitioning (#7589)
1 parent 7f8f368 commit e56caba

File tree

7 files changed

+516
-333
lines changed

7 files changed

+516
-333
lines changed

‎ydb/core/protos/flat_scheme_op.proto

+1
Original file line numberDiff line numberDiff line change
@@ -1855,6 +1855,7 @@ message TDescribeOptions {
18551855
optional bool ReturnChannelsBinding = 8 [default = false];
18561856
optional bool ReturnRangeKey = 9 [default = true];
18571857
optional bool ReturnSetVal = 10 [default = false];
1858+
optional bool ReturnIndexTableBoundaries = 11 [default = false];
18581859
}
18591860

18601861
// Request to read scheme for a specific path

‎ydb/core/tx/schemeshard/schemeshard_export_flow_proposals.cpp

+1
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ static NKikimrSchemeOp::TPathDescription GetTableDescription(TSchemeShard* ss, c
7676
opts.SetReturnPartitioningInfo(false);
7777
opts.SetReturnPartitionConfig(true);
7878
opts.SetReturnBoundaries(true);
79+
opts.SetReturnIndexTableBoundaries(true);
7980

8081
auto desc = DescribePath(ss, TlsActivationContext->AsActorContext(), pathId, opts);
8182
auto record = desc->GetRecord();

‎ydb/core/tx/schemeshard/schemeshard_path_describer.cpp

+82-60
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,79 @@ static void FillTableStats(NKikimrSchemeOp::TPathDescription& pathDescription, c
7373
FillTableMetrics(pathDescription.MutableTabletMetrics(), stats);
7474
}
7575

76+
static void FillColumns(
77+
const TTableInfo& tableInfo,
78+
google::protobuf::RepeatedPtrField<NKikimrSchemeOp::TColumnDescription>& out
79+
) {
80+
bool familyNamesBuilt = false;
81+
THashMap<ui32, TString> familyNames;
82+
83+
out.Reserve(tableInfo.Columns.size());
84+
for (const auto& col : tableInfo.Columns) {
85+
const auto& cinfo = col.second;
86+
if (cinfo.IsDropped())
87+
continue;
88+
89+
auto* colDescr = out.Add();
90+
colDescr->SetName(cinfo.Name);
91+
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
92+
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
93+
colDescr->SetTypeId(columnType.TypeId);
94+
if (columnType.TypeInfo) {
95+
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
96+
}
97+
colDescr->SetId(cinfo.Id);
98+
colDescr->SetNotNull(cinfo.NotNull);
99+
100+
if (cinfo.Family != 0) {
101+
colDescr->SetFamily(cinfo.Family);
102+
103+
if (!familyNamesBuilt) {
104+
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
105+
if (family.HasName() && family.HasId()) {
106+
familyNames[family.GetId()] = family.GetName();
107+
}
108+
}
109+
familyNamesBuilt = true;
110+
}
111+
112+
auto it = familyNames.find(cinfo.Family);
113+
if (it != familyNames.end() && !it->second.empty()) {
114+
colDescr->SetFamilyName(it->second);
115+
}
116+
}
117+
118+
colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);
119+
120+
switch (cinfo.DefaultKind) {
121+
case ETableColumnDefaultKind::None:
122+
break;
123+
case ETableColumnDefaultKind::FromSequence:
124+
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
125+
break;
126+
case ETableColumnDefaultKind::FromLiteral:
127+
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
128+
cinfo.DefaultValue));
129+
break;
130+
}
131+
}
132+
}
133+
134+
static void FillKeyColumns(
135+
const TTableInfo& tableInfo,
136+
google::protobuf::RepeatedPtrField<TProtoStringType>& names,
137+
google::protobuf::RepeatedField<ui32>& ids
138+
) {
139+
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
140+
names.Reserve(tableInfo.KeyColumnIds.size());
141+
ids.Reserve(tableInfo.KeyColumnIds.size());
142+
for (ui32 keyColId : tableInfo.KeyColumnIds) {
143+
*names.Add() = tableInfo.Columns.at(keyColId).Name;
144+
*ids.Add() = keyColId;
145+
}
146+
147+
}
148+
76149
void TPathDescriber::FillPathDescr(NKikimrSchemeOp::TDirEntry* descr, TPathElement::TPtr pathEl, TPathElement::EPathSubType subType) {
77150
FillChildDescr(descr, pathEl);
78151

@@ -303,6 +376,7 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
303376
bool returnBoundaries = false;
304377
bool returnRangeKey = true;
305378
bool returnSetVal = Params.GetOptions().GetReturnSetVal();
379+
bool returnIndexTableBoundaries = Params.GetOptions().GetReturnIndexTableBoundaries();
306380
if (Params.HasOptions()) {
307381
returnConfig = Params.GetOptions().GetReturnPartitionConfig();
308382
returnPartitioning = Params.GetOptions().GetReturnPartitioningInfo();
@@ -427,7 +501,9 @@ void TPathDescriber::DescribeTable(const TActorContext& ctx, TPathId pathId, TPa
427501

428502
switch (childPath->PathType) {
429503
case NKikimrSchemeOp::EPathTypeTableIndex:
430-
Self->DescribeTableIndex(childPathId, childName, returnConfig, false, *entry->AddTableIndexes());
504+
Self->DescribeTableIndex(
505+
childPathId, childName, returnConfig, returnIndexTableBoundaries, *entry->AddTableIndexes()
506+
);
431507
break;
432508
case NKikimrSchemeOp::EPathTypeCdcStream:
433509
Self->DescribeCdcStream(childPathId, childName, *entry->AddCdcStreams());
@@ -1189,67 +1265,10 @@ void TSchemeShard::DescribeTable(
11891265
) const
11901266
{
11911267
Y_UNUSED(typeRegistry);
1192-
THashMap<ui32, TString> familyNames;
1193-
bool familyNamesBuilt = false;
11941268

11951269
entry->SetTableSchemaVersion(tableInfo.AlterVersion);
1196-
entry->MutableColumns()->Reserve(tableInfo.Columns.size());
1197-
for (auto col : tableInfo.Columns) {
1198-
const auto& cinfo = col.second;
1199-
if (cinfo.IsDropped())
1200-
continue;
1201-
1202-
auto colDescr = entry->AddColumns();
1203-
colDescr->SetName(cinfo.Name);
1204-
colDescr->SetType(NScheme::TypeName(cinfo.PType, cinfo.PTypeMod));
1205-
auto columnType = NScheme::ProtoColumnTypeFromTypeInfoMod(cinfo.PType, cinfo.PTypeMod);
1206-
colDescr->SetTypeId(columnType.TypeId);
1207-
if (columnType.TypeInfo) {
1208-
*colDescr->MutableTypeInfo() = *columnType.TypeInfo;
1209-
}
1210-
colDescr->SetId(cinfo.Id);
1211-
colDescr->SetNotNull(cinfo.NotNull);
1212-
1213-
if (cinfo.Family != 0) {
1214-
colDescr->SetFamily(cinfo.Family);
1215-
1216-
if (!familyNamesBuilt) {
1217-
for (const auto& family : tableInfo.PartitionConfig().GetColumnFamilies()) {
1218-
if (family.HasName() && family.HasId()) {
1219-
familyNames[family.GetId()] = family.GetName();
1220-
}
1221-
}
1222-
familyNamesBuilt = true;
1223-
}
1224-
1225-
auto it = familyNames.find(cinfo.Family);
1226-
if (it != familyNames.end() && !it->second.empty()) {
1227-
colDescr->SetFamilyName(it->second);
1228-
}
1229-
}
1230-
1231-
colDescr->SetIsBuildInProgress(cinfo.IsBuildInProgress);
1232-
1233-
switch (cinfo.DefaultKind) {
1234-
case ETableColumnDefaultKind::None:
1235-
break;
1236-
case ETableColumnDefaultKind::FromSequence:
1237-
colDescr->SetDefaultFromSequence(cinfo.DefaultValue);
1238-
break;
1239-
case ETableColumnDefaultKind::FromLiteral:
1240-
Y_ABORT_UNLESS(colDescr->MutableDefaultFromLiteral()->ParseFromString(
1241-
cinfo.DefaultValue));
1242-
break;
1243-
}
1244-
}
1245-
Y_ABORT_UNLESS(!tableInfo.KeyColumnIds.empty());
1246-
1247-
entry->MutableKeyColumnNames()->Reserve(tableInfo.KeyColumnIds.size());
1248-
entry->MutableKeyColumnIds()->Reserve(tableInfo.KeyColumnIds.size());
1249-
for (ui32 keyColId : tableInfo.KeyColumnIds) {
1250-
entry->AddKeyColumnNames(tableInfo.Columns.at(keyColId).Name);
1251-
entry->AddKeyColumnIds(keyColId);
1252-
}
1270+
FillColumns(tableInfo, *entry->MutableColumns());
1271+
FillKeyColumns(tableInfo, *entry->MutableKeyColumnNames(), *entry->MutableKeyColumnIds());
12531272

12541273
if (fillConfig) {
12551274
FillPartitionConfig(tableInfo.PartitionConfig(), *entry->MutablePartitionConfig());
@@ -1328,6 +1347,9 @@ void TSchemeShard::DescribeTableIndex(const TPathId& pathId, const TString& name
13281347
FillPartitionConfig(tableInfo.PartitionConfig(), *tableDescription->MutablePartitionConfig());
13291348
}
13301349
if (fillBoundaries) {
1350+
// column info is necessary for split boundary type conversion
1351+
FillColumns(tableInfo, *tableDescription->MutableColumns());
1352+
FillKeyColumns(tableInfo, *tableDescription->MutableKeyColumnNames(), *tableDescription->MutableKeyColumnIds());
13311353
FillTableBoundaries(tableDescription->MutableSplitBoundary(), tableInfo);
13321354
}
13331355
}

‎ydb/core/ydb_convert/table_description.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -852,8 +852,8 @@ void FillGlobalIndexSettings(Ydb::Table::GlobalIndexSettings& settings,
852852
NKikimrMiniKQL::TType splitKeyType;
853853
Ydb::Table::DescribeTableResult unused;
854854
FillColumnDescription(unused, splitKeyType, indexImplTableDescription);
855-
FillTableBoundaryImpl(
856-
*settings.mutable_partition_at_keys(),
855+
FillTableBoundaryImpl<Ydb::Table::GlobalIndexSettings>(
856+
settings,
857857
indexImplTableDescription,
858858
splitKeyType
859859
);

‎ydb/public/sdk/cpp/client/ydb_table/table.cpp

+17-4
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,10 @@ class TTableDescription::TImpl {
469469
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns));
470470
}
471471

472+
void AddSecondaryIndex(const TIndexDescription& indexDescription) {
473+
Indexes_.emplace_back(indexDescription);
474+
}
475+
472476
void AddVectorIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVectorIndexSettings& vectorIndexSettings) {
473477
Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, vectorIndexSettings));
474478
}
@@ -749,6 +753,10 @@ void TTableDescription::AddSecondaryIndex(const TString& indexName, EIndexType t
749753
Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
750754
}
751755

756+
void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) {
757+
Impl_->AddSecondaryIndex(indexDescription);
758+
}
759+
752760
void TTableDescription::AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns) {
753761
AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns);
754762
}
@@ -1173,6 +1181,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const TString& primaryKeyColum
11731181
return *this;
11741182
}
11751183

1184+
TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) {
1185+
TableDescription_.AddSecondaryIndex(indexDescription);
1186+
return *this;
1187+
}
1188+
11761189
TTableBuilder& TTableBuilder::AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns) {
11771190
TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns);
11781191
return *this;
@@ -2364,7 +2377,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) {
23642377
default:
23652378
return EVectorType::Unknown;
23662379
}
2367-
};
2380+
};
23682381

23692382

23702383
auto metricFromProto = [&](const auto& proto) -> TVectorIndexSettings::TMetric {
@@ -2376,7 +2389,7 @@ TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) {
23762389
default:
23772390
return {};
23782391
}
2379-
};
2392+
};
23802393

23812394
return {
23822395
.Metric = metricFromProto(proto),
@@ -2424,8 +2437,8 @@ void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings
24242437
return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UNSPECIFIED;
24252438
}
24262439
};
2427-
2428-
2440+
2441+
24292442
if (const auto* distance = std::get_if<EDistance>(&Metric)) {
24302443
settings.set_distance(convertDistance(*distance));
24312444
} else if (const auto* similarity = std::get_if<ESimilarity>(&Metric)) {

‎ydb/public/sdk/cpp/client/ydb_table/table.h

+4-2
Original file line numberDiff line numberDiff line change
@@ -174,10 +174,10 @@ struct TExplicitPartitions {
174174
using TSelf = TExplicitPartitions;
175175

176176
FLUENT_SETTING_VECTOR(TValue, SplitPoints);
177-
177+
178178
template <typename TProto>
179179
static TExplicitPartitions FromProto(const TProto& proto);
180-
180+
181181
void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
182182
};
183183

@@ -642,6 +642,7 @@ class TTableDescription {
642642
// common
643643
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
644644
void AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
645+
void AddSecondaryIndex(const TIndexDescription& indexDescription);
645646
// sync
646647
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns);
647648
void AddSyncSecondaryIndex(const TString& indexName, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
@@ -855,6 +856,7 @@ class TTableBuilder {
855856
TTableBuilder& SetPrimaryKeyColumn(const TString& primaryKeyColumn);
856857

857858
// common
859+
TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription);
858860
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns, const TVector<TString>& dataColumns);
859861
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TVector<TString>& indexColumns);
860862
TTableBuilder& AddSecondaryIndex(const TString& indexName, EIndexType type, const TString& indexColumn);

0 commit comments

Comments
 (0)