Skip to content

Commit 781ca4f

Browse files
jepett0jepett0
committed
add split boundaries backup / restore unit tests
1 parent cccb02d commit 781ca4f

File tree

1 file changed

+227
-0
lines changed

1 file changed

+227
-0
lines changed

‎ydb/services/ydb/backup_ut/ydb_backup_ut.cpp

+227
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
#include <library/cpp/testing/unittest/registar.h>
1717

1818
#include <aws/core/Aws.h>
19+
#include <google/protobuf/util/message_differencer.h>
1920

2021
using namespace NYdb;
2122
using namespace NYdb::NTable;
@@ -90,6 +91,22 @@ void CheckTableDescription(TSession& session, const TString& path, auto&& checke
9091

9192
}
9293

94+
namespace NYdb::NTable {
95+
96+
bool operator==(const TValue& lhs, const TValue& rhs) {
97+
return google::protobuf::util::MessageDifferencer::Equals(lhs.GetProto(), rhs.GetProto());
98+
}
99+
100+
bool operator==(const TKeyBound& lhs, const TKeyBound& rhs) {
101+
return lhs.GetValue() == rhs.GetValue() && lhs.IsInclusive() == rhs.IsInclusive();
102+
}
103+
104+
bool operator==(const TKeyRange& lhs, const TKeyRange& rhs) {
105+
return lhs.From() == lhs.From() && lhs.To() == rhs.To();
106+
}
107+
108+
}
109+
93110
Y_UNIT_TEST_SUITE(BackupRestore) {
94111

95112
void Restore(NDump::TClient& client, const TFsPath& sourceFile, const TString& dbPath) {
@@ -241,6 +258,114 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
241258
CheckTableDescription(session, indexTablePath, CreateMinPartitionsChecker(minIndexPartitions, DEBUG_HINT));
242259
}
243260

261+
Y_UNIT_TEST(RestoreTableSplitBoundaries) {
262+
TKikimrWithGrpcAndRootSchema server;
263+
auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort())));
264+
TTableClient tableClient(driver);
265+
auto session = tableClient.GetSession().ExtractValueSync().GetSession();
266+
267+
constexpr const char* table = "/Root/table";
268+
constexpr int partitions = 10;
269+
270+
ExecuteDataDefinitionQuery(session, Sprintf(R"(
271+
CREATE TABLE `%s` (
272+
Key Uint32,
273+
Value Utf8,
274+
PRIMARY KEY (Key)
275+
)
276+
WITH (
277+
UNIFORM_PARTITIONS = %d,
278+
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d,
279+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = %d
280+
);
281+
)",
282+
table, partitions, partitions, partitions
283+
));
284+
const auto originalTableDescription = GetTableDescription(session, table,
285+
TDescribeTableSettings().WithTableStatistics(true)
286+
);
287+
UNIT_ASSERT_VALUES_EQUAL(originalTableDescription.GetPartitionsCount(), partitions);
288+
const auto originalKeyRanges = originalTableDescription.GetKeyRanges();
289+
290+
TTempDir tempDir;
291+
const auto& pathToBackup = tempDir.Path();
292+
// TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder
293+
NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, true, false);
294+
295+
ExecuteDataDefinitionQuery(session, Sprintf(R"(
296+
DROP TABLE `%s`;
297+
)", table
298+
));
299+
300+
// restore deleted table
301+
NDump::TClient backupClient(driver);
302+
Restore(backupClient, pathToBackup, "/Root");
303+
const auto restoredTableDescription = GetTableDescription(session, table,
304+
TDescribeTableSettings().WithTableStatistics(true)
305+
);
306+
UNIT_ASSERT_VALUES_EQUAL(restoredTableDescription.GetPartitionsCount(), partitions);
307+
UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges);
308+
}
309+
310+
Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) {
311+
TKikimrWithGrpcAndRootSchema server;
312+
auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%d", server.GetPort())));
313+
TTableClient tableClient(driver);
314+
auto session = tableClient.GetSession().ExtractValueSync().GetSession();
315+
316+
constexpr const char* table = "/Root/table";
317+
constexpr const char* textTableDescription = R"(
318+
Name: "table"
319+
Columns { Name: "Key" Type: "Uint32"}
320+
Columns { Name: "Value" Type: "Uint32"}
321+
KeyColumnNames: ["Key"]
322+
)";
323+
NKikimrSchemeOp::TTableDescription protoTableDescription;
324+
UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(textTableDescription, &protoTableDescription));
325+
326+
constexpr const char* index = "byValue";
327+
const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable");
328+
constexpr int indexPartitions = 5;
329+
330+
{
331+
TClient testClient(*server.ServerSettings);
332+
const auto status = testClient.CreateTableWithUniformShardedIndex(
333+
"/Root",
334+
protoTableDescription,
335+
index,
336+
{ "Value" },
337+
NKikimrSchemeOp::EIndexTypeGlobal,
338+
{},
339+
indexPartitions
340+
);
341+
UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::EResponseStatus::MSTATUS_OK);
342+
}
343+
const auto originalIndexTableDescription = GetTableDescription(session, indexTablePath,
344+
TDescribeTableSettings().WithTableStatistics(true)
345+
);
346+
UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions);
347+
const auto originalKeyRanges = originalIndexTableDescription.GetKeyRanges();
348+
349+
TTempDir tempDir;
350+
const auto& pathToBackup = tempDir.Path();
351+
// TO DO: implement NDump::TClient::Dump and call it instead of BackupFolder
352+
NYdb::NBackup::BackupFolder(driver, "/Root", ".", pathToBackup, {}, true, false);
353+
354+
ExecuteDataDefinitionQuery(session, Sprintf(R"(
355+
DROP TABLE `%s`;
356+
)", table
357+
));
358+
359+
// restore deleted table
360+
NDump::TClient backupClient(driver);
361+
Restore(backupClient, pathToBackup, "/Root");
362+
const auto restoredIndexTableDescription = GetTableDescription(session, indexTablePath,
363+
TDescribeTableSettings().WithTableStatistics(true)
364+
);
365+
UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions);
366+
UNIT_ASSERT_EQUAL(restoredIndexTableDescription.GetKeyRanges(), originalKeyRanges);
367+
}
368+
244369
}
245370

246371
Y_UNIT_TEST_SUITE(BackupRestoreS3) {
@@ -489,4 +614,106 @@ Y_UNIT_TEST_SUITE(BackupRestoreS3) {
489614
);
490615
}
491616

617+
Y_UNIT_TEST(RestoreTableSplitBoundaries) {
618+
TS3TestEnv testEnv;
619+
620+
constexpr const char* table = "/Root/table";
621+
constexpr int partitions = 10;
622+
623+
ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"(
624+
CREATE TABLE `%s` (
625+
Key Uint32,
626+
Value Utf8,
627+
PRIMARY KEY (Key)
628+
)
629+
WITH (
630+
UNIFORM_PARTITIONS = %d,
631+
AUTO_PARTITIONING_MIN_PARTITIONS_COUNT = %d,
632+
AUTO_PARTITIONING_MAX_PARTITIONS_COUNT = %d
633+
);
634+
)",
635+
table, partitions, partitions, partitions
636+
));
637+
const auto originalTableDescription = GetTableDescription(testEnv.GetSession(), table,
638+
TDescribeTableSettings().WithTableStatistics(true)
639+
);
640+
UNIT_ASSERT_VALUES_EQUAL(originalTableDescription.GetPartitionsCount(), partitions);
641+
const auto originalKeyRanges = originalTableDescription.GetKeyRanges();
642+
643+
NExport::TExportClient exportClient(testEnv.GetDriver());
644+
NImport::TImportClient importClient(testEnv.GetDriver());
645+
NOperation::TOperationClient operationClient(testEnv.GetDriver());
646+
647+
ExportToS3(exportClient, testEnv.GetS3Port(), operationClient, table, "table");
648+
649+
// The table needs to be dropped before importing from S3 can proceed successfully.
650+
ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"(
651+
DROP TABLE `%s`;
652+
)", table
653+
));
654+
655+
ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table);
656+
const auto restoredTableDescription = GetTableDescription(testEnv.GetSession(), table,
657+
TDescribeTableSettings().WithTableStatistics(true)
658+
);
659+
UNIT_ASSERT_VALUES_EQUAL(restoredTableDescription.GetPartitionsCount(), partitions);
660+
UNIT_ASSERT_EQUAL(restoredTableDescription.GetKeyRanges(), originalKeyRanges);
661+
}
662+
663+
Y_UNIT_TEST(RestoreIndexTableSplitBoundaries) {
664+
TS3TestEnv testEnv;
665+
666+
constexpr const char* table = "/Root/table";
667+
constexpr const char* textTableDescription = R"(
668+
Name: "table"
669+
Columns { Name: "Key" Type: "Uint32"}
670+
Columns { Name: "Value" Type: "Uint32"}
671+
KeyColumnNames: ["Key"]
672+
)";
673+
NKikimrSchemeOp::TTableDescription protoTableDescription;
674+
UNIT_ASSERT(google::protobuf::TextFormat::ParseFromString(textTableDescription, &protoTableDescription));
675+
676+
constexpr const char* index = "byValue";
677+
const TString indexTablePath = JoinFsPaths(table, index, "indexImplTable");
678+
constexpr int indexPartitions = 5;
679+
680+
{
681+
TClient testClient(*testEnv.GetServer().ServerSettings);
682+
const auto status = testClient.CreateTableWithUniformShardedIndex(
683+
"/Root",
684+
protoTableDescription,
685+
index,
686+
{ "Value" },
687+
NKikimrSchemeOp::EIndexTypeGlobal,
688+
{},
689+
indexPartitions
690+
);
691+
UNIT_ASSERT_VALUES_EQUAL(status, NMsgBusProxy::EResponseStatus::MSTATUS_OK);
692+
}
693+
const auto originalIndexTableDescription = GetTableDescription(testEnv.GetSession(), indexTablePath,
694+
TDescribeTableSettings().WithTableStatistics(true)
695+
);
696+
UNIT_ASSERT_VALUES_EQUAL(originalIndexTableDescription.GetPartitionsCount(), indexPartitions);
697+
const auto originalKeyRanges = originalIndexTableDescription.GetKeyRanges();
698+
699+
NExport::TExportClient exportClient(testEnv.GetDriver());
700+
NImport::TImportClient importClient(testEnv.GetDriver());
701+
NOperation::TOperationClient operationClient(testEnv.GetDriver());
702+
703+
ExportToS3(exportClient, testEnv.GetS3Port(), operationClient, table, "table");
704+
705+
// The table needs to be dropped before importing from S3 can proceed successfully.
706+
ExecuteDataDefinitionQuery(testEnv.GetSession(), Sprintf(R"(
707+
DROP TABLE `%s`;
708+
)", table
709+
));
710+
711+
ImportFromS3(importClient, testEnv.GetS3Port(), operationClient, "table", table);
712+
const auto restoredIndexTableDescription = GetTableDescription(testEnv.GetSession(), indexTablePath,
713+
TDescribeTableSettings().WithTableStatistics(true)
714+
);
715+
UNIT_ASSERT_VALUES_EQUAL(restoredIndexTableDescription.GetPartitionsCount(), indexPartitions);
716+
UNIT_ASSERT_EQUAL(restoredIndexTableDescription.GetKeyRanges(), originalKeyRanges);
717+
}
718+
492719
}

0 commit comments

Comments
 (0)