Skip to content

Olap TableStore read/write test #8858

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 7, 2024
Merged

Olap TableStore read/write test #8858

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a7376f4
fix
nikvas0 Sep 6, 2024
b57ff4e
fix
nikvas0 Sep 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/config/muted_ya.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ ydb/core/keyvalue/ut_trace TKeyValueTracingTest.*
ydb/core/kqp/provider/ut KikimrIcGateway.TestLoadBasicSecretValueFromExternalDataSourceMetadata
ydb/core/kqp/ut/olap KqpOlapBlobsSharing.*
ydb/core/kqp/ut/olap KqpOlapStatistics.StatsUsageWithTTL
ydb/core/kqp/ut/olap KqpOlap.TableSinkWithOlapStore
ydb/core/kqp/ut/pg KqpPg.CreateIndex
ydb/core/kqp/ut/query KqpLimits.QueryReplySize
ydb/core/kqp/ut/query KqpQuery.QueryTimeout
Expand Down
8 changes: 7 additions & 1 deletion ydb/core/kqp/ut/olap/helpers/local.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,14 @@ class TLocalHelper: public Tests::NCS::THelper {

void CreateTestOlapTable(TString tableName = "olapTable", TString storeName = "olapStore",
ui32 storeShardsCount = 4, ui32 tableShardsCount = 3) {
CreateOlapTableWithStore(tableName, storeName, storeShardsCount, tableShardsCount);
CreateOlapTablesWithStore({tableName}, storeName, storeShardsCount, tableShardsCount);
}

void CreateTestOlapTables(TVector<TString> tableNames = {"olapTable0", "olapTable1"}, TString storeName = "olapStore",
ui32 storeShardsCount = 4, ui32 tableShardsCount = 3) {
CreateOlapTablesWithStore(tableNames, storeName, storeShardsCount, tableShardsCount);
}

using TBase::TBase;

TLocalHelper(TKikimrRunner& runner)
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/ut/olap/helpers/typed_local.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ TString TTypedLocalHelper::GetMultiColumnTestTableSchema(ui32 reps) const {
}

void TTypedLocalHelper::CreateMultiColumnOlapTableWithStore(ui32 reps, ui32 storeShardsCount, ui32 tableShardsCount) {
CreateSchemaOlapTableWithStore(GetMultiColumnTestTableSchema(reps), TableName, "olapStore", storeShardsCount, tableShardsCount);
CreateSchemaOlapTablesWithStore(GetMultiColumnTestTableSchema(reps), {TableName}, "olapStore", storeShardsCount, tableShardsCount);
}

void TTypedLocalHelper::ExecuteSchemeQuery(const TString& alterQuery, const NYdb::EStatus expectedStatus /*= EStatus::SUCCESS*/) const {
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/ut/olap/helpers/typed_local.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ class TTypedLocalHelper: public Tests::NCS::THelper {
void FillPKOnly(const double pkKff = 0, const ui32 numRows = 800000) const;

void CreateTestOlapTable(ui32 storeShardsCount = 4, ui32 tableShardsCount = 3) {
CreateOlapTableWithStore(TableName, StoreName, storeShardsCount, tableShardsCount);
CreateOlapTablesWithStore({TableName}, StoreName, storeShardsCount, tableShardsCount);
}

TString GetMultiColumnTestTableSchema(ui32 reps) const;
Expand Down
26 changes: 26 additions & 0 deletions ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2883,6 +2883,32 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
CompareYson("[[200u]]", result);
}
}

Y_UNIT_TEST(TableSinkWithOlapStore) {
NKikimrConfig::TAppConfig appConfig;
appConfig.MutableTableServiceConfig()->SetEnableOlapSink(true);
auto settings = TKikimrSettings()
.SetAppConfig(appConfig)
.SetWithSampleTables(false);
TKikimrRunner kikimr(settings);
Tests::NCommon::TLoggerInit(kikimr).Initialize();

TLocalHelper(kikimr).CreateTestOlapTables();

WriteTestData(kikimr, "/Root/olapStore/olapTable0", 0, 1000000, 3, true);


auto client = kikimr.GetQueryClient();
{
auto result = client.ExecuteQuery(R"(
SELECT * FROM `/Root/olapStore/olapTable0` ORDER BY timestamp;
INSERT INTO `/Root/olapStore/olapTable1` SELECT * FROM `/Root/olapStore/olapTable0`;
INSERT INTO `/Root/olapStore/olapTable0` SELECT * FROM `/Root/olapStore/olapTable1`;
SELECT * FROM `/Root/olapStore/olapTable1` ORDER BY timestamp;
)", NYdb::NQuery::TTxControl::BeginTx().CommitTx()).ExtractValueSync();
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
}
}
}

}
26 changes: 14 additions & 12 deletions ydb/core/testlib/cs_helper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ TString THelper::GetTestTableSchema() const {
return sb;
}

void THelper::CreateSchemaOlapTableWithStore(const TString tableSchema, TString tableName /*= "olapTable"*/, TString storeName /*= "olapStore"*/, ui32 storeShardsCount /*= 4*/, ui32 tableShardsCount /*= 3*/) {
void THelper::CreateSchemaOlapTablesWithStore(const TString tableSchema, TVector<TString> tableNames /*= "olapTable"*/, TString storeName /*= "olapStore"*/, ui32 storeShardsCount /*= 4*/, ui32 tableShardsCount /*= 3*/) {
TActorId sender = Server.GetRuntime()->AllocateEdgeActor();
CreateTestOlapStore(sender, Sprintf(R"(
Name: "%s"
Expand All @@ -213,19 +213,21 @@ void THelper::CreateSchemaOlapTableWithStore(const TString tableSchema, TString

const TString shardingColumns = "[\"" + JoinSeq("\",\"", GetShardingColumns()) + "\"]";

TBase::CreateTestOlapTable(sender, storeName, Sprintf(R"(
Name: "%s"
ColumnShardCount: %d
Sharding {
HashSharding {
Function: %s
Columns: %s
}
})", tableName.c_str(), tableShardsCount, ShardingMethod.data(), shardingColumns.c_str()));
for (const TString& tableName : tableNames) {
TBase::CreateTestOlapTable(sender, storeName, Sprintf(R"(
Name: "%s"
ColumnShardCount: %d
Sharding {
HashSharding {
Function: %s
Columns: %s
}
})", tableName.c_str(), tableShardsCount, ShardingMethod.data(), shardingColumns.c_str()));
}
}

void THelper::CreateOlapTableWithStore(TString tableName /*= "olapTable"*/, TString storeName /*= "olapStore"*/, ui32 storeShardsCount /*= 4*/, ui32 tableShardsCount /*= 3*/) {
CreateSchemaOlapTableWithStore(GetTestTableSchema(), tableName, storeName, storeShardsCount, tableShardsCount);
void THelper::CreateOlapTablesWithStore(TVector<TString> tableNames /*= {"olapTable"}*/, TString storeName /*= "olapStore"*/, ui32 storeShardsCount /*= 4*/, ui32 tableShardsCount /*= 3*/) {
CreateSchemaOlapTablesWithStore(GetTestTableSchema(), tableNames, storeName, storeShardsCount, tableShardsCount);
}

// Clickbench table
Expand Down
5 changes: 3 additions & 2 deletions ydb/core/testlib/cs_helper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,11 @@ class THelper: public THelperSchemaless {
private:
bool WithSomeNulls_ = false;
protected:
void CreateSchemaOlapTableWithStore(const TString tableSchema, TString tableName = "olapTable", TString storeName = "olapStore",
void CreateSchemaOlapTablesWithStore(const TString tableSchema, TVector<TString> tableName = {"olapTable"}, TString storeName = "olapStore",
ui32 storeShardsCount = 4, ui32 tableShardsCount = 3);
void CreateOlapTableWithStore(TString tableName = "olapTable", TString storeName = "olapStore",
void CreateOlapTablesWithStore(TVector<TString> tableName = {"olapTable"}, TString storeName = "olapStore",
ui32 storeShardsCount = 4, ui32 tableShardsCount = 3);

public:
using TBase::TBase;

Expand Down