Skip to content

YQ WM move resource pools into metadata folder #7741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Aug 14, 2024
Merged

YQ WM move resource pools into metadata folder #7741

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
49728b1
Refactored '.resource_pools' -> '.metadata/workload_manager/pools'
GrigoriyPA Aug 14, 2024
cba77ce
Fixed unit test TestTablesIsNotCreatingForUnlimitedPool
GrigoriyPA Aug 14, 2024
7773f85
Fixed TestCreateWorkloadSerivceTables
GrigoriyPA Aug 14, 2024
d630a65
Fixed classifiers table path
GrigoriyPA Aug 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ydb/core/kqp/gateway/behaviour/resource_pool/manager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -241,23 +241,23 @@ void TResourcePoolManager::PrepareCreateResourcePool(NKqpProto::TKqpSchemeOperat
}

auto& schemeTx = *schemeOperation.MutableCreateResourcePool();
schemeTx.SetWorkingDir(JoinPath({context.GetExternalData().GetDatabase(), ".resource_pools/"}));
schemeTx.SetWorkingDir(JoinPath({context.GetExternalData().GetDatabase(), ".metadata/workload_manager/pools/"}));
schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateResourcePool);

FillResourcePoolDescription(*schemeTx.MutableCreateResourcePool(), settings);
}

void TResourcePoolManager::PrepareAlterResourcePool(NKqpProto::TKqpSchemeOperation& schemeOperation, const NYql::TDropObjectSettings& settings, TInternalModificationContext& context) const {
auto& schemeTx = *schemeOperation.MutableAlterResourcePool();
schemeTx.SetWorkingDir(JoinPath({context.GetExternalData().GetDatabase(), ".resource_pools/"}));
schemeTx.SetWorkingDir(JoinPath({context.GetExternalData().GetDatabase(), ".metadata/workload_manager/pools/"}));
schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpAlterResourcePool);

FillResourcePoolDescription(*schemeTx.MutableCreateResourcePool(), settings);
}

void TResourcePoolManager::PrepareDropResourcePool(NKqpProto::TKqpSchemeOperation& schemeOperation, const NYql::TDropObjectSettings& settings, TInternalModificationContext& context) const {
auto& schemeTx = *schemeOperation.MutableDropResourcePool();
schemeTx.SetWorkingDir(JoinPath({context.GetExternalData().GetDatabase(), ".resource_pools/"}));
schemeTx.SetWorkingDir(JoinPath({context.GetExternalData().GetDatabase(), ".metadata/workload_manager/pools/"}));
schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpDropResourcePool);

schemeTx.MutableDrop()->SetName(settings.GetObjectId());
Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/gateway/behaviour/resource_pool_classifier/behaviour.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ NMetadata::NModifications::IOperationsManager::TPtr TResourcePoolClassifierBehav
}

TString TResourcePoolClassifierBehaviour::GetInternalStorageTablePath() const {
return "resource_pools/resource_pools_classifiers";
return "workload_manager/classifiers/resource_pool_classifiers";
}

TString TResourcePoolClassifierBehaviour::GetTypeId() const {
Expand Down
6 changes: 3 additions & 3 deletions ydb/core/kqp/provider/yql_kikimr_gateway_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,7 @@ void TestCreateResourcePool(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGat
{"concurrent_query_limit", "10"},
{"queue_size", "100"}
});
const auto& resourcePool = TestCreateObjectCommon(runtime, gateway, settings, TStringBuilder() << "/Root/.resource_pools/" << poolId);
const auto& resourcePool = TestCreateObjectCommon(runtime, gateway, settings, TStringBuilder() << "/Root/.metadata/workload_manager/pools/" << poolId);

UNIT_ASSERT_VALUES_EQUAL(resourcePool.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindResourcePool);
UNIT_ASSERT(resourcePool.ResourcePoolInfo);
Expand All @@ -249,7 +249,7 @@ void TestAlterResourcePool(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGate
}, {
"queue_size"
});
const auto& resourcePool = TestAlterObjectCommon(runtime, gateway, settings, TStringBuilder() << "/Root/.resource_pools/" << poolId);
const auto& resourcePool = TestAlterObjectCommon(runtime, gateway, settings, TStringBuilder() << "/Root/.metadata/workload_manager/pools/" << poolId);

UNIT_ASSERT_VALUES_EQUAL(resourcePool.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindResourcePool);
UNIT_ASSERT(resourcePool.ResourcePoolInfo);
Expand All @@ -262,7 +262,7 @@ void TestAlterResourcePool(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGate

void TestDropResourcePool(TTestActorRuntime& runtime, TIntrusivePtr<IKikimrGateway> gateway, const TString& poolId) {
TDropObjectSettings settings("RESOURCE_POOL", poolId, {});
TestDropObjectCommon(runtime, gateway, settings, TStringBuilder() << "/Root/.resource_pools/" << poolId);
TestDropObjectCommon(runtime, gateway, settings, TStringBuilder() << "/Root/.metadata/workload_manager/pools/" << poolId);
}

TKikimrRunner GetKikimrRunnerWithResourcePools() {
Expand Down
12 changes: 6 additions & 6 deletions ydb/core/kqp/ut/scheme/kqp_scheme_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6593,7 +6593,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());

auto& runtime = *kikimr.GetTestServer().GetRuntime();
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.resource_pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.metadata/workload_manager/pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
const auto& resourcePool = resourcePoolDesc->ResultSet.at(0);
UNIT_ASSERT_VALUES_EQUAL(resourcePool.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindResourcePool);
UNIT_ASSERT(resourcePool.ResourcePoolInfo);
Expand Down Expand Up @@ -6625,7 +6625,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());

auto& runtime = *kikimr.GetTestServer().GetRuntime();
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.resource_pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.metadata/workload_manager/pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
UNIT_ASSERT_VALUES_EQUAL(resourcePoolDesc->ResultSet.at(0).Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindResourcePool);
}

Expand All @@ -6636,7 +6636,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
);)";
auto result = session.ExecuteSchemeQuery(query).GetValueSync();
UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), EStatus::GENERIC_ERROR);
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Check failed: path: '/Root/.resource_pools/MyResourcePool', error: path exist");
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), "Check failed: path: '/Root/.metadata/workload_manager/pools/MyResourcePool', error: path exist");
}
}

Expand All @@ -6661,7 +6661,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());

auto& runtime = *kikimr.GetTestServer().GetRuntime();
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.resource_pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.metadata/workload_manager/pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
const auto& properties = resourcePoolDesc->ResultSet.at(0).ResourcePoolInfo->Description.GetProperties().GetProperties();
UNIT_ASSERT_VALUES_EQUAL(properties.size(), 2);
UNIT_ASSERT_VALUES_EQUAL(properties.at("concurrent_query_limit"), "20");
Expand All @@ -6678,7 +6678,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());

auto& runtime = *kikimr.GetTestServer().GetRuntime();
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.resource_pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.metadata/workload_manager/pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
const auto& properties = resourcePoolDesc->ResultSet.at(0).ResourcePoolInfo->Description.GetProperties().GetProperties();
UNIT_ASSERT_VALUES_EQUAL(properties.size(), 3);
UNIT_ASSERT_VALUES_EQUAL(properties.at("concurrent_query_limit"), "30");
Expand Down Expand Up @@ -6734,7 +6734,7 @@ Y_UNIT_TEST_SUITE(KqpScheme) {
}

auto& runtime = *kikimr.GetTestServer().GetRuntime();
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.resource_pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
auto resourcePoolDesc = Navigate(runtime, runtime.AllocateEdgeActor(), "Root/.metadata/workload_manager/pools/MyResourcePool", NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
const auto& resourcePool = resourcePoolDesc->ResultSet.at(0);
UNIT_ASSERT_VALUES_EQUAL(resourcePoolDesc->ErrorCount, 1);
UNIT_ASSERT_VALUES_EQUAL(resourcePool.Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindUnknown);
Expand Down
4 changes: 2 additions & 2 deletions ydb/core/kqp/workload_service/actors/scheme_actors.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ class TPoolFetcherActor : public TSchemeActorBase<TPoolFetcherActor> {
void StartRequest() override {
LOG_D("Start pool fetching");
auto event = NTableCreator::BuildSchemeCacheNavigateRequest(
{{".resource_pools", PoolId}},
{{".metadata/workload_manager/pools", PoolId}},
Database,
UserToken
);
Expand Down Expand Up @@ -326,7 +326,7 @@ class TPoolCreatorActor : public TSchemeActorBase<TPoolCreatorActor> {
auto event = std::make_unique<TEvTxUserProxy::TEvProposeTransaction>();

auto& schemeTx = *event->Record.MutableTransaction()->MutableModifyScheme();
schemeTx.SetWorkingDir(JoinPath({Database, ".resource_pools"}));
schemeTx.SetWorkingDir(JoinPath({Database, ".metadata/workload_manager/pools"}));
schemeTx.SetOperationType(NKikimrSchemeOp::ESchemeOpCreateResourcePool);
schemeTx.SetInternal(true);

Expand Down
2 changes: 1 addition & 1 deletion ydb/core/kqp/workload_service/ut/common/kqp_workload_service_ut_common.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ class TWorkloadServiceYdbSetup : public IYdbSetup {
auto token = NACLib::TUserToken(userSID, {});

WaitFor(FUTURE_WAIT_TIMEOUT, "pool acl", [this, token, access, poolId](TString& errorString) {
auto response = Navigate(TStringBuilder() << ".resource_pools/" << (poolId ? poolId : Settings_.PoolId_));
auto response = Navigate(TStringBuilder() << ".metadata/workload_manager/pools/" << (poolId ? poolId : Settings_.PoolId_));
if (!response) {
errorString = "empty response";
return false;
Expand Down
8 changes: 4 additions & 4 deletions ydb/core/kqp/workload_service/ut/kqp_workload_service_actors_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceActors) {

const TString& userSID = "user@test";
TSampleQueries::CheckSuccess(ydb->ExecuteQuery(TStringBuilder() << R"(
GRANT DESCRIBE SCHEMA ON `/Root/.resource_pools/)" << ydb->GetSettings().PoolId_ << "` TO `" << userSID << "`;"
GRANT DESCRIBE SCHEMA ON `/Root/.metadata/workload_manager/pools/)" << ydb->GetSettings().PoolId_ << "` TO `" << userSID << "`;"
));
ydb->WaitPoolAccess(userSID, NACLib::EAccessRights::DescribeSchema);

Expand All @@ -67,7 +67,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceActors) {
UNIT_ASSERT_STRING_CONTAINS(failedResponse->Get()->Issues.ToString(), TStringBuilder() << "You don't have access permissions for resource pool " << ydb->GetSettings().PoolId_);

TSampleQueries::CheckSuccess(ydb->ExecuteQuery(TStringBuilder() << R"(
GRANT SELECT ROW ON `/Root/.resource_pools/)" << ydb->GetSettings().PoolId_ << "` TO `" << userSID << "`;"
GRANT SELECT ROW ON `/Root/.metadata/workload_manager/pools/)" << ydb->GetSettings().PoolId_ << "` TO `" << userSID << "`;"
));
ydb->WaitPoolAccess(userSID, NACLib::EAccessRights::SelectRow);

Expand All @@ -86,7 +86,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceActors) {
Y_UNIT_TEST(TestCreateDefaultPool) {
auto ydb = TYdbSetupSettings().Create();

const TString path = TStringBuilder() << ".resource_pools/" << NResourcePool::DEFAULT_POOL_ID;
const TString path = TStringBuilder() << ".metadata/workload_manager/pools/" << NResourcePool::DEFAULT_POOL_ID;
auto response = ydb->Navigate(path, NSchemeCache::TSchemeCacheNavigate::EOp::OpUnknown);
UNIT_ASSERT_VALUES_EQUAL(response->ErrorCount, 1);
UNIT_ASSERT_VALUES_EQUAL(response->ResultSet.at(0).Kind, NSchemeCache::TSchemeCacheNavigate::EKind::KindUnknown);
Expand Down Expand Up @@ -224,7 +224,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceSubscriptions) {

const TString& userSID = "test@user";
ydb->ExecuteSchemeQuery(TStringBuilder() << R"(
GRANT ALL ON `/Root/.resource_pools/)" << ydb->GetSettings().PoolId_ << R"(` TO `)" << userSID << R"(`;
GRANT ALL ON `/Root/.metadata/workload_manager/pools/)" << ydb->GetSettings().PoolId_ << R"(` TO `)" << userSID << R"(`;
)");

const auto& response = ydb->GetRuntime()->GrabEdgeEvent<TEvUpdatePoolInfo>(subscriber, FUTURE_WAIT_TIMEOUT);
Expand Down
9 changes: 4 additions & 5 deletions ydb/core/kqp/workload_service/ut/kqp_workload_service_tables_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceTables) {
CanonizePath({ydb->GetSettings().DomainName_, ".metadata/workload_manager"})
).GetValue(FUTURE_WAIT_TIMEOUT);
UNIT_ASSERT_VALUES_EQUAL_C(listResult.GetStatus(), NYdb::EStatus::SUCCESS, listResult.GetIssues().ToString());
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren().size(), 2);
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren().size(), 3);
}

Y_UNIT_TEST(TestTablesIsNotCreatingForUnlimitedPool) {
Expand All @@ -83,12 +83,11 @@ Y_UNIT_TEST_SUITE(KqpWorkloadServiceTables) {

// Check that there is no .metadata folder
auto listResult = ydb->GetSchemeClient().ListDirectory(
CanonizePath(ydb->GetSettings().DomainName_)
CanonizePath({ydb->GetSettings().DomainName_, ".metadata", "workload_manager"})
).GetValue(FUTURE_WAIT_TIMEOUT);
UNIT_ASSERT_VALUES_EQUAL_C(listResult.GetStatus(), NYdb::EStatus::SUCCESS, listResult.GetIssues().ToString());
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren().size(), 2);
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren()[0].Name, ".resource_pools");
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren()[1].Name, ".sys");
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren().size(), 1);
UNIT_ASSERT_VALUES_EQUAL(listResult.GetChildren()[0].Name, "pools");
}

Y_UNIT_TEST(TestPoolStateFetcherActor) {
Expand Down
8 changes: 4 additions & 4 deletions ydb/core/kqp/workload_service/ut/kqp_workload_service_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -480,7 +480,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolsDdl) {
);

IYdbSetup::WaitFor(FUTURE_WAIT_TIMEOUT, "pool drop", [ydb, poolId](TString& errorString) {
auto kind = ydb->Navigate(TStringBuilder() << ".resource_pools/" << poolId)->ResultSet.at(0).Kind;
auto kind = ydb->Navigate(TStringBuilder() << ".metadata/workload_manager/pools/" << poolId)->ResultSet.at(0).Kind;

errorString = TStringBuilder() << "kind = " << kind;
return kind == NSchemeCache::TSchemeCacheNavigate::EKind::KindUnknown;
Expand All @@ -500,7 +500,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolsDdl) {
CREATE RESOURCE POOL )" << poolId << R"( WITH (
CONCURRENT_QUERY_LIMIT=1
);
GRANT DESCRIBE SCHEMA ON `/Root/.resource_pools/)" << poolId << "` TO `" << userSID << "`;"
GRANT DESCRIBE SCHEMA ON `/Root/.metadata/workload_manager/pools/)" << poolId << "` TO `" << userSID << "`;"
);
ydb->WaitPoolAccess(userSID, NACLib::EAccessRights::DescribeSchema, poolId);

Expand All @@ -510,7 +510,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolsDdl) {
UNIT_ASSERT_STRING_CONTAINS(result.GetIssues().ToString(), TStringBuilder() << "You don't have access permissions for resource pool " << poolId);

ydb->ExecuteSchemeQuery(TStringBuilder() << R"(
GRANT SELECT ROW ON `/Root/.resource_pools/)" << poolId << "` TO `" << userSID << "`;"
GRANT SELECT ROW ON `/Root/.metadata/workload_manager/pools/)" << poolId << "` TO `" << userSID << "`;"
);
ydb->WaitPoolAccess(userSID, NACLib::EAccessRights::SelectRow, poolId);
TSampleQueries::TSelect42::CheckResult(ydb->ExecuteQuery(TSampleQueries::TSelect42::Query, settings));
Expand All @@ -524,7 +524,7 @@ Y_UNIT_TEST_SUITE(ResourcePoolClassifiersDdl) {
const TString& userSID = "user@test";
ydb->ExecuteSchemeQuery(TStringBuilder() << R"(
GRANT DESCRIBE SCHEMA ON `/Root` TO `)" << userSID << R"(`;
GRANT DESCRIBE SCHEMA, SELECT ROW ON `/Root/.resource_pools/)" << ydb->GetSettings().PoolId_ << "` TO `" << userSID << "`;"
GRANT DESCRIBE SCHEMA, SELECT ROW ON `/Root/.metadata/workload_manager/pools/)" << ydb->GetSettings().PoolId_ << "` TO `" << userSID << "`;"
);
ydb->WaitPoolAccess(userSID, NACLib::EAccessRights::DescribeSchema | NACLib::EAccessRights::SelectRow);

Expand Down
2 changes: 1 addition & 1 deletion ydb/core/tx/schemeshard/schemeshard__operation_common_resource_pool.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ TPath::TChecker IsParentPathValid(const TPath& parentPath) {
}

bool IsParentPathValid(const THolder<TProposeResponse>& result, const TPath& parentPath) {
const TString& resourcePoolsDir = JoinPath({parentPath.GetDomainPathString(), ".resource_pools"});
const TString& resourcePoolsDir = JoinPath({parentPath.GetDomainPathString(), ".metadata/workload_manager/pools"});
if (parentPath.PathString() != resourcePoolsDir) {
result->SetError(NKikimrScheme::EStatus::StatusSchemeError, TStringBuilder() << "Resource pools shoud be placed in " << resourcePoolsDir);
return false;
Expand Down
Loading
Loading