Skip to content

Commit b2da93a

Browse files
swalrus1swalrus1
authored
configure tiering on CS via ttl (#12095)
1 parent abb79c8 commit b2da93a

File tree

76 files changed

+495
-1260
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

76 files changed

+495
-1260
lines changed

‎ydb/core/grpc_services/rpc_create_table.cpp‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,6 @@ class TCreateTableRPC : public TRpcSchemeRequestActor<TCreateTableRPC, TEvCreate
136136
return false;
137137
}
138138
}
139-
tableDesc->MutableTtlSettings()->SetUseTiering(req.tiering());
140139

141140
return true;
142141
}

‎ydb/core/grpc_services/rpc_log_store.cpp‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -435,8 +435,6 @@ class TCreateLogTableRPC : public TRpcSchemeRequestActor<TCreateLogTableRPC, TEv
435435
if (!FillTtlSettings(*create->MutableTtlSettings()->MutableEnabled(), req->ttl_settings(), status, error)) {
436436
return Reply(status, error, NKikimrIssues::TIssuesIds::DEFAULT_ERROR, ctx);
437437
}
438-
} else if (req->has_tiering_settings()) {
439-
create->MutableTtlSettings()->SetUseTiering(req->tiering_settings().tiering_id());
440438
}
441439

442440
create->SetColumnShardCount(req->shards_count());
@@ -600,12 +598,6 @@ class TAlterLogTableRPC : public TRpcSchemeRequestActor<TAlterLogTableRPC, TEvAl
600598
alter->MutableAlterTtlSettings()->MutableDisabled();
601599
}
602600

603-
if (req->has_set_tiering_settings()) {
604-
alter->MutableAlterTtlSettings()->SetUseTiering(req->set_tiering_settings().tiering_id());
605-
} else if (req->has_drop_tiering_settings()) {
606-
alter->MutableAlterTtlSettings()->SetUseTiering("");
607-
}
608-
609601
ctx.Send(MakeTxProxyID(), proposeRequest.release());
610602
}
611603
};

‎ydb/core/kqp/host/kqp_gateway_proxy.cpp‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -276,16 +276,6 @@ bool ConvertCreateTableSettingsToProto(NYql::TKikimrTableMetadataPtr metadata, Y
276276
}
277277
}
278278

279-
if (const auto& tiering = metadata->TableSettings.Tiering) {
280-
if (tiering.IsSet()) {
281-
proto.set_tiering(tiering.GetValueSet());
282-
} else {
283-
code = Ydb::StatusIds::BAD_REQUEST;
284-
error = "Can't reset TIERING";
285-
return false;
286-
}
287-
}
288-
289279
if (metadata->TableSettings.StoreExternalBlobs) {
290280
auto& storageSettings = *proto.mutable_storage_settings();
291281
TString value = to_lower(metadata->TableSettings.StoreExternalBlobs.GetRef());
@@ -520,7 +510,15 @@ bool FillCreateColumnTableDesc(NYql::TKikimrTableMetadataPtr metadata,
520510
const auto& inputSettings = metadata->TableSettings.TtlSettings.GetValueSet();
521511
auto& resultSettings = *tableDesc.MutableTtlSettings();
522512
resultSettings.MutableEnabled()->SetColumnName(inputSettings.ColumnName);
523-
resultSettings.MutableEnabled()->SetExpireAfterSeconds(inputSettings.ExpireAfter.Seconds());
513+
for (const auto& tier : inputSettings.Tiers) {
514+
auto* tierProto = resultSettings.MutableEnabled()->AddTiers();
515+
tierProto->SetApplyAfterSeconds(tier.ApplyAfter.Seconds());
516+
if (tier.StorageName) {
517+
tierProto->MutableEvictToExternalStorage()->SetStorageName(*tier.StorageName);
518+
} else {
519+
tierProto->MutableDelete();
520+
}
521+
}
524522
if (inputSettings.ColumnUnit) {
525523
resultSettings.MutableEnabled()->SetColumnUnit(static_cast<NKikimrSchemeOp::TTTLSettings::EUnit>(*inputSettings.ColumnUnit));
526524
}

‎ydb/core/kqp/provider/yql_kikimr_exec.cpp‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1686,13 +1686,6 @@ class TKiSinkCallableExecutionTransformer : public TAsyncCallbackTransformer<TKi
16861686
ConvertTtlSettingsToProto(ttlSettings, *alterTableRequest.mutable_set_ttl_settings());
16871687
} else if (name == "resetTtlSettings") {
16881688
alterTableRequest.mutable_drop_ttl_settings();
1689-
} else if (name == "setTiering") {
1690-
const auto tieringName = TString(
1691-
setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value()
1692-
);
1693-
alterTableRequest.set_set_tiering(tieringName);
1694-
} else if (name == "resetTiering") {
1695-
alterTableRequest.mutable_drop_tiering();
16961689
} else {
16971690
ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()),
16981691
TStringBuilder() << "Unknown table profile setting: " << name));

‎ydb/core/kqp/provider/yql_kikimr_gateway.cpp‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -105,42 +105,36 @@ bool TTtlSettings::TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSetti
105105
if (name == "columnName") {
106106
YQL_ENSURE(field.Value().Maybe<TCoAtom>());
107107
settings.ColumnName = field.Value().Cast<TCoAtom>().StringValue();
108-
} else if (name == "expireAfter") {
109-
// TODO (yentsovsemyon): remove this clause after extending TTL syntax in YQL
110-
YQL_ENSURE(field.Value().Maybe<TCoInterval>());
111-
auto value = FromString<i64>(field.Value().Cast<TCoInterval>().Literal().Value());
112-
if (value < 0) {
113-
error = "Interval value cannot be negative";
114-
return false;
115-
}
116-
117-
settings.ExpireAfter = TDuration::FromValue(value);
118108
} else if (name == "tiers") {
119109
YQL_ENSURE(field.Value().Maybe<TExprList>());
120110
auto listNode = field.Value().Cast<TExprList>();
121111

122112
for (size_t i = 0; i < listNode.Size(); ++i) {
123113
auto tierNode = listNode.Item(i);
124114

115+
std::optional<TString> storageName;
116+
TDuration evictionDelay;
125117
YQL_ENSURE(tierNode.Maybe<TCoNameValueTupleList>());
126118
for (const auto& tierField : tierNode.Cast<TCoNameValueTupleList>()) {
127119
auto tierFieldName = tierField.Name().Value();
128120
if (tierFieldName == "storageName") {
129-
error = "TTL cannot contain tiered storage: tiering in TTL syntax is not supported";
130-
return false;
121+
YQL_ENSURE(tierField.Value().Maybe<TCoAtom>());
122+
storageName = tierField.Value().Cast<TCoAtom>().StringValue();
131123
} else if (tierFieldName == "evictionDelay") {
132124
YQL_ENSURE(tierField.Value().Maybe<TCoInterval>());
133125
auto value = FromString<i64>(tierField.Value().Cast<TCoInterval>().Literal().Value());
134126
if (value < 0) {
135127
error = "Interval value cannot be negative";
136128
return false;
137129
}
138-
settings.ExpireAfter = TDuration::FromValue(value);
130+
evictionDelay = TDuration::FromValue(value);
139131
} else {
140132
error = TStringBuilder() << "Unknown field: " << tierFieldName;
141133
return false;
142134
}
143135
}
136+
137+
settings.Tiers.emplace_back(evictionDelay, storageName);
144138
}
145139
} else if (name == "columnUnit") {
146140
YQL_ENSURE(field.Value().Maybe<TCoAtom>());
@@ -322,9 +316,15 @@ void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::T
322316
opts.set_column_name(settings.ColumnName);
323317
opts.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(*settings.ColumnUnit));
324318
}
325-
auto* deleteTier = proto.add_tiers();
326-
deleteTier->set_apply_after_seconds(settings.ExpireAfter.Seconds());
327-
deleteTier->mutable_delete_();
319+
for (const auto& tier : settings.Tiers) {
320+
auto* tierProto = proto.add_tiers();
321+
tierProto->set_apply_after_seconds(tier.ApplyAfter.Seconds());
322+
if (tier.StorageName) {
323+
tierProto->mutable_evict_to_external_storage()->set_storage_name(*tier.StorageName);
324+
} else {
325+
tierProto->mutable_delete_();
326+
}
327+
}
328328
}
329329

330330
Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) {

‎ydb/core/kqp/provider/yql_kikimr_gateway.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -222,9 +222,14 @@ struct TTtlSettings {
222222
Nanoseconds = 4,
223223
};
224224

225+
struct TTier {
226+
TDuration ApplyAfter;
227+
std::optional<TString> StorageName;
228+
};
229+
225230
TString ColumnName;
226-
TDuration ExpireAfter;
227231
TMaybe<EUnit> ColumnUnit;
232+
std::vector<TTier> Tiers;
228233

229234
static bool TryParse(const NNodes::TCoNameValueTupleList& node, TTtlSettings& settings, TString& error);
230235
};
@@ -242,7 +247,6 @@ struct TTableSettings {
242247
TMaybe<TString> KeyBloomFilter;
243248
TMaybe<TString> ReadReplicasSettings;
244249
TResetableSetting<TTtlSettings, void> TtlSettings;
245-
TResetableSetting<TString, void> Tiering;
246250
TMaybe<TString> PartitionByHashFunction;
247251
TMaybe<TString> StoreExternalBlobs;
248252

‎ydb/core/kqp/provider/yql_kikimr_type_ann.cpp‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1251,14 +1251,6 @@ virtual TStatus HandleCreateTable(TKiCreateTable create, TExprContext& ctx) over
12511251
ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()),
12521252
"Can't reset TTL settings"));
12531253
return TStatus::Error;
1254-
} else if (name == "setTiering") {
1255-
meta->TableSettings.Tiering.Set(TString(
1256-
setting.Value().Cast<TCoDataCtor>().Literal().Cast<TCoAtom>().Value()
1257-
));
1258-
} else if (name == "resetTiering") {
1259-
ctx.AddError(TIssue(ctx.GetPosition(setting.Name().Pos()),
1260-
"Can't reset TIERING"));
1261-
return TStatus::Error;
12621254
} else if (name == "storeType") {
12631255
TMaybe<TString> storeType = TString(setting.Value().Cast<TCoAtom>().Value());
12641256
if (storeType && to_lower(storeType.GetRef()) == "column") {

‎ydb/core/kqp/ut/common/columnshard.cpp‎

Lines changed: 3 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -68,29 +68,14 @@ namespace NKqp {
6868
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
6969
}
7070

71-
TString TTestHelper::CreateTieringRule(const TString& tierName, const TString& columnName) {
72-
const TString ruleName = tierName + "_" + columnName;
73-
const TString configTieringStr = TStringBuilder() << R"({
74-
"rules" : [
75-
{
76-
"tierName" : ")" << tierName << R"(",
77-
"durationForEvict" : "10d"
78-
}
79-
]
80-
})";
81-
auto result = GetSession().ExecuteSchemeQuery("CREATE OBJECT IF NOT EXISTS " + ruleName + " (TYPE TIERING_RULE) WITH (defaultColumn = " + columnName + ", description = `" + configTieringStr + "`)").GetValueSync();
82-
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
83-
return ruleName;
84-
}
85-
86-
void TTestHelper::SetTiering(const TString& tableName, const TString& ruleName) {
87-
auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` SET (TIERING = '" << ruleName << "')";
71+
void TTestHelper::SetTiering(const TString& tableName, const TString& tierName, const TString& columnName) {
72+
auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` SET TTL Interval(\"P10D\") TO EXTERNAL DATA SOURCE `" << tierName << "` ON `" << columnName << "`;";
8873
auto result = GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync();
8974
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
9075
}
9176

9277
void TTestHelper::ResetTiering(const TString& tableName) {
93-
auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` RESET (TIERING)";
78+
auto alterQuery = TStringBuilder() << "ALTER TABLE `" << tableName << "` RESET (TTL)";
9479
auto result = GetSession().ExecuteSchemeQuery(alterQuery).GetValueSync();
9580
UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), EStatus::SUCCESS, result.GetIssues().ToString());
9681
}

‎ydb/core/kqp/ut/common/columnshard.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,10 @@ class TTestHelper {
109109
NYdb::NTable::TSession& GetSession();
110110
void CreateTable(const TColumnTableBase& table, const NYdb::EStatus expectedStatus = NYdb::EStatus::SUCCESS);
111111
void DropTable(const TString& tableName);
112+
void EnsureSecret(const TString& name, const TString& value);
112113
void CreateTier(const TString& tierName);
113114
TString CreateTieringRule(const TString& tierName, const TString& columnName);
114-
void SetTiering(const TString& tableName, const TString& ruleName);
115+
void SetTiering(const TString& tableName, const TString& tierName, const TString& columnName);
115116
void ResetTiering(const TString& tableName);
116117
void BulkUpsert(
117118
const TColumnTable& table, TTestHelper::TUpdatesBuilder& updates, const Ydb::StatusIds_StatusCode& opStatus = Ydb::StatusIds::SUCCESS);

0 commit comments

Comments
 (0)