Skip to content

Commit 4743a95

Browse files
author
yentsovsemyon
committed
Extend TTL syntax to support tiers
RFC: **[nda.ya.ru/t/JsIT3hp679nYxn](https://nda.ya.ru/t/JsIT3hp679nYxn)** commit_hash:a0a4f65b24ee591cb76fd3cf253ffe24a01bfaf5
1 parent 90a8f8c commit 4743a95

File tree

10 files changed

+332
-25
lines changed

10 files changed

+332
-25
lines changed

‎yql/essentials/sql/v1/SQLv1.g.in‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -791,10 +791,16 @@ table_setting_value:
791791
| STRING_VALUE
792792
| integer
793793
| split_boundaries
794-
| expr ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
794+
| ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
795795
| bool_value
796796
;
797797

798+
ttl_tier_list: expr (ttl_tier_action (COMMA expr ttl_tier_action)*)?;
799+
ttl_tier_action:
800+
TO EXTERNAL DATA SOURCE an_id
801+
| DELETE
802+
;
803+
798804
family_entry: FAMILY an_id family_settings;
799805
family_settings: LPAREN (family_settings_entry (COMMA family_settings_entry)*)? RPAREN;
800806
family_settings_entry: an_id EQUALS family_setting_value;

‎yql/essentials/sql/v1/SQLv1Antlr4.g.in‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -790,10 +790,16 @@ table_setting_value:
790790
| STRING_VALUE
791791
| integer
792792
| split_boundaries
793-
| expr ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
793+
| ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
794794
| bool_value
795795
;
796796

797+
ttl_tier_list: expr (ttl_tier_action (COMMA expr ttl_tier_action)*)?;
798+
ttl_tier_action:
799+
TO EXTERNAL DATA SOURCE an_id
800+
| DELETE
801+
;
802+
797803
family_entry: FAMILY an_id family_settings;
798804
family_settings: LPAREN (family_settings_entry (COMMA family_settings_entry)*)? RPAREN;
799805
family_settings_entry: an_id EQUALS family_setting_value;

‎yql/essentials/sql/v1/format/sql_format.cpp‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2495,6 +2495,66 @@ friend struct TStaticData;
24952495
Visit(msg.GetToken5());
24962496
}
24972497

2498+
void VisitTableSettingValue(const TRule_table_setting_value& msg) {
2499+
switch (msg.GetAltCase()) {
2500+
case TRule_table_setting_value::kAltTableSettingValue5: {
2501+
// | ttl_tier_list ON an_id (AS (SECONDS | MILLISECONDS | MICROSECONDS | NANOSECONDS))?
2502+
const auto& ttlSettings = msg.GetAlt_table_setting_value5();
2503+
const auto& tierList = ttlSettings.GetRule_ttl_tier_list1();
2504+
const bool needIndent = tierList.HasBlock2() && tierList.GetBlock2().Block2Size() > 0; // more then one tier
2505+
if (needIndent) {
2506+
NewLine();
2507+
PushCurrentIndent();
2508+
Visit(tierList.GetRule_expr1());
2509+
VisitTtlTierAction(tierList.GetBlock2().GetRule_ttl_tier_action1());
2510+
2511+
for (const auto& tierEntry : tierList.GetBlock2().GetBlock2()) {
2512+
Visit(tierEntry.GetToken1()); // comma
2513+
NewLine();
2514+
Visit(tierEntry.GetRule_expr2());
2515+
VisitTtlTierAction(tierEntry.GetRule_ttl_tier_action3());
2516+
}
2517+
2518+
PopCurrentIndent();
2519+
NewLine();
2520+
} else {
2521+
Visit(tierList.GetRule_expr1());
2522+
if (tierList.HasBlock2()) {
2523+
VisitTtlTierAction(tierList.GetBlock2().GetRule_ttl_tier_action1());
2524+
}
2525+
}
2526+
2527+
VisitKeyword(ttlSettings.GetToken2());
2528+
Visit(ttlSettings.GetRule_an_id3());
2529+
if (ttlSettings.HasBlock4()) {
2530+
VisitKeyword(ttlSettings.GetBlock4().GetToken1());
2531+
VisitKeyword(ttlSettings.GetBlock4().GetToken2());
2532+
}
2533+
} break;
2534+
default:
2535+
VisitAllFields(TRule_table_setting_value::GetDescriptor(), msg);
2536+
}
2537+
}
2538+
2539+
void VisitTtlTierAction(const TRule_ttl_tier_action& msg) {
2540+
switch (msg.GetAltCase()) {
2541+
case TRule_ttl_tier_action::kAltTtlTierAction1:
2542+
// | TO EXTERNAL DATA SOURCE an_id
2543+
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken1());
2544+
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken2());
2545+
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken3());
2546+
VisitKeyword(msg.GetAlt_ttl_tier_action1().GetToken4());
2547+
Visit(msg.GetAlt_ttl_tier_action1().GetRule_an_id5());
2548+
break;
2549+
case TRule_ttl_tier_action::kAltTtlTierAction2:
2550+
// | DELETE
2551+
VisitKeyword(msg.GetAlt_ttl_tier_action2().GetToken1());
2552+
break;
2553+
case TRule_ttl_tier_action::ALT_NOT_SET:
2554+
break;
2555+
}
2556+
}
2557+
24982558
void VisitExpr(const TRule_expr& msg) {
24992559
if (msg.HasAlt_expr2()) {
25002560
Visit(msg.GetAlt_expr2());
@@ -2783,6 +2843,8 @@ TStaticData::TStaticData()
27832843
{TRule_case_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitCaseExpr)},
27842844
{TRule_when_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitWhenExpr)},
27852845
{TRule_with_table_settings::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitWithTableSettingsExpr)},
2846+
{TRule_table_setting_value::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitTableSettingValue)},
2847+
{TRule_ttl_tier_action::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitTtlTierAction)},
27862848

27872849
{TRule_expr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitExpr)},
27882850
{TRule_or_subexpr::GetDescriptor(), MakePrettyFunctor(&TPrettyVisitor::VisitOrSubexpr)},

‎yql/essentials/sql/v1/format/sql_format_ut.h‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,16 @@ Y_UNIT_TEST(CreateTable) {
226226
"CREATE TABLE user (\n\tuser int32\n)\nWITH (ttl = interval('P1D') ON user AS MICROSECONDS);\n"},
227227
{"create table user(user int32) with (ttl=interval('P1D') on user as nAnOsEcOnDs)",
228228
"CREATE TABLE user (\n\tuser int32\n)\nWITH (ttl = interval('P1D') ON user AS NANOSECONDS);\n"},
229+
{"create table user(user int32) with (ttl=interval('P1D') delete on user as nAnOsEcOnDs)",
230+
"CREATE TABLE user (\n\tuser int32\n)\nWITH (ttl = interval('P1D') DELETE ON user AS NANOSECONDS);\n"},
231+
{"create table user(user int32) with (ttl=interval('P1D')to external data source tier1 ,interval('P10D')delete on user as seconds)",
232+
"CREATE TABLE user (\n"
233+
"\tuser int32\n"
234+
")\n"
235+
"WITH (ttl =\n"
236+
"\tinterval('P1D') TO EXTERNAL DATA SOURCE tier1,\n"
237+
"\tinterval('P10D') DELETE\n"
238+
"ON user AS SECONDS);\n"},
229239
{"create table user(index user global unique sync on (user,user) with (user=user,user=user))",
230240
"CREATE TABLE user (\n\tINDEX user GLOBAL UNIQUE SYNC ON (user, user) WITH (user = user, user = user)\n);\n"},
231241
{"create table user(index user global async on (user) with (user=user,))",

‎yql/essentials/sql/v1/node.cpp‎

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1894,9 +1894,14 @@ TMaybe<TStringContent> StringContentOrIdContent(TContext& ctx, TPosition pos, co
18941894
(ctx.AnsiQuotedIdentifiers && input.StartsWith('"'))? EStringContentMode::AnsiIdent : EStringContentMode::Default);
18951895
}
18961896

1897-
TTtlSettings::TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr, const TMaybe<EUnit>& columnUnit)
1897+
TTtlSettings::TTierSettings::TTierSettings(const TNodePtr& evictionDelay, const std::optional<TIdentifier>& storageName)
1898+
: EvictionDelay(evictionDelay)
1899+
, StorageName(storageName) {
1900+
}
1901+
1902+
TTtlSettings::TTtlSettings(const TIdentifier& columnName, const std::vector<TTierSettings>& tiers, const TMaybe<EUnit>& columnUnit)
18981903
: ColumnName(columnName)
1899-
, Expr(expr)
1904+
, Tiers(tiers)
19001905
, ColumnUnit(columnUnit)
19011906
{
19021907
}
@@ -3131,10 +3136,10 @@ class TCalcOverWindow final: public INode {
31313136
Y_DEBUG_ABORT_UNLESS(FuncNode);
31323137
FuncNode->VisitTree(func, visited);
31333138
}
3134-
3139+
31353140
void CollectPreaggregateExprs(TContext& ctx, ISource& src, TVector<INode::TPtr>& exprs) override {
31363141
if (ctx.DistinctOverWindow) {
3137-
FuncNode->CollectPreaggregateExprs(ctx, src, exprs);
3142+
FuncNode->CollectPreaggregateExprs(ctx, src, exprs);
31383143
} else {
31393144
INode::CollectPreaggregateExprs(ctx, src, exprs);
31403145
}
@@ -3274,7 +3279,7 @@ TSourcePtr TryMakeSourceFromExpression(TPosition pos, TContext& ctx, const TStri
32743279
return nullptr;
32753280
}
32763281

3277-
auto wrappedNode = new TAstListNodeImpl(pos, {
3282+
auto wrappedNode = new TAstListNodeImpl(pos, {
32783283
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
32793284
node
32803285
});
@@ -3303,7 +3308,7 @@ void MakeTableFromExpression(TPosition pos, TContext& ctx, TNodePtr node, TDefer
33033308
node = node->Y("Concat", node->Y("String", node->Q(prefix)), node);
33043309
}
33053310

3306-
auto wrappedNode = new TAstListNodeImpl(pos, {
3311+
auto wrappedNode = new TAstListNodeImpl(pos, {
33073312
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
33083313
node
33093314
});
@@ -3320,7 +3325,7 @@ TDeferredAtom MakeAtomFromExpression(TPosition pos, TContext& ctx, TNodePtr node
33203325
node = node->Y("Concat", node->Y("String", node->Q(prefix)), node);
33213326
}
33223327

3323-
auto wrappedNode = new TAstListNodeImpl(pos, {
3328+
auto wrappedNode = new TAstListNodeImpl(pos, {
33243329
new TAstAtomNodeImpl(pos, "EvaluateAtom", TNodeFlags::Default),
33253330
node
33263331
});
@@ -3462,7 +3467,7 @@ bool TVectorIndexSettings::Validate(TContext& ctx) const {
34623467
if (!Distance && !Similarity) {
34633468
ctx.Error() << "either distance or similarity should be set";
34643469
return false;
3465-
}
3470+
}
34663471
if (!VectorType) {
34673472
ctx.Error() << "vector_type should be set";
34683473
return false;

‎yql/essentials/sql/v1/node.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1112,11 +1112,18 @@ namespace NSQLTranslationV1 {
11121112
Nanoseconds /* "nanoseconds" */,
11131113
};
11141114

1115+
struct TTierSettings {
1116+
TNodePtr EvictionDelay;
1117+
std::optional<TIdentifier> StorageName;
1118+
1119+
TTierSettings(const TNodePtr& evictionDelay, const std::optional<TIdentifier>& storageName = std::nullopt);
1120+
};
1121+
11151122
TIdentifier ColumnName;
1116-
TNodePtr Expr;
1123+
std::vector<TTierSettings> Tiers;
11171124
TMaybe<EUnit> ColumnUnit;
11181125

1119-
TTtlSettings(const TIdentifier& columnName, const TNodePtr& expr, const TMaybe<EUnit>& columnUnit = {});
1126+
TTtlSettings(const TIdentifier& columnName, const std::vector<TTierSettings>& tiers, const TMaybe<EUnit>& columnUnit = {});
11201127
};
11211128

11221129
struct TTableSettings {

‎yql/essentials/sql/v1/query.cpp‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,17 @@ static INode::TPtr CreateTableSettings(const TTableSettings& tableSettings, ETab
240240
auto opts = Y();
241241

242242
opts = L(opts, Q(Y(Q("columnName"), BuildQuotedAtom(ttlSettings.ColumnName.Pos, ttlSettings.ColumnName.Name))));
243-
opts = L(opts, Q(Y(Q("expireAfter"), ttlSettings.Expr)));
243+
244+
auto tiersDesc = Y();
245+
for (const auto& tier : ttlSettings.Tiers) {
246+
auto tierDesc = Y();
247+
tierDesc = L(tierDesc, Q(Y(Q("evictionDelay"), tier.EvictionDelay)));
248+
if (tier.StorageName) {
249+
tierDesc = L(tierDesc, Q(Y(Q("storageName"), BuildQuotedAtom(tier.StorageName->Pos, tier.StorageName->Name))));
250+
}
251+
tiersDesc = L(tiersDesc, Q(tierDesc));
252+
}
253+
opts = L(opts, Q(Y(Q("tiers"), Q(tiersDesc))));
244254

245255
if (ttlSettings.ColumnUnit) {
246256
opts = L(opts, Q(Y(Q("columnUnit"), Q(ToString(*ttlSettings.ColumnUnit)))));

‎yql/essentials/sql/v1/sql_translation.cpp‎

Lines changed: 57 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1963,19 +1963,68 @@ namespace {
19631963
return true;
19641964
}
19651965

1966-
bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to,
1967-
TSqlExpression& expr, TContext& ctx, TTranslation& txc) {
1966+
bool FillTieringInterval(const TRule_expr& from, TNodePtr& tieringInterval, TSqlExpression& expr, TContext& ctx) {
1967+
auto exprNode = expr.Build(from);
1968+
if (!exprNode) {
1969+
return false;
1970+
}
1971+
1972+
if (exprNode->GetOpName() != "Interval") {
1973+
ctx.Error() << "Literal of Interval type is expected for TTL";
1974+
return false;
1975+
}
1976+
1977+
tieringInterval = exprNode;
1978+
return true;
1979+
}
1980+
1981+
bool FillTierAction(const TRule_ttl_tier_action& from, std::optional<TIdentifier>& storageName, TTranslation& txc) {
1982+
switch (from.GetAltCase()) {
1983+
case TRule_ttl_tier_action::kAltTtlTierAction1:
1984+
storageName = IdEx(from.GetAlt_ttl_tier_action1().GetRule_an_id5(), txc);
1985+
break;
1986+
case TRule_ttl_tier_action::kAltTtlTierAction2:
1987+
storageName.reset();
1988+
break;
1989+
case TRule_ttl_tier_action::ALT_NOT_SET:
1990+
Y_ABORT("You should change implementation according to grammar changes");
1991+
}
1992+
return true;
1993+
}
1994+
1995+
bool StoreTtlSettings(const TRule_table_setting_value& from, TResetableSetting<TTtlSettings, void>& to, TSqlExpression& expr, TContext& ctx,
1996+
TTranslation& txc) {
19681997
switch (from.Alt_case()) {
19691998
case TRule_table_setting_value::kAltTableSettingValue5: {
19701999
auto columnName = IdEx(from.GetAlt_table_setting_value5().GetRule_an_id3(), txc);
1971-
auto exprNode = expr.Build(from.GetAlt_table_setting_value5().GetRule_expr1());
1972-
if (!exprNode) {
2000+
auto tiersLiteral = from.GetAlt_table_setting_value5().GetRule_ttl_tier_list1();
2001+
2002+
TNodePtr firstInterval;
2003+
if (!FillTieringInterval(tiersLiteral.GetRule_expr1(), firstInterval, expr, ctx)) {
19732004
return false;
19742005
}
19752006

1976-
if (exprNode->GetOpName() != "Interval") {
1977-
ctx.Error() << "Literal of Interval type is expected for TTL";
1978-
return false;
2007+
std::vector<TTtlSettings::TTierSettings> tiers;
2008+
if (!tiersLiteral.HasBlock2()) {
2009+
tiers.emplace_back(firstInterval);
2010+
} else {
2011+
std::optional<TIdentifier> firstStorageName;
2012+
if (!FillTierAction(tiersLiteral.GetBlock2().GetRule_ttl_tier_action1(), firstStorageName, txc)) {
2013+
return false;
2014+
}
2015+
tiers.emplace_back(firstInterval, firstStorageName);
2016+
2017+
for (const auto& tierLiteral : tiersLiteral.GetBlock2().GetBlock2()) {
2018+
TNodePtr intervalExpr;
2019+
if (!FillTieringInterval(tierLiteral.GetRule_expr2(), intervalExpr, expr, ctx)) {
2020+
return false;
2021+
}
2022+
std::optional<TIdentifier> storageName;
2023+
if (!FillTierAction(tierLiteral.GetRule_ttl_tier_action3(), storageName, txc)) {
2024+
return false;
2025+
}
2026+
tiers.emplace_back(intervalExpr, storageName);
2027+
}
19792028
}
19802029

19812030
TMaybe<TTtlSettings::EUnit> columnUnit;
@@ -1988,7 +2037,7 @@ namespace {
19882037
}
19892038
}
19902039

1991-
to.Set(TTtlSettings(columnName, exprNode, columnUnit));
2040+
to.Set(TTtlSettings(columnName, tiers, columnUnit));
19922041
break;
19932042
}
19942043
default:

0 commit comments

Comments
 (0)