Skip to content

Commit cde1863

Browse files
swalrus1swalrus1
authored
Merge f60784a into 7365427 
2 parents 7365427 + f60784a commit cde1863

File tree

17 files changed

+486
-182
lines changed

17 files changed

+486
-182
lines changed

‎ydb/core/grpc_services/rpc_log_store.cpp‎

Lines changed: 1 addition & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -508,30 +508,7 @@ class TDescribeLogTableRPC : public TRpcSchemeRequestActor<TDescribeLogTableRPC,
508508
}
509509

510510
if (tableDescription.HasTtlSettings() && tableDescription.GetTtlSettings().HasEnabled()) {
511-
const auto& inTTL = tableDescription.GetTtlSettings().GetEnabled();
512-
513-
switch (inTTL.GetColumnUnit()) {
514-
case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
515-
auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_date_type_column();
516-
outTTL.set_column_name(inTTL.GetColumnName());
517-
outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
518-
break;
519-
}
520-
521-
case NKikimrSchemeOp::TTTLSettings::UNIT_SECONDS:
522-
case NKikimrSchemeOp::TTTLSettings::UNIT_MILLISECONDS:
523-
case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS:
524-
case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: {
525-
auto& outTTL = *describeLogTableResult.mutable_ttl_settings()->mutable_value_since_unix_epoch();
526-
outTTL.set_column_name(inTTL.GetColumnName());
527-
outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit()));
528-
outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
529-
break;
530-
}
531-
532-
default:
533-
break;
534-
}
511+
FillTtlSettings(*describeLogTableResult.mutable_ttl_settings(), tableDescription.GetTtlSettings().GetEnabled());
535512
}
536513

537514
return ReplyWithResult(Ydb::StatusIds::SUCCESS, describeLogTableResult, ctx);

‎ydb/core/kqp/provider/yql_kikimr_gateway.cpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -313,13 +313,14 @@ void ConvertTtlSettingsToProto(const NYql::TTtlSettings& settings, Ydb::Table::T
313313
if (!settings.ColumnUnit) {
314314
auto& opts = *proto.mutable_date_type_column();
315315
opts.set_column_name(settings.ColumnName);
316-
opts.set_expire_after_seconds(settings.ExpireAfter.Seconds());
317316
} else {
318317
auto& opts = *proto.mutable_value_since_unix_epoch();
319318
opts.set_column_name(settings.ColumnName);
320319
opts.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(*settings.ColumnUnit));
321-
opts.set_expire_after_seconds(settings.ExpireAfter.Seconds());
322320
}
321+
auto* deleteTier = proto.add_tiers();
322+
deleteTier->set_evict_after_seconds(settings.ExpireAfter.Seconds());
323+
deleteTier->mutable_delete_();
323324
}
324325

325326
Ydb::FeatureFlag::Status GetFlagValue(const TMaybe<bool>& value) {

‎ydb/core/persqueue/writer/metadata_initializers.cpp‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,10 @@ void TSrcIdMetaInitializer::DoPrepare(NInitializer::IInitializerInput::TPtr cont
5555
auto* ttlSettings = request.mutable_ttl_settings();
5656
auto* columnTtl = ttlSettings->mutable_value_since_unix_epoch();
5757
columnTtl->set_column_name("AccessTime");
58-
columnTtl->set_expire_after_seconds(1382400);
5958
columnTtl->set_column_unit(Ydb::Table::ValueSinceUnixEpochModeSettings::UNIT_MILLISECONDS);
59+
auto* deleteTier = ttlSettings->add_tiers();
60+
deleteTier->mutable_delete_();
61+
deleteTier->set_evict_after_seconds(1382400);
6062
}
6163

6264
result.emplace_back(new NInitializer::TGenericTableModifier<NRequest::TDialogCreateTable>(request, "create"));

‎ydb/core/protos/flat_scheme_op.proto‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -221,11 +221,24 @@ message TTTLSettings {
221221
optional uint32 MaxShardsInFlight = 6 [default = 0]; // zero means no limit
222222
}
223223

224+
message TEvictionToExternalStorageSettings {
225+
optional string StorageName = 1;
226+
}
227+
228+
message TTier {
229+
optional uint32 EvictAfterSeconds = 1;
230+
oneof Action {
231+
google.protobuf.Empty Delete = 2;
232+
TEvictionToExternalStorageSettings EvictToExternalStorage = 3;
233+
}
234+
}
235+
224236
message TEnabled {
225237
optional string ColumnName = 1;
226-
optional uint32 ExpireAfterSeconds = 2;
238+
optional uint32 ExpireAfterSeconds = 2 [deprecated = true];
227239
optional EUnit ColumnUnit = 3;
228240
optional TSysSettings SysSettings = 4;
241+
repeated TTier Tiers = 5;
229242
}
230243

231244
message TDisabled {
@@ -570,10 +583,11 @@ message TColumnDataLifeCycle {
570583
message TTtl {
571584
optional string ColumnName = 1;
572585
oneof Expire {
573-
uint32 ExpireAfterSeconds = 2;
586+
uint32 ExpireAfterSeconds = 2 [deprecated = true];
574587
uint64 ExpireAfterBytes = 4;
575588
}
576589
optional TTTLSettings.EUnit ColumnUnit = 3;
590+
repeated TTTLSettings.TTier Tiers = 5;
577591
}
578592

579593
message TDisabled {

‎ydb/core/tx/schemeshard/schemeshard__conditional_erase.cpp‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,20 @@ struct TSchemeShard::TTxRunConditionalErase: public TSchemeShard::TRwTxBase {
141141
}
142142

143143
const auto& settings = tableInfo->TTLSettings().GetEnabled();
144-
const TDuration expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds());
144+
145+
TDuration expireAfter;
146+
if (settings.TiersSize()) {
147+
if (settings.TiersSize() > 1 || !settings.GetTiers(0).HasDelete()) {
148+
LOG_WARN_S(ctx, NKikimrServices::FLAT_TX_SCHEMESHARD, "Invalid ttl setting: external storage tiers"
149+
<< ": shardIdx: " << tableShardInfo.ShardIdx
150+
<< ": pathId: " << shardInfo.PathId
151+
<< ", at schemeshard: " << Self->TabletID());
152+
return false;
153+
}
154+
expireAfter = TDuration::Seconds(settings.GetTiers(0).GetEvictAfterSeconds());
155+
} else {
156+
expireAfter = TDuration::Seconds(settings.GetExpireAfterSeconds());
157+
}
145158
const TInstant wallClock = ctx.Now() - expireAfter;
146159

147160
NKikimrTxDataShard::TEvConditionalEraseRowsRequest request;

‎ydb/core/tx/schemeshard/schemeshard_validate_ttl.cpp‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,8 +58,19 @@ bool ValidateTtlSettings(const NKikimrSchemeOp::TTTLSettings& ttl,
5858
return false;
5959
}
6060

61+
ui32 expireInSeconds;
62+
if (enabled.TiersSize()) {
63+
if (enabled.TiersSize() > 1 || !enabled.GetTiers(0).HasDelete()) {
64+
errStr = Sprintf("Eviction to external storage via TTL is not allowed for row-oriented tables");
65+
return false;
66+
}
67+
expireInSeconds = enabled.GetTiers(0).GetEvictAfterSeconds();
68+
} else {
69+
expireInSeconds = enabled.GetExpireAfterSeconds();
70+
}
71+
6172
const TInstant now = TInstant::Now();
62-
if (enabled.GetExpireAfterSeconds() > now.Seconds()) {
73+
if (expireInSeconds > now.Seconds()) {
6374
errStr = Sprintf("TTL should be less than %" PRIu64 " seconds (%" PRIu64 " days, %" PRIu64 " years). The ttl behaviour is undefined before 1970.", now.Seconds(), now.Days(), now.Days() / 365);
6475
return false;
6576
}

‎ydb/core/tx/schemeshard/ut_helpers/ls_checks.cpp‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1139,6 +1139,9 @@ TCheckFunc HasTtlEnabled(const TString& columnName, const TDuration& expireAfter
11391139
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), columnName);
11401140
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnUnit(), columnUnit);
11411141
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), expireAfter.Seconds());
1142+
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 1);
1143+
UNIT_ASSERT(ttl.GetEnabled().GetTiers(0).HasDelete());
1144+
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetTiers(0).GetEvictAfterSeconds(), expireAfter.Seconds());
11421145
};
11431146
}
11441147

‎ydb/core/tx/schemeshard/ut_ttl/ut_ttl.cpp‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ void CheckTtlSettings(const TTtlSettings& ttl, const char* ttlColumnName) {
1414
UNIT_ASSERT(ttl.HasEnabled());
1515
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetColumnName(), ttlColumnName);
1616
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetExpireAfterSeconds(), 3600);
17+
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().TiersSize(), 1);
18+
UNIT_ASSERT(ttl.GetEnabled().GetTiers(0).HasDelete());
19+
UNIT_ASSERT_VALUES_EQUAL(ttl.GetEnabled().GetTiers(0).GetEvictAfterSeconds(), 3600);
1720
}
1821

1922
void OltpTtlChecker(const NKikimrScheme::TEvDescribeSchemeResult& record) {
@@ -54,6 +57,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
5457
ColumnName: "modified_at"
5558
ExpireAfterSeconds: 3600
5659
ColumnUnit: %s
60+
Tiers: {
61+
EvictAfterSeconds: 3600
62+
Delete: {}
63+
}
5764
}
5865
}
5966
)", name, ttlColumnType, unit));
@@ -185,6 +192,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
185192
Enabled {
186193
ColumnName: "modified_at"
187194
ExpireAfterSeconds: 3153600000
195+
Tiers: {
196+
EvictAfterSeconds: 3153600000
197+
Delete: {}
198+
}
188199
}
189200
}
190201
)", {NKikimrScheme::StatusSchemeError});
@@ -205,6 +216,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
205216
Enabled {
206217
ColumnName: "modified_at"
207218
ExpireAfterSeconds: 3600
219+
Tiers: {
220+
EvictAfterSeconds: 3600
221+
Delete: {}
222+
}
208223
}
209224
}
210225
}
@@ -246,6 +261,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
246261
Enabled {
247262
ColumnName: "modified_at"
248263
ExpireAfterSeconds: 3600
264+
Tiers: {
265+
EvictAfterSeconds: 3600
266+
Delete: {}
267+
}
249268
}
250269
}
251270
)");
@@ -296,6 +315,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
296315
Enabled {
297316
ColumnName: "modified_at"
298317
ExpireAfterSeconds: 3600
318+
Tiers: {
319+
EvictAfterSeconds: 3600
320+
Delete: {}
321+
}
299322
}
300323
}
301324
)");
@@ -353,6 +376,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
353376
Enabled {
354377
ColumnName: "modified_at"
355378
ExpireAfterSeconds: 3600
379+
Tiers: {
380+
EvictAfterSeconds: 3600
381+
Delete: {}
382+
}
356383
}
357384
}
358385
)");
@@ -384,6 +411,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
384411
Enabled {
385412
ColumnName: "modified_at"
386413
ExpireAfterSeconds: 3600
414+
Tiers: {
415+
EvictAfterSeconds: 3600
416+
Delete: {}
417+
}
387418
}
388419
}
389420
)");
@@ -524,6 +555,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
524555
Enabled {
525556
ColumnName: "ts"
526557
ExpireAfterSeconds: 3600
558+
Tiers: {
559+
EvictAfterSeconds: 3600
560+
Delete: {}
561+
}
527562
}
528563
}
529564
)");
@@ -785,6 +820,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
785820
Enabled {
786821
ColumnName: "modified_at"
787822
ExpireAfterSeconds: 3600
823+
Tiers: {
824+
EvictAfterSeconds: 3600
825+
Delete: {}
826+
}
788827
}
789828
}
790829
)");
@@ -803,6 +842,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
803842
SysSettings {
804843
RunInterval: 1800000000
805844
}
845+
Tiers: {
846+
EvictAfterSeconds: 3600
847+
Delete: {}
848+
}
806849
}
807850
}
808851
)");
@@ -821,6 +864,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTests) {
821864
SysSettings {
822865
RunInterval: 1799999999
823866
}
867+
Tiers: {
868+
EvictAfterSeconds: 3600
869+
Delete: {}
870+
}
824871
}
825872
}
826873
)", {NKikimrScheme::StatusSchemeError});
@@ -1089,6 +1136,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) {
10891136
ColumnName: "modified_at"
10901137
ExpireAfterSeconds: 3600
10911138
ColumnUnit: %s
1139+
Tiers: {
1140+
EvictAfterSeconds: 3600
1141+
Delete: {}
1142+
}
10921143
}
10931144
}
10941145
)", name, ttlColumnType, unit));
@@ -1125,6 +1176,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) {
11251176
Enabled {
11261177
ColumnName: "modified_at"
11271178
ExpireAfterSeconds: 3600
1179+
Tiers: {
1180+
EvictAfterSeconds: 3600
1181+
Delete: {}
1182+
}
11281183
}
11291184
}
11301185
)", ct), {NKikimrScheme::StatusSchemeError});
@@ -1183,6 +1238,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) {
11831238
Enabled {
11841239
ColumnName: "modified_at"
11851240
ExpireAfterSeconds: 3600
1241+
Tiers: {
1242+
EvictAfterSeconds: 3600
1243+
Delete: {}
1244+
}
11861245
}
11871246
}
11881247
)");
@@ -1247,6 +1306,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardColumnTableTTL) {
12471306
Enabled {
12481307
ColumnName: "str"
12491308
ExpireAfterSeconds: 3600
1309+
Tiers: {
1310+
EvictAfterSeconds: 3600
1311+
Delete: {}
1312+
}
12501313
}
12511314
}
12521315
)", {NKikimrScheme::StatusSchemeError});
@@ -1266,6 +1329,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) {
12661329
Enabled {
12671330
ColumnName: "modified_at"
12681331
ExpireAfterSeconds: 3600
1332+
Tiers: {
1333+
EvictAfterSeconds: 3600
1334+
Delete: {}
1335+
}
12691336
}
12701337
}
12711338
)");
@@ -1298,6 +1365,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) {
12981365
Enabled {
12991366
ColumnName: "modified_at"
13001367
ExpireAfterSeconds: 3600
1368+
Tiers: {
1369+
EvictAfterSeconds: 3600
1370+
Delete: {}
1371+
}
13011372
}
13021373
}
13031374
)");
@@ -1324,6 +1395,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) {
13241395
Enabled {
13251396
ColumnName: "modified_at"
13261397
ExpireAfterSeconds: 3600
1398+
Tiers: {
1399+
EvictAfterSeconds: 3600
1400+
Delete: {}
1401+
}
13271402
}
13281403
}
13291404
)");
@@ -1354,6 +1429,10 @@ Y_UNIT_TEST_SUITE(TSchemeShardTTLTestsWithReboots) {
13541429
Enabled {
13551430
ColumnName: "modified_at"
13561431
ExpireAfterSeconds: 3600
1432+
Tiers: {
1433+
EvictAfterSeconds: 3600
1434+
Delete: {}
1435+
}
13571436
}
13581437
}
13591438
)");

0 commit comments

Comments
 (0)