Merge 3bf480f into 7b9c7e3

Author: swalrus1

ydb/core/protos/flat_scheme_op.proto

@@ -221,11 +221,18 @@ message TTTLSettings {
         optional uint32 MaxShardsInFlight = 6 [default = 0]; // zero means no limit
     }
 
+    message TTier {
+        optional uint32 EvictAfterSeconds = 1;
+        optional string StorageName = 2;
+        optional TCompressionOptions Compression = 3;
+    }
+
     message TEnabled {
         optional string ColumnName = 1;
         optional uint32 ExpireAfterSeconds = 2;
         optional EUnit ColumnUnit = 3;
         optional TSysSettings SysSettings = 4;
+        repeated TTier Tiers = 5;
     }
 
     message TDisabled {
@@ -555,6 +562,7 @@ message TColumnDataLifeCycle {
             uint64 ExpireAfterBytes = 4;
         }
         optional TTTLSettings.EUnit ColumnUnit = 3;
+        repeated TTTLSettings.TTier Tiers = 5;
     }
 
     message TDisabled {

ydb/core/ydb_convert/table_settings.h

@@ -28,15 +28,25 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
         return false;
     };
 
+    static const auto& fillCommonFields = []<class TModeSettings>(TTtlSettingsEnabled& out, const TModeSettings& in) {
+        out.SetColumnName(in.column_name());
+        if (in.has_expire_after_seconds()) {
+            out.SetExpireAfterSeconds(in.expire_after_seconds());
+        }
+        for (const auto& in_tier : in.storage_tiers()) {
+            auto* out_tier = out.AddTiers();
+            out_tier->SetEvictAfterSeconds(in_tier.evict_after_seconds());
+            out_tier->SetStorageName(in_tier.storage_name());
+        }
+    };
+
     switch (in.mode_case()) {
     case Ydb::Table::TtlSettings::kDateTypeColumn:
-        out.SetColumnName(in.date_type_column().column_name());
-        out.SetExpireAfterSeconds(in.date_type_column().expire_after_seconds());
+        fillCommonFields(out, in.date_type_column());
         break;
 
     case Ydb::Table::TtlSettings::kValueSinceUnixEpoch:
-        out.SetColumnName(in.value_since_unix_epoch().column_name());
-        out.SetExpireAfterSeconds(in.value_since_unix_epoch().expire_after_seconds());
+        fillCommonFields(out, in.value_since_unix_epoch());
 
         #define CASE_UNIT(type) \
             case Ydb::Table::ValueSinceUnixEpochModeSettings::type: \

ydb/public/api/protos/ydb_table.proto

@@ -433,6 +433,13 @@ message ColumnMeta {
     }
 }
 
+message EvictionTier {
+    // Name of tiered storage
+    string storage_name = 1;
+
+    uint32 evict_after_seconds = 2;
+}
+
 message DateTypeColumnModeSettings {
     // The row will be considered as expired at the moment of time, when the value
     // stored in <column_name> is less than or equal to the current time (in epoch
@@ -442,7 +449,9 @@ message DateTypeColumnModeSettings {
     // The column type must be a date type
     string column_name = 1;
 
-    uint32 expire_after_seconds = 2;
+    optional uint32 expire_after_seconds = 2;
+
+    repeated EvictionTier storage_tiers = 3;
 }
 
 message ValueSinceUnixEpochModeSettings {
@@ -468,7 +477,9 @@ message ValueSinceUnixEpochModeSettings {
 
     // This option is always interpreted as seconds regardless of the
     // <column_unit> value.
-    uint32 expire_after_seconds = 3;
+    optional uint32 expire_after_seconds = 3;
+
+    repeated EvictionTier storage_tiers = 4;
 }
 
 message TtlSettings {
@@ -624,7 +635,7 @@ message CreateTableRequest {
     Ydb.FeatureFlag.Status key_bloom_filter = 16;
     // Read replicas settings for table
     ReadReplicasSettings read_replicas_settings = 17;
-    // Tiering rules name. It specifies how data migrates from one tier (logical storage) to another.
+    // Deprecated. Use TTL instead.
     string tiering = 18;
     // Is temporary table
     bool temporary = 19;
@@ -704,7 +715,7 @@ message AlterTableRequest {
     repeated string drop_changefeeds = 20;
     // Rename existed index
     repeated RenameIndexItem rename_indexes = 21;
-    // Setup or remove tiering
+    // Deprecated. Use ttl_action instead
     oneof tiering_action {
         string set_tiering = 22;
         google.protobuf.Empty drop_tiering = 23;

ydb/public/lib/experimental/ydb_logstore.cpp

@@ -199,8 +199,6 @@ void TLogTableDescription::SerializeTo(Ydb::LogStore::CreateLogTableRequest& req
 
     if (TtlSettings) {
         TtlSettings->SerializeTo(*request.mutable_ttl_settings());
-    } else if (TieringSettings) {
-        TieringSettings->SerializeTo(*request.mutable_tiering_settings());
     }
 }
 

ydb/public/lib/experimental/ydb_logstore.h

@@ -152,21 +152,6 @@ struct TLogTableSharding {
     TLogTableSharding(const Ydb::LogStore::DescribeLogTableResult& desc);
 };
 
-class TTieringSettings {
-private:
-    TString TieringId;
-public:
-    TTieringSettings(const TString& tieringId)
-        : TieringId(tieringId) {
-
-    }
-
-    void SerializeTo(Ydb::LogStore::TieringSettings& proto) const {
-        proto.set_tiering_id(TieringId);
-    }
-
-};
-
 class TLogTableDescription {
 public:
     TLogTableDescription(const TString& schemaPresetName, const TLogTableSharding& sharding);
@@ -200,16 +185,11 @@ class TLogTableDescription {
         TtlSettings = settings;
         return *this;
     }
-    TLogTableDescription& SetTieringSettings(const TTieringSettings& settings) {
-        TieringSettings = settings;
-        return *this;
-    }
 private:
     const TString SchemaPresetName;
     const TSchema Schema;
     const TLogTableSharding Sharding;
     TMaybe<TTtlSettings> TtlSettings;
-    TMaybe<TTieringSettings> TieringSettings;
     TString Owner;
     TVector<NScheme::TPermissions> Permissions;
     TVector<NScheme::TPermissions> EffectivePermissions;

ydb/public/sdk/cpp/client/ydb_table/table.cpp

@@ -345,11 +345,6 @@ class TTableDescription::TImpl {
             break;
         }
 
-        // tiering
-        if (proto.tiering().size()) {
-            Tiering_ = proto.tiering();
-        }
-
         if (proto.store_type()) {
             StoreType_ = (proto.store_type() == Ydb::Table::STORE_TYPE_COLUMN) ? EStoreType::Column : EStoreType::Row;
         }
@@ -580,10 +575,6 @@ class TTableDescription::TImpl {
         return TtlSettings_;
     }
 
-    const TMaybe<TString>& GetTiering() const {
-        return Tiering_;
-    }
-
     EStoreType GetStoreType() const {
         return StoreType_;
     }
@@ -664,7 +655,6 @@ class TTableDescription::TImpl {
     TVector<TIndexDescription> Indexes_;
     TVector<TChangefeedDescription> Changefeeds_;
     TMaybe<TTtlSettings> TtlSettings_;
-    TMaybe<TString> Tiering_;
     TString Owner_;
     TVector<NScheme::TPermissions> Permissions_;
     TVector<NScheme::TPermissions> EffectivePermissions_;
@@ -731,7 +721,7 @@ TMaybe<TTtlSettings> TTableDescription::GetTtlSettings() const {
 }
 
 TMaybe<TString> TTableDescription::GetTiering() const {
-    return Impl_->GetTiering();
+    return Nothing();
 }
 
 EStoreType TTableDescription::GetStoreType() const {
@@ -954,10 +944,6 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con
         ttl->SerializeTo(*request.mutable_ttl_settings());
     }
 
-    if (const auto& tiering = Impl_->GetTiering()) {
-        request.set_tiering(*tiering);
-    }
-
     if (Impl_->GetStoreType() == EStoreType::Column) {
         request.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN);
     }
@@ -2941,22 +2927,80 @@ bool operator!=(const TChangefeedDescription& lhs, const TChangefeedDescription&
 
 ////////////////////////////////////////////////////////////////////////////////
 
+namespace {
+
+TVector<TEvictionTierSettings> DeserializeTiers(const NProtoBuf::RepeatedPtrField<Ydb::Table::EvictionTier>& serialized) {
+    TVector<TEvictionTierSettings> tiers;
+    for (const auto& tier : serialized) {
+        tiers.push_back(TEvictionTierSettings(tier));
+    }
+    return tiers;
+}
+
+}
+
+TEvictionTierSettings::TEvictionTierSettings(TString storageName, TDuration evictionDelay)
+    : StorageName_(storageName)
+    , EvictionDelay_(evictionDelay)
+{}
+
+TEvictionTierSettings::TEvictionTierSettings(const Ydb::Table::EvictionTier& tier)
+    : StorageName_(tier.storage_name())
+    , EvictionDelay_(TDuration::Seconds(tier.evict_after_seconds()))
+{}
+
+void TEvictionTierSettings::SerializeTo(Ydb::Table::EvictionTier& proto) const {
+    proto.set_storage_name(StorageName_);
+    proto.set_evict_after_seconds(EvictionDelay_.Seconds());
+}
+
+const TString& TEvictionTierSettings::GetStorageName() const {
+    return StorageName_;
+}
+
+TDuration TEvictionTierSettings::GetEvictionDelay() const {
+    return EvictionDelay_;
+}
+
 TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const TString& columnName, const TDuration& expireAfter)
     : ColumnName_(columnName)
     , ExpireAfter_(expireAfter)
 {}
 
+TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const TString& columnName, const std::optional<TDuration>& expireAfter, const TVector<TEvictionTierSettings>& tiers)
+    : ColumnName_(columnName)
+    , ExpireAfter_(expireAfter)
+    , Tiers_(tiers)
+{}
+
 void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const {
     proto.set_column_name(ColumnName_);
-    proto.set_expire_after_seconds(ExpireAfter_.Seconds());
+    if (ExpireAfter_) {
+        proto.set_expire_after_seconds(ExpireAfter_->Seconds());
+    }
+    for (const auto& tier : Tiers_) {
+        tier.SerializeTo(*proto.add_storage_tiers());
+    }
 }
 
 const TString& TDateTypeColumnModeSettings::GetColumnName() const {
     return ColumnName_;
 }
 
 const TDuration& TDateTypeColumnModeSettings::GetExpireAfter() const {
-    return ExpireAfter_;
+    if (ExpireAfter_) {
+        return *ExpireAfter_;
+    }
+    static constexpr TDuration DurationMax = TDuration::Max();
+    return DurationMax;
+}
+
+bool TDateTypeColumnModeSettings::HasExpireAfter() const {
+    return !!ExpireAfter_;
+}
+
+const TVector<TEvictionTierSettings>& TDateTypeColumnModeSettings::GetTiers() const {
+    return Tiers_;
 }
 
 TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter)
@@ -2965,10 +3009,22 @@ TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString
     , ExpireAfter_(expireAfter)
 {}
 
+TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const TString& columnName, EUnit columnUnit, const std::optional<TDuration>& expireAfter, const TVector<TEvictionTierSettings>& tiers)
+    : ColumnName_(columnName)
+    , ColumnUnit_(columnUnit)
+    , ExpireAfter_(expireAfter)
+    , Tiers_(tiers)
+{}
+
 void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const {
     proto.set_column_name(ColumnName_);
     proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_));
-    proto.set_expire_after_seconds(ExpireAfter_.Seconds());
+    if (ExpireAfter_) {
+        proto.set_expire_after_seconds(ExpireAfter_->Seconds());
+    }
+    for (const auto& tier : Tiers_) {
+        tier.SerializeTo(*proto.Addstorage_tiers());
+    }
 }
 
 const TString& TValueSinceUnixEpochModeSettings::GetColumnName() const {
@@ -2980,7 +3036,19 @@ TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::GetCol
 }
 
 const TDuration& TValueSinceUnixEpochModeSettings::GetExpireAfter() const {
-    return ExpireAfter_;
+    if (ExpireAfter_) {
+        return *ExpireAfter_;
+    }
+    static constexpr TDuration DurationMax = TDuration::Max();
+    return DurationMax;
+}
+
+bool TValueSinceUnixEpochModeSettings::HasExpireAfter() const {
+    return !!ExpireAfter_;
+}
+
+const TVector<TEvictionTierSettings>& TValueSinceUnixEpochModeSettings::GetTiers() const {
+    return Tiers_;
 }
 
 void TValueSinceUnixEpochModeSettings::Out(IOutputStream& out, EUnit unit) {
@@ -3023,27 +3091,37 @@ TValueSinceUnixEpochModeSettings::EUnit TValueSinceUnixEpochModeSettings::UnitFr
     return EUnit::Unknown;
 }
 
+TTtlSettings::TTtlSettings(const TString& columnName, const std::optional<TDuration>& expireAfter, const TVector<TEvictionTierSettings>& tiers)
+    : Mode_(TDateTypeColumnModeSettings(columnName, expireAfter, tiers))
+{}
+
 TTtlSettings::TTtlSettings(const TString& columnName, const TDuration& expireAfter)
-    : Mode_(TDateTypeColumnModeSettings(columnName, expireAfter))
+    : TTtlSettings(columnName, expireAfter, {})
 {}
 
 TTtlSettings::TTtlSettings(const Ydb::Table::DateTypeColumnModeSettings& mode, ui32 runIntervalSeconds)
-    : TTtlSettings(mode.column_name(), TDuration::Seconds(mode.expire_after_seconds()))
-{
+    : TTtlSettings(mode.column_name(),
+          mode.has_expire_after_seconds() ? std::optional<TDuration>(TDuration::Seconds(mode.expire_after_seconds())) : std::nullopt,
+          DeserializeTiers(mode.storage_tiers())) {
     RunInterval_ = TDuration::Seconds(runIntervalSeconds);
 }
 
 const TDateTypeColumnModeSettings& TTtlSettings::GetDateTypeColumn() const {
     return std::get<TDateTypeColumnModeSettings>(Mode_);
 }
 
+TTtlSettings::TTtlSettings(const TString& columnName, EUnit columnUnit, const std::optional<TDuration>& expireAfter, const TVector<TEvictionTierSettings>& tiers)
+    : Mode_(TValueSinceUnixEpochModeSettings(columnName, columnUnit, expireAfter, tiers))
+{}
+
 TTtlSettings::TTtlSettings(const TString& columnName, EUnit columnUnit, const TDuration& expireAfter)
-    : Mode_(TValueSinceUnixEpochModeSettings(columnName, columnUnit, expireAfter))
+    : TTtlSettings(columnName, columnUnit, expireAfter, {})
 {}
 
 TTtlSettings::TTtlSettings(const Ydb::Table::ValueSinceUnixEpochModeSettings& mode, ui32 runIntervalSeconds)
-    : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()), TDuration::Seconds(mode.expire_after_seconds()))
-{
+    : TTtlSettings(mode.column_name(), TProtoAccessor::FromProto(mode.column_unit()),
+          mode.has_expire_after_seconds() ? std::optional<TDuration>(TDuration::Seconds(mode.expire_after_seconds())) : std::nullopt,
+          DeserializeTiers(mode.storage_tiers())) {
     RunInterval_ = TDuration::Seconds(runIntervalSeconds);
 }