Merge 3536ce2 into 481ccad

Author: swalrus1

ydb/core/protos/flat_scheme_op.proto

@@ -221,11 +221,24 @@ message TTTLSettings {
         optional uint32 MaxShardsInFlight = 6 [default = 0]; // zero means no limit
     }
 
+    message TEvictionToExternalStorageSettings {
+        optional string StorageName = 1;
+    }
+
+    message TTier {
+        optional uint32 EvictAfterSeconds = 1;
+        oneof Action {
+            google.protobuf.Empty Delete = 2;
+            TEvictionToExternalStorageSettings EvictToExternalStorage = 3;
+        }
+    }
+
     message TEnabled {
         optional string ColumnName = 1;
-        optional uint32 ExpireAfterSeconds = 2;
+        optional uint32 ExpireAfterSeconds = 2 [deprecated = true];
         optional EUnit ColumnUnit = 3;
         optional TSysSettings SysSettings = 4;
+        repeated TTier Tiers = 5;
     }
 
     message TDisabled {
@@ -570,10 +583,11 @@ message TColumnDataLifeCycle {
     message TTtl {
         optional string ColumnName = 1;
         oneof Expire {
-            uint32 ExpireAfterSeconds = 2;
+            uint32 ExpireAfterSeconds = 2 [deprecated = true];
             uint64 ExpireAfterBytes = 4;
         }
         optional TTTLSettings.EUnit ColumnUnit = 3;
+        repeated TTTLSettings.TTier Tiers = 5;
     }
 
     message TDisabled {

ydb/core/ydb_convert/table_description.cpp

@@ -48,8 +48,6 @@ THashSet<EAlterOperationKind> GetAlterOperationKinds(const Ydb::Table::AlterTabl
         req->alter_columns_size() ||
         req->ttl_action_case() !=
             Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET ||
-        req->tiering_action_case() !=
-            Ydb::Table::AlterTableRequest::TIERING_ACTION_NOT_SET ||
         req->has_alter_storage_settings() || req->add_column_families_size() ||
         req->alter_column_families_size() || req->set_compaction_policy() ||
         req->has_alter_partitioning_settings() ||
@@ -509,11 +507,25 @@ Ydb::Type* AddColumn<NKikimrSchemeOp::TColumnDescription>(Ydb::Table::ColumnMeta
 
 template <typename TYdbProto, typename TTtl>
 static void AddTtl(TYdbProto& out, const TTtl& inTTL) {
+    for (const auto& inTier : inTTL.GetTiers()) {
+        auto* outTier = out.mutable_ttl_settings()->add_tiers();
+        outTier->set_evict_after_seconds(inTier.GetEvictAfterSeconds());
+        switch (inTier.GetActionCase()) {
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kDelete:
+                outTier->mutable_delete_();
+                break;
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kEvictToExternalStorage:
+                outTier->mutable_evict_to_external_storage()->set_storage_name(inTier.GetEvictToExternalStorage().GetStorageName());
+                break;
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::ACTION_NOT_SET:
+                break;
+        }
+    }
+
     switch (inTTL.GetColumnUnit()) {
     case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
         auto& outTTL = *out.mutable_ttl_settings()->mutable_date_type_column();
         outTTL.set_column_name(inTTL.GetColumnName());
-        outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
         break;
     }
 
@@ -524,7 +536,6 @@ static void AddTtl(TYdbProto& out, const TTtl& inTTL) {
         auto& outTTL = *out.mutable_ttl_settings()->mutable_value_since_unix_epoch();
         outTTL.set_column_name(inTTL.GetColumnName());
         outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit()));
-        outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
         break;
     }
 
@@ -572,10 +583,6 @@ void FillColumnDescriptionImpl(TYdbProto& out,
         if (in.GetTTLSettings().HasEnabled()) {
             AddTtl(out, in.GetTTLSettings().GetEnabled());
         }
-
-        if (in.GetTTLSettings().HasUseTiering()) {
-            out.set_tiering(in.GetTTLSettings().GetUseTiering());
-        }
     }
 }
 
@@ -612,10 +619,6 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc
         if (in.GetTtlSettings().HasEnabled()) {
             AddTtl(out, in.GetTtlSettings().GetEnabled());
         }
-
-        if (in.GetTtlSettings().HasUseTiering()) {
-            out.set_tiering(in.GetTtlSettings().GetUseTiering());
-        }
     }
 
     out.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN);
@@ -829,12 +832,6 @@ bool BuildAlterColumnTableModifyScheme(const TString& path, const Ydb::Table::Al
         } else if (req->has_drop_ttl_settings()) {
             alterColumnTable->MutableAlterTtlSettings()->MutableDisabled();
         }
-
-        if (req->has_set_tiering()) {
-            alterColumnTable->MutableAlterTtlSettings()->SetUseTiering(req->set_tiering());
-        } else if (req->has_drop_tiering()) {
-            alterColumnTable->MutableAlterTtlSettings()->SetUseTiering("");
-        }
     }
 
     return true;

ydb/core/ydb_convert/table_settings.h

@@ -28,22 +28,23 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
         return false;
     };
 
-    switch (in.mode_case()) {
-    case Ydb::Table::TtlSettings::kDateTypeColumn:
-        out.SetColumnName(in.date_type_column().column_name());
-        out.SetExpireAfterSeconds(in.date_type_column().expire_after_seconds());
-        break;
+    static const auto& fillModeSettings = []<class TModeSettings>(TTtlSettingsEnabled& out, const TModeSettings& in) {
+        out.SetColumnName(in.column_name());
+    };
 
-    case Ydb::Table::TtlSettings::kValueSinceUnixEpoch:
-        out.SetColumnName(in.value_since_unix_epoch().column_name());
-        out.SetExpireAfterSeconds(in.value_since_unix_epoch().expire_after_seconds());
+    static const auto& fillDeleteTier = []<class TModeSettings>(TTtlSettingsEnabled& out, const TModeSettings& in) {
+        auto* deleteTier = out.AddTiers();
+        deleteTier->SetEvictAfterSeconds(in.expire_after_seconds());
+        deleteTier->MutableDelete();
+    };
 
+    static const auto& fillColumnUnit = []<class TModeSettings> (TTtlSettingsEnabled& out, const TModeSettings& in) -> bool {
         #define CASE_UNIT(type) \
             case Ydb::Table::ValueSinceUnixEpochModeSettings::type: \
                 out.SetColumnUnit(NKikimrSchemeOp::TTTLSettings::type); \
                 break
 
-        switch (in.value_since_unix_epoch().column_unit()) {
+        switch (in.column_unit()) {
         CASE_UNIT(UNIT_SECONDS);
         CASE_UNIT(UNIT_MILLISECONDS);
         CASE_UNIT(UNIT_MICROSECONDS);
@@ -54,12 +55,62 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
         }
 
         #undef CASE_UNIT
+    };
+
+    if (in.tiers_size()) {
+        for (const auto& inTier : in.tiers()) {
+            auto* outTier = out.AddTiers();
+            outTier->SetEvictAfterSeconds(inTier.evict_after_seconds());
+            switch (inTier.action_case()) {
+                case Ydb::Table::TtlTier::kDelete:
+                    outTier->MutableDelete();
+                    break;
+                case Ydb::Table::TtlTier::kEvictToExternalStorage:
+                    outTier->MutableEvictToExternalStorage()->SetStorageName(inTier.evict_to_external_storage().storage_name());
+                    break;
+                case Ydb::Table::TtlTier::ACTION_NOT_SET:
+                    break;
+            }
+        }
+    }
+
+    switch (in.mode_case()) {
+    case Ydb::Table::TtlSettings::kDeprecatedDateTypeColumn:
+        fillModeSettings(out, in.deprecated_date_type_column());
+        fillDeleteTier(out, in.deprecated_date_type_column());
         break;
 
-    default:
+    case Ydb::Table::TtlSettings::kDeprecatedValueSinceUnixEpoch:
+        fillModeSettings(out, in.deprecated_value_since_unix_epoch());
+        fillDeleteTier(out, in.deprecated_date_type_column());
+        if (!fillColumnUnit(out, in.value_since_unix_epoch())) {
+            return false;
+        }
+        break;
+
+    case Ydb::Table::TtlSettings::kDateTypeColumn:
+        fillModeSettings(out, in.date_type_column());
+        break;
+
+    case Ydb::Table::TtlSettings::kValueSinceUnixEpoch:
+        fillModeSettings(out, in.value_since_unix_epoch());
+        if (!fillColumnUnit(out, in.value_since_unix_epoch())) {
+            return false;
+        }
+        break;
+
+    case Ydb::Table::TtlSettings::MODE_NOT_SET:
         return unsupported("Unsupported ttl settings");
     }
 
+    std::optional<ui32> expireInSeconds = 0;
+    for (const auto& tier : out.GetTiers()) {
+        if (tier.HasDelete()) {
+            expireInSeconds = tier.GetEvictInSeconds();
+        }
+    }
+    out.SetExpireInSeconds(expireInSeconds.value_or(std::numeric_limits<uint32_t>::max()));
+
     if constexpr (std::is_same_v<TTtlSettingsEnabled, NKikimrSchemeOp::TTTLSettings::TEnabled>) {
         if (in.run_interval_seconds()) {
             out.MutableSysSettings()->SetRunInterval(TDuration::Seconds(in.run_interval_seconds()).GetValue());

ydb/public/api/protos/ydb_table.proto

@@ -433,16 +433,33 @@ message ColumnMeta {
     }
 }
 
+message EvictionToExternalStorageSettings {
+    // Path to external data source
+    string storage_name = 1;
+}
+
+message EvictionToFamilySettings {
+    // Name of column family
+    string family_name = 1;
+}
+
+message TtlTier {
+    uint32 evict_after_seconds = 1;
+
+    oneof action {
+        google.protobuf.Empty delete = 2;
+        EvictionToExternalStorageSettings evict_to_external_storage = 3;
+    }
+}
+
 message DateTypeColumnModeSettings {
-    // The row will be considered as expired at the moment of time, when the value
+    // The row will be assigned a tier at the moment of time, when the value
     // stored in <column_name> is less than or equal to the current time (in epoch
-    // time format), and <expire_after_seconds> has passed since that moment;
-    // i.e. the expiration threshold is the value of <column_name> plus <expire_after_seconds>.
+    // time format), and <tier.evict_after_seconds> has passed since that moment;
+    // i.e. the eviction threshold is the value of <column_name> plus <tier.evict_after_seconds>.
 
     // The column type must be a date type
     string column_name = 1;
-
-    uint32 expire_after_seconds = 2;
 }
 
 message ValueSinceUnixEpochModeSettings {
@@ -465,16 +482,27 @@ message ValueSinceUnixEpochModeSettings {
 
     // Interpretation of the value stored in <column_name>
     Unit column_unit = 2;
+}
 
-    // This option is always interpreted as seconds regardless of the
-    // <column_unit> value.
+message DeprecatedDateTypeColumnModeSettings {
+    // Deprecated TTL mode for internal use only
+    string column_name = 1;
+    uint32 expire_after_seconds = 2;
+}
+
+message DeprecatedValueSinceUnixEpochModeSettings {
+    // Deprecated TTL mode for internal use only
+    string column_name = 1;
+    ValueSinceUnixEpochModeSettings.Unit column_unit = 2;
     uint32 expire_after_seconds = 3;
 }
 
 message TtlSettings {
     oneof mode {
-        DateTypeColumnModeSettings date_type_column = 1;
-        ValueSinceUnixEpochModeSettings value_since_unix_epoch = 2;
+        DeprecatedDateTypeColumnModeSettings deprecated_date_type_column = 1 [deprecated = true];
+        DeprecatedValueSinceUnixEpochModeSettings deprecated_value_since_unix_epoch = 2 [deprecated = true];
+        DateTypeColumnModeSettings date_type_column = 4;
+        ValueSinceUnixEpochModeSettings value_since_unix_epoch = 5;
     }
 
     // There is no guarantee that expired row will be deleted immediately upon
@@ -490,6 +518,8 @@ message TtlSettings {
     // How often to run BRO on the same partition.
     // BRO will not be started more often, but may be started less often.
     uint32 run_interval_seconds = 3;
+
+    repeated TtlTier tiers = 6;
 }
 
 message StorageSettings {
@@ -624,8 +654,7 @@ message CreateTableRequest {
     Ydb.FeatureFlag.Status key_bloom_filter = 16;
     // Read replicas settings for table
     ReadReplicasSettings read_replicas_settings = 17;
-    // Tiering rules name. It specifies how data migrates from one tier (logical storage) to another.
-    string tiering = 18;
+    reserved 18; // tiering
     // Is temporary table
     bool temporary = 19;
     // Is table column or row oriented
@@ -704,11 +733,7 @@ message AlterTableRequest {
     repeated string drop_changefeeds = 20;
     // Rename existed index
     repeated RenameIndexItem rename_indexes = 21;
-    // Setup or remove tiering
-    oneof tiering_action {
-        string set_tiering = 22;
-        google.protobuf.Empty drop_tiering = 23;
-    }
+    reserved 22, 23; // tiering_action
 }
 
 message AlterTableResponse {

ydb/public/lib/experimental/ydb_logstore.cpp

@@ -16,23 +16,10 @@ namespace NYdb {
 namespace NLogStore {
 
 TMaybe<TTtlSettings> TtlSettingsFromProto(const Ydb::Table::TtlSettings& proto) {
-    switch (proto.mode_case()) {
-    case Ydb::Table::TtlSettings::kDateTypeColumn:
-        return TTtlSettings(
-            proto.date_type_column(),
-            proto.run_interval_seconds()
-        );
-
-    case Ydb::Table::TtlSettings::kValueSinceUnixEpoch:
-        return TTtlSettings(
-            proto.value_since_unix_epoch(),
-            proto.run_interval_seconds()
-        );
-
-    default:
-        break;
+    if (auto settings = TTtlSettings::DeserializeFromProto(proto)) {
+        return *settings;
     }
-    return {};
+    return Nothing();
 }
 
 static TCompression CompressionFromProto(const Ydb::LogStore::Compression& compression) {
@@ -199,8 +186,6 @@ void TLogTableDescription::SerializeTo(Ydb::LogStore::CreateLogTableRequest& req
 
     if (TtlSettings) {
         TtlSettings->SerializeTo(*request.mutable_ttl_settings());
-    } else if (TieringSettings) {
-        TieringSettings->SerializeTo(*request.mutable_tiering_settings());
     }
 }
 

ydb/public/lib/experimental/ydb_logstore.h

@@ -152,21 +152,6 @@ struct TLogTableSharding {
     TLogTableSharding(const Ydb::LogStore::DescribeLogTableResult& desc);
 };
 
-class TTieringSettings {
-private:
-    TString TieringId;
-public:
-    TTieringSettings(const TString& tieringId)
-        : TieringId(tieringId) {
-
-    }
-
-    void SerializeTo(Ydb::LogStore::TieringSettings& proto) const {
-        proto.set_tiering_id(TieringId);
-    }
-
-};
-
 class TLogTableDescription {
 public:
     TLogTableDescription(const TString& schemaPresetName, const TLogTableSharding& sharding);
@@ -200,16 +185,11 @@ class TLogTableDescription {
         TtlSettings = settings;
         return *this;
     }
-    TLogTableDescription& SetTieringSettings(const TTieringSettings& settings) {
-        TieringSettings = settings;
-        return *this;
-    }
 private:
     const TString SchemaPresetName;
     const TSchema Schema;
     const TLogTableSharding Sharding;
     TMaybe<TTtlSettings> TtlSettings;
-    TMaybe<TTieringSettings> TieringSettings;
     TString Owner;
     TVector<NScheme::TPermissions> Permissions;
     TVector<NScheme::TPermissions> EffectivePermissions;