Merge eea3575 into d353cb0

swalrus1 · web-flow · commit f8d448e1138a · 2024-11-13T08:17:02.000Z
diff --git a/ydb/core/protos/flat_scheme_op.proto b/ydb/core/protos/flat_scheme_op.proto
@@ -221,11 +221,29 @@ message TTTLSettings {
         optional uint32 MaxShardsInFlight = 6 [default = 0]; // zero means no limit
     }
 
+    message TEvictionToExternalStorageSettings {
+        optional string StorageName = 1;
+    }
+
+    message TEvictionToColumnFamilySettings {
+        optional string FamilyName = 1;
+    }
+
+    message TTier {
+        optional uint32 EvictAfterSeconds = 1;
+        oneof Action {
+            google.protobuf.Empty Delete = 2;
+            TEvictionToExternalStorageSettings EvictToExternalStorage = 3;
+            TEvictionToColumnFamilySettings EvictToColumnFamily = 4;
+        }
+    }
+
     message TEnabled {
         optional string ColumnName = 1;
-        optional uint32 ExpireAfterSeconds = 2;
+        optional uint32 ExpireAfterSeconds = 2 [deprecated = true];
         optional EUnit ColumnUnit = 3;
         optional TSysSettings SysSettings = 4;
+        repeated TTier Tiers = 5;
     }
 
     message TDisabled {
@@ -570,10 +588,11 @@ message TColumnDataLifeCycle {
     message TTtl {
         optional string ColumnName = 1;
         oneof Expire {
-            uint32 ExpireAfterSeconds = 2;
+            uint32 ExpireAfterSeconds = 2 [deprecated = true];
             uint64 ExpireAfterBytes = 4;
         }
         optional TTTLSettings.EUnit ColumnUnit = 3;
+        repeated TTTLSettings.TTier Tiers = 5;
     }
 
     message TDisabled {
diff --git a/ydb/core/ydb_convert/table_description.cpp b/ydb/core/ydb_convert/table_description.cpp
@@ -48,8 +48,6 @@ THashSet<EAlterOperationKind> GetAlterOperationKinds(const Ydb::Table::AlterTabl
         req->alter_columns_size() ||
         req->ttl_action_case() !=
             Ydb::Table::AlterTableRequest::TTL_ACTION_NOT_SET ||
-        req->tiering_action_case() !=
-            Ydb::Table::AlterTableRequest::TIERING_ACTION_NOT_SET ||
         req->has_alter_storage_settings() || req->add_column_families_size() ||
         req->alter_column_families_size() || req->set_compaction_policy() ||
         req->has_alter_partitioning_settings() ||
@@ -509,11 +507,45 @@ Ydb::Type* AddColumn<NKikimrSchemeOp::TColumnDescription>(Ydb::Table::ColumnMeta
 
 template <typename TYdbProto, typename TTtl>
 static void AddTtl(TYdbProto& out, const TTtl& inTTL) {
+    static const auto& fillCommonFields = []<class TModeSettings>(TModeSettings& out, const TTtl& in) {
+        out.set_column_name(in.GetColumnName());
+        if (in.TiersSize()) {
+            std::optional<ui32> expireInSeconds;
+            for (const auto& inTier : in.GetTiers()) {
+                if (inTier.HasDelete()) {
+                    expireInSeconds = inTier.GetEvictAfterSeconds();
+                    break;
+                }
+            }
+            out.set_expire_after_seconds(expireInSeconds.value_or(std::numeric_limits<uint32_t>::max()));
+        } else {
+            // legacy schema
+            out.set_expire_after_seconds(in.GetExpireAfterSeconds());
+        }
+    };
+
+    for (const auto& inTier : inTTL.GetTiers()) {
+        auto* outTier = out.mutable_ttl_settings()->add_tiers();
+        outTier->set_evict_after_seconds(inTier.GetEvictAfterSeconds());
+        switch (inTier.GetActionCase()) {
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kDelete:
+                outTier->mutable_delete_();
+                break;
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kEvictToExternalStorage:
+                outTier->mutable_evict_to_external_storage()->set_storage_name(inTier.GetEvictToExternalStorage().GetStorageName());
+                break;
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::kEvictToColumnFamily:
+                outTier->mutable_evict_to_column_family()->set_family_name(inTier.GetEvictToColumnFamily().GetFamilyName());
+                break;
+            case NKikimrSchemeOp::TTTLSettings::TTier::ActionCase::ACTION_NOT_SET:
+                break;
+        }
+    }
+
     switch (inTTL.GetColumnUnit()) {
     case NKikimrSchemeOp::TTTLSettings::UNIT_AUTO: {
         auto& outTTL = *out.mutable_ttl_settings()->mutable_date_type_column();
-        outTTL.set_column_name(inTTL.GetColumnName());
-        outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
+        fillCommonFields(outTTL, inTTL);
         break;
     }
 
@@ -522,9 +554,8 @@ static void AddTtl(TYdbProto& out, const TTtl& inTTL) {
     case NKikimrSchemeOp::TTTLSettings::UNIT_MICROSECONDS:
     case NKikimrSchemeOp::TTTLSettings::UNIT_NANOSECONDS: {
         auto& outTTL = *out.mutable_ttl_settings()->mutable_value_since_unix_epoch();
-        outTTL.set_column_name(inTTL.GetColumnName());
+        fillCommonFields(outTTL, inTTL);
         outTTL.set_column_unit(static_cast<Ydb::Table::ValueSinceUnixEpochModeSettings::Unit>(inTTL.GetColumnUnit()));
-        outTTL.set_expire_after_seconds(inTTL.GetExpireAfterSeconds());
         break;
     }
 
@@ -572,10 +603,6 @@ void FillColumnDescriptionImpl(TYdbProto& out,
         if (in.GetTTLSettings().HasEnabled()) {
             AddTtl(out, in.GetTTLSettings().GetEnabled());
         }
-
-        if (in.GetTTLSettings().HasUseTiering()) {
-            out.set_tiering(in.GetTTLSettings().GetUseTiering());
-        }
     }
 }
 
@@ -612,10 +639,6 @@ void FillColumnDescription(Ydb::Table::DescribeTableResult& out, const NKikimrSc
         if (in.GetTtlSettings().HasEnabled()) {
             AddTtl(out, in.GetTtlSettings().GetEnabled());
         }
-
-        if (in.GetTtlSettings().HasUseTiering()) {
-            out.set_tiering(in.GetTtlSettings().GetUseTiering());
-        }
     }
 
     out.set_store_type(Ydb::Table::StoreType::STORE_TYPE_COLUMN);
@@ -829,12 +852,6 @@ bool BuildAlterColumnTableModifyScheme(const TString& path, const Ydb::Table::Al
         } else if (req->has_drop_ttl_settings()) {
             alterColumnTable->MutableAlterTtlSettings()->MutableDisabled();
         }
-
-        if (req->has_set_tiering()) {
-            alterColumnTable->MutableAlterTtlSettings()->SetUseTiering(req->set_tiering());
-        } else if (req->has_drop_tiering()) {
-            alterColumnTable->MutableAlterTtlSettings()->SetUseTiering("");
-        }
     }
 
     return true;
diff --git a/ydb/core/ydb_convert/table_settings.h b/ydb/core/ydb_convert/table_settings.h
@@ -28,15 +28,43 @@ bool FillTtlSettings(TTtlSettingsEnabled& out, const Ydb::Table::TtlSettings& in
         return false;
     };
 
+    static const auto& fillCommonFields = []<class TModeSettings>(TTtlSettingsEnabled& out, const TModeSettings& in, std::optional<ui32> expireAfterSeconds) {
+        out.SetColumnName(in.column_name());
+        out.SetExpireAfterSeconds(expireAfterSeconds.value_or(in.expire_after_seconds()));
+    };
+
+    std::optional<ui32> expireAfterSeconds;
+    if (in.tiers_size()) {
+        for (const auto& inTier : in.tiers()) {
+            auto* outTier = out.AddTiers();
+            outTier->SetEvictAfterSeconds(inTier.evict_after_seconds());
+            switch (inTier.action_case()) {
+                case Ydb::Table::TtlTier::kDelete:
+                    outTier->MutableDelete();
+                    expireAfterSeconds = inTier.evict_after_seconds();
+                    break;
+                case Ydb::Table::TtlTier::kEvictToExternalStorage:
+                    outTier->MutableEvictToExternalStorage()->SetStorageName(inTier.evict_to_external_storage().storage_name());
+                    break;
+                case Ydb::Table::TtlTier::kEvictToColumnFamily:
+                    outTier->MutableEvictToColumnFamily()->SetFamilyName(inTier.evict_to_column_family().family_name());
+                    break;
+                case Ydb::Table::TtlTier::ACTION_NOT_SET:
+                    break;
+            }
+        }
+        if (!expireAfterSeconds) {
+            expireAfterSeconds = std::numeric_limits<uint32_t>::max();
+        }
+    }
+
     switch (in.mode_case()) {
     case Ydb::Table::TtlSettings::kDateTypeColumn:
-        out.SetColumnName(in.date_type_column().column_name());
-        out.SetExpireAfterSeconds(in.date_type_column().expire_after_seconds());
+        fillCommonFields(out, in.date_type_column(), expireAfterSeconds);
         break;
 
     case Ydb::Table::TtlSettings::kValueSinceUnixEpoch:
-        out.SetColumnName(in.value_since_unix_epoch().column_name());
-        out.SetExpireAfterSeconds(in.value_since_unix_epoch().expire_after_seconds());
+        fillCommonFields(out, in.value_since_unix_epoch(), expireAfterSeconds);
 
         #define CASE_UNIT(type) \
             case Ydb::Table::ValueSinceUnixEpochModeSettings::type: \
diff --git a/ydb/public/api/protos/ydb_table.proto b/ydb/public/api/protos/ydb_table.proto
@@ -433,15 +433,37 @@ message ColumnMeta {
     }
 }
 
+message EvictionToExternalStorageSettings {
+    // Path to external data source
+    string storage_name = 1;
+}
+
+message EvictionToFamilySettings {
+    // Name of column family
+    string family_name = 1;
+}
+
+message TtlTier {
+    uint32 evict_after_seconds = 1;
+
+    oneof action {
+        google.protobuf.Empty delete = 2;
+        EvictionToExternalStorageSettings evict_to_external_storage = 3;
+        EvictionToFamilySettings evict_to_column_family = 3;
+    }
+}
+
 message DateTypeColumnModeSettings {
-    // The row will be considered as expired at the moment of time, when the value
+    // The row will be belong to a tier at the moment of time, when the value
     // stored in <column_name> is less than or equal to the current time (in epoch
-    // time format), and <expire_after_seconds> has passed since that moment;
-    // i.e. the expiration threshold is the value of <column_name> plus <expire_after_seconds>.
+    // time format), and <evict_after_seconds> has passed since that moment;
+    // i.e. the eviction threshold is the value of <column_name> plus <evict_after_seconds>.
 
     // The column type must be a date type
     string column_name = 1;
 
+    // Deprecated. Use TtlSettings.tiers instead. If TtlSettings.tiers is not empty,
+    // expire_after_seconds does not take effect
     uint32 expire_after_seconds = 2;
 }
 
@@ -466,8 +488,8 @@ message ValueSinceUnixEpochModeSettings {
     // Interpretation of the value stored in <column_name>
     Unit column_unit = 2;
 
-    // This option is always interpreted as seconds regardless of the
-    // <column_unit> value.
+    // Deprecated. Use TtlSettings.tiers instead. If TtlSettings.tiers is not empty,
+    // expire_after_seconds does not take effect
     uint32 expire_after_seconds = 3;
 }
 
@@ -490,6 +512,8 @@ message TtlSettings {
     // How often to run BRO on the same partition.
     // BRO will not be started more often, but may be started less often.
     uint32 run_interval_seconds = 3;
+
+    repeated TtlTier tiers = 4;
 }
 
 message StorageSettings {
@@ -624,7 +648,7 @@ message CreateTableRequest {
     Ydb.FeatureFlag.Status key_bloom_filter = 16;
     // Read replicas settings for table
     ReadReplicasSettings read_replicas_settings = 17;
-    // Tiering rules name. It specifies how data migrates from one tier (logical storage) to another.
+    // Deprecated. Use TTL instead.
     string tiering = 18;
     // Is temporary table
     bool temporary = 19;
@@ -704,7 +728,7 @@ message AlterTableRequest {
     repeated string drop_changefeeds = 20;
     // Rename existed index
     repeated RenameIndexItem rename_indexes = 21;
-    // Setup or remove tiering
+    // Deprecated. Use ttl_action instead
     oneof tiering_action {
         string set_tiering = 22;
         google.protobuf.Empty drop_tiering = 23;
diff --git a/ydb/public/lib/experimental/ydb_logstore.cpp b/ydb/public/lib/experimental/ydb_logstore.cpp
@@ -16,23 +16,10 @@ namespace NYdb {
 namespace NLogStore {
 
 TMaybe<TTtlSettings> TtlSettingsFromProto(const Ydb::Table::TtlSettings& proto) {
-    switch (proto.mode_case()) {
-    case Ydb::Table::TtlSettings::kDateTypeColumn:
-        return TTtlSettings(
-            proto.date_type_column(),
-            proto.run_interval_seconds()
-        );
-
-    case Ydb::Table::TtlSettings::kValueSinceUnixEpoch:
-        return TTtlSettings(
-            proto.value_since_unix_epoch(),
-            proto.run_interval_seconds()
-        );
-
-    default:
-        break;
+    if (auto settings = TTtlSettings::DeserializeFromProto(proto)) {
+        return *settings;
     }
-    return {};
+    return Nothing();
 }
 
 static TCompression CompressionFromProto(const Ydb::LogStore::Compression& compression) {
@@ -199,8 +186,6 @@ void TLogTableDescription::SerializeTo(Ydb::LogStore::CreateLogTableRequest& req
 
     if (TtlSettings) {
         TtlSettings->SerializeTo(*request.mutable_ttl_settings());
-    } else if (TieringSettings) {
-        TieringSettings->SerializeTo(*request.mutable_tiering_settings());
     }
 }
 
diff --git a/ydb/public/lib/experimental/ydb_logstore.h b/ydb/public/lib/experimental/ydb_logstore.h
@@ -152,21 +152,6 @@ struct TLogTableSharding {
     TLogTableSharding(const Ydb::LogStore::DescribeLogTableResult& desc);
 };
 
-class TTieringSettings {
-private:
-    TString TieringId;
-public:
-    TTieringSettings(const TString& tieringId)
-        : TieringId(tieringId) {
-
-    }
-
-    void SerializeTo(Ydb::LogStore::TieringSettings& proto) const {
-        proto.set_tiering_id(TieringId);
-    }
-
-};
-
 class TLogTableDescription {
 public:
     TLogTableDescription(const TString& schemaPresetName, const TLogTableSharding& sharding);
@@ -200,16 +185,11 @@ class TLogTableDescription {
         TtlSettings = settings;
         return *this;
     }
-    TLogTableDescription& SetTieringSettings(const TTieringSettings& settings) {
-        TieringSettings = settings;
-        return *this;
-    }
 private:
     const TString SchemaPresetName;
     const TSchema Schema;
     const TLogTableSharding Sharding;
     TMaybe<TTtlSettings> TtlSettings;
-    TMaybe<TTieringSettings> TieringSettings;
     TString Owner;
     TVector<NScheme::TPermissions> Permissions;
     TVector<NScheme::TPermissions> EffectivePermissions;
diff --git a/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp b/ydb/public/lib/ydb_cli/commands/ydb_service_scheme.cpp
@@ -845,7 +845,7 @@ namespace {
 
     void PrintPartitionInfo(const NTable::TTableDescription& tableDescription, bool showBoundaries, bool showStats) {
         const TVector<NTable::TKeyRange>& ranges = tableDescription.GetKeyRanges();
-        const TVector<NTable::TPartitionStats>& stats = tableDescription.GetPartitionStats();
+        const TVector<NTable::TTPartitionStats>& stats = tableDescription.GetPartitionStats();
         if (showBoundaries) {
             if (showStats) {
                 Cout << Endl << "Partitions info:" << Endl;
@@ -921,7 +921,7 @@ namespace {
                 }
             }
             if (showStats) {
-                const NTable::TPartitionStats& partStats = stats[i];
+                const NTable::TTPartitionStats& partStats = stats[i];
                 row.Column(j++, partStats.Rows);
                 row.Column(j++, PrettySize(partStats.Size));
             }
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.cpp b/ydb/public/sdk/cpp/client/ydb_table/table.cpp
diff --git a/ydb/public/sdk/cpp/client/ydb_table/table.h b/ydb/public/sdk/cpp/client/ydb_table/table.h

Original file line number	Diff line number	Diff line change
`@@ -845,7 +845,7 @@ namespace {`
`845`	`845`
`846`	`846`	`void PrintPartitionInfo(const NTable::TTableDescription& tableDescription, bool showBoundaries, bool showStats) {`
`847`	`847`	`const TVector<NTable::TKeyRange>& ranges = tableDescription.GetKeyRanges();`
`848`		`- const TVector<NTable::TPartitionStats>& stats = tableDescription.GetPartitionStats();`
	`848`	`+ const TVector<NTable::TTPartitionStats>& stats = tableDescription.GetPartitionStats();`
`849`	`849`	`if (showBoundaries) {`
`850`	`850`	`if (showStats) {`
`851`	`851`	`Cout << Endl << "Partitions info:" << Endl;`
`@@ -921,7 +921,7 @@ namespace {`
`921`	`921`	`}`
`922`	`922`	`}`
`923`	`923`	`if (showStats) {`
`924`		`- const NTable::TPartitionStats& partStats = stats[i];`
	`924`	`+ const NTable::TTPartitionStats& partStats = stats[i];`
`925`	`925`	`row.Column(j++, partStats.Rows);`
`926`	`926`	`row.Column(j++, PrettySize(partStats.Size));`
`927`	`927`	`}`