sort statistics by name (#3144)

ivanmorozov333 · web-flow · commit d64073bde17e · 2024-03-26T10:01:37.000+03:00
diff --git a/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp b/ydb/core/kqp/ut/olap/kqp_olap_ut.cpp
@@ -4129,7 +4129,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
             helper.GetStats(stats, true);
             for (auto&& i : stats) {
                 AFL_VERIFY(i.ScalarsSize() == 2);
-                AFL_VERIFY(i.GetScalars()[1].GetUint32() == 3);
+                AFL_VERIFY(i.GetScalars()[0].GetUint32() == 3);
             }
         }
         {
@@ -4151,7 +4151,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {
             helper.GetStats(stats, true);
             for (auto&& i : stats) {
                 AFL_VERIFY(i.ScalarsSize() == 2);
-                AFL_VERIFY(i.GetScalars()[1].GetUint32() == 3);
+                AFL_VERIFY(i.GetScalars()[0].GetUint32() == 3);
             }
         }
     }
diff --git a/ydb/core/tx/columnshard/columnshard_ut_common.h b/ydb/core/tx/columnshard/columnshard_ut_common.h
@@ -246,7 +246,7 @@ struct TTestSchema {
                 continue;
             }
             if (NOlap::NStatistics::NMax::TOperator::IsAvailableType(columns[i].GetType())) {
-                *schema->AddStatistics() = NOlap::NStatistics::TOperatorContainer(std::make_shared<NOlap::NStatistics::NMax::TOperator>(i + 1)).SerializeToProto();
+                *schema->AddStatistics() = NOlap::NStatistics::TOperatorContainer("MAX::" + columns[i].GetName(), std::make_shared<NOlap::NStatistics::NMax::TOperator>(i + 1)).SerializeToProto();
             }
         }
 
diff --git a/ydb/core/tx/columnshard/engines/portions/with_blobs.cpp b/ydb/core/tx/columnshard/engines/portions/with_blobs.cpp
@@ -169,7 +169,7 @@ bool TPortionInfoWithBlobs::ExtractColumnChunks(const ui32 columnId, std::vector
 
 void TPortionInfoWithBlobs::FillStatistics(const TIndexInfo& index) {
     NStatistics::TPortionStorage storage;
-    for (auto&& i : index.GetStatistics()) {
+    for (auto&& i : index.GetStatisticsByName()) {
         THashMap<ui32, std::vector<std::shared_ptr<IPortionDataChunk>>> data;
         for (auto&& entityId : i.second->GetEntityIds()) {
             data.emplace(entityId, GetEntityChunks(entityId));
@@ -225,9 +225,9 @@ TPortionInfoWithBlobs TPortionInfoWithBlobs::SyncPortion(TPortionInfoWithBlobs&&
     result.GetPortionInfo().MutableMeta().SetTierName(targetTier);
 
     NStatistics::TPortionStorage storage;
-    for (auto&& i : to->GetIndexInfo().GetStatistics()) {
-        auto it = from->GetIndexInfo().GetStatistics().find(i.first);
-        if (it != from->GetIndexInfo().GetStatistics().end()) {
+    for (auto&& i : to->GetIndexInfo().GetStatisticsByName()) {
+        auto it = from->GetIndexInfo().GetStatisticsByName().find(i.first);
+        if (it != from->GetIndexInfo().GetStatisticsByName().end()) {
             i.second->CopyData(it->second.GetCursorVerified(), source.PortionInfo.GetMeta().GetStatisticsStorage(), storage);
         } else {
             i.second->FillStatisticsData(entityChunksNew, storage, to->GetIndexInfo());
diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.cpp b/ydb/core/tx/columnshard/engines/scheme/index_info.cpp
@@ -355,12 +355,14 @@ bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema&
     }
 
     {
-        NStatistics::TPortionStorageCursor cursor;
         for (const auto& stat : schema.GetStatistics()) {
             NStatistics::TOperatorContainer container;
             AFL_VERIFY(container.DeserializeFromProto(stat));
+            AFL_VERIFY(StatisticsByName.emplace(container.GetName(), std::move(container)).second);
+        }
+        NStatistics::TPortionStorageCursor cursor;
+        for (auto&& [_, container] : StatisticsByName) {
             container.SetCursor(cursor);
-            Statistics.emplace(container->GetIdentifier(), container);
             container->ShiftCursor(cursor);
         }
     }
diff --git a/ydb/core/tx/columnshard/engines/scheme/index_info.h b/ydb/core/tx/columnshard/engines/scheme/index_info.h
@@ -43,7 +43,7 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo {
     THashMap<ui32, TColumnFeatures> ColumnFeatures;
     THashMap<ui32, std::shared_ptr<arrow::Field>> ArrowColumnByColumnIdCache;
     THashMap<ui32, NIndexes::TIndexMetaContainer> Indexes;
-    std::map<NStatistics::TIdentifier, NStatistics::TOperatorContainer> Statistics;
+    std::map<TString, NStatistics::TOperatorContainer> StatisticsByName;
     TIndexInfo(const TString& name);
     bool SchemeNeedActualization = false;
     bool DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema& schema, const std::shared_ptr<IStoragesManager>& operators);
@@ -78,14 +78,15 @@ struct TIndexInfo : public NTable::TScheme::TTableSchema, public IIndexInfo {
 
     std::vector<std::shared_ptr<IPortionDataChunk>> MakeEmptyChunks(const ui32 columnId, const std::vector<ui32>& pages, const TSimpleColumnInfo& columnInfo) const;
 
-    const std::map<NStatistics::TIdentifier, NStatistics::TOperatorContainer>& GetStatistics() const {
-        return Statistics;
+    const std::map<TString, NStatistics::TOperatorContainer>& GetStatisticsByName() const {
+        return StatisticsByName;
     }
 
     NStatistics::TOperatorContainer GetStatistics(const NStatistics::TIdentifier& id) const {
-        auto it = Statistics.find(id);
-        if (it != Statistics.end()) {
-            return it->second;
+        for (auto&& i : StatisticsByName) {
+            if (i.second->GetIdentifier() == id) {
+                return i.second;
+            }
         }
         return NStatistics::TOperatorContainer();
     }
diff --git a/ydb/core/tx/columnshard/engines/scheme/statistics/abstract/constructor.h b/ydb/core/tx/columnshard/engines/scheme/statistics/abstract/constructor.h
@@ -20,7 +20,7 @@ class IConstructor {
     YDB_READONLY(EType, Type, EType::Undefined);
     IConstructor() = default;
 protected:
-    virtual TConclusion<TOperatorContainer> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const = 0;
+    virtual TConclusion<std::shared_ptr<IOperator>> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const = 0;
     virtual bool DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) = 0;
     virtual void DoSerializeToProto(NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) const = 0;
     virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonData) = 0;
@@ -39,8 +39,12 @@ class IConstructor {
         return DoDeserializeFromJson(jsonData);
     }
 
-    TConclusion<TOperatorContainer> CreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
-        return DoCreateOperator(currentSchema);
+    TConclusion<TOperatorContainer> CreateOperator(const TString& name, const NSchemeShard::TOlapSchema& currentSchema) const {
+        auto result = DoCreateOperator(currentSchema);
+        if (!result) {
+            return result.GetError();
+        }
+        return TOperatorContainer(name, result.DetachResult());
     }
 
     TString GetClassName() const {
diff --git a/ydb/core/tx/columnshard/engines/scheme/statistics/abstract/operator.h b/ydb/core/tx/columnshard/engines/scheme/statistics/abstract/operator.h
@@ -68,10 +68,18 @@ class IOperator {
 
 class TOperatorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IOperator> {
 private:
+    YDB_READONLY_DEF(TString, Name);
     std::optional<TPortionStorageCursor> Cursor;
     using TBase = NBackgroundTasks::TInterfaceProtoContainer<IOperator>;
 public:
-    using TBase::TBase;
+    TOperatorContainer() = default;
+
+    TOperatorContainer(const TString& name, const std::shared_ptr<IOperator>& object)
+        : TBase(object)
+        , Name(name)
+    {
+        AFL_VERIFY(Name);
+    }
 
     const TPortionStorageCursor& GetCursorVerified() const {
         AFL_VERIFY(Cursor);
@@ -87,6 +95,30 @@ class TOperatorContainer: public NBackgroundTasks::TInterfaceProtoContainer<IOpe
         AFL_VERIFY(!!Cursor);
         return storage.GetScalarVerified(*Cursor);
     }
+
+    NKikimrColumnShardStatisticsProto::TOperatorContainer SerializeToProto() const {
+        NKikimrColumnShardStatisticsProto::TOperatorContainer result = TBase::SerializeToProto();
+        result.SetName(Name);
+        AFL_VERIFY(Name);
+        return result;
+    }
+
+    void SerializeToProto(NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) const {
+        TBase::SerializeToProto(proto);
+        proto.SetName(Name);
+        AFL_VERIFY(Name);
+    }
+
+    bool DeserializeFromProto(const NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) {
+        Name = proto.GetName();
+        if (!Name) {
+            return false;
+        }
+        if (!TBase::DeserializeFromProto(proto)) {
+            return false;
+        }
+        return true;
+    }
 };
 
 }
diff --git a/ydb/core/tx/columnshard/engines/scheme/statistics/max/constructor.cpp b/ydb/core/tx/columnshard/engines/scheme/statistics/max/constructor.cpp
@@ -3,12 +3,12 @@
 
 namespace NKikimr::NOlap::NStatistics::NMax {
 
-NKikimr::TConclusion<TOperatorContainer> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
+NKikimr::TConclusion<std::shared_ptr<IOperator>> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
     auto column = currentSchema.GetColumns().GetByName(ColumnName);
     if (!TOperator::IsAvailableType(column->GetType())) {
         return TConclusionStatus::Fail("incorrect type for stat calculation");
     }
-    return TOperatorContainer(std::make_shared<TOperator>(column->GetId()));
+    return std::make_shared<TOperator>(column->GetId());
 }
 
 bool TConstructor::DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) {
diff --git a/ydb/core/tx/columnshard/engines/scheme/statistics/max/constructor.h b/ydb/core/tx/columnshard/engines/scheme/statistics/max/constructor.h
@@ -12,7 +12,7 @@ class TConstructor: public IConstructor {
     static inline const auto Registrator = TFactory::TRegistrator<TConstructor>(::ToString(EType::Max));
     YDB_READONLY(TString, ColumnName, 0);
 protected:
-    virtual TConclusion<TOperatorContainer> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
+    virtual TConclusion<std::shared_ptr<IOperator>> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
     virtual bool DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) override;
     virtual void DoSerializeToProto(NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) const override;
     virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonData) override;
diff --git a/ydb/core/tx/columnshard/engines/scheme/statistics/variability/constructor.cpp b/ydb/core/tx/columnshard/engines/scheme/statistics/variability/constructor.cpp
@@ -3,12 +3,12 @@
 
 namespace NKikimr::NOlap::NStatistics::NVariability {
 
-NKikimr::TConclusion<TOperatorContainer> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
+NKikimr::TConclusion<std::shared_ptr<IOperator>> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {
     auto column = currentSchema.GetColumns().GetByName(ColumnName);
     if (!TOperator::IsAvailableType(column->GetType())) {
         return TConclusionStatus::Fail("incorrect type for stat calculation");
     }
-    return TOperatorContainer(std::make_shared<TOperator>(column->GetId()));
+    return std::make_shared<TOperator>(column->GetId());
 }
 
 bool TConstructor::DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) {
diff --git a/ydb/core/tx/columnshard/engines/scheme/statistics/variability/constructor.h b/ydb/core/tx/columnshard/engines/scheme/statistics/variability/constructor.h
@@ -12,7 +12,7 @@ class TConstructor: public IConstructor {
     static inline const auto Registrator = TFactory::TRegistrator<TConstructor>(::ToString(EType::Variability));
     YDB_READONLY(TString, ColumnName, 0);
 protected:
-    virtual TConclusion<TOperatorContainer> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
+    virtual TConclusion<std::shared_ptr<IOperator>> DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const override;
     virtual bool DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) override;
     virtual void DoSerializeToProto(NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) const override;
     virtual TConclusionStatus DoDeserializeFromJson(const NJson::TJsonValue& jsonData) override;
diff --git a/ydb/core/tx/columnshard/test_helper/controllers.h b/ydb/core/tx/columnshard/test_helper/controllers.h
@@ -26,6 +26,21 @@ class TWaitCompactionController: public NYDBTest::NColumnShard::TController {
     virtual bool DoOnStartCompaction(std::shared_ptr<NKikimr::NOlap::TColumnEngineChanges>& changes) override;
     virtual bool DoOnWriteIndexComplete(const NKikimr::NOlap::TColumnEngineChanges& changes, const NKikimr::NColumnShard::TColumnShard& /*shard*/) override;
     virtual bool DoOnWriteIndexStart(const ui64 /*tabletId*/, const TString& changeClassName) override;
+    virtual bool NeedForceCompactionBacketsConstruction() const override {
+        return true;
+    }
+    virtual ui64 GetSmallPortionSizeDetector(const ui64 /*def*/) const override {
+        return 0;
+    }
+    virtual TDuration GetOptimizerFreshnessCheckDuration(const TDuration /*defaultValue*/) const override {
+        return TDuration::Zero();
+    }
+    virtual TDuration GetLagForCompactionBeforeTierings(const TDuration /*def*/) const override {
+        return TDuration::Zero();
+    }
+    virtual TDuration GetTTLDefaultWaitingDuration(const TDuration /*defaultValue*/) const override {
+        return TDuration::Seconds(1);
+    }
 public:
     ui32 GetFinishedExportsCount() const {
         return ExportsFinishedCount.Val();
diff --git a/ydb/core/tx/schemeshard/olap/statistics/schema.cpp b/ydb/core/tx/schemeshard/olap/statistics/schema.cpp
@@ -4,24 +4,22 @@
 namespace NKikimr::NSchemeShard {
 
 void TOlapStatisticsSchema::SerializeToProto(NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) const {
-    proto.SetName(Name);
     Operator.SerializeToProto(proto);
 }
 
 bool TOlapStatisticsSchema::DeserializeFromProto(const NKikimrColumnShardStatisticsProto::TOperatorContainer& proto) {
-    Name = proto.GetName();
     AFL_VERIFY(Operator.DeserializeFromProto(proto))("incorrect_proto", proto.DebugString());
     return true;
 }
 
 bool TOlapStatisticsSchema::ApplyUpdate(const TOlapSchema& /*currentSchema*/, const TOlapStatisticsUpsert& upsert, IErrorCollector& errors) {
-    AFL_VERIFY(upsert.GetName() == GetName());
+    AFL_VERIFY(upsert.GetName() == Operator.GetName());
     AFL_VERIFY(!!upsert.GetConstructor());
     if (upsert.GetConstructor().GetClassName() != Operator.GetClassName()) {
         errors.AddError("different index classes: " + upsert.GetConstructor().GetClassName() + " vs " + Operator.GetClassName());
         return false;
     }
-    errors.AddError("cannot modify statistics calculation for " + GetName() + ". not implemented currently.");
+    errors.AddError("cannot modify statistics calculation for " + Operator.GetName() + ". not implemented currently.");
     return false;
 }
 
@@ -33,12 +31,12 @@ bool TOlapStatisticsDescription::ApplyUpdate(const TOlapSchema& currentSchema, c
                 return false;
             }
         } else {
-            auto meta = stat.GetConstructor()->CreateOperator(currentSchema);
+            auto meta = stat.GetConstructor()->CreateOperator(stat.GetName(), currentSchema);
             if (!meta) {
                 errors.AddError(meta.GetErrorMessage());
                 return false;
             }
-            TOlapStatisticsSchema object(stat.GetName(), meta.DetachResult());
+            TOlapStatisticsSchema object(meta.DetachResult());
             Y_ABORT_UNLESS(ObjectsByName.emplace(stat.GetName(), std::move(object)).second);
         }
     }
diff --git a/ydb/core/tx/schemeshard/olap/statistics/schema.h b/ydb/core/tx/schemeshard/olap/statistics/schema.h
@@ -7,16 +7,14 @@ class TOlapSchema;
 
 class TOlapStatisticsSchema {
 private:
-    YDB_READONLY_DEF(TString, Name);
     YDB_READONLY_DEF(NOlap::NStatistics::TOperatorContainer, Operator);
 public:
     TOlapStatisticsSchema() = default;
 
-    TOlapStatisticsSchema(const TString& name, const NOlap::NStatistics::TOperatorContainer& container)
-        : Name(name)
-        , Operator(container)
+    TOlapStatisticsSchema(const NOlap::NStatistics::TOperatorContainer& container)
+        : Operator(container)
     {
-
+        AFL_VERIFY(container.GetName());
     }
 
     bool ApplyUpdate(const TOlapSchema& currentSchema, const TOlapStatisticsUpsert& upsert, IErrorCollector& errors);
diff --git a/ydb/library/conclusion/result.h b/ydb/library/conclusion/result.h
@@ -30,11 +30,13 @@ class TConclusion {
         Y_ABORT_UNLESS(IsFail());
     }
 
-    TConclusion(TResult&& result)
+    template <class TResultArg>
+    TConclusion(TResultArg&& result)
         : Result(std::move(result)) {
     }
 
-    TConclusion(const TResult& result)
+    template <class TResultArg>
+    TConclusion(const TResultArg& result)
         : Result(result) {
     }
 

Original file line number	Diff line number	Diff line change
`@@ -4129,7 +4129,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {`
`4129`	`4129`	`helper.GetStats(stats, true);`
`4130`	`4130`	`for (auto&& i : stats) {`
`4131`	`4131`	`AFL_VERIFY(i.ScalarsSize() == 2);`
`4132`		`- AFL_VERIFY(i.GetScalars()[1].GetUint32() == 3);`
	`4132`	`+ AFL_VERIFY(i.GetScalars()[0].GetUint32() == 3);`
`4133`	`4133`	`}`
`4134`	`4134`	`}`
`4135`	`4135`	`{`
`@@ -4151,7 +4151,7 @@ Y_UNIT_TEST_SUITE(KqpOlap) {`
`4151`	`4151`	`helper.GetStats(stats, true);`
`4152`	`4152`	`for (auto&& i : stats) {`
`4153`	`4153`	`AFL_VERIFY(i.ScalarsSize() == 2);`
`4154`		`- AFL_VERIFY(i.GetScalars()[1].GetUint32() == 3);`
	`4154`	`+ AFL_VERIFY(i.GetScalars()[0].GetUint32() == 3);`
`4155`	`4155`	`}`
`4156`	`4156`	`}`
`4157`	`4157`	`}`
Original file line number	Diff line number	Diff line change
`@@ -246,7 +246,7 @@ struct TTestSchema {`
`246`	`246`	`continue;`
`247`	`247`	`}`
`248`	`248`	`if (NOlap::NStatistics::NMax::TOperator::IsAvailableType(columns[i].GetType())) {`
`249`		`- *schema->AddStatistics() = NOlap::NStatistics::TOperatorContainer(std::make_shared<NOlap::NStatistics::NMax::TOperator>(i + 1)).SerializeToProto();`
	`249`	`+ *schema->AddStatistics() = NOlap::NStatistics::TOperatorContainer("MAX::" + columns[i].GetName(), std::make_shared<NOlap::NStatistics::NMax::TOperator>(i + 1)).SerializeToProto();`
`250`	`250`	`}`
`251`	`251`	`}`
`252`	`252`
Original file line number	Diff line number	Diff line change
`@@ -355,12 +355,14 @@ bool TIndexInfo::DeserializeFromProto(const NKikimrSchemeOp::TColumnTableSchema&`
`355`	`355`	`}`
`356`	`356`
`357`	`357`	`{`
`358`		`- NStatistics::TPortionStorageCursor cursor;`
`359`	`358`	`for (const auto& stat : schema.GetStatistics()) {`
`360`	`359`	`NStatistics::TOperatorContainer container;`
`361`	`360`	`AFL_VERIFY(container.DeserializeFromProto(stat));`
	`361`	`+ AFL_VERIFY(StatisticsByName.emplace(container.GetName(), std::move(container)).second);`
	`362`	`+ }`
	`363`	`+ NStatistics::TPortionStorageCursor cursor;`
	`364`	`+ for (auto&& [_, container] : StatisticsByName) {`
`362`	`365`	`container.SetCursor(cursor);`
`363`		`- Statistics.emplace(container->GetIdentifier(), container);`
`364`	`366`	`container->ShiftCursor(cursor);`
`365`	`367`	`}`
`366`	`368`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,12 +3,12 @@`
`3`	`3`
`4`	`4`	`namespace NKikimr::NOlap::NStatistics::NMax {`
`5`	`5`
`6`		`-NKikimr::TConclusion<TOperatorContainer> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {`
	`6`	`+NKikimr::TConclusion<std::shared_ptr<IOperator>> TConstructor::DoCreateOperator(const NSchemeShard::TOlapSchema& currentSchema) const {`
`7`	`7`	`auto column = currentSchema.GetColumns().GetByName(ColumnName);`
`8`	`8`	`if (!TOperator::IsAvailableType(column->GetType())) {`
`9`	`9`	`return TConclusionStatus::Fail("incorrect type for stat calculation");`
`10`	`10`	`}`
`11`		`- return TOperatorContainer(std::make_shared<TOperator>(column->GetId()));`
	`11`	`+ return std::make_shared<TOperator>(column->GetId());`
`12`	`12`	`}`
`13`	`13`
`14`	`14`	`bool TConstructor::DoDeserializeFromProto(const NKikimrColumnShardStatisticsProto::TConstructorContainer& proto) {`