split chunks from portion to optional usage method

Author: ivanmorozov333

ydb/core/tx/columnshard/blobs_action/transaction/tx_blobs_written.cpp

@@ -99,7 +99,7 @@ void TTxBlobsWritingFinished::DoComplete(const TActorContext& ctx) {
                     Self->GetOperationsManager().AddEventForLock(*Self, op->GetLockId(), evWrite);
                 }
             }
-            granule.InsertPortionOnComplete(portion.GetPortionInfo().MutablePortionInfoPtr());
+            granule.InsertPortionOnComplete(portion.GetPortionInfo(), index);
         }
         if (op->GetBehaviour() == EOperationBehaviour::NoTxWrite) {
             AFL_VERIFY(CommitSnapshot);

ydb/core/tx/columnshard/columnshard__statistics.cpp

@@ -74,7 +74,9 @@ void TColumnShard::Handle(NStat::TEvStatistics::TEvStatisticsRequest::TPtr& ev,
                 }
                 AFL_VERIFY(indexMeta->GetColumnIds().size() == 1);
 
-                const std::vector<TString> data = portionInfo->GetIndexInplaceDataVerified(indexMeta->GetIndexId());
+                const std::vector<TString> data
+                //= portionInfo->GetIndexInplaceDataVerified(indexMeta->GetIndexId())
+                ;
 
                 for (const auto& sketchAsString : data) {
                     auto sketch = std::unique_ptr<TCountMinSketch>(TCountMinSketch::FromString(sketchAsString.data(), sketchAsString.size()));

ydb/core/tx/columnshard/columnshard_impl.cpp

@@ -1238,15 +1238,15 @@ class TTxAskPortionChunks: public TTransactionBase<TColumnShard> {
         for (auto&& i : PortionsByPath) {
             while (i.second.size()) {
                 auto p = i.second.back();
-                auto rowset = db.Table<NColumnShard::Schema::IndexColumnsV1>().Prefix(p->GetPathId(), p->GetPortionId()).Select();
-                NOlap::TPortionInfoConstructor constructor(*p, false, true, true);
+                std::vector<NOlap::TColumnChunkLoadContextV1> records;
+                std::vector<NOlap::TIndexChunkLoadContext> indexes;
                 {
+                    auto rowset = db.Table<NColumnShard::Schema::IndexColumnsV1>().Prefix(p->GetPathId(), p->GetPortionId()).Select();
                     if (!rowset.IsReady()) {
                         return false;
                     }
                     while (!rowset.EndOfSet()) {
-                        NOlap::TColumnChunkLoadContextV1 chunkLoadContext(rowset);
-                        constructor.LoadRecord(chunkLoadContext);
+                        records.emplace_back(NOlap::TColumnChunkLoadContextV1(rowset));
                         if (!rowset.Next()) {
                             return false;
                         }
@@ -1258,13 +1258,13 @@ class TTxAskPortionChunks: public TTransactionBase<TColumnShard> {
                         return false;
                     }
                     while (!rowset.EndOfSet()) {
-                        NOlap::TIndexChunkLoadContext chunkLoadContext(rowset, &selector);
-                        constructor.LoadIndex(chunkLoadContext);
+                        indexes.emplace_back(NOlap::TIndexChunkLoadContext(rowset, &selector));
                         if (!rowset.Next()) {
                             return false;
                         }
                     }
                 }
+                NOlap::TPortionInfoConstructor constructor = NOlap::TPortionInfoConstructor::BuildForLoading(p, std::move(records), std::move(indexes));
                 Accessors[i.first].emplace_back(constructor.Build(true));
                 i.second.pop_back();
             }

ydb/core/tx/columnshard/counters/portions.cpp

@@ -30,7 +30,6 @@ void TSimplePortionsGroupInfo::AddPortion(const TPortionInfo& p) {
     RawBytes += p.GetTotalRawBytes();
     Count += 1;
     RecordsCount += p.GetRecordsCount();
-    ChunksCount += p.GetChunksCount();
 }
 
 void TSimplePortionsGroupInfo::RemovePortion(const std::shared_ptr<const NOlap::TPortionInfo>& p) {
@@ -42,12 +41,10 @@ void TSimplePortionsGroupInfo::RemovePortion(const TPortionInfo& p) {
     RawBytes -= p.GetTotalRawBytes();
     Count -= 1;
     RecordsCount -= p.GetRecordsCount();
-    ChunksCount -= p.GetChunksCount();
     AFL_VERIFY(RawBytes >= 0);
     AFL_VERIFY(BlobBytes >= 0);
     AFL_VERIFY(Count >= 0);
     AFL_VERIFY(RecordsCount >= 0);
-    AFL_VERIFY(ChunksCount >= 0);
 }
 
 }   // namespace NKikimr::NOlap

ydb/core/tx/columnshard/counters/portions.h

@@ -15,7 +15,6 @@ class TSimplePortionsGroupInfo {
     YDB_READONLY(i64, RawBytes, 0);
     YDB_READONLY(i64, Count, 0);
     YDB_READONLY(i64, RecordsCount, 0);
-    YDB_READONLY(i64, ChunksCount, 0);
 
 public:
     NJson::TJsonValue SerializeToJson() const {
@@ -24,7 +23,6 @@ class TSimplePortionsGroupInfo {
         result.InsertValue("raw_bytes", RawBytes);
         result.InsertValue("count", Count);
         result.InsertValue("records_count", RecordsCount);
-        result.InsertValue("chunks_count", ChunksCount);
         return result;
     }
 
@@ -47,7 +45,6 @@ class TSimplePortionsGroupInfo {
         result.RawBytes = RawBytes + item.RawBytes;
         result.Count = Count + item.Count;
         result.RecordsCount = RecordsCount + item.RecordsCount;
-        result.ChunksCount = ChunksCount + item.ChunksCount;
         return result;
     }
 

ydb/core/tx/columnshard/data_accessor/controller.h

@@ -34,22 +34,27 @@ class IGranuleDataAccessor {
 class TMemDataAccessor: public IGranuleDataAccessor {
 private:
     using TBase = IGranuleDataAccessor;
+    THashMap<ui64, TPortionDataAccessor> Accessors;
     virtual void DoAskData(const std::shared_ptr<TDataAccessorsRequest>& request) override {
         std::vector<TPortionDataAccessor> accessors;
         auto& portions = request->StartFetching(GetPathId());
         for (auto&& i : portions) {
-            accessors.emplace_back(TPortionDataAccessor(i));
+            auto it = Accessors.find(i->GetPortionId());
+            AFL_VERIFY(it != Accessors.end());
+            accessors.emplace_back(it->second);
         }
         request->AddData(GetPathId(), std::move(accessors));
     }
-    virtual void DoModifyPortions(const std::vector<TPortionDataAccessor>& /*add*/, const std::vector<ui64>& /*remove*/) override {
+    virtual void DoModifyPortions(const std::vector<TPortionDataAccessor>& add, const std::vector<ui64>& remove) override {
+        for (auto&& i : remove) {
+            AFL_VERIFY(Accessors.erase(i));
+        }
+        for (auto&& i : add) {
+            AFL_VERIFY(Accessors.emplace(i.GetPortionInfo().GetPortionId(), i).second);
+        }
     }
 
 public:
-    TPortionDataAccessor BuildAccessor(const TPortionInfo::TConstPtr& portion) const {
-        return TPortionDataAccessor(portion);
-    }
-
     TMemDataAccessor(const ui64 pathId)
         : TBase(pathId) {
     }
@@ -64,10 +69,6 @@ class TTabletDataAccessor: public IGranuleDataAccessor {
     }
 
 public:
-    TPortionDataAccessor BuildAccessor(const TPortionInfo::TConstPtr& portion) const {
-        return TPortionDataAccessor(portion);
-    }
-
     TTabletDataAccessor(const ui64 pathId)
         : TBase(pathId) {
     }

ydb/core/tx/columnshard/data_sharing/common/session/common.cpp

@@ -27,7 +27,7 @@ TConclusionStatus TCommonSession::TryStart(NColumnShard::TColumnShard& shard) {
             if (shard.GetDataLocksManager()->IsLocked(*p.second, { "sharing_session:" + GetSessionId() })) {
                 return TConclusionStatus::Fail("failed to start cursor: portion is locked");
             }
-            portionsByPath[i].emplace_back(p.second);
+//            portionsByPath[i].emplace_back(p.second);
         }
     }
 

ydb/core/tx/columnshard/data_sharing/destination/session/destination.cpp

@@ -20,7 +20,7 @@ NKikimr::TConclusionStatus TDestinationSession::DataReceived(
         AFL_VERIFY(it != PathIds.end())("path_id_undefined", i.first);
         for (auto&& portion : i.second.DetachPortions()) {
             portion.MutablePortionInfo().SetPathId(it->second);
-            index.AppendPortion(portion.MutablePortionInfoPtr());
+            index.AppendPortion(portion);
         }
     }
     return TConclusionStatus::Success();

ydb/core/tx/columnshard/data_sharing/source/session/cursor.cpp

@@ -26,8 +26,8 @@ void TSourceCursor::BuildSelection(const std::shared_ptr<IStoragesManager>& stor
                 NextPortionId = itPortion->first;
             } else {
                 portions.emplace_back(itPortion->second);
-                chunksCount += portions.back().GetRecords().size();
-                chunksCount += portions.back().GetIndexes().size();
+                chunksCount += portions.back().GetRecordsVerified().size();
+                chunksCount += portions.back().GetIndexesVerified().size();
                 ++count;
             }
         }

ydb/core/tx/columnshard/engines/changes/with_appended.cpp

@@ -113,7 +113,7 @@ void TChangesWithAppend::DoWriteIndexOnComplete(NColumnShard::TColumnShard* self
             context.EngineLogs.AddCleanupPortion(i);
         }
         for (auto& portionBuilder : AppendedPortions) {
-            context.EngineLogs.AppendPortion(portionBuilder.GetPortionResult().MutablePortionInfoPtr());
+            context.EngineLogs.AppendPortion(portionBuilder.GetPortionResult());
         }
     }
 }