fixes

ivanmorozov333 · ivanmorozov333 · commit 4f189c89f3c0 · 2025-03-06T19:45:23.000+03:00
diff --git a/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp b/ydb/core/tx/columnshard/engines/reader/common_reader/iterator/fetching.cpp
@@ -214,23 +214,6 @@ TConclusion<bool> TProgramStepPrepare::DoExecuteInplace(const std::shared_ptr<ID
         logic->Start(readActions, context);
         AFL_VERIFY(fetchers.emplace(i.GetColumnId(), logic).second)("column_id", i.GetColumnId());
     }
-    auto logicIndexesList = Step->GetIndexesToFetch(source->GetSourceSchema());
-    for (auto&& i : Step.GetOriginalIndexesToUse()) {
-        const NIndexes::TIndexMetaContainer indexMeta = source->GetSourceSchema()->GetIndexVerified(i.GetIndexId());
-        auto chunks = source->GetStageData().GetPortionAccessor().GetIndexChunksPointers(i.GetColumnId());
-        std::vector<NIndexes::TChunkOriginalData> originalChunks;
-        for (auto&& c : chunks) {
-            if (auto range = c->GetBlobRangeOptional()) {
-                originalChunks.emplace_back(source->GetStageData().GetPortionAccessor().GetPortionInfo().RestoreBlobRange(*range));
-            } else {
-                originalChunks.emplace_back(c->GetBlobDataVerified());
-            }
-        }
-        std::shared_ptr<IKernelFetchLogic> logic = indexMeta->BuildFetchTask(
-            i, originalChunks, *source->GetStageData().GetIndexes(), indexMeta, source->GetContext()->GetCommonContext()->GetStoragesManager());
-        logic->Start(readActions, context);
-        AFL_VERIFY(fetchers.emplace(i.GetColumnId(), logic).second)("column_id", i.GetColumnId());
-    }
     if (readActions.IsEmpty()) {
         NBlobOperations::NRead::TCompositeReadBlobs blobs;
         for (auto&& i : fetchers) {
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/context.cpp
@@ -48,7 +48,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c
                 return false;
         }
     }();
-    const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false);
+    const bool useIndexes = (IndexChecker ? source->HasIndexes(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds())) : false);
     const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly() && source->IsSourceInMemory();
     const bool needSnapshots = GetReadMetadata()->GetRequestSnapshot() < source->GetRecordSnapshotMax() || !isWholeExclusiveSource;
     const bool hasDeletions = source->GetHasDeletions();
diff --git a/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/plain_reader/iterator/source.cpp
@@ -174,8 +174,8 @@ void TPortionDataSource::DoAbort() {
 }
 
 void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) {
-    THashMap<ui32, std::vector<TString>> indexBlobs;
-    std::set<ui32> indexIds = indexChecker->GetIndexIds();
+    THashMap<NIndexes::TIndexDataAddress, std::vector<TString>> indexBlobs;
+    std::set<ui32> indexIds = NIndexes::TIndexDataAddress::ExtractIndexIds(indexChecker->GetIndexIds());
     //    NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString());
     std::vector<TPortionDataAccessor::TPage> pages = GetStageData().GetPortionAccessor().BuildPages();
     NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter();
@@ -185,12 +185,12 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in
                 continue;
             }
             if (i->HasBlobData()) {
-                indexBlobs[i->GetIndexId()].emplace_back(i->GetBlobDataVerified());
+                indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(i->GetBlobDataVerified());
             } else {
-                indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetAddress()));
+                indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(StageData->ExtractBlob(i->GetAddress()));
             }
         }
-        for (auto&& i : indexIds) {
+        for (auto&& i : indexChecker->GetIndexIds()) {
             if (!indexBlobs.contains(i)) {
                 return;
             }
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/context.cpp
@@ -32,7 +32,8 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c
                 return false;
         }
     }();
-    const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false);
+
+    const bool useIndexes = (IndexChecker ? source->HasIndexes(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds())) : false);
     const bool hasDeletions = source->GetHasDeletions();
     bool needShardingFilter = false;
     if (!!GetReadMetadata()->GetRequestShardingInfo()) {
diff --git a/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp b/ydb/core/tx/columnshard/engines/reader/simple_reader/iterator/source.cpp
@@ -171,8 +171,8 @@ void TPortionDataSource::DoAbort() {
 }
 
 void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) {
-    THashMap<ui32, std::vector<TString>> indexBlobs;
-    const std::set<TIndexDataAddress> indexIds = indexChecker->GetIndexIds();
+    THashMap<NIndexes::TIndexDataAddress, std::vector<TString>> indexBlobs;
+    const auto indexIds = NIndexes::TIndexDataAddress::ExtractIndexIds(indexChecker->GetIndexIds());
     //    NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetPortionId());
     std::vector<TPortionDataAccessor::TPage> pages = GetStageData().GetPortionAccessor().BuildPages();
     NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter();
@@ -182,12 +182,12 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in
                 continue;
             }
             if (i->HasBlobData()) {
-                indexBlobs[i->GetIndexId()].emplace_back(i->GetBlobDataVerified());
+                indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(i->GetBlobDataVerified());
             } else {
-                indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetAddress()));
+                indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(StageData->ExtractBlob(i->GetAddress()));
             }
         }
-        for (auto&& i : indexIds) {
+        for (auto&& i : indexChecker->GetIndexIds()) {
             if (!indexBlobs.contains(i)) {
                 GetContext()->GetCommonContext()->GetCounters().OnNotIndexBlobs();
                 return;
diff --git a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/checker.h b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/checker.h
@@ -16,6 +16,15 @@ class TIndexDataAddress {
     YDB_READONLY_DEF(std::optional<ui64>, Category);
 
 public:
+    template <class TContainer>
+    static std::set<ui32> ExtractIndexIds(const TContainer& addresses) {
+        std::set<ui32> result;
+        for (auto&& i : addresses) {
+            result.emplace(i.GetIndexId());
+        }
+        return result;
+    }
+
     TIndexDataAddress() = default;
 
     explicit TIndexDataAddress(const ui32 indexId)
diff --git a/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/ya.make b/ydb/core/tx/columnshard/engines/scheme/indexes/abstract/ya.make
@@ -9,6 +9,7 @@ SRCS(
     collection.cpp
     header.cpp
     fetcher.cpp
+    abstract.cpp
     meta.cpp
     checker.cpp
     GLOBAL composite.cpp
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.cpp
@@ -366,11 +366,11 @@ ui32 ISnapshotSchema::GetIndexesCount() const {
     return GetIndexInfo().GetIndexes().size();
 }
 
-std::shared_ptr<NIndexes::IIndexMeta> ISnapshotSchema::GetIndexVerified(const ui32 indexId) const {
+NIndexes::TIndexMetaContainer ISnapshotSchema::GetIndexVerified(const ui32 indexId) const {
     return GetIndexInfo().GetIndexVerified(indexId);
 }
 
-std::shared_ptr<NIndexes::IIndexMeta> ISnapshotSchema::GetIndexOptional(const ui32 indexId) const {
+NIndexes::TIndexMetaContainer ISnapshotSchema::GetIndexOptional(const ui32 indexId) const {
     return GetIndexInfo().GetIndexOptional(indexId);
 }
 
diff --git a/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h b/ydb/core/tx/columnshard/engines/scheme/versions/abstract_scheme.h
@@ -14,7 +14,7 @@
 namespace NKikimr::NOlap {
 
 namespace NIndexes {
-class IIndexMeta;
+class TIndexMetaContainer;
 }
 
 struct TIndexInfo;
@@ -33,8 +33,8 @@ class ISnapshotSchema {
     std::shared_ptr<NArrow::NAccessor::TColumnLoader> GetColumnLoaderVerified(const ui32 columnId) const;
     std::shared_ptr<NArrow::NAccessor::TColumnLoader> GetColumnLoaderOptional(const std::string& columnName) const;
     std::shared_ptr<NArrow::NAccessor::TColumnLoader> GetColumnLoaderVerified(const std::string& columnName) const;
-    std::shared_ptr<NIndexes::IIndexMeta> GetIndexVerified(const ui32 indexId) const;
-    std::shared_ptr<NIndexes::IIndexMeta> GetIndexOptional(const ui32 indexId) const;
+    NIndexes::TIndexMetaContainer GetIndexVerified(const ui32 indexId) const;
+    NIndexes::TIndexMetaContainer GetIndexOptional(const ui32 indexId) const;
 
     bool IsSpecialColumnId(const ui32 columnId) const;
     virtual TColumnIdsView GetColumnIds() const = 0;
diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/checker.h b/ydb/core/tx/columnshard/engines/storage/indexes/bloom_ngramm/checker.h
@@ -22,7 +22,7 @@ class TFilterChecker: public TSimpleIndexChecker {
 public:
     TFilterChecker() = default;
     TFilterChecker(const ui32 indexId, std::set<ui64>&& hashes)
-        : TBase(indexId)
+        : TBase(TIndexDataAddress(indexId))
         , HashValues(std::move(hashes)) {
     }
     virtual TString GetClassName() const override {
diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h b/ydb/core/tx/columnshard/engines/storage/indexes/count_min_sketch/checker.h
@@ -21,7 +21,7 @@ class TCountMinSketchChecker: public TSimpleIndexChecker {
 public:
     TCountMinSketchChecker() = default;
     TCountMinSketchChecker(const ui32 indexId)
-        : TBase(indexId)
+        : TBase(TIndexDataAddress(indexId))
     {}
 
     virtual TString GetClassName() const override {
diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/portions/extractor/default.cpp b/ydb/core/tx/columnshard/engines/storage/indexes/portions/extractor/default.cpp
@@ -63,7 +63,7 @@ THashMap<ui64, ui32> TDefaultDataExtractor::DoGetIndexHitsCount(const std::share
         }
         {
             auto& stats = subColumns->GetOthersData().GetStats();
-            for (ui32 i = 0; i < stats.GetOthersData(); ++i) {
+            for (ui32 i = 0; i < stats.GetColumnsCount(); ++i) {
                 result[NRequest::TOriginalDataAddress::CalcSubColumnHash(stats.GetColumnName(i))] += stats.GetColumnRecordsCount(i);
             }
         }
diff --git a/ydb/core/tx/columnshard/engines/storage/indexes/portions/extractor/sub_column.h b/ydb/core/tx/columnshard/engines/storage/indexes/portions/extractor/sub_column.h
@@ -49,7 +49,7 @@ class TSubColumnDataExtractor: public IReadDataExtractor {
         return request.GetSubColumnName() == SubColumnName;
     }
 
-    virtual ui32 DoGetIndexHitsCount(const std::shared_ptr<NArrow::NAccessor::IChunkedArray>& dataArray) const override;
+    virtual THashMap<ui64, ui32> DoGetIndexHitsCount(const std::shared_ptr<NArrow::NAccessor::IChunkedArray>& dataArray) const override;
 
 public:
     virtual TString GetClassName() const override {

Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ std::shared_ptr<TFetchingScript> TSpecialReadContext::DoGetColumnsFetchingPlan(c`
`48`	`48`	`return false;`
`49`	`49`	`}`
`50`	`50`	`}();`
`51`		`- const bool useIndexes = (IndexChecker ? source->HasIndexes(IndexChecker->GetIndexIds()) : false);`
	`51`	`+ const bool useIndexes = (IndexChecker ? source->HasIndexes(NIndexes::TIndexDataAddress::ExtractIndexIds(IndexChecker->GetIndexIds())) : false);`
`52`	`52`	`const bool isWholeExclusiveSource = source->GetExclusiveIntervalOnly() && source->IsSourceInMemory();`
`53`	`53`	`const bool needSnapshots = GetReadMetadata()->GetRequestSnapshot() < source->GetRecordSnapshotMax() \|\| !isWholeExclusiveSource;`
`54`	`54`	`const bool hasDeletions = source->GetHasDeletions();`
Original file line number	Diff line number	Diff line change
`@@ -174,8 +174,8 @@ void TPortionDataSource::DoAbort() {`
`174`	`174`	`}`
`175`	`175`
`176`	`176`	`void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& indexChecker) {`
`177`		`- THashMap<ui32, std::vector<TString>> indexBlobs;`
`178`		`- std::set<ui32> indexIds = indexChecker->GetIndexIds();`
	`177`	`+ THashMap<NIndexes::TIndexDataAddress, std::vector<TString>> indexBlobs;`
	`178`	`+ std::set<ui32> indexIds = NIndexes::TIndexDataAddress::ExtractIndexIds(indexChecker->GetIndexIds());`
`179`	`179`	`// NActors::TLogContextGuard gLog = NActors::TLogContextBuilder::Build()("records_count", GetRecordsCount())("portion_id", Portion->GetAddress().DebugString());`
`180`	`180`	`std::vector<TPortionDataAccessor::TPage> pages = GetStageData().GetPortionAccessor().BuildPages();`
`181`	`181`	`NArrow::TColumnFilter constructor = NArrow::TColumnFilter::BuildAllowFilter();`
`@@ -185,12 +185,12 @@ void TPortionDataSource::DoApplyIndex(const NIndexes::TIndexCheckerContainer& in`
`185`	`185`	`continue;`
`186`	`186`	`}`
`187`	`187`	`if (i->HasBlobData()) {`
`188`		`- indexBlobs[i->GetIndexId()].emplace_back(i->GetBlobDataVerified());`
	`188`	`+ indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(i->GetBlobDataVerified());`
`189`	`189`	`} else {`
`190`		`- indexBlobs[i->GetIndexId()].emplace_back(StageData->ExtractBlob(i->GetAddress()));`
	`190`	`+ indexBlobs[NIndexes::TIndexDataAddress(i->GetIndexId())].emplace_back(StageData->ExtractBlob(i->GetAddress()));`
`191`	`191`	`}`
`192`	`192`	`}`
`193`		`- for (auto&& i : indexIds) {`
	`193`	`+ for (auto&& i : indexChecker->GetIndexIds()) {`
`194`	`194`	`if (!indexBlobs.contains(i)) {`
`195`	`195`	`return;`
`196`	`196`	`}`
Original file line number	Diff line number	Diff line change
`@@ -366,11 +366,11 @@ ui32 ISnapshotSchema::GetIndexesCount() const {`
`366`	`366`	`return GetIndexInfo().GetIndexes().size();`
`367`	`367`	`}`
`368`	`368`
`369`		`-std::shared_ptr<NIndexes::IIndexMeta> ISnapshotSchema::GetIndexVerified(const ui32 indexId) const {`
	`369`	`+NIndexes::TIndexMetaContainer ISnapshotSchema::GetIndexVerified(const ui32 indexId) const {`
`370`	`370`	`return GetIndexInfo().GetIndexVerified(indexId);`
`371`	`371`	`}`
`372`	`372`
`373`		`-std::shared_ptr<NIndexes::IIndexMeta> ISnapshotSchema::GetIndexOptional(const ui32 indexId) const {`
	`373`	`+NIndexes::TIndexMetaContainer ISnapshotSchema::GetIndexOptional(const ui32 indexId) const {`
`374`	`374`	`return GetIndexInfo().GetIndexOptional(indexId);`
`375`	`375`	`}`
`376`	`376`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ class TFilterChecker: public TSimpleIndexChecker {`
`22`	`22`	`public:`
`23`	`23`	`TFilterChecker() = default;`
`24`	`24`	`TFilterChecker(const ui32 indexId, std::set<ui64>&& hashes)`
`25`		`- : TBase(indexId)`
	`25`	`+ : TBase(TIndexDataAddress(indexId))`
`26`	`26`	`, HashValues(std::move(hashes)) {`
`27`	`27`	`}`
`28`	`28`	`virtual TString GetClassName() const override {`