Skip to content

Commit ce83f08

Browse files
ivanmorozov333ivanmorozov333
authored
Merge c7b504b into 5f44598 
2 parents 5f44598 + c7b504b commit ce83f08

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

60 files changed

+797
-991
lines changed

‎ydb/core/formats/arrow/accessor/plain/accessor.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,10 @@ class TChunkAccessor {
4343
return (ui64)ChunkedArray->num_chunks();
4444
}
4545
ui64 GetChunkLength(const ui32 idx) const {
46-
return (ui64)ChunkedArray->chunk(idx)->length();
46+
return (ui64)ChunkedArray->chunks()[idx]->length();
4747
}
4848
void OnArray(const ui32 idx, const ui32 startPosition) const {
49-
const auto& arr = ChunkedArray->chunk(idx);
50-
*Result = IChunkedArray::TLocalDataAddress(arr, startPosition, idx);
49+
*Result = IChunkedArray::TLocalDataAddress(ChunkedArray->chunk(idx), startPosition, idx);
5150
}
5251
};
5352

‎ydb/core/kqp/executer_actor/kqp_executer_impl.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1597,7 +1597,7 @@ class TKqpExecuterBase : public TActorBootstrapped<TDerived> {
15971597
}
15981598
}
15991599

1600-
if (!AppData()->FeatureFlags.GetEnableSeparationComputeActorsFromRead() || (!isOlapScan && readSettings.Sorted)) {
1600+
if (!AppData()->FeatureFlags.GetEnableSeparationComputeActorsFromRead()) {
16011601
for (auto&& pair : nodeShards) {
16021602
auto& shardsInfo = pair.second;
16031603
for (auto&& shardInfo : shardsInfo) {

‎ydb/core/tx/columnshard/columnshard_impl.cpp‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1397,11 +1397,8 @@ class TTxAskPortionChunks: public TTransactionBase<TColumnShard> {
13971397
for (auto&& i : PortionsByPath) {
13981398
AFL_INFO(NKikimrServices::TX_COLUMNSHARD)("event", "TTxAskPortionChunks::Execute")("size", i.second.size())("path_id", i.first);
13991399
for (auto&& p : i.second) {
1400-
{
1401-
auto rowset = db.Table<NColumnShard::Schema::IndexColumnsV2>().Prefix(p->GetPathId(), p->GetPortionId()).Select();
1402-
if (!rowset.IsReady()) {
1403-
reask = true;
1404-
}
1400+
if (!p->GetSchema(Self->GetIndexAs<NOlap::TColumnEngineForLogs>().GetVersionedIndex())->GetIndexesCount()) {
1401+
continue;
14051402
}
14061403
{
14071404
auto rowset = db.Table<NColumnShard::Schema::IndexIndexes>().Prefix(p->GetPathId(), p->GetPortionId()).Select();
@@ -1433,8 +1430,8 @@ class TTxAskPortionChunks: public TTransactionBase<TColumnShard> {
14331430
}
14341431
}
14351432
}
1436-
{
1437-
std::vector<NOlap::TIndexChunkLoadContext> indexes;
1433+
std::vector<NOlap::TIndexChunkLoadContext> indexes;
1434+
if (p->GetSchema(Self->GetIndexAs<NOlap::TColumnEngineForLogs>().GetVersionedIndex())->GetIndexesCount()) {
14381435
auto rowset = db.Table<NColumnShard::Schema::IndexIndexes>().Prefix(p->GetPathId(), p->GetPortionId()).Select();
14391436
if (!rowset.IsReady()) {
14401437
return false;
@@ -1445,8 +1442,8 @@ class TTxAskPortionChunks: public TTransactionBase<TColumnShard> {
14451442
return false;
14461443
}
14471444
}
1448-
constructor.SetIndexes(std::move(indexes));
14491445
}
1446+
constructor.SetIndexes(std::move(indexes));
14501447
FetchedAccessors.emplace_back(std::move(constructor));
14511448
i.second.pop_back();
14521449
}

‎ydb/core/tx/columnshard/data_accessor/request.h‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,11 @@ class TPathFetchingState {
145145
AFL_VERIFY(Portions.emplace(portion->GetPortionId(), portion).second);
146146
}
147147

148-
void AddAccessor(const TPortionDataAccessor& accessor) {
148+
void AddAccessor(
149+
const TPortionDataAccessor& accessor, const std::optional<std::set<ui32>>& columnIds, const std::optional<std::set<ui32>>& indexIds) {
149150
AFL_VERIFY(Stage == EFetchStage::Fetching);
150151
AFL_VERIFY(Portions.erase(accessor.GetPortionInfo().GetPortionId()));
151-
AFL_VERIFY(PortionAccessors.emplace(accessor.GetPortionInfo().GetPortionId(), accessor).second);
152+
AFL_VERIFY(PortionAccessors.emplace(accessor.GetPortionInfo().GetPortionId(), accessor.Extract(columnIds, indexIds)).second);
152153
if (Portions.empty()) {
153154
AFL_VERIFY(Stage == EFetchStage::Fetching);
154155
Stage = EFetchStage::Fetched;
@@ -176,8 +177,8 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter<TDat
176177
THashMap<ui64, TPathFetchingState> PathIdStatus;
177178
THashSet<ui64> PathIds;
178179
TDataAccessorsResult AccessorsByPathId;
179-
std::optional<std::vector<ui32>> ColumnIds;
180-
std::optional<std::vector<ui32>> IndexIds;
180+
YDB_READONLY_DEF(std::optional<std::set<ui32>>, ColumnIds);
181+
std::optional<std::set<ui32>> IndexIds;
181182

182183
TAtomicCounter PreparingCount = 0;
183184
TAtomicCounter FetchingCount = 0;
@@ -197,6 +198,11 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter<TDat
197198
}
198199

199200
public:
201+
void SetColumnIds(const std::set<ui32>& columnIds) {
202+
AFL_VERIFY(!ColumnIds);
203+
ColumnIds = columnIds;
204+
}
205+
200206
TString DebugString() const {
201207
TStringBuilder sb;
202208
sb << "request_id=" << RequestId << ";";
@@ -291,7 +297,7 @@ class TDataAccessorsRequest: public NColumnShard::TMonitoringObjectsCounter<TDat
291297
{
292298
auto itStatus = PathIdStatus.find(pathId);
293299
AFL_VERIFY(itStatus != PathIdStatus.end());
294-
itStatus->second.AddAccessor(accessor);
300+
itStatus->second.AddAccessor(accessor, ColumnIds, IndexIds);
295301
if (itStatus->second.IsFinished()) {
296302
AFL_VERIFY(FetchingCount.Dec() >= 0);
297303
ReadyCount.Inc();

‎ydb/core/tx/columnshard/data_locks/locks/abstract.h‎

Lines changed: 49 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,89 @@
11
#pragma once
22
#include <ydb/library/accessor/accessor.h>
33

4-
#include <util/generic/string.h>
54
#include <util/generic/hash_set.h>
5+
#include <util/generic/string.h>
66

7-
#include <optional>
87
#include <memory>
8+
#include <optional>
9+
#include <set>
910
#include <vector>
1011

1112
namespace NKikimr::NOlap {
1213
class TPortionInfo;
1314
class TGranuleMeta;
14-
}
15+
} // namespace NKikimr::NOlap
1516

1617
namespace NKikimr::NOlap::NDataLocks {
1718

19+
enum class ELockCategory : ui32 {
20+
Compaction = 0,
21+
Cleanup,
22+
Sharing,
23+
Actualization,
24+
Tables,
25+
MAX
26+
};
27+
28+
static const inline std::array<std::set<ELockCategory>, (ui32)ELockCategory::MAX> LockCategoriesInteraction = {
29+
//Compaction
30+
std::set<ELockCategory>({ ELockCategory::Compaction, ELockCategory::Actualization, ELockCategory::Tables }),
31+
//Cleanup
32+
std::set<ELockCategory>({ ELockCategory::Cleanup, ELockCategory::Sharing, ELockCategory::Tables }),
33+
//Sharing
34+
std::set<ELockCategory>({ ELockCategory::Sharing, ELockCategory::Cleanup, ELockCategory::Tables }),
35+
//Actualization
36+
std::set<ELockCategory>({ ELockCategory::Actualization, ELockCategory::Compaction, ELockCategory::Tables }),
37+
//Tables
38+
std::set<ELockCategory>(
39+
{ ELockCategory::Cleanup, ELockCategory::Sharing, ELockCategory::Actualization, ELockCategory::Compaction, ELockCategory::Tables }),
40+
};
41+
1842
class ILock {
1943
private:
2044
YDB_READONLY_DEF(TString, LockName);
2145
YDB_READONLY_FLAG(ReadOnly, false);
46+
const ELockCategory Category;
47+
2248
protected:
23-
virtual std::optional<TString> DoIsLocked(const TPortionInfo& portion, const THashSet<TString>& excludedLocks = {}) const = 0;
24-
virtual std::optional<TString> DoIsLocked(const TGranuleMeta& granule, const THashSet<TString>& excludedLocks = {}) const = 0;
49+
virtual std::optional<TString> DoIsLocked(
50+
const TPortionInfo& portion, const ELockCategory category, const THashSet<TString>& excludedLocks = {}) const = 0;
51+
virtual std::optional<TString> DoIsLocked(
52+
const TGranuleMeta& granule, const ELockCategory category, const THashSet<TString>& excludedLocks = {}) const = 0;
2553
virtual bool DoIsEmpty() const = 0;
54+
2655
public:
27-
ILock(const TString& lockName, const bool isReadOnly = false)
56+
ILock(const TString& lockName, const ELockCategory category, const bool isReadOnly = false)
2857
: LockName(lockName)
2958
, ReadOnlyFlag(isReadOnly)
30-
{
31-
59+
, Category(category) {
3260
}
3361

3462
virtual ~ILock() = default;
3563

36-
std::optional<TString> IsLocked(const TPortionInfo& portion, const THashSet<TString>& excludedLocks = {}, const bool readOnly = false) const {
64+
std::optional<TString> IsLocked(const TPortionInfo& portion, const ELockCategory portionForLock, const THashSet<TString>& excludedLocks = {},
65+
const bool readOnly = false) const {
3766
if (IsReadOnly() && readOnly) {
3867
return {};
3968
}
40-
return DoIsLocked(portion, excludedLocks);
69+
if (!LockCategoriesInteraction[(ui32)Category].contains(portionForLock)) {
70+
return {};
71+
}
72+
return DoIsLocked(portion, portionForLock, excludedLocks);
4173
}
42-
std::optional<TString> IsLocked(const TGranuleMeta& g, const THashSet<TString>& excludedLocks = {}, const bool readOnly = false) const {
74+
std::optional<TString> IsLocked(const TGranuleMeta& g, const ELockCategory portionForLock, const THashSet<TString>& excludedLocks = {},
75+
const bool readOnly = false) const {
4376
if (IsReadOnly() && readOnly) {
4477
return {};
4578
}
46-
return DoIsLocked(g, excludedLocks);
79+
if (!LockCategoriesInteraction[(ui32)Category].contains(portionForLock)) {
80+
return {};
81+
}
82+
return DoIsLocked(g, portionForLock, excludedLocks);
4783
}
4884
bool IsEmpty() const {
4985
return DoIsEmpty();
5086
}
5187
};
5288

53-
}
89+
} // namespace NKikimr::NOlap::NDataLocks

‎ydb/core/tx/columnshard/data_locks/locks/composite.h‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -8,23 +8,24 @@ class TCompositeLock: public ILock {
88
using TBase = ILock;
99
std::vector<std::shared_ptr<ILock>> Locks;
1010
protected:
11-
virtual std::optional<TString> DoIsLocked(const TPortionInfo& portion, const THashSet<TString>& excludedLocks) const override {
11+
virtual std::optional<TString> DoIsLocked(const TPortionInfo& portion, const ELockCategory category, const THashSet<TString>& excludedLocks) const override {
1212
for (auto&& i : Locks) {
1313
if (excludedLocks.contains(i->GetLockName())) {
1414
continue;
1515
}
16-
if (auto lockName = i->IsLocked(portion)) {
16+
if (auto lockName = i->IsLocked(portion, category)) {
1717
return lockName;
1818
}
1919
}
2020
return {};
2121
}
22-
virtual std::optional<TString> DoIsLocked(const TGranuleMeta& granule, const THashSet<TString>& excludedLocks) const override {
22+
virtual std::optional<TString> DoIsLocked(
23+
const TGranuleMeta& granule, const ELockCategory category, const THashSet<TString>& excludedLocks) const override {
2324
for (auto&& i : Locks) {
2425
if (excludedLocks.contains(i->GetLockName())) {
2526
continue;
2627
}
27-
if (auto lockName = i->IsLocked(granule)) {
28+
if (auto lockName = i->IsLocked(granule, category)) {
2829
return lockName;
2930
}
3031
}
@@ -34,8 +35,9 @@ class TCompositeLock: public ILock {
3435
return Locks.empty();
3536
}
3637
public:
37-
TCompositeLock(const TString& lockName, const std::vector<std::shared_ptr<ILock>>& locks, const bool readOnly = false)
38-
: TBase(lockName, readOnly)
38+
TCompositeLock(
39+
const TString& lockName, const std::vector<std::shared_ptr<ILock>>& locks, const ELockCategory category, const bool readOnly = false)
40+
: TBase(lockName, category, readOnly)
3941
{
4042
for (auto&& l : locks) {
4143
if (!l || l->IsEmpty()) {
@@ -45,8 +47,9 @@ class TCompositeLock: public ILock {
4547
}
4648
}
4749

48-
TCompositeLock(const TString& lockName, std::initializer_list<std::shared_ptr<ILock>> locks, const bool readOnly = false)
49-
: TBase(lockName, readOnly)
50+
TCompositeLock(
51+
const TString& lockName, std::initializer_list<std::shared_ptr<ILock>> locks, const ELockCategory category, const bool readOnly = false)
52+
: TBase(lockName, category, readOnly)
5053
{
5154
for (auto&& l : locks) {
5255
if (!l || l->IsEmpty()) {

‎ydb/core/tx/columnshard/data_locks/locks/list.h‎

Lines changed: 30 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,15 @@ class TListPortionsLock: public ILock {
1111
THashSet<TPortionAddress> Portions;
1212
THashSet<ui64> Granules;
1313
protected:
14-
virtual std::optional<TString> DoIsLocked(const TPortionInfo& portion, const THashSet<TString>& /*excludedLocks*/) const override {
14+
virtual std::optional<TString> DoIsLocked(
15+
const TPortionInfo& portion, const ELockCategory /*category*/, const THashSet<TString>& /*excludedLocks*/) const override {
1516
if (Portions.contains(portion.GetAddress())) {
1617
return GetLockName();
1718
}
1819
return {};
1920
}
20-
virtual std::optional<TString> DoIsLocked(const TGranuleMeta& granule, const THashSet<TString>& /*excludedLocks*/) const override {
21+
virtual std::optional<TString> DoIsLocked(
22+
const TGranuleMeta& granule, const ELockCategory /*category*/, const THashSet<TString>& /*excludedLocks*/) const override {
2123
if (Granules.contains(granule.GetPathId())) {
2224
return GetLockName();
2325
}
@@ -27,42 +29,46 @@ class TListPortionsLock: public ILock {
2729
return Portions.empty();
2830
}
2931
public:
30-
TListPortionsLock(const TString& lockName, const std::vector<TPortionDataAccessor>& portions, const bool readOnly = false)
31-
: TBase(lockName, readOnly)
32+
TListPortionsLock(const TString& lockName, const std::vector<TPortionDataAccessor>& portions, const ELockCategory category, const bool readOnly = false)
33+
: TBase(lockName, category, readOnly)
3234
{
3335
for (auto&& p : portions) {
3436
Portions.emplace(p.GetPortionInfo().GetAddress());
3537
Granules.emplace(p.GetPortionInfo().GetPathId());
3638
}
3739
}
3840

39-
TListPortionsLock(const TString& lockName, const std::vector<std::shared_ptr<TPortionInfo>>& portions, const bool readOnly = false)
40-
: TBase(lockName, readOnly) {
41+
TListPortionsLock(const TString& lockName, const std::vector<std::shared_ptr<TPortionInfo>>& portions, const ELockCategory category,
42+
const bool readOnly = false)
43+
: TBase(lockName, category, readOnly) {
4144
for (auto&& p : portions) {
4245
Portions.emplace(p->GetAddress());
4346
Granules.emplace(p->GetPathId());
4447
}
4548
}
4649

47-
TListPortionsLock(const TString& lockName, const std::vector<TPortionInfo::TConstPtr>& portions, const bool readOnly = false)
48-
: TBase(lockName, readOnly) {
50+
TListPortionsLock(
51+
const TString& lockName, const std::vector<TPortionInfo::TConstPtr>& portions, const ELockCategory category, const bool readOnly = false)
52+
: TBase(lockName, category, readOnly) {
4953
for (auto&& p : portions) {
5054
Portions.emplace(p->GetAddress());
5155
Granules.emplace(p->GetPathId());
5256
}
5357
}
5458

55-
TListPortionsLock(const TString& lockName, const std::vector<TPortionInfo>& portions, const bool readOnly = false)
56-
: TBase(lockName, readOnly) {
59+
TListPortionsLock(
60+
const TString& lockName, const std::vector<TPortionInfo>& portions, const ELockCategory category, const bool readOnly = false)
61+
: TBase(lockName, category, readOnly) {
5762
for (auto&& p : portions) {
5863
Portions.emplace(p.GetAddress());
5964
Granules.emplace(p.GetPathId());
6065
}
6166
}
6267

6368
template <class T, class TGetter>
64-
TListPortionsLock(const TString& lockName, const std::vector<T>& portions, const TGetter& g, const bool readOnly = false)
65-
: TBase(lockName, readOnly) {
69+
TListPortionsLock(
70+
const TString& lockName, const std::vector<T>& portions, const TGetter& g, const ELockCategory category, const bool readOnly = false)
71+
: TBase(lockName, category, readOnly) {
6672
for (auto&& p : portions) {
6773
const auto address = g(p);
6874
Portions.emplace(address);
@@ -71,17 +77,19 @@ class TListPortionsLock: public ILock {
7177
}
7278

7379
template <class T>
74-
TListPortionsLock(const TString& lockName, const THashMap<TPortionAddress, T>& portions, const bool readOnly = false)
75-
: TBase(lockName, readOnly) {
80+
TListPortionsLock(
81+
const TString& lockName, const THashMap<TPortionAddress, T>& portions, const ELockCategory category, const bool readOnly = false)
82+
: TBase(lockName, category, readOnly) {
7683
for (auto&& p : portions) {
7784
const auto address = p.first;
7885
Portions.emplace(address);
7986
Granules.emplace(address.GetPathId());
8087
}
8188
}
8289

83-
TListPortionsLock(const TString& lockName, const THashSet<TPortionAddress>& portions, const bool readOnly = false)
84-
: TBase(lockName, readOnly) {
90+
TListPortionsLock(
91+
const TString& lockName, const THashSet<TPortionAddress>& portions, const ELockCategory category, const bool readOnly = false)
92+
: TBase(lockName, category, readOnly) {
8593
for (auto&& address : portions) {
8694
Portions.emplace(address);
8795
Granules.emplace(address.GetPathId());
@@ -94,13 +102,15 @@ class TListTablesLock: public ILock {
94102
using TBase = ILock;
95103
THashSet<ui64> Tables;
96104
protected:
97-
virtual std::optional<TString> DoIsLocked(const TPortionInfo& portion, const THashSet<TString>& /*excludedLocks*/) const override {
105+
virtual std::optional<TString> DoIsLocked(
106+
const TPortionInfo& portion, const ELockCategory /*category*/, const THashSet<TString>& /*excludedLocks*/) const override {
98107
if (Tables.contains(portion.GetPathId())) {
99108
return GetLockName();
100109
}
101110
return {};
102111
}
103-
virtual std::optional<TString> DoIsLocked(const TGranuleMeta& granule, const THashSet<TString>& /*excludedLocks*/) const override {
112+
virtual std::optional<TString> DoIsLocked(
113+
const TGranuleMeta& granule, const ELockCategory /*category*/, const THashSet<TString>& /*excludedLocks*/) const override {
104114
if (Tables.contains(granule.GetPathId())) {
105115
return GetLockName();
106116
}
@@ -110,8 +120,8 @@ class TListTablesLock: public ILock {
110120
return Tables.empty();
111121
}
112122
public:
113-
TListTablesLock(const TString& lockName, const THashSet<ui64>& tables, const bool readOnly = false)
114-
: TBase(lockName, readOnly)
123+
TListTablesLock(const TString& lockName, const THashSet<ui64>& tables, const ELockCategory category, const bool readOnly = false)
124+
: TBase(lockName, category, readOnly)
115125
, Tables(tables)
116126
{
117127
}

0 commit comments

Comments
 (0)