Skip to content

Commit 5ad1dcc

Browse files
kungakunga
committed
KIKIMR-19139 Load index in TForward (temporary solution)
1 parent 0ea90c0 commit 5ad1dcc

File tree

6 files changed

+87
-34
lines changed

6 files changed

+87
-34
lines changed

‎ydb/core/tablet_flat/flat_fwd_cache.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ namespace NTable {
1111
namespace NFwd {
1212

1313
class TCache : public IPageLoadingLogic {
14+
using TGroupId = NPage::TGroupId;
1415

1516
template<size_t Items>
1617
struct TRound {
@@ -57,8 +58,8 @@ namespace NFwd {
5758
public:
5859
TCache() = delete;
5960

60-
TCache(const NPage::TIndex& index, const TIntrusiveConstPtr<TSlices>& bounds = nullptr)
61-
: Index(index, 1, bounds)
61+
TCache(const TPart* part, IPages* env, TGroupId groupId, const TIntrusiveConstPtr<TSlices>& bounds = nullptr)
62+
: Index(part, env, groupId, 1, bounds)
6263
{ }
6364

6465
~TCache()

‎ydb/core/tablet_flat/flat_fwd_env.h

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -348,18 +348,16 @@ namespace NFwd {
348348
}
349349
}
350350

351-
TEgg MakeCache(const TPart *part, NPage::TGroupId groupId, TIntrusiveConstPtr<TSlices> bounds) const noexcept
351+
TEgg MakeCache(const TPart *part, NPage::TGroupId groupId, TIntrusiveConstPtr<TSlices> bounds) noexcept
352352
{
353353
auto *partStore = dynamic_cast<const TPartStore*>(part);
354354

355355
Y_VERIFY(groupId.Index < partStore->PageCollections.size(), "Got part without enough page collections");
356356

357357
auto& cache = partStore->PageCollections[groupId.Index];
358-
auto& index = part->GetGroupIndex(groupId);
359-
if (cache->PageCollection->Total() < index.UpperPage())
360-
Y_FAIL("Part index last page is out of storage capacity");
361-
362-
return { new NFwd::TCache(index, bounds), cache->PageCollection };
358+
359+
auto* fwd = new NFwd::TCache(part, this, groupId, bounds);
360+
return { fwd, cache->PageCollection };
363361
}
364362

365363
TEgg MakeExtern(const TPart *part, TIntrusiveConstPtr<TSlices> bounds) const noexcept

‎ydb/core/tablet_flat/flat_part_forward.h

Lines changed: 50 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#pragma once
22

3+
#include "flat_part_index_iter.h"
34
#include "flat_table_part.h"
45
#include "flat_page_index.h"
56
#include "flat_part_slice.h"
@@ -9,6 +10,8 @@ namespace NTable {
910

1011
class TForward {
1112
using TIndex = NPage::TIndex;
13+
using TGroupId = NPage::TGroupId;
14+
using TPageId = NPage::TPageId;
1215
using TIter = TIndex::TIter;
1316

1417
public:
@@ -28,40 +31,29 @@ namespace NTable {
2831
TPageId PageId = Max<TPageId>();
2932
};
3033

31-
explicit TForward(const TIndex& index, ui32 trace, const TIntrusiveConstPtr<TSlices>& bounds = nullptr)
34+
explicit TForward(const TPart* part, IPages* env, TGroupId groupId, ui32 trace, const TIntrusiveConstPtr<TSlices>& bounds = nullptr)
3235
: Trace(Max(ui32(1), trace))
33-
, Tail(index->Begin())
34-
, Head(index->Begin())
35-
, Edge(index->Begin())
36-
, End(index->End())
36+
, Index(part, env, groupId)
3737
{
3838
if (bounds && !bounds->empty()) {
39-
if (auto rowId = bounds->front().BeginRowId(); rowId > 0) {
40-
// Find the first page we would allow to load
41-
Tail = Head = Edge = index.LookupRow(rowId);
42-
}
43-
if (auto rowId = bounds->back().EndRowId(); rowId < index.GetEndRowId()) {
44-
// Find the first page we would refuse to load
45-
End = index.LookupRow(rowId);
46-
if (End && End->GetRowId() < rowId) {
47-
++End;
48-
}
49-
}
39+
BeginBoundRowId = bounds->front().BeginRowId();
40+
EndBoundRowId = bounds->back().EndRowId();
41+
} else {
42+
BeginBoundRowId = 0;
43+
EndBoundRowId = Max<TRowId>();
5044
}
5145
}
5246

53-
inline TPageId On(bool head = false) const noexcept
47+
inline TPageId On(bool head = false) noexcept
5448
{
49+
EnsureStarted();
5550
return PageOf(head ? Head : Edge);
5651
}
5752

58-
inline size_t Span() const noexcept
59-
{
60-
return Head - Tail;
61-
}
62-
6353
TResult Clean(TPageId until) noexcept
6454
{
55+
EnsureStarted();
56+
6557
if (PageOf(Tail) > until) {
6658
Y_FAIL("Part lookups goes below of its trace pages");
6759
} else {
@@ -84,6 +76,8 @@ namespace NTable {
8476

8577
TResult More(TPageId until) noexcept
8678
{
79+
EnsureStarted();
80+
8781
if (Head != End && ((Edge - Head >= 0) || PageOf(Head) < until)) {
8882
return (Head++)->GetPageId();
8983
}
@@ -92,6 +86,37 @@ namespace NTable {
9286
}
9387

9488
private:
89+
void EnsureStarted()
90+
{
91+
if (Started) {
92+
return;
93+
}
94+
95+
auto index = Index.TryLoadRaw();
96+
97+
// temporary solution: its too hard to handle index page faults in Clean/More methods
98+
Y_VERIFY(index, "Index should have been loaded before using TForward");
99+
100+
Tail = (*index)->Begin();
101+
Head = (*index)->Begin();
102+
Edge = (*index)->Begin();
103+
End = (*index)->End();
104+
105+
if (BeginBoundRowId > 0) {
106+
// Find the first page we would allow to load
107+
Tail = Head = Edge = index->LookupRow(BeginBoundRowId);
108+
}
109+
if (EndBoundRowId < index->GetEndRowId()) {
110+
// Find the first page we would refuse to load
111+
End = index->LookupRow(EndBoundRowId);
112+
if (End && End->GetRowId() < EndBoundRowId) {
113+
++End;
114+
}
115+
}
116+
117+
Started = true;
118+
}
119+
95120
inline TPageId PageOf(const TIter &it) const
96121
{
97122
return it == End ? Max<TPageId>() : it->GetPageId();
@@ -101,7 +126,10 @@ namespace NTable {
101126
const ui32 Trace = 0;
102127

103128
private:
129+
TPartIndexIt Index;
130+
bool Started = false;
104131
TIter Tail, Head, Edge, End;
132+
TRowId BeginBoundRowId, EndBoundRowId;
105133
};
106134
}
107135
}

‎ydb/core/tablet_flat/flat_part_index_iter.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,7 @@ class TPartIndexIt {
9393
return bool(Iter);
9494
}
9595

96-
// for precharge only
96+
// for precharge and TForward only
9797
TIndex* TryLoadRaw() {
9898
return TryGetIndex();
9999
}

‎ydb/core/tablet_flat/flat_scan_feed.h

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -334,7 +334,7 @@ namespace NTable {
334334
}
335335

336336
PrepareBoots();
337-
SeekState = ESeekState::SeekBoots;
337+
SeekState = ESeekState::LoadIndexes;
338338
return true;
339339
}
340340

@@ -356,6 +356,20 @@ namespace NTable {
356356
return LoadingParts == 0;
357357
}
358358

359+
bool LoadIndexes() noexcept
360+
{
361+
bool ready = true;
362+
for (const auto& partView : Subset.Flatten) {
363+
for (auto indexPageId : partView->IndexPages.Groups) {
364+
ready &= bool(CurrentEnv->TryGetPage(partView.Part.Get(), indexPageId));
365+
}
366+
for (auto indexPageId : partView->IndexPages.Historic) {
367+
ready &= bool(CurrentEnv->TryGetPage(partView.Part.Get(), indexPageId));
368+
}
369+
}
370+
return ready;
371+
}
372+
359373
void PrepareBoots() noexcept
360374
{
361375
auto keyDefaults = Subset.Scheme->Keys;
@@ -433,6 +447,9 @@ namespace NTable {
433447
return Boots.empty();
434448
}
435449

450+
/**
451+
* @return true on page fault
452+
*/
436453
bool Seek() noexcept
437454
{
438455
switch (SeekState) {
@@ -444,6 +461,12 @@ namespace NTable {
444461
[[fallthrough]];
445462
case ESeekState::PrepareBoots:
446463
PrepareBoots();
464+
SeekState = ESeekState::LoadIndexes;
465+
[[fallthrough]];
466+
case ESeekState::LoadIndexes:
467+
if (!LoadIndexes()) {
468+
return true;
469+
}
447470
SeekState = ESeekState::SeekBoots;
448471
[[fallthrough]];
449472
case ESeekState::SeekBoots:
@@ -473,6 +496,7 @@ namespace NTable {
473496
enum class ESeekState {
474497
LoadColdParts,
475498
PrepareBoots,
499+
LoadIndexes,
476500
SeekBoots,
477501
Finished,
478502
};

‎ydb/core/tablet_flat/test/libs/table/test_envs.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,8 @@ namespace NTest {
274274
for (ui32 room : xrange(partStore->Store->GetRoomCount())) {
275275
if (room < partStore->Store->GetGroupCount()) {
276276
NPage::TGroupId groupId(room);
277-
slots.push_back(Settle(partStore, room, new NFwd::TCache(partStore->GetGroupIndex(groupId))));
277+
auto *cache = new NFwd::TCache(part, this, groupId);
278+
slots.push_back(Settle(partStore, room, cache));
278279
} else if (room == partStore->Store->GetOuterRoom()) {
279280
slots.push_back(Settle(partStore, room, MakeOuter(partStore)));
280281
} else if (room == partStore->Store->GetExternRoom()) {
@@ -285,7 +286,8 @@ namespace NTest {
285286
}
286287
for (ui32 group : xrange(part->HistoricGroupsCount)) {
287288
NPage::TGroupId groupId(group, true);
288-
slots.push_back(Settle(partStore, group, new NFwd::TCache(partStore->GetGroupIndex(groupId))));
289+
auto *cache = new NFwd::TCache(part, this, groupId);
290+
slots.push_back(Settle(partStore, group, cache));
289291
}
290292
}
291293

0 commit comments

Comments
 (0)