Skip to content

Increase index flags capacity in ODS14 #8340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Feb 20, 2025
Merged

Increase index flags capacity in ODS14 #8340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2a00a58
Increase index flags capacity. Store transaction number as plain int6…
dyemanov Dec 4, 2024
428dd18
Revert back private members of irt_repeat
dyemanov Dec 5, 2024
fefe8c1
Fix alignment incompatibility between 32-bit and 64-bit builds. Parti…
dyemanov Jan 20, 2025
e4d31a3
Merge branch 'master' into ods14-irt-repeat
dyemanov Feb 15, 2025
78a5a56
Sync with the metacache branch
dyemanov Feb 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 14 additions & 13 deletions src/jrd/btr.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -924,12 +924,12 @@ bool BTR_delete_index(thread_db* tdbb, WIN* window, USHORT id)
{
index_root_page::irt_repeat* irt_desc = root->irt_rpt + id;
CCH_MARK(tdbb, window);
const PageNumber next(window->win_page.getPageSpaceID(), irt_desc->getRoot());
tree_exists = (irt_desc->getRoot() != 0);
const ULONG rootPage = irt_desc->getRoot();
const PageNumber next(window->win_page.getPageSpaceID(), rootPage);
tree_exists = (rootPage != 0);

// remove the pointer to the top-level index page before we delete it
irt_desc->setRoot(0);
irt_desc->irt_flags = 0;
irt_desc->setEmpty();
const PageNumber prior = window->win_page;
const USHORT relation_id = root->irt_relation;

Expand Down Expand Up @@ -962,11 +962,12 @@ bool BTR_description(thread_db* tdbb, jrd_rel* relation, index_root_page* root,

const index_root_page::irt_repeat* irt_desc = &root->irt_rpt[id];

if (irt_desc->getRoot() == 0)
const ULONG rootPage = irt_desc->getRoot();
if (!rootPage)
return false;

idx->idx_id = id;
idx->idx_root = irt_desc->getRoot();
idx->idx_root = rootPage;
idx->idx_count = irt_desc->irt_keys;
idx->idx_flags = irt_desc->irt_flags;
idx->idx_runtime_flags = 0;
Expand Down Expand Up @@ -2124,18 +2125,18 @@ bool BTR_next_index(thread_db* tdbb, jrd_rel* relation, jrd_tra* transaction, in
for (; id < root->irt_count; ++id)
{
const index_root_page::irt_repeat* irt_desc = root->irt_rpt + id;
if (irt_desc->getTransaction() && transaction)
const TraNumber inProgressTrans = irt_desc->inProgress();
if (inProgressTrans && transaction)
{
const TraNumber trans = irt_desc->getTransaction();
CCH_RELEASE(tdbb, window);
const int trans_state = TRA_wait(tdbb, transaction, trans, jrd_tra::tra_wait);
const int trans_state = TRA_wait(tdbb, transaction, inProgressTrans, jrd_tra::tra_wait);
if ((trans_state == tra_dead) || (trans_state == tra_committed))
{
// clean up this left-over index
root = (index_root_page*) CCH_FETCH(tdbb, window, LCK_write, pag_root);
irt_desc = root->irt_rpt + id;

if (irt_desc->getTransaction() == trans)
if (irt_desc->inProgress() == inProgressTrans)
BTR_delete_index(tdbb, window, id);
else
CCH_RELEASE(tdbb, window);
Expand Down Expand Up @@ -2359,7 +2360,7 @@ void BTR_reserve_slot(thread_db* tdbb, IndexCreation& creation)
fb_assert(idx->idx_count <= MAX_UCHAR);
slot->irt_keys = (UCHAR) idx->idx_count;
slot->irt_flags = idx->idx_flags;
slot->setTransaction(transaction->tra_number);
slot->setInProgress(transaction->tra_number);

// Exploit the fact idx_repeat structure matches ODS IRTD one
memcpy(desc, idx->idx_rpt, len);
Expand Down Expand Up @@ -2393,13 +2394,13 @@ void BTR_selectivity(thread_db* tdbb, jrd_rel* relation, USHORT id, SelectivityL
if (!root)
return;

ULONG page;
if (id >= root->irt_count || !(page = root->irt_rpt[id].getRoot()))
if (id >= root->irt_count || !root->irt_rpt[id].getRoot())
{
CCH_RELEASE(tdbb, &window);
return;
}

ULONG page = root->irt_rpt[id].getRoot();
const bool descending = (root->irt_rpt[id].irt_flags & irt_descending);
const ULONG segments = root->irt_rpt[id].irt_keys;

Expand Down
2 changes: 1 addition & 1 deletion src/jrd/btr.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ struct index_desc
ULONG idx_root; // Index root
float idx_selectivity; // selectivity of index
USHORT idx_id;
UCHAR idx_flags;
USHORT idx_flags;
UCHAR idx_runtime_flags; // flags used at runtime, not stored on disk
USHORT idx_primary_index; // id for primary key partner index
USHORT idx_primary_relation; // id for primary key partner relation
Expand Down
2 changes: 1 addition & 1 deletion src/jrd/idx.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ struct ini_idx_t
{
UCHAR ini_idx_index_id;
UCHAR ini_idx_relid;
UCHAR ini_idx_flags;
USHORT ini_idx_flags;
UCHAR ini_idx_segment_count;
USHORT ini_idx_ods;
struct ini_idx_segment_t
Expand Down
2 changes: 1 addition & 1 deletion src/jrd/ini.epp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1636,7 +1636,7 @@ static void store_indices(thread_db* tdbb, USHORT odsVersion)
PAD(relation->rel_name, X.RDB$RELATION_NAME);
PAD(indexName, X.RDB$INDEX_NAME);

X.RDB$UNIQUE_FLAG = index->ini_idx_flags & idx_unique;
X.RDB$UNIQUE_FLAG = (index->ini_idx_flags & idx_unique) ? 1 : 0;
X.RDB$SEGMENT_COUNT = index->ini_idx_segment_count;

if (index->ini_idx_flags & idx_descending)
Expand Down
114 changes: 75 additions & 39 deletions src/jrd/ods.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,42 +365,51 @@ struct index_root_page
{
pag irt_header;
USHORT irt_relation; // relation id (for consistency)
USHORT irt_count; // Number of indices
USHORT irt_count; // number of indices
ULONG irt_dummy; // so far used as a padding to ensure the same
// alignment between 32-bit and 64-bit builds
struct irt_repeat
{
friend class index_root_page; // to allow offset check for private members

private:
friend struct index_root_page; // to allow offset check for private members
ULONG irt_root; // page number of index root if irt_in_progress is NOT set, or
// highest 32 bit of transaction if irt_in_progress is set
ULONG irt_transaction; // transaction in progress (lowest 32 bits)
FB_UINT64 irt_transaction; // transaction in progress
ULONG irt_page_num; // page number
ULONG irt_page_space_id; // page space
public:
USHORT irt_desc; // offset to key descriptions
UCHAR irt_keys; // number of keys in index
UCHAR irt_flags;
USHORT irt_desc; // offset to key descriptions
USHORT irt_flags; // index flags
UCHAR irt_state; // index state
UCHAR irt_keys; // number of keys in index
private:
USHORT irt_dummy; // alignment to 8-byte boundary

ULONG getRoot() const;
void setRoot(ULONG root_page);
public:
TraNumber inProgress() const;
void setInProgress(TraNumber traNumber);

TraNumber getTransaction() const;
void setTransaction(TraNumber traNumber);
ULONG getRoot() const;
void setRoot(ULONG rootPage);

bool isUsed() const;

void setEmpty();
} irt_rpt[1];

static_assert(sizeof(struct irt_repeat) == 12, "struct irt_repeat size mismatch");
static_assert(offsetof(struct irt_repeat, irt_root) == 0, "irt_root offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_transaction) == 4, "irt_transaction offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_desc) == 8, "irt_desc offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_keys) == 10, "irt_keys offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_flags) == 11, "irt_flags offset mismatch");
static_assert(sizeof(struct irt_repeat) == 24, "struct irt_repeat size mismatch");
static_assert(offsetof(struct irt_repeat, irt_transaction) == 0, "irt_transaction offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_page_num) == 8, "irt_root offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_page_space_id) == 12, "irt_root offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_desc) == 16, "irt_desc offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_flags) == 18, "irt_flags offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_state) == 20, "irt_state offset mismatch");
static_assert(offsetof(struct irt_repeat, irt_keys) == 21, "irt_keys offset mismatch");
};

static_assert(sizeof(struct index_root_page) == 32, "struct index_root_page size mismatch");
static_assert(sizeof(struct index_root_page) == 48, "struct index_root_page size mismatch");
static_assert(offsetof(struct index_root_page, irt_header) == 0, "irt_header offset mismatch");
static_assert(offsetof(struct index_root_page, irt_relation) == 16, "irt_relation offset mismatch");
static_assert(offsetof(struct index_root_page, irt_count) == 18, "irt_count offset mismatch");
static_assert(offsetof(struct index_root_page, irt_rpt) == 20, "irt_rpt offset mismatch");
static_assert(offsetof(struct index_root_page, irt_rpt) == 24, "irt_rpt offset mismatch");

// key descriptor

Expand All @@ -419,38 +428,65 @@ static_assert(offsetof(struct irtd, irtd_selectivity) == 4, "irtd_selectivity of
// irt_flags, must match the idx_flags (see btr.h)
inline constexpr USHORT irt_unique = 1;
inline constexpr USHORT irt_descending = 2;
inline constexpr USHORT irt_in_progress = 4;
inline constexpr USHORT irt_foreign = 8;
inline constexpr USHORT irt_primary = 16;
inline constexpr USHORT irt_expression = 32;
inline constexpr USHORT irt_condition = 64;
inline constexpr USHORT irt_foreign = 4;
inline constexpr USHORT irt_primary = 8;
inline constexpr USHORT irt_expression = 16;
inline constexpr USHORT irt_condition = 32;

// possible index states
inline constexpr UCHAR irt_unused = 0; // empty slot
inline constexpr UCHAR irt_in_progress = 1; // under construction - sort & merge
inline constexpr UCHAR irt_rollback = 2; // index to be removed when irt_transaction dead (rolled back)
inline constexpr UCHAR irt_normal = 3; // normal working state of index
inline constexpr UCHAR irt_kill = 4; // index to be removed when irt_transaction ended (both commit/rollback)
inline constexpr UCHAR irt_commit = 5; // start index removal (switch to irt_drop) when irt_transaction committed
inline constexpr UCHAR irt_drop = 6; // index to be removed when OAT > irt_transaction

inline ULONG index_root_page::irt_repeat::getRoot() const
inline bool index_root_page::irt_repeat::isUsed() const
{
return (irt_flags & irt_in_progress) ? 0 : irt_root;
return (irt_state != irt_unused);
}

inline void index_root_page::irt_repeat::setRoot(ULONG root_page)
inline void index_root_page::irt_repeat::setEmpty()
{
irt_root = root_page;
irt_flags &= ~irt_in_progress;
irt_transaction = 0;
irt_page_num = 0;
irt_page_space_id = 0;
irt_flags = 0;
irt_state = irt_unused;
}

inline TraNumber index_root_page::irt_repeat::getTransaction() const
inline TraNumber index_root_page::irt_repeat::inProgress() const
{
return (irt_flags & irt_in_progress) ? ((TraNumber) irt_root << BITS_PER_LONG) | irt_transaction : 0;
return irt_transaction;
}

inline void index_root_page::irt_repeat::setTransaction(TraNumber traNumber)
inline void index_root_page::irt_repeat::setInProgress(TraNumber traNumber)
{
irt_root = ULONG(traNumber >> BITS_PER_LONG);
irt_transaction = ULONG(traNumber);
irt_flags |= irt_in_progress;
fb_assert(irt_state == irt_unused);
fb_assert(!irt_page_num && !irt_page_space_id);

irt_transaction = traNumber;
irt_page_num = 0;
irt_page_space_id = 0;
irt_state = irt_in_progress;
}

inline bool index_root_page::irt_repeat::isUsed() const
inline ULONG index_root_page::irt_repeat::getRoot() const
{
return (irt_state == irt_unused) ? 0 : irt_page_num;
}

inline void index_root_page::irt_repeat::setRoot(ULONG rootPage)
{
return (irt_flags & irt_in_progress) || (irt_root != 0);
fb_assert(irt_state == irt_in_progress);
fb_assert(!irt_page_num && !irt_page_space_id);
fb_assert(rootPage);

irt_transaction = 0;
irt_page_num = rootPage;
irt_page_space_id = 0;
irt_state = irt_normal;
}


Expand Down
27 changes: 13 additions & 14 deletions src/jrd/validation.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1956,7 +1956,7 @@ void Validation::walk_generators()
}
}

Validation::RTN Validation::walk_index(jrd_rel* relation, index_root_page& root_page, USHORT id)
Validation::RTN Validation::walk_index(jrd_rel* relation, index_root_page* root_page, USHORT id)
{
/**************************************
*
Expand All @@ -1976,25 +1976,24 @@ Validation::RTN Validation::walk_index(jrd_rel* relation, index_root_page& root_
**************************************/
Database* dbb = vdr_tdbb->getDatabase();

const ULONG page_number = root_page.irt_rpt[id].getRoot();
if (!page_number) {
const ULONG page_number = root_page->irt_rpt[id].getRoot();
if (!page_number)
return rtn_ok;
}

const bool unique = (root_page.irt_rpt[id].irt_flags & (irt_unique | idx_primary));
const bool descending = (root_page.irt_rpt[id].irt_flags & irt_descending);
const bool condition = (root_page.irt_rpt[id].irt_flags & irt_condition);
const bool unique = (root_page->irt_rpt[id].irt_flags & (irt_unique | idx_primary));
const bool descending = (root_page->irt_rpt[id].irt_flags & irt_descending);
const bool condition = (root_page->irt_rpt[id].irt_flags & irt_condition);

temporary_key nullKey, *null_key = 0;
temporary_key nullKey, *null_key = nullptr;
if (unique)
{
index_desc idx;
{
// No need to evaluate index expression and/or condition
AutoSetRestoreFlag<UCHAR> flags(&root_page.irt_rpt[id].irt_flags,
AutoSetRestoreFlag<USHORT> flags(&root_page->irt_rpt[id].irt_flags,
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it necessary to specify template arg, could it be auto deducted from ctor arguments by compiler ?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's C++17 feature (https://en.cppreference.com/w/cpp/language/class_template_argument_deduction)
AFAIK, we were not using it, but we could.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've found at least one such usage of AutoSetRestore in remote.cpp - see PortsCleanup::closePorts().

Copy link
Member Author

@dyemanov dyemanov Dec 5, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Auto deduction works for irt_expression, but fails for irt_expression | irt_condition, it should be explicitly casted to USHORT to work and it somewhat defeats the idea.

Nevertheless, I'd suggest (outside this PR) to remove AutoSetRestoreFlag here at all and instead add the needed lookup flags to BTR_description. Because the index root page may be locked as LCK_read but we modify data there (even if temporarily) -- this looks hackery.

irt_expression | irt_condition, false);

BTR_description(vdr_tdbb, relation, &root_page, &idx, id);
BTR_description(vdr_tdbb, relation, root_page, &idx, id);
}

null_key = &nullKey;
Expand Down Expand Up @@ -3210,13 +3209,13 @@ Validation::RTN Validation::walk_root(jrd_rel* relation, bool getInfo)
if (!relPages->rel_index_root)
return corrupt(VAL_INDEX_ROOT_MISSING, relation);

index_root_page* page = 0;
index_root_page* page = nullptr;
WIN window(DB_PAGE_SPACE, -1);
fetch_page(!getInfo, relPages->rel_index_root, pag_root, &window, &page);

for (USHORT i = 0; i < page->irt_count; i++)
{
if (page->irt_rpt[i].getRoot() == 0)
if (!page->irt_rpt[i].getRoot())
continue;

MetaName index;
Expand All @@ -3242,7 +3241,7 @@ Validation::RTN Validation::walk_root(jrd_rel* relation, bool getInfo)
if (page->irt_rpt[i].irt_flags & irt_condition)
{
// No need to evaluate index expression
AutoSetRestoreFlag<UCHAR> flag(&page->irt_rpt[i].irt_flags, irt_expression, false);
AutoSetRestoreFlag<USHORT> flag(&page->irt_rpt[i].irt_flags, irt_expression, false);

IdxInfo info;
if (BTR_description(vdr_tdbb, relation, page, &info.m_desc, i))
Expand All @@ -3252,7 +3251,7 @@ Validation::RTN Validation::walk_root(jrd_rel* relation, bool getInfo)
}

output("Index %d (%s)\n", i + 1, index.c_str());
walk_index(relation, *page, i);
walk_index(relation, page, i);
}

release_page(&window);
Expand Down
2 changes: 1 addition & 1 deletion src/jrd/validation.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ class Validation
RTN walk_data_page(jrd_rel*, ULONG, ULONG, UCHAR&);
void walk_database();
void walk_generators();
RTN walk_index(jrd_rel*, Ods::index_root_page&, USHORT);
RTN walk_index(jrd_rel*, Ods::index_root_page*, USHORT);
void walk_pip();
RTN walk_pointer_page(jrd_rel*, ULONG);
RTN walk_record(jrd_rel*, const Ods::rhd*, USHORT, RecordNumber, bool);
Expand Down
Loading
Loading