Skip to content

Commit

Permalink
[Enhancement] Resolve a protobuf/glibc incompatability. (StarRocks#29331
Browse files Browse the repository at this point in the history 
)
  • Loading branch information
@DorianZheng
DorianZheng authored Aug 18, 2023
1 parent 2e6b460 commit 7e0e1b9
Show file tree
Hide file tree
Showing 17 changed files with 192 additions and 178 deletions.
12 changes: 6 additions & 6 deletions be/src/script/script.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,12 @@ static DataDir* tablet_data_dir(Tablet& tablet) {
return tablet.data_dir();
}

static uint64_t get_major(EditVersion& self) {
return self.major();
static uint64_t get_major_number(EditVersion& self) {
return self.major_number();
}

static uint64_t get_minor(EditVersion& self) {
return self.minor();
static uint64_t get_minor_number(EditVersion& self) {
return self.minor_number();
}

static void bind_common(ForeignModule& m) {
Expand Down Expand Up @@ -400,8 +400,8 @@ class StorageEngineRef {
}
{
auto& cls = m.klass<EditVersion>("EditVersion");
cls.funcExt<&get_major>("major");
cls.funcExt<&get_minor>("minor");
cls.funcExt<&get_major_number>("major_number");
cls.funcExt<&get_minor_number>("minor_number");
cls.func<&EditVersion::to_string>("toString");
}
{
Expand Down
10 changes: 5 additions & 5 deletions be/src/storage/edit_version.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,17 +23,17 @@ namespace starrocks {
struct EditVersion {
unsigned __int128 value = 0;
EditVersion() = default;
EditVersion(const EditVersionPB& pb) : EditVersion(pb.major(), pb.minor()) {}
EditVersion(const EditVersionPB& pb) : EditVersion(pb.major_number(), pb.minor_number()) {}
EditVersion(int64_t major, int64_t minor) { value = (((unsigned __int128)major) << 64) | minor; }
int64_t major() const { return (int64_t)(value >> 64); }
int64_t minor() const { return (int64_t)(value & 0xffffffffUL); }
int64_t major_number() const { return (int64_t)(value >> 64); }
int64_t minor_number() const { return (int64_t)(value & 0xffffffffUL); }
std::string to_string() const;
bool operator<(const EditVersion& rhs) const { return value < rhs.value; }
bool operator==(const EditVersion& rhs) const { return value == rhs.value; }

void to_pb(EditVersionPB* pb) const {
pb->set_major(major());
pb->set_minor(minor());
pb->set_major_number(major_number());
pb->set_minor_number(minor_number());
}
};

Expand Down
16 changes: 8 additions & 8 deletions be/src/storage/lake/lake_local_persistent_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,9 @@ Status LakeLocalPersistentIndex::load_from_lake_tablet(starrocks::lake::Tablet*
EditVersion version = index_meta.version();

// Compaction of Lake tablet also generate a new version
// here just use version.major so that PersistentIndexMetaPB need't to be modified,
// the minor is meaningless
if (version.major() == base_version) {
// here just use version.major_number so that PersistentIndexMetaPB need't to be modified,
// the minor_number is meaningless
if (version.major_number() == base_version) {
// If format version is not equal to PERSISTENT_INDEX_VERSION_2, this maybe upgrade from
// PERSISTENT_INDEX_VERSION_2.
// We need to rebuild persistent index because the meta structure is changed
Expand All @@ -86,23 +86,23 @@ Status LakeLocalPersistentIndex::load_from_lake_tablet(starrocks::lake::Tablet*
if (index_meta.has_l0_meta()) {
EditVersion l0_version = index_meta.l0_meta().snapshot().version();
std::string l0_file_name =
strings::Substitute("index.l0.$0.$1", l0_version.major(), l0_version.minor());
strings::Substitute("index.l0.$0.$1", l0_version.major_number(), l0_version.minor_number());
Status st = FileSystem::Default()->delete_file(l0_file_name);
LOG(WARNING) << "delete error l0 index file: " << l0_file_name << ", status: " << st;
}
if (index_meta.has_l1_version()) {
EditVersion l1_version = index_meta.l1_version();
std::string l1_file_name =
strings::Substitute("index.l1.$0.$1", l1_version.major(), l1_version.minor());
strings::Substitute("index.l1.$0.$1", l1_version.major_number(), l1_version.minor_number());
Status st = FileSystem::Default()->delete_file(l1_file_name);
LOG(WARNING) << "delete error l1 index file: " << l1_file_name << ", status: " << st;
}
if (index_meta.l2_versions_size() > 0) {
for (int i = 0; i < index_meta.l2_versions_size(); i++) {
EditVersion l2_version = index_meta.l2_versions(i);
std::string l2_file_name =
strings::Substitute("index.l2.$0.$1$2", l2_version.major(), l2_version.minor(),
index_meta.l2_version_merged(i) ? MergeSuffix : "");
std::string l2_file_name = strings::Substitute(
"index.l2.$0.$1$2", l2_version.major_number(), l2_version.minor_number(),
index_meta.l2_version_merged(i) ? MergeSuffix : "");
Status st = FileSystem::Default()->delete_file(l2_file_name);
LOG(WARNING) << "delete error l2 index file: " << l2_file_name << ", status: " << st;
}
Expand Down
2 changes: 1 addition & 1 deletion be/src/storage/local_tablet_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ Status LocalTabletReader::multi_get(const Chunk& keys, const std::vector<uint32_
EditVersion edit_version;
std::vector<uint64_t> rowids(n);
RETURN_IF_ERROR(_tablet->updates()->get_rss_rowids_by_pk(_tablet.get(), *pk_column, &edit_version, &rowids));
if (edit_version.major() != _version) {
if (edit_version.major_number() != _version) {
return Status::InternalError(
strings::Substitute("multi_get version not match tablet:$0 current_version:$1 read_version:$2",
_tablet->tablet_id(), edit_version.to_string(), _version));
Expand Down
68 changes: 36 additions & 32 deletions be/src/storage/persistent_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ T pad(T v, P p) {
}

static std::string get_l0_index_file_name(std::string& dir, const EditVersion& version) {
return strings::Substitute("$0/index.l0.$1.$2", dir, version.major(), version.minor());
return strings::Substitute("$0/index.l0.$1.$2", dir, version.major_number(), version.minor_number());
}

struct IndexHash {
Expand Down Expand Up @@ -1880,7 +1880,7 @@ Status ShardByLengthMutableIndex::flush_to_immutable_index(const std::string& pa
auto writer = std::make_unique<ImmutableIndexWriter>();
std::string idx_file_path;
if (!write_tmp_l1) {
idx_file_path = strings::Substitute("$0/index.l1.$1.$2", path, version.major(), version.minor());
idx_file_path = strings::Substitute("$0/index.l1.$1.$2", path, version.major_number(), version.minor_number());
} else {
idx_file_path = path;
}
Expand Down Expand Up @@ -2422,7 +2422,8 @@ Status PersistentIndex::_load(const PersistentIndexMetaPB& index_meta, bool relo
// so we use index file size as the _usage and the _usage will be adjusted in subsequent compaction.
if (index_meta.has_l1_version()) {
EditVersion version = index_meta.l1_version();
auto l1_file_path = strings::Substitute("$0/index.l1.$1.$2", _path, version.major(), version.minor());
auto l1_file_path =
strings::Substitute("$0/index.l1.$1.$2", _path, version.major_number(), version.minor_number());
auto l1_st = _fs->get_file_size(l1_file_path);
if (!l1_st.ok()) {
return l1_st.status();
Expand Down Expand Up @@ -2459,7 +2460,8 @@ Status PersistentIndex::_load(const PersistentIndexMetaPB& index_meta, bool relo
}
if (index_meta.has_l1_version()) {
_l1_version = index_meta.l1_version();
auto l1_block_path = strings::Substitute("$0/index.l1.$1.$2", _path, _l1_version.major(), _l1_version.minor());
auto l1_block_path =
strings::Substitute("$0/index.l1.$1.$2", _path, _l1_version.major_number(), _l1_version.minor_number());
ASSIGN_OR_RETURN(auto l1_rfile, _fs->new_random_access_file(l1_block_path));
auto l1_st = ImmutableIndex::load(std::move(l1_rfile));
if (!l1_st.ok()) {
Expand All @@ -2480,9 +2482,9 @@ Status PersistentIndex::_load(const PersistentIndexMetaPB& index_meta, bool relo
}
if (index_meta.l2_versions_size() > 0) {
for (int i = 0; i < index_meta.l2_versions_size(); i++) {
auto l2_block_path = strings::Substitute("$0/index.l2.$1.$2$3", _path, index_meta.l2_versions(i).major(),
index_meta.l2_versions(i).minor(),
index_meta.l2_version_merged(i) ? MergeSuffix : "");
auto l2_block_path = strings::Substitute(
"$0/index.l2.$1.$2$3", _path, index_meta.l2_versions(i).major_number(),
index_meta.l2_versions(i).minor_number(), index_meta.l2_version_merged(i) ? MergeSuffix : "");
ASSIGN_OR_RETURN(auto l2_rfile, _fs->new_random_access_file(l2_block_path));
ASSIGN_OR_RETURN(auto l2_index, ImmutableIndex::load(std::move(l2_rfile)));
{
Expand Down Expand Up @@ -2700,23 +2702,23 @@ Status PersistentIndex::load_from_tablet(Tablet* tablet) {
if (index_meta.has_l0_meta()) {
EditVersion l0_version = index_meta.l0_meta().snapshot().version();
std::string l0_file_name =
strings::Substitute("index.l0.$0.$1", l0_version.major(), l0_version.minor());
strings::Substitute("index.l0.$0.$1", l0_version.major_number(), l0_version.minor_number());
Status st = FileSystem::Default()->delete_file(l0_file_name);
LOG(WARNING) << "delete error l0 index file: " << l0_file_name << ", status: " << st;
}
if (index_meta.has_l1_version()) {
EditVersion l1_version = index_meta.l1_version();
std::string l1_file_name =
strings::Substitute("index.l1.$0.$1", l1_version.major(), l1_version.minor());
strings::Substitute("index.l1.$0.$1", l1_version.major_number(), l1_version.minor_number());
Status st = FileSystem::Default()->delete_file(l1_file_name);
LOG(WARNING) << "delete error l1 index file: " << l1_file_name << ", status: " << st;
}
if (index_meta.l2_versions_size() > 0) {
for (int i = 0; i < index_meta.l2_versions_size(); i++) {
EditVersion l2_version = index_meta.l2_versions(i);
std::string l2_file_name =
strings::Substitute("index.l2.$0.$1$2", l2_version.major(), l2_version.minor(),
index_meta.l2_version_merged(i) ? MergeSuffix : "");
std::string l2_file_name = strings::Substitute(
"index.l2.$0.$1$2", l2_version.major_number(), l2_version.minor_number(),
index_meta.l2_version_merged(i) ? MergeSuffix : "");
Status st = FileSystem::Default()->delete_file(l2_file_name);
LOG(WARNING) << "delete error l2 index file: " << l2_file_name << ", status: " << st;
}
Expand Down Expand Up @@ -3395,8 +3397,8 @@ Status PersistentIndex::try_replace(size_t n, const Slice* keys, const IndexValu
Status PersistentIndex::flush_advance() {
// flush l0 into _l1_vec
int idx = _l1_vec.size();
std::string l1_tmp_file =
strings::Substitute("$0/index.l1.$1.$2.$3.tmp", _path, _version.major(), _version.minor(), idx);
std::string l1_tmp_file = strings::Substitute("$0/index.l1.$1.$2.$3.tmp", _path, _version.major_number(),
_version.minor_number(), idx);
std::map<size_t, std::unique_ptr<BloomFilter>> bf_map;
if (_need_bloom_filter) {
RETURN_IF_ERROR(_l0->flush_to_immutable_index(l1_tmp_file, _version, true, &bf_map));
Expand Down Expand Up @@ -3491,8 +3493,10 @@ static StatusOr<EditVersion> parse_l2_filename(const std::string& filename) {

Status PersistentIndex::_delete_expired_index_file(const EditVersion& l0_version, const EditVersion& l1_version,
const EditVersion& min_l2_version) {
std::string l0_file_name = strings::Substitute("index.l0.$0.$1", l0_version.major(), l0_version.minor());
std::string l1_file_name = strings::Substitute("index.l1.$0.$1", l1_version.major(), l1_version.minor());
std::string l0_file_name =
strings::Substitute("index.l0.$0.$1", l0_version.major_number(), l0_version.minor_number());
std::string l1_file_name =
strings::Substitute("index.l1.$0.$1", l1_version.major_number(), l1_version.minor_number());
std::string l0_prefix("index.l0");
std::string l1_prefix("index.l1");
std::string l2_prefix("index.l2");
Expand Down Expand Up @@ -3964,7 +3968,7 @@ size_t PersistentIndex::_get_tmp_l1_count() {
Status PersistentIndex::_minor_compaction(PersistentIndexMetaPB* index_meta) {
// 1. flush l0 to l1
const std::string new_l1_filename =
strings::Substitute("$0/index.l1.$1.$2", _path, _version.major(), _version.minor());
strings::Substitute("$0/index.l1.$1.$2", _path, _version.major_number(), _version.minor_number());
const size_t tmp_l1_cnt = _get_tmp_l1_count();
// maybe need to append wal in 1.a
bool need_append_wal = false;
Expand Down Expand Up @@ -4009,9 +4013,9 @@ Status PersistentIndex::_minor_compaction(PersistentIndexMetaPB* index_meta) {
if (_has_l1) {
// just link old l1 file to l2
const std::string l2_file_path =
strings::Substitute("$0/index.l2.$1.$2", _path, _l1_version.major(), _l1_version.minor());
strings::Substitute("$0/index.l2.$1.$2", _path, _l1_version.major_number(), _l1_version.minor_number());
const std::string old_l1_file_path =
strings::Substitute("$0/index.l1.$1.$2", _path, _l1_version.major(), _l1_version.minor());
strings::Substitute("$0/index.l1.$1.$2", _path, _l1_version.major_number(), _l1_version.minor_number());
LOG(INFO) << "PersistentIndex minor compaction, link from " << old_l1_file_path << " to " << l2_file_path;
RETURN_IF_ERROR(FileSystem::Default()->link_file(old_l1_file_path, l2_file_path));
_l1_version.to_pb(index_meta->add_l2_versions());
Expand All @@ -4036,14 +4040,14 @@ Status PersistentIndex::_merge_compaction() {
if (_l0->size() == 0) {
if (!_has_l1 && _l1_vec.size() == 1) {
const std::string idx_file_path =
strings::Substitute("$0/index.l1.$1.$2", _path, _version.major(), _version.minor());
strings::Substitute("$0/index.l1.$1.$2", _path, _version.major_number(), _version.minor_number());
const std::string idx_file_path_tmp = _l1_vec[0]->_file->filename();
return FileSystem::Default()->rename_file(idx_file_path_tmp, idx_file_path);
}
}
auto writer = std::make_unique<ImmutableIndexWriter>();
const std::string idx_file_path =
strings::Substitute("$0/index.l1.$1.$2", _path, _version.major(), _version.minor());
strings::Substitute("$0/index.l1.$1.$2", _path, _version.major_number(), _version.minor_number());
RETURN_IF_ERROR(writer->init(idx_file_path, _version, true));
RETURN_IF_ERROR(
_merge_compaction_internal(writer.get(), 0, _l1_vec.size(), _usage_and_size_by_key_length, false, nullptr));
Expand All @@ -4059,8 +4063,8 @@ Status PersistentIndex::_merge_compaction() {
Status PersistentIndex::_merge_compaction_advance() {
DCHECK(_l1_vec.size() >= config::max_tmp_l1_num);
auto writer = std::make_unique<ImmutableIndexWriter>();
const std::string idx_file_path_tmp =
strings::Substitute("$0/index.l1.$1.$2.$3.tmp", _path, _version.major(), _version.minor(), _l1_vec.size());
const std::string idx_file_path_tmp = strings::Substitute(
"$0/index.l1.$1.$2.$3.tmp", _path, _version.major_number(), _version.minor_number(), _l1_vec.size());
RETURN_IF_ERROR(writer->init(idx_file_path_tmp, _version, false));
int merge_l1_start_idx = _l1_vec.size() - config::max_tmp_l1_num;
int merge_l1_end_idx = _l1_vec.size();
Expand Down Expand Up @@ -4124,8 +4128,8 @@ Status PersistentIndex::_merge_compaction_advance() {
}
}

const std::string idx_file_path = strings::Substitute("$0/index.l1.$1.$2.$3.tmp", _path, _version.major(),
_version.minor(), new_l1_vec.size());
const std::string idx_file_path = strings::Substitute("$0/index.l1.$1.$2.$3.tmp", _path, _version.major_number(),
_version.minor_number(), new_l1_vec.size());
RETURN_IF_ERROR(FileSystem::Default()->rename_file(idx_file_path_tmp, idx_file_path));
std::unique_ptr<RandomAccessFile> l1_rfile;
ASSIGN_OR_RETURN(l1_rfile, _fs->new_random_access_file(idx_file_path));
Expand Down Expand Up @@ -4180,8 +4184,8 @@ StatusOr<EditVersion> PersistentIndex::_major_compaction_impl(
auto writer = std::make_unique<ImmutableIndexWriter>();
// use latest l2 edit version as new l2 edit version
EditVersion new_l2_version = l2_versions.back();
const std::string idx_file_path = strings::Substitute("$0/index.l2.$1.$2$3", _path, new_l2_version.major(),
new_l2_version.minor(), MergeSuffix);
const std::string idx_file_path = strings::Substitute("$0/index.l2.$1.$2$3", _path, new_l2_version.major_number(),
new_l2_version.minor_number(), MergeSuffix);
RETURN_IF_ERROR(writer->init(idx_file_path, new_l2_version, true));
std::map<uint32_t, std::pair<int64_t, int64_t>> usage_and_size_stat;
_get_l2_stat(l2_vec, usage_and_size_stat);
Expand Down Expand Up @@ -4300,8 +4304,8 @@ Status PersistentIndex::TEST_major_compaction(PersistentIndexMetaPB& index_meta)
std::vector<std::unique_ptr<ImmutableIndex>> l2_vec;
for (int i = 0; i < index_meta.l2_versions_size(); i++) {
l2_versions.emplace_back(index_meta.l2_versions(i));
auto l2_block_path = strings::Substitute("$0/index.l2.$1.$2$3", _path, index_meta.l2_versions(i).major(),
index_meta.l2_versions(i).minor(),
auto l2_block_path = strings::Substitute("$0/index.l2.$1.$2$3", _path, index_meta.l2_versions(i).major_number(),
index_meta.l2_versions(i).minor_number(),
index_meta.l2_version_merged(i) ? MergeSuffix : "");
ASSIGN_OR_RETURN(auto l2_rfile, _fs->new_random_access_file(l2_block_path));
ASSIGN_OR_RETURN(auto l2_index, ImmutableIndex::load(std::move(l2_rfile)));
Expand Down Expand Up @@ -4336,9 +4340,9 @@ Status PersistentIndex::major_compaction(Tablet* tablet) {
std::vector<std::unique_ptr<ImmutableIndex>> l2_vec;
for (int i = 0; i < prev_index_meta.l2_versions_size(); i++) {
l2_versions.emplace_back(prev_index_meta.l2_versions(i));
auto l2_block_path = strings::Substitute("$0/index.l2.$1.$2$3", _path, prev_index_meta.l2_versions(i).major(),
prev_index_meta.l2_versions(i).minor(),
prev_index_meta.l2_version_merged(i) ? MergeSuffix : "");
auto l2_block_path = strings::Substitute(
"$0/index.l2.$1.$2$3", _path, prev_index_meta.l2_versions(i).major_number(),
prev_index_meta.l2_versions(i).minor_number(), prev_index_meta.l2_version_merged(i) ? MergeSuffix : "");
ASSIGN_OR_RETURN(auto l2_rfile, _fs->new_random_access_file(l2_block_path));
ASSIGN_OR_RETURN(auto l2_index, ImmutableIndex::load(std::move(l2_rfile)));
l2_vec.emplace_back(std::move(l2_index));
Expand Down
Loading

0 comments on commit 7e0e1b9

Please sign in to comment.