Skip to content

refactor: flat param string cleanup #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
JalinWang merged 3 commits into from
Jan 26, 2026
Merged

refactor: flat param string cleanup #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
60ba9ab
refactor: flat param string cleanup
JalinWang Jan 20, 2026
42b01a4
fix
JalinWang Jan 20, 2026
2ff5cf3
Merge branch 'main' into refactor/flat_param_string
JalinWang Jan 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions src/core/algorithm/flat/flat_builder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,

// Set the major order
bool column_major_order = false;
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order)) {
if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order)) {
meta_.set_major_order(column_major_order ? IndexMeta::MO_COLUMN
: IndexMeta::MO_ROW);
}
Expand Down Expand Up @@ -71,7 +71,7 @@ int FlatBuilder<BATCH_SIZE>::init(const IndexMeta &meta,

std::string tag = std::to_string(BATCH_SIZE);
ailego::Params searcher_params;
searcher_params.set(FLAT_SEARCHER_BATCH_SIZE, BATCH_SIZE);
searcher_params.set(PARAM_FLAT_BATCH_SIZE, BATCH_SIZE);
meta_.set_searcher("FlatSearcher" + tag, 0, searcher_params);
meta_.set_builder("FlatBuilder" + tag, 0, params);
return 0;
Expand Down Expand Up @@ -151,7 +151,7 @@ int FlatBuilder<BATCH_SIZE>::write_keys(const std::vector<uint64_t> &keys,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size, keys_padding_size,
0);
}

Expand Down Expand Up @@ -179,7 +179,7 @@ int FlatBuilder<BATCH_SIZE>::write_mapping(const std::vector<uint64_t> &keys,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size,
mapping_padding_size, 0);
}

Expand Down Expand Up @@ -236,7 +236,7 @@ int FlatBuilder<BATCH_SIZE>::write_column_index(IndexDumper *dumper,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}

Expand Down Expand Up @@ -271,7 +271,7 @@ int FlatBuilder<BATCH_SIZE>::write_row_index(IndexDumper *dumper,
return IndexError_WriteData;
}
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}

Expand Down
12 changes: 6 additions & 6 deletions src/core/algorithm/flat/flat_searcher.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,10 @@ IndexProvider::Pointer FlatSearcher<BATCH_SIZE>::create_provider(void) const {
std::lock_guard<std::mutex> lock(mapping_mutex_);

if (mapping_.empty()) {
auto mapping_segment = container_->get(PARAM_FLAT_SEGMENT_MAPPING);
auto mapping_segment = container_->get(FLAT_SEGMENT_MAPPING_SEG_ID);
if (!mapping_segment) {
LOG_ERROR("Failed to fetch segment %s",
PARAM_FLAT_SEGMENT_MAPPING.c_str());
FLAT_SEGMENT_MAPPING_SEG_ID.c_str());
return nullptr;
}

Expand Down Expand Up @@ -115,14 +115,14 @@ int FlatSearcher<BATCH_SIZE>::load(IndexStorage::Pointer cntr,
}
}

auto keys_segment = cntr->get(PARAM_FLAT_SEGMENT_KEYS);
auto keys_segment = cntr->get(FLAT_SEGMENT_KEYS_SEG_ID);
if (!keys_segment) {
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str());
return IndexError_NoExist;
}
features_segment_ = cntr->get(PARAM_FLAT_SEGMENT_FEATURES);
features_segment_ = cntr->get(FLAT_SEGMENT_FEATURES_SEG_ID);
if (!features_segment_) {
LOG_ERROR("Failed to fetch segment %s", PARAM_FLAT_SEGMENT_KEYS.c_str());
LOG_ERROR("Failed to fetch segment %s", FLAT_SEGMENT_KEYS_SEG_ID.c_str());
return IndexError_NoExist;
}

Expand Down
6 changes: 3 additions & 3 deletions src/core/algorithm/flat/flat_searcher.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@ class FlatSearcher : public IndexSearcher {
//! Initialize Searcher
int init(const ailego::Params &index_params) override {
params_ = index_params;
read_block_size_ = FLAT_READ_BLOCK_SIZE;
index_params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE;
index_params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_);
return 0;
}

Expand Down Expand Up @@ -165,7 +165,7 @@ class FlatSearcher : public IndexSearcher {
const uint64_t *keys_{nullptr};
std::map<key_t, node_id_t> key_id_mapping_;
uint32_t magic_{IndexContext::GenerateMagic()};
uint32_t read_block_size_{FLAT_READ_BLOCK_SIZE};
uint32_t read_block_size_{FLAT_DEFAULT_READ_BLOCK_SIZE};
bool column_major_order_{false};
IndexMeta meta_{};
IndexStorage::Pointer container_{};
Expand Down
8 changes: 4 additions & 4 deletions src/core/algorithm/flat/flat_streamer.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
}

// 参数设置
if (params.get(FLAT_BUILDER_COLUMN_MAJOR_ORDER, &column_major_order_)) {
if (params.get(PARAM_FLAT_COLUMN_MAJOR_ORDER, &column_major_order_)) {
meta_.set_major_order(column_major_order_ ? IndexMeta::MO_COLUMN
: IndexMeta::MO_ROW);
}
Expand Down Expand Up @@ -99,9 +99,9 @@ int FlatStreamer<BATCH_SIZE>::init(const IndexMeta &imeta,
return IndexError_InvalidArgument;
}

read_block_size_ = FLAT_READ_BLOCK_SIZE;
params.get(FLAT_SEARCHER_READ_BLOCK_SIZE, &read_block_size_);
params.get(FLAT_SEARCHER_USE_ID_MAP, &use_key_info_map_);
read_block_size_ = FLAT_DEFAULT_READ_BLOCK_SIZE;
params.get(PARAM_FLAT_READ_BLOCK_SIZE, &read_block_size_);
params.get(PARAM_FLAT_USE_ID_MAP, &use_key_info_map_);

// entity init
uint32_t block_vector_count = kDefaultBlockVecCount;
Expand Down
10 changes: 5 additions & 5 deletions src/core/algorithm/flat/flat_streamer_dumper.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}

Expand Down Expand Up @@ -177,7 +177,7 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_FEATURES, features_size,
return dumper->append(FLAT_SEGMENT_FEATURES_SEG_ID, features_size,
features_padding_size, 0);
}

Expand All @@ -200,8 +200,8 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_KEYS, keys_size, keys_padding_size,
0);
return dumper->append(FLAT_SEGMENT_KEYS_SEG_ID, keys_size,
keys_padding_size, 0);
}

int write_mapping(const std::vector<uint64_t> &keys, IndexDumper *dumper) {
Expand Down Expand Up @@ -230,7 +230,7 @@ class FlatStreamerDumper {
}
dump_size_ += padding.size();
}
return dumper->append(PARAM_FLAT_SEGMENT_MAPPING, mapping_size,
return dumper->append(FLAT_SEGMENT_MAPPING_SEG_ID, mapping_size,
mapping_padding_size, 0);
}

Expand Down
4 changes: 2 additions & 2 deletions src/core/algorithm/flat/flat_streamer_entity.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ int FlatStreamerEntity::load_storage(IndexStorage::Pointer storage) {
size_t index_size = hd_segment->capacity();
for (size_t i = 1; i <= meta_.segment_count; ++i) {
std::string segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i);
auto seg = storage->get(segment_id);
if (!seg || seg->data_size() < meta_.header.block_size) {
LOG_ERROR("Failed to get segment %s, or invalid segment size",
Expand Down Expand Up @@ -756,7 +756,7 @@ int FlatStreamerEntity::alloc_segment(void) {
}

std::string segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, index);
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, index);
size_t size =
ailego_align(meta_.segment_size, ailego::MemoryHelper::PageSize());
auto segment = storage_->get(segment_id);
Expand Down
2 changes: 1 addition & 1 deletion src/core/algorithm/flat/flat_streamer_entity.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -219,7 +219,7 @@ class FlatStreamerEntity {
const IndexStorage::Segment::Pointer get_segment(size_t index) const {
for (size_t i = segments_.size(); i <= index; ++i) {
auto segment_id =
ailego::StringHelper::Concat(PARAM_FLAT_SEGMENT_FEATURES, i);
ailego::StringHelper::Concat(FLAT_SEGMENT_FEATURES_SEG_ID, i);
auto segment = storage_->get(segment_id);
if (!segment) {
LOG_ERROR("Failed to get segment %s", segment_id.c_str());
Expand Down
37 changes: 13 additions & 24 deletions src/core/algorithm/flat/flat_utility.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,21 @@ namespace zvec {
namespace core {

//! The default size of reading a block
static constexpr uint32_t FLAT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
static const std::string FLAT_LINEAR_META_SEG_ID = "linear.linear_meta";
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID =
"linear.linear_list_head";

// reducer params
static const std::string PARAM_FLAT_REDUCER_WORKING_PATH(
"proxima.linear.reducer.working_path");
static const std::string PARAM_FLAT_REDUCER_NUM_OF_ADD_THREADS(
"proxima.linear.reducer.num_of_add_threads");
static const std::string PARAM_FLAT_REDUCER_INDEX_NAME(
"proxima.linear.reducer.index_name");
static constexpr uint32_t FLAT_DEFAULT_READ_BLOCK_SIZE = 4 * 1024 * 1024;
static const std::string FLAT_LINEAR_META_SEG_ID = "flat.linear_meta";
static const std::string FLAT_LINEAR_LIST_HEAD_SEG_ID = "flat.linear_list_head";

static const std::string FLAT_SEGMENT_KEYS_SEG_ID("flat.keys");
static const std::string FLAT_SEGMENT_FEATURES_SEG_ID("flat.features");
static const std::string FLAT_SEGMENT_MAPPING_SEG_ID("flat.mapping");

// index params
static const std::string PARAM_FLAT_SEGMENT_KEYS("linear.keys");
static const std::string PARAM_FLAT_SEGMENT_FEATURES("linear.features");
static const std::string PARAM_FLAT_SEGMENT_MAPPING("linear.mapping");

static const std::string FLAT_BUILDER_COLUMN_MAJOR_ORDER(
"proxima.linear.builder.column_major_order");
static const std::string FLAT_SEARCHER_BATCH_SIZE(
"proxima.linear.searcher.batch_size");
static const std::string FLAT_SEARCHER_READ_BLOCK_SIZE(
"proxima.linear.searcher.read_block_size");
static const std::string FLAT_SEARCHER_USE_ID_MAP(
"proxima.linear.searcher.use_id_map");
static const std::string PARAM_FLAT_COLUMN_MAJOR_ORDER(
"proxima.flat.column_major_order");
static const std::string PARAM_FLAT_BATCH_SIZE("proxima.flat.batch_size");
static const std::string PARAM_FLAT_READ_BLOCK_SIZE(
"proxima.flat.read_block_size");
static const std::string PARAM_FLAT_USE_ID_MAP("proxima.flat.use_id_map");

//! Determines if a number is equal to two to the power of n.
template <size_t K>
Expand Down
4 changes: 2 additions & 2 deletions src/core/interface/indexes/flat_index.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,9 +25,9 @@ namespace zvec::core_interface {
int FlatIndex::CreateAndInitStreamer(const BaseIndexParam &param) {
param_ = dynamic_cast<const FlatIndexParam &>(param);

proxima_index_params_.set(core::FLAT_BUILDER_COLUMN_MAJOR_ORDER,
proxima_index_params_.set(core::PARAM_FLAT_COLUMN_MAJOR_ORDER,
param_.major_order == IndexMeta::MO_COLUMN);
proxima_index_params_.set(core::FLAT_SEARCHER_USE_ID_MAP, param_.use_id_map);
proxima_index_params_.set(core::PARAM_FLAT_USE_ID_MAP, param_.use_id_map);
if (is_sparse_) {
streamer_ = core::IndexFactory::CreateStreamer("FlatSparseStreamer");
} else {
Expand Down
15 changes: 0 additions & 15 deletions src/core/utility/utility_params.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,21 +21,6 @@ namespace core {
static const std::string INDEX_META_SEGMENT_NAME = "IndexMeta";
static const std::string INDEX_VERSION_SEGMENT_NAME = "IndexVersion";

//! BruteForceBuilder/BruteForceSearcher
static const std::string BRUTE_FORCE_SEGMENT_KEYS = "linear.keys";
static const std::string BRUTE_FORCE_SEGMENT_FEATURES = "linear.features";
static const std::string BRUTE_FORCE_SEGMENT_MAPPING = "linear.mapping";

//! BruteForceBuilder
static const std::string BRUTE_FORCE_BUILDER_COLUMN_MAJOR_ORDER =
"proxima.linear.builder.column_major_order";

//! BruteForceSearcher
static const std::string BRUTE_FORCE_SEARCHER_BATCH_SIZE =
"proxima.linear.searcher.batch_size";
static const std::string BRUTE_FORCE_SEARCHER_READ_BLOCK_SIZE =
"proxima.linear.searcher.read_block_size";

//! FileLogger
static const std::string FILE_LOGGER_PATH = "proxima.file.logger.path";

Expand Down
8 changes: 4 additions & 4 deletions tests/core/algorithm/flat/flat_searcher_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -883,7 +883,7 @@ TEST(FlatSearcher, ColumnBatch_Binary32) {
meta1.set_major_order(IndexMeta::MO_ROW);

Params params1;
params1.set("proxima.linear.builder.column_major_order", true);
params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, true);
BuildIndex(meta1, params1, holder, INDEX_PATH + ".1");

IndexMeta meta2;
Expand Down Expand Up @@ -984,7 +984,7 @@ TEST(FlatSearcher, ColumnBatch_FP32) {
// column_major_order in params

Params params1;
params1.set("proxima.linear.builder.column_major_order",
params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER,
true); // make it MO_COLUMN
BuildIndex(meta1, params1, holder, INDEX_PATH + ".1");

Expand Down Expand Up @@ -1090,7 +1090,7 @@ TEST(FlatSearcher, ColumnBatch_FP16) {
// column_major_order in params

Params params1;
params1.set("proxima.linear.builder.column_major_order",
params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER,
true); // make it MO_COLUMN
BuildIndex(meta1, params1, holder, INDEX_PATH + ".1");

Expand Down Expand Up @@ -1196,7 +1196,7 @@ TEST(FlatSearcher, ColumnBatch_INT8) {
// column_major_order in params

Params params1;
params1.set("proxima.linear.builder.column_major_order",
params1.set(PARAM_FLAT_COLUMN_MAJOR_ORDER,
true); // make it MO_COLUMN
BuildIndex(meta1, params1, holder, INDEX_PATH + ".1");

Expand Down
6 changes: 4 additions & 2 deletions tests/core/algorithm/flat/flat_streamer_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@
#include <framework/index_framework.h>
#include <framework/index_streamer.h>
#include <gtest/gtest.h>
#include "ailego/encoding/json/mod_json.h"
#include "algorithm/flat/flat_utility.h"

#if defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
Expand Down Expand Up @@ -353,7 +355,7 @@ TEST_F(FlatStreamerTest, TestOpenClose) {
IndexMeta meta(IndexMeta::DataType::DT_FP32, dim);
meta.set_metric("SquaredEuclidean", 0, Params());
Params params;
// params.set("proxima.linear.builder.column_major_order", false);
// params.set(PARAM_FLAT_COLUMN_MAJOR_ORDER, false);
auto storage1 = IndexFactory::CreateStorage("MMapFileStorage");
auto storage2 = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_NE(nullptr, storage1);
Expand Down Expand Up @@ -1151,7 +1153,7 @@ TEST_F(FlatStreamerTest, TestAddAndSearchWithID2) {
IndexStreamer::Pointer read_streamer =
IndexFactory::CreateStreamer("FlatStreamer");
Params read_params;
read_params.set("proxima.linear.searcher.use_id_map", false);
read_params.set(PARAM_FLAT_USE_ID_MAP, false);
Params read_stg_params;
auto read_storage = IndexFactory::CreateStorage("MMapFileStorage");
ASSERT_EQ(0, read_storage->init(read_stg_params));
Expand Down