refactor: clarify HNSW 'm' as max neighbors of upper layer

src/core/algorithm/hnsw/hnsw_builder.cc

@@ -37,41 +37,43 @@ int HnswBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
   size_t memory_quota = 0UL;
   params.get(PARAM_HNSW_BUILDER_MEMORY_QUOTA, &memory_quota);
   params.get(PARAM_HNSW_BUILDER_THREAD_COUNT, &thread_cnt_);
-  params.get(PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT, &neighbor_cnt_);
   params.get(PARAM_HNSW_BUILDER_MIN_NEIGHBOR_COUNT, &min_neighbor_cnt_);
-  float ratio = HnswEntity::kDefaultNeighborPruneRatio;
-  params.get(PARAM_HNSW_BUILDER_UPPER_NEIGHBOR_RATIO, &ratio);
-  upper_neighbor_cnt_ = ratio * neighbor_cnt_;
   params.get(PARAM_HNSW_BUILDER_EFCONSTRUCTION, &ef_construction_);
   params.get(PARAM_HNSW_BUILDER_SCALING_FACTOR, &scaling_factor_);
   params.get(PARAM_HNSW_BUILDER_CHECK_INTERVAL_SECS, &check_interval_secs_);
-  ratio = HnswEntity::kDefaultNeighborPruneRatio;
-  params.get(PARAM_HNSW_BUILDER_NEIGHBOR_PRUNE_RATIO, &ratio);
 
-  size_t prune_cnt = neighbor_cnt_ * ratio;
+  params.get(PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT, &upper_max_neighbor_cnt_);
+  float multiplier = HnswEntity::kDefaultL0MaxNeighborCntMultiplier;
+  params.get(PARAM_HNSW_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER, &multiplier);
+  l0_max_neighbor_cnt_ = multiplier * upper_max_neighbor_cnt_;
+
+  multiplier = HnswEntity::kDefaultNeighborPruneMultiplier;
+  params.get(PARAM_HNSW_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER, &multiplier);
+  size_t prune_cnt = multiplier * upper_max_neighbor_cnt_;
 
   if (ef_construction_ == 0) {
     ef_construction_ = HnswEntity::kDefaultEfConstruction;
   }
-  if (neighbor_cnt_ == 0) {
-    neighbor_cnt_ = HnswEntity::kDefaultNeighborCnt;
+  if (upper_max_neighbor_cnt_ == 0) {
+    upper_max_neighbor_cnt_ = HnswEntity::kDefaultUpperMaxNeighborCnt;
   }
-  if (neighbor_cnt_ > kMaxNeighborCnt) {
+  if (upper_max_neighbor_cnt_ > kMaxNeighborCnt) {
     LOG_ERROR("[%s] must be in range (0,%d]",
               PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT.c_str(), kMaxNeighborCnt);
     return IndexError_InvalidArgument;
   }
-  if (min_neighbor_cnt_ > neighbor_cnt_) {
+  if (min_neighbor_cnt_ > upper_max_neighbor_cnt_) {
     LOG_ERROR("[%s]-[%d] must be <= [%s]-[%d]",
               PARAM_HNSW_BUILDER_MIN_NEIGHBOR_COUNT.c_str(), min_neighbor_cnt_,
-              PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT.c_str(), neighbor_cnt_);
+              PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT.c_str(),
+              upper_max_neighbor_cnt_);
     return IndexError_InvalidArgument;
   }
-  if (upper_neighbor_cnt_ == 0) {
-    neighbor_cnt_ = HnswEntity::kDefaultUpperNeighborCnt;
+  if (l0_max_neighbor_cnt_ == 0) {
+    l0_max_neighbor_cnt_ = HnswEntity::kDefaultUpperMaxNeighborCnt;
   }
-  if (upper_neighbor_cnt_ > HnswEntity::kMaxNeighborCnt) {
-    LOG_ERROR("UpperNeighborCnt must be in range (0,%d)",
+  if (l0_max_neighbor_cnt_ > HnswEntity::kMaxNeighborCnt) {
+    LOG_ERROR("L0MaxNeighborCnt must be in range (0,%d)",
               HnswEntity::kMaxNeighborCnt);
     return IndexError_InvalidArgument;
   }
@@ -92,7 +94,7 @@ int HnswBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
              std::thread::hardware_concurrency());
   }
   if (prune_cnt == 0UL) {
-    prune_cnt = upper_neighbor_cnt_;
+    prune_cnt = upper_max_neighbor_cnt_;
   }
 
   metric_ = IndexFactory::CreateMetric(meta_.metric_name());
@@ -109,9 +111,9 @@ int HnswBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
   entity_.set_vector_size(meta_.element_size());
 
   entity_.set_ef_construction(ef_construction_);
-  entity_.set_neighbor_cnt(neighbor_cnt_);
+  entity_.set_l0_neighbor_cnt(l0_max_neighbor_cnt_);
   entity_.set_min_neighbor_cnt(min_neighbor_cnt_);
-  entity_.set_upper_neighbor_cnt(upper_neighbor_cnt_);
+  entity_.set_upper_neighbor_cnt(upper_max_neighbor_cnt_);
   entity_.set_scaling_factor(scaling_factor_);
   entity_.set_memory_quota(memory_quota);
   entity_.set_prune_cnt(prune_cnt);
@@ -131,10 +133,10 @@ int HnswBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
   state_ = BUILD_STATE_INITED;
   LOG_INFO(
       "End HnswBuilder::init, params: vectorSize=%u efConstruction=%u "
-      "neighborCnt=%u upperNeighborCnt=%u scalingFactor=%u "
+      "l0NeighborCnt=%u upperNeighborCnt=%u scalingFactor=%u "
       "memoryQuota=%zu neighborPruneCnt=%zu metricName=%s ",
-      meta_.element_size(), ef_construction_, neighbor_cnt_,
-      upper_neighbor_cnt_, scaling_factor_, memory_quota, prune_cnt,
+      meta_.element_size(), ef_construction_, l0_max_neighbor_cnt_,
+      upper_max_neighbor_cnt_, scaling_factor_, memory_quota, prune_cnt,
       meta_.metric_name().c_str());
 
   return 0;
@@ -143,9 +145,9 @@ int HnswBuilder::init(const IndexMeta &meta, const ailego::Params &params) {
 int HnswBuilder::cleanup(void) {
   LOG_INFO("Begin HnswBuilder::cleanup");
 
-  neighbor_cnt_ = HnswEntity::kDefaultNeighborCnt;
+  l0_max_neighbor_cnt_ = HnswEntity::kDefaultL0MaxNeighborCnt;
   min_neighbor_cnt_ = 0;
-  upper_neighbor_cnt_ = HnswEntity::kDefaultUpperNeighborCnt;
+  upper_max_neighbor_cnt_ = HnswEntity::kDefaultUpperMaxNeighborCnt;
   ef_construction_ = HnswEntity::kDefaultEfConstruction;
   scaling_factor_ = HnswEntity::kDefaultScalingFactor;
   check_interval_secs_ = kDefaultLogIntervalSecs;

src/core/algorithm/hnsw/hnsw_builder.h

@@ -71,9 +71,9 @@ class HnswBuilder : public IndexBuilder {
   HnswBuilderEntity entity_{};
   HnswAlgorithm::UPointer alg_;  // impl graph algorithm
   uint32_t thread_cnt_{0};
-  uint32_t neighbor_cnt_{HnswEntity::kDefaultNeighborCnt};
   uint32_t min_neighbor_cnt_{0};
-  uint32_t upper_neighbor_cnt_{HnswEntity::kDefaultUpperNeighborCnt};
+  uint32_t upper_max_neighbor_cnt_{HnswEntity::kDefaultUpperMaxNeighborCnt};
+  uint32_t l0_max_neighbor_cnt_{HnswEntity::kDefaultL0MaxNeighborCnt};
   uint32_t ef_construction_{HnswEntity::kDefaultEfConstruction};
   uint32_t scaling_factor_{HnswEntity::kDefaultScalingFactor};
   uint32_t check_interval_secs_{kDefaultLogIntervalSecs};

src/core/algorithm/hnsw/hnsw_builder_entity.h

@@ -97,7 +97,7 @@ class HnswBuilderEntity : public HnswEntity {
 
   //! Get neighbors size
   inline size_t neighbors_size() const {
-    return sizeof(NeighborsHeader) + neighbor_cnt() * sizeof(node_id_t);
+    return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
   }
 
   //! Get upper neighbors size

src/core/algorithm/hnsw/hnsw_context.cc

@@ -47,7 +47,7 @@ int HnswContext::init(ContextType type) {
         return ret;
       }
       candidates_.limit(max_scan_num_);
-      update_heap_.limit(entity_->neighbor_cnt() + 1);
+      update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
       break;
 
     case kSearcherContext:
@@ -74,7 +74,7 @@ int HnswContext::init(ContextType type) {
         return ret;
       }
 
-      update_heap_.limit(entity_->neighbor_cnt() + 1);
+      update_heap_.limit(entity_->l0_neighbor_cnt() + 1);
       candidates_.limit(max_scan_num_);
 
       check_need_adjuct_ctx();
@@ -240,7 +240,7 @@ int HnswContext::update_context(ContextType type, const IndexMeta &meta,
         return IndexError_Runtime;
       }
 
-      update_heap_.limit(entity->neighbor_cnt() + 1);
+      update_heap_.limit(entity->l0_neighbor_cnt() + 1);
       candidates_.limit(max_scan_num_);
       topk_heap_.limit(std::max(topk_, ef_));
       break;

src/core/algorithm/hnsw/hnsw_entity.cc

@@ -224,7 +224,7 @@ int64_t HnswEntity::dump_graph_neighbors(
   graph_meta.reserve(doc_cnt());
   size_t offset = 0;
   uint32_t crc = 0;
-  node_id_t mapping[neighbor_cnt()];
+  node_id_t mapping[l0_neighbor_cnt()];
 
   uint32_t min_neighbor_count = 10000;
   uint32_t max_neighbor_count = 0;
@@ -234,7 +234,8 @@ int64_t HnswEntity::dump_graph_neighbors(
     const Neighbors neighbors =
         get_neighbors(0, reorder_mapping.empty() ? id : reorder_mapping[id]);
     ailego_assert_with(!!neighbors.data, "invalid neighbors");
-    ailego_assert_with(neighbors.size() <= neighbor_cnt(), "invalid neighbors");
+    ailego_assert_with(neighbors.size() <= l0_neighbor_cnt(),
+                       "invalid neighbors");
 
     uint32_t neighbor_count = neighbors.size();
     if (neighbor_count < min_neighbor_count) {

src/core/algorithm/hnsw/hnsw_entity.h

@@ -42,7 +42,7 @@ struct GraphHeader {
   uint32_t doc_count;
   uint32_t vector_size;
   uint32_t node_size;
-  uint32_t max_neighbor_count;
+  uint32_t l0_neighbor_count;
   uint32_t prune_type;
   uint32_t prune_neighbor_count;
   uint32_t ef_construction;
@@ -58,7 +58,7 @@ static_assert(sizeof(GraphHeader) % 32 == 0,
 struct HnswHeader {
   uint32_t size;      // header size
   uint32_t revision;  // current total docs of the graph
-  uint32_t thumb_neighbor_count;
+  uint32_t upper_neighbor_count;
   uint32_t ef_construction;
   uint32_t scaling_factor;
   uint32_t max_level;
@@ -100,12 +100,12 @@ struct HNSWHeader {
     hnsw.size = sizeof(HnswHeader);
   }
 
-  size_t neighbor_cnt() const {
-    return graph.max_neighbor_count;
+  size_t l0_neighbor_cnt() const {
+    return graph.l0_neighbor_count;
   }
 
   size_t upper_neighbor_cnt() const {
-    return hnsw.thumb_neighbor_count;
+    return hnsw.upper_neighbor_count;
   }
 
   size_t vector_size() const {
@@ -201,13 +201,13 @@ class HnswEntity {
 
   //! Get max neighbor size of graph level
   inline size_t neighbor_cnt(level_t level) const {
-    return level == 0 ? header_.graph.max_neighbor_count
-                      : header_.hnsw.thumb_neighbor_count;
+    return level == 0 ? header_.graph.l0_neighbor_count
+                      : header_.hnsw.upper_neighbor_count;
   }
 
   //! get max neighbor size of graph level 0
-  inline size_t neighbor_cnt() const {
-    return header_.graph.max_neighbor_count;
+  inline size_t l0_neighbor_cnt() const {
+    return header_.graph.l0_neighbor_count;
   }
 
   //! get min neighbor size of graph
@@ -217,7 +217,7 @@ class HnswEntity {
 
   //! get upper neighbor size of graph level other than 0
   inline size_t upper_neighbor_cnt() const {
-    return header_.hnsw.thumb_neighbor_count;
+    return header_.hnsw.upper_neighbor_count;
   }
 
   //! Get current total doc of the hnsw graph
@@ -276,16 +276,16 @@ class HnswEntity {
     header_.hnsw.scaling_factor = val;
   }
 
-  void set_neighbor_cnt(size_t cnt) {
-    header_.graph.max_neighbor_count = cnt;
+  void set_l0_neighbor_cnt(size_t cnt) {
+    header_.graph.l0_neighbor_count = cnt;
   }
 
   void set_min_neighbor_cnt(size_t cnt) {
     header_.graph.min_neighbor_count = cnt;
   }
 
   void set_upper_neighbor_cnt(size_t cnt) {
-    header_.hnsw.thumb_neighbor_count = cnt;
+    header_.hnsw.upper_neighbor_count = cnt;
   }
 
   void set_ef_construction(size_t ef) {
@@ -499,8 +499,8 @@ class HnswEntity {
   constexpr static size_t kMaxGraphLayers = 15;
   constexpr static uint32_t kDefaultEfConstruction = 500;
   constexpr static uint32_t kDefaultEf = 500;
-  constexpr static uint32_t kDefaultNeighborCnt = 100;
-  constexpr static uint32_t kDefaultUpperNeighborCnt = 50;
+  constexpr static uint32_t kDefaultUpperMaxNeighborCnt = 50;  // M of HNSW
+  constexpr static uint32_t kDefaultL0MaxNeighborCnt = 100;
   constexpr static uint32_t kMaxNeighborCnt = 65535;
   constexpr static float kDefaultScanRatio = 0.1f;
   constexpr static uint32_t kDefaultMinScanLimit = 10000;
@@ -514,10 +514,10 @@ class HnswEntity {
   constexpr static size_t kMaxChunkSize = 0xFFFFFFFF;
   constexpr static size_t kDefaultChunkSize = 2UL * 1024UL * 1024UL;
   constexpr static size_t kDefaultMaxChunkCnt = 50000UL;
-  constexpr static float kDefaultNeighborPruneRatio =
-      0.5f;  // prune count / neighbor count
-  constexpr static float kDefaultUpperNeighborRatio =
-      0.5f;  // upper neighbor count / neighbor count
+  constexpr static float kDefaultNeighborPruneMultiplier =
+      1.0f;  // prune_cnt = upper_max_neighbor_cnt * multiplier
+  constexpr static float kDefaultL0MaxNeighborCntMultiplier =
+      2.0f;  // l0_max_neighbor_cnt = upper_max_neighbor_cnt * multiplier
 
  protected:
   HNSWHeader header_{};

src/core/algorithm/hnsw/hnsw_params.h

@@ -28,20 +28,14 @@ static const std::string PARAM_HNSW_BUILDER_SCALING_FACTOR(
     "proxima.hnsw.builder.scaling_factor");
 static const std::string PARAM_HNSW_BUILDER_CHECK_INTERVAL_SECS(
     "proxima.hnsw.builder.check_interval_secs");
-static const std::string PARAM_HNSW_BUILDER_NEIGHBOR_PRUNE_RATIO(
-    "proxima.hnsw.builder.neighbor_prune_ratio");
-static const std::string PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT(
-    "proxima.hnsw.builder.max_neighbor_count");
+static const std::string PARAM_HNSW_BUILDER_NEIGHBOR_PRUNE_MULTIPLIER(
+    "proxima.hnsw.builder.neighbor_prune_multiplier");
 static const std::string PARAM_HNSW_BUILDER_MIN_NEIGHBOR_COUNT(
     "proxima.hnsw.builder.min_neighbor_count");
-static const std::string PARAM_HNSW_BUILDER_UPPER_NEIGHBOR_RATIO(
-    "proxima.hnsw.builder.upper_neighbor_ratio");
-static const std::string PARAM_HNSW_BUILDER_HYBRID_VECTOR_ENABLE(
-    "proxima.hnsw.builder.hybrid_vector_enable");
-static const std::string PARAM_HNSW_BUILDER_HYBRID_VECTOR_SEPARATE_NEIGHBOR(
-    "proxima.hnsw.builder.hybrid_vector_separate_neighbor");
-static const std::string PARAM_HNSW_BUILDER_SPARSE_NEIGHBOR_RATIO(
-    "proxima.hnsw.builder.sparse_neighbor_ratio");
+static const std::string PARAM_HNSW_BUILDER_MAX_NEIGHBOR_COUNT(
+    "proxima.hnsw.builder.max_neighbor_count");
+static const std::string PARAM_HNSW_BUILDER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
+    "proxima.hnsw.builder.l0_max_neighbor_count_multiplier");
 
 static const std::string PARAM_HNSW_SEARCHER_EF("proxima.hnsw.searcher.ef");
 static const std::string PARAM_HNSW_SEARCHER_BRUTE_FORCE_THRESHOLD(
@@ -70,8 +64,8 @@ static const std::string PARAM_HNSW_STREAMER_EFCONSTRUCTION(
     "proxima.hnsw.streamer.efconstruction");
 static const std::string PARAM_HNSW_STREAMER_MAX_NEIGHBOR_COUNT(
     "proxima.hnsw.streamer.max_neighbor_count");
-static const std::string PARAM_HNSW_STREAMER_UPPER_NEIGHBOR_RATIO(
-    "proxima.hnsw.streamer.upper_neighbor_ratio");
+static const std::string PARAM_HNSW_STREAMER_L0_MAX_NEIGHBOR_COUNT_MULTIPLIER(
+    "proxima.hnsw.streamer.l0_max_neighbor_count_multiplier");
 static const std::string PARAM_HNSW_STREAMER_SCALING_FACTOR(
     "proxima.hnsw.streamer.scaling_factor");
 static const std::string PARAM_HNSW_STREAMER_BRUTE_FORCE_THRESHOLD(
@@ -88,8 +82,8 @@ static const std::string PARAM_HNSW_STREAMER_VISIT_BLOOMFILTER_NEGATIVE_PROB(
     "proxima.hnsw.streamer.visit_bloomfilter_negative_prob");
 static const std::string PARAM_HNSW_STREAMER_CHECK_CRC_ENABLE(
     "proxima.hnsw.streamer.check_crc_enable");
-static const std::string PARAM_HNSW_STREAMER_NEIGHBOR_PRUNE_RATIO(
-    "proxima.hnsw.streamer.neighbor_prune_ratio");
+static const std::string PARAM_HNSW_STREAMER_NEIGHBOR_PRUNE_MULTIPLIER(
+    "proxima.hnsw.streamer.neighbor_prune_multiplier");
 static const std::string PARAM_HNSW_STREAMER_CHUNK_SIZE(
     "proxima.hnsw.streamer.chunk_size");
 static const std::string PARAM_HNSW_STREAMER_FILTER_SAME_KEY(
@@ -100,12 +94,6 @@ static const std::string PARAM_HNSW_STREAMER_MIN_NEIGHBOR_COUNT(
     "proxima.hnsw.streamer.min_neighbor_count");
 static const std::string PARAM_HNSW_STREAMER_FORCE_PADDING_RESULT_ENABLE(
     "proxima.hnsw.streamer.force_padding_result_enable");
-static const std::string PARAM_HNSW_STREAMER_HYBRID_VECTOR_ENABLE(
-    "proxima.hnsw.streamer.hybrid_vector_enable");
-static const std::string PARAM_HNSW_STREAMER_HYBRID_VECTOR_SEPARATE_NEIGHBOR(
-    "proxima.hnsw.streamer.hybrid_vector_separate_neighbor");
-static const std::string PARAM_HNSW_STREAMER_SPARSE_NEIGHBOR_RATIO(
-    "proxima.hnsw.streamer.sparse_neighbor_ratio");
 static const std::string PARAM_HNSW_STREAMER_ESTIMATE_DOC_COUNT(
     "proxima.hnsw.streamer.estimate_doc_count");
 static const std::string PARAM_HNSW_STREAMER_USE_ID_MAP(

src/core/algorithm/hnsw/hnsw_searcher_entity.cc

@@ -223,11 +223,11 @@ int HnswSearcherEntity::load(const IndexStorage::Pointer &container,
 
   LOG_INFO(
       "Index info: docCnt=%u entryPoint=%u maxLevel=%d efConstruct=%zu "
-      "neighborCnt=%zu upperNeighborCnt=%zu scalingFactor=%zu "
+      "l0NeighborCnt=%zu upperNeighborCnt=%zu scalingFactor=%zu "
       "vectorSize=%zu nodeSize=%zu vectorSegmentSize=%zu keySegmentSize=%zu "
       "neighborsSegmentSize=%zu neighborsMetaSegmentSize=%zu ",
       doc_cnt(), entry_point(), cur_max_level(), ef_construction(),
-      neighbor_cnt(), upper_neighbor_cnt(), scaling_factor(), vector_size(),
+      l0_neighbor_cnt(), upper_neighbor_cnt(), scaling_factor(), vector_size(),
       node_size(), vectors_->data_size(), keys_->data_size(),
       neighbors_ == nullptr ? 0 : neighbors_->data_size(),
       neighbors_meta_ == nullptr ? 0 : neighbors_meta_->data_size());
@@ -263,7 +263,7 @@ int HnswSearcherEntity::load_segments(bool check_crc) {
   }
   memcpy(&hd.hnsw, data, sizeof(hd.hnsw));
   *mutable_header() = hd;
-  segment_datas_.resize(std::max(neighbor_cnt(), upper_neighbor_cnt()));
+  segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()));
 
   vectors_ = storage_->get(kGraphFeaturesSegmentId);
   if (!vectors_) {
@@ -403,7 +403,7 @@ int HnswSearcherEntity::get_fixed_neighbors(
     return IndexError_InvalidArgument;
   }
 
-  size_t fixed_neighbor_cnt = neighbor_cnt();
+  size_t fixed_neighbor_cnt = l0_neighbor_cnt();
   fixed_neighbors->resize((fixed_neighbor_cnt + 1) * doc_cnt(), kInvalidNodeId);
 
   size_t neighbors_cnt_offset = fixed_neighbor_cnt * doc_cnt();

src/core/algorithm/hnsw/hnsw_searcher_entity.h

@@ -112,15 +112,15 @@ class HnswSearcherEntity : public HnswEntity {
         upper_neighbors_(neighbor_group.upper_neighbors),
         upper_neighbors_meta_(neighbor_group.upper_neighbors_meta),
         neighbors_in_memory_enabled_(neighbors_in_memory_enabled) {
-    segment_datas_.resize(std::max(neighbor_cnt(), upper_neighbor_cnt()),
+    segment_datas_.resize(std::max(l0_neighbor_cnt(), upper_neighbor_cnt()),
                           IndexStorage::SegmentData(0U, 0U));
     fixed_neighbors_ = fixed_neighbors;
   }
 
   bool do_crc_check(std::vector<SegmentPointer> &segments) const;
 
   inline size_t neighbors_size() const {
-    return sizeof(NeighborsHeader) + neighbor_cnt() * sizeof(node_id_t);
+    return sizeof(NeighborsHeader) + l0_neighbor_cnt() * sizeof(node_id_t);
   }
 
   inline size_t upper_neighbors_size() const {