Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix the disparity between disk and memory search for Universal label #347

Merged
merged 9 commits into from
May 24, 2023
Merged

Fix the disparity between disk and memory search for Universal label #347

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
b89deef
UNV Search Fix for Memory
NeelamMahapatro May 11, 2023
797b757
two places to update
NeelamMahapatro May 12, 2023
2b60e02
clang format
NeelamMahapatro May 12, 2023
99270d7
unify find_common_filters function
NeelamMahapatro May 12, 2023
9c3ef0f
fix comments
NeelamMahapatro May 16, 2023
8ed9d67
dummy comments
NeelamMahapatro May 17, 2023
93e7dd7
clang format
NeelamMahapatro May 17, 2023
b929f05
Reduce repetitive calls
NeelamMahapatro May 17, 2023
5b4e304
changing name and return type of function
NeelamMahapatro May 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions include/index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ template <typename T, typename TagT = uint32_t, typename LabelT = uint32_t> clas
DISKANN_DLLEXPORT size_t get_num_points();
DISKANN_DLLEXPORT size_t get_max_points();

DISKANN_DLLEXPORT bool detect_common_filters(uint32_t point_id, bool search_invocation,
const std::vector<LabelT> &incoming_labels);

// Batch build from a file. Optionally pass tags vector.
DISKANN_DLLEXPORT void build(const char *filename, const size_t num_points_to_load,
const IndexWriteParameters &parameters,
Expand Down
59 changes: 34 additions & 25 deletions src/index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -837,6 +837,38 @@ template <typename T, typename TagT, typename LabelT> std::vector<uint32_t> Inde
return init_ids;
}

// Find common filter between a node's labels and a given set of labels, while taking into account universal label
template <typename T, typename TagT, typename LabelT>
bool Index<T, TagT, LabelT>::detect_common_filters(uint32_t point_id, bool search_invocation,
const std::vector<LabelT> &incoming_labels)
{
auto &curr_node_labels = _pts_to_labels[point_id];
std::vector<LabelT> common_filters;
std::set_intersection(incoming_labels.begin(), incoming_labels.end(), curr_node_labels.begin(),
gopalrs marked this conversation as resolved.
Show resolved Hide resolved
curr_node_labels.end(), std::back_inserter(common_filters));
if (common_filters.size() > 0)
{
// This is to reduce the repetitive calls. If common_filters size is > 0 , we dont need to check further for
// universal label
return true;
}
if (_use_universal_label)
{
if (!search_invocation)
{
if (std::find(incoming_labels.begin(), incoming_labels.end(), _universal_label) != incoming_labels.end() ||
std::find(curr_node_labels.begin(), curr_node_labels.end(), _universal_label) != curr_node_labels.end())
common_filters.push_back(_universal_label);
}
else
{
if (std::find(curr_node_labels.begin(), curr_node_labels.end(), _universal_label) != curr_node_labels.end())
common_filters.push_back(_universal_label);
}
}
return (common_filters.size() > 0);
}

template <typename T, typename TagT, typename LabelT>
std::pair<uint32_t, uint32_t> Index<T, TagT, LabelT>::iterate_to_fixed_point(
const T *query, const uint32_t Lsize, const std::vector<uint32_t> &init_ids, InMemQueryScratch<T> *scratch,
Expand Down Expand Up @@ -933,18 +965,7 @@ std::pair<uint32_t, uint32_t> Index<T, TagT, LabelT>::iterate_to_fixed_point(

if (use_filter)
{
std::vector<LabelT> common_filters;
auto &x = _pts_to_labels[id];
std::set_intersection(filter_label.begin(), filter_label.end(), x.begin(), x.end(),
std::back_inserter(common_filters));
if (_use_universal_label)
{
if (std::find(filter_label.begin(), filter_label.end(), _universal_label) != filter_label.end() ||
std::find(x.begin(), x.end(), _universal_label) != x.end())
common_filters.emplace_back(_universal_label);
}

if (common_filters.size() == 0)
if (!detect_common_filters(id, search_invocation, filter_label))
continue;
}

Expand Down Expand Up @@ -1012,19 +1033,7 @@ std::pair<uint32_t, uint32_t> Index<T, TagT, LabelT>::iterate_to_fixed_point(
if (use_filter)
{
// NOTE: NEED TO CHECK IF THIS CORRECT WITH NEW LOCKS.
std::vector<LabelT> common_filters;
auto &x = _pts_to_labels[id];
std::set_intersection(filter_label.begin(), filter_label.end(), x.begin(), x.end(),
std::back_inserter(common_filters));
if (_use_universal_label)
{
if (std::find(filter_label.begin(), filter_label.end(), _universal_label) !=
filter_label.end() ||
std::find(x.begin(), x.end(), _universal_label) != x.end())
common_filters.emplace_back(_universal_label);
}

if (common_filters.size() == 0)
if (!detect_common_filters(id, search_invocation, filter_label))
continue;
}

Expand Down