Skip to content

Commit

Permalink
Removed the logger and verified that the logging capability is the ro…
Browse files Browse the repository at this point in the history 
…ot cause of our consistent segfault errors in python. Perhaps it also will fix any issues in our label test too? I'd like to push it to GH and see.
  • Loading branch information
@daxpryce
daxpryce committed Jul 12, 2023
1 parent a7b2087 commit 7561009
Show file tree
Hide file tree
Showing 32 changed files with 367 additions and 572 deletions.
6 changes: 3 additions & 3 deletions apps/build_disk_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ int main(int argc, char **argv)
use_filters, label_file, universal_label, filter_threshold, Lf);
else
{
diskann::cerr << "Error. Unsupported data type" << std::endl;
std::cerr << "Error. Unsupported data type" << std::endl;
return -1;
}
}
Expand All @@ -170,15 +170,15 @@ int main(int argc, char **argv)
universal_label, filter_threshold, Lf);
else
{
diskann::cerr << "Error. Unsupported data type" << std::endl;
std::cerr << "Error. Unsupported data type" << std::endl;
return -1;
}
}
}
catch (const std::exception &e)
{
std::cout << std::string(e.what()) << std::endl;
diskann::cerr << "Index build failed." << std::endl;
std::cerr << "Index build failed." << std::endl;
return -1;
}
}
4 changes: 2 additions & 2 deletions apps/build_memory_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ int main(int argc, char **argv)

try
{
diskann::cout << "Starting index build with R: " << R << " Lbuild: " << L << " alpha: " << alpha
std::cout << "Starting index build with R: " << R << " Lbuild: " << L << " alpha: " << alpha
<< " #threads: " << num_threads << std::endl;

size_t data_num, data_dim;
Expand Down Expand Up @@ -196,7 +196,7 @@ int main(int argc, char **argv)
catch (const std::exception &e)
{
std::cout << std::string(e.what()) << std::endl;
diskann::cerr << "Index build failed." << std::endl;
std::cerr << "Index build failed." << std::endl;
return -1;
}
}
4 changes: 2 additions & 2 deletions apps/build_stitched_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ void prune_and_save(path final_index_path_prefix, path full_index_path_prefix, p
path label_data_path, uint32_t num_threads)
{
size_t dimension, number_of_label_points;
auto diskann_cout_buffer = diskann::cout.rdbuf(nullptr);
auto diskann_cout_buffer = std::cout.rdbuf(nullptr);
auto std_cout_buffer = std::cout.rdbuf(nullptr);
auto pruning_index_timer = std::chrono::high_resolution_clock::now();

Expand All @@ -285,7 +285,7 @@ void prune_and_save(path final_index_path_prefix, path full_index_path_prefix, p
index.prune_all_neighbors(stitched_R, 750, 1.2);
index.save((final_index_path_prefix).c_str());

diskann::cout.rdbuf(diskann_cout_buffer);
std::cout.rdbuf(diskann_cout_buffer);
std::cout.rdbuf(std_cout_buffer);
std::chrono::duration<double> pruning_index_time = std::chrono::high_resolution_clock::now() - pruning_index_timer;
std::cout << "pruning performed in " << pruning_index_time.count() << " seconds\n" << std::endl;
Expand Down
48 changes: 24 additions & 24 deletions apps/range_search_disk_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,18 +35,18 @@ namespace po = boost::program_options;

void print_stats(std::string category, std::vector<float> percentiles, std::vector<float> results)
{
diskann::cout << std::setw(20) << category << ": " << std::flush;
std::cout << std::setw(20) << category << ": " << std::flush;
for (uint32_t s = 0; s < percentiles.size(); s++)
{
diskann::cout << std::setw(8) << percentiles[s] << "%";
std::cout << std::setw(8) << percentiles[s] << "%";
}
diskann::cout << std::endl;
diskann::cout << std::setw(22) << " " << std::flush;
std::cout << std::endl;
std::cout << std::setw(22) << " " << std::flush;
for (uint32_t s = 0; s < percentiles.size(); s++)
{
diskann::cout << std::setw(9) << results[s];
std::cout << std::setw(9) << results[s];
}
diskann::cout << std::endl;
std::cout << std::endl;
}

template <typename T, typename LabelT = uint32_t>
Expand All @@ -58,11 +58,11 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
std::string disk_index_file = index_path_prefix + "_disk.index";
std::string warmup_query_file = index_path_prefix + "_sample_data.bin";

diskann::cout << "Search parameters: #threads: " << num_threads << ", ";
std::cout << "Search parameters: #threads: " << num_threads << ", ";
if (beamwidth <= 0)
diskann::cout << "beamwidth to be optimized for each L value" << std::endl;
std::cout << "beamwidth to be optimized for each L value" << std::endl;
else
diskann::cout << " beamwidth: " << beamwidth << std::endl;
std::cout << " beamwidth: " << beamwidth << std::endl;

// load query bin
T *query = nullptr;
Expand All @@ -79,7 +79,7 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
// groundtruth_ids, gt_num); // use for traditional truthset
if (gt_num != query_num)
{
diskann::cout << "Error. Mismatch in number of queries and ground truth data" << std::endl;
std::cout << "Error. Mismatch in number of queries and ground truth data" << std::endl;
return -1;
}
calc_recall_flag = true;
Expand Down Expand Up @@ -107,7 +107,7 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
}
// cache bfs levels
std::vector<uint32_t> node_list;
diskann::cout << "Caching " << num_nodes_to_cache << " BFS nodes around medoid(s)" << std::endl;
std::cout << "Caching " << num_nodes_to_cache << " BFS nodes around medoid(s)" << std::endl;
_pFlashIndex->cache_bfs_levels(num_nodes_to_cache, node_list);
// _pFlashIndex->generate_cache_list_from_sample_queries(
// warmup_query_file, 15, 6, num_nodes_to_cache, num_threads,
Expand Down Expand Up @@ -146,7 +146,7 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
}
}
}
diskann::cout << "Warming up index... " << std::flush;
std::cout << "Warming up index... " << std::flush;
std::vector<uint64_t> warmup_result_ids_64(warmup_num, 0);
std::vector<float> warmup_result_dists(warmup_num, 0);

Expand All @@ -157,23 +157,23 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
warmup_result_ids_64.data() + (i * 1),
warmup_result_dists.data() + (i * 1), 4);
}
diskann::cout << "..done" << std::endl;
std::cout << "..done" << std::endl;
}

diskann::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
diskann::cout.precision(2);
std::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
std::cout.precision(2);

std::string recall_string = "Recall@rng=" + std::to_string(search_range);
diskann::cout << std::setw(6) << "L" << std::setw(12) << "Beamwidth" << std::setw(16) << "QPS" << std::setw(16)
std::cout << std::setw(6) << "L" << std::setw(12) << "Beamwidth" << std::setw(16) << "QPS" << std::setw(16)
<< "Mean Latency" << std::setw(16) << "99.9 Latency" << std::setw(16) << "Mean IOs" << std::setw(16)
<< "CPU (s)";
if (calc_recall_flag)
{
diskann::cout << std::setw(16) << recall_string << std::endl;
std::cout << std::setw(16) << recall_string << std::endl;
}
else
diskann::cout << std::endl;
diskann::cout << "==============================================================="
std::cout << std::endl;
std::cout << "==============================================================="
"==========================================="
<< std::endl;

Expand Down Expand Up @@ -247,18 +247,18 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
ratio_of_sums = (1.0 * total_true_positive) / (1.0 * total_positive);
}

diskann::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps
std::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps
<< std::setw(16) << mean_latency << std::setw(16) << latency_999 << std::setw(16) << mean_ios
<< std::setw(16) << mean_cpuus;
if (calc_recall_flag)
{
diskann::cout << std::setw(16) << recall << "," << ratio_of_sums << std::endl;
std::cout << std::setw(16) << recall << "," << ratio_of_sums << std::endl;
}
else
diskann::cout << std::endl;
std::cout << std::endl;
}

diskann::cout << "Done searching. " << std::endl;
std::cout << "Done searching. " << std::endl;

diskann::aligned_free(query);
if (warmup != nullptr)
Expand Down Expand Up @@ -359,7 +359,7 @@ int main(int argc, char **argv)
catch (const std::exception &e)
{
std::cout << std::string(e.what()) << std::endl;
diskann::cerr << "Index search failed." << std::endl;
std::cerr << "Index search failed." << std::endl;
return -1;
}
}
56 changes: 28 additions & 28 deletions apps/search_disk_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,18 +32,18 @@ namespace po = boost::program_options;

void print_stats(std::string category, std::vector<float> percentiles, std::vector<float> results)
{
diskann::cout << std::setw(20) << category << ": " << std::flush;
std::cout << std::setw(20) << category << ": " << std::flush;
for (uint32_t s = 0; s < percentiles.size(); s++)
{
diskann::cout << std::setw(8) << percentiles[s] << "%";
std::cout << std::setw(8) << percentiles[s] << "%";
}
diskann::cout << std::endl;
diskann::cout << std::setw(22) << " " << std::flush;
std::cout << std::endl;
std::cout << std::setw(22) << " " << std::flush;
for (uint32_t s = 0; s < percentiles.size(); s++)
{
diskann::cout << std::setw(9) << results[s];
std::cout << std::setw(9) << results[s];
}
diskann::cout << std::endl;
std::cout << std::endl;
}

template <typename T, typename LabelT = uint32_t>
Expand All @@ -54,15 +54,15 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
const std::vector<uint32_t> &Lvec, const float fail_if_recall_below,
const std::vector<std::string> &query_filters, const bool use_reorder_data = false)
{
diskann::cout << "Search parameters: #threads: " << num_threads << ", ";
std::cout << "Search parameters: #threads: " << num_threads << ", ";
if (beamwidth <= 0)
diskann::cout << "beamwidth to be optimized for each L value" << std::flush;
std::cout << "beamwidth to be optimized for each L value" << std::flush;
else
diskann::cout << " beamwidth: " << beamwidth << std::flush;
std::cout << " beamwidth: " << beamwidth << std::flush;
if (search_io_limit == std::numeric_limits<uint32_t>::max())
diskann::cout << "." << std::endl;
std::cout << "." << std::endl;
else
diskann::cout << ", io_limit: " << search_io_limit << "." << std::endl;
std::cout << ", io_limit: " << search_io_limit << "." << std::endl;

std::string warmup_query_file = index_path_prefix + "_sample_data.bin";

Expand Down Expand Up @@ -92,7 +92,7 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
diskann::load_truthset(gt_file, gt_ids, gt_dists, gt_num, gt_dim);
if (gt_num != query_num)
{
diskann::cout << "Error. Mismatch in number of queries and ground truth data" << std::endl;
std::cout << "Error. Mismatch in number of queries and ground truth data" << std::endl;
}
calc_recall_flag = true;
}
Expand All @@ -119,7 +119,7 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
}
// cache bfs levels
std::vector<uint32_t> node_list;
diskann::cout << "Caching " << num_nodes_to_cache << " BFS nodes around medoid(s)" << std::endl;
std::cout << "Caching " << num_nodes_to_cache << " BFS nodes around medoid(s)" << std::endl;
//_pFlashIndex->cache_bfs_levels(num_nodes_to_cache, node_list);
if (num_nodes_to_cache > 0)
_pFlashIndex->generate_cache_list_from_sample_queries(warmup_query_file, 15, 6, num_nodes_to_cache, num_threads,
Expand Down Expand Up @@ -158,7 +158,7 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
}
}
}
diskann::cout << "Warming up index... " << std::flush;
std::cout << "Warming up index... " << std::flush;
std::vector<uint64_t> warmup_result_ids_64(warmup_num, 0);
std::vector<float> warmup_result_dists(warmup_num, 0);

Expand All @@ -169,23 +169,23 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
warmup_result_ids_64.data() + (i * 1),
warmup_result_dists.data() + (i * 1), 4);
}
diskann::cout << "..done" << std::endl;
std::cout << "..done" << std::endl;
}

diskann::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
diskann::cout.precision(2);
std::cout.setf(std::ios_base::fixed, std::ios_base::floatfield);
std::cout.precision(2);

std::string recall_string = "Recall@" + std::to_string(recall_at);
diskann::cout << std::setw(6) << "L" << std::setw(12) << "Beamwidth" << std::setw(16) << "QPS" << std::setw(16)
std::cout << std::setw(6) << "L" << std::setw(12) << "Beamwidth" << std::setw(16) << "QPS" << std::setw(16)
<< "Mean Latency" << std::setw(16) << "99.9 Latency" << std::setw(16) << "Mean IOs" << std::setw(16)
<< "CPU (s)";
if (calc_recall_flag)
{
diskann::cout << std::setw(16) << recall_string << std::endl;
std::cout << std::setw(16) << recall_string << std::endl;
}
else
diskann::cout << std::endl;
diskann::cout << "==============================================================="
std::cout << std::endl;
std::cout << "==============================================================="
"======================================================="
<< std::endl;

Expand All @@ -202,13 +202,13 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre

if (L < recall_at)
{
diskann::cout << "Ignoring search with L:" << L << " since it's smaller than K:" << recall_at << std::endl;
std::cout << "Ignoring search with L:" << L << " since it's smaller than K:" << recall_at << std::endl;
continue;
}

if (beamwidth <= 0)
{
diskann::cout << "Tuning beamwidth.." << std::endl;
std::cout << "Tuning beamwidth.." << std::endl;
optimized_beamwidth =
optimize_beamwidth(_pFlashIndex, warmup, warmup_num, warmup_aligned_dim, L, optimized_beamwidth);
}
Expand Down Expand Up @@ -277,19 +277,19 @@ int search_disk_index(diskann::Metric &metric, const std::string &index_path_pre
best_recall = std::max(recall, best_recall);
}

diskann::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps
std::cout << std::setw(6) << L << std::setw(12) << optimized_beamwidth << std::setw(16) << qps
<< std::setw(16) << mean_latency << std::setw(16) << latency_999 << std::setw(16) << mean_ios
<< std::setw(16) << mean_cpuus;
if (calc_recall_flag)
{
diskann::cout << std::setw(16) << recall << std::endl;
std::cout << std::setw(16) << recall << std::endl;
}
else
diskann::cout << std::endl;
std::cout << std::endl;
delete[] stats;
}

diskann::cout << "Done searching. Now saving results " << std::endl;
std::cout << "Done searching. Now saving results " << std::endl;
uint64_t test_id = 0;
for (auto L : Lvec)
{
Expand Down Expand Up @@ -475,7 +475,7 @@ int main(int argc, char **argv)
catch (const std::exception &e)
{
std::cout << std::string(e.what()) << std::endl;
diskann::cerr << "Index search failed." << std::endl;
std::cerr << "Index search failed." << std::endl;
return -1;
}
}
8 changes: 4 additions & 4 deletions apps/search_memory_index.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
}
else
{
diskann::cout << " Truthset file " << truthset_file << " not found. Not computing recall." << std::endl;
std::cout << " Truthset file " << truthset_file << " not found. Not computing recall." << std::endl;
}

bool filtered_search = false;
Expand Down Expand Up @@ -147,7 +147,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
uint32_t L = Lvec[test_id];
if (L < recall_at)
{
diskann::cout << "Ignoring search with L:" << L << " since it's smaller than K:" << recall_at << std::endl;
std::cout << "Ignoring search with L:" << L << " since it's smaller than K:" << recall_at << std::endl;
continue;
}

Expand Down Expand Up @@ -244,7 +244,7 @@ int search_memory_index(diskann::Metric &metric, const std::string &index_path,
{
if (L < recall_at)
{
diskann::cout << "Ignoring search with L:" << L << " since it's smaller than K:" << recall_at << std::endl;
std::cout << "Ignoring search with L:" << L << " since it's smaller than K:" << recall_at << std::endl;
continue;
}
std::string cur_result_path_prefix = result_path_prefix + "_" + std::to_string(L);
Expand Down Expand Up @@ -441,7 +441,7 @@ int main(int argc, char **argv)
catch (std::exception &e)
{
std::cout << std::string(e.what()) << std::endl;
diskann::cerr << "Index search failed." << std::endl;
std::cerr << "Index search failed." << std::endl;
return -1;
}
}
Loading

0 comments on commit 7561009

Please sign in to comment.