Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ impl<I: VectorId> NeighborProvider<I> {

/// Create a snapshot of the adjacency list index
///
pub fn snapshot(&self) {
self.adjacency_list_index.snapshot();
pub fn snapshot(&self) -> std::path::PathBuf {
self.adjacency_list_index.snapshot()
}

/// Return the maximum degree (number of neighbors per vector)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1852,6 +1852,28 @@ impl BfTreePaths {
}
}

/// Copy a snapshot file to the target path if they differ.
/// This handles the case where the index was built with a different prefix
/// than the one being saved to.
async fn copy_snapshot_if_needed(
snapshot_path: std::path::PathBuf,
target_path: std::path::PathBuf,
) -> ANNResult<()> {
if snapshot_path != target_path {
tokio::task::spawn_blocking(move || {
std::fs::copy(&snapshot_path, &target_path).map_err(|e| {
ANNError::log_index_error(format!(
"Failed to copy snapshot from {:?} to {:?}: {}",
snapshot_path, target_path, e
))
})
})
.await
.map_err(|e| ANNError::log_index_error(format!("Blocking copy task failed: {}", e)))??;
}
Ok(())
}

// SaveWith/LoadWith for BfTreeProvider with TableDeleteProviderAsync

impl<T> SaveWith<String> for BfTreeProvider<T, NoStore, TableDeleteProviderAsync>
Expand Down Expand Up @@ -1897,8 +1919,20 @@ where
}

// Save vectors and neighbors
self.full_vectors.snapshot();
self.neighbor_provider.snapshot();
let vectors_snapshot_path = self.full_vectors.snapshot();
let neighbors_snapshot_path = self.neighbor_provider.snapshot();

// Copy snapshot files to the target prefix location if they differ
copy_snapshot_if_needed(
vectors_snapshot_path,
BfTreePaths::vectors_bftree(&saved_params.prefix),
)
.await?;
copy_snapshot_if_needed(
neighbors_snapshot_path,
BfTreePaths::neighbors_bftree(&saved_params.prefix),
)
.await?;

// Save delete bitmap
{
Expand Down Expand Up @@ -2035,9 +2069,26 @@ where
}

// Save vectors, neighbors, and quant vectors
self.full_vectors.snapshot();
self.neighbor_provider.snapshot();
self.quant_vectors.snapshot();
let vectors_snapshot_path = self.full_vectors.snapshot();
let neighbors_snapshot_path = self.neighbor_provider.snapshot();
let quant_snapshot_path = self.quant_vectors.snapshot();

// Copy snapshot files to the target prefix location if they differ
copy_snapshot_if_needed(
vectors_snapshot_path,
BfTreePaths::vectors_bftree(&saved_params.prefix),
)
.await?;
copy_snapshot_if_needed(
neighbors_snapshot_path,
BfTreePaths::neighbors_bftree(&saved_params.prefix),
)
.await?;
copy_snapshot_if_needed(
quant_snapshot_path,
BfTreePaths::quant_bftree(&saved_params.prefix),
)
.await?;

// Save PQ table metadata and data using PQStorage format
let filename = BfTreePaths::pq_pivots_bin(&saved_params.prefix);
Expand Down Expand Up @@ -2455,12 +2506,19 @@ mod tests {

let storage = FileStorageProvider;

provider.save_with(&storage, &prefix).await.unwrap();
// Save to a different prefix to exercise the snapshot copy logic
let save_dir = tempdir().unwrap();
let save_prefix = save_dir
.path()
.join("saved_bf_tree_provider")
.to_string_lossy()
.to_string();
provider.save_with(&storage, &save_prefix).await.unwrap();

// Load using trait method (includes delete bitmap)
let loaded_provider = BfTreeProvider::<f32, NoStore, TableDeleteProviderAsync>::load_with(
&storage,
&prefix.clone(),
&save_prefix,
)
.await
.unwrap();
Expand Down Expand Up @@ -2623,13 +2681,20 @@ mod tests {

let storage = FileStorageProvider;

provider.save_with(&storage, &prefix).await.unwrap();
// Save to a different prefix to exercise the snapshot copy logic
let save_dir = tempdir().unwrap();
let save_prefix = save_dir
.path()
.join("saved_bf_tree_provider_quant")
.to_string_lossy()
.to_string();
provider.save_with(&storage, &save_prefix).await.unwrap();

// Load using trait method (includes delete bitmap and quantization)
let loaded_provider =
BfTreeProvider::<f32, QuantVectorProvider, TableDeleteProviderAsync>::load_with(
&storage,
&prefix.clone(),
&save_prefix,
)
.await
.unwrap();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ impl QuantVectorProvider {

/// Create a snapshot of the quant vector index
///
pub fn snapshot(&self) {
self.quant_vector_index.snapshot();
pub fn snapshot(&self) -> std::path::PathBuf {
self.quant_vector_index.snapshot()
}

/// Create a new instance from an existing BfTree (for loading from snapshot)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,8 +104,8 @@ impl<T: VectorRepr, I: VectorId> VectorProvider<T, I> {
/// Create a snapshot of the vector index
///
#[inline(always)]
pub fn snapshot(&self) {
self.vector_index.snapshot();
pub fn snapshot(&self) -> std::path::PathBuf {
self.vector_index.snapshot()
}

/// Set vector with Id, `i``, to `v`
Expand Down
Loading