Skip to content
This repository was archived by the owner on Jan 2, 2025. It is now read-only.
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 4 additions & 13 deletions server/bleep/src/agent.rs
Original file line number Diff line number Diff line change
Expand Up @@ -344,10 +344,7 @@ impl Agent {
&self,
query: parser::Literal<'_>,
paths: Vec<String>,
limit: u64,
offset: u64,
threshold: f32,
retrieve_more: bool,
params: semantic::SemanticSearchParams,
) -> Result<Vec<semantic::Payload>> {
let paths_set = paths
.into_iter()
Expand Down Expand Up @@ -392,20 +389,14 @@ impl Agent {
};

debug!(?query, %self.thread_id, "executing semantic query");
self.app
.semantic
.search(&query, limit, offset, threshold, retrieve_more)
.await
self.app.semantic.search(&query, params).await
}

#[allow(dead_code)]
async fn batch_semantic_search(
&self,
queries: Vec<parser::Literal<'_>>,
limit: u64,
offset: u64,
threshold: f32,
retrieve_more: bool,
params: semantic::SemanticSearchParams,
) -> Result<Vec<semantic::Payload>> {
let queries = queries
.iter()
Expand All @@ -421,7 +412,7 @@ impl Agent {
debug!(?queries, %self.thread_id, "executing semantic query");
self.app
.semantic
.batch_search(queries.as_slice(), limit, offset, threshold, retrieve_more)
.batch_search(queries.as_slice(), params)
.await
}

Expand Down
23 changes: 21 additions & 2 deletions server/bleep/src/agent/tools/code.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crate::{
},
analytics::EventData,
llm_gateway,
semantic::SemanticSearchParams,
};

impl Agent {
Expand All @@ -23,7 +24,16 @@ impl Agent {
.await?;

let mut results = self
.semantic_search(query.into(), vec![], CODE_SEARCH_LIMIT, 0, 0.3, true)
.semantic_search(
query.into(),
vec![],
SemanticSearchParams {
limit: CODE_SEARCH_LIMIT,
offset: 0,
threshold: 0.3,
exact_match: false,
},
)
.await?;

debug!("returned {} results", results.len());
Expand All @@ -35,7 +45,16 @@ impl Agent {
if !hyde_docs.is_empty() {
let hyde_doc = hyde_docs.first().unwrap().into();
let hyde_results = self
.semantic_search(hyde_doc, vec![], CODE_SEARCH_LIMIT, 0, 0.3, true)
.semantic_search(
hyde_doc,
vec![],
SemanticSearchParams {
limit: CODE_SEARCH_LIMIT,
offset: 0,
threshold: 0.3,
exact_match: false,
},
)
.await?;

debug!("returned {} HyDE results", results.len());
Expand Down
12 changes: 11 additions & 1 deletion server/bleep/src/agent/tools/path.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use crate::{
Agent,
},
analytics::EventData,
semantic::SemanticSearchParams,
};

impl Agent {
Expand All @@ -34,7 +35,16 @@ impl Agent {
// If there are no lexical results, perform a semantic search.
if paths.is_empty() {
let semantic_paths = self
.semantic_search(query.into(), vec![], 30, 0, 0.0, true)
.semantic_search(
query.into(),
vec![],
SemanticSearchParams {
limit: 30,
offset: 0,
threshold: 0.0,
exact_match: false,
},
)
.await?
.into_iter()
.map(|chunk| chunk.relative_path)
Expand Down
12 changes: 11 additions & 1 deletion server/bleep/src/agent/tools/proc.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use crate::{
Agent,
},
analytics::EventData,
semantic::SemanticSearchParams,
};

impl Agent {
Expand All @@ -31,7 +32,16 @@ impl Agent {
.await?;

let results = self
.semantic_search(query.into(), paths.clone(), 10, 0, 0.0, true)
.semantic_search(
query.into(),
paths.clone(),
SemanticSearchParams {
limit: 10,
offset: 0,
threshold: 0.0,
exact_match: true,
},
)
.await?;

let mut chunks = results
Expand Down
81 changes: 56 additions & 25 deletions server/bleep/src/semantic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,14 @@ pub enum SemanticError {
},
}

#[derive(Debug, Clone)]
pub struct SemanticSearchParams {
pub limit: u64,
pub offset: u64,
pub threshold: f32,
pub exact_match: bool, // keyword match for all filters
}

#[derive(Clone)]
pub struct Semantic {
qdrant: Arc<QdrantClient>,
Expand Down Expand Up @@ -329,10 +337,11 @@ impl Semantic {
limit: u64,
offset: u64,
threshold: f32,
exact: bool,
) -> anyhow::Result<Vec<ScoredPoint>> {
let hybrid_filter = Some(Filter {
should: build_conditions_lexical(parsed_query),
must: build_conditions(parsed_query),
must: build_conditions(parsed_query, exact),
..Default::default()
});

Expand Down Expand Up @@ -361,6 +370,7 @@ impl Semantic {
limit: u64,
offset: u64,
threshold: f32,
exact: bool,
) -> anyhow::Result<Vec<ScoredPoint>> {
let response = self
.qdrant
Expand All @@ -374,7 +384,7 @@ impl Semantic {
selector_options: Some(with_payload_selector::SelectorOptions::Enable(true)),
}),
filter: Some(Filter {
must: build_conditions(parsed_query),
must: build_conditions(parsed_query, exact),
..Default::default()
}),
with_vectors: Some(WithVectorsSelector {
Expand All @@ -398,6 +408,7 @@ impl Semantic {
limit: u64,
offset: u64,
threshold: f32,
exact: bool,
) -> anyhow::Result<Vec<ScoredPoint>> {
// FIXME: This method uses `search_points` internally, and not `search_batch_points`. It's
// not clear why, but it seems that the `batch` variant of the `qdrant` calls leads to
Expand All @@ -412,7 +423,7 @@ impl Semantic {

// Queries should contain the same filters, so we get the first one
let parsed_query = parsed_queries.first().unwrap();
let filters = &build_conditions(parsed_query);
let filters = &build_conditions(parsed_query, exact);

let responses = stream::iter(vectors.into_iter())
.map(|vector| async move {
Expand Down Expand Up @@ -517,15 +528,18 @@ impl Semantic {
pub async fn search<'a>(
&self,
parsed_query: &SemanticQuery<'a>,
limit: u64,
offset: u64,
threshold: f32,
retrieve_more: bool,
params: SemanticSearchParams,
) -> anyhow::Result<Vec<Payload>> {
let Some(query) = parsed_query.target() else {
anyhow::bail!("no search target for query");
};
let vector = self.embedder.embed(&query).await?;
let SemanticSearchParams {
limit,
offset,
threshold,
exact_match: exact,
} = params;

// TODO: Remove the need for `retrieve_more`. It's here because:
// In /q `limit` is the maximum number of results returned (the actual number will often be lower due to deduplication)
Expand All @@ -534,9 +548,10 @@ impl Semantic {
.search_with(
parsed_query,
vector.clone(),
if retrieve_more { limit * 2 } else { limit }, // Retrieve double `limit` and deduplicate
limit * 2, // Retrieve double `limit` and deduplicate
offset,
threshold,
exact,
)
.await
.map(|raw| {
Expand All @@ -550,9 +565,10 @@ impl Semantic {
.search_lexical(
parsed_query,
vector.clone(),
if retrieve_more { limit * 2 } else { limit },
limit * 2, // Retrieve double `limit` and deduplicate
offset,
0.0,
exact,
)
.await
.map(|raw| {
Expand All @@ -575,10 +591,7 @@ impl Semantic {
pub async fn batch_search<'a>(
&self,
parsed_queries: &[&SemanticQuery<'a>],
limit: u64,
offset: u64,
threshold: f32,
retrieve_more: bool,
params: SemanticSearchParams,
) -> anyhow::Result<Vec<Payload>> {
if parsed_queries.iter().any(|q| q.target().is_none()) {
anyhow::bail!("no search target for query");
Expand All @@ -593,15 +606,23 @@ impl Semantic {
.into_iter()
.collect::<anyhow::Result<Vec<_>>>()?;

let SemanticSearchParams {
limit,
offset,
threshold,
exact_match: exact,
} = params;

trace!(?parsed_queries, "performing qdrant batch search");

let result = self
.batch_search_with(
parsed_queries,
vectors.clone(),
if retrieve_more { limit * 2 } else { limit }, // Retrieve double `limit` and deduplicate
limit * 2, // Retrieve double `limit` and deduplicate
offset,
threshold,
exact,
)
.await;

Expand Down Expand Up @@ -756,20 +777,22 @@ fn build_conditions_lexical(
.collect()
}

fn build_conditions(query: &SemanticQuery<'_>) -> Vec<qdrant_client::qdrant::Condition> {
let repo_filter = {
fn build_conditions(
query: &SemanticQuery<'_>,
exact_match: bool,
) -> Vec<qdrant_client::qdrant::Condition> {
let path_filter = {
let conditions = query
.repos()
.paths()
.map(|r| {
if r.contains('/') && !r.starts_with("github.com/") {
format!("github.com/{r}")
if exact_match {
make_kv_keyword_filter("relative_path", r.as_ref())
} else {
r.to_string()
make_kv_text_filter("relative_path", r.as_ref())
}
.into()
})
.map(|r| make_kv_keyword_filter("repo_name", r.as_ref()).into())
.collect::<Vec<_>>();
// one of the above repos should match
if conditions.is_empty() {
None
} else {
Expand All @@ -780,11 +803,19 @@ fn build_conditions(query: &SemanticQuery<'_>) -> Vec<qdrant_client::qdrant::Con
}
};

let path_filter = {
let repo_filter = {
let conditions = query
.paths()
.map(|r| make_kv_text_filter("relative_path", r.as_ref()).into())
.repos()
.map(|r| {
if r.contains('/') && !r.starts_with("github.com/") {
format!("github.com/{r}")
} else {
r.to_string()
}
})
.map(|r| make_kv_keyword_filter("repo_name", r.as_ref()).into())
.collect::<Vec<_>>();
// one of the above repos should match
if conditions.is_empty() {
None
} else {
Expand Down
11 changes: 7 additions & 4 deletions server/bleep/src/semantic/execute.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use crate::{
execute::{ApiQuery, PagingMetadata, QueryResponse, QueryResult, ResultStats},
parser::SemanticQuery,
},
semantic::SemanticSearchParams,
snippet::Snippet,
};

Expand All @@ -20,10 +21,12 @@ pub async fn execute(
let results = semantic
.search(
&query,
params.page_size as u64,
((params.page + 1) * params.page_size) as u64,
0.0,
false,
SemanticSearchParams {
limit: params.page_size as u64,
offset: ((params.page + 1) * params.page_size) as u64,
threshold: 0.0,
exact_match: false,
},
)
.await?;

Expand Down