[BLO-1840] Determinism semantic search (first call) (#1121)

* gpt3 temperature 0, and using modified rake that gives always same order and do not ignore underscores and slashes

* gpt3 temperature 0, and using modified rake that gives always same order and do not ignore underscores and slashes

* rebase and add modified github repo to Cargo toml

* handling the case where keywords is empty

* move stopwords removal logic into bleep

* attribution

* address review comments

* revert

* Update server/bleep/src/query/stopwords.rs

Co-authored-by: akshay <nerdy@peppe.rs>

* address more comments

---------

Co-authored-by: Gabriel Gordon-Hall <ggordonhall@gmail.com>
Co-authored-by: akshay <nerdy@peppe.rs>

Author: 3 people

Cargo.lock

@@ -111,7 +111,6 @@ git-version = "0.3.5"
 gix = { git = "https://github.com/BloopAI/gitoxide", version="0.55.2", features = ["blocking-http-transport-reqwest-rust-tls-no-trust-dns", "pack-cache-lru-static"] }
 
 # semantic
-rake = "0.1"
 qdrant-client = { version = "1.5.0", default-features = false }
 tiktoken-rs = "0.4.5"
 tokenizers = { version = "0.14.0", default-features = false, features = ["progressbar", "cli", "onig", "esaxx_fast"] }

server/bleep/Cargo.toml

@@ -2,16 +2,14 @@ use std::{sync::Arc, time::Duration};
 
 use anyhow::{anyhow, Context, Result};
 use futures::{Future, TryStreamExt};
-use once_cell::sync::OnceCell;
-use rake::*;
 use tokio::sync::mpsc::Sender;
 use tracing::{debug, error, info, instrument};
 
 use crate::{
     analytics::{EventData, QueryEvent},
     indexes::reader::{ContentDocument, FileDocument},
     llm_gateway::{self, api::FunctionCall},
-    query::parser,
+    query::{parser, stopwords::remove_stopwords},
     repo::RepoRef,
     semantic,
     webserver::{
@@ -43,19 +41,6 @@ mod tools {
     pub mod proc;
 }
 
-static STOPWORDS: OnceCell<StopWords> = OnceCell::new();
-static STOP_WORDS_LIST: &str = include_str!("stopwords.txt");
-
-fn stop_words() -> &'static StopWords {
-    STOPWORDS.get_or_init(|| {
-        let mut sw = StopWords::new();
-        for w in STOP_WORDS_LIST.lines() {
-            sw.insert(w.to_string());
-        }
-        sw
-    })
-}
-
 pub enum Error {
     Timeout(Duration),
     Processing(anyhow::Error),
@@ -196,23 +181,14 @@ impl Agent {
 
                 // Always make a code search for the user query on the first exchange
                 if self.exchanges.len() == 1 {
-                    // Extract keywords from the query
                     let keywords = {
-                        let sw = stop_words();
-                        let r = Rake::new(sw.clone());
-                        let keywords = r.run(s);
-
-                        if keywords.is_empty() {
+                        let keys = remove_stopwords(s);
+                        if keys.is_empty() {
                             s.clone()
                         } else {
-                            keywords
-                                .iter()
-                                .map(|k| k.keyword.clone())
-                                .collect::<Vec<_>>()
-                                .join(" ")
+                            keys
                         }
                     };
-
                     self.code_search(&keywords).await?;
                 }
                 s.clone()

server/bleep/src/agent.rs

@@ -110,6 +110,7 @@ impl Agent {
             .llm_gateway
             .clone()
             .model("gpt-3.5-turbo-0613")
+            .temperature(0.0)
             .chat(&prompt, None)
             .await?;
 

server/bleep/src/agent/tools/code.rs

@@ -4,3 +4,4 @@ pub mod languages;
 pub mod parser;
 pub mod planner;
 pub mod ranking;
+pub mod stopwords;