feat: add LlamaModel::load_from_file_async

Author: scriptis

Cargo.lock

@@ -15,6 +15,6 @@ derive_more = "0.99.17"
 flume = "0.11.0"
 llama_cpp_sys = { version = "^0.2.0", path = "../llama_cpp_sys" }
 num_cpus = "1.16.0"
-thiserror = "1.0.49"
-tokio = { version = "1.33.0", features = ["sync"] }
+thiserror = "1.0.50"
+tokio = { version = "1.33.0", features = ["sync", "rt"] }
 tracing = "0.1.39"

crates/llama_cpp/Cargo.toml

@@ -291,6 +291,15 @@ impl LlamaModel {
         }
     }
 
+    /// Loads a LLaMA model from a compatible GGUF (`.gguf`) file asyncronously.
+    ///
+    /// This is a thin `tokio` wrapper over [`LlamaModel::load_from_file`].
+    pub async fn load_from_file_async(file_path: impl AsRef<Path>) -> Result<Self, LlamaLoadError> {
+        let path = file_path.as_ref().to_owned();
+
+        tokio::task::spawn_blocking(move || Self::load_from_file(path)).await.unwrap()
+    }
+
     /// Converts `content` into a vector of tokens that are valid input for this model.
     ///
     /// This temporarily allocates at the amount of memory consumed by `content`, but shrinks that
@@ -690,7 +699,6 @@ mod detail {
 
     use std::ffi::{c_char, c_void, CStr};
     use std::ptr::slice_from_raw_parts;
-    use tokio::sync::OwnedSemaphorePermit;
 
     use tracing::{error, info, trace, warn};