From 599a35436c64d26225d0cff7ad63a5fb427f15d6 Mon Sep 17 00:00:00 2001 From: Metalflame12 Date: Fri, 31 Mar 2023 03:16:08 -0300 Subject: [PATCH] Fix for the --batch-size parameter (#94) * Added support to cached sessions in REPL mode * Fixed the --batch-size parameter, the default value was being used regardless of the value provided by the user. --- llama-rs/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llama-rs/src/lib.rs b/llama-rs/src/lib.rs index c718dda1..f764a954 100644 --- a/llama-rs/src/lib.rs +++ b/llama-rs/src/lib.rs @@ -1461,7 +1461,7 @@ impl InferenceSession { return Err(InferenceError::ContextFull); } - for batch in prompt_tokens.chunks(8) { + for batch in prompt_tokens.chunks(params.n_batch) { model.evaluate(self, params, batch, &mut EvaluateOutputRequest::default()); for &tk in batch { // NOTE: No string ever tokenizes to the end of sentence. So we