Skip to content

Commit

Permalink
Fix read timeout for ollama (#18417)
Browse files Browse the repository at this point in the history
Supercedes: #18310

Release Notes:

- Fixed `low_speed_timeout_in_seconds` for Ollama
  • Loading branch information
ConradIrwin authored Sep 27, 2024
1 parent 1be3c44 commit 02d0561
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions crates/ollama/src/ollama.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use anyhow::{anyhow, Context, Result};
use futures::{io::BufReader, stream::BoxStream, AsyncBufReadExt, AsyncReadExt, StreamExt};
use http_client::{http, AsyncBody, HttpClient, Method, Request as HttpRequest};
use http_client::{http, AsyncBody, HttpClient, HttpRequestExt, Method, Request as HttpRequest};
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use serde_json::{value::RawValue, Value};
Expand Down Expand Up @@ -262,14 +262,18 @@ pub async fn stream_chat_completion(
client: &dyn HttpClient,
api_url: &str,
request: ChatRequest,
_: Option<Duration>,
low_speed_timeout: Option<Duration>,
) -> Result<BoxStream<'static, Result<ChatResponseDelta>>> {
let uri = format!("{api_url}/api/chat");
let request_builder = http::Request::builder()
let mut request_builder = http::Request::builder()
.method(Method::POST)
.uri(uri)
.header("Content-Type", "application/json");

if let Some(low_speed_timeout) = low_speed_timeout {
request_builder = request_builder.read_timeout(low_speed_timeout);
}

let request = request_builder.body(AsyncBody::from(serde_json::to_string(&request)?))?;
let mut response = client.send(request).await?;
if response.status().is_success() {
Expand Down

0 comments on commit 02d0561

Please sign in to comment.