Skip to content
This repository was archived by the owner on Jan 2, 2025. It is now read-only.

Track LLM gateway errors #575

Merged
calyptobai merged 3 commits into from
May 30, 2023
Merged

Track LLM gateway errors #575

Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e15ebf1
Track LLM gateway errors
calyptobai May 26, 2023
33cffa3
Log errors in Sentry instead of Rudderstack
calyptobai May 30, 2023
f770669
Address review comments
calyptobai May 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions server/bleep/src/webserver/answer/llm_gateway.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ use anyhow::{anyhow, bail};
use axum::http::StatusCode;
use futures::{Stream, StreamExt};
use reqwest_eventsource::EventSource;
use tracing::warn;
use tracing::{debug, warn};

pub mod api {
#[derive(serde::Serialize, serde::Deserialize, Debug, Clone, PartialEq)]
Expand Down Expand Up @@ -143,16 +143,18 @@ impl Client {
for _ in 0..self.max_retries {
match self.chat_oneshot(messages).await {
Err(ChatError::TooManyRequests) => {
warn!("LLM request failed, retrying ...");
warn!("too many LLM requests, retrying in {delay:?}...");
tokio::time::sleep(delay).await;
delay = Duration::from_millis((delay.as_millis() as f32 * SCALE_FACTOR) as u64);
}
Err(ChatError::BadRequest) => {
warn!("LLM request failed, request not eligible for retry");
debug!("LLM message list: {messages:?}");
error!("LLM request failed, request not eligible for retry");
bail!("request not eligible for retry");
}
Err(ChatError::Other(e)) => {
warn!("{e}");
debug!("LLM message list: {messages:?}");
error!("LLM request failed due to unknown reason: {e}");
return Err(e);
}
Ok(stream) => return Ok(stream),
Expand Down