log into files in addition to stderr (#802)

* log into files in addition to stderr

this changeset introduces persistent logging through the
`tracing-appender` crate. logs are written into
`<index_dir>/logs/bloop.log.YYYY-MM-DD-HH` and rotated hourly. logs
written to files are limited to the `bleep` crate.

* disable log write on cloud instances

* improve log filters

* rotate logs daily instead of hourly, tweak log filters

Author: oppiliappan

Cargo.lock

@@ -40,4 +40,4 @@ COPY model /model
 COPY --from=builder /bleep /
 COPY --from=builder /dylib /dylib
 COPY --from=frontend /build/client/dist /frontend
-ENTRYPOINT ["/bleep", "--host=0.0.0.0", "--source-dir=/repos", "--index-dir=/data", "--model-dir=/model", "--dylib-dir=/dylib"]
+ENTRYPOINT ["/bleep", "--host=0.0.0.0", "--source-dir=/repos", "--index-dir=/data", "--model-dir=/model", "--dylib-dir=/dylib", "--disable-log-write"]

Dockerfile

@@ -69,6 +69,8 @@ where
         )
     };
 
+    Application::install_logging(&configuration);
+
     if let Some(dsn) = &configuration.sentry_dsn {
         initialize_sentry(dsn);
     }

apps/desktop/src-tauri/src/backend.rs

@@ -3,8 +3,6 @@
     windows_subsystem = "windows"
 )]
 
-use bleep::Application;
-
 mod backend;
 mod qdrant;
 
@@ -36,8 +34,6 @@ fn relative_command_path(command: impl AsRef<str>) -> Option<PathBuf> {
 
 #[tokio::main]
 async fn main() {
-    Application::install_logging();
-
     tauri::Builder::default()
         .plugin(qdrant::QdrantSupervisor::default())
         .setup(backend::bleep)

apps/desktop/src-tauri/src/main.rs

@@ -72,6 +72,7 @@ spec:
           - --sentry-dsn={{ .Values.bloop.sentryDsn }}
           - --sentry-dsn-fe={{ .Values.bloop.sentryDsnFE }}
           - --analytics-key-fe={{ .Values.bloop.analyticsKeyFE }}
+          - --disable-log-write
           env:
           - name: BLOOP_LOG
             value: {{ .Values.bloop.logLevel }}

helm/bloop/templates/bloop-deployment.yaml

@@ -36,6 +36,7 @@ rayon = "1.7.0"
 clap = { version = "4.3.11", features = ["derive"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.17", features = ["env-filter", "registry"] }
+tracing-appender = "0.2.2"
 color-eyre = "0.6.2"
 sqlx = { version = "0.6.3", features = ["sqlite", "migrate", "offline", "runtime-tokio-rustls", "chrono"] }
 

server/bleep/Cargo.toml

@@ -3,14 +3,10 @@ use bleep::{Application, Configuration, Environment};
 
 #[tokio::main]
 async fn main() -> Result<()> {
-    Application::install_logging();
-    let app = Application::initialize(
-        Environment::server(),
-        Configuration::cli_overriding_config_file()?,
-        None,
-        None,
-    )
-    .await?;
+    let config = Configuration::cli_overriding_config_file()?;
+
+    Application::install_logging(&config);
+    let app = Application::initialize(Environment::server(), config, None, None).await?;
 
     app.initialize_sentry();
     app.run().await

server/bleep/src/bin/bleep.rs

@@ -41,6 +41,13 @@ pub struct Configuration {
     /// Disable system-native notification backends to detect new git commits immediately.
     pub disable_fsevents: bool,
 
+    #[clap(long, default_value_t = false)]
+    #[serde(default)]
+    /// Avoid writing logs to files.
+    ///
+    /// If this flag is not set to `true`, logs are written to <index_dir>/logs/bloop.log.YYYY-MM-DD-HH
+    pub disable_log_write: bool,
+
     #[clap(short, long, default_value_t = default_buffer_size())]
     #[serde(default = "default_buffer_size")]
     /// Size of memory to use for file indexes
@@ -226,6 +233,8 @@ impl Configuration {
 
             disable_fsevents: b.disable_fsevents | a.disable_fsevents,
 
+            disable_log_write: b.disable_log_write | a.disable_log_write,
+
             buffer_size: right_if_default!(b.buffer_size, a.buffer_size, default_buffer_size()),
 
             repo_buffer_size: right_if_default!(

server/bleep/src/config.rs

@@ -603,10 +603,7 @@ impl RepoFile {
                 // we have a graph, use that
                 Ok(graph) => SymbolLocations::TreeSitter(graph),
                 // no graph, it's empty
-                Err(err) => {
-                    warn!(?err, %lang_str, "failed to build scope graph");
-                    SymbolLocations::Empty
-                }
+                Err(_) => SymbolLocations::Empty,
             }
         };
 

server/bleep/src/indexes/file.rs

@@ -34,7 +34,12 @@ use once_cell::sync::OnceCell;
 use sentry_tracing::{EventFilter, SentryLayer};
 use std::{path::Path, sync::Arc};
 use tracing::{debug, error, info, warn, Level};
-use tracing_subscriber::EnvFilter;
+use tracing_subscriber::{
+    filter::{LevelFilter, Targets},
+    fmt,
+    prelude::*,
+    EnvFilter,
+};
 
 mod background;
 mod cache;
@@ -67,6 +72,7 @@ pub use env::Environment;
 const LOG_ENV_VAR: &str = "BLOOP_LOG";
 static LOGGER_INSTALLED: OnceCell<bool> = OnceCell::new();
 static SENTRY_GUARD: OnceCell<sentry::ClientInitGuard> = OnceCell::new();
+static LOGGER_GUARD: OnceCell<tracing_appender::non_blocking::WorkerGuard> = OnceCell::new();
 
 /// The global state
 #[derive(Clone)]
@@ -218,12 +224,12 @@ impl Application {
         _ = SENTRY_GUARD.set(guard);
     }
 
-    pub fn install_logging() {
+    pub fn install_logging(config: &Configuration) {
         if let Some(true) = LOGGER_INSTALLED.get() {
             return;
         }
 
-        if !tracing_subscribe() {
+        if !tracing_subscribe(config) {
             warn!("Failed to install tracing_subscriber. There's probably one already...");
         };
 
@@ -239,7 +245,7 @@ impl Application {
     }
 
     pub async fn run(self) -> Result<()> {
-        Self::install_logging();
+        Self::install_logging(&self.config);
 
         let mut joins = tokio::task::JoinSet::new();
 
@@ -338,25 +344,36 @@ impl FromRef<Application> for axum_extra::extract::cookie::Key {
     }
 }
 
-#[cfg(all(tokio_unstable, feature = "debug"))]
-fn tracing_subscribe() -> bool {
-    use tracing_subscriber::{fmt, prelude::*};
-    let env_filter = fmt::layer().with_filter(EnvFilter::from_env(LOG_ENV_VAR));
-    tracing_subscriber::registry()
-        .with(env_filter)
-        .with(sentry_layer())
-        .with(console_subscriber::spawn())
-        .try_init()
-        .is_ok()
-}
+fn tracing_subscribe(config: &Configuration) -> bool {
+    let env_filter_layer = fmt::layer().with_filter(EnvFilter::from_env(LOG_ENV_VAR));
+    let sentry_layer = sentry_layer();
+    let log_writer_layer = (!config.disable_log_write).then(|| {
+        let log_dir = config.index_dir.join("logs");
+        let file_appender = tracing_appender::rolling::daily(log_dir, "bloop.log");
+        let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
+        _ = LOGGER_GUARD.set(guard);
+        fmt::layer()
+            .with_writer(non_blocking)
+            .with_ansi(false)
+            .with_filter(
+                Targets::new()
+                    .with_target("bleep", LevelFilter::DEBUG)
+                    .with_target("bleep::indexes::file", LevelFilter::WARN)
+                    .with_target("bleep::semantic", LevelFilter::WARN),
+            )
+    });
+
+    #[cfg(all(tokio_unstable, feature = "debug"))]
+    let console_subscriber_layer = Some(console_subscriber::spawn());
+    #[cfg(not(all(tokio_unstable, feature = "debug")))]
+    let console_subscriber_layer: Option<Box<dyn tracing_subscriber::Layer<_> + Send + Sync>> =
+        None;
 
-#[cfg(not(all(tokio_unstable, feature = "debug")))]
-fn tracing_subscribe() -> bool {
-    use tracing_subscriber::{fmt, prelude::*};
-    let env_filter = fmt::layer().with_filter(EnvFilter::from_env(LOG_ENV_VAR));
     tracing_subscriber::registry()
-        .with(env_filter)
-        .with(sentry_layer())
+        .with(log_writer_layer)
+        .with(env_filter_layer)
+        .with(sentry_layer)
+        .with(console_subscriber_layer)
         .try_init()
         .is_ok()
 }